<commit_before>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

"""
Run Regression Test Suite

This module calls down into individual test cases via subprocess. It will
forward all unrecognized arguments onto the individual test scripts, other
than:

    - `-extended`: run the "extended" test suite in addition to the basic one.
    - `-win`: signal that this is running in a Windows environment, and we
      should run the tests.
    - `--coverage`: this generates a basic coverage report for the RPC
      interface.

For a description of arguments recognized by test scripts, see
`qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`.

"""

import os
import time
import shutil
import sys
import subprocess
import tempfile
import re

sys.path.append("qa/pull-tester/")
from tests_config import *

BOLD = ("","")
if os.name == 'posix':
    # primitive formatting on supported
    # terminal via ANSI escape sequences:
    BOLD = ('\033[0m', '\033[1m')

RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/'

#If imported values are not defined then set to zero (or disabled)
if 'ENABLE_WALLET' not in vars():
    ENABLE_WALLET=0
if 'ENABLE_BITCOIND' not in vars():
    ENABLE_BITCOIND=0
if 'ENABLE_UTILS' not in vars():
    ENABLE_UTILS=0
if 'ENABLE_ZMQ' not in vars():
    ENABLE_ZMQ=0

ENABLE_COVERAGE=0

#Create a set to store arguments and create the passon string
opts = set()
passon_args = []
PASSON_REGEX = re.compile("^--")
PARALLEL_REGEX = re.compile('^-parallel=')

print_help = False
run_parallel = 4

for arg in sys.argv[1:]:
    if arg == "--help" or arg == "-h" or arg == "-?":
        print_help = True
        break
    if arg == '--coverage':
        ENABLE_COVERAGE = 1
    elif PASSON_REGEX.match(arg):
        passon_args.append(arg)
    elif PARALLEL_REGEX.match(arg):
        run_parallel = int(arg.split(sep='=', maxsplit=1)[1])
    else:
        opts.add(arg)

#Set env vars
if "BITCOIND" not in os.environ:
    os.environ["BITCOIND"] = BUILDDIR + '/src/qtumd' + EXEEXT

if EXEEXT == ".exe" and "-win" not in opts:
    # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
    # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
    print("Win tests currently disabled by default.  Use -win option to enable")
    sys.exit(0)

if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
    print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")
    sys.exit(0)

# python3-zmq may not be installed. Handle this gracefully and with some helpful info
if ENABLE_ZMQ:
    try:
        import zmq
    except ImportError:
        print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or "
              "to run zmq tests, see dependency info in /qa/README.md.")
        # ENABLE_ZMQ=0
        raise

testScripts = [
    # longest test should go first, to favor running tests in parallel
    'wallet-hd.py',
    'walletbackup.py',
    # vv Tests less than 5m vv
    'p2p-fullblocktest.py',
    'fundrawtransaction.py',
    'p2p-compactblocks.py',
    'segwit.py',
    # vv Tests less than 2m vv
    'wallet.py',
    'wallet-accounts.py',
    'p2p-segwit.py',
    'wallet-dump.py',
    'listtransactions.py',
    # vv Tests less than 60s vv
    'sendheaders.py',
    'zapwallettxes.py',
    'importmulti.py',
    'mempool_limit.py',
    'merkle_blocks.py',
    'receivedby.py',
    'abandonconflict.py',
    'bip68-112-113-p2p.py',
    'rawtransactions.py',
    'reindex.py',
    # vv Tests less than 30s vv
    'mempool_resurrect_test.py',
    'txn_doublespend.py --mineblock',
    'txn_clone.py',
    'getchaintips.py',
    'rest.py',
    'mempool_spendcoinbase.py',
    'mempool_reorg.py',
    'httpbasics.py',
    'multi_rpc.py',
    'proxy_test.py',
    'signrawtransactions.py',
    'nodehandling.py',
    'decodescript.py',
    'blockchain.py',
    'disablewallet.py',
    'keypool.py',
    'p2p-mempool.py',
    'prioritise_transaction.py',
    'invalidblockrequest.py',
    'invalidtxrequest.py',
    'p2p-versionbits-warning.py',
    'preciousblock.py',
    'importprunedfunds.py',
    'signmessages.py',
    'nulldummy.py',
    'import-rescan.py',
    'bumpfee.py',
    'rpcnamedargs.py',
    'listsinceblock.py',
    'p2p-leaktests.py',
]
if ENABLE_ZMQ:
    testScripts.append('zmq_test.py')

testScriptsExt = [
    'pruning.py',
    # vv Tests less than 20m vv
    'smartfees.py',
    # vv Tests less than 5m vv
    'maxuploadtarget.py',
    'mempool_packages.py',
    # vv Tests less than 2m vv
    'bip68-sequence.py',
    'getblocktemplate_longpoll.py',
    'p2p-timeouts.py',
    # vv Tests less than 60s vv
    'bip9-softforks.py',
    'p2p-feefilter.py',
    'rpcbind_test.py',
    # vv Tests less than 30s vv
    'bip65-cltv.py',
    'bip65-cltv-p2p.py',
    'bipdersig-p2p.py',
    'bipdersig.py',
    'getblocktemplate_proposals.py',
    'txn_doublespend.py',
    'txn_clone.py --mineblock',
    'forknotify.py',
    'invalidateblock.py',
    'maxblocksinflight.py',
    'p2p-acceptblock.py',
    'replace-by-fee.py',
]

def runtests():
    test_list = []
    if '-extended' in opts:
        test_list = testScripts + testScriptsExt
    elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts):
        test_list = testScripts
    else:
        for t in testScripts + testScriptsExt:
            if t in opts or re.sub(".py$", "", t) in opts:
                test_list.append(t)

    if print_help:
        # Only print help of the first script and exit
        subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h'])
        sys.exit(0)

    coverage = None

    if ENABLE_COVERAGE:
        coverage = RPCCoverage()
        print("Initializing coverage directory at %s\n" % coverage.dir)
    flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args
    flags.append("--cachedir=%s/qa/cache" % BUILDDIR)
    if coverage:
        flags.append(coverage.flag)

    if len(test_list) > 1 and run_parallel > 1:
        # Populate cache
        subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags)

    #Run Tests
    max_len_name = len(max(test_list, key=len))
    time_sum = 0
    time0 = time.time()
    job_queue = RPCTestHandler(run_parallel, test_list, flags)
    results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
    all_passed = True
    for _ in range(len(test_list)):
        (name, stdout, stderr, passed, duration) = job_queue.get_next()
        all_passed = all_passed and passed
        time_sum += duration

        print('\n' + BOLD[1] + name + BOLD[0] + ":")
        print('' if passed else stdout + '\n', end='')
        print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
        results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
        print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
    results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
    print(results)
    print("\nRuntime: %s s" % (int(time.time() - time0)))

    if coverage:
        coverage.report_rpc_coverage()

        print("Cleaning up coverage data")
        coverage.cleanup()

    sys.exit(not all_passed)

class RPCTestHandler:
    """
    Trigger the testscrips passed in via the list.
    """

    def __init__(self, num_tests_parallel, test_list=None, flags=None):
        assert(num_tests_parallel >= 1)
        self.num_jobs = num_tests_parallel
        self.test_list = test_list
        self.flags = flags
        self.num_running = 0
        # In case there is a graveyard of zombie bitcoinds, we can apply a
        # pseudorandom offset to hopefully jump over them.
        # (625 is PORT_RANGE/MAX_NODES)
        self.portseed_offset = int(time.time() * 1000) % 625
        self.jobs = []

    def get_next(self):
        while self.num_running < self.num_jobs and self.test_list:
            # Add tests
            self.num_running += 1
            t = self.test_list.pop(0)
            port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]
            log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
            log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
            self.jobs.append((t,
                              time.time(),
                              subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed,
                                               universal_newlines=True,
                                               stdout=log_stdout,
                                               stderr=log_stderr),
                              log_stdout,
                              log_stderr))
        if not self.jobs:
            raise IndexError('pop from empty list')
        while True:
            # Return first proc that finishes
            time.sleep(.5)
            for j in self.jobs:
                (name, time0, proc, log_out, log_err) = j
                if proc.poll() is not None:
                    log_out.seek(0), log_err.seek(0)
                    [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
                    log_out.close(), log_err.close()
                    passed = stderr == "" and proc.returncode == 0
                    self.num_running -= 1
                    self.jobs.remove(j)
                    return name, stdout, stderr, passed, int(time.time() - time0)
            print('.', end='', flush=True)

class RPCCoverage(object):
    """
    Coverage reporting utilities for pull-tester.

    Coverage calculation works by having each test script subprocess write
    coverage files into a particular directory. These files contain the RPC
    commands invoked during testing, as well as a complete listing of RPC
    commands per `bitcoin-cli help` (`rpc_interface.txt`).

    After all tests complete, the commands run are combined and diff'd against
    the complete list to calculate uncovered RPC commands.

    See also: qa/rpc-tests/test_framework/coverage.py

    """
    def __init__(self):
        self.dir = tempfile.mkdtemp(prefix="coverage")
        self.flag = '--coveragedir=%s' % self.dir

    def report_rpc_coverage(self):
        """
        Print out RPC commands that were unexercised by tests.

        """
        uncovered = self._get_uncovered_rpc_commands()

        if uncovered:
            print("Uncovered RPC commands:")
            print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
        else:
            print("All RPC commands covered.")

    def cleanup(self):
        return shutil.rmtree(self.dir)

    def _get_uncovered_rpc_commands(self):
        """
        Return a set of currently untested RPC commands.

        """
        # This is shared from `qa/rpc-tests/test-framework/coverage.py`
        REFERENCE_FILENAME = 'rpc_interface.txt'
        COVERAGE_FILE_PREFIX = 'coverage.'

        coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME)
        coverage_filenames = set()
        all_cmds = set()
        covered_cmds = set()

        if not os.path.isfile(coverage_ref_filename):
            raise RuntimeError("No coverage reference found")

        with open(coverage_ref_filename, 'r') as f:
            all_cmds.update([i.strip() for i in f.readlines()])

        for root, dirs, files in os.walk(self.dir):
            for filename in files:
                if filename.startswith(COVERAGE_FILE_PREFIX):
                    coverage_filenames.add(os.path.join(root, filename))

        for filename in coverage_filenames:
            with open(filename, 'r') as f:
                covered_cmds.update([i.strip() for i in f.readlines()])

        return all_cmds - covered_cmds

if __name__ == '__main__':
    runtests()
<commit_msg>Temporarily disabled p2p-segwit due to it requiring an increase in the max message size<commit_after>#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

"""
Run Regression Test Suite

This module calls down into individual test cases via subprocess. It will
forward all unrecognized arguments onto the individual test scripts, other
than:

    - `-extended`: run the "extended" test suite in addition to the basic one.
    - `-win`: signal that this is running in a Windows environment, and we
      should run the tests.
    - `--coverage`: this generates a basic coverage report for the RPC
      interface.

For a description of arguments recognized by test scripts, see
`qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`.

"""

import os
import time
import shutil
import sys
import subprocess
import tempfile
import re

sys.path.append("qa/pull-tester/")
from tests_config import *

BOLD = ("","")
if os.name == 'posix':
    # primitive formatting on supported
    # terminal via ANSI escape sequences:
    BOLD = ('\033[0m', '\033[1m')

RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/'

#If imported values are not defined then set to zero (or disabled)
if 'ENABLE_WALLET' not in vars():
    ENABLE_WALLET=0
if 'ENABLE_BITCOIND' not in vars():
    ENABLE_BITCOIND=0
if 'ENABLE_UTILS' not in vars():
    ENABLE_UTILS=0
if 'ENABLE_ZMQ' not in vars():
    ENABLE_ZMQ=0

ENABLE_COVERAGE=0

#Create a set to store arguments and create the passon string
opts = set()
passon_args = []
PASSON_REGEX = re.compile("^--")
PARALLEL_REGEX = re.compile('^-parallel=')

print_help = False
run_parallel = 4

for arg in sys.argv[1:]:
    if arg == "--help" or arg == "-h" or arg == "-?":
        print_help = True
        break
    if arg == '--coverage':
        ENABLE_COVERAGE = 1
    elif PASSON_REGEX.match(arg):
        passon_args.append(arg)
    elif PARALLEL_REGEX.match(arg):
        run_parallel = int(arg.split(sep='=', maxsplit=1)[1])
    else:
        opts.add(arg)

#Set env vars
if "BITCOIND" not in os.environ:
    os.environ["BITCOIND"] = BUILDDIR + '/src/qtumd' + EXEEXT

if EXEEXT == ".exe" and "-win" not in opts:
    # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
    # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
    print("Win tests currently disabled by default.  Use -win option to enable")
    sys.exit(0)

if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
    print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")
    sys.exit(0)

# python3-zmq may not be installed. Handle this gracefully and with some helpful info
if ENABLE_ZMQ:
    try:
        import zmq
    except ImportError:
        print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or "
              "to run zmq tests, see dependency info in /qa/README.md.")
        # ENABLE_ZMQ=0
        raise

testScripts = [
    # longest test should go first, to favor running tests in parallel
    'wallet-hd.py',
    'walletbackup.py',
    # vv Tests less than 5m vv
    'p2p-fullblocktest.py',
    'fundrawtransaction.py',
    'p2p-compactblocks.py',
    'segwit.py',
    # vv Tests less than 2m vv
    'wallet.py',
    'wallet-accounts.py',
    #
    # Disabled the segwit test since it requires an increase in net.h
    # Will be reenabled when this value has been increased.
    #'p2p-segwit.py',
    'wallet-dump.py',
    'listtransactions.py',
    # vv Tests less than 60s vv
    'sendheaders.py',
    'zapwallettxes.py',
    'importmulti.py',
    'mempool_limit.py',
    'merkle_blocks.py',
    'receivedby.py',
    'abandonconflict.py',
    'bip68-112-113-p2p.py',
    'rawtransactions.py',
    'reindex.py',
    # vv Tests less than 30s vv
    'mempool_resurrect_test.py',
    'txn_doublespend.py --mineblock',
    'txn_clone.py',
    'getchaintips.py',
    'rest.py',
    'mempool_spendcoinbase.py',
    'mempool_reorg.py',
    'httpbasics.py',
    'multi_rpc.py',
    'proxy_test.py',
    'signrawtransactions.py',
    'nodehandling.py',
    'decodescript.py',
    'blockchain.py',
    'disablewallet.py',
    'keypool.py',
    'p2p-mempool.py',
    'prioritise_transaction.py',
    'invalidblockrequest.py',
    'invalidtxrequest.py',
    'p2p-versionbits-warning.py',
    'preciousblock.py',
    'importprunedfunds.py',
    'signmessages.py',
    'nulldummy.py',
    'import-rescan.py',
    'bumpfee.py',
    'rpcnamedargs.py',
    'listsinceblock.py',
    'p2p-leaktests.py',
]
if ENABLE_ZMQ:
    testScripts.append('zmq_test.py')

testScriptsExt = [
    'pruning.py',
    # vv Tests less than 20m vv
    'smartfees.py',
    # vv Tests less than 5m vv
    'maxuploadtarget.py',
    'mempool_packages.py',
    # vv Tests less than 2m vv
    'bip68-sequence.py',
    'getblocktemplate_longpoll.py',
    'p2p-timeouts.py',
    # vv Tests less than 60s vv
    'bip9-softforks.py',
    'p2p-feefilter.py',
    'rpcbind_test.py',
    # vv Tests less than 30s vv
    'bip65-cltv.py',
    'bip65-cltv-p2p.py',
    'bipdersig-p2p.py',
    'bipdersig.py',
    'getblocktemplate_proposals.py',
    'txn_doublespend.py',
    'txn_clone.py --mineblock',
    'forknotify.py',
    'invalidateblock.py',
    'maxblocksinflight.py',
    'p2p-acceptblock.py',
    'replace-by-fee.py',
]

def runtests():
    test_list = []
    if '-extended' in opts:
        test_list = testScripts + testScriptsExt
    elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts):
        test_list = testScripts
    else:
        for t in testScripts + testScriptsExt:
            if t in opts or re.sub(".py$", "", t) in opts:
                test_list.append(t)

    if print_help:
        # Only print help of the first script and exit
        subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h'])
        sys.exit(0)

    coverage = None

    if ENABLE_COVERAGE:
        coverage = RPCCoverage()
        print("Initializing coverage directory at %s\n" % coverage.dir)
    flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args
    flags.append("--cachedir=%s/qa/cache" % BUILDDIR)
    if coverage:
        flags.append(coverage.flag)

    if len(test_list) > 1 and run_parallel > 1:
        # Populate cache
        subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags)

    #Run Tests
    max_len_name = len(max(test_list, key=len))
    time_sum = 0
    time0 = time.time()
    job_queue = RPCTestHandler(run_parallel, test_list, flags)
    results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
    all_passed = True
    for _ in range(len(test_list)):
        (name, stdout, stderr, passed, duration) = job_queue.get_next()
        all_passed = all_passed and passed
        time_sum += duration

        print('\n' + BOLD[1] + name + BOLD[0] + ":")
        print('' if passed else stdout + '\n', end='')
        print('' if stderr == '' else 'stderr:\n' + stderr + '\n', end='')
        results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
        print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
    results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
    print(results)
    print("\nRuntime: %s s" % (int(time.time() - time0)))

    if coverage:
        coverage.report_rpc_coverage()

        print("Cleaning up coverage data")
        coverage.cleanup()

    sys.exit(not all_passed)

class RPCTestHandler:
    """
    Trigger the testscrips passed in via the list.
    """

    def __init__(self, num_tests_parallel, test_list=None, flags=None):
        assert(num_tests_parallel >= 1)
        self.num_jobs = num_tests_parallel
        self.test_list = test_list
        self.flags = flags
        self.num_running = 0
        # In case there is a graveyard of zombie bitcoinds, we can apply a
        # pseudorandom offset to hopefully jump over them.
        # (625 is PORT_RANGE/MAX_NODES)
        self.portseed_offset = int(time.time() * 1000) % 625
        self.jobs = []

    def get_next(self):
        while self.num_running < self.num_jobs and self.test_list:
            # Add tests
            self.num_running += 1
            t = self.test_list.pop(0)
            port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)]
            log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
            log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
            self.jobs.append((t,
                              time.time(),
                              subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed,
                                               universal_newlines=True,
                                               stdout=log_stdout,
                                               stderr=log_stderr),
                              log_stdout,
                              log_stderr))
        if not self.jobs:
            raise IndexError('pop from empty list')
        while True:
            # Return first proc that finishes
            time.sleep(.5)
            for j in self.jobs:
                (name, time0, proc, log_out, log_err) = j
                if proc.poll() is not None:
                    log_out.seek(0), log_err.seek(0)
                    [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
                    log_out.close(), log_err.close()
                    passed = stderr == "" and proc.returncode == 0
                    self.num_running -= 1
                    self.jobs.remove(j)
                    return name, stdout, stderr, passed, int(time.time() - time0)
            print('.', end='', flush=True)

class RPCCoverage(object):
    """
    Coverage reporting utilities for pull-tester.

    Coverage calculation works by having each test script subprocess write
    coverage files into a particular directory. These files contain the RPC
    commands invoked during testing, as well as a complete listing of RPC
    commands per `bitcoin-cli help` (`rpc_interface.txt`).

    After all tests complete, the commands run are combined and diff'd against
    the complete list to calculate uncovered RPC commands.

    See also: qa/rpc-tests/test_framework/coverage.py

    """
    def __init__(self):
        self.dir = tempfile.mkdtemp(prefix="coverage")
        self.flag = '--coveragedir=%s' % self.dir

    def report_rpc_coverage(self):
        """
        Print out RPC commands that were unexercised by tests.

        """
        uncovered = self._get_uncovered_rpc_commands()

        if uncovered:
            print("Uncovered RPC commands:")
            print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
        else:
            print("All RPC commands covered.")

    def cleanup(self):
        return shutil.rmtree(self.dir)

    def _get_uncovered_rpc_commands(self):
        """
        Return a set of currently untested RPC commands.

        """
        # This is shared from `qa/rpc-tests/test-framework/coverage.py`
        REFERENCE_FILENAME = 'rpc_interface.txt'
        COVERAGE_FILE_PREFIX = 'coverage.'

        coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME)
        coverage_filenames = set()
        all_cmds = set()
        covered_cmds = set()

        if not os.path.isfile(coverage_ref_filename):
            raise RuntimeError("No coverage reference found")

        with open(coverage_ref_filename, 'r') as f:
            all_cmds.update([i.strip() for i in f.readlines()])

        for root, dirs, files in os.walk(self.dir):
            for filename in files:
                if filename.startswith(COVERAGE_FILE_PREFIX):
                    coverage_filenames.add(os.path.join(root, filename))

        for filename in coverage_filenames:
            with open(filename, 'r') as f:
                covered_cmds.update([i.strip() for i in f.readlines()])

        return all_cmds - covered_cmds

if __name__ == '__main__':
    runtests()
<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import TestCase, main

from qiita_core.util import qiita_test_checker
from qiita_db.util import get_count, check_count

@qiita_test_checker()
class SetupTest(TestCase):
    """Tests that the test database have been successfully populated"""

    def test_qitta_user(self):
        self.assertEqual(get_count("qiita.qiita_user"), 4)

    def test_study_person(self):
        self.assertEqual(get_count("qiita.study_person"), 3)

    def test_study(self):
        self.assertEqual(get_count("qiita.study"), 1)

    def test_study_users(self):
        self.assertEqual(get_count("qiita.study_users"), 2)

    def test_investigation(self):
        self.assertEqual(get_count("qiita.investigation"), 1)

    def test_investigation_study(self):
        self.assertEqual(get_count("qiita.investigation_study"), 1)

    def test_study_experimental_factor(self):
        self.assertEqual(get_count("qiita.study_experimental_factor"), 1)

    def test_filepath(self):
        self.assertEqual(get_count("qiita.filepath"), 15)

    def test_filepath_type(self):
        self.assertEqual(get_count("qiita.filepath_type"), 13)

    def test_raw_data(self):
        self.assertEqual(get_count("qiita.raw_data"), 2)

    def test_raw_filepath(self):
        self.assertEqual(get_count("qiita.raw_filepath"), 4)

    def test_study_raw_data(self):
        self.assertEqual(get_count("qiita.study_raw_data"), 2)

    def test_required_sample_info(self):
        self.assertEqual(get_count("qiita.required_sample_info"), 27)

    def test_study_sample_columns(self):
        self.assertEqual(get_count("qiita.study_sample_columns"), 21)

    def test_sample_1(self):
        self.assertEqual(get_count("qiita.sample_1"), 27)

    def test_common_prep_info(self):
        self.assertEqual(get_count("qiita.common_prep_info"), 27)

    def test_raw_data_prep_columns(self):
        self.assertEqual(get_count("qiita.raw_data_prep_columns"), 20)

    def test_prep_1(self):
        self.assertEqual(get_count("qiita.prep_1"), 27)

    def test_preprocessed_data(self):
        self.assertEqual(get_count("qiita.preprocessed_data"), 2)

    def test_raw_preprocessed_data(self):
        self.assertEqual(get_count("qiita.raw_preprocessed_data"), 2)

    def test_study_preprocessed_data(self):
        self.assertEqual(get_count("qiita.study_preprocessed_data"), 2)

    def test_preprocessed_filepath(self):
        self.assertEqual(get_count("qiita.preprocessed_filepath"), 3)

    def test_preprocessed_sequence_illumina_params(self):
        self.assertEqual(
            get_count("qiita.preprocessed_sequence_illumina_params"), 2)

    def test_processed_data(self):
        self.assertEqual(get_count("qiita.processed_data"), 1)

    def test_preprocessed_processed_data(self):
        self.assertEqual(get_count("qiita.preprocessed_processed_data"), 1)

    def test_reference(self):
        self.assertEqual(get_count("qiita.reference"), 1)

    def test_processed_params_uclust(self):
        self.assertEqual(get_count("qiita.processed_params_uclust"), 1)

    def test_processed_filepath(self):
        self.assertEqual(get_count("qiita.processed_filepath"), 1)

    def test_job(self):
        self.assertEqual(get_count("qiita.job"), 3)

    def test_analysis(self):
        self.assertEqual(get_count("qiita.analysis"), 2)

    def test_analysis_job(self):
        self.assertEqual(get_count("qiita.analysis_job"), 3)

    def test_analysis_filepath(self):
        self.assertEqual(get_count("qiita.analysis_filepath"), 2)

    def test_analysis_sample(self):
        self.assertEqual(get_count("qiita.analysis_sample"), 9)

    def test_analysis_users(self):
        self.assertEqual(get_count("qiita.analysis_users"), 1)

    def test_job_results_filepath(self):
        self.assertEqual(get_count("qiita.job_results_filepath"), 2)

    def test_command_data_type(self):
        self.assertEqual(get_count("qiita.command_data_type"), 14)

    def test_ontology(self):
        self.assertTrue(check_count('qiita.ontology', 1))

    def test_ontology_terms(self):
        self.assertTrue(check_count('qiita.term', 14))

if __name__ == '__main__':
    main()
<commit_msg>fix test for share change<commit_after># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import TestCase, main

from qiita_core.util import qiita_test_checker
from qiita_db.util import get_count, check_count

@qiita_test_checker()
class SetupTest(TestCase):
    """Tests that the test database have been successfully populated"""

    def test_qitta_user(self):
        self.assertEqual(get_count("qiita.qiita_user"), 4)

    def test_study_person(self):
        self.assertEqual(get_count("qiita.study_person"), 3)

    def test_study(self):
        self.assertEqual(get_count("qiita.study"), 1)

    def test_study_users(self):
        self.assertEqual(get_count("qiita.study_users"), 1)

    def test_investigation(self):
        self.assertEqual(get_count("qiita.investigation"), 1)

    def test_investigation_study(self):
        self.assertEqual(get_count("qiita.investigation_study"), 1)

    def test_study_experimental_factor(self):
        self.assertEqual(get_count("qiita.study_experimental_factor"), 1)

    def test_filepath(self):
        self.assertEqual(get_count("qiita.filepath"), 15)

    def test_filepath_type(self):
        self.assertEqual(get_count("qiita.filepath_type"), 13)

    def test_raw_data(self):
        self.assertEqual(get_count("qiita.raw_data"), 2)

    def test_raw_filepath(self):
        self.assertEqual(get_count("qiita.raw_filepath"), 4)

    def test_study_raw_data(self):
        self.assertEqual(get_count("qiita.study_raw_data"), 2)

    def test_required_sample_info(self):
        self.assertEqual(get_count("qiita.required_sample_info"), 27)

    def test_study_sample_columns(self):
        self.assertEqual(get_count("qiita.study_sample_columns"), 21)

    def test_sample_1(self):
        self.assertEqual(get_count("qiita.sample_1"), 27)

    def test_common_prep_info(self):
        self.assertEqual(get_count("qiita.common_prep_info"), 27)

    def test_raw_data_prep_columns(self):
        self.assertEqual(get_count("qiita.raw_data_prep_columns"), 20)

    def test_prep_1(self):
        self.assertEqual(get_count("qiita.prep_1"), 27)

    def test_preprocessed_data(self):
        self.assertEqual(get_count("qiita.preprocessed_data"), 2)

    def test_raw_preprocessed_data(self):
        self.assertEqual(get_count("qiita.raw_preprocessed_data"), 2)

    def test_study_preprocessed_data(self):
        self.assertEqual(get_count("qiita.study_preprocessed_data"), 2)

    def test_preprocessed_filepath(self):
        self.assertEqual(get_count("qiita.preprocessed_filepath"), 3)

    def test_preprocessed_sequence_illumina_params(self):
        self.assertEqual(
            get_count("qiita.preprocessed_sequence_illumina_params"), 2)

    def test_processed_data(self):
        self.assertEqual(get_count("qiita.processed_data"), 1)

    def test_preprocessed_processed_data(self):
        self.assertEqual(get_count("qiita.preprocessed_processed_data"), 1)

    def test_reference(self):
        self.assertEqual(get_count("qiita.reference"), 1)

    def test_processed_params_uclust(self):
        self.assertEqual(get_count("qiita.processed_params_uclust"), 1)

    def test_processed_filepath(self):
        self.assertEqual(get_count("qiita.processed_filepath"), 1)

    def test_job(self):
        self.assertEqual(get_count("qiita.job"), 3)

    def test_analysis(self):
        self.assertEqual(get_count("qiita.analysis"), 2)

    def test_analysis_job(self):
        self.assertEqual(get_count("qiita.analysis_job"), 3)

    def test_analysis_filepath(self):
        self.assertEqual(get_count("qiita.analysis_filepath"), 2)

    def test_analysis_sample(self):
        self.assertEqual(get_count("qiita.analysis_sample"), 9)

    def test_analysis_users(self):
        self.assertEqual(get_count("qiita.analysis_users"), 1)

    def test_job_results_filepath(self):
        self.assertEqual(get_count("qiita.job_results_filepath"), 2)

    def test_command_data_type(self):
        self.assertEqual(get_count("qiita.command_data_type"), 14)

    def test_ontology(self):
        self.assertTrue(check_count('qiita.ontology', 1))

    def test_ontology_terms(self):
        self.assertTrue(check_count('qiita.term', 14))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import TestCase, main

from qiita_db.util import get_count, check_count

class SetupTest(TestCase):
    """Tests that the test database have been successfully populated"""

    def test_qitta_user(self):
        self.assertEqual(get_count("qiita.qiita_user"), 4)

    def test_study_person(self):
        self.assertEqual(get_count("qiita.study_person"), 3)

    def test_study(self):
        self.assertEqual(get_count("qiita.study"), 1)

    def test_study_users(self):
        self.assertEqual(get_count("qiita.study_users"), 1)

    def test_investigation(self):
        self.assertEqual(get_count("qiita.investigation"), 1)

    def test_investigation_study(self):
        self.assertEqual(get_count("qiita.investigation_study"), 1)

    def test_study_experimental_factor(self):
        self.assertEqual(get_count("qiita.study_experimental_factor"), 1)

    def test_filepath(self):
        self.assertEqual(get_count("qiita.filepath"), 16)

    def test_filepath_type(self):
        self.assertEqual(get_count("qiita.filepath_type"), 19)

    def test_study_prep_template(self):
        self.assertEqual(get_count("qiita.study_prep_template"), 1)

    def test_required_sample_info(self):
        self.assertEqual(get_count("qiita.study_sample"), 27)

    def test_study_sample_columns(self):
        self.assertEqual(get_count("qiita.study_sample_columns"), 31)

    def test_sample_1(self):
        self.assertEqual(get_count("qiita.sample_1"), 27)

    def test_prep_template(self):
        self.assertEqual(get_count("qiita.prep_template"), 1)

    def test_prep_template_sample(self):
        self.assertEqual(get_count("qiita.prep_template_sample"), 27)

    def test_prep_columns(self):
        self.assertEqual(get_count("qiita.prep_columns"), 23)

    def test_prep_1(self):
        self.assertEqual(get_count("qiita.prep_1"), 27)

    def test_preprocessed_sequence_illumina_params(self):
        self.assertEqual(
            get_count("qiita.preprocessed_sequence_illumina_params"), 7)

    def test_reference(self):
        self.assertEqual(get_count("qiita.reference"), 1)

    def test_processed_params_uclust(self):
        self.assertEqual(get_count("qiita.processed_params_uclust"), 1)

    def test_job(self):
        self.assertEqual(get_count("qiita.job"), 3)

    def test_analysis(self):
        self.assertEqual(get_count("qiita.analysis"), 10)

    def test_analysis_job(self):
        self.assertEqual(get_count("qiita.analysis_job"), 3)

    def test_analysis_workflow(self):
        self.assertEqual(get_count("qiita.analysis_workflow"), 2)

    def test_analysis_filepath(self):
        self.assertEqual(get_count("qiita.analysis_filepath"), 2)

    def test_analysis_sample(self):
        self.assertEqual(get_count("qiita.analysis_sample"), 13)

    def test_analysis_users(self):
        self.assertEqual(get_count("qiita.analysis_users"), 1)

    def test_job_results_filepath(self):
        self.assertEqual(get_count("qiita.job_results_filepath"), 2)

    def test_command_data_type(self):
        self.assertEqual(get_count("qiita.command_data_type"), 14)

    def test_ontology(self):
        self.assertTrue(check_count('qiita.ontology', 1))

    def test_ontology_terms(self):
        self.assertTrue(check_count('qiita.term', 14))

if __name__ == '__main__':
    main()
<commit_msg>Fixing test_setup.py<commit_after># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import TestCase, main

from qiita_db.util import get_count, check_count

class SetupTest(TestCase):
    """Tests that the test database have been successfully populated"""

    def test_qitta_user(self):
        self.assertEqual(get_count("qiita.qiita_user"), 4)

    def test_study_person(self):
        self.assertEqual(get_count("qiita.study_person"), 3)

    def test_study(self):
        self.assertEqual(get_count("qiita.study"), 1)

    def test_study_users(self):
        self.assertEqual(get_count("qiita.study_users"), 1)

    def test_investigation(self):
        self.assertEqual(get_count("qiita.investigation"), 1)

    def test_investigation_study(self):
        self.assertEqual(get_count("qiita.investigation_study"), 1)

    def test_study_experimental_factor(self):
        self.assertEqual(get_count("qiita.study_experimental_factor"), 1)

    def test_filepath(self):
        self.assertEqual(get_count("qiita.filepath"), 16)

    def test_filepath_type(self):
        self.assertEqual(get_count("qiita.filepath_type"), 19)

    def test_study_prep_template(self):
        self.assertEqual(get_count("qiita.study_prep_template"), 1)

    def test_required_sample_info(self):
        self.assertEqual(get_count("qiita.study_sample"), 27)

    def test_study_sample_columns(self):
        self.assertEqual(get_count("qiita.study_sample_columns"), 31)

    def test_sample_1(self):
        self.assertEqual(get_count("qiita.sample_1"), 27)

    def test_prep_template(self):
        self.assertEqual(get_count("qiita.prep_template"), 1)

    def test_prep_template_sample(self):
        self.assertEqual(get_count("qiita.prep_template_sample"), 27)

    def test_prep_columns(self):
        self.assertEqual(get_count("qiita.prep_columns"), 23)

    def test_prep_1(self):
        self.assertEqual(get_count("qiita.prep_1"), 27)

    def test_reference(self):
        self.assertEqual(get_count("qiita.reference"), 1)

    def test_job(self):
        self.assertEqual(get_count("qiita.job"), 3)

    def test_analysis(self):
        self.assertEqual(get_count("qiita.analysis"), 10)

    def test_analysis_job(self):
        self.assertEqual(get_count("qiita.analysis_job"), 3)

    def test_analysis_workflow(self):
        self.assertEqual(get_count("qiita.analysis_workflow"), 2)

    def test_analysis_filepath(self):
        self.assertEqual(get_count("qiita.analysis_filepath"), 2)

    def test_analysis_sample(self):
        self.assertEqual(get_count("qiita.analysis_sample"), 13)

    def test_analysis_users(self):
        self.assertEqual(get_count("qiita.analysis_users"), 1)

    def test_job_results_filepath(self):
        self.assertEqual(get_count("qiita.job_results_filepath"), 2)

    def test_command_data_type(self):
        self.assertEqual(get_count("qiita.command_data_type"), 14)

    def test_ontology(self):
        self.assertTrue(check_count('qiita.ontology', 1))

    def test_ontology_terms(self):
        self.assertTrue(check_count('qiita.term', 14))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys

if sys.hexversion < 0x02070000:
    import unittest2 as unittest
else:
    import unittest

from hypothesis import given
from hypothesis.extra.datetime import datetimes

from datetime import date

from qual.calendars import DateWithCalendar, ProlepticGregorianCalendar, JulianCalendar, EnglishHistoricalCalendar

class CalendarTest(unittest.TestCase):
    def check_valid_date(self, year, month, day):
        d = self.calendar.date(year, month, day)
        self.assertIsNotNone(d)

    def check_invalid_date(self, year, month, day):
        self.assertRaises(Exception, lambda : self.calendar(year, month, day))

class TestProlepticGregorianCalendar(CalendarTest):
    def setUp(self):
        self.calendar = ProlepticGregorianCalendar()

    def test_leap_year_from_before_1582(self):
        """Pope Gregory introduced the calendar in 1582"""
        self.check_valid_date(1200, 2, 29)

    def test_day_missed_out_in_British_calendar_change(self):
        """This date never happened in English law:
           It was missed when changing from the Julian to
           Gregorian. This test proves that we are not
           using a historical British calendar."""
        self.check_valid_date(1752, 9, 3)

    def test_Julian_leap_day_is_not_a_valid_date(self):
        """This day /was/ a leap day contemporaneously,
           but is not a valid date of the Gregorian calendar."""
        self.check_invalid_date(1300, 2, 29)

    def test_calendar_reference(self):
        """Can we retrieve the calendar from the date"""
        d = self.calendar.date(2010, 8, 1)
        self.assertEqual(d.calendar, ProlepticGregorianCalendar)

class TestJulianCalendar(CalendarTest):
    def setUp(self):
        self.calendar = JulianCalendar()

class TestEnglishHistoricalCalendar(CalendarTest):
    def setUp(self):
        self.calendar = EnglishHistoricalCalendar()

    def test_before_switch(self):
        self.check_valid_date(1752, 9, 2)

class JulianGregorianConversion(unittest.TestCase):
    def setUp(self):
        self.gregorian = ProlepticGregorianCalendar()
        self.julian = JulianCalendar()
        self.addTypeEqualityFunc(
            DateWithCalendar,
            DateWithCalendar.make_assertEqual(self)
        )

    def Julian_to_Gregorian_conversion(self, julian_args, gregorian_args):
        julian_date = self.julian.date(*julian_args)
        gregorian_date = self.gregorian.date(*gregorian_args)
        converted_date = julian_date.convert_to(self.gregorian)
        self.assertEqual(
            converted_date,
            gregorian_date,
        )

    def Gregorian_to_Julian_conversion(self, julian_args, gregorian_args):
        gregorian_date = self.gregorian.date(*gregorian_args)
        julian_date = self.julian.date(*julian_args)
        converted_date = gregorian_date.convert_to(self.julian)
        self.assertEqual(
            converted_date,
            julian_date,
        )

    def check_both_conversions(self, julian_args, gregorian_args):
        self.Julian_to_Gregorian_conversion(julian_args, gregorian_args)
        self.Gregorian_to_Julian_conversion(julian_args, gregorian_args)

    def test_first_day_of_Gregorian_calendar(self):
        """ This date is the first day the Gregorian calendar was used
            in Catholic countries which adopted the new calendar immediately.
            We should be able to convert it from Julian to Gregorian."""
        self.check_both_conversions(
            (1582, 10, 5),
            (1582, 10, 15)
        )

    def test_a_Julian_only_leap_day(self):
        """ This date is a leap day of the Julian calendar.
            The corresponding day of the Gregorian calendar is not a leap day,
            and that year is not leap in the Gregorian calendar. """
        self.check_both_conversions(
            (1500, 2, 29),
            (1500, 3, 10)
        )

    def test_another_Julian_only_leap_day(self):
        """ This date is a leap day of the Julian calendar.
            The corresponding day of the Gregorian calendar is not a leap day,
            and that year is not leap in the Gregorian calendar. """
        self.check_both_conversions(
            (1400, 2, 29),
            (1400, 3, 9)
        )

    def test_the_day_before_a_Julian_only_leap_day(self):
        """ This date is Feb 28th before leap day of the Julian calendar.
            The corresponding day of the Gregorian calendar is not a leap day,
            and that year is not leap in the Gregorian calendar. """
        self.check_both_conversions(
            (1500, 2, 28),
            (1500, 3, 9)
        )

    def test_a_Gregorian_March_1st(self):
        """ This Gregorian date is March 1st, of a Julian-only leap year.
            Although the Julian date falls before the leap day,
            since the Gregorian date falls after it, the number of
            days difference between the dates in the two calendars
            depends which calendar you use to measure it. """
        self.check_both_conversions(
            (1500, 2, 20),
            (1500, 3, 1)
        )

    def test_a_Gregorian_Feb_28th(self):
        """ This Gregorian date is Feb 28th, of a Julian-only leap year.
            Both the Gregorian date, and the Julian date, are unaffected
            by the Julian leap day later in that year."""
        self.check_both_conversions(
            (1500, 2, 19),
            (1500, 2, 28)
        )

    @given(datetimes(timezones=[]))
    def test_round_trip_Gregorian_Julian_Gregorian(self, dt):
        original_gregorian_date = self.gregorian.from_date(dt.date())
        converted_date = original_gregorian_date.convert_to(self.julian)
        round_tripped_date = converted_date.convert_to(self.gregorian)
        self.assertEqual(original_gregorian_date, round_tripped_date)

    @given(datetimes(timezones=[]))
    def test_round_trip_Julian_Gregorian_Julian(self, dt):
        original_julian_date = self.julian.from_date(dt.date())
        converted_date = original_julian_date.convert_to(self.gregorian)
        round_tripped_date = converted_date.convert_to(self.julian)
        self.assertEqual(original_julian_date, round_tripped_date)

<commit_msg>Check that a valid date has the right calendar.<commit_after>import sys

if sys.hexversion < 0x02070000:
    import unittest2 as unittest
else:
    import unittest

from hypothesis import given
from hypothesis.extra.datetime import datetimes

from datetime import date

from qual.calendars import DateWithCalendar, ProlepticGregorianCalendar, JulianCalendar, EnglishHistoricalCalendar

class CalendarTest(unittest.TestCase):
    def check_valid_date(self, year, month, day):
        d = self.calendar.date(year, month, day)
        self.assertIsNotNone(d)
        self.assertEqual(d.calendar, self.calendar.__class__)

    def check_invalid_date(self, year, month, day):
        self.assertRaises(Exception, lambda : self.calendar(year, month, day))

class TestProlepticGregorianCalendar(CalendarTest):
    def setUp(self):
        self.calendar = ProlepticGregorianCalendar()

    def test_leap_year_from_before_1582(self):
        """Pope Gregory introduced the calendar in 1582"""
        self.check_valid_date(1200, 2, 29)

    def test_day_missed_out_in_British_calendar_change(self):
        """This date never happened in English law:
           It was missed when changing from the Julian to
           Gregorian. This test proves that we are not
           using a historical British calendar."""
        self.check_valid_date(1752, 9, 3)

    def test_Julian_leap_day_is_not_a_valid_date(self):
        """This day /was/ a leap day contemporaneously,
           but is not a valid date of the Gregorian calendar."""
        self.check_invalid_date(1300, 2, 29)

    def test_calendar_reference(self):
        """Can we retrieve the calendar from the date"""
        d = self.calendar.date(2010, 8, 1)
        self.assertEqual(d.calendar, ProlepticGregorianCalendar)

class TestJulianCalendar(CalendarTest):
    def setUp(self):
        self.calendar = JulianCalendar()

class TestEnglishHistoricalCalendar(CalendarTest):
    def setUp(self):
        self.calendar = EnglishHistoricalCalendar()

    def test_before_switch(self):
        self.check_valid_date(1752, 9, 2)

class JulianGregorianConversion(unittest.TestCase):
    def setUp(self):
        self.gregorian = ProlepticGregorianCalendar()
        self.julian = JulianCalendar()
        self.addTypeEqualityFunc(
            DateWithCalendar,
            DateWithCalendar.make_assertEqual(self)
        )

    def Julian_to_Gregorian_conversion(self, julian_args, gregorian_args):
        julian_date = self.julian.date(*julian_args)
        gregorian_date = self.gregorian.date(*gregorian_args)
        converted_date = julian_date.convert_to(self.gregorian)
        self.assertEqual(
            converted_date,
            gregorian_date,
        )

    def Gregorian_to_Julian_conversion(self, julian_args, gregorian_args):
        gregorian_date = self.gregorian.date(*gregorian_args)
        julian_date = self.julian.date(*julian_args)
        converted_date = gregorian_date.convert_to(self.julian)
        self.assertEqual(
            converted_date,
            julian_date,
        )

    def check_both_conversions(self, julian_args, gregorian_args):
        self.Julian_to_Gregorian_conversion(julian_args, gregorian_args)
        self.Gregorian_to_Julian_conversion(julian_args, gregorian_args)

    def test_first_day_of_Gregorian_calendar(self):
        """ This date is the first day the Gregorian calendar was used
            in Catholic countries which adopted the new calendar immediately.
            We should be able to convert it from Julian to Gregorian."""
        self.check_both_conversions(
            (1582, 10, 5),
            (1582, 10, 15)
        )

    def test_a_Julian_only_leap_day(self):
        """ This date is a leap day of the Julian calendar.
            The corresponding day of the Gregorian calendar is not a leap day,
            and that year is not leap in the Gregorian calendar. """
        self.check_both_conversions(
            (1500, 2, 29),
            (1500, 3, 10)
        )

    def test_another_Julian_only_leap_day(self):
        """ This date is a leap day of the Julian calendar.
            The corresponding day of the Gregorian calendar is not a leap day,
            and that year is not leap in the Gregorian calendar. """
        self.check_both_conversions(
            (1400, 2, 29),
            (1400, 3, 9)
        )

    def test_the_day_before_a_Julian_only_leap_day(self):
        """ This date is Feb 28th before leap day of the Julian calendar.
            The corresponding day of the Gregorian calendar is not a leap day,
            and that year is not leap in the Gregorian calendar. """
        self.check_both_conversions(
            (1500, 2, 28),
            (1500, 3, 9)
        )

    def test_a_Gregorian_March_1st(self):
        """ This Gregorian date is March 1st, of a Julian-only leap year.
            Although the Julian date falls before the leap day,
            since the Gregorian date falls after it, the number of
            days difference between the dates in the two calendars
            depends which calendar you use to measure it. """
        self.check_both_conversions(
            (1500, 2, 20),
            (1500, 3, 1)
        )

    def test_a_Gregorian_Feb_28th(self):
        """ This Gregorian date is Feb 28th, of a Julian-only leap year.
            Both the Gregorian date, and the Julian date, are unaffected
            by the Julian leap day later in that year."""
        self.check_both_conversions(
            (1500, 2, 19),
            (1500, 2, 28)
        )

    @given(datetimes(timezones=[]))
    def test_round_trip_Gregorian_Julian_Gregorian(self, dt):
        original_gregorian_date = self.gregorian.from_date(dt.date())
        converted_date = original_gregorian_date.convert_to(self.julian)
        round_tripped_date = converted_date.convert_to(self.gregorian)
        self.assertEqual(original_gregorian_date, round_tripped_date)

    @given(datetimes(timezones=[]))
    def test_round_trip_Julian_Gregorian_Julian(self, dt):
        original_julian_date = self.julian.from_date(dt.date())
        converted_date = original_julian_date.convert_to(self.gregorian)
        round_tripped_date = converted_date.convert_to(self.julian)
        self.assertEqual(original_julian_date, round_tripped_date)

<|endoftext|>


<commit_before>__version__ = '2.2.9'
<commit_msg>update version num<commit_after>__version__ = '2.2.11'
<|endoftext|>


<commit_before>
from rest_framework.permissions import IsAuthenticated

from accelerator_abstract.models.base_user_utils import (
    is_employee,
    is_expert,
)
from accelerator.models import UserRole
from impact.permissions import (
    settings,
    BasePermission,
)

SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS']
DEFAULT_PERMISSION_DENIED_DETAIL = ("You do not have permission to perform "
                                    "this action.")
CREATE_PERMISSION_DENIED_DETAIL = ("You do not have permission to create "
                                   "this office hour")
EDIT_PERMISSION_DENIED_DETAIL = ("You do not have permission to edit this "
                                 "office hour")
RESERVE_PERMISSION_DENIED_DETAIL = ("You do not have permission to reserve "
                                    "this office hour")
CANCEL_SESSION_PERMISSION_DENIED_DETAIL = ("You do not have permission to "
                                           "cancel this office hour")
CANCEL_RESERVATION_PERMISSION_DENIED_DETAIL = ("You do not have permission "
                                               "to cancel this session")
OFFICE_HOUR_RESERVERS = [UserRole.FINALIST, UserRole.AIR, UserRole.ALUM]

def can_view_user_details_page(request):
    user_id = request.parser_context['kwargs'].get('pk', '')
    if user_id and request.method in SAFE_METHODS:
        return str(user_id) == str(request.user.id)

class V1APIPermissions(BasePermission):
    authenticated_users_only = True

    def has_permission(self, request, view):
        return (is_employee(request.user) or
                request.user.groups.filter(
                    name=settings.V1_API_GROUP).exists())

class UserDetailViewPermission(V1APIPermissions):
    def has_permission(self, request, view):
        return (super().has_permission(request, view) or
                can_view_user_details_page(request))

class OfficeHourMentorPermission(BasePermission):
    # User has permission to act as mentor on this office hour
    message = CANCEL_SESSION_PERMISSION_DENIED_DETAIL

    def has_object_permission(self, request, view, office_hour):
        return (is_employee(request.user) or
                office_hour.mentor == request.user)

class OfficeHourFinalistPermission(BasePermission):
    # User has permission to act as finalist on this office hour
    message = CANCEL_RESERVATION_PERMISSION_DENIED_DETAIL

    def has_object_permission(self, request, view, office_hour):
        return (is_employee(request.user) or
                office_hour.finalist == request.user)

class IsExpertUser(IsAuthenticated):
    def has_permission(self, request, view):
        return (super().has_permission(request, view) and
                is_expert(request.user))

class OfficeHourPermission(IsAuthenticated):
    def has_permission(self, request, view):
        self.get_message(request)
        roles = [UserRole.MENTOR, UserRole.AIR]
        return super().has_permission(request, view) and (
            is_employee(request.user) or
            request.user.programrolegrant_set.filter(
                program_role__user_role__name__in=roles,
                program_role__program__program_status='active',
            ).exists())

    def has_object_permission(self, request, view, office_hour):
        is_reserved = office_hour.finalist
        return (is_employee(request.user) or
                office_hour.mentor == request.user and not is_reserved)

    def get_message(self, request):
        detail_messages = {'POST': CREATE_PERMISSION_DENIED_DETAIL,
                           'PATCH': EDIT_PERMISSION_DENIED_DETAIL}
        self.message = detail_messages.get(request.method,
                                           DEFAULT_PERMISSION_DENIED_DETAIL)
<commit_msg>[AC-7953] Replace string literal with defined constant<commit_after>
from rest_framework.permissions import IsAuthenticated

from accelerator_abstract.models.base_user_utils import (
    is_employee,
    is_expert,
)
from accelerator_abstract.models import ACTIVE_PROGRAM_STATUS

from accelerator.models import UserRole
from impact.permissions import (
    settings,
    BasePermission,
)

SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS']
DEFAULT_PERMISSION_DENIED_DETAIL = ("You do not have permission to perform "
                                    "this action.")
CREATE_PERMISSION_DENIED_DETAIL = ("You do not have permission to create "
                                   "this office hour")
EDIT_PERMISSION_DENIED_DETAIL = ("You do not have permission to edit this "
                                 "office hour")
RESERVE_PERMISSION_DENIED_DETAIL = ("You do not have permission to reserve "
                                    "this office hour")
CANCEL_SESSION_PERMISSION_DENIED_DETAIL = ("You do not have permission to "
                                           "cancel this office hour")
CANCEL_RESERVATION_PERMISSION_DENIED_DETAIL = ("You do not have permission "
                                               "to cancel this session")
OFFICE_HOUR_RESERVERS = [UserRole.FINALIST, UserRole.AIR, UserRole.ALUM]

def can_view_user_details_page(request):
    user_id = request.parser_context['kwargs'].get('pk', '')
    if user_id and request.method in SAFE_METHODS:
        return str(user_id) == str(request.user.id)

class V1APIPermissions(BasePermission):
    authenticated_users_only = True

    def has_permission(self, request, view):
        return (is_employee(request.user) or
                request.user.groups.filter(
                    name=settings.V1_API_GROUP).exists())

class UserDetailViewPermission(V1APIPermissions):
    def has_permission(self, request, view):
        return (super().has_permission(request, view) or
                can_view_user_details_page(request))

class OfficeHourMentorPermission(BasePermission):
    # User has permission to act as mentor on this office hour
    message = CANCEL_SESSION_PERMISSION_DENIED_DETAIL

    def has_object_permission(self, request, view, office_hour):
        return (is_employee(request.user) or
                office_hour.mentor == request.user)

class OfficeHourFinalistPermission(BasePermission):
    # User has permission to act as finalist on this office hour
    message = CANCEL_RESERVATION_PERMISSION_DENIED_DETAIL

    def has_object_permission(self, request, view, office_hour):
        return (is_employee(request.user) or
                office_hour.finalist == request.user)

class IsExpertUser(IsAuthenticated):
    def has_permission(self, request, view):
        return (super().has_permission(request, view) and
                is_expert(request.user))

class OfficeHourPermission(IsAuthenticated):
    def has_permission(self, request, view):
        self.get_message(request)
        roles = [UserRole.MENTOR, UserRole.AIR]
        return super().has_permission(request, view) and (
            is_employee(request.user) or
            request.user.programrolegrant_set.filter(
                program_role__user_role__name__in=roles,
                program_role__program__program_status=ACTIVE_PROGRAM_STATUS,
            ).exists())

    def has_object_permission(self, request, view, office_hour):
        is_reserved = office_hour.finalist
        return (is_employee(request.user) or
                office_hour.mentor == request.user and not is_reserved)

    def get_message(self, request):
        detail_messages = {'POST': CREATE_PERMISSION_DENIED_DETAIL,
                           'PATCH': EDIT_PERMISSION_DENIED_DETAIL}
        self.message = detail_messages.get(request.method,
                                           DEFAULT_PERMISSION_DENIED_DETAIL)
<|endoftext|>


<commit_before>import re
import json
import logging
from typing import Text, Optional, List

from flask import Blueprint, request, jsonify, make_response, Response
from slackclient import SlackClient

from rasa_core.channels import InputChannel
from rasa_core.channels.channel import UserMessage, OutputChannel

logger = logging.getLogger(__name__)

class SlackBot(SlackClient, OutputChannel):
    """A Slack communication channel"""

    @classmethod
    def name(cls):
        return "slack"

    def __init__(self,
                 token: Text,
                 slack_channel: Optional[Text] = None) -> None:

        self.slack_channel = slack_channel
        super(SlackBot, self).__init__(token)

    def send_text_message(self, recipient_id, message):
        recipient = self.slack_channel or recipient_id
        for message_part in message.split("\n\n"):
            super(SlackBot, self).api_call("chat.postMessage",
                                           channel=recipient,
                                           as_user=True, text=message_part)

    def send_image_url(self, recipient_id, image_url, message=""):
        image_attachment = [{"image_url": image_url,
                             "text": message}]
        recipient = self.slack_channel or recipient_id
        return super(SlackBot, self).api_call("chat.postMessage",
                                              channel=recipient,
                                              as_user=True,
                                              attachments=image_attachment)

    def send_attachment(self, recipient_id, attachment, message=""):
        recipient = self.slack_channel or recipient_id
        return super(SlackBot, self).api_call("chat.postMessage",
                                              channel=recipient,
                                              as_user=True,
                                              text=message,
                                              attachments=attachment)

    @staticmethod
    def _convert_to_slack_buttons(buttons):
        return [{"text": b['title'],
                 "name": b['payload'],
                 "type": "button"} for b in buttons]

    def send_text_with_buttons(self, recipient_id, message, buttons, **kwargs):
        recipient = self.slack_channel or recipient_id

        if len(buttons) > 5:
            logger.warning("Slack API currently allows only up to 5 buttons. "
                           "If you add more, all will be ignored.")
            return self.send_text_message(recipient, message)

        button_attachment = [{"fallback": message,
                              "callback_id": message.replace(' ', '_')[:20],
                              "actions": self._convert_to_slack_buttons(
                                  buttons)}]

        super(SlackBot, self).api_call("chat.postMessage",
                                       channel=recipient,
                                       as_user=True,
                                       text=message,
                                       attachments=button_attachment)

class SlackInput(InputChannel):
    """Slack input channel implementation. Based on the HTTPInputChannel."""

    @classmethod
    def name(cls):
        return "slack"

    @classmethod
    def from_credentials(cls, credentials):
        if not credentials:
            cls.raise_missing_credentials_exception()

        return cls(credentials.get("slack_token"),
                   credentials.get("slack_channel"))

    def __init__(self,
                 slack_token: Text,
                 slack_channel: Optional[Text] = None,
                 errors_ignore_retry: Optional[List[Text]] = None) -> None:
        """Create a Slack input channel.

        Needs a couple of settings to properly authenticate and validate
        messages. Details to setup:

        https://github.com/slackapi/python-slackclient

        Args:
            slack_token: Your Slack Authentication token. You can find or
                generate a test token
                `here <https://api.slack.com/docs/oauth-test-tokens>`_.
            slack_channel: the string identifier for a channel to which
                the bot posts, or channel name (e.g. 'C1234ABC', 'bot-test'
                or '#bot-test') If unset, messages will be sent back
                to the user they came from.
            errors_ignore_retry: If error code given by slack
                included in this list then it will ignore the event.
                The code is listed here:
                https://api.slack.com/events-api#errors
        """
        self.slack_token = slack_token
        self.slack_channel = slack_channel
        self.errors_ignore_retry = errors_ignore_retry or ('http_timeout',)

    @staticmethod
    def _is_user_message(slack_event):
        return (slack_event.get('event') and
                (slack_event.get('event').get('type') == u'message' or
                 slack_event.get('event').get('type') == u'app_mention') and
                slack_event.get('event').get('text') and not
                slack_event.get('event').get('bot_id'))

    @staticmethod
    def _is_button_reply(slack_event):
        return (slack_event.get('payload') and
                slack_event['payload'][0] and
                'name' in slack_event['payload'][0])

    @staticmethod
    def _get_button_reply(slack_event):
        return json.loads(slack_event['payload'][0])['actions'][0]['name']

    @staticmethod
    def _sanitize_user_message(text, uids_to_remove):
        """
        Probably a good starting point for pre-formatting of user-provided text,
        to make NLUs life easier in case they go funky to the power of extreme.

        In the current state will just drop self-mentions of bot itself

        :param text: raw message
        :param users_to_drop: a list of users to remove from the content
        :return text:
        """
        for uid_to_remove in uids_to_remove:
            # the seeming duplication turns out to be an enough heuristic
            # to format majority of cases OK
            # can be adjusted to taste later on if needed,
            # but is a good first approximation
            text = re.sub(
                r'<@{}>\s'.format(uid_to_remove),
                '',
                text
            )
            text = re.sub(
                r'\s<@{}>'.format(uid_to_remove),
                '',
                text
            )

        return text.rstrip()  # drop an optional extra space at the end

    def process_message(self, on_new_message, text, sender_id):
        """Slack retry to post messages up to 3 times based on
        failure conditions defined here:
        https://api.slack.com/events-api#failure_conditions
        """
        retry_reason = request.headers.environ.get('HTTP_X_SLACK_RETRY_REASON')
        retry_count = request.headers.environ.get('HTTP_X_SLACK_RETRY_NUM')
        if retry_count and retry_reason in self.errors_ignore_retry:
            logger.warning("Received retry #{} request from slack"
                           " due to {}".format(retry_count, retry_reason))

            return Response(status=201, headers={'X-Slack-No-Retry': 1})

        try:
            out_channel = SlackBot(self.slack_token, self.slack_channel)
            user_msg = UserMessage(text, out_channel, sender_id,
                                   input_channel=self.name())
            on_new_message(user_msg)
        except Exception as e:
            logger.error("Exception when trying to handle "
                         "message.{0}".format(e))
            logger.error(str(e), exc_info=True)

        return make_response()

    def blueprint(self, on_new_message):
        slack_webhook = Blueprint('slack_webhook', __name__)

        @slack_webhook.route("/", methods=['GET'])
        def health():
            return jsonify({"status": "ok"})

        @slack_webhook.route("/webhook", methods=['GET', 'POST'])
        def webhook():
            request.get_data()
            if request.json:
                output = request.json
                if "challenge" in output:
                    return make_response(output.get("challenge"), 200,
                                         {"content_type": "application/json"})
                elif self._is_user_message(output):
                    return self.process_message(
                        on_new_message,
                        text=self._sanitize_user_message(
                            output['event']['text'],
                            output['authed_users']),
                        sender_id=output.get('event').get('user'))
            elif request.form:
                output = dict(request.form)
                if self._is_button_reply(output):
                    sender_id = json.loads(output['payload'][0])['user']['id']
                    return self.process_message(
                        on_new_message,
                        text=self._get_button_reply(output),
                        sender_id=sender_id)

            return make_response()

        return slack_webhook
<commit_msg>only docstring and descriptions tweaks<commit_after>import re
import json
import logging
from typing import Text, Optional, List

from flask import Blueprint, request, jsonify, make_response, Response
from slackclient import SlackClient

from rasa_core.channels import InputChannel
from rasa_core.channels.channel import UserMessage, OutputChannel

logger = logging.getLogger(__name__)

class SlackBot(SlackClient, OutputChannel):
    """A Slack communication channel"""

    @classmethod
    def name(cls):
        return "slack"

    def __init__(self,
                 token: Text,
                 slack_channel: Optional[Text] = None) -> None:

        self.slack_channel = slack_channel
        super(SlackBot, self).__init__(token)

    def send_text_message(self, recipient_id, message):
        recipient = self.slack_channel or recipient_id
        for message_part in message.split("\n\n"):
            super(SlackBot, self).api_call("chat.postMessage",
                                           channel=recipient,
                                           as_user=True, text=message_part)

    def send_image_url(self, recipient_id, image_url, message=""):
        image_attachment = [{"image_url": image_url,
                             "text": message}]
        recipient = self.slack_channel or recipient_id
        return super(SlackBot, self).api_call("chat.postMessage",
                                              channel=recipient,
                                              as_user=True,
                                              attachments=image_attachment)

    def send_attachment(self, recipient_id, attachment, message=""):
        recipient = self.slack_channel or recipient_id
        return super(SlackBot, self).api_call("chat.postMessage",
                                              channel=recipient,
                                              as_user=True,
                                              text=message,
                                              attachments=attachment)

    @staticmethod
    def _convert_to_slack_buttons(buttons):
        return [{"text": b['title'],
                 "name": b['payload'],
                 "type": "button"} for b in buttons]

    def send_text_with_buttons(self, recipient_id, message, buttons, **kwargs):
        recipient = self.slack_channel or recipient_id

        if len(buttons) > 5:
            logger.warning("Slack API currently allows only up to 5 buttons. "
                           "If you add more, all will be ignored.")
            return self.send_text_message(recipient, message)

        button_attachment = [{"fallback": message,
                              "callback_id": message.replace(' ', '_')[:20],
                              "actions": self._convert_to_slack_buttons(
                                  buttons)}]

        super(SlackBot, self).api_call("chat.postMessage",
                                       channel=recipient,
                                       as_user=True,
                                       text=message,
                                       attachments=button_attachment)

class SlackInput(InputChannel):
    """Slack input channel implementation. Based on the HTTPInputChannel."""

    @classmethod
    def name(cls):
        return "slack"

    @classmethod
    def from_credentials(cls, credentials):
        if not credentials:
            cls.raise_missing_credentials_exception()

        return cls(credentials.get("slack_token"),
                   credentials.get("slack_channel"))

    def __init__(self,
                 slack_token: Text,
                 slack_channel: Optional[Text] = None,
                 errors_ignore_retry: Optional[List[Text]] = None) -> None:
        """Create a Slack input channel.

        Needs a couple of settings to properly authenticate and validate
        messages. Details to setup:

        https://github.com/slackapi/python-slackclient

        Args:
            slack_token: Your Slack Authentication token. You can find or
                generate a test token
                `here <https://api.slack.com/docs/oauth-test-tokens>`_.
            slack_channel: the string identifier for a channel to which
                the bot posts, or channel name (e.g. 'C1234ABC', 'bot-test'
                or '#bot-test') If unset, messages will be sent back
                to the user they came from.
            errors_ignore_retry: If error code given by slack
                included in this list then it will ignore the event.
                The code is listed here:
                https://api.slack.com/events-api#errors
        """
        self.slack_token = slack_token
        self.slack_channel = slack_channel
        self.errors_ignore_retry = errors_ignore_retry or ('http_timeout',)

    @staticmethod
    def _is_user_message(slack_event):
        return (slack_event.get('event') and
                (slack_event.get('event').get('type') == u'message' or
                 slack_event.get('event').get('type') == u'app_mention') and
                slack_event.get('event').get('text') and not
                slack_event.get('event').get('bot_id'))

    @staticmethod
    def _is_button_reply(slack_event):
        return (slack_event.get('payload') and
                slack_event['payload'][0] and
                'name' in slack_event['payload'][0])

    @staticmethod
    def _get_button_reply(slack_event):
        return json.loads(slack_event['payload'][0])['actions'][0]['name']

    @staticmethod
    def _sanitize_user_message(text, uids_to_remove):
        """Remove superfluous/wrong/problematic tokens from a message
        Probably a good starting point for pre-formatting of user-provided text,
        to make NLU's life easier in case they go funky to the power of extreme.

        In the current state will just drop self-mentions of bot itself

        Args:
            text: raw message as sent from slack
            uids_to_remove: a list of user ids to remove from the content

        Returns text: a parsed and cleaned string
        """
        for uid_to_remove in uids_to_remove:
            # the seeming duplication turns out to be an enough heuristic
            # to format majority of cases OK
            # can be adjusted to taste later on if needed,
            # but is a good first approximation
            text = re.sub(
                r'<@{}>\s'.format(uid_to_remove),
                '',
                text
            )
            text = re.sub(
                r'\s<@{}>'.format(uid_to_remove),
                '',
                text
            )

        return text.rstrip()  # drop an optional extra space at the end

    def process_message(self, on_new_message, text, sender_id):
        """Slack retry to post messages up to 3 times based on
        failure conditions defined here:
        https://api.slack.com/events-api#failure_conditions
        """
        retry_reason = request.headers.environ.get('HTTP_X_SLACK_RETRY_REASON')
        retry_count = request.headers.environ.get('HTTP_X_SLACK_RETRY_NUM')
        if retry_count and retry_reason in self.errors_ignore_retry:
            logger.warning("Received retry #{} request from slack"
                           " due to {}".format(retry_count, retry_reason))

            return Response(status=201, headers={'X-Slack-No-Retry': 1})

        try:
            out_channel = SlackBot(self.slack_token, self.slack_channel)
            user_msg = UserMessage(text, out_channel, sender_id,
                                   input_channel=self.name())
            on_new_message(user_msg)
        except Exception as e:
            logger.error("Exception when trying to handle "
                         "message.{0}".format(e))
            logger.error(str(e), exc_info=True)

        return make_response()

    def blueprint(self, on_new_message):
        slack_webhook = Blueprint('slack_webhook', __name__)

        @slack_webhook.route("/", methods=['GET'])
        def health():
            return jsonify({"status": "ok"})

        @slack_webhook.route("/webhook", methods=['GET', 'POST'])
        def webhook():
            request.get_data()
            if request.json:
                output = request.json
                if "challenge" in output:
                    return make_response(output.get("challenge"), 200,
                                         {"content_type": "application/json"})
                elif self._is_user_message(output):
                    return self.process_message(
                        on_new_message,
                        text=self._sanitize_user_message(
                            output['event']['text'],
                            output['authed_users']),
                        sender_id=output.get('event').get('user'))
            elif request.form:
                output = dict(request.form)
                if self._is_button_reply(output):
                    sender_id = json.loads(output['payload'][0])['user']['id']
                    return self.process_message(
                        on_new_message,
                        text=self._get_button_reply(output),
                        sender_id=sender_id)

            return make_response()

        return slack_webhook
<|endoftext|>