<commit_before>#!/usr/bin/python

import os, sys, re, shutil, unittest, doctest

WITH_CYTHON = True
CLEANUP_WORKDIR = True

from distutils.dist import Distribution
from distutils.core import Extension
from distutils.command.build_ext import build_ext
distutils_distro = Distribution()

TEST_DIRS = ['compile', 'errors', 'run']
TEST_RUN_DIRS = ['run']

INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
CFLAGS = os.getenv('CFLAGS', '').split()

class ErrorWriter(object):
    match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
    def __init__(self):
        self.output = []
        self.write = self.output.append

    def geterrors(self):
        s = ''.join(self.output)
        errors = []
        for line in s.split('\n'):
            match = self.match_error(line)
            if match:
                line, column, message = match.groups()
                errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
        return errors

class TestBuilder(object):
    def __init__(self, rootdir, workdir, selectors, annotate):
        self.rootdir = rootdir
        self.workdir = workdir
        self.selectors = selectors
        self.annotate = annotate

    def build_suite(self):
        suite = unittest.TestSuite()
        filenames = os.listdir(self.rootdir)
        filenames.sort()
        for filename in filenames:
            if not WITH_CYTHON and filename == "errors":
                # we won't get any errors without running Cython
                continue
            path = os.path.join(self.rootdir, filename)
            if os.path.isdir(path) and filename in TEST_DIRS:
                suite.addTest(
                    self.handle_directory(path, filename))
        return suite

    def handle_directory(self, path, context):
        workdir = os.path.join(self.workdir, context)
        if not os.path.exists(workdir):
            os.makedirs(workdir)
        if workdir not in sys.path:
            sys.path.insert(0, workdir)

        expect_errors = (context == 'errors')
        suite = unittest.TestSuite()
        filenames = os.listdir(path)
        filenames.sort()
        for filename in filenames:
            if not filename.endswith(".pyx"):
                continue
            module = filename[:-4]
            fqmodule = "%s.%s" % (context, module)
            if not [ 1 for match in self.selectors
                     if match(fqmodule) ]:
                continue
            if context in TEST_RUN_DIRS:
                test = CythonRunTestCase(
                    path, workdir, module, self.annotate)
            else:
                test = CythonCompileTestCase(
                    path, workdir, module, expect_errors, self.annotate)
            suite.addTest(test)
        return suite

class CythonCompileTestCase(unittest.TestCase):
    def __init__(self, directory, workdir, module,
                 expect_errors=False, annotate=False):
        self.directory = directory
        self.workdir = workdir
        self.module = module
        self.expect_errors = expect_errors
        self.annotate = annotate
        unittest.TestCase.__init__(self)

    def shortDescription(self):
        return "compiling " + self.module

    def tearDown(self):
        cleanup_c_files = WITH_CYTHON and CLEANUP_WORKDIR
        if os.path.exists(self.workdir):
            for rmfile in os.listdir(self.workdir):
                if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
                    continue
                if self.annotate and rmfile.endswith(".html"):
                    continue
                try:
                    rmfile = os.path.join(self.workdir, rmfile)
                    if os.path.isdir(rmfile):
                        shutil.rmtree(rmfile, ignore_errors=True)
                    else:
                        os.remove(rmfile)
                except IOError:
                    pass
        else:
            os.makedirs(self.workdir)

    def runTest(self):
        self.compile(self.directory, self.module, self.workdir,
                     self.directory, self.expect_errors, self.annotate)

    def split_source_and_output(self, directory, module, workdir):
        source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
        out = open(os.path.join(workdir, module + '.pyx'), 'w')
        for line in source_and_output:
            last_line = line
            if line.startswith("_ERRORS"):
                out.close()
                out = ErrorWriter()
            else:
                out.write(line)
        try:
            geterrors = out.geterrors
        except AttributeError:
            return []
        else:
            return geterrors()

    def run_cython(self, directory, module, targetdir, incdir, annotate):
        include_dirs = INCLUDE_DIRS[:]
        if incdir:
            include_dirs.append(incdir)
        source = os.path.join(directory, module + '.pyx')
        target = os.path.join(targetdir, module + '.c')
        options = CompilationOptions(
            pyrex_default_options,
            include_path = include_dirs,
            output_file = target,
            annotate = annotate,
            use_listing_file = False, cplus = False, generate_pxi = False)
        cython_compile(source, options=options,
                       full_module_name=module)

    def run_distutils(self, module, workdir, incdir):
        cwd = os.getcwd()
        os.chdir(workdir)
        try:
            build_extension = build_ext(distutils_distro)
            build_extension.include_dirs = INCLUDE_DIRS[:]
            if incdir:
                build_extension.include_dirs.append(incdir)
            build_extension.finalize_options()

            extension = Extension(
                module,
                sources = [module + '.c'],
                extra_compile_args = CFLAGS,
                )
            build_extension.extensions = [extension]
            build_extension.build_temp = workdir
            build_extension.build_lib  = workdir
            build_extension.run()
        finally:
            os.chdir(cwd)

    def compile(self, directory, module, workdir, incdir,
                expect_errors, annotate):
        expected_errors = errors = ()
        if expect_errors:
            expected_errors = self.split_source_and_output(
                directory, module, workdir)
            directory = workdir

        if WITH_CYTHON:
            old_stderr = sys.stderr
            try:
                sys.stderr = ErrorWriter()
                self.run_cython(directory, module, workdir, incdir, annotate)
                errors = sys.stderr.geterrors()
            finally:
                sys.stderr = old_stderr

        if errors or expected_errors:
            for expected, error in zip(expected_errors, errors):
                self.assertEquals(expected, error)
            if len(errors) < len(expected_errors):
                expected_error = expected_errors[len(errors)]
                self.assertEquals(expected_error, None)
            elif len(errors) > len(expected_errors):
                unexpected_error = errors[len(expected_errors)]
                self.assertEquals(None, unexpected_error)
        else:
            self.run_distutils(module, workdir, incdir)

class CythonRunTestCase(CythonCompileTestCase):
    def shortDescription(self):
        return "compiling and running " + self.module

    def run(self, result=None):
        if result is None:
            result = self.defaultTestResult()
        try:
            self.runTest()
            doctest.DocTestSuite(self.module).run(result)
        except Exception:
            result.startTest(self)
            result.addError(self, sys.exc_info())
            result.stopTest(self)
        try:
            self.tearDown()
        except Exception:
            pass

if __name__ == '__main__':
    try:
        sys.argv.remove("--no-cython")
    except ValueError:
        WITH_CYTHON = True
    else:
        WITH_CYTHON = False

    if WITH_CYTHON:
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile

    from distutils.dist import Distribution
    from distutils.core import Extension
    from distutils.command.build_ext import build_ext
    distutils_distro = Distribution()

    # RUN ALL TESTS!
    ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
    WORKDIR = os.path.join(os.getcwd(), 'BUILD')
    if WITH_CYTHON:
        if os.path.exists(WORKDIR):
            shutil.rmtree(WORKDIR, ignore_errors=True)
    if not os.path.exists(WORKDIR):
        os.makedirs(WORKDIR)

    if WITH_CYTHON:
        from Cython.Compiler.Version import version
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile
        print("Running tests against Cython %s" % version)
    else:
        print("Running tests without Cython.")
    print("Python %s" % sys.version)
    print("")

    try:
        sys.argv.remove("-C")
    except ValueError:
        coverage = None
    else:
        import coverage
        coverage.erase()

    try:
        sys.argv.remove("--no-cleanup")
    except ValueError:
        CLEANUP_WORKDIR = True
    else:
        CLEANUP_WORKDIR = False

    try:
        sys.argv.remove("-a")
    except ValueError:
        annotate_source = False
    else:
        annotate_source = True

    import re
    selectors = [ re.compile(r, re.I|re.U).search for r in sys.argv[1:] ]
    if not selectors:
        selectors = [ lambda x:True ]

    tests = TestBuilder(ROOTDIR, WORKDIR, selectors, annotate_source)
    test_suite = tests.build_suite()

    if coverage is not None:
        coverage.start()

    unittest.TextTestRunner(verbosity=2).run(test_suite)

    if coverage is not None:
        coverage.stop()
        ignored_modules = ('Options', 'Version', 'DebugFlags')
        modules = [ module for name, module in sys.modules.items()
                    if module is not None and
                    name.startswith('Cython.Compiler.') and
                    name[len('Cython.Compiler.'):] not in ignored_modules ]
        coverage.report(modules, show_missing=0)
<commit_msg>better test output<commit_after>#!/usr/bin/python

import os, sys, re, shutil, unittest, doctest

WITH_CYTHON = True
CLEANUP_WORKDIR = True

from distutils.dist import Distribution
from distutils.core import Extension
from distutils.command.build_ext import build_ext
distutils_distro = Distribution()

TEST_DIRS = ['compile', 'errors', 'run']
TEST_RUN_DIRS = ['run']

INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
CFLAGS = os.getenv('CFLAGS', '').split()

class ErrorWriter(object):
    match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
    def __init__(self):
        self.output = []
        self.write = self.output.append

    def geterrors(self):
        s = ''.join(self.output)
        errors = []
        for line in s.split('\n'):
            match = self.match_error(line)
            if match:
                line, column, message = match.groups()
                errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
        return errors

class TestBuilder(object):
    def __init__(self, rootdir, workdir, selectors, annotate):
        self.rootdir = rootdir
        self.workdir = workdir
        self.selectors = selectors
        self.annotate = annotate

    def build_suite(self):
        suite = unittest.TestSuite()
        filenames = os.listdir(self.rootdir)
        filenames.sort()
        for filename in filenames:
            if not WITH_CYTHON and filename == "errors":
                # we won't get any errors without running Cython
                continue
            path = os.path.join(self.rootdir, filename)
            if os.path.isdir(path) and filename in TEST_DIRS:
                suite.addTest(
                    self.handle_directory(path, filename))
        return suite

    def handle_directory(self, path, context):
        workdir = os.path.join(self.workdir, context)
        if not os.path.exists(workdir):
            os.makedirs(workdir)
        if workdir not in sys.path:
            sys.path.insert(0, workdir)

        expect_errors = (context == 'errors')
        suite = unittest.TestSuite()
        filenames = os.listdir(path)
        filenames.sort()
        for filename in filenames:
            if not filename.endswith(".pyx"):
                continue
            module = filename[:-4]
            fqmodule = "%s.%s" % (context, module)
            if not [ 1 for match in self.selectors
                     if match(fqmodule) ]:
                continue
            if context in TEST_RUN_DIRS:
                test = CythonRunTestCase(
                    path, workdir, module, self.annotate)
            else:
                test = CythonCompileTestCase(
                    path, workdir, module, expect_errors, self.annotate)
            suite.addTest(test)
        return suite

class CythonCompileTestCase(unittest.TestCase):
    def __init__(self, directory, workdir, module,
                 expect_errors=False, annotate=False):
        self.directory = directory
        self.workdir = workdir
        self.module = module
        self.expect_errors = expect_errors
        self.annotate = annotate
        unittest.TestCase.__init__(self)

    def shortDescription(self):
        return "compiling " + self.module

    def tearDown(self):
        cleanup_c_files = WITH_CYTHON and CLEANUP_WORKDIR
        if os.path.exists(self.workdir):
            for rmfile in os.listdir(self.workdir):
                if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
                    continue
                if self.annotate and rmfile.endswith(".html"):
                    continue
                try:
                    rmfile = os.path.join(self.workdir, rmfile)
                    if os.path.isdir(rmfile):
                        shutil.rmtree(rmfile, ignore_errors=True)
                    else:
                        os.remove(rmfile)
                except IOError:
                    pass
        else:
            os.makedirs(self.workdir)

    def runTest(self):
        self.compile(self.directory, self.module, self.workdir,
                     self.directory, self.expect_errors, self.annotate)

    def split_source_and_output(self, directory, module, workdir):
        source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
        out = open(os.path.join(workdir, module + '.pyx'), 'w')
        for line in source_and_output:
            last_line = line
            if line.startswith("_ERRORS"):
                out.close()
                out = ErrorWriter()
            else:
                out.write(line)
        try:
            geterrors = out.geterrors
        except AttributeError:
            return []
        else:
            return geterrors()

    def run_cython(self, directory, module, targetdir, incdir, annotate):
        include_dirs = INCLUDE_DIRS[:]
        if incdir:
            include_dirs.append(incdir)
        source = os.path.join(directory, module + '.pyx')
        target = os.path.join(targetdir, module + '.c')
        options = CompilationOptions(
            pyrex_default_options,
            include_path = include_dirs,
            output_file = target,
            annotate = annotate,
            use_listing_file = False, cplus = False, generate_pxi = False)
        cython_compile(source, options=options,
                       full_module_name=module)

    def run_distutils(self, module, workdir, incdir):
        cwd = os.getcwd()
        os.chdir(workdir)
        try:
            build_extension = build_ext(distutils_distro)
            build_extension.include_dirs = INCLUDE_DIRS[:]
            if incdir:
                build_extension.include_dirs.append(incdir)
            build_extension.finalize_options()

            extension = Extension(
                module,
                sources = [module + '.c'],
                extra_compile_args = CFLAGS,
                )
            build_extension.extensions = [extension]
            build_extension.build_temp = workdir
            build_extension.build_lib  = workdir
            build_extension.run()
        finally:
            os.chdir(cwd)

    def compile(self, directory, module, workdir, incdir,
                expect_errors, annotate):
        expected_errors = errors = ()
        if expect_errors:
            expected_errors = self.split_source_and_output(
                directory, module, workdir)
            directory = workdir

        if WITH_CYTHON:
            old_stderr = sys.stderr
            try:
                sys.stderr = ErrorWriter()
                self.run_cython(directory, module, workdir, incdir, annotate)
                errors = sys.stderr.geterrors()
            finally:
                sys.stderr = old_stderr

        if errors or expected_errors:
            for expected, error in zip(expected_errors, errors):
                self.assertEquals(expected, error)
            if len(errors) < len(expected_errors):
                expected_error = expected_errors[len(errors)]
                self.assertEquals(expected_error, None)
            elif len(errors) > len(expected_errors):
                unexpected_error = errors[len(expected_errors)]
                self.assertEquals(None, unexpected_error)
        else:
            self.run_distutils(module, workdir, incdir)

class CythonRunTestCase(CythonCompileTestCase):
    def shortDescription(self):
        return "compiling and running " + self.module

    def run(self, result=None):
        if result is None:
            result = self.defaultTestResult()
        result.startTest(self)
        try:
            self.runTest()
            doctest.DocTestSuite(self.module).run(result)
        except Exception:
            result.addError(self, sys.exc_info())
            result.stopTest(self)
        try:
            self.tearDown()
        except Exception:
            pass

if __name__ == '__main__':
    try:
        sys.argv.remove("--no-cython")
    except ValueError:
        WITH_CYTHON = True
    else:
        WITH_CYTHON = False

    if WITH_CYTHON:
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile

    from distutils.dist import Distribution
    from distutils.core import Extension
    from distutils.command.build_ext import build_ext
    distutils_distro = Distribution()

    # RUN ALL TESTS!
    ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
    WORKDIR = os.path.join(os.getcwd(), 'BUILD')
    if WITH_CYTHON:
        if os.path.exists(WORKDIR):
            shutil.rmtree(WORKDIR, ignore_errors=True)
    if not os.path.exists(WORKDIR):
        os.makedirs(WORKDIR)

    if WITH_CYTHON:
        from Cython.Compiler.Version import version
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile
        print("Running tests against Cython %s" % version)
    else:
        print("Running tests without Cython.")
    print("Python %s" % sys.version)
    print("")

    try:
        sys.argv.remove("-C")
    except ValueError:
        coverage = None
    else:
        import coverage
        coverage.erase()

    try:
        sys.argv.remove("--no-cleanup")
    except ValueError:
        CLEANUP_WORKDIR = True
    else:
        CLEANUP_WORKDIR = False

    try:
        sys.argv.remove("-a")
    except ValueError:
        annotate_source = False
    else:
        annotate_source = True

    import re
    selectors = [ re.compile(r, re.I|re.U).search for r in sys.argv[1:] ]
    if not selectors:
        selectors = [ lambda x:True ]

    tests = TestBuilder(ROOTDIR, WORKDIR, selectors, annotate_source)
    test_suite = tests.build_suite()

    if coverage is not None:
        coverage.start()

    unittest.TextTestRunner(verbosity=2).run(test_suite)

    if coverage is not None:
        coverage.stop()
        ignored_modules = ('Options', 'Version', 'DebugFlags')
        modules = [ module for name, module in sys.modules.items()
                    if module is not None and
                    name.startswith('Cython.Compiler.') and
                    name[len('Cython.Compiler.'):] not in ignored_modules ]
        coverage.report(modules, show_missing=0)
<|endoftext|>


<commit_before>#! /usr/bin/env python
from __future__ import print_function

import os
import subprocess
import sys

import pytest

PYTEST_ARGS = {
    'default': ['tests', '--tb=short', '-s'],
    'fast': ['tests', '--tb=short', '-q', '-s'],
}

FLAKE8_ARGS = ['rest_framework_mongoengine', 'tests', '--ignore=E501']

ISORT_ARGS = ['--recursive', '--check-only', '--verbose', 'rest_framework_mongoengine', 'tests']

sys.path.append(os.path.dirname(__file__))

def exit_on_failure(ret, message=None):
    if ret:
        sys.exit(ret)

def flake8_main(args):
    print('Running flake8 code linting')
    ret = subprocess.call(['flake8'] + args)
    print('flake8 failed' if ret else 'flake8 passed')
    return ret

def isort_main(args):
    print('Running isort code checking')
    ret = subprocess.call(['isort'] + args)

    if ret:
        print('isort failed: Some modules have incorrectly ordered imports. Fix by running `isort --recursive .`')
    else:
        print('isort passed')

    return ret

def split_class_and_function(string):
    class_string, function_string = string.split('.', 1)
    return "%s and %s" % (class_string, function_string)

def is_function(string):
    # `True` if it looks like a test function is included in the string.
    return string.startswith('test_') or '.test_' in string

def is_class(string):
    # `True` if first character is uppercase - assume it's a class name.
    return string[0] == string[0].upper()

if __name__ == "__main__":
    try:
        sys.argv.remove('--nolint')
    except ValueError:
        run_flake8 = True
        run_isort = True
    else:
        run_flake8 = False
        run_isort = False

    try:
        sys.argv.remove('--lintonly')
    except ValueError:
        run_tests = True
    else:
        run_tests = False

    try:
        sys.argv.remove('--fast')
    except ValueError:
        style = 'default'
    else:
        style = 'fast'
        run_flake8 = False
        run_isort = False

    if len(sys.argv) > 1:
        pytest_args = sys.argv[1:]
        first_arg = pytest_args[0]

        try:
            pytest_args.remove('--coverage')
        except ValueError:
            pass
        else:
            pytest_args = [
                '--cov-report',
                'xml',
                '--cov',
                'rest_framework'] + pytest_args

        if first_arg.startswith('-'):
            # `runtests.py [flags]`
            pytest_args = ['tests'] + pytest_args
        elif is_class(first_arg) and is_function(first_arg):
            # `runtests.py TestCase.test_function [flags]`
            expression = split_class_and_function(first_arg)
            pytest_args = ['tests', '-k', expression] + pytest_args[1:]
        elif is_class(first_arg) or is_function(first_arg):
            # `runtests.py TestCase [flags]`
            # `runtests.py test_function [flags]`
            pytest_args = ['tests', '-k', pytest_args[0]] + pytest_args[1:]
    else:
        pytest_args = PYTEST_ARGS[style]

    if run_tests:
        exit_on_failure(pytest.main(pytest_args))

    if run_flake8:
        exit_on_failure(flake8_main(FLAKE8_ARGS))

    if run_isort:
        exit_on_failure(isort_main(ISORT_ARGS))
<commit_msg>Replaced --check-only with --diff to troubleshoot travis.<commit_after>#! /usr/bin/env python
from __future__ import print_function

import os
import subprocess
import sys

import pytest

PYTEST_ARGS = {
    'default': ['tests', '--tb=short', '-s'],
    'fast': ['tests', '--tb=short', '-q', '-s'],
}

FLAKE8_ARGS = ['rest_framework_mongoengine', 'tests', '--ignore=E501']

ISORT_ARGS = ['--recursive', '--diff', '--verbose', 'rest_framework_mongoengine', 'tests']

sys.path.append(os.path.dirname(__file__))

def exit_on_failure(ret, message=None):
    if ret:
        sys.exit(ret)

def flake8_main(args):
    print('Running flake8 code linting')
    ret = subprocess.call(['flake8'] + args)
    print('flake8 failed' if ret else 'flake8 passed')
    return ret

def isort_main(args):
    print('Running isort code checking')
    ret = subprocess.call(['isort'] + args)

    if ret:
        print('isort failed: Some modules have incorrectly ordered imports. Fix by running `isort --recursive .`')
    else:
        print('isort passed')

    return ret

def split_class_and_function(string):
    class_string, function_string = string.split('.', 1)
    return "%s and %s" % (class_string, function_string)

def is_function(string):
    # `True` if it looks like a test function is included in the string.
    return string.startswith('test_') or '.test_' in string

def is_class(string):
    # `True` if first character is uppercase - assume it's a class name.
    return string[0] == string[0].upper()

if __name__ == "__main__":
    try:
        sys.argv.remove('--nolint')
    except ValueError:
        run_flake8 = True
        run_isort = True
    else:
        run_flake8 = False
        run_isort = False

    try:
        sys.argv.remove('--lintonly')
    except ValueError:
        run_tests = True
    else:
        run_tests = False

    try:
        sys.argv.remove('--fast')
    except ValueError:
        style = 'default'
    else:
        style = 'fast'
        run_flake8 = False
        run_isort = False

    if len(sys.argv) > 1:
        pytest_args = sys.argv[1:]
        first_arg = pytest_args[0]

        try:
            pytest_args.remove('--coverage')
        except ValueError:
            pass
        else:
            pytest_args = [
                '--cov-report',
                'xml',
                '--cov',
                'rest_framework'] + pytest_args

        if first_arg.startswith('-'):
            # `runtests.py [flags]`
            pytest_args = ['tests'] + pytest_args
        elif is_class(first_arg) and is_function(first_arg):
            # `runtests.py TestCase.test_function [flags]`
            expression = split_class_and_function(first_arg)
            pytest_args = ['tests', '-k', expression] + pytest_args[1:]
        elif is_class(first_arg) or is_function(first_arg):
            # `runtests.py TestCase [flags]`
            # `runtests.py test_function [flags]`
            pytest_args = ['tests', '-k', pytest_args[0]] + pytest_args[1:]
    else:
        pytest_args = PYTEST_ARGS[style]

    if run_tests:
        exit_on_failure(pytest.main(pytest_args))

    if run_flake8:
        exit_on_failure(flake8_main(FLAKE8_ARGS))

    if run_isort:
        exit_on_failure(isort_main(ISORT_ARGS))
<|endoftext|>


<commit_before>from .models import Campaign

def connect_campaign(sender, **kwargs):
    reference = kwargs.get('reference')
    if not reference:
        return
    parts = reference.split(':', 1)
    if len(parts) != 2:
        return
    namespace = parts[0]
    try:
        campaign = Campaign.objects.get(ident=namespace)
    except Campaign.DoesNotExist:
        return
    sender.campaign = campaign
    sender.save()
<commit_msg>Add tags of campaign to user<commit_after>from .models import Campaign

def connect_campaign(sender, **kwargs):
    reference = kwargs.get('reference')
    if not reference:
        return
    parts = reference.split(':', 1)
    if len(parts) != 2:
        return
    namespace = parts[0]
    try:
        campaign = Campaign.objects.get(ident=namespace)
    except Campaign.DoesNotExist:
        return
    sender.campaign = campaign
    sender.save()

    sender.user.tags.add(campaign.ident)
    if not sender.user.is_active:
        # First-time requester
        sender.user.tags.add('%s-first' % campaign.ident)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
from optparse import OptionParser

from django.conf import settings

if not settings.configured:
    settings.configure(
        DATABASES = {
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
                'NAME': ':memory:',
                'USER': '',
                'PASSWORD': '',
                'HOST': '',
                'PORT': '',
            }
        },

        INSTALLED_APPS=[
            'django.contrib.contenttypes',
            'celery_rpc',
            'celery_rpc.tests',
        ],
        ROOT_URLCONF='',
        DEBUG=True,

        CELERY_RPC_CONFIG = {
            'CELERY_ALWAYS_EAGER': True
        },
    )

from django_nose import NoseTestSuiteRunner

def runtests(*test_args, **kwargs):
    if 'south' in settings.INSTALLED_APPS:
        from south.management.commands import patch_for_test_db_setup
        patch_for_test_db_setup()

    if not test_args:
        test_args = ['celery_rpc']

    test_runner = NoseTestSuiteRunner(**kwargs)

    failures = test_runner.run_tests(test_args)
    sys.exit(failures)

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option('--verbosity', dest='verbosity', action='store',
                      default=1, type=int)
    parser.add_options(NoseTestSuiteRunner.options)
    (options, args) = parser.parse_args()

    runtests(*args, **options.__dict__)
<commit_msg>Fix settings for travis<commit_after>#!/usr/bin/env python

import sys
from optparse import OptionParser

from django.conf import settings

if not settings.configured:
    from celery_rpc.runtests import settings as test_settings
    kwargs = {k: getattr(test_settings, k) for k in dir(test_settings)
              if not k.startswith('_')}
    settings.configure(**kwargs)

from django_nose import NoseTestSuiteRunner

def runtests(*test_args, **kwargs):
    if 'south' in settings.INSTALLED_APPS:
        from south.management.commands import patch_for_test_db_setup
        patch_for_test_db_setup()

    if not test_args:
        test_args = ['celery_rpc']

    test_runner = NoseTestSuiteRunner(**kwargs)

    failures = test_runner.run_tests(test_args)
    sys.exit(failures)

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option('--verbosity', dest='verbosity', action='store',
                      default=1, type=int)
    parser.add_options(NoseTestSuiteRunner.options)
    (options, args) = parser.parse_args()

    runtests(*args, **options.__dict__)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim:ts=4:sw=4:et:

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals

import unittest
import os
import os.path
import sys
# Ensure that we can find pywatchman
sys.path.insert(0, os.path.join(os.getcwd(), 'python'))
sys.path.insert(0, os.path.join(os.getcwd(), 'tests', 'integration'))
import tempfile
import shutil
import subprocess
import traceback
import time
import argparse
import atexit
import WatchmanTapTests
import WatchmanInstance
import WatchmanTestCase
import NodeTests
import glob
import threading
import multiprocessing
import math
import signal
import Interrupt
import random

try:
    import queue
except Exception:
    import Queue
    queue = Queue

parser = argparse.ArgumentParser(
    description="Run the watchman unit and integration tests")
parser.add_argument('-v', '--verbosity', default=2,
                    help="test runner verbosity")
parser.add_argument(
    "--keep",
    action='store_true',
    help="preserve all temporary files created during test execution")
parser.add_argument(
    "--keep-if-fail",
    action='store_true',
    help="preserve all temporary files created during test execution if failed")

parser.add_argument(
    "files",
    nargs='*',
    help='specify which test files to run')

parser.add_argument(
    '--method',
    action='append',
    help='specify which python test method names to run')

parser.add_argument(
    '--concurrency',
    default=int(min(8, math.ceil(1.5 * multiprocessing.cpu_count()))),
    type=int,
    help='How many tests to run at once')

parser.add_argument(
    '--watcher',
    action='store',
    default='auto',
    help='Specify which watcher should be used to run the tests')

args = parser.parse_args()

# We test for this in a test case
os.environ['WATCHMAN_EMPTY_ENV_VAR'] = ''

# Ensure that we find the watchman we built in the tests
os.environ['PATH'] = '%s%s%s' % (
        os.path.abspath(os.path.dirname(__file__)),
        os.pathsep,
        os.environ['PATH'])

# We'll put all our temporary stuff under one dir so that we
# can clean it all up at the end
temp_dir = os.path.realpath(tempfile.mkdtemp(prefix='watchmantest'))
# Redirect all temporary files to that location
tempfile.tempdir = temp_dir

def interrupt_handler(signo, frame):
    Interrupt.setInterrupted()
signal.signal(signal.SIGINT, interrupt_handler)

def retry_rmtree(top):
    # Keep trying to remove it; on Windows it may take a few moments
    # for any outstanding locks/handles to be released
    for i in range(1, 10):
        shutil.rmtree(top, ignore_errors=True)
        if not os.path.isdir(top):
            return
        time.sleep(0.2)
    sys.stdout.write('Failed to completely remove ' + top)

def cleanup():
    if args.keep:
        sys.stdout.write('Preserving output in %s\n' % temp_dir)
        return
    retry_rmtree(temp_dir)

atexit.register(cleanup)

class Result(unittest.TestResult):
    # Make it easier to spot success/failure by coloring the status
    # green for pass, red for fail and yellow for skip.
    # also print the elapsed time per test
    transport = None
    encoding = None

    def shouldStop(self):
        if Interrupt.wasInterrupted():
            return True
        return super(Result, self).shouldStop()

    def startTest(self, test):
        self.startTime = time.time()
        super(Result, self).startTest(test)

    def setFlavour(self, transport, encoding):
        self.transport = transport
        self.encoding = encoding

    def flavour(self, test):
        if self.transport:
            return '%s [%s, %s]' % (test.id(), self.transport, self.encoding)
        return test.id()

    def addSuccess(self, test):
        elapsed = time.time() - self.startTime
        super(Result, self).addSuccess(test)
        print('\033[32mPASS\033[0m %s (%.3fs)' % (self.flavour(test), elapsed))

    def addSkip(self, test, reason):
        elapsed = time.time() - self.startTime
        super(Result, self).addSkip(test, reason)
        print('\033[33mSKIP\033[0m %s (%.3fs) %s' %
              (self.flavour(test), elapsed, reason))

    def __printFail(self, test, err):
        elapsed = time.time() - self.startTime
        t, val, trace = err
        print('\033[31mFAIL\033[0m %s (%.3fs)\n%s' % (
            self.flavour(test),
            elapsed,
            ''.join(traceback.format_exception(t, val, trace))))

    def addFailure(self, test, err):
        self.__printFail(test, err)
        super(Result, self).addFailure(test, err)

    def addError(self, test, err):
        self.__printFail(test, err)
        super(Result, self).addError(test, err)

def expandFilesList(files):
    """ expand any dir names into a full list of files """
    res = []
    for g in args.files:
        if os.path.isdir(g):
            for dirname, dirs, files in os.walk(g):
                for f in files:
                    if not f.startswith('.'):
                        res.append(os.path.normpath(os.path.join(dirname, f)))
        else:
            res.append(os.path.normpath(g))
    return res

if args.files:
    args.files = expandFilesList(args.files)

def shouldIncludeTestFile(filename):
    """ used by our loader to respect the set of tests to run """
    global args
    fname = os.path.relpath(filename.replace('.pyc', '.py'))
    if args.files:
        for f in args.files:
            if f == fname:
                return True
        return False

    if args.method:
        # implies python tests only
        if not fname.endswith('.py'):
            return False

    return True

def shouldIncludeTestName(name):
    """ used by our loader to respect the set of tests to run """
    global args
    if args.method:
        method = name.split('.').pop()
        for f in args.method:
            if method == f:
                return True
        return False
    return True

class Loader(unittest.TestLoader):
    """ allows us to control the subset of which tests are run """

    def __init__(self):
        super(Loader, self).__init__()

    def loadTestsFromTestCase(self, testCaseClass):
        return super(Loader, self).loadTestsFromTestCase(testCaseClass)

    def getTestCaseNames(self, testCaseClass):
        names = super(Loader, self).getTestCaseNames(testCaseClass)
        return filter(lambda name: shouldIncludeTestName(name), names)

    def loadTestsFromModule(self, module, *args, **kw):
        if not shouldIncludeTestFile(module.__file__):
            return unittest.TestSuite()
        return super(Loader, self).loadTestsFromModule(module, *args, **kw)

loader = Loader()
suite = unittest.TestSuite()
for d in ['python/tests', 'tests/integration']:
    suite.addTests(loader.discover(d, top_level_dir=d))

if os.name == 'nt':
    t_globs = 'tests/*.exe'
else:
    t_globs = 'tests/*.t'

suite.addTests(WatchmanTapTests.discover(
    shouldIncludeTestFile, t_globs))
suite.addTests(WatchmanTapTests.discover(
    shouldIncludeTestFile, 'tests/integration/*.php'))

suite.addTests(NodeTests.discover(
    shouldIncludeTestFile, 'node/test/*.js'))
suite.addTests(NodeTests.discover(
    shouldIncludeTestFile, 'tests/integration/*.js'))

# Manage printing from concurrent threads
# http://stackoverflow.com/a/3030755/149111
class ThreadSafeFile(object):
    def __init__(self, f):
        self.f = f
        self.lock = threading.RLock()
        self.nesting = 0

    def _getlock(self):
        self.lock.acquire()
        self.nesting += 1

    def _droplock(self):
        nesting = self.nesting
        self.nesting = 0
        for i in range(nesting):
            self.lock.release()

    def __getattr__(self, name):
        if name == 'softspace':
            return tls.softspace
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        if name == 'softspace':
            tls.softspace = value
        else:
            return object.__setattr__(self, name, value)

    def write(self, data):
        self._getlock()
        self.f.write(data)
        if data == '\n':
            self._droplock()

sys.stdout = ThreadSafeFile(sys.stdout)

tests_queue = queue.Queue()
results_queue = queue.Queue()

def runner():
    global results_queue
    global tests_queue

    broken = False
    try:
        # Start up a shared watchman instance for the tests.
        inst = WatchmanInstance.Instance({
            "watcher": args.watcher
        })
        inst.start()
        # Allow tests to locate this default instance
        WatchmanInstance.setSharedInstance(inst)
    except Exception as e:
        print('while starting watchman: %s' % str(e))
        broken = True

    while True:
        test = tests_queue.get()
        try:
            if test == 'terminate':
                break

            if Interrupt.wasInterrupted() or broken:
                continue

            try:
                result = Result()
                test.run(result)
                results_queue.put(result)
            except Exception as e:
                print(e)

        finally:
            tests_queue.task_done()

    if not broken:
        inst.stop()

def expand_suite(suite, target=None):
    """ recursively expand a TestSuite into a list of TestCase """
    if target is None:
        target = []
    for i, test in enumerate(suite):
        if isinstance(test, unittest.TestSuite):
            expand_suite(test, target)
        elif isinstance(test, WatchmanTestCase.WatchmanTestCase):
            for cfg in test.expandConfigurations():
                target.append(cfg)
        else:
            target.append(test)

    # randomize both because we don't want tests to have relatively
    # dependency ordering and also because this can help avoid clumping
    # longer running tests together
    random.shuffle(target)
    return target

def queue_jobs(tests):
    for test in tests:
        tests_queue.put(test)

all_tests = expand_suite(suite)
if len(all_tests) < args.concurrency:
    args.concurrency = len(all_tests)
queue_jobs(all_tests)

for i in range(args.concurrency):
    t = threading.Thread(target=runner)
    t.daemon = True
    t.start()
    # also send a termination sentinel
    tests_queue.put('terminate')

# Wait for all tests to have been dispatched
tests_queue.join()

# Now pull out and aggregate the results
tests_run = 0
tests_failed = 0
tests_skipped = 0
while not results_queue.empty():
    res = results_queue.get()
    tests_run = tests_run + res.testsRun
    tests_failed = tests_failed + len(res.errors) + len(res.failures)
    tests_skipped = tests_skipped + len(res.skipped)

print('Ran %d, failed %d, skipped %d, concurrency %d' % (
    tests_run, tests_failed, tests_skipped, args.concurrency))

if 'APPVEYOR' in os.environ:
    shutil.copytree(temp_dir, 'logs')
    subprocess.call(['7z', 'a', 'logs.zip', 'logs'])
    subprocess.call(['appveyor', 'PushArtifact', 'logs.zip'])

if tests_failed or (tests_run == 0):
    if args.keep_if_fail:
        args.keep = True
    sys.exit(1)
<commit_msg>runtests.py: implement flush on ThreadSafeFile<commit_after>#!/usr/bin/env python
# vim:ts=4:sw=4:et:

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# no unicode literals

import unittest
import os
import os.path
import sys
# Ensure that we can find pywatchman
sys.path.insert(0, os.path.join(os.getcwd(), 'python'))
sys.path.insert(0, os.path.join(os.getcwd(), 'tests', 'integration'))
import tempfile
import shutil
import subprocess
import traceback
import time
import argparse
import atexit
import WatchmanTapTests
import WatchmanInstance
import WatchmanTestCase
import NodeTests
import glob
import threading
import multiprocessing
import math
import signal
import Interrupt
import random

try:
    import queue
except Exception:
    import Queue
    queue = Queue

parser = argparse.ArgumentParser(
    description="Run the watchman unit and integration tests")
parser.add_argument('-v', '--verbosity', default=2,
                    help="test runner verbosity")
parser.add_argument(
    "--keep",
    action='store_true',
    help="preserve all temporary files created during test execution")
parser.add_argument(
    "--keep-if-fail",
    action='store_true',
    help="preserve all temporary files created during test execution if failed")

parser.add_argument(
    "files",
    nargs='*',
    help='specify which test files to run')

parser.add_argument(
    '--method',
    action='append',
    help='specify which python test method names to run')

parser.add_argument(
    '--concurrency',
    default=int(min(8, math.ceil(1.5 * multiprocessing.cpu_count()))),
    type=int,
    help='How many tests to run at once')

parser.add_argument(
    '--watcher',
    action='store',
    default='auto',
    help='Specify which watcher should be used to run the tests')

args = parser.parse_args()

# We test for this in a test case
os.environ['WATCHMAN_EMPTY_ENV_VAR'] = ''

# Ensure that we find the watchman we built in the tests
os.environ['PATH'] = '%s%s%s' % (
        os.path.abspath(os.path.dirname(__file__)),
        os.pathsep,
        os.environ['PATH'])

# We'll put all our temporary stuff under one dir so that we
# can clean it all up at the end
temp_dir = os.path.realpath(tempfile.mkdtemp(prefix='watchmantest'))
# Redirect all temporary files to that location
tempfile.tempdir = temp_dir

def interrupt_handler(signo, frame):
    Interrupt.setInterrupted()
signal.signal(signal.SIGINT, interrupt_handler)

def retry_rmtree(top):
    # Keep trying to remove it; on Windows it may take a few moments
    # for any outstanding locks/handles to be released
    for i in range(1, 10):
        shutil.rmtree(top, ignore_errors=True)
        if not os.path.isdir(top):
            return
        time.sleep(0.2)
    sys.stdout.write('Failed to completely remove ' + top)

def cleanup():
    if args.keep:
        sys.stdout.write('Preserving output in %s\n' % temp_dir)
        return
    retry_rmtree(temp_dir)

atexit.register(cleanup)

class Result(unittest.TestResult):
    # Make it easier to spot success/failure by coloring the status
    # green for pass, red for fail and yellow for skip.
    # also print the elapsed time per test
    transport = None
    encoding = None

    def shouldStop(self):
        if Interrupt.wasInterrupted():
            return True
        return super(Result, self).shouldStop()

    def startTest(self, test):
        self.startTime = time.time()
        super(Result, self).startTest(test)

    def setFlavour(self, transport, encoding):
        self.transport = transport
        self.encoding = encoding

    def flavour(self, test):
        if self.transport:
            return '%s [%s, %s]' % (test.id(), self.transport, self.encoding)
        return test.id()

    def addSuccess(self, test):
        elapsed = time.time() - self.startTime
        super(Result, self).addSuccess(test)
        print('\033[32mPASS\033[0m %s (%.3fs)' % (self.flavour(test), elapsed))

    def addSkip(self, test, reason):
        elapsed = time.time() - self.startTime
        super(Result, self).addSkip(test, reason)
        print('\033[33mSKIP\033[0m %s (%.3fs) %s' %
              (self.flavour(test), elapsed, reason))

    def __printFail(self, test, err):
        elapsed = time.time() - self.startTime
        t, val, trace = err
        print('\033[31mFAIL\033[0m %s (%.3fs)\n%s' % (
            self.flavour(test),
            elapsed,
            ''.join(traceback.format_exception(t, val, trace))))

    def addFailure(self, test, err):
        self.__printFail(test, err)
        super(Result, self).addFailure(test, err)

    def addError(self, test, err):
        self.__printFail(test, err)
        super(Result, self).addError(test, err)

def expandFilesList(files):
    """ expand any dir names into a full list of files """
    res = []
    for g in args.files:
        if os.path.isdir(g):
            for dirname, dirs, files in os.walk(g):
                for f in files:
                    if not f.startswith('.'):
                        res.append(os.path.normpath(os.path.join(dirname, f)))
        else:
            res.append(os.path.normpath(g))
    return res

if args.files:
    args.files = expandFilesList(args.files)

def shouldIncludeTestFile(filename):
    """ used by our loader to respect the set of tests to run """
    global args
    fname = os.path.relpath(filename.replace('.pyc', '.py'))
    if args.files:
        for f in args.files:
            if f == fname:
                return True
        return False

    if args.method:
        # implies python tests only
        if not fname.endswith('.py'):
            return False

    return True

def shouldIncludeTestName(name):
    """ used by our loader to respect the set of tests to run """
    global args
    if args.method:
        method = name.split('.').pop()
        for f in args.method:
            if method == f:
                return True
        return False
    return True

class Loader(unittest.TestLoader):
    """ allows us to control the subset of which tests are run """

    def __init__(self):
        super(Loader, self).__init__()

    def loadTestsFromTestCase(self, testCaseClass):
        return super(Loader, self).loadTestsFromTestCase(testCaseClass)

    def getTestCaseNames(self, testCaseClass):
        names = super(Loader, self).getTestCaseNames(testCaseClass)
        return filter(lambda name: shouldIncludeTestName(name), names)

    def loadTestsFromModule(self, module, *args, **kw):
        if not shouldIncludeTestFile(module.__file__):
            return unittest.TestSuite()
        return super(Loader, self).loadTestsFromModule(module, *args, **kw)

loader = Loader()
suite = unittest.TestSuite()
for d in ['python/tests', 'tests/integration']:
    suite.addTests(loader.discover(d, top_level_dir=d))

if os.name == 'nt':
    t_globs = 'tests/*.exe'
else:
    t_globs = 'tests/*.t'

suite.addTests(WatchmanTapTests.discover(
    shouldIncludeTestFile, t_globs))
suite.addTests(WatchmanTapTests.discover(
    shouldIncludeTestFile, 'tests/integration/*.php'))

suite.addTests(NodeTests.discover(
    shouldIncludeTestFile, 'node/test/*.js'))
suite.addTests(NodeTests.discover(
    shouldIncludeTestFile, 'tests/integration/*.js'))

# Manage printing from concurrent threads
# http://stackoverflow.com/a/3030755/149111
class ThreadSafeFile(object):
    def __init__(self, f):
        self.f = f
        self.lock = threading.RLock()
        self.nesting = 0

    def _getlock(self):
        self.lock.acquire()
        self.nesting += 1

    def _droplock(self):
        nesting = self.nesting
        self.nesting = 0
        for i in range(nesting):
            self.lock.release()

    def __getattr__(self, name):
        if name == 'softspace':
            return tls.softspace
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        if name == 'softspace':
            tls.softspace = value
        else:
            return object.__setattr__(self, name, value)

    def write(self, data):
        self._getlock()
        self.f.write(data)
        if data == '\n':
            self._droplock()

    def flush(self):
        self._getlock()
        self.f.flush()
        self._droplock()

sys.stdout = ThreadSafeFile(sys.stdout)

tests_queue = queue.Queue()
results_queue = queue.Queue()

def runner():
    global results_queue
    global tests_queue

    broken = False
    try:
        # Start up a shared watchman instance for the tests.
        inst = WatchmanInstance.Instance({
            "watcher": args.watcher
        })
        inst.start()
        # Allow tests to locate this default instance
        WatchmanInstance.setSharedInstance(inst)
    except Exception as e:
        print('while starting watchman: %s' % str(e))
        broken = True

    while True:
        test = tests_queue.get()
        try:
            if test == 'terminate':
                break

            if Interrupt.wasInterrupted() or broken:
                continue

            try:
                result = Result()
                test.run(result)
                results_queue.put(result)
            except Exception as e:
                print(e)

        finally:
            tests_queue.task_done()

    if not broken:
        inst.stop()

def expand_suite(suite, target=None):
    """ recursively expand a TestSuite into a list of TestCase """
    if target is None:
        target = []
    for i, test in enumerate(suite):
        if isinstance(test, unittest.TestSuite):
            expand_suite(test, target)
        elif isinstance(test, WatchmanTestCase.WatchmanTestCase):
            for cfg in test.expandConfigurations():
                target.append(cfg)
        else:
            target.append(test)

    # randomize both because we don't want tests to have relatively
    # dependency ordering and also because this can help avoid clumping
    # longer running tests together
    random.shuffle(target)
    return target

def queue_jobs(tests):
    for test in tests:
        tests_queue.put(test)

all_tests = expand_suite(suite)
if len(all_tests) < args.concurrency:
    args.concurrency = len(all_tests)
queue_jobs(all_tests)

for i in range(args.concurrency):
    t = threading.Thread(target=runner)
    t.daemon = True
    t.start()
    # also send a termination sentinel
    tests_queue.put('terminate')

# Wait for all tests to have been dispatched
tests_queue.join()

# Now pull out and aggregate the results
tests_run = 0
tests_failed = 0
tests_skipped = 0
while not results_queue.empty():
    res = results_queue.get()
    tests_run = tests_run + res.testsRun
    tests_failed = tests_failed + len(res.errors) + len(res.failures)
    tests_skipped = tests_skipped + len(res.skipped)

print('Ran %d, failed %d, skipped %d, concurrency %d' % (
    tests_run, tests_failed, tests_skipped, args.concurrency))

if 'APPVEYOR' in os.environ:
    shutil.copytree(temp_dir, 'logs')
    subprocess.call(['7z', 'a', 'logs.zip', 'logs'])
    subprocess.call(['appveyor', 'PushArtifact', 'logs.zip'])

if tests_failed or (tests_run == 0):
    if args.keep_if_fail:
        args.keep = True
    sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys

from django import VERSION
from django.conf import settings
from django.core.management import execute_from_command_line

if not settings.configured:
    settings.configure(
        DEBUG = False,
        SITE_ID = 1,
        SECRET_KEY = 'o29a3w)gmsf1d^)yjizb4=f751i*-j92%i1)1sx^_0q%wwwnxs',
        LANGUAGE_CODE = 'en-us',
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
            }
        },
        OPPS_DB_HOST='127.0.0.1',
        OPPS_DB_PORT=6379,
        OPPS_DB_NAME='opps',
        OPPS_DB_ENGINE='opps.db._redis.Redis',
        TEMPLATE_CONTEXT_PROCESSORS = (
            "django.contrib.auth.context_processors.auth",
            "django.core.context_processors.i18n",
            "django.core.context_processors.media",
            "django.core.context_processors.static",
            "django.core.context_processors.request",
            'django.contrib.messages.context_processors.messages',
            'opps.channels.context_processors.channel_context',
        ),
        MIDDLEWARE_CLASSES = (
            'django.middleware.common.CommonMiddleware',
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.contrib.redirects.middleware.RedirectFallbackMiddleware'
        ),
        INSTALLED_APPS = (
            'opps.contrib.admin',
            'opps.contrib.fileupload',
            'django.contrib.admin',
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.messages',
            'django.contrib.staticfiles',
            'django.contrib.sites',
            'django.contrib.redirects',

            'opps.core',
            'opps.core.tags',
            'opps.containers',
            'opps.boxes',
            'opps.channels',
            'opps.articles',
            'opps.images',
            'opps.sitemaps',
            'opps.flatpages',
            'opps.archives',
            'opps.views',
            'opps.fields',
            'opps.db',
            'opps.contrib.notifications',

            'djcelery',
        ),
        ROOT_URLCONF = "opps.urls",
        TEST_RUNNER = 'django_coverage.coverage_runner.CoverageRunner',
        STATIC_URL = '/static/',
        BROKER_BACKEND = "redis",
        REDIS_PORT = 6379,
        REDIS_HOST = "localhost",
        BROKER_URL = "redis://localhost:6379/0",
        REDIS_DB = 0,
        REDIS_CONNECT_RETRY = True,
        CELERY_RESULT_BACKEND ='redis',
        OPPS_MIRROR_CHANNEL = True,
    )

def runtests():
    argv = sys.argv[:1] + ['test'] + sys.argv[1:]
    if VERSION[2] <= 5:
        argv = sys.argv[:1] + ['test'] + ['core', 'containers', 'articles',
                                          'boxes', 'channels', 'images',
                                          'sitemaps', 'flatpages', 'archives',
                                          'views', 'fields', 'db', 'notifications']
    import pdb; pdb.set_trace()
    execute_from_command_line(argv)

if __name__ == '__main__':
    runtests()

<commit_msg>Fix django validation version on run tests<commit_after>#!/usr/bin/env python
import sys

from django import VERSION
from django.conf import settings
from django.core.management import execute_from_command_line

if not settings.configured:
    settings.configure(
        DEBUG = False,
        SITE_ID = 1,
        SECRET_KEY = 'o29a3w)gmsf1d^)yjizb4=f751i*-j92%i1)1sx^_0q%wwwnxs',
        LANGUAGE_CODE = 'en-us',
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
            }
        },
        OPPS_DB_HOST='127.0.0.1',
        OPPS_DB_PORT=6379,
        OPPS_DB_NAME='opps',
        OPPS_DB_ENGINE='opps.db._redis.Redis',
        TEMPLATE_CONTEXT_PROCESSORS = (
            "django.contrib.auth.context_processors.auth",
            "django.core.context_processors.i18n",
            "django.core.context_processors.media",
            "django.core.context_processors.static",
            "django.core.context_processors.request",
            'django.contrib.messages.context_processors.messages',
            'opps.channels.context_processors.channel_context',
        ),
        MIDDLEWARE_CLASSES = (
            'django.middleware.common.CommonMiddleware',
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.contrib.redirects.middleware.RedirectFallbackMiddleware'
        ),
        INSTALLED_APPS = (
            'opps.contrib.admin',
            'opps.contrib.fileupload',
            'django.contrib.admin',
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.messages',
            'django.contrib.staticfiles',
            'django.contrib.sites',
            'django.contrib.redirects',

            'opps.core',
            'opps.core.tags',
            'opps.containers',
            'opps.boxes',
            'opps.channels',
            'opps.articles',
            'opps.images',
            'opps.sitemaps',
            'opps.flatpages',
            'opps.archives',
            'opps.views',
            'opps.fields',
            'opps.db',
            'opps.contrib.notifications',

            'djcelery',
        ),
        ROOT_URLCONF = "opps.urls",
        TEST_RUNNER = 'django_coverage.coverage_runner.CoverageRunner',
        STATIC_URL = '/static/',
        BROKER_BACKEND = "redis",
        REDIS_PORT = 6379,
        REDIS_HOST = "localhost",
        BROKER_URL = "redis://localhost:6379/0",
        REDIS_DB = 0,
        REDIS_CONNECT_RETRY = True,
        CELERY_RESULT_BACKEND ='redis',
        OPPS_MIRROR_CHANNEL = True,
    )

def runtests():
    argv = sys.argv[:1] + ['test'] + sys.argv[1:]
    if VERSION[1] <= 5:
        argv = sys.argv[:1] + ['test'] + ['core', 'containers', 'articles',
                                          'boxes', 'channels', 'images',
                                          'sitemaps', 'flatpages', 'archives',
                                          'views', 'fields', 'db', 'notifications']
    execute_from_command_line(argv)

if __name__ == '__main__':
    runtests()

<|endoftext|>


<commit_before>import wx

class ClientFrame(wx.Frame):
    """ Main program. (Client)
    This class includes the main code.
    """

    def __init__(self):
        """ Initializes a new frame """
        # Creates a new frame
        wx.Frame.__init__(self,
                          None,
                          id = wx.ID_ANY,
                          title = u"LT Toolkit",
                          style = wx.MINIMIZE_BOX
                          | wx.CLOSE_BOX
                          | wx.SYSTEM_MENU
                          | wx.CAPTION
                          | wx.CLIP_CHILDREN,
                          size = (640, 480))
        # Creates a new tab control
        self.tab_control = wx.Notebook(self,
                                       id = wx.ID_ANY,
                                       style = wx.NB_TOP,
                                       size = self.GetSize())
        page1 = ClientFrame.ListenPanel(self.tab_control)
        page3 = ClientFrame.HistoryPanel(self.tab_control)
        self.tab_control.AddPage(page1, "Audience")
        self.tab_control.AddPage(page3, "History")
        # Shows this frame
        self.Center()
        self.Show()

    class ListenPanel(wx.Panel):
        """ Creates a panel of the audience
        This class is a panel which has all functions for the audience.
        """
        def __init__(self, parent):
            """Initializes ListenPanel class"""
            # new panel
            wx.Panel.__init__(self,
                              parent)
            # new font
            DEFAULT_FONT = wx.Font(12,
                           wx.FONTFAMILY_DEFAULT,
                           wx.FONTSTYLE_NORMAL,
                           wx.FONTWEIGHT_NORMAL)
            # Creates sizer
            layout_top = wx.BoxSizer(wx.HORIZONTAL)
            # text = "Connect to:"
            text_connect_to = wx.StaticText(self,
                                            id = wx.ID_ANY,
                                            label = u"Connect to ",
                                            style = wx.ALIGN_RIGHT,
                                            size = (96, -1))
            text_connect_to.SetFont(DEFAULT_FONT)
            # Creates an text control to enter the hostname into
            self.hostname = wx.TextCtrl(self,
                                        id = wx.ID_ANY,
                                        size = (256, 32))
            self.hostname.SetFont(DEFAULT_FONT)
            # Sets sizer
            layout_top.Add(text_connect_to)
            layout_top.Add(self.hostname)
            self.SetSizer(layout_top)
            return

    class HistoryPanel(wx.Panel):
        """ Creates a panel of the comment list
        This class is a panel of the comment list on the tab control.
        """
        def __init__(self, parent):
            """Initializes HistoryPanel class"""
            wx.Panel.__init__(self, parent)
            return

def main():
    # Initializes a new application
    client_application = wx.App()
    # Creates the frame
    ClientFrame()
    # Main loop
    client_application.MainLoop()
    return

if __name__ == "__main__":
    main()
<commit_msg>added : some interfaces<commit_after>import wx

class ClientFrame(wx.Frame):
    """ Main program. (Client)
    This class includes the main code.
    """

    def __init__(self):
        """ Initializes a new frame """
        # Creates a new frame
        wx.Frame.__init__(self,
                          None,
                          id = wx.ID_ANY,
                          title = u"LT Toolkit",
                          style = wx.MINIMIZE_BOX
                          | wx.CLOSE_BOX
                          | wx.SYSTEM_MENU
                          | wx.CAPTION
                          | wx.CLIP_CHILDREN,
                          size = (640, 480))
        # Creates a new tab control
        self.tab_control = wx.Notebook(self,
                                       id = wx.ID_ANY,
                                       style = wx.NB_TOP,
                                       size = self.GetSize())
        page1 = ClientFrame.ListenPanel(self.tab_control)
        page3 = ClientFrame.HistoryPanel(self.tab_control)
        self.tab_control.AddPage(page1, "Audience")
        self.tab_control.AddPage(page3, "History")
        # Shows this frame
        self.Center()
        self.Show()

    class ListenPanel(wx.Panel):
        """ Creates a panel of the audience
        This class is a panel which has all functions for the audience.
        """
        def __init__(self, parent):
            """Initializes ListenPanel class"""
            # variable
            self.message_color = (240, 240, 240)
            # new panel
            wx.Panel.__init__(self,
                              parent)
            # new font
            DEFAULT_FONT = wx.Font(12,
                           wx.FONTFAMILY_DEFAULT,
                           wx.FONTSTYLE_NORMAL,
                           wx.FONTWEIGHT_NORMAL)
            # Creates sizer
            layout_main = wx.BoxSizer(wx.VERTICAL)
            layout = []
            for i in range(3):
                layout.append(wx.BoxSizer(wx.HORIZONTAL))
            # text = "Connect to:"
            label = wx.StaticText(self,
                                  id = wx.ID_ANY,
                                  label = u"Connect to ",)
            label.SetFont(DEFAULT_FONT)
            layout[0].Add(label,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 8)
            # Creates a text control to enter the hostname into
            self.text_ip = wx.TextCtrl(self,
                                       id = wx.ID_ANY,
                                       value = u"127.0.0.1",
                                       size = (256, 32))
            self.text_ip.SetFont(DEFAULT_FONT)
            layout[0].Add(self.text_ip,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 0)
            # text = "with port"
            label = wx.StaticText(self,
                                  id = wx.ID_ANY,
                                  label = u"with port")
            label.SetFont(DEFAULT_FONT)
            layout[0].Add(label,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 8)
            # Creates a text control to enter the port number into
            self.text_port = wx.TextCtrl(self,
                                         id = wx.ID_ANY,
                                         value = u"8080",
                                         size = (64, 32))
            self.text_port.SetFont(DEFAULT_FONT)
            layout[0].Add(self.text_port,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 0)
            # Creates a list control to display the history on
            self.list_history = wx.ListCtrl(self,
                                            id = wx.ID_ANY,
                                            style = wx.LC_REPORT | wx.SUNKEN_BORDER,
                                            size = (-1, -1))
            self.list_history.Show(True)
            self.list_history.InsertColumn(0, u"Message")
            self.list_history.InsertColumn(1, u"Date")
            self.list_history.InsertColumn(2, u"ID")
            layout[1].Add(self.list_history,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 0)
            # Creates a button to configure the comment color
            self.button_color = wx.Button(self,
                                          id = wx.ID_ANY,
                                          label = u"COLOR",
                                          size = (64, 32))
            self.button_color.SetForegroundColour(self.message_color)
            self.button_color.SetToolTipString(u"message color")
            self.button_color.Bind(wx.EVT_BUTTON, self.configure_color)
            layout[2].Add(self.button_color,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 0)
            # Creates a text control to enter the message into
            self.text_message = wx.TextCtrl(self,
                                            id = wx.ID_ANY,
                                            size = (256, 32))
            self.text_message.SetFont(DEFAULT_FONT)
            self.text_message.SetToolTipString("message")
            layout[2].Add(self.text_message,
                          flag = wx.EXPAND | wx.ALIGN_LEFT | wx.ALL,
                          border = 0)
            # Sets sizer
            layout_main.Add(layout[0])
            layout_main.Add(layout[1])
            layout_main.Add(layout[2])
            self.SetSizer(layout_main)
            return

        def configure_color(self, event):
            """ Asks the message color """
            data = wx.ColourData()
            dialog = wx.ColourDialog(None)
            dialog.GetColourData().SetChooseFull(True)
            dialog.GetColourData().SetColour(wx.Colour(self.message_color[0],
                                                       self.message_color[1],
                                                       self.message_color[2]))
            if dialog.ShowModal() == wx.ID_OK:
                data = dialog.GetColourData()
                self.message_color = data.GetColour().Get()
                self.button_color.SetForegroundColour(self.message_color)
            dialog.Destroy()
            return

    class HistoryPanel(wx.Panel):
        """ Creates a panel of the message list
        This class is a panel of the message list on the tab control.
        """
        def __init__(self, parent):
            """Initializes HistoryPanel class"""
            wx.Panel.__init__(self, parent)
            return

def main():
    # Initializes a new application
    client_application = wx.App()
    # Creates the frame
    ClientFrame()
    # Main loop
    client_application.MainLoop()
    return

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""rwrtrack

Usage:
    rwrtrack.py [-q|-v] get [<pages>]
    rwrtrack.py [-q|-v] analyse <name> [<othername>] [-d <dates>]
    rwrtrack.py [-q|-v] average <metric> [-d <dates>] [-x <pre>] [-y <pst>]
    rwrtrack.py [-q|-v] rank <metric> [<n>] [-d <dates>] [-x <pre>] [-y <pst>]
    rwrtrack.py [-q|-v] sum [-d <dates>]
    rwrtrack.py [-q|-v] _db_migrate_csv

Options:
    -q          Quiet mode, reduces logging output to errors and above
    -v          Verbose output, with full stdout logging
    -d <dates>  Date, or two dates separated by a hyphen
                [default: latest]
                Other shortcut options: day, week, month
    -x <pre>    Prefilter specifier
    -y <pst>    Postfilter specifier
"""

import logging
import logging.config
import operator
import statistics
import sys
from datetime import datetime, timedelta
from pathlib import Path

from docopt import docopt

from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy import and_

from rwrtrack.core import DbInfo, Account, Record, sesh, get_account_from_db, \
                            get_records_on_date, update_db_from_stats
from rwrtrack.avg import print_avg
from rwrtrack.get_stats import get_stats
from rwrtrack.ranking import print_ranking
from rwrtrack.stats_csv import load_stats_from_csv, write_stats_to_csv
from rwrtrack.sums import sum_stats_and_analyse

script_dir = Path(__file__).parent
log_conf_p = (script_dir / "logging.conf").resolve()
log_p = (script_dir / "rwrtrackpy.log").resolve()
logger = logging.getLogger(__name__)

csv_hist_path = Path(__file__).parent / Path("csv_historical")

def _process_numeric_dates(date_string):
    if date_string.isnumeric():
        return "single", int(date_string)
    else:
        # Handle date ranges
        dates = date_string.split("-")
        if dates[0] > dates[1]:
            raise ValueError("Dates must be older-newer")
        d_older = int(dates[0])
        d_newer = int(dates[1])
        return "range", (d_older, d_newer)

def _unpack_filters(fs):
    _fs = []
    for f in fs.split(","):
        # logger.debug(f"Unpacking {f}")
        m, o, v = f.split(":")
        # logger.debug(f"metric: {m}, op: {o}, value: {v}")
        _fs.append((m, o, int(v)))
    return _fs

if __name__ == '__main__':
    args = docopt(__doc__)

    log_opts = {"logfilename": log_p.as_posix(), "consoleloglvl": "INFO"}
    if args["-q"]:
        log_opts["consoleloglvl"] = "ERROR"
    elif args["-v"]:
        log_opts["consoleloglvl"] = "DEBUG"
    logging.config.fileConfig(log_conf_p.as_posix(),
                              disable_existing_loggers=False,
                              defaults=log_opts)

    logger.debug(f"Logging configured from {str(log_conf_p)}")
    logger.debug(f"Logging output will be written to {str(log_p)}")
    logger.debug(f"Running rwrtrack.py with arguments: {sys.argv[1:]}")
    logger.debug(f"docopt output:\n{args}")

    if args["get"]:
        # TODO: Rewrite to use write to db as well
        num_pages = int(args["<pages>"]) if args["<pages>"] else 10
        stats = get_stats(num_pages)
        write_stats_to_csv(stats)

    elif args["analyse"]:
        # analyse <name> [<othername>] [-d <dates>]
        username = args["<name>"]
        othername = args["<othername>"]
        dates = args["-d"]

        logger.info(f"Performing individual analysis for '{username}'...")
        try:
            account = get_account_from_db(username)
            account_id = account._id
        except NoResultFound as e:
            logger.error(f"'{username}' not found in database.")
            sys.exit(1)
        if not othername:
            if dates.isalpha():
                if dates == "latest":
                    print(f"'{username}' on {account.latest_date}:")
                    print(account.latest_record.as_table())
                elif dates == "first":
                    r = account.on_date(account.first_date)
                    print(f"'{username}' on {account.first_date}:")
                    print(r.as_table())
                elif dates == "day":
                    r_newer = account.latest_record
                    r_older = account.on_date(account.latest_date, days=-1)
                    d = r_newer - r_older
                    print(f"'{username}' from {d.dates[1]} to {d.dates[0]}:")
                    print(d.as_table())
                elif dates == "week":
                    r_newer = account.latest_record
                    r_older = account.on_date(account.latest_date, weeks=-1)
                    d = r_newer - r_older
                    print(f"'{username}' from {d.dates[1]} to {d.dates[0]}:")
                    print(d.as_table())
                else:
                    date_opt = dates
                    raise ValueError(f"Date(s) option '{date_opt}' invalid")
            else:
                dt, d = _process_numeric_dates(dates)
                if dt == "single":
                    # TODO: Improve handling if record for date not in db
                    r = account.on_date(d)
                    print(f"'{username}' on {r.date}:")
                    print(r.as_table())
                elif dt == "range":
                    r_newer = account.on_date(d[1])
                    r_older = account.on_date(d[0])
                    d = r_newer - r_older
                    print(f"'{username}' from {d.dates[1]} to {d.dates[0]}:")
                    print(d.as_table())
        else:
            # print(">do a comparative analysis")
            # TODO: Implement... eventually...
            raise NotImplementedError("GO COMPARE... elsewhere until later...")

    elif args["average"]:
        # Old:
        # average <metric> [<minxp>] [-d <dates>]
        # New (provisionally):
        # average <metric> [-d <dates>] [-x <pre>] [-y <pst>]
        # TODO: Rewrite to use write to db as well
        # stats_pruned = [s for s in stats_list if s.time_played > 0]
        # min_xp = int(args["<minxp>"]) if args["<minxp>"] else 0
        # print_avg(stats_pruned, metric, min_xp)
        # TODO: Rework to allow a list of permissible metrics to be specified
        metric = args["<metric>"]
        # Fail fast if the metric specified is not one of those listed below
        if metric not in ["xp", "time_played_hours", "kills", "deaths",
                          "score", "kdr", "kill_streak", "targets_destroyed",
                          "vehicles_destroyed", "soldiers_healed",
                          "team_kills", "distance_moved_km", "shots_fired",
                          "throwables_thrown", "xp_ph", "kills_ph",
                          "deaths_ph", "targets_destroyed_ph",
                          "vehicles_destroyed_ph", "soldiers_healed_ph",
                          "team_kills_ph", "distance_moved_km_ph",
                          "shots_fired_ph", "throwables_thrown_ph",
                          "xp_pk", "xp_pb", "shots_fired_pk",
                          "team_kills_pk", "runs_around_the_equator"]:
            raise ValueError("Bad metric!")
        dates = args["-d"]
        prefilters = args["-x"]
        pstfilters = args["-y"]
        # TODO: Process filter specifiers into stuff we can operate with...
        logger.debug(f"Prefilters specified: '{prefilters}'")
        prefs = _unpack_filters(prefilters) if prefilters else []
        # logger.debug(f"prefs: {prefs}")
        logger.debug(f"Postfilters specified: '{pstfilters}'")
        pstfs = _unpack_filters(pstfilters) if pstfilters else []
        # logger.debug(f"pstfs: {pstfs}")

        try:
            db_info = sesh.query(DbInfo).one()
        except NoResultFound:
            logger.error("Database is blank... (or non-existent)... Exit.")
            sys.exit(1)

        logger.info(f"Calculating average of '{metric}' for '{dates}'...")
        if dates.isalpha():
            if dates == "latest":
                d = db_info.latest_date
                rs = get_records_on_date(d)
                # TODO: Apply pref here, pstf irrelevant for individual dates
                for pref in prefs:
                    m, o, v = pref
                    _opmap = {">": operator.ge, "<": operator.le}
                    _op = _opmap.get(o, None)
                    if _op:
                        rs = [r for r in rs if _op(getattr(r, m), v)]
                    else:
                        logger.error(f"Operator '{o}' not workable... ignored")
                logger.info(f"Averaging {len(rs)} records for {d}...")
                meanv = statistics.mean(getattr(r, metric) for r in rs)
                medianv = statistics.median(getattr(r, metric) for r in rs)
                print(f"Mean '{metric}' is {meanv:.2f}")
                print(f"Median '{metric}' is {medianv:.2f}")
            elif dates == "first":
                d = db_info.first_date
                rs = get_records_on_date(d)
                # TODO: Apply pref here, pstf irrelevant for individual dates
                logger.info(f"Averaging {len(rs)} records for {d}...")
                meanv = statistics.mean(getattr(r, metric) for r in rs)
                medianv = statistics.median(getattr(r, metric) for r in rs)
                print(f"Mean '{metric}' is {meanv:.2f}")
                print(f"Median '{metric}' is {medianv:.2f}")
            elif dates == "day":
                # TODO: Calculate average for metric for day
                raise NotImplementedError("No diffs for averages... yet...")
            elif dates == "week":
                # TODO: Calculate average for metric for week
                raise NotImplementedError("No diffs for averages... yet...")
            else:
                date_opt = dates
                raise ValueError(f"Date(s) option '{date_opt}' invalid")
        else:
            dt, d = _process_numeric_dates(dates)
            if dt == "single":
                # TODO: Improve handling if record for date not in db
                rs = get_records_on_date(d)
                # TODO: Apply pref here, pstf irrelevant for individual dates
                logger.info(f"Averaging {len(rs)} records for {d}...")
                meanv = statistics.mean(getattr(r, metric) for r in rs)
                medianv = statistics.median(getattr(r, metric) for r in rs)
                print(f"Mean '{metric}' is {meanv:.2f}")
                print(f"Median '{metric}' is {medianv:.2f}")
            elif dt == "range":
                # TODO: Calculate average for metric across date range
                raise NotImplementedError("No diffs for averages... yet...")

    elif args["rank"]:
        # Old:
        # rank <metric> [<minxp>] [<upto>] [-d <dates>]
        # New (provisionally):
        # rank <metric> [<n>] [-d <dates>] [-x <pre>] [-y <pst>]
        # TODO: Rewrite to use write to db as well
        stats_pruned = [s for s in stats_list if s.time_played > 0]
        metric = args["<metric>"]
        min_xp = int(args["<minxp>"]) if args["<minxp>"] else 0
        upto = int(args["<upto>"]) if args["<upto>"] else 25
        print_ranking(stats_pruned, metric, min_xp, upto)

    elif args["sum"]:
        # TODO: Rewrite to use write to db as well
        output_at_rows = [10, 100, 1000]
        sum_stats_and_analyse(stats_list, output_at_rows)

    elif args["_db_migrate_csv"]:
        logger.info("Migrating CSV to database...")
        # Get all CSV files and filter
        logger.info("Finding CSV files for migration...")
        csv_file_paths = sorted(list(csv_hist_path.glob("*.csv")))
        # Filter out CSV files that are not being migrated (for reasons...)
        csv_file_paths = filter(lambda x: "2017" not in x.stem,
                                csv_file_paths)
        try:
            # Attempt to access the DbInfo
            db_info = sesh.query(DbInfo).one()
        except NoResultFound:
            # If no DbInfo, db is blank, initialise from origin CSV file
            logger.info("Blank database found - beginning full migration...")
            first_csv_path = next(csv_file_paths)
            logger.info(f"First CSV file at '{first_csv_path}'")
            # Fix dates
            d = datetime.strptime(first_csv_path.stem, "%Y-%m-%d").date()
            d = d - timedelta(days=1)
            d = int(d.strftime("%Y%m%d"))
            # Populate _dbinfo table with the initial CSV in migration
            db_info = DbInfo(first_date=d, latest_date=d)
            sesh.add(db_info)
            # Add stats from the first file in the filter generator
            stats = load_stats_from_csv(first_csv_path)
            update_db_from_stats(stats, d)
        else:
            logger.info("Existing database found - continuing migration...")
            # Step the csv_file_paths filter until we are at the first new file
            while True:
                try:
                    csv_file_path = next(csv_file_paths)
                except StopIteration:
                    logger.info("No new CSV files to migrate")
                    sys.exit(1)
                d = datetime.strptime(csv_file_path.stem, "%Y-%m-%d").date()
                d = d - timedelta(days=1)
                d = int(d.strftime("%Y%m%d"))
                if (d > db_info.latest_date):
                    # Update latest_date in _dbinfo table
                    db_info.latest_date = d
                    # Add stats from the first new file in the filter generator
                    stats = load_stats_from_csv(csv_file_path)
                    update_db_from_stats(stats, d)
                    break
        finally:
            for csv_file_path in csv_file_paths:
                stats = load_stats_from_csv(csv_file_path)
                # Fix dates
                d = datetime.strptime(csv_file_path.stem, "%Y-%m-%d").date()
                d = d - timedelta(days=1)
                d = int(d.strftime("%Y%m%d"))
                # Update latest_date in _dbinfo table
                db_info.latest_date = d
                update_db_from_stats(stats, d)

        logger.info("Committing changes to database...")
        sesh.commit()

    else:
        print(f"BAD USAGE!\n{__doc__}")
<commit_msg>Refactor filtering<commit_after>"""rwrtrack

Usage:
    rwrtrack.py [-q|-v] get [<pages>]
    rwrtrack.py [-q|-v] analyse <name> [<othername>] [-d <dates>]
    rwrtrack.py [-q|-v] average <metric> [-d <dates>] [-x <pre>] [-y <pst>]
    rwrtrack.py [-q|-v] rank <metric> [<n>] [-d <dates>] [-x <pre>] [-y <pst>]
    rwrtrack.py [-q|-v] sum [-d <dates>]
    rwrtrack.py [-q|-v] _db_migrate_csv

Options:
    -q          Quiet mode, reduces logging output to errors and above
    -v          Verbose output, with full stdout logging
    -d <dates>  Date, or two dates separated by a hyphen
                [default: latest]
                Other shortcut options: day, week, month
    -x <pre>    Prefilter specifier
    -y <pst>    Postfilter specifier
"""

import logging
import logging.config
import operator
import statistics
import sys
from datetime import datetime, timedelta
from pathlib import Path

from docopt import docopt

from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy import and_

from rwrtrack.core import DbInfo, Account, Record, sesh, get_account_from_db, \
                            get_records_on_date, update_db_from_stats
from rwrtrack.avg import print_avg
from rwrtrack.get_stats import get_stats
from rwrtrack.ranking import print_ranking
from rwrtrack.stats_csv import load_stats_from_csv, write_stats_to_csv
from rwrtrack.sums import sum_stats_and_analyse

script_dir = Path(__file__).parent
log_conf_p = (script_dir / "logging.conf").resolve()
log_p = (script_dir / "rwrtrackpy.log").resolve()
logger = logging.getLogger(__name__)

csv_hist_path = Path(__file__).parent / Path("csv_historical")

def _process_numeric_dates(date_string):
    if date_string.isnumeric():
        return "single", int(date_string)
    else:
        # Handle date ranges
        dates = date_string.split("-")
        if dates[0] > dates[1]:
            raise ValueError("Dates must be older-newer")
        d_older = int(dates[0])
        d_newer = int(dates[1])
        return "range", (d_older, d_newer)

def _unpack_filters(fs):
    _fs = []
    for f in fs.split(","):
        m, o, v = f.split(":")
        _fs.append((m, o, int(v)))
    return _fs

def apply_filters(rs, filters):
    fs = _unpack_filters(filters)
    for m, o, v in fs:
        _opmap = {">": operator.ge, "<": operator.le}
        _op = _opmap.get(o, None)
        if _op:
            rs = [r for r in rs if _op(getattr(r, m), v)]
        else:
            logger.error(f"Operator '{o}' not workable... ignored")
    return rs

if __name__ == '__main__':
    args = docopt(__doc__)

    log_opts = {"logfilename": log_p.as_posix(), "consoleloglvl": "INFO"}
    if args["-q"]:
        log_opts["consoleloglvl"] = "ERROR"
    elif args["-v"]:
        log_opts["consoleloglvl"] = "DEBUG"
    logging.config.fileConfig(log_conf_p.as_posix(),
                              disable_existing_loggers=False,
                              defaults=log_opts)

    logger.debug(f"Logging configured from {str(log_conf_p)}")
    logger.debug(f"Logging output will be written to {str(log_p)}")
    logger.debug(f"Running rwrtrack.py with arguments: {sys.argv[1:]}")
    logger.debug(f"docopt output:\n{args}")

    if args["get"]:
        # TODO: Rewrite to use write to db as well
        num_pages = int(args["<pages>"]) if args["<pages>"] else 10
        stats = get_stats(num_pages)
        write_stats_to_csv(stats)

    elif args["analyse"]:
        # analyse <name> [<othername>] [-d <dates>]
        username = args["<name>"]
        othername = args["<othername>"]
        dates = args["-d"]

        logger.info(f"Performing individual analysis for '{username}'...")
        try:
            account = get_account_from_db(username)
            account_id = account._id
        except NoResultFound as e:
            logger.error(f"'{username}' not found in database.")
            sys.exit(1)
        if not othername:
            if dates.isalpha():
                if dates == "latest":
                    print(f"'{username}' on {account.latest_date}:")
                    print(account.latest_record.as_table())
                elif dates == "first":
                    r = account.on_date(account.first_date)
                    print(f"'{username}' on {account.first_date}:")
                    print(r.as_table())
                elif dates == "day":
                    r_newer = account.latest_record
                    r_older = account.on_date(account.latest_date, days=-1)
                    d = r_newer - r_older
                    print(f"'{username}' from {d.dates[1]} to {d.dates[0]}:")
                    print(d.as_table())
                elif dates == "week":
                    r_newer = account.latest_record
                    r_older = account.on_date(account.latest_date, weeks=-1)
                    d = r_newer - r_older
                    print(f"'{username}' from {d.dates[1]} to {d.dates[0]}:")
                    print(d.as_table())
                else:
                    date_opt = dates
                    raise ValueError(f"Date(s) option '{date_opt}' invalid")
            else:
                dt, d = _process_numeric_dates(dates)
                if dt == "single":
                    # TODO: Improve handling if record for date not in db
                    r = account.on_date(d)
                    print(f"'{username}' on {r.date}:")
                    print(r.as_table())
                elif dt == "range":
                    r_newer = account.on_date(d[1])
                    r_older = account.on_date(d[0])
                    d = r_newer - r_older
                    print(f"'{username}' from {d.dates[1]} to {d.dates[0]}:")
                    print(d.as_table())
        else:
            # print(">do a comparative analysis")
            # TODO: Implement... eventually...
            raise NotImplementedError("GO COMPARE... elsewhere until later...")

    elif args["average"]:
        # Old:
        # average <metric> [<minxp>] [-d <dates>]
        # New (provisionally):
        # average <metric> [-d <dates>] [-x <pre>] [-y <pst>]
        # TODO: Rewrite to use write to db as well
        # stats_pruned = [s for s in stats_list if s.time_played > 0]
        # min_xp = int(args["<minxp>"]) if args["<minxp>"] else 0
        # print_avg(stats_pruned, metric, min_xp)
        # TODO: Rework to allow a list of permissible metrics to be specified
        metric = args["<metric>"]
        # Fail fast if the metric specified is not one of those listed below
        if metric not in ["xp", "time_played_hours", "kills", "deaths",
                          "score", "kdr", "kill_streak", "targets_destroyed",
                          "vehicles_destroyed", "soldiers_healed",
                          "team_kills", "distance_moved_km", "shots_fired",
                          "throwables_thrown", "xp_ph", "kills_ph",
                          "deaths_ph", "targets_destroyed_ph",
                          "vehicles_destroyed_ph", "soldiers_healed_ph",
                          "team_kills_ph", "distance_moved_km_ph",
                          "shots_fired_ph", "throwables_thrown_ph",
                          "xp_pk", "xp_pb", "shots_fired_pk",
                          "team_kills_pk", "runs_around_the_equator"]:
            raise ValueError("Bad metric!")
        dates = args["-d"]
        prefilters = args["-x"]
        pstfilters = args["-y"]
        # TODO: Process filter specifiers into stuff we can operate with...
        logger.debug(f"Prefilters specified: '{prefilters}'")
        logger.debug(f"Postfilters specified: '{pstfilters}'")

        try:
            db_info = sesh.query(DbInfo).one()
        except NoResultFound:
            logger.error("Database is blank... (or non-existent)... Exit.")
            sys.exit(1)

        logger.info(f"Calculating average of '{metric}' for '{dates}'...")
        if dates.isalpha():
            if dates == "latest":
                d = db_info.latest_date
                rs = get_records_on_date(d)
                # Apply prefilters
                if prefilters:
                    logger.info(f"Applying prefilters: '{prefilters}'")
                    rs = apply_filters(rs, prefilters)
                if len(rs) == 0:
                    logger.error("No records to average... exit.")
                    sys.exit(1)
                logger.info(f"Averaging {len(rs)} records for {d}...")
                meanv = statistics.mean(getattr(r, metric) for r in rs)
                medianv = statistics.median(getattr(r, metric) for r in rs)
                print(f"Mean '{metric}' is {meanv:.2f}")
                print(f"Median '{metric}' is {medianv:.2f}")
            elif dates == "first":
                d = db_info.first_date
                rs = get_records_on_date(d)
                # Apply prefilters
                if prefilters:
                    logger.info(f"Applying prefilters: '{prefilters}'")
                    rs = apply_filters(rs, prefilters)
                logger.info(f"Averaging {len(rs)} records for {d}...")
                meanv = statistics.mean(getattr(r, metric) for r in rs)
                medianv = statistics.median(getattr(r, metric) for r in rs)
                print(f"Mean '{metric}' is {meanv:.2f}")
                print(f"Median '{metric}' is {medianv:.2f}")
            elif dates == "day":
                # TODO: Calculate average for metric for day
                raise NotImplementedError("No diffs for averages... yet...")
            elif dates == "week":
                # TODO: Calculate average for metric for week
                raise NotImplementedError("No diffs for averages... yet...")
            else:
                date_opt = dates
                raise ValueError(f"Date(s) option '{date_opt}' invalid")
        else:
            dt, d = _process_numeric_dates(dates)
            if dt == "single":
                # TODO: Improve handling if record for date not in db
                rs = get_records_on_date(d)
                # Apply prefilters
                if prefilters:
                    logger.info(f"Applying prefilters: '{prefilters}'")
                    rs = apply_filters(rs, prefilters)
                logger.info(f"Averaging {len(rs)} records for {d}...")
                meanv = statistics.mean(getattr(r, metric) for r in rs)
                medianv = statistics.median(getattr(r, metric) for r in rs)
                print(f"Mean '{metric}' is {meanv:.2f}")
                print(f"Median '{metric}' is {medianv:.2f}")
            elif dt == "range":
                # TODO: Calculate average for metric across date range
                raise NotImplementedError("No diffs for averages... yet...")

    elif args["rank"]:
        # Old:
        # rank <metric> [<minxp>] [<upto>] [-d <dates>]
        # New (provisionally):
        # rank <metric> [<n>] [-d <dates>] [-x <pre>] [-y <pst>]
        # TODO: Rewrite to use write to db as well
        stats_pruned = [s for s in stats_list if s.time_played > 0]
        metric = args["<metric>"]
        min_xp = int(args["<minxp>"]) if args["<minxp>"] else 0
        upto = int(args["<upto>"]) if args["<upto>"] else 25
        print_ranking(stats_pruned, metric, min_xp, upto)

    elif args["sum"]:
        # TODO: Rewrite to use write to db as well
        output_at_rows = [10, 100, 1000]
        sum_stats_and_analyse(stats_list, output_at_rows)

    elif args["_db_migrate_csv"]:
        logger.info("Migrating CSV to database...")
        # Get all CSV files and filter
        logger.info("Finding CSV files for migration...")
        csv_file_paths = sorted(list(csv_hist_path.glob("*.csv")))
        # Filter out CSV files that are not being migrated (for reasons...)
        csv_file_paths = filter(lambda x: "2017" not in x.stem,
                                csv_file_paths)
        try:
            # Attempt to access the DbInfo
            db_info = sesh.query(DbInfo).one()
        except NoResultFound:
            # If no DbInfo, db is blank, initialise from origin CSV file
            logger.info("Blank database found - beginning full migration...")
            first_csv_path = next(csv_file_paths)
            logger.info(f"First CSV file at '{first_csv_path}'")
            # Fix dates
            d = datetime.strptime(first_csv_path.stem, "%Y-%m-%d").date()
            d = d - timedelta(days=1)
            d = int(d.strftime("%Y%m%d"))
            # Populate _dbinfo table with the initial CSV in migration
            db_info = DbInfo(first_date=d, latest_date=d)
            sesh.add(db_info)
            # Add stats from the first file in the filter generator
            stats = load_stats_from_csv(first_csv_path)
            update_db_from_stats(stats, d)
        else:
            logger.info("Existing database found - continuing migration...")
            # Step the csv_file_paths filter until we are at the first new file
            while True:
                try:
                    csv_file_path = next(csv_file_paths)
                except StopIteration:
                    logger.info("No new CSV files to migrate")
                    sys.exit(1)
                d = datetime.strptime(csv_file_path.stem, "%Y-%m-%d").date()
                d = d - timedelta(days=1)
                d = int(d.strftime("%Y%m%d"))
                if (d > db_info.latest_date):
                    # Update latest_date in _dbinfo table
                    db_info.latest_date = d
                    # Add stats from the first new file in the filter generator
                    stats = load_stats_from_csv(csv_file_path)
                    update_db_from_stats(stats, d)
                    break
        finally:
            for csv_file_path in csv_file_paths:
                stats = load_stats_from_csv(csv_file_path)
                # Fix dates
                d = datetime.strptime(csv_file_path.stem, "%Y-%m-%d").date()
                d = d - timedelta(days=1)
                d = int(d.strftime("%Y%m%d"))
                # Update latest_date in _dbinfo table
                db_info.latest_date = d
                update_db_from_stats(stats, d)

        logger.info("Committing changes to database...")
        sesh.commit()

    else:
        print(f"BAD USAGE!\n{__doc__}")
<|endoftext|>