<commit_before># Copyright 2020 MONAI Consortium
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import sys
import time
import unittest

from monai.utils import PerfContext

results: dict = dict()

class TimeLoggingTestResult(unittest.TextTestResult):
    """Overload the default results so that we can store the results."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.timed_tests = dict()

    def startTest(self, test):  # noqa: N802
        """Start timer, print test name, do normal test."""
        self.start_time = time.time()
        name = self.getDescription(test)
        self.stream.write(f"Starting test: {name}...\n")
        super().startTest(test)

    def stopTest(self, test):  # noqa: N802
        """On test end, get time, print, store and do normal behaviour."""
        elapsed = time.time() - self.start_time
        name = self.getDescription(test)
        self.stream.write(f"Finished test: {name} ({elapsed:.03}s)\n")
        if name in results:
            raise AssertionError("expected all keys to be unique")
        results[name] = elapsed
        super().stopTest(test)

def print_results(results, discovery_time, thresh, status):
    # only keep results >= threshold
    results = dict(filter(lambda x: x[1] > thresh, results.items()))
    if len(results) == 0:
        return
    print(f"\n\n{status}, printing completed times >{thresh}s in ascending order...\n")
    timings = dict(sorted(results.items(), key=lambda item: item[1]))

    for r in timings:
        if timings[r] >= thresh:
            print(f"{r} ({timings[r]:.03}s)")
    print(f"test discovery time: {discovery_time:.03}s")
    print(f"total testing time: {sum(results.values()):.03}s")
    print("Remember to check above times for any errors!")

def parse_args():
    parser = argparse.ArgumentParser(description="Runner for MONAI unittests with timing.")
    parser.add_argument(
        "-s", action="store", dest="path", default=".", help="Directory to start discovery ('.' default)"
    )
    parser.add_argument(
        "-p", action="store", dest="pattern", default=None, help="Pattern to match tests (default is unittest default)"
    )
    parser.add_argument(
        "-t",
        "--thresh",
        dest="thresh",
        default=10.0,
        type=float,
        help="Display tests longer than given threshold default: 10)",
    )
    parser.add_argument("-q", "--quick", action="store_true", dest="quick", default=False, help="Only do quick tests")
    args = parser.parse_args()
    print(f"Running tests in folder: '{args.path}'")
    if args.pattern:
        print(f"With file pattern: '{args.pattern}'")

    return args.path, args.pattern, args.thresh, args.quick

if __name__ == "__main__":
    # Parse input arguments
    path, pattern, thresh, quick = parse_args()

    # If quick is desired, set environment variable
    if any(q in sys.argv for q in ["-q", "--quick"]):
        os.environ["QUICKTEST"] = "True"

    # Get all test names (optionally from some path with some pattern)
    loader_args = [path, pattern] if pattern else [path]
    loader = unittest.TestLoader()
    with PerfContext() as pc:
        tests = loader.discover(*loader_args)
    discovery_time = pc.total_time
    print(f"time to discover tests: {discovery_time}s")

    test_runner = unittest.runner.TextTestRunner(resultclass=TimeLoggingTestResult)

    # Use try catches to print the current results if encountering exception or keyboard interruption
    try:
        test_result = test_runner.run(tests)
        print_results(results, discovery_time, thresh, "tests finished")
        sys.exit(not test_result.wasSuccessful())
    except KeyboardInterrupt:
        print_results(results, discovery_time, thresh, "tests cancelled")
        sys.exit(1)
    except Exception:
        print_results(results, discovery_time, thresh, "exception reached")
        raise
<commit_msg>Runner failfast (#1207)<commit_after># Copyright 2020 MONAI Consortium
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import inspect
import os
import sys
import time
import unittest

from monai.utils import PerfContext

results: dict = dict()

class TimeLoggingTestResult(unittest.TextTestResult):
    """Overload the default results so that we can store the results."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.timed_tests = dict()

    def startTest(self, test):  # noqa: N802
        """Start timer, print test name, do normal test."""
        self.start_time = time.time()
        name = self.getDescription(test)
        self.stream.write(f"Starting test: {name}...\n")
        super().startTest(test)

    def stopTest(self, test):  # noqa: N802
        """On test end, get time, print, store and do normal behaviour."""
        elapsed = time.time() - self.start_time
        name = self.getDescription(test)
        self.stream.write(f"Finished test: {name} ({elapsed:.03}s)\n")
        if name in results:
            raise AssertionError("expected all keys to be unique")
        results[name] = elapsed
        super().stopTest(test)

def print_results(results, discovery_time, thresh, status):
    # only keep results >= threshold
    results = dict(filter(lambda x: x[1] > thresh, results.items()))
    if len(results) == 0:
        return
    print(f"\n\n{status}, printing completed times >{thresh}s in ascending order...\n")
    timings = dict(sorted(results.items(), key=lambda item: item[1]))

    for r in timings:
        if timings[r] >= thresh:
            print(f"{r} ({timings[r]:.03}s)")
    print(f"test discovery time: {discovery_time:.03}s")
    print(f"total testing time: {sum(results.values()):.03}s")
    print("Remember to check above times for any errors!")

def parse_args(default_pattern):
    parser = argparse.ArgumentParser(description="Runner for MONAI unittests with timing.")
    parser.add_argument(
        "-s", action="store", dest="path", default=".", help="Directory to start discovery (default: '%(default)s')"
    )
    parser.add_argument(
        "-p",
        action="store",
        dest="pattern",
        default=default_pattern,
        help="Pattern to match tests (default: '%(default)s')",
    )
    parser.add_argument(
        "-t",
        "--thresh",
        dest="thresh",
        default=10.0,
        type=float,
        help="Display tests longer than given threshold (default: %(default)d)",
    )
    parser.add_argument(
        "-v",
        "--verbosity",
        action="store",
        dest="verbosity",
        type=int,
        default=1,
        help="Verbosity level (default: %(default)d)",
    )
    parser.add_argument("-q", "--quick", action="store_true", dest="quick", default=False, help="Only do quick tests")
    parser.add_argument(
        "-f", "--failfast", action="store_true", dest="failfast", default=False, help="Stop testing on first failure"
    )
    args = parser.parse_args()
    print(f"Running tests in folder: '{args.path}'")
    if args.pattern:
        print(f"With file pattern: '{args.pattern}'")

    return args

def get_default_pattern(loader):
    signature = inspect.signature(loader.discover)
    params = {k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty}
    return params["pattern"]

if __name__ == "__main__":

    loader = unittest.TestLoader()
    default_pattern = get_default_pattern(loader)

    # Parse input arguments
    args = parse_args(default_pattern)

    # If quick is desired, set environment variable
    if args.quick:
        os.environ["QUICKTEST"] = "True"

    # Get all test names (optionally from some path with some pattern)
    with PerfContext() as pc:
        tests = loader.discover(args.path, args.pattern)
    discovery_time = pc.total_time
    print(f"time to discover tests: {discovery_time}s")

    test_runner = unittest.runner.TextTestRunner(
        resultclass=TimeLoggingTestResult, verbosity=args.verbosity, failfast=args.failfast
    )

    # Use try catches to print the current results if encountering exception or keyboard interruption
    try:
        test_result = test_runner.run(tests)
        print_results(results, discovery_time, args.thresh, "tests finished")
        sys.exit(not test_result.wasSuccessful())
    except KeyboardInterrupt:
        print_results(results, discovery_time, args.thresh, "tests cancelled")
        sys.exit(1)
    except Exception:
        print_results(results, discovery_time, args.thresh, "exception reached")
        raise
<|endoftext|>


<commit_before>import unittest
import subprocess

def callTest(fileNameInput, fileNameOutput = None, extraarg = []):
    if fileNameOutput == None:
        fileNameOutput = fileNameInput
    with open('testfile', 'wb') as f:
        ret = subprocess.check_call(["python3", "solomagic.py", "tests/" + fileNameInput] + extraarg, stdout=f)
        if ret != 0:
            return 1

    # I think whitespace changes don't matter
    return subprocess.check_call(["diff", "-ub", "tests/" + fileNameOutput, "testfile"])

class TestIO(unittest.TestCase):

    def test_invariant(self):
        self.assertEqual(callTest("a.text.in"), 0)
        self.assertEqual(callTest("b.text.in"), 0)
        self.assertEqual(callTest("ivaena.text.in"), 0)

        self.assertEqual(callTest("testapostrophe_mt.txt"), 0)
        self.assertEqual(callTest("testivena_apostrophe_mt.txt"), 0)
        self.assertEqual(callTest("testivena_mt.txt"), 0)
        self.assertEqual(callTest("testivena_ma.txt"), 0)
        self.assertEqual(callTest("bv_cs_nihenatuna_pt.txt"), 0)

    def test_process(self):
        self.assertEqual(callTest("a.text.in", "a.text.out", ["QToApostrophe"]), 0)
        self.assertEqual(callTest("b.text.in", "b.text.out", ["createMa", "QToApostrophe"]), 0)
        self.assertEqual(callTest("apostropheivaena.text.in", "apostropheivaena.text.ou", ["createMa", "QToApostrophe", "IvenaToIvaEna"]), 0)

    def test_ivaena(self):
        self.assertEqual(callTest("ivaena.text.in", "ivaena.text.out", ["IvenaToIvaEna"]), 0)

        self.assertEqual(callTest("testivena_mt.txt", "testivena_ma.txt", ["IvenaToIvaEna"]), 0)

        self.assertEqual(callTest("realivaena.text.in", "realivaena.text.out", ["IvenaToIvaEna"]), 0)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Better version of testS4.py<commit_after>import unittest
import subprocess

def callTest(fileNameInput, fileNameOutput = None, extraarg = []):
    if fileNameOutput == None:
        fileNameOutput = fileNameInput
    with open('testfile', 'wb') as f:
        ret = subprocess.check_call(["python3", "solomagic.py", "tests/" + fileNameInput] + extraarg, stdout=f)
        if ret != 0:
            return 1

    # I think whitespace changes don't matter
    return subprocess.check_call(["diff", "-ub", "tests/" + fileNameOutput, "testfile"])

class TestIO(unittest.TestCase):

    def test_invariant(self):
        self.assertEqual(callTest("a.text.in"), 0)
        self.assertEqual(callTest("b.text.in"), 0)
        self.assertEqual(callTest("ivaena.text.in"), 0)

        self.assertEqual(callTest("testapostrophe_mt.txt"), 0)
        self.assertEqual(callTest("testivena_apostrophe_mt.txt"), 0)
        self.assertEqual(callTest("testivena_mt.txt"), 0)
        self.assertEqual(callTest("testivena_ma.txt"), 0)
        self.assertEqual(callTest("bv_cs_nihenatuna_pt.txt"), 0)

    def test_process(self):
        self.assertEqual(callTest("a.text.in", "a.text.out", ["QToApostrophe"]), 0)
        self.assertEqual(callTest("b.text.in", "b.text.out", ["createMa", "QToApostrophe"]), 0)
        self.assertEqual(callTest("apostropheivaena.text.in", "apostropheivaena.text.out", ["createMa", "QToApostrophe", "IvenaToIvaEna"]), 0)

    def test_ivaena(self):
        self.assertEqual(callTest("ivaena.text.in", "ivaena.text.out", ["IvenaToIvaEna"]), 0)

        self.assertEqual(callTest("testivena_mt.txt", "testivena_ma.txt", ["IvenaToIvaEna"]), 0)

        self.assertEqual(callTest("realivaena.text.in", "realivaena.text.out", ["IvenaToIvaEna"]), 0)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import json
import requests
from contextlib import contextmanager

from .webdriver import ResumableRemote

__all__ = ['Client', 'default_client ', 'reserve_browser ', 'release_browser ',
           'destroy_browser ', 'running_browsers ', 'browser_metadata ',
           'set_browser_tags ', 'refresh_browser ', 'browser ']

class ClientResumableRemote(ResumableRemote):
    def __init__(self, client, *args, **kwargs):
        super(ClientResumableRemote, self).__init__(*args, **kwargs)
        original_execute = self.command_executor.execute
        def releasable_execute(*args, **kwargs):
            if self.released:
                raise ValueError('The associated browser has been released, and'
                                 ' should no longer be used.')
            return original_execute(*args, **kwargs)
        self.command_executor.execute = releasable_execute
        self.client = client
        self.released = False

    def reserve(self):
        self.client.reserve_browser(self)

    def release(self):
        self.client.release_browser(self)
        self.released = True

    def destroy(self):
        self.client.destroy_browser(browser=self)

    def refresh(self):
        self.client.refresh_browser(browser=self)

    @property
    def tags(self):
        return tuple(
            self.client.browser_metadata(self.session_id).get('tags', []))

    @tags.setter
    def tags(self, new_tags):
        self.client.set_tags(self.session_id, new_tags)

class Client(object):
    def __init__(self, url_root='http://localhost:5000'):
        self.url_root = url_root
        self.headers = {'Content-Type': 'application/json'}

    def get_or_create_browser(self, browser_name='phantomjs', tags=None,
                              hub=None, reserve=False, force_create=False,
                              extra_desired_capabilities=None):

        tags = tags or []

        data = {
            'tags': tags,
            'browser_name': browser_name,
            'extra_desired_capabilities': extra_desired_capabilities,
        }
        if hub is not None:
            data['hub'] = hub
        resp = requests.post('{}/sessions/'.format(self.url_root),
                             data=json.dumps(data),
                             params={'force_create':force_create,
                                     'reserve':reserve},
                             headers=self.headers)
        resp_data = json.loads(resp.content.decode('UTF-8'))
        if reserve:
            return ClientResumableRemote(client=self,
                    session_id=resp_data['id'], command_executor=resp_data['hub'])
        return resp_data

    def create_browser(self, browser_name='phantomjs', tags=None,
                       hub=None, extra_desired_capabilities=None,
                       reserve=False):
        return self.get_or_create_browser(browser_name=browser_name, tags=tags,
                                          hub=hub, reserve=reserve,
                                          force_create=True)

    def reserve_browser(self, session_id):
        resp = requests.put('{}/sessions/{}'.format(self.url_root, session_id),
                data=json.dumps({'reserved': True}), headers=self.headers)
        resp_data = json.loads(resp.content.decode('UTF-8'))
        return ClientResumableRemote(client=self, session_id = resp_data['id'],
                command_executor=resp_data['hub'])

    def release_browser(self, browser):
        requests.put('{}/sessions/{}'.format(self.url_root, browser.session_id),
                data=json.dumps({'reserved': False}), headers=self.headers)

    def destroy_browser(self, browser=None, session_id=None):
        to_delete = session_id or getattr(browser, 'session_id')
        requests.delete('{}/sessions/{}'.format(self.url_root, to_delete))

    def running_browsers(self, reserved=None, browser_name=None,
            with_tags=None, without_tags=None):

        with_tags = with_tags or []
        without_tags = without_tags or []

        resp = requests.get('{}/sessions/'.format(self.url_root))
        browser_data = json.loads(resp.content.decode('UTF-8'))
        browser_data = (b for b in browser_data
                        if (reserved is None
                            or (b.get('reserved', False) == reserved)))
        browser_data = (b for b in browser_data
                        if (browser_name is None
                            or (b.get('browser_name', None) == browser_name)))
        browser_data = (b for b in browser_data
                        if (not with_tags
                            or (set(with_tags) <= set(b.get('tags', [])))))
        browser_data = (b for b in browser_data
                        if (not without_tags
                            or (set(b.get('tags', [])).isdisjoint(set(without_tags)))))
        return tuple(browser_data)

    def browser_metadata(self, session_id):
        resp = requests.get('{}/sessions/{}'.format(self.url_root, session_id))
        return json.loads(resp.content.decode('UTF-8'))

    def set_browser_tags(self, session_id, tags=None):

        tags = tags or []

        requests.put('{}/sessions/{}'.format(self.url_root, session_id),
            data=json.dumps({'tags': tags}), headers=self.headers)

    def refresh_browser(self, browser=None, session_id=None):
        to_refresh = session_id or getattr(browser, 'session_id')
        if to_refresh is None:
            requests.post('{}/sessions/refresh'.format(self.url_root))
        else:
            url = '{}/sessions/{}/refresh'.format(self.url_root, to_refresh)
            requests.post(url)

    @contextmanager
    def browser(self, session_id=None, browser_name=None, tags=None, hub=None,
                extra_desired_capabilities=None):
        if session_id:
            browser = self.reserve_browser(session_id)
        else:
            browser = self.get_or_create_browser(
                    browser_name=browser_name, tags=tags, hub=hub,
                    extra_desired_capabilities=extra_desired_capabilities,
                    reserve=True)
        try:
            yield browser
        finally:
            browser.release()

default_client = Client()

get_or_create_browser = default_client.get_or_create_browser

create_browser = default_client.create_browser

reserve_browser = default_client.reserve_browser

release_browser = default_client.reserve_browser

destroy_browser = default_client.destroy_browser

running_browsers = default_client.running_browsers

browser_metadata = default_client.browser_metadata

set_browser_tags = default_client.set_browser_tags

refresh_browser = default_client.refresh_browser

browser = default_client.browser
<commit_msg>Update the client to use nodes over hubs.<commit_after>import json
import requests
from contextlib import contextmanager

from .webdriver import ResumableRemote

__all__ = ['Client', 'default_client ', 'reserve_browser ', 'release_browser ',
           'destroy_browser ', 'running_browsers ', 'browser_metadata ',
           'set_browser_tags ', 'refresh_browser ', 'browser ']

class ClientResumableRemote(ResumableRemote):
    def __init__(self, client, *args, **kwargs):
        super(ClientResumableRemote, self).__init__(*args, **kwargs)
        original_execute = self.command_executor.execute
        def releasable_execute(*args, **kwargs):
            if self.released:
                raise ValueError('The associated browser has been released, and'
                                 ' should no longer be used.')
            return original_execute(*args, **kwargs)
        self.command_executor.execute = releasable_execute
        self.client = client
        self.released = False

    def reserve(self):
        self.client.reserve_browser(self)

    def release(self):
        self.client.release_browser(self)
        self.released = True

    def destroy(self):
        self.client.destroy_browser(browser=self)

    def refresh(self):
        self.client.refresh_browser(browser=self)

    @property
    def tags(self):
        return tuple(
            self.client.browser_metadata(self.session_id).get('tags', []))

    @tags.setter
    def tags(self, new_tags):
        self.client.set_tags(self.session_id, new_tags)

class Client(object):
    def __init__(self, url_root='http://localhost:5000'):
        self.url_root = url_root
        self.headers = {'Content-Type': 'application/json'}

    def get_or_create_browser(self, browser_name='phantomjs', tags=None,
                              node=None, reserve=False, force_create=False,
                              extra_desired_capabilities=None):

        tags = tags or []

        data = {
            'tags': tags,
            'browser_name': browser_name,
            'extra_desired_capabilities': extra_desired_capabilities,
        }
        if node is not None:
            data['node'] = node
        resp = requests.post('{}/sessions/'.format(self.url_root),
                             data=json.dumps(data),
                             params={'force_create':force_create,
                                     'reserve':reserve},
                             headers=self.headers)
        resp_data = json.loads(resp.content.decode('UTF-8'))
        if reserve:
            return ClientResumableRemote(client=self,
                    session_id=resp_data['id'], command_executor=resp_data['node'])
        return resp_data

    def create_browser(self, browser_name='phantomjs', tags=None,
                       node=None, extra_desired_capabilities=None,
                       reserve=False):
        return self.get_or_create_browser(browser_name=browser_name, tags=tags,
                                          node=node, reserve=reserve,
                                          force_create=True)

    def reserve_browser(self, session_id):
        resp = requests.put('{}/sessions/{}'.format(self.url_root, session_id),
                data=json.dumps({'reserved': True}), headers=self.headers)
        resp_data = json.loads(resp.content.decode('UTF-8'))
        return ClientResumableRemote(client=self, session_id = resp_data['id'],
                command_executor=resp_data['node'])

    def release_browser(self, browser):
        requests.put('{}/sessions/{}'.format(self.url_root, browser.session_id),
                data=json.dumps({'reserved': False}), headers=self.headers)

    def destroy_browser(self, browser=None, session_id=None):
        to_delete = session_id or getattr(browser, 'session_id')
        requests.delete('{}/sessions/{}'.format(self.url_root, to_delete))

    def running_browsers(self, reserved=None, browser_name=None,
            with_tags=None, without_tags=None):

        with_tags = with_tags or []
        without_tags = without_tags or []

        resp = requests.get('{}/sessions/'.format(self.url_root))
        browser_data = json.loads(resp.content.decode('UTF-8'))
        browser_data = (b for b in browser_data
                        if (reserved is None
                            or (b.get('reserved', False) == reserved)))
        browser_data = (b for b in browser_data
                        if (browser_name is None
                            or (b.get('browser_name', None) == browser_name)))
        browser_data = (b for b in browser_data
                        if (not with_tags
                            or (set(with_tags) <= set(b.get('tags', [])))))
        browser_data = (b for b in browser_data
                        if (not without_tags
                            or (set(b.get('tags', [])).isdisjoint(set(without_tags)))))
        return tuple(browser_data)

    def browser_metadata(self, session_id):
        resp = requests.get('{}/sessions/{}'.format(self.url_root, session_id))
        return json.loads(resp.content.decode('UTF-8'))

    def set_browser_tags(self, session_id, tags=None):

        tags = tags or []

        requests.put('{}/sessions/{}'.format(self.url_root, session_id),
            data=json.dumps({'tags': tags}), headers=self.headers)

    def refresh_browser(self, browser=None, session_id=None):
        to_refresh = session_id or getattr(browser, 'session_id')
        if to_refresh is None:
            requests.post('{}/sessions/refresh'.format(self.url_root))
        else:
            url = '{}/sessions/{}/refresh'.format(self.url_root, to_refresh)
            requests.post(url)

    @contextmanager
    def browser(self, session_id=None, browser_name=None, tags=None, node=None,
                extra_desired_capabilities=None):
        if session_id:
            browser = self.reserve_browser(session_id)
        else:
            browser = self.get_or_create_browser(
                    browser_name=browser_name, tags=tags, node=node,
                    extra_desired_capabilities=extra_desired_capabilities,
                    reserve=True)
        try:
            yield browser
        finally:
            browser.release()

default_client = Client()

get_or_create_browser = default_client.get_or_create_browser

create_browser = default_client.create_browser

reserve_browser = default_client.reserve_browser

release_browser = default_client.reserve_browser

destroy_browser = default_client.destroy_browser

running_browsers = default_client.running_browsers

browser_metadata = default_client.browser_metadata

set_browser_tags = default_client.set_browser_tags

refresh_browser = default_client.refresh_browser

browser = default_client.browser
<|endoftext|>


<commit_before>import functools, string, googletrans
from Cogs import DisplayName, Message, PickList
from discord.ext import commands

def setup(bot):
    # Add the bot and deps
    settings = bot.get_cog("Settings")
    bot.add_cog(Translate(bot, settings))

# Requires the mtranslate module be installed

class Translate(commands.Cog):

    def __init__(self, bot, settings):
        self.bot = bot
        self.settings = settings
        self.translator = googletrans.Translator(service_urls=["translate.googleapis.com"])
        global Utils, DisplayName
        Utils = self.bot.get_cog("Utils")
        DisplayName = self.bot.get_cog("DisplayName")
        self.langcodes = googletrans.LANGCODES
        self.languages = googletrans.LANGUAGES

    @commands.command(pass_context=True)
    async def langlist(self, ctx):
        """Lists available languages."""
        description = ""
        for lang in googletrans.LANGCODES:
            description += "**{}** - {}\n".format(string.capwords(lang), googletrans.LANGCODES[lang])
        await PickList.PagePicker(
            title="Language List",
            description=description,
            color=ctx.author,
            ctx=ctx
        ).pick()

    @commands.command(pass_context=True)
    async def detectlang(self, ctx, *, text):
        """Reports the detected language and certainty of the passed text."""
        if text == None: return await ctx.send("Usage: `{}detectlang [text to identify]`".format(ctx.prefix))
        lang_detect = await self.bot.loop.run_in_executor(None, self.translator.detect, text)
        await Message.EmbedText(
            title="Detected Language",
            description="Detected **{}** ({}) with {:.0%} confidence.".format(
                string.capwords(googletrans.LANGUAGES.get(lang_detect.lang.lower(), "Martian?")),
                lang_detect.lang.lower(),
                lang_detect.confidence
            ),
            color=ctx.author
        ).send(ctx)

    def _unpack(self, value):
        if isinstance(value, list):
            while True:
                if any((isinstance(x, list) for x in value)): value = sum(value, [])
                else: break
            if value: value = value[0]
        return value

    @commands.command(name="translate", aliases=["tr"], pass_context=True)
    async def translate(self, ctx, *, translate=None):
        """Translate some stuff!  Takes a phrase, the from language identifier and the to language identifier (optional).
        To see a number of potential language identifiers, use the langlist command.

        Example Translation:
        $tr Hello there, how are you? en es

        Would translate from English to Spanish resulting in:
        ¿Hola como estás?

        If you do not specify the from language, Google translate will attempt to automatically determine it.
        If you do not specify the to language, it will default to English."""

        usage = "Usage: `{}tr [words] [from code (optional)] [to code (optional)]`".format(ctx.prefix)
        if translate == None: return await ctx.send(usage)

        word_list = translate.split(" ")
        if len(word_list) < 1: return await ctx.send(usage)

        to_lang = word_list[-1].lower() if word_list[-1].lower() in self.langcodes.values() else None  # check for to_lang
        if to_lang: word_list.pop()  # Remove the last word from the list, i.e. the to_lang
        else: to_lang = "en"  # Default to english

        # there should be at least 2 words left after we remove the to_lang, in case the user specifies a source language
        from_lang = word_list[-1].lower() if len(word_list) >= 2 and word_list[-1].lower() in self.langcodes.values() else None
        if from_lang: word_list.pop()  # remove the last word from the list, i.e. the from_lang (since the to_lang has been removed already)

        # Get the from language name from the passed code
        if from_lang: from_lang_name = self.languages.get(from_lang, None)
        else: from_lang_name = None

        # Get the to language name from the passed code
        if to_lang: to_lang_name = self.languages.get(to_lang, None)
        else: to_lang_name = None

        if not to_lang_name:  # No dice on the language :(
            return await Message.EmbedText(
                title="Something went wrong...",
                description="I couldn't find that language!",
                color=ctx.author
            ).send(ctx)

        # Get our words joined with spaces
        to_translate = " ".join(word_list) if word_list else ""

        if from_lang_name:
            result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=to_translate, src=from_lang, dest=to_lang))
        else:
            # We'll leave Google Translate to figure out the source language if we don't have it
            result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=to_translate, dest=to_lang))

        # Explore the results!
        result.text = self._unpack(result.text)
        if not result.text:
            return await Message.EmbedText(
                title="Something went wrong...",
                description="I wasn't able to translate that!",
                color=ctx.author
            ).send(ctx)

        if result.text == to_translate:
            # We got back what we put in...
            return await Message.EmbedText(
                title="Something went wrong...",
                description="The text returned from Google was the same as the text put in.  Either the translation failed - or you were translating from/to the same language ({src} -> {src})".format(
                    src=result.src
                ),
                color=ctx.author
            ).send(ctx)

        # We got a translation!
        embed = Message.Embed(
            title="{}, your translation is:".format(DisplayName.name(ctx.author)),
            force_pm=True,
            color=ctx.author,
            description=result.text,
            footer="{} --> {}".format(
                self.languages.get(result.src.lower(), "Unknown").title(),
                self.languages.get(result.dest.lower(), "Unknown").title()
            )
        )

        # If we have a valid pronunciation, add it to the embed!
        result.pronunciation = self._unpack(result.pronunciation)
        if result.pronunciation:
            if isinstance(result.pronunciation, list):
                result.pronunciation = result.pronunciation[0]
            if not any(result.pronunciation == x for x in (result.text, to_translate)):
                embed.add_field(name="Pronunciation", value=result.pronunciation, inline=False)

        await embed.send(ctx)

    @commands.command(name="pronounce", aliases=["pr"])
    async def pronounce(self, ctx, *, text=None):
        """Pronunciation for a sentence in the English language.\n
        $pronounce こんにちは --> returns \"Kon'nichiwa\""""

        if text == None: return await ctx.send("Usage: `{}pronounce [text to identify]`".format(ctx.prefix))

        if text.split()[-1] in self.langcodes.values():
            source_lang = text.split()[-1]
            text = " ".join(text.split()[:-1])  # We are removing the last word from the list, since it's the language code
        else:
            source_lang = None

        if len(text) > 1000:  # We need to keep it under 1024 characters due to field size limits
            return await ctx.send("Text too long. Please keep it under 1000 characters.")

        detect_result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.detect, text))  # We are detecting the language of the text

        if not source_lang:
            if isinstance(detect_result.confidence, list):  # We got multiple results
                source_lang = detect_result.lang[0]
                lang_confidence = detect_result.confidence[0]
            else:
                source_lang = detect_result.lang
                lang_confidence = detect_result.confidence
        else:
            lang_confidence = 1  # When the user specifies a source language, we assume they know what they're doing

        pronunciation_result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=text, src=source_lang, dest=source_lang))
        # We don't need to translate to another language, we just get the pronunciation
        if not pronunciation_result.pronunciation or pronunciation_result.pronunciation == pronunciation_result.src:
            return await ctx.send("I couldn't find a pronunciation for that text!")  # We don't want a pronunciation that's the same as the text

        english_translation = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=text, src=source_lang, dest="en"))
        if not self.languages.get(source_lang):
            return await ctx.send("I couldn't detect the language of the text!")

        embed = Message.Embed(title="{}, your pronunciation is:".format(ctx.author.display_name),
                              color=ctx.author.color)
        embed.description = pronunciation_result.pronunciation
        embed.add_field(name="Source Text", value=pronunciation_result.text, inline=False)
        embed.add_field(name="Translated to English", value=english_translation.text, inline=False)
        if int(lang_confidence) != 1:
            embed.footer = {
                "text": "Language: {} (Confidence: {}%)".format(
                    self.languages.get(source_lang).title() if self.languages.get(source_lang) else 'Unknown',
                    round(lang_confidence * 100, 2))
            }
        else:
            embed.footer = {
                "text": "Language: {}".format(self.languages.get(source_lang).title() if self.languages.get(source_lang) else 'Unknown')
            }
        await embed.send(ctx)
<commit_msg>Compare None with "is", report errors in embeds<commit_after>import functools, string, googletrans
from Cogs import DisplayName, Message, PickList
from discord.ext import commands

def setup(bot):
    # Add the bot and deps
    settings = bot.get_cog("Settings")
    bot.add_cog(Translate(bot, settings))

# Requires the mtranslate module be installed

class Translate(commands.Cog):

    def __init__(self, bot, settings):
        self.bot = bot
        self.settings = settings
        self.translator = googletrans.Translator(service_urls=["translate.googleapis.com"])
        global Utils, DisplayName
        Utils = self.bot.get_cog("Utils")
        DisplayName = self.bot.get_cog("DisplayName")
        self.langcodes = googletrans.LANGCODES
        self.languages = googletrans.LANGUAGES

    @commands.command(pass_context=True)
    async def langlist(self, ctx):
        """Lists available languages."""
        description = ""
        for lang in googletrans.LANGCODES:
            description += "**{}** - {}\n".format(string.capwords(lang), googletrans.LANGCODES[lang])
        await PickList.PagePicker(
            title="Language List",
            description=description,
            color=ctx.author,
            ctx=ctx
        ).pick()

    @commands.command(pass_context=True)
    async def detectlang(self, ctx, *, text):
        """Reports the detected language and certainty of the passed text."""
        if text is None: return await ctx.send("Usage: `{}detectlang [text to identify]`".format(ctx.prefix))
        lang_detect = await self.bot.loop.run_in_executor(None, self.translator.detect, text)
        await Message.EmbedText(
            title="Detected Language",
            description="Detected **{}** ({}) with {:.0%} confidence.".format(
                string.capwords(googletrans.LANGUAGES.get(lang_detect.lang.lower(),"Martian?")),
                lang_detect.lang.lower(),
                lang_detect.confidence
            ),
            color=ctx.author
        ).send(ctx)

    def _unpack(self,value):
        if isinstance(value,list):
            while True:
                if any((isinstance(x,list) for x in value)): value = sum(value,[])
                else: break
            if value: value = value[0]
        return value

    @commands.command(name="translate", aliases=["tr"], pass_context=True)
    async def translate(self, ctx, *, translate=None):
        """Translate some stuff!  Takes a phrase, the from language identifier and the to language identifier (optional).
        To see a number of potential language identifiers, use the langlist command.

        Example Translation:
        $tr Hello there, how are you? en es

        Would translate from English to Spanish resulting in:
        ¿Hola como estás?

        If you do not specify the from language, Google translate will attempt to automatically determine it.
        If you do not specify the to language, it will default to English."""

        usage = "Usage: `{}tr [words] [from code (optional)] [to code (optional)]`".format(ctx.prefix)
        if translate is None: return await ctx.send(usage)

        word_list = translate.split(" ")
        if len(word_list) < 1: return await ctx.send(usage)

        to_lang = word_list[-1].lower() if word_list[-1].lower() in self.langcodes.values() else None  # check for to_lang
        if to_lang: word_list.pop()  # Remove the last word from the list, i.e. the to_lang
        else: to_lang = "en"  # Default to english

        # there should be at least 2 words left after we remove the to_lang, in case the user specifies a source language
        from_lang = word_list[-1].lower() if len(word_list) >= 2 and word_list[-1].lower() in self.langcodes.values() else None
        if from_lang: word_list.pop()  # remove the last word from the list, i.e. the from_lang (since the to_lang has been removed already)

        # Get the from language name from the passed code
        if from_lang: from_lang_name = self.languages.get(from_lang, None)
        else: from_lang_name = None

        # Get the to language name from the passed code
        if to_lang: to_lang_name = self.languages.get(to_lang, None)
        else: to_lang_name = None

        if not to_lang_name:  # No dice on the language :(
            return await Message.EmbedText(
                title="Something went wrong...",
                description="I couldn't find that language!",
                color=ctx.author
            ).send(ctx)

        # Get our words joined with spaces
        to_translate = " ".join(word_list) if word_list else ""

        if from_lang_name:
            result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=to_translate, src=from_lang, dest=to_lang))
        else:
            # We'll leave Google Translate to figure out the source language if we don't have it
            result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=to_translate, dest=to_lang))

        # Explore the results!
        result.text = self._unpack(result.text)
        if not result.text:
            return await Message.EmbedText(
                title="Something went wrong...",
                description="I wasn't able to translate that!",
                color=ctx.author
            ).send(ctx)

        if result.text == to_translate:
            # We got back what we put in...
            return await Message.EmbedText(
                title="Something went wrong...",
                description="The text returned from Google was the same as the text put in.  Either the translation failed - or you were translating from/to the same language ({src} -> {src})".format(
                    src=result.src
                ),
                color=ctx.author
            ).send(ctx)

        # We got a translation!
        embed = Message.Embed(
            title="{}, your translation is:".format(DisplayName.name(ctx.author)),
            force_pm=True,
            color=ctx.author,
            description=result.text,
            footer="{} --> {}".format(
                self.languages.get(result.src.lower(), "Unknown").title(),
                self.languages.get(result.dest.lower(), "Unknown").title()
            )
        )

        # If we have a valid pronunciation, add it to the embed!
        result.pronunciation = self._unpack(result.pronunciation)
        if result.pronunciation:
            if isinstance(result.pronunciation,list):
                result.pronunciation = result.pronunciation[0]
            if not any(result.pronunciation == x for x in (result.text,to_translate)):
                embed.add_field(name="Pronunciation", value=result.pronunciation, inline=False)

        await embed.send(ctx)

    @commands.command(name="pronounce", aliases=["pr"])
    async def pronounce(self, ctx, *, text=None):
        """Pronunciation for a sentence in the English language.\n
        $pronounce こんにちは --> returns \"Kon'nichiwa\""""

        if text is None: return await ctx.send("Usage: `{}pronounce [text to identify]`".format(ctx.prefix))

        if text.split()[-1] in self.langcodes.values():
            source_lang = text.split()[-1]
            text = " ".join(text.split()[:-1])  # We are removing the last word from the list, since it's the language code
        else:
            source_lang = None

        if len(text) > 1000:  # We need to keep it under 1024 characters due to field size limits
            return await Message.EmbedText(
                title="Something went wrong...",
                description="Text entered is too long. Please keep it under 1000 characters.",
                color=ctx.author
            ).send(ctx)

        detect_result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.detect, text))  # We are detecting the language of the text

        if not source_lang:
            if isinstance(detect_result.confidence, list):  # We got multiple results
                source_lang = detect_result.lang[0]
                lang_confidence = detect_result.confidence[0]
            else:
                source_lang = detect_result.lang
                lang_confidence = detect_result.confidence
        else:
            lang_confidence = 1  # When the user specifies a source language, we assume they know what they're doing

        pronunciation_result = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=text, src=source_lang, dest=source_lang))
        # We don't need to translate to another language, we just get the pronunciation
        if not pronunciation_result.pronunciation or pronunciation_result.pronunciation == pronunciation_result.src:
            return await Message.EmbedText(
                title="Something went wrong...",
                description="I couldn't find a pronunciation for that text!",
                color=ctx.author
            ).send(ctx) # We don't want a pronunciation that's the same as the text

        english_translation = await self.bot.loop.run_in_executor(None, functools.partial(self.translator.translate, text=text, src=source_lang, dest="en"))
        if not self.languages.get(source_lang):
            return await Message.EmbedText(
                title="Something went wrong...",
                description="I couldn't detect the language of the text!",
                color=ctx.author
            ).send(ctx)

        embed = Message.Embed(title="{}, your pronunciation is:".format(ctx.author.display_name),
                              color=ctx.author.color)
        embed.description = pronunciation_result.pronunciation
        embed.add_field(name="Source Text", value=pronunciation_result.text, inline=False)
        embed.add_field(name="Translated to English", value=english_translation.text, inline=False)
        if int(lang_confidence) != 1:
            embed.footer = {
                "text": "Language: {} (Confidence: {}%)".format(
                    self.languages.get(source_lang).title() if self.languages.get(source_lang) else 'Unknown',
                    round(lang_confidence * 100, 2))
            }
        else:
            embed.footer = {
                "text": "Language: {}".format(self.languages.get(source_lang).title() if self.languages.get(source_lang) else 'Unknown')
            }
        await embed.send(ctx)
<|endoftext|>


<commit_before>#!/usr/bin/python3
# vim: set tw=79:

"""
Print to file differences in two directory structures, ignoring file contents.

Looking recursively in two directories (which are defined below),
print to file the differences found in the subdirectory names,
highlighting odd subdirectories.

Reason for this:
SyncBackFree is a powerful synchronisation program, but with two drawbacks:
    * big changes in directory structure are hard to interpret and act upon
    * it seems to sync the existence of empty directories without reporting
So when I've changed a lot on my netbook, and wish to sync my directory
with my portable hard drive, I can run this module first, and use the output
to guide me through some manual directory synching before running SyncBackFree.

This program does not change anything, but just (re)writes the result file,
which is automatically named after this program.
"""
import os
import socket
import sys
import time
import datetime
import re

# Get the start time, to be able to check against later:
start = time.time()
# Get the datetime:
startd = datetime.datetime.now().isoformat(' ')

# Specify the parent directory's location:
drv = ['D:/', 'E:/']

# List of empty directories not to be flagged up
# (add regex's to this as required):
known_empty = [r'\.dropbox\.cache',
               r'Cross-platform\\ImageMagick',
               r'IT\\Cross-platform\\Browsing\\Firefox\\profile\\',
               r'Firefox\\.*\.default\\',
               r'GT-S7560 Camera',
               r'MSWinUser\\.*\\.*\.default\\',
               r'MSWinUser\\.*\\user\\',
               r'MSWin\\AV\\ImageMagick-6.8.9\\',
               r'Sharon\'s Music\\',
               r'SOFTWARE\\DOTNET\\ENGLISH',
               r'WriteBackup',
               r'\\CDExPortable\\',
               r'\\Gigabyte GA M57SLI-S4\\',
               r'\\tempera\\admin\\images\\schemes',
               r'\\\.git\\',
               r'\\\.vim\\',
               r'\\vim-markdown\\\.git\\',
               r'\\vimfiles\\',
               r'backup-.*_josep396\\',
               r'debian-7.5.0-i386-xfce-CD-1\\Snapshots',
               r'config\\soffice\.cfg\\modules\\']

# Get this script's directory path:
sdp = os.getcwd()[3:]
if(sdp):
    sdp = sdp + '/'
# Get this script's filename without extension,
# for use as the name of the directory to be synched:
dtbs = os.path.basename(os.path.splitext(sys.argv[0])[0])
# and for use as the name of the output file:
outfile = dtbs + '.txt'
print('When this is done, you should open', outfile, '\n')

def dirlister(dirTolist):
    """ Prepare a subdirectory list, with relative paths included.
    (Progress is reported, and odd directories are highlighted.)"""
    print('Looking at contents of', dirTolist, ':')
    dlc = olc = 0
    # Initialise two lists just with the base folder path:
    dirList = [dirTolist]
    oddList = [dirTolist]
    for root, folders, files in os.walk(dirTolist):
        for subd in folders:
            # Report a progress count:
            dlc += 1
            print('\r', dlc, end=' ')
            # subd is just the folder name, need to add its path:
            abspath = os.path.join(root, subd)
            relpath = abspath.replace(dirTolist+"\\", "")
            # First check there's not an unwriteable
            # Unicode escape character gotten in there:
            if '\ufffd' in abspath:
                # Store such messed-up directory paths as ASCII:
                oddList.append(ascii(relpath))
                olc += 1
            else:
                # Check there's something in the subdirectory:
                if (os.listdir(abspath) or
                        # (treat known empty subdirectories normally:)
                        any(re.search(ke, abspath) for ke in known_empty)):
                    # Add relative path to the subdirectory name, and store it
                    # in the list for the directory to be compared:
                    dirList.append(relpath)
                else:
                    # Add relative path to the subdirectory name, and store it
                    # in the list of empty directories:
                    oddList.append(relpath)
                    olc += 1
                    # Store the full path of an empty directory in the list
                    # (thus highlighting it):
                    # dirList.append(abspath)
    print(' - subdirectory records loaded in')
    # Adjust the first (title) items:
    dlctxt = (' ---> contains ' + str(dlc) + ' subdirectories, these ones ')
    oddList[0] += (dlctxt + 'are oddly Empty:')
    dirList[0] += (dlctxt + 'are Unmatched:')
    return dirList, (dlc - olc), oddList

# Now build two lists of subdirectories, with counts.
# Initialise a list to contain the two pairs of lists:
list = [[], []]
# (taking care to ensure they've separate id's)
empt = [[], []]
# and the two counts:
sdc = [0, 0]
# Get the lists:
for d in range(0, 2):
    list[d], sdc[d], empt[d] = dirlister(drv[d] + sdp + dtbs)

# Identify the index of the list to be picked through:
if sdc[0] < sdc[1]:
    d = 0
# Prepare an empty list to move unmatched items into:
listd = []
# Identify an index for the other (possibly longer) list:
dl = abs(d-1)

# Pick through one list now, comparing items with with the other list:
print('Cross-checking now for differences :')
for _ in range(sdc[d]+1):
    # Pull off the first item from the pick-list:
    item = list[d].pop(0)
    print('\r', _, end=' ')
    # If the item's in the other list, remove it from there too:
    try:
        list[dl].remove(item)
    # If not, save it to the (new) unmatched list:
    except ValueError:
        listd.append(item)
print(' - subdirectory records compared')

# Create a file object for output:
fo = open(outfile, 'w')

# Create a nice header:
wrt1 = '\n' + socket.gethostname()+' folder changes at '+startd+'\n'
wrt2 = '\n'+'\n'.join(empt[0])+'\n\n'+'\n'.join(empt[1])+'\n'
# Next the two lists of unmatched items
# (automatically headed by their title lines):
wrt3 = '\n'+'\n'.join(list[dl])+'\n\n'+'\n'.join(listd)+'\n'
# And the time taken:
wrt4 = '\ntook '+str(time.time()-start)+' seconds to find the differences'

# Write and close the file object:
fo.write(wrt1+wrt2+wrt3+wrt4)
fo.close()
<commit_msg>tidied known empty list<commit_after>#!/usr/bin/python3
# vim: set tw=79:

"""
Print to file differences in two directory structures, ignoring file contents.

Looking recursively in two directories (which are defined below),
print to file the differences found in the subdirectory names,
highlighting odd subdirectories.

Reason for this:
SyncBackFree is a powerful synchronisation program, but with two drawbacks:
    * big changes in directory structure are hard to interpret and act upon
    * it seems to sync the existence of empty directories without reporting
So when I've changed a lot on my netbook, and wish to sync my directory
with my portable hard drive, I can run this module first, and use the output
to guide me through some manual directory synching before running SyncBackFree.

This program does not change anything, but just (re)writes the result file,
which is automatically named after this program.
"""
import os
import socket
import sys
import time
import datetime
import re

# Get the start time, to be able to check against later:
start = time.time()
# Get the datetime:
startd = datetime.datetime.now().isoformat(' ')

# Specify the parent directory's location:
drv = ['D:/', 'E:/']

# List of empty directories not to be flagged up
# (add regex's to this as required):
known_empty = [r'\.dropbox\.cache',
               r'Cross-platform\\ImageMagick',
               # anonymized Firefox profile folders:
               r'IT\\Cross-platform\\Browsing\\Firefox\\profile\\',
               r'GT-S7560 Camera',
               r'Gigabyte GA M57SLI-S4\\',
               # my Firefox profile folders:
               r'MSWinUser\\.*\\.*\.default\\',
               # my LibreOffice profile folders:
               r'MSWinUser\\.*\\user\\',
               r'MSWin\\AV\\ImageMagick-6.8.9\\',
               r'Sharon\'s Music\\',
               r'SOFTWARE\\DOTNET\\ENGLISH',
               r'WriteBackup',
               r'\\CDExPortable\\',
               r'\\tempera\\admin\\images\\schemes',
               r'\\\.git\\',
               r'\\\.vim\\',
               r'\\vimfiles\\',
               r'backup-.*_josep396\\',
               r'debian-7.5.0-i386-xfce-CD-1\\Snapshots',
               r'config\\soffice\.cfg\\modules\\']

# Get this script's directory path:
sdp = os.getcwd()[3:]
if(sdp):
    sdp = sdp + '/'
# Get this script's filename without extension,
# for use as the name of the directory to be synched:
dtbs = os.path.basename(os.path.splitext(sys.argv[0])[0])
# and for use as the name of the output file:
outfile = dtbs + '.txt'
print('When this is done, you should open', outfile, '\n')

def dirlister(dirTolist):
    """ Prepare a subdirectory list, with relative paths included.
    (Progress is reported, and odd directories are highlighted.)"""
    print('Looking at contents of', dirTolist, ':')
    dlc = olc = 0
    # Initialise two lists just with the base folder path:
    dirList = [dirTolist]
    oddList = [dirTolist]
    for root, folders, files in os.walk(dirTolist):
        for subd in folders:
            # Report a progress count:
            dlc += 1
            print('\r', dlc, end=' ')
            # subd is just the folder name, need to add its path:
            abspath = os.path.join(root, subd)
            relpath = abspath.replace(dirTolist+"\\", "")
            # First check there's not an unwriteable
            # Unicode escape character gotten in there:
            if '\ufffd' in abspath:
                # Store such messed-up directory paths as ASCII:
                oddList.append(ascii(relpath))
                olc += 1
            else:
                # Check there's something in the subdirectory:
                if (os.listdir(abspath) or
                        # (treat known empty subdirectories normally:)
                        any(re.search(ke, abspath) for ke in known_empty)):
                    # Add relative path to the subdirectory name, and store it
                    # in the list for the directory to be compared:
                    dirList.append(relpath)
                else:
                    # Add relative path to the subdirectory name, and store it
                    # in the list of empty directories:
                    oddList.append(relpath)
                    olc += 1
                    # Store the full path of an empty directory in the list
                    # (thus highlighting it):
                    # dirList.append(abspath)
    print(' - subdirectory records loaded in')
    # Adjust the first (title) items:
    dlctxt = (' ---> contains ' + str(dlc) + ' subdirectories, these ones ')
    oddList[0] += (dlctxt + 'are oddly Empty:')
    dirList[0] += (dlctxt + 'are Unmatched:')
    return dirList, (dlc - olc), oddList

# Now build two lists of subdirectories, with counts.
# Initialise a list to contain the two pairs of lists:
list = [[], []]
# (taking care to ensure they've separate id's)
empt = [[], []]
# and the two counts:
sdc = [0, 0]
# Get the lists:
for d in range(0, 2):
    list[d], sdc[d], empt[d] = dirlister(drv[d] + sdp + dtbs)

# Identify the index of the list to be picked through:
if sdc[0] < sdc[1]:
    d = 0
# Prepare an empty list to move unmatched items into:
listd = []
# Identify an index for the other (possibly longer) list:
dl = abs(d-1)

# Pick through one list now, comparing items with with the other list:
print('Cross-checking now for differences :')
for _ in range(sdc[d]+1):
    # Pull off the first item from the pick-list:
    item = list[d].pop(0)
    print('\r', _, end=' ')
    # If the item's in the other list, remove it from there too:
    try:
        list[dl].remove(item)
    # If not, save it to the (new) unmatched list:
    except ValueError:
        listd.append(item)
print(' - subdirectory records compared')

# Create a file object for output:
fo = open(outfile, 'w')

# Create a nice header:
wrt1 = '\n' + socket.gethostname()+' folder changes at '+startd+'\n'
wrt2 = '\n'+'\n'.join(empt[0])+'\n\n'+'\n'.join(empt[1])+'\n'
# Next the two lists of unmatched items
# (automatically headed by their title lines):
wrt3 = '\n'+'\n'.join(list[dl])+'\n\n'+'\n'.join(listd)+'\n'
# And the time taken:
wrt4 = '\ntook '+str(time.time()-start)+' seconds to find the differences'

# Write and close the file object:
fo.write(wrt1+wrt2+wrt3+wrt4)
fo.close()
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""Effortlessly read a text file using counting numbers.

Created 2013-2017 Caleb Ely
<https://CodeTri.net>

Licensed under The MIT License
<http://opensource.org/licenses/MIT/>

"""

from __future__ import unicode_literals

import re
import sys
import locale

# Grab Python 3.0+ open() function
if sys.version_info[:2] < (3, 0):
    from io import open

__all__ = ("LineScan")

class LineScan(object):
    """Main linescan class.

    Instance using linescan.LineScan()
    Primary entry point is scan() function.
    """

    def __init__(self):
        """Initialize private and public variables."""
        self.__scans = {}
        self.__enable_exceptions = False

        self.start_line = None
        self.end_line = None
        self.encoding = None
        self.file_name = ""

    # ------- Private Methods ------- #
    def __len__(self):
        """Expose the total number of files scanned.

        @returns {Integer}
        """
        return len(self.__scans)

    def __set_details(self, filename, lineno, endline, encoding):
        """Store scan details."""
        self.start_line = lineno
        self.end_line = endline
        self.encoding = encoding
        self.file_name = filename

    def __clear_details(self):
        """Reset scan details."""
        self.start_line = None
        self.end_line = None
        self.encoding = None
        self.file_name = ""

    def __check_bool(self, value):
        """Used to check if various options should be enabled."""
        return value is True

    def __create_pointer(self):
        """Construct the comma-separated pointer for the specified file."""
        filePointer = "{0},{1}".format(self.file_name, self.start_line)

        # Append the ending line if one is specified
        if self.end_line is not None:
            filePointer = "{0},{1}".format(
                filePointer, self.end_line)

        # An encoding is always specified even if
        # the user did not provide one
        filePointer = "{0},encode={1}".format(filePointer, self.encoding)
        return filePointer

    def __raise_exception(self, exc, generic=True):
        """Handle errors per exception option."""
        # Raise an exception if they are enabled
        if self.__enable_exceptions:
            # A "generic" exception should be raised
            if generic:
                raise exc

            # A rescaning error should be raised instead
            # Raise FileNotFoundError exception on Python 3.3+
            if sys.version_info[:2] >= (3, 3):
                raise FileNotFoundError(exc)  # noqa

            # Raise the old IOError on Python 3.2 and lower
            else:
                raise IOError(exc)

        # If exceptions have not been enabled, simply return False
        return False

    def __scanner(self):
        """Perform the actual scan."""
        try:
            # Since line numbers start at 0, get the starting line number.
            # No need to do the same for the ending line,
            # as it is not modified
            startLine = self.start_line - 1

            # Open the file for scanning using specified encoding
            with open(self.file_name, "rt", encoding=self.encoding) as f:

                # The user wants to scan only one line.
                if self.end_line is None:
                    lines = f.readlines()[startLine]

                # The user wants to scan until the end of the file.
                elif self.end_line == "end":
                    lines = f.readlines()[startLine:]

                # The user wants to scan multiple specified lines.
                else:
                    lines = f.readlines()[startLine:self.end_line]

            # Break the multiple lines from the returned list.
            lines = "".join(lines)

            return lines

        except Exception as exc:
            return self.__raise_exception(exc)

    # ------- Public Methods ------- #
    def clearscans(self):
        """Clear any stored scans."""
        self.__scans = {}
        self.__clear_details()

    def show_errors(self, error=False):
        """Enable exception raising instead of returning False on error.

        @param [error=False] Passing a value of True enables exception raising.
        """
        self.__enable_exceptions = self.__check_bool(error)

    def scan(self, filename, lineno, endline=None, encoding=None):
        """Scan both single and multiple lines with option of custom encoding.

        filename (String): The desired file to scan.
        lineno (Integer): The line you wish to scan.
        endline (Optional, Integer, String): The last line to want to scan.
        Specify when scanning multiple lines. Specifying 'end' will scan
        the file from lineno to the end of the file.
        encoding (Optional, String): Specify a file encoding to use.
        Defaults to default system encoding.
        """
        # Use the system default encoding if one is not specified.
        if encoding is None:
            encoding = locale.getpreferredencoding(False)

        # Store the scan details for use elsewhere
        self.__set_details(filename, lineno, endline, encoding)

        # Create a file pointer
        filePointer = self.__create_pointer()

        # If the pointer has been used already, return the stored value
        if filePointer in self.__scans:
            return self.__scans[filePointer]

        # The pointer could not be found, proceed to scan the file
        else:
            # Perform the actual scan
            theScan = self.__scanner()

            # Store the scan only if it is valid.
            if theScan:
                self.__scans[filePointer] = theScan
            return theScan

    def rescan(self, filename=None):
        """Rescan filename to update stored scans with file changes."""
        _filenames = []
        for pointer in self.__scans.keys():
            # A file was not specified, rescan all stored scans
            if filename is None:
                _filenames = list(self.__scans.keys())
                break
            # A file was specified and the pointer has been already be stored
            else:
                if filename in pointer:
                    _filenames = [pointer]
                    break

        # The file specified has not been scanned before
        if not _filenames:
            return self.__raise_exception(
                "{0} has not been previously scanned.".format(filename),
                False
            )

        # We have file(s) to rescan
        for _key in _filenames:
            _keySplit = _key.split(",")
            # An ending line number was not specified
            if len(_keySplit) == 3:
                fileName, startLine, encode = _keySplit

                # Only one line needs to be rescanned
                endLine = None

            # An ending line number (or "end" string) was specified
            else:
                fileName, startLine, endLine, encode = _keySplit

            # Trim encoding string for use,
            # convert `endLine` to an integer under proper conditions
            encode = re.sub(r"encode=", "", encode)
            if endLine is not None and endLine != "end":
                endLine = int(endLine)

            # Now that we have the proper data, preform the rescan
            self.__set_details(fileName, int(startLine), endLine, encode)
            _newScan = self.__scanner()

            # Update the stored scan with the new scan
            self.__scans[_key] = _newScan
            return _newScan
<commit_msg>Rename parameters<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""Effortlessly read a text file using counting numbers.

Created 2013-2017 Caleb Ely
<https://CodeTri.net>

Licensed under The MIT License
<http://opensource.org/licenses/MIT/>

"""

from __future__ import unicode_literals

import re
import sys
import locale

# Grab Python 3.0+ open() function
if sys.version_info[:2] < (3, 0):
    from io import open

__all__ = ("LineScan")

class LineScan(object):
    """Main linescan class.

    Instance using linescan.LineScan()
    Primary entry point is scan() function.
    """

    def __init__(self):
        """Initialize private and public variables."""
        self.__scans = {}
        self.__enable_exceptions = False

        self.start_line = None
        self.end_line = None
        self.encoding = None
        self.file_name = ""

    # ------- Private Methods ------- #
    def __len__(self):
        """Expose the total number of files scanned.

        @returns {Integer}
        """
        return len(self.__scans)

    def __set_details(self, filename, lineno, endline, encoding):
        """Store scan details."""
        self.start_line = lineno
        self.end_line = endline
        self.encoding = encoding
        self.file_name = filename

    def __clear_details(self):
        """Reset scan details."""
        self.start_line = None
        self.end_line = None
        self.encoding = None
        self.file_name = ""

    def __check_bool(self, value):
        """Used to check if various options should be enabled."""
        return value is True

    def __create_pointer(self):
        """Construct the comma-separated pointer for the specified file."""
        filePointer = "{0},{1}".format(self.file_name, self.start_line)

        # Append the ending line if one is specified
        if self.end_line is not None:
            filePointer = "{0},{1}".format(
                filePointer, self.end_line)

        # An encoding is always specified even if
        # the user did not provide one
        filePointer = "{0},encode={1}".format(filePointer, self.encoding)
        return filePointer

    def __raise_exception(self, exc, generic=True):
        """Handle errors per exception option."""
        # Raise an exception if they are enabled
        if self.__enable_exceptions:
            # A "generic" exception should be raised
            if generic:
                raise exc

            # A rescaning error should be raised instead
            # Raise FileNotFoundError exception on Python 3.3+
            if sys.version_info[:2] >= (3, 3):
                raise FileNotFoundError(exc)  # noqa

            # Raise the old IOError on Python 3.2 and lower
            else:
                raise IOError(exc)

        # If exceptions have not been enabled, simply return False
        return False

    def __scanner(self):
        """Perform the actual scan."""
        try:
            # Since line numbers start at 0, get the starting line number.
            # No need to do the same for the ending line,
            # as it is not modified
            startLine = self.start_line - 1

            # Open the file for scanning using specified encoding
            with open(self.file_name, "rt", encoding=self.encoding) as f:

                # The user wants to scan only one line.
                if self.end_line is None:
                    lines = f.readlines()[startLine]

                # The user wants to scan until the end of the file.
                elif self.end_line == "end":
                    lines = f.readlines()[startLine:]

                # The user wants to scan multiple specified lines.
                else:
                    lines = f.readlines()[startLine:self.end_line]

            # Break the multiple lines from the returned list.
            lines = "".join(lines)

            return lines

        except Exception as exc:
            return self.__raise_exception(exc)

    # ------- Public Methods ------- #
    def clearscans(self):
        """Clear any stored scans."""
        self.__scans = {}
        self.__clear_details()

    def show_errors(self, error=False):
        """Enable exception raising instead of returning False on error.

        @param [error=False] Passing a value of True enables exception raising.
        """
        self.__enable_exceptions = self.__check_bool(error)

    def scan(self, file_name, start_line, end_line=None, encoding=None):
        """Scan both single and multiple lines with option of custom encoding.

        filename (String): The desired file to scan.
        lineno (Integer): The line you wish to scan.
        endline (Optional, Integer, String): The last line to want to scan.
        Specify when scanning multiple lines. Specifying 'end' will scan
        the file from lineno to the end of the file.
        encoding (Optional, String): Specify a file encoding to use.
        Defaults to default system encoding.
        """
        # Use the system default encoding if one is not specified.
        if encoding is None:
            encoding = locale.getpreferredencoding(False)

        # Store the scan details for use elsewhere
        self.__set_details(file_name, start_line, end_line, encoding)

        # Create a file pointer
        filePointer = self.__create_pointer()

        # If the pointer has been used already, return the stored value
        if filePointer in self.__scans:
            return self.__scans[filePointer]

        # The pointer could not be found, proceed to scan the file
        else:
            # Perform the actual scan
            theScan = self.__scanner()

            # Store the scan only if it is valid.
            if theScan:
                self.__scans[filePointer] = theScan
            return theScan

    def rescan(self, file_name=None):
        """Rescan filename to update stored scans with file changes."""
        _filenames = []
        for pointer in self.__scans.keys():
            # A file was not specified, rescan all stored scans
            if file_name is None:
                _filenames = list(self.__scans.keys())
                break
            # A file was specified and the pointer has been already be stored
            else:
                if file_name in pointer:
                    _filenames = [pointer]
                    break

        # The file specified has not been scanned before
        if not _filenames:
            return self.__raise_exception(
                "{0} has not been previously scanned.".format(file_name),
                False
            )

        # We have file(s) to rescan
        for _key in _filenames:
            _keySplit = _key.split(",")
            # An ending line number was not specified
            if len(_keySplit) == 3:
                fileName, startLine, encode = _keySplit

                # Only one line needs to be rescanned
                endLine = None

            # An ending line number (or "end" string) was specified
            else:
                fileName, startLine, endLine, encode = _keySplit

            # Trim encoding string for use,
            # convert `endLine` to an integer under proper conditions
            encode = re.sub(r"encode=", "", encode)
            if endLine is not None and endLine != "end":
                endLine = int(endLine)

            # Now that we have the proper data, preform the rescan
            self.__set_details(fileName, int(startLine), endLine, encode)
            _newScan = self.__scanner()

            # Update the stored scan with the new scan
            self.__scans[_key] = _newScan
            return _newScan
<|endoftext|>


<commit_before>"""
Test for the basis function module.
"""
import pytest
from dials.array_family import flex
from dials.algorithms.scaling.model.components.scale_components import \
  SingleBScaleFactor, SingleScaleFactor
from dials.algorithms.scaling.basis_functions import basis_function
from dials.algorithms.scaling.parameter_handler import \
  scaling_active_parameter_manager

@pytest.fixture
def small_reflection_table():
  """Generate reflection table to test the basis function."""
  reflections = flex.reflection_table()
  reflections['d'] = flex.double([2.0, 0.8, 2.0]) #don't change
  reflections['xyzobs.px.value'] = flex.vec3_double([(0.0, 0.0, 0.0),
    (0.0, 0.0, 5.0), (0.0, 0.0, 10.0)])
  return reflections

def test_basis_function(small_reflection_table):
  """Test for the basis function class. This calculates scale factors and
  derivatives for reflections based on the model components."""

  # To test the basis function, need a scaling active parameter manager - to set
  # this up we need a components dictionary with some reflection data.

  # Let's use KB model components for simplicity - and have an extra fake 'abs'
  # component.
  rt = small_reflection_table
  components = {
    'scale' : SingleScaleFactor(flex.double([1.0])),
    'decay' : SingleBScaleFactor(flex.double([0.0])),
    'abs' : SingleScaleFactor(flex.double([1.0]))} #Create empty components.
  for component in components.itervalues():
    component.update_reflection_data(rt) #Add some data to components.

  apm = scaling_active_parameter_manager(components, ['decay', 'scale'])

  # First test that scale factors can be successfully updated.
  # Manually change the parameters in the apm.
  decay = components['decay'] # Define alias
  _ = components['scale'] # Define alias
  # Note, order of params in apm.x depends on order in scaling model components.
  new_B = 1.0
  new_S = 2.0
  apm.set_param_vals(flex.double([new_S, new_B]))
  basis_fn = basis_function(curvatures=True)
  s, d, c = basis_fn.calculate_scales_and_derivatives(apm, 0)
  slist, dlist, clist = basis_fn.calc_component_scales_derivatives(apm, 0)
  # Now test that the inverse scale factor is correctly calculated.
  calculated_sfs = s
  assert list(calculated_sfs) == list(new_S * flex.exp(new_B/
    (2.0*(decay.d_values[0]**2))))

  # Now check that the derivative matrix is correctly calculated.
  calc_derivs = d
  assert calc_derivs[0, 0] == dlist[0][0, 0] * slist[1][0]
  assert calc_derivs[1, 0] == dlist[0][1, 0] * slist[1][1]
  assert calc_derivs[2, 0] == dlist[0][2, 0] * slist[1][2]
  assert calc_derivs[0, 1] == dlist[1][0, 0] * slist[0][0]
  assert calc_derivs[1, 1] == dlist[1][1, 0] * slist[0][1]
  assert calc_derivs[2, 1] == dlist[1][2, 0] * slist[0][2]

  # Test that the curvatures matrix is correctly composed.
  calc_curvs = c
  assert calc_curvs[0, 0] == clist[0][0, 0] * slist[1][0]
  assert calc_curvs[1, 0] == clist[0][1, 0] * slist[1][1]
  assert calc_curvs[2, 0] == clist[0][2, 0] * slist[1][2]
  assert calc_curvs[0, 1] == clist[1][0, 0] * slist[0][0]
  assert calc_curvs[1, 1] == clist[1][1, 0] * slist[0][1]
  assert calc_curvs[2, 1] == clist[1][2, 0] * slist[0][2]

  # Repeat the test when there is only one active parameter.
  # First reset the parameters
  components['decay'].parameters = flex.double([0.0])
  components['scale'].parameters = flex.double([1.0])
  components['abs'].parameters = flex.double([1.0])
  components['decay'].calculate_scales_and_derivatives()
  components['scale'].calculate_scales_and_derivatives()
  components['abs'].calculate_scales_and_derivatives()

  # Now generate a parameter manager for a single component.
  apm = scaling_active_parameter_manager(components, ['scale'])
  new_S = 2.0
  apm.set_param_vals(flex.double(components['scale'].n_params, new_S))
  basis_fn = basis_function(curvatures=True)
  s, d, c = basis_fn.calculate_scales_and_derivatives(apm, 0)
  slist, dlist, clist = basis_fn.calc_component_scales_derivatives(apm, 0)
  # Test that the scales and derivatives were correctly calculated
  assert list(s) == list([new_S] *
    slist[0].size())
  assert d[0, 0] == dlist[0][0, 0]
  assert d[1, 0] == dlist[0][1, 0]
  assert d[2, 0] == dlist[0][2, 0]

  # Test again for two components, with curvatures.
  components['decay'].parameters = flex.double([0.0])
  components['scale'].parameters = flex.double([1.0])
  components['abs'].parameters = flex.double([1.0])
  components['decay'].calculate_scales_and_derivatives(curvatures=True)
  components['scale'].calculate_scales_and_derivatives(curvatures=True)
  components['abs'].calculate_scales_and_derivatives(curvatures=True)

  apm = scaling_active_parameter_manager(components, ['scale', 'decay'])
  basis_fn = basis_function(curvatures=True)
  _, __, ___ = basis_fn.calculate_scales_and_derivatives(apm, 0)

  #Test for no components
  apm = scaling_active_parameter_manager(components, [])
  basis_fn = basis_function(curvatures=True)
  _, d, c = basis_fn.calculate_scales_and_derivatives(apm, 0)
  assert d is None
  assert c is None
  basis_fn = basis_function()
  _, d = basis_fn.calculate_scales_and_derivatives(apm, 0)
  assert d is None
<commit_msg>Use pytest.approx for scaling test comparisons<commit_after>"""
Test for the basis function module.
"""
import pytest
from dials.array_family import flex
from dials.algorithms.scaling.model.components.scale_components import \
  SingleBScaleFactor, SingleScaleFactor
from dials.algorithms.scaling.basis_functions import basis_function
from dials.algorithms.scaling.parameter_handler import \
  scaling_active_parameter_manager

@pytest.fixture
def small_reflection_table():
  """Generate reflection table to test the basis function."""
  reflections = flex.reflection_table()
  reflections['d'] = flex.double([2.0, 0.8, 2.0]) #don't change
  reflections['xyzobs.px.value'] = flex.vec3_double([(0.0, 0.0, 0.0),
    (0.0, 0.0, 5.0), (0.0, 0.0, 10.0)])
  return reflections

def test_basis_function(small_reflection_table):
  """Test for the basis function class. This calculates scale factors and
  derivatives for reflections based on the model components."""

  # To test the basis function, need a scaling active parameter manager - to set
  # this up we need a components dictionary with some reflection data.

  # Let's use KB model components for simplicity - and have an extra fake 'abs'
  # component.
  rt = small_reflection_table
  components = {
    'scale' : SingleScaleFactor(flex.double([1.0])),
    'decay' : SingleBScaleFactor(flex.double([0.0])),
    'abs' : SingleScaleFactor(flex.double([1.0]))} #Create empty components.
  for component in components.itervalues():
    component.update_reflection_data(rt) #Add some data to components.

  apm = scaling_active_parameter_manager(components, ['decay', 'scale'])

  # First test that scale factors can be successfully updated.
  # Manually change the parameters in the apm.
  decay = components['decay'] # Define alias
  _ = components['scale'] # Define alias
  # Note, order of params in apm.x depends on order in scaling model components.
  new_B = 1.0
  new_S = 2.0
  apm.set_param_vals(flex.double([new_S, new_B]))
  basis_fn = basis_function(curvatures=True)
  s, d, c = basis_fn.calculate_scales_and_derivatives(apm, 0)
  slist, dlist, clist = basis_fn.calc_component_scales_derivatives(apm, 0)
  # Now test that the inverse scale factor is correctly calculated.
  calculated_sfs = s
  assert list(calculated_sfs) == pytest.approx(list(new_S * flex.exp(new_B/
    (2.0*(decay.d_values[0]**2)))))

  # Now check that the derivative matrix is correctly calculated.
  calc_derivs = d
  assert calc_derivs[0, 0] == dlist[0][0, 0] * slist[1][0]
  assert calc_derivs[1, 0] == dlist[0][1, 0] * slist[1][1]
  assert calc_derivs[2, 0] == dlist[0][2, 0] * slist[1][2]
  assert calc_derivs[0, 1] == dlist[1][0, 0] * slist[0][0]
  assert calc_derivs[1, 1] == dlist[1][1, 0] * slist[0][1]
  assert calc_derivs[2, 1] == dlist[1][2, 0] * slist[0][2]

  # Test that the curvatures matrix is correctly composed.
  calc_curvs = c
  assert calc_curvs[0, 0] == clist[0][0, 0] * slist[1][0]
  assert calc_curvs[1, 0] == clist[0][1, 0] * slist[1][1]
  assert calc_curvs[2, 0] == clist[0][2, 0] * slist[1][2]
  assert calc_curvs[0, 1] == clist[1][0, 0] * slist[0][0]
  assert calc_curvs[1, 1] == clist[1][1, 0] * slist[0][1]
  assert calc_curvs[2, 1] == clist[1][2, 0] * slist[0][2]

  # Repeat the test when there is only one active parameter.
  # First reset the parameters
  components['decay'].parameters = flex.double([0.0])
  components['scale'].parameters = flex.double([1.0])
  components['abs'].parameters = flex.double([1.0])
  components['decay'].calculate_scales_and_derivatives()
  components['scale'].calculate_scales_and_derivatives()
  components['abs'].calculate_scales_and_derivatives()

  # Now generate a parameter manager for a single component.
  apm = scaling_active_parameter_manager(components, ['scale'])
  new_S = 2.0
  apm.set_param_vals(flex.double(components['scale'].n_params, new_S))
  basis_fn = basis_function(curvatures=True)
  s, d, c = basis_fn.calculate_scales_and_derivatives(apm, 0)
  slist, dlist, clist = basis_fn.calc_component_scales_derivatives(apm, 0)
  # Test that the scales and derivatives were correctly calculated
  assert list(s) == list([new_S] *
    slist[0].size())
  assert d[0, 0] == dlist[0][0, 0]
  assert d[1, 0] == dlist[0][1, 0]
  assert d[2, 0] == dlist[0][2, 0]

  # Test again for two components, with curvatures.
  components['decay'].parameters = flex.double([0.0])
  components['scale'].parameters = flex.double([1.0])
  components['abs'].parameters = flex.double([1.0])
  components['decay'].calculate_scales_and_derivatives(curvatures=True)
  components['scale'].calculate_scales_and_derivatives(curvatures=True)
  components['abs'].calculate_scales_and_derivatives(curvatures=True)

  apm = scaling_active_parameter_manager(components, ['scale', 'decay'])
  basis_fn = basis_function(curvatures=True)
  _, __, ___ = basis_fn.calculate_scales_and_derivatives(apm, 0)

  #Test for no components
  apm = scaling_active_parameter_manager(components, [])
  basis_fn = basis_function(curvatures=True)
  _, d, c = basis_fn.calculate_scales_and_derivatives(apm, 0)
  assert d is None
  assert c is None
  basis_fn = basis_function()
  _, d = basis_fn.calculate_scales_and_derivatives(apm, 0)
  assert d is None
<|endoftext|>


<commit_before># Tai Sakuma <tai.sakuma@gmail.com>

##__________________________________________________________________||
import collections

##__________________________________________________________________||
EventBuilderConfig = collections.namedtuple(
    'EventBuilderConfig',
    'inputPaths treeName maxEvents start name'
)
# 'name' is used by loop.EventLoopProgressReportWriter

##__________________________________________________________________||
<commit_msg>add comment<commit_after># Tai Sakuma <tai.sakuma@gmail.com>

##__________________________________________________________________||
import collections

##__________________________________________________________________||
# EventBuilderConfig is deprecated as EventBuilder is being replaced
# with BuildEvents, which uses dict instead of EventBuilderConfig

EventBuilderConfig = collections.namedtuple(
    'EventBuilderConfig',
    'inputPaths treeName maxEvents start name'
)
# 'name' is used by loop.EventLoopProgressReportWriter

##__________________________________________________________________||
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from django.utils.six import moves

import sys
import warnings
import tablib

try:
    from tablib.compat import xlrd

    XLS_IMPORT = True
except ImportError:
    try:
        import xlrd  # NOQA

        XLS_IMPORT = True
    except ImportError:
        xls_warning = "Installed `tablib` library does not include"
        "import support for 'xls' format and xlrd module is not found."
        warnings.warn(xls_warning, ImportWarning)
        XLS_IMPORT = False

try:
    import openpyxl
    XLSX_IMPORT = True
except ImportError:
    try:
        from tablib.compat import openpyxl
        XLSX_IMPORT = hasattr(openpyxl, 'load_workbook')
    except ImportError:
        xlsx_warning = "Installed `tablib` library does not include"
        "import support for 'xlsx' format and openpyxl module is not found."
        warnings.warn(xlsx_warning, ImportWarning)
        XLSX_IMPORT = False

try:
    from importlib import import_module
except ImportError:
    from django.utils.importlib import import_module

class Format(object):
    def get_title(self):
        return type(self)

    def create_dataset(self, in_stream):
        """
        Create dataset from given string.
        """
        raise NotImplementedError()

    def export_data(self, dataset, **kwargs):
        """
        Returns format representation for given dataset.
        """
        raise NotImplementedError()

    def is_binary(self):
        """
        Returns if this format is binary.
        """
        return True

    def get_read_mode(self):
        """
        Returns mode for opening files.
        """
        return 'rb'

    def get_extension(self):
        """
        Returns extension for this format files.
        """
        return ""

    def get_content_type(self):
        # For content types see
        # http://www.iana.org/assignments/media-types/media-types.xhtml
        return 'application/octet-stream'

    def can_import(self):
        return False

    def can_export(self):
        return False

class TablibFormat(Format):
    TABLIB_MODULE = None
    CONTENT_TYPE = 'application/octet-stream'

    def get_format(self):
        """
        Import and returns tablib module.
        """
        return import_module(self.TABLIB_MODULE)

    def get_title(self):
        return self.get_format().title

    def create_dataset(self, in_stream, **kwargs):
        data = tablib.Dataset()
        self.get_format().import_set(data, in_stream, **kwargs)
        return data

    def export_data(self, dataset, **kwargs):
        return self.get_format().export_set(dataset, **kwargs)

    def get_extension(self):
        # we support both 'extentions' and 'extensions' because currently
        # tablib's master branch uses 'extentions' (which is a typo) but it's
        # dev branch already uses 'extension'.
        # TODO - remove this once the typo is fixxed in tablib's master branch
        if hasattr(self.get_format(), 'extentions'):
            return self.get_format().extentions[0]
        return self.get_format().extensions[0]

    def get_content_type(self):
        return self.CONTENT_TYPE

    def can_import(self):
        return hasattr(self.get_format(), 'import_set')

    def can_export(self):
        return hasattr(self.get_format(), 'export_set')

class TextFormat(TablibFormat):
    def get_read_mode(self):
        if sys.version_info[0] < 3:  # backwards compatibility for python 2.7
            return 'rU'
        else:
            return 'r'

    def is_binary(self):
        return False

class CSV(TextFormat):
    TABLIB_MODULE = 'tablib.formats._csv'
    CONTENT_TYPE = 'text/csv'

    def create_dataset(self, in_stream, **kwargs):
        if sys.version_info[0] < 3:
            # python 2.7 csv does not do unicode
            return super(CSV, self).create_dataset(in_stream.encode('utf-8'), **kwargs)
        return super(CSV, self).create_dataset(in_stream, **kwargs)

class JSON(TextFormat):
    TABLIB_MODULE = 'tablib.formats._json'
    CONTENT_TYPE = 'application/json'

class YAML(TextFormat):
    TABLIB_MODULE = 'tablib.formats._yaml'
    # See http://stackoverflow.com/questions/332129/yaml-mime-type
    CONTENT_TYPE = 'text/yaml'

class TSV(TextFormat):
    TABLIB_MODULE = 'tablib.formats._tsv'
    CONTENT_TYPE = 'text/tab-separated-values'

    def create_dataset(self, in_stream, **kwargs):
        if sys.version_info[0] < 3:
            # python 2.7 csv does not do unicode
            return super(TSV, self).create_dataset(in_stream.encode('utf-8'), **kwargs)
        return super(TSV, self).create_dataset(in_stream, **kwargs)

class ODS(TextFormat):
    TABLIB_MODULE = 'tablib.formats._ods'
    CONTENT_TYPE = 'application/vnd.oasis.opendocument.spreadsheet'

class HTML(TextFormat):
    TABLIB_MODULE = 'tablib.formats._html'
    CONTENT_TYPE = 'text/html'

class XLS(TablibFormat):
    TABLIB_MODULE = 'tablib.formats._xls'
    CONTENT_TYPE = 'application/vnd.ms-excel'

    def can_import(self):
        return XLS_IMPORT

    def create_dataset(self, in_stream):
        """
        Create dataset from first sheet.
        """
        assert XLS_IMPORT
        xls_book = xlrd.open_workbook(file_contents=in_stream)
        dataset = tablib.Dataset()
        sheet = xls_book.sheets()[0]

        dataset.headers = sheet.row_values(0)
        for i in moves.range(1, sheet.nrows):
            dataset.append(sheet.row_values(i))
        return dataset

class XLSX(TablibFormat):
    TABLIB_MODULE = 'tablib.formats._xlsx'
    CONTENT_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'

    def can_import(self):
        return XLSX_IMPORT

    def create_dataset(self, in_stream):
        """
        Create dataset from first sheet.
        """
        assert XLSX_IMPORT
        from io import BytesIO
        xlsx_book = openpyxl.load_workbook(BytesIO(in_stream), read_only=True)

        dataset = tablib.Dataset()
        sheet = xlsx_book.active

        # obtain generator
        rows = sheet.rows
        dataset.headers = [cell.value for cell in next(rows)]

        for row in rows:
            row_values = [cell.value for cell in row]
            dataset.append(row_values)
        return dataset

#: These are the default formats for import and export. Whether they can be
#: used or not is depending on their implementation in the tablib library.
DEFAULT_FORMATS = (
    CSV,
    XLS,
    XLSX,
    TSV,
    ODS,
    JSON,
    YAML,
    HTML,
)
<commit_msg>Remove workaround for older Pythons<commit_after>from __future__ import unicode_literals
from django.utils.six import moves

import sys
import warnings
import tablib
from importlib import import_module

try:
    from tablib.compat import xlrd

    XLS_IMPORT = True
except ImportError:
    try:
        import xlrd  # NOQA

        XLS_IMPORT = True
    except ImportError:
        xls_warning = "Installed `tablib` library does not include"
        "import support for 'xls' format and xlrd module is not found."
        warnings.warn(xls_warning, ImportWarning)
        XLS_IMPORT = False

try:
    import openpyxl
    XLSX_IMPORT = True
except ImportError:
    try:
        from tablib.compat import openpyxl
        XLSX_IMPORT = hasattr(openpyxl, 'load_workbook')
    except ImportError:
        xlsx_warning = "Installed `tablib` library does not include"
        "import support for 'xlsx' format and openpyxl module is not found."
        warnings.warn(xlsx_warning, ImportWarning)
        XLSX_IMPORT = False

class Format(object):
    def get_title(self):
        return type(self)

    def create_dataset(self, in_stream):
        """
        Create dataset from given string.
        """
        raise NotImplementedError()

    def export_data(self, dataset, **kwargs):
        """
        Returns format representation for given dataset.
        """
        raise NotImplementedError()

    def is_binary(self):
        """
        Returns if this format is binary.
        """
        return True

    def get_read_mode(self):
        """
        Returns mode for opening files.
        """
        return 'rb'

    def get_extension(self):
        """
        Returns extension for this format files.
        """
        return ""

    def get_content_type(self):
        # For content types see
        # http://www.iana.org/assignments/media-types/media-types.xhtml
        return 'application/octet-stream'

    def can_import(self):
        return False

    def can_export(self):
        return False

class TablibFormat(Format):
    TABLIB_MODULE = None
    CONTENT_TYPE = 'application/octet-stream'

    def get_format(self):
        """
        Import and returns tablib module.
        """
        return import_module(self.TABLIB_MODULE)

    def get_title(self):
        return self.get_format().title

    def create_dataset(self, in_stream, **kwargs):
        data = tablib.Dataset()
        self.get_format().import_set(data, in_stream, **kwargs)
        return data

    def export_data(self, dataset, **kwargs):
        return self.get_format().export_set(dataset, **kwargs)

    def get_extension(self):
        # we support both 'extentions' and 'extensions' because currently
        # tablib's master branch uses 'extentions' (which is a typo) but it's
        # dev branch already uses 'extension'.
        # TODO - remove this once the typo is fixxed in tablib's master branch
        if hasattr(self.get_format(), 'extentions'):
            return self.get_format().extentions[0]
        return self.get_format().extensions[0]

    def get_content_type(self):
        return self.CONTENT_TYPE

    def can_import(self):
        return hasattr(self.get_format(), 'import_set')

    def can_export(self):
        return hasattr(self.get_format(), 'export_set')

class TextFormat(TablibFormat):
    def get_read_mode(self):
        if sys.version_info[0] < 3:  # backwards compatibility for python 2.7
            return 'rU'
        else:
            return 'r'

    def is_binary(self):
        return False

class CSV(TextFormat):
    TABLIB_MODULE = 'tablib.formats._csv'
    CONTENT_TYPE = 'text/csv'

    def create_dataset(self, in_stream, **kwargs):
        if sys.version_info[0] < 3:
            # python 2.7 csv does not do unicode
            return super(CSV, self).create_dataset(in_stream.encode('utf-8'), **kwargs)
        return super(CSV, self).create_dataset(in_stream, **kwargs)

class JSON(TextFormat):
    TABLIB_MODULE = 'tablib.formats._json'
    CONTENT_TYPE = 'application/json'

class YAML(TextFormat):
    TABLIB_MODULE = 'tablib.formats._yaml'
    # See http://stackoverflow.com/questions/332129/yaml-mime-type
    CONTENT_TYPE = 'text/yaml'

class TSV(TextFormat):
    TABLIB_MODULE = 'tablib.formats._tsv'
    CONTENT_TYPE = 'text/tab-separated-values'

    def create_dataset(self, in_stream, **kwargs):
        if sys.version_info[0] < 3:
            # python 2.7 csv does not do unicode
            return super(TSV, self).create_dataset(in_stream.encode('utf-8'), **kwargs)
        return super(TSV, self).create_dataset(in_stream, **kwargs)

class ODS(TextFormat):
    TABLIB_MODULE = 'tablib.formats._ods'
    CONTENT_TYPE = 'application/vnd.oasis.opendocument.spreadsheet'

class HTML(TextFormat):
    TABLIB_MODULE = 'tablib.formats._html'
    CONTENT_TYPE = 'text/html'

class XLS(TablibFormat):
    TABLIB_MODULE = 'tablib.formats._xls'
    CONTENT_TYPE = 'application/vnd.ms-excel'

    def can_import(self):
        return XLS_IMPORT

    def create_dataset(self, in_stream):
        """
        Create dataset from first sheet.
        """
        assert XLS_IMPORT
        xls_book = xlrd.open_workbook(file_contents=in_stream)
        dataset = tablib.Dataset()
        sheet = xls_book.sheets()[0]

        dataset.headers = sheet.row_values(0)
        for i in moves.range(1, sheet.nrows):
            dataset.append(sheet.row_values(i))
        return dataset

class XLSX(TablibFormat):
    TABLIB_MODULE = 'tablib.formats._xlsx'
    CONTENT_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'

    def can_import(self):
        return XLSX_IMPORT

    def create_dataset(self, in_stream):
        """
        Create dataset from first sheet.
        """
        assert XLSX_IMPORT
        from io import BytesIO
        xlsx_book = openpyxl.load_workbook(BytesIO(in_stream), read_only=True)

        dataset = tablib.Dataset()
        sheet = xlsx_book.active

        # obtain generator
        rows = sheet.rows
        dataset.headers = [cell.value for cell in next(rows)]

        for row in rows:
            row_values = [cell.value for cell in row]
            dataset.append(row_values)
        return dataset

#: These are the default formats for import and export. Whether they can be
#: used or not is depending on their implementation in the tablib library.
DEFAULT_FORMATS = (
    CSV,
    XLS,
    XLSX,
    TSV,
    ODS,
    JSON,
    YAML,
    HTML,
)
<|endoftext|>