<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Base classes for our unit tests.
Allows overriding of flags for use of fakes,
and some black magic for inline callbacks.
"""

import sys
import time

import mox
import stubout
from tornado import ioloop
from twisted.internet import defer
from twisted.trial import unittest

from nova import fakerabbit
from nova import flags

FLAGS = flags.FLAGS
flags.DEFINE_bool('fake_tests', True,
                  'should we use everything for testing')

def skip_if_fake(func):
    """Decorator that skips a test if running in fake mode"""
    def _skipper(*args, **kw):
        """Wrapped skipper function"""
        if FLAGS.fake_tests:
            raise unittest.SkipTest('Test cannot be run in fake mode')
        else:
            return func(*args, **kw)
    return _skipper

class TrialTestCase(unittest.TestCase):
    """Test case base class for all unit tests"""
    def setUp(self): # pylint: disable=C0103
        """Run before each test method to initialize test environment"""
        super(TrialTestCase, self).setUp()

        # emulate some of the mox stuff, we can't use the metaclass
        # because it screws with our generators
        self.mox = mox.Mox()
        self.stubs = stubout.StubOutForTesting()
        self.flag_overrides = {}

    def tearDown(self): # pylint: disable=C0103
        """Runs after each test method to finalize/tear down test environment"""
        super(TrialTestCase, self).tearDown()
        self.reset_flags()
        self.mox.UnsetStubs()
        self.stubs.UnsetAll()
        self.stubs.SmartUnsetAll()
        self.mox.VerifyAll()

        if FLAGS.fake_rabbit:
            fakerabbit.reset_all()

    def flags(self, **kw):
        """Override flag variables for a test"""
        for k, v in kw.iteritems():
            if k in self.flag_overrides:
                self.reset_flags()
                raise Exception(
                        'trying to override already overriden flag: %s' % k)
            self.flag_overrides[k] = getattr(FLAGS, k)
            setattr(FLAGS, k, v)

    def reset_flags(self):
        """Resets all flag variables for the test.  Runs after each test"""
        for k, v in self.flag_overrides.iteritems():
            setattr(FLAGS, k, v)

class BaseTestCase(TrialTestCase):
    # TODO(jaypipes): Can this be moved into the TrialTestCase class?
    """Base test case class for all unit tests."""
    def setUp(self): # pylint: disable=C0103
        """Run before each test method to initialize test environment"""
        super(BaseTestCase, self).setUp()
        # TODO(termie): we could possibly keep a more global registry of
        #               the injected listeners... this is fine for now though
        self.injected = []
        self.ioloop = ioloop.IOLoop.instance()

        self._waiting = None
        self._done_waiting = False
        self._timed_out = False

    def tearDown(self):# pylint: disable=C0103
        """Runs after each test method to finalize/tear down test environment"""
        super(BaseTestCase, self).tearDown()
        for x in self.injected:
            x.stop()
        if FLAGS.fake_rabbit:
            fakerabbit.reset_all()

    def _wait_for_test(self, timeout=60):
        """ Push the ioloop along to wait for our test to complete. """
        self._waiting = self.ioloop.add_timeout(time.time() + timeout,
                                                self._timeout)
        def _wait():
            """Wrapped wait function. Called on timeout."""
            if self._timed_out:
                self.fail('test timed out')
                self._done()
            if self._done_waiting:
                self.ioloop.stop()
                return
            # we can use add_callback here but this uses less cpu when testing
            self.ioloop.add_timeout(time.time() + 0.01, _wait)

        self.ioloop.add_callback(_wait)
        self.ioloop.start()

    def _done(self):
        """Callback used for cleaning up deferred test methods."""
        if self._waiting:
            try:
                self.ioloop.remove_timeout(self._waiting)
            except Exception: # pylint: disable=W0703
                # TODO(jaypipes): This produces a pylint warning.  Should
                # we really be catching Exception and then passing here?
                pass
            self._waiting = None
        self._done_waiting = True

    def _maybe_inline_callbacks(self, func):
        """ If we're doing async calls in our tests, wait on them.

        This is probably the most complicated hunk of code we have so far.

        First up, if the function is normal (not async) we just act normal
        and return.

        Async tests will use the "Inline Callbacks" pattern, which means
        you yield Deferreds at every "waiting" step of your code instead
        of making epic callback chains.

        Example (callback chain, ugly):

        d = self.compute.terminate_instance(instance_id) # a Deferred instance
        def _describe(_):
            d_desc = self.compute.describe_instances() # another Deferred instance
            return d_desc
        def _checkDescribe(rv):
            self.assertEqual(rv, [])
        d.addCallback(_describe)
        d.addCallback(_checkDescribe)
        d.addCallback(lambda x: self._done())
        self._wait_for_test()

        Example (inline callbacks! yay!):

        yield self.compute.terminate_instance(instance_id)
        rv = yield self.compute.describe_instances()
        self.assertEqual(rv, [])

        If the test fits the Inline Callbacks pattern we will automatically
        handle calling wait and done.
        """
        # TODO(termie): this can be a wrapper function instead and
        #               and we can make a metaclass so that we don't
        #               have to copy all that "run" code below.
        g = func()
        if not hasattr(g, 'send'):
            self._done()
            return defer.succeed(g)

        inlined = defer.inlineCallbacks(func)
        d = inlined()
        return d

    def _catch_exceptions(self, result, failure):
        """Catches all exceptions and handles keyboard interrupts."""
        exc = (failure.type, failure.value, failure.getTracebackObject())
        if isinstance(failure.value, self.failureException):
            result.addFailure(self, exc)
        elif isinstance(failure.value, KeyboardInterrupt):
            raise
        else:
            result.addError(self, exc)

        self._done()

    def _timeout(self):
        """Helper method which trips the timeouts"""
        self._waiting = False
        self._timed_out = True

    def run(self, result=None):
        """Runs the test case"""

        result.startTest(self)
        test_method = getattr(self, self._testMethodName)
        try:
            try:
                self.setUp()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info())
                return

            ok = False
            try:
                d = self._maybe_inline_callbacks(test_method)
                d.addErrback(lambda x: self._catch_exceptions(result, x))
                d.addBoth(lambda x: self._done() and x)
                self._wait_for_test()
                ok = True
            except self.failureException:
                result.addFailure(self, sys.exc_info())
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info())

            try:
                self.tearDown()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info())
                ok = False
            if ok:
                result.addSuccess(self)
        finally:
            result.stopTest(self)
<commit_msg>Changed our minds: keep pylint equal to Ubuntu Lucid version, and use disable-msg throughout.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Base classes for our unit tests.
Allows overriding of flags for use of fakes,
and some black magic for inline callbacks.
"""

import sys
import time

import mox
import stubout
from tornado import ioloop
from twisted.internet import defer
from twisted.trial import unittest

from nova import fakerabbit
from nova import flags

FLAGS = flags.FLAGS
flags.DEFINE_bool('fake_tests', True,
                  'should we use everything for testing')

def skip_if_fake(func):
    """Decorator that skips a test if running in fake mode"""
    def _skipper(*args, **kw):
        """Wrapped skipper function"""
        if FLAGS.fake_tests:
            raise unittest.SkipTest('Test cannot be run in fake mode')
        else:
            return func(*args, **kw)
    return _skipper

class TrialTestCase(unittest.TestCase):
    """Test case base class for all unit tests"""
    def setUp(self): # pylint: disable-msg=C0103
        """Run before each test method to initialize test environment"""
        super(TrialTestCase, self).setUp()

        # emulate some of the mox stuff, we can't use the metaclass
        # because it screws with our generators
        self.mox = mox.Mox()
        self.stubs = stubout.StubOutForTesting()
        self.flag_overrides = {}

    def tearDown(self): # pylint: disable-msg=C0103
        """Runs after each test method to finalize/tear down test environment"""
        super(TrialTestCase, self).tearDown()
        self.reset_flags()
        self.mox.UnsetStubs()
        self.stubs.UnsetAll()
        self.stubs.SmartUnsetAll()
        self.mox.VerifyAll()

        if FLAGS.fake_rabbit:
            fakerabbit.reset_all()

    def flags(self, **kw):
        """Override flag variables for a test"""
        for k, v in kw.iteritems():
            if k in self.flag_overrides:
                self.reset_flags()
                raise Exception(
                        'trying to override already overriden flag: %s' % k)
            self.flag_overrides[k] = getattr(FLAGS, k)
            setattr(FLAGS, k, v)

    def reset_flags(self):
        """Resets all flag variables for the test.  Runs after each test"""
        for k, v in self.flag_overrides.iteritems():
            setattr(FLAGS, k, v)

class BaseTestCase(TrialTestCase):
    # TODO(jaypipes): Can this be moved into the TrialTestCase class?
    """Base test case class for all unit tests."""
    def setUp(self): # pylint: disable-msg=C0103
        """Run before each test method to initialize test environment"""
        super(BaseTestCase, self).setUp()
        # TODO(termie): we could possibly keep a more global registry of
        #               the injected listeners... this is fine for now though
        self.injected = []
        self.ioloop = ioloop.IOLoop.instance()

        self._waiting = None
        self._done_waiting = False
        self._timed_out = False

    def tearDown(self):# pylint: disable-msg=C0103
        """Runs after each test method to finalize/tear down test environment"""
        super(BaseTestCase, self).tearDown()
        for x in self.injected:
            x.stop()
        if FLAGS.fake_rabbit:
            fakerabbit.reset_all()

    def _wait_for_test(self, timeout=60):
        """ Push the ioloop along to wait for our test to complete. """
        self._waiting = self.ioloop.add_timeout(time.time() + timeout,
                                                self._timeout)
        def _wait():
            """Wrapped wait function. Called on timeout."""
            if self._timed_out:
                self.fail('test timed out')
                self._done()
            if self._done_waiting:
                self.ioloop.stop()
                return
            # we can use add_callback here but this uses less cpu when testing
            self.ioloop.add_timeout(time.time() + 0.01, _wait)

        self.ioloop.add_callback(_wait)
        self.ioloop.start()

    def _done(self):
        """Callback used for cleaning up deferred test methods."""
        if self._waiting:
            try:
                self.ioloop.remove_timeout(self._waiting)
            except Exception: # pylint: disable-msg=W0703
                # TODO(jaypipes): This produces a pylint warning.  Should
                # we really be catching Exception and then passing here?
                pass
            self._waiting = None
        self._done_waiting = True

    def _maybe_inline_callbacks(self, func):
        """ If we're doing async calls in our tests, wait on them.

        This is probably the most complicated hunk of code we have so far.

        First up, if the function is normal (not async) we just act normal
        and return.

        Async tests will use the "Inline Callbacks" pattern, which means
        you yield Deferreds at every "waiting" step of your code instead
        of making epic callback chains.

        Example (callback chain, ugly):

        d = self.compute.terminate_instance(instance_id) # a Deferred instance
        def _describe(_):
            d_desc = self.compute.describe_instances() # another Deferred instance
            return d_desc
        def _checkDescribe(rv):
            self.assertEqual(rv, [])
        d.addCallback(_describe)
        d.addCallback(_checkDescribe)
        d.addCallback(lambda x: self._done())
        self._wait_for_test()

        Example (inline callbacks! yay!):

        yield self.compute.terminate_instance(instance_id)
        rv = yield self.compute.describe_instances()
        self.assertEqual(rv, [])

        If the test fits the Inline Callbacks pattern we will automatically
        handle calling wait and done.
        """
        # TODO(termie): this can be a wrapper function instead and
        #               and we can make a metaclass so that we don't
        #               have to copy all that "run" code below.
        g = func()
        if not hasattr(g, 'send'):
            self._done()
            return defer.succeed(g)

        inlined = defer.inlineCallbacks(func)
        d = inlined()
        return d

    def _catch_exceptions(self, result, failure):
        """Catches all exceptions and handles keyboard interrupts."""
        exc = (failure.type, failure.value, failure.getTracebackObject())
        if isinstance(failure.value, self.failureException):
            result.addFailure(self, exc)
        elif isinstance(failure.value, KeyboardInterrupt):
            raise
        else:
            result.addError(self, exc)

        self._done()

    def _timeout(self):
        """Helper method which trips the timeouts"""
        self._waiting = False
        self._timed_out = True

    def run(self, result=None):
        """Runs the test case"""

        result.startTest(self)
        test_method = getattr(self, self._testMethodName)
        try:
            try:
                self.setUp()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info())
                return

            ok = False
            try:
                d = self._maybe_inline_callbacks(test_method)
                d.addErrback(lambda x: self._catch_exceptions(result, x))
                d.addBoth(lambda x: self._done() and x)
                self._wait_for_test()
                ok = True
            except self.failureException:
                result.addFailure(self, sys.exc_info())
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info())

            try:
                self.tearDown()
            except KeyboardInterrupt:
                raise
            except:
                result.addError(self, sys.exc_info())
                ok = False
            if ok:
                result.addSuccess(self)
        finally:
            result.stopTest(self)
<|endoftext|>


<commit_before>#! /usr/bin/env python3

# Suggest name to z3 binary based on it its sha

import sys
import words
import subprocess
import argparse
import os.path
import shutil

class Z3Namer(object):

    def __init__(self):
        self._name = 'name_z3'
        self._help = 'Name for z3 binary'

    def mk_arg_parser(self, ap):
        ap.add_argument('z3bin',
                        metavar='Z3_BIN',
                        help='Full path to z3 binary')
        ap.add_argument('-o',
                        type=str,
                        metavar="DIR",
                        help='Copies renamed binary to given directory',
                        default=None)
        return ap

    def run(self, args=None):
        copy_mode = args.o is not None
        out_dir = args.o

        if copy_mode and not os.path.isdir(out_dir):
            print(f"Error: '{out_dir}' is not a directory", file=sys.stderr)
            return 1

        z3bin = args.z3bin
        z3ver_str = subprocess.check_output(f'{z3bin} --version',
                                            shell=True,
                                            encoding='utf-8')
        z3ver = z3ver_str.split()
        git_sha = z3ver[9]
        short_git_sha = git_sha[0:7]
        noun = words.get_a_noun(length=5, bound='atmost', seed=git_sha)
        z3bin_base = os.path.basename(z3bin).replace('-', '_')

        z3_name = f'{z3bin_base}-{noun}-{short_git_sha}'
        if copy_mode:
            shutil.copy2(z3bin, os.path.join(out_dir, z3_name))
        else:
            print(z3_name)
        return 0

    def main(self, argv):
        ap = argparse.ArgumentParser(prog=self._name, description=self._help)
        ap = self.mk_arg_parser(ap)
        args = ap.parse_args(argv)
        return self.run(args)

def main():
    cmd = Z3Namer()
    return cmd.main(sys.argv[1:])

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Fix for z3 without build hashcode<commit_after>#! /usr/bin/env python3

# Suggest name to z3 binary based on it its sha

import sys
import words
import subprocess
import argparse
import os.path
import shutil

class Z3Namer(object):

    def __init__(self):
        self._name = 'name_z3'
        self._help = 'Name for z3 binary'

    def mk_arg_parser(self, ap):
        ap.add_argument('z3bin',
                        metavar='Z3_BIN',
                        help='Full path to z3 binary')
        ap.add_argument('-o',
                        type=str,
                        metavar="DIR",
                        help='Copies renamed binary to given directory',
                        default=None)
        return ap

    def run(self, args=None):
        copy_mode = args.o is not None
        out_dir = args.o

        if copy_mode and not os.path.isdir(out_dir):
            print(f"Error: '{out_dir}' is not a directory", file=sys.stderr)
            return 1

        z3bin = args.z3bin
        z3ver_str = subprocess.check_output(f'{z3bin} --version',
                                            shell=True,
                                            encoding='utf-8')
        z3ver = z3ver_str.split()

        if len(z3ver) == 10:
            # z3 version with build hashcode
            git_sha = z3ver[9]
            short_git_sha = git_sha[0:7]
        else:
            # z3 release without build hashcode
            # use z3_version instead of the hashcode
            git_sha = z3ver[2].replace('.', '_')
            short_git_sha = git_sha

        noun = words.get_a_noun(length=5, bound='atmost', seed=git_sha)
        z3bin_base = os.path.basename(z3bin).replace('-', '_')

        z3_name = f'{z3bin_base}-{noun}-{short_git_sha}'
        if copy_mode:
            shutil.copy2(z3bin, os.path.join(out_dir, z3_name))
        else:
            print(z3_name)
        return 0

    def main(self, argv):
        ap = argparse.ArgumentParser(prog=self._name, description=self._help)
        ap = self.mk_arg_parser(ap)
        args = ap.parse_args(argv)
        return self.run(args)

def main():
    cmd = Z3Namer()
    return cmd.main(sys.argv[1:])

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import os
import json
from itertools import chain, groupby
from collections import defaultdict
from operator import itemgetter

from jinja2 import Environment, FileSystemLoader, environmentfilter
from jinja2.nodes import Getattr

@environmentfilter
def merge_imports(env, objects):
    """Jinja filter to merge all imports from a list of objects"""
    imports = chain(*(env.getattr(obj, 'imports') for obj in objects))
    modules = defaultdict(set)
    for imp in imports:
        modules[env.getattr(imp, 'module')] |= set(env.getattr(imp, 'names'))
    return ["from {0} import {1}".format(module, ', '.join(sorted(names)))
            for module, names in sorted(modules.items())]

def getpath(*args):
    """Get absolute path of joined directories relative to this file"""
    return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))

# Construct the jinja template environment
JINJA_ENV = Environment(loader=FileSystemLoader(getpath('templates')))
JINJA_ENV.filters['repr'] = repr
JINJA_ENV.filters['merge_imports'] = merge_imports

TYPE_MAP = {'oneOf': 'Union',
            "array": "List",
            "boolean": "Bool",
            "number": "CFloat",
            "string": "Unicode",
            "object": "Any",
            }

class SchemaProperty(object):
    """Class Wrapper for a property in a VegaLite Schema

    This class exposes methods used in the templating.
    """
    def __init__(self, schema, name, top):
        self.name = name
        self.schema = schema
        self.top = top

        self.properties = {k: SchemaProperty(v, k, self.top)
                           for k, v in self.schema.get('properties',
                                                       {}).items()}

    def rename(self, newname):
        return self.__class__(self.schema, newname, self.top)

    @property
    def subtypes(self):
        trait = self.trait_name
        if trait == 'Union':
            return [self.__class__(s, '', self.top)
                    for s in self.schema['oneOf']]
        elif trait == 'List':
            return [self.__class__(self.schema['items'], '', self.top)]
        else:
            return []

    @property
    def type(self):
        if '$ref' in self.schema:
            return '$ref'
        elif 'oneOf' in self.schema:
            return 'oneOf'
        else:
            return self.schema.get('type', '')

    @property
    def trait_name(self):
        if self.refname:
            trait = self.top.definitions[self.refname].trait_name
            if trait == 'Any':
                return 'Instance'
            else:
                return self.refname
        else:
            return TYPE_MAP.get(self.type, "Any")

    @property
    def trait_descr(self):
        trait = self.trait_name
        if trait == 'Instance':
            return self.refname
        elif self.subtypes:
            return '{0}({1})'.format(trait, ', '.join(t.trait_descr
                                                      for t in self.subtypes))
        else:
            return trait

    @property
    def trait_fulldef(self):
        trait = self.trait_name
        kwds = "allow_none=True, default_value=None"
        if 'minimum' in self.schema:
            kwds += ', min={0}'.format(self.schema['minimum'])
        if 'maximum' in self.schema:
            kwds += ', max={0}'.format(self.schema['maximum'])
        if self.description:
            kwds += ', help="""{0}"""'.format(self.short_description)

        if trait == 'Union':
            return ('T.Union([{0}])'
                    ''.format(', '.join(t.trait_fulldef
                                        for t in self.subtypes)))
        elif trait == 'List':
            return 'T.List({0}, {1})'.format(*(t.trait_fulldef
                                               for t in self.subtypes), kwds)
        elif trait == 'Instance':
            return 'T.Instance({0}, {1})'.format(self.refname, kwds)
        elif trait == self.refname:
            return '{0}({1})'.format(self.refname, kwds)
        else:
            return 'T.{0}({1})'.format(trait, kwds)

    @property
    def trait_or_subtrait(self):
        if self.refname is not None:
            return self.refname
        elif self.trait_name in ['Union', 'List']:
            for t in self.subtypes:
                if t.trait_or_subtrait is not None:
                    return t.trait_or_subtrait
            else:
                return None
        else:
            return None

    @property
    def description(self):
        return self.schema.get('description', '')

    @property
    def short_description(self):
        if self.description:
            return self.description.split('(e.g.')[0].split('.')[0] + '.'
        else:
            return ''

    @property
    def long_description(self):
        return self.description

    @property
    def refname(self):
        if '$ref' in self.schema:
            return self.schema['$ref'].split('/')[-1]
        else:
            return None

    @property
    def imports(self):
        def gen_imports():
            if self.refname:
                yield self.refname
            for t in self.subtypes:
                yield from t.imports
            for v in self.properties.values():
                yield from v.imports
        return sorted(set(gen_imports()))

    @property
    def attributes(self):
        return [v for k, v in sorted(self.properties.items())]

    @property
    def enum(self):
        return self.schema.get('enum', [])

class VegaLiteSchema(SchemaProperty):
    """
    This is a wrapper for the vegalite JSON schema that provides tools to
    export Python wrappers.
    """
    def __init__(self, schema_file=None):
        if schema_file is None:
            schema_file = getpath('..', 'altair', 'schema',
                                  'vega-lite-schema.json')
        with open(schema_file) as f:
            schema = json.load(f)

        self.definitions = {k: SchemaProperty(v, k, self)
                            for k, v in schema['definitions'].items()}

        super(VegaLiteSchema, self).__init__(schema, 'VegaLiteSchema', self)

    def wrappers(self):
        """Iterator over all class aliases to generate.

        These will be passed via jinja to ``templates/{template}.tpl``
        and saved to ``altair/schema/_wrappers/{template}.py``
        """

        # Channel wrapper classes
        for base in ['PositionChannelDef', 'ChannelDefWithLegend',
                     'FieldDef', 'OrderChannelDef']:
            wrappername = base.replace('Def', '')
            yield dict(name=wrappername,
                       base=self.definitions[base],
                       imports=[dict(module='.._interface', names=[base])],
                       root='channel_wrappers')

        # Encoding channel specializations
        encoding = self.definitions['Encoding']
        for attr in encoding.attributes:
            base = attr.trait_or_subtrait
            wrappername = base.replace('Def', '')
            yield dict(name=attr.name.title(),
                       base=self.definitions[base].rename(wrappername),
                       imports=[dict(module='.channel_wrappers', names=[wrappername])],
                       root='encoding_channels')

    def write_interface(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing code to {0}".format(path))

        # Write Init File
        template = JINJA_ENV.get_template('interface_init.tpl')
        header = "Auto-generated Python wrappers for Vega-Lite Schema"
        print(" - Writing __init__.py")
        objects = [dict(module=obj.lower(), classname=obj)
                   for obj in sorted(self.definitions)]
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=objects,
                                    header=header))

        # Write Class Definition files
        templates = {'string': JINJA_ENV.get_template('interface_enum.tpl'),
                     'object': JINJA_ENV.get_template('interface_object.tpl')}
        for key, prop in sorted(self.definitions.items()):
            if prop.type not in templates:
                raise ValueError("No template for type={0}".format(prop.type))
            outfile = os.path.join(path, '{0}.py'.format(key.lower()))
            print(" - Writing {0}".format(os.path.basename(outfile)))
            with open(outfile, 'w') as f:
                f.write(templates[prop.type].render(cls=prop))

    def write_interface_tests(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface', 'tests')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing tests to {0}".format(path))

        # Write test Init File
        template = JINJA_ENV.get_template('interface_init.tpl')
        header = 'Auto-generated tests for Vega-Lite Schema wrappers'
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=[], header=header))

        # Write test file
        template = JINJA_ENV.get_template('interface_test.tpl')
        classes = [prop for key, prop in sorted(self.definitions.items())]
        print(" - Writing test_instantiations.py")
        with open(os.path.join(path, 'test_instantiations.py'), 'w') as f:
            f.write(template.render(classes=classes))

    def write_wrappers(self, path=None):
        # make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_wrappers')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing wrappers to {0}".format(path))

        # Write wrapper definition files
        groups = groupby(self.wrappers(), itemgetter('root'))
        for root, wrappers in groups:
            template = JINJA_ENV.get_template('{0}.tpl'.format(root))
            filename = os.path.join(path, '{0}.py'.format(root))
            print("- writing {0}".format(filename))
            with open(filename, 'w') as f:
                f.write(template.render(objects=list(wrappers)))

        # Write __init__.py file
        template = JINJA_ENV.get_template('wrapper_init.tpl')
        filename = os.path.join(path, '__init__.py')
        header = 'Wrappers for low-level schema objects'
        print("- writing {0}".format(filename))
        with open(filename, 'w') as f:
            f.write(template.render(objects=list(self.wrappers()),
                                    header=header))

if __name__ == '__main__':
    schema = VegaLiteSchema()
    schema.write_interface()
    schema.write_interface_tests()
    schema.write_wrappers()
<commit_msg>remove unused import<commit_after>import os
import json
from itertools import chain, groupby
from collections import defaultdict
from operator import itemgetter

from jinja2 import Environment, FileSystemLoader, environmentfilter

@environmentfilter
def merge_imports(env, objects):
    """Jinja filter to merge all imports from a list of objects"""
    imports = chain(*(env.getattr(obj, 'imports') for obj in objects))
    modules = defaultdict(set)
    for imp in imports:
        modules[env.getattr(imp, 'module')] |= set(env.getattr(imp, 'names'))
    return ["from {0} import {1}".format(module, ', '.join(sorted(names)))
            for module, names in sorted(modules.items())]

def getpath(*args):
    """Get absolute path of joined directories relative to this file"""
    return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))

# Construct the jinja template environment
JINJA_ENV = Environment(loader=FileSystemLoader(getpath('templates')))
JINJA_ENV.filters['repr'] = repr
JINJA_ENV.filters['merge_imports'] = merge_imports

TYPE_MAP = {'oneOf': 'Union',
            "array": "List",
            "boolean": "Bool",
            "number": "CFloat",
            "string": "Unicode",
            "object": "Any",
            }

class SchemaProperty(object):
    """Class Wrapper for a property in a VegaLite Schema

    This class exposes methods used in the templating.
    """
    def __init__(self, schema, name, top):
        self.name = name
        self.schema = schema
        self.top = top

        self.properties = {k: SchemaProperty(v, k, self.top)
                           for k, v in self.schema.get('properties',
                                                       {}).items()}

    def rename(self, newname):
        return self.__class__(self.schema, newname, self.top)

    @property
    def subtypes(self):
        trait = self.trait_name
        if trait == 'Union':
            return [self.__class__(s, '', self.top)
                    for s in self.schema['oneOf']]
        elif trait == 'List':
            return [self.__class__(self.schema['items'], '', self.top)]
        else:
            return []

    @property
    def type(self):
        if '$ref' in self.schema:
            return '$ref'
        elif 'oneOf' in self.schema:
            return 'oneOf'
        else:
            return self.schema.get('type', '')

    @property
    def trait_name(self):
        if self.refname:
            trait = self.top.definitions[self.refname].trait_name
            if trait == 'Any':
                return 'Instance'
            else:
                return self.refname
        else:
            return TYPE_MAP.get(self.type, "Any")

    @property
    def trait_descr(self):
        trait = self.trait_name
        if trait == 'Instance':
            return self.refname
        elif self.subtypes:
            return '{0}({1})'.format(trait, ', '.join(t.trait_descr
                                                      for t in self.subtypes))
        else:
            return trait

    @property
    def trait_fulldef(self):
        trait = self.trait_name
        kwds = "allow_none=True, default_value=None"
        if 'minimum' in self.schema:
            kwds += ', min={0}'.format(self.schema['minimum'])
        if 'maximum' in self.schema:
            kwds += ', max={0}'.format(self.schema['maximum'])
        if self.description:
            kwds += ', help="""{0}"""'.format(self.short_description)

        if trait == 'Union':
            return ('T.Union([{0}])'
                    ''.format(', '.join(t.trait_fulldef
                                        for t in self.subtypes)))
        elif trait == 'List':
            return 'T.List({0}, {1})'.format(*(t.trait_fulldef
                                               for t in self.subtypes), kwds)
        elif trait == 'Instance':
            return 'T.Instance({0}, {1})'.format(self.refname, kwds)
        elif trait == self.refname:
            return '{0}({1})'.format(self.refname, kwds)
        else:
            return 'T.{0}({1})'.format(trait, kwds)

    @property
    def trait_or_subtrait(self):
        if self.refname is not None:
            return self.refname
        elif self.trait_name in ['Union', 'List']:
            for t in self.subtypes:
                if t.trait_or_subtrait is not None:
                    return t.trait_or_subtrait
            else:
                return None
        else:
            return None

    @property
    def description(self):
        return self.schema.get('description', '')

    @property
    def short_description(self):
        if self.description:
            return self.description.split('(e.g.')[0].split('.')[0] + '.'
        else:
            return ''

    @property
    def long_description(self):
        return self.description

    @property
    def refname(self):
        if '$ref' in self.schema:
            return self.schema['$ref'].split('/')[-1]
        else:
            return None

    @property
    def imports(self):
        def gen_imports():
            if self.refname:
                yield self.refname
            for t in self.subtypes:
                yield from t.imports
            for v in self.properties.values():
                yield from v.imports
        return sorted(set(gen_imports()))

    @property
    def attributes(self):
        return [v for k, v in sorted(self.properties.items())]

    @property
    def enum(self):
        return self.schema.get('enum', [])

class VegaLiteSchema(SchemaProperty):
    """
    This is a wrapper for the vegalite JSON schema that provides tools to
    export Python wrappers.
    """
    def __init__(self, schema_file=None):
        if schema_file is None:
            schema_file = getpath('..', 'altair', 'schema',
                                  'vega-lite-schema.json')
        with open(schema_file) as f:
            schema = json.load(f)

        self.definitions = {k: SchemaProperty(v, k, self)
                            for k, v in schema['definitions'].items()}

        super(VegaLiteSchema, self).__init__(schema, 'VegaLiteSchema', self)

    def wrappers(self):
        """Iterator over all class aliases to generate.

        These will be passed via jinja to ``templates/{template}.tpl``
        and saved to ``altair/schema/_wrappers/{template}.py``
        """

        # Channel wrapper classes
        for base in ['PositionChannelDef', 'ChannelDefWithLegend',
                     'FieldDef', 'OrderChannelDef']:
            wrappername = base.replace('Def', '')
            yield dict(name=wrappername,
                       base=self.definitions[base],
                       imports=[dict(module='.._interface', names=[base])],
                       root='channel_wrappers')

        # Encoding channel specializations
        encoding = self.definitions['Encoding']
        for attr in encoding.attributes:
            base = attr.trait_or_subtrait
            wrappername = base.replace('Def', '')
            yield dict(name=attr.name.title(),
                       base=self.definitions[base].rename(wrappername),
                       imports=[dict(module='.channel_wrappers', names=[wrappername])],
                       root='encoding_channels')

    def write_interface(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing code to {0}".format(path))

        # Write Init File
        template = JINJA_ENV.get_template('interface_init.tpl')
        header = "Auto-generated Python wrappers for Vega-Lite Schema"
        print(" - Writing __init__.py")
        objects = [dict(module=obj.lower(), classname=obj)
                   for obj in sorted(self.definitions)]
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=objects,
                                    header=header))

        # Write Class Definition files
        templates = {'string': JINJA_ENV.get_template('interface_enum.tpl'),
                     'object': JINJA_ENV.get_template('interface_object.tpl')}
        for key, prop in sorted(self.definitions.items()):
            if prop.type not in templates:
                raise ValueError("No template for type={0}".format(prop.type))
            outfile = os.path.join(path, '{0}.py'.format(key.lower()))
            print(" - Writing {0}".format(os.path.basename(outfile)))
            with open(outfile, 'w') as f:
                f.write(templates[prop.type].render(cls=prop))

    def write_interface_tests(self, path=None):
        # Make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_interface', 'tests')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing tests to {0}".format(path))

        # Write test Init File
        template = JINJA_ENV.get_template('interface_init.tpl')
        header = 'Auto-generated tests for Vega-Lite Schema wrappers'
        print(" - Writing __init__.py")
        with open(os.path.join(path, '__init__.py'), 'w') as f:
            f.write(template.render(objects=[], header=header))

        # Write test file
        template = JINJA_ENV.get_template('interface_test.tpl')
        classes = [prop for key, prop in sorted(self.definitions.items())]
        print(" - Writing test_instantiations.py")
        with open(os.path.join(path, 'test_instantiations.py'), 'w') as f:
            f.write(template.render(classes=classes))

    def write_wrappers(self, path=None):
        # make sure the path is valid
        if path is None:
            path = getpath('..', 'altair', 'schema', '_wrappers')
        if not os.path.exists(path):
            os.makedirs(path)

        print("Writing wrappers to {0}".format(path))

        # Write wrapper definition files
        groups = groupby(self.wrappers(), itemgetter('root'))
        for root, wrappers in groups:
            template = JINJA_ENV.get_template('{0}.tpl'.format(root))
            filename = os.path.join(path, '{0}.py'.format(root))
            print("- writing {0}".format(filename))
            with open(filename, 'w') as f:
                f.write(template.render(objects=list(wrappers)))

        # Write __init__.py file
        template = JINJA_ENV.get_template('wrapper_init.tpl')
        filename = os.path.join(path, '__init__.py')
        header = 'Wrappers for low-level schema objects'
        print("- writing {0}".format(filename))
        with open(filename, 'w') as f:
            f.write(template.render(objects=list(self.wrappers()),
                                    header=header))

if __name__ == '__main__':
    schema = VegaLiteSchema()
    schema.write_interface()
    schema.write_interface_tests()
    schema.write_wrappers()
<|endoftext|>


<commit_before># coding: utf8
from __future__ import print_function
# NB! This breaks in plac on Python 2!!
#from __future__ import unicode_literals

import plac
from spacy.cli import download as cli_download
from spacy.cli import link as cli_link
from spacy.cli import info as cli_info
from spacy.cli import package as cli_package
from spacy.cli import train as cli_train
from spacy.cli import model as cli_model
from spacy.cli import convert as cli_convert

@plac.annotations(
    model=("model to download (shortcut or model name)", "positional", None, str),
    direct=("force direct download. Needs model name with version and won't "
            "perform compatibility check", "flag", "d", bool)
)
def download(model, direct=False):
    """
    Download compatible model from default download path using pip. Model
    can be shortcut, model name or, if --direct flag is set, full model name
    with version.
    """
    cli_download(model, direct)

@plac.annotations(
    origin=("package name or local path to model", "positional", None, str),
    link_name=("name of shortuct link to create", "positional", None, str),
    force=("force overwriting of existing link", "flag", "f", bool)
)
def link(origin, link_name, force=False):
    """
    Create a symlink for models within the spacy/data directory. Accepts
    either the name of a pip package, or the local path to the model data
    directory. Linking models allows loading them via spacy.load(link_name).
    """
    cli_link(origin, link_name, force)

@plac.annotations(
    model=("optional: shortcut link of model", "positional", None, str),
    markdown=("generate Markdown for GitHub issues", "flag", "md", str)
)
def info(model=None, markdown=False):
    """
    Print info about spaCy installation. If a model shortcut link is
    speficied as an argument, print model information. Flag --markdown
    prints details in Markdown for easy copy-pasting to GitHub issues.
    """
    cli_info(model, markdown)

@plac.annotations(
    input_dir=("directory with model data", "positional", None, str),
    output_dir=("output parent directory", "positional", None, str),
    meta=("path to meta.json", "option", "m", str),
    force=("force overwriting of existing folder in output directory", "flag", "f", bool)
)
def package(input_dir, output_dir, meta=None, force=False):
    """
    Generate Python package for model data, including meta and required
    installation files. A new directory will be created in the specified
    output directory, and model data will be copied over.
    """
    cli_package(input_dir, output_dir, meta, force)

@plac.annotations(
    lang=("model language", "positional", None, str),
    output_dir=("output directory to store model in", "positional", None, str),
    train_data=("location of JSON-formatted training data", "positional", None, str),
    dev_data=("location of JSON-formatted development data (optional)", "positional", None, str),
    n_iter=("number of iterations", "option", "n", int),
    nsents=("number of sentences", "option", None, int),
    parser_L1=("L1 regularization penalty for parser", "option", "L", float),
    use_gpu=("Use GPU", "flag", "g", bool),
    no_tagger=("Don't train tagger", "flag", "T", bool),
    no_parser=("Don't train parser", "flag", "P", bool),
    no_entities=("Don't train NER", "flag", "N", bool)
)
def train(lang, output_dir, train_data, dev_data=None, n_iter=15,
          nsents=0, parser_L1=0.0, use_gpu=False,
          no_tagger=False, no_parser=False, no_entities=False):
    """
    Train a model. Expects data in spaCy's JSON format.
    """
    nsents = nsents or None
    cli_train(lang, output_dir, train_data, dev_data, n_iter, nsents,
              use_gpu, no_tagger, no_parser, no_entities, parser_L1)

@plac.annotations(
    input_file=("input file", "positional", None, str),
    output_dir=("output directory for converted file", "positional", None, str),
    n_sents=("Number of sentences per doc", "option", "n", float),
    morphology=("Enable appending morphology to tags", "flag", "m", bool)
)
def convert(input_file, output_dir, n_sents=10, morphology=False):
    """
    Convert files into JSON format for use with train command and other
    experiment management functions.
    """
    cli_convert(input_file, output_dir, n_sents, morphology)

@plac.annotations(
    lang=("model language", "positional", None, str),
    model_dir=("output directory to store model in", "positional", None, str),
    freqs_data=("tab-separated frequencies file", "positional", None, str),
    clusters_data=("Brown clusters file", "positional", None, str),
    vectors_data=("word vectors file", "positional", None, str)
)
def model(lang, model_dir, freqs_data, clusters_data=None, vectors_data=None):
    """
    Initialize a new model and its data directory.
    """
    cli_model(lang, model_dir, freqs_data, clusters_data, vectors_data)

@plac.annotations(
    lang=("model language", "positional", None, str),
    output_dir=("output directory to store model in", "positional", None, str),
    train_data=("location of JSON-formatted training data", "positional", None, str),
    dev_data=("location of JSON-formatted development data (optional)", "positional", None, str),
    n_iter=("number of iterations", "option", "n", int),
    nsents=("number of sentences", "option", None, int),
    use_gpu=("Use GPU", "flag", "g", bool),
    no_tagger=("Don't train tagger", "flag", "T", bool),
    no_parser=("Don't train parser", "flag", "P", bool),
    no_entities=("Don't train NER", "flag", "N", bool)
)
def train(self, lang, output_dir, train_data, dev_data=None, n_iter=15,
          nsents=0, use_gpu=False,
          no_tagger=False, no_parser=False, no_entities=False):
    """
    Train a model. Expects data in spaCy's JSON format.
    """
    print(train_data, dev_data)
    nsents = nsents or None
    cli_train(lang, output_dir, train_data, dev_data, n_iter, nsents,
              use_gpu, no_tagger, no_parser, no_entities)

if __name__ == '__main__':
    import plac
    import sys
    if sys.argv[1] == 'train':
        plac.call(train)
    if sys.argv[1] == 'convert':
        plac.call(convert)
<commit_msg>Simplify CLI implementation for subcommands. Remove model command.<commit_after># coding: utf8
from __future__ import print_function
# NB! This breaks in plac on Python 2!!
#from __future__ import unicode_literals

import plac
from spacy.cli import download as cli_download
from spacy.cli import link as cli_link
from spacy.cli import info as cli_info
from spacy.cli import package as cli_package
from spacy.cli import train as cli_train
from spacy.cli import model as cli_model
from spacy.cli import convert as cli_convert

@plac.annotations(
    model=("model to download (shortcut or model name)", "positional", None, str),
    direct=("force direct download. Needs model name with version and won't "
            "perform compatibility check", "flag", "d", bool)
)
def download(model, direct=False):
    """
    Download compatible model from default download path using pip. Model
    can be shortcut, model name or, if --direct flag is set, full model name
    with version.
    """
    cli_download(model, direct)

@plac.annotations(
    origin=("package name or local path to model", "positional", None, str),
    link_name=("name of shortuct link to create", "positional", None, str),
    force=("force overwriting of existing link", "flag", "f", bool)
)
def link(origin, link_name, force=False):
    """
    Create a symlink for models within the spacy/data directory. Accepts
    either the name of a pip package, or the local path to the model data
    directory. Linking models allows loading them via spacy.load(link_name).
    """
    cli_link(origin, link_name, force)

@plac.annotations(
    model=("optional: shortcut link of model", "positional", None, str),
    markdown=("generate Markdown for GitHub issues", "flag", "md", str)
)
def info(model=None, markdown=False):
    """
    Print info about spaCy installation. If a model shortcut link is
    speficied as an argument, print model information. Flag --markdown
    prints details in Markdown for easy copy-pasting to GitHub issues.
    """
    cli_info(model, markdown)

@plac.annotations(
    input_dir=("directory with model data", "positional", None, str),
    output_dir=("output parent directory", "positional", None, str),
    meta=("path to meta.json", "option", "m", str),
    force=("force overwriting of existing folder in output directory", "flag", "f", bool)
)
def package(input_dir, output_dir, meta=None, force=False):
    """
    Generate Python package for model data, including meta and required
    installation files. A new directory will be created in the specified
    output directory, and model data will be copied over.
    """
    cli_package(input_dir, output_dir, meta, force)

@plac.annotations(
    input_file=("input file", "positional", None, str),
    output_dir=("output directory for converted file", "positional", None, str),
    n_sents=("Number of sentences per doc", "option", "n", float),
    morphology=("Enable appending morphology to tags", "flag", "m", bool)
)
def convert(input_file, output_dir, n_sents=10, morphology=False):
    """
    Convert files into JSON format for use with train command and other
    experiment management functions.
    """
    cli_convert(input_file, output_dir, n_sents, morphology)

@plac.annotations(
    lang=("model language", "positional", None, str),
    output_dir=("output directory to store model in", "positional", None, str),
    train_data=("location of JSON-formatted training data", "positional", None, str),
    dev_data=("location of JSON-formatted development data (optional)", "positional", None, str),
    n_iter=("number of iterations", "option", "n", int),
    nsents=("number of sentences", "option", None, int),
    use_gpu=("Use GPU", "flag", "g", bool),
    no_tagger=("Don't train tagger", "flag", "T", bool),
    no_parser=("Don't train parser", "flag", "P", bool),
    no_entities=("Don't train NER", "flag", "N", bool)
)
def train(lang, output_dir, train_data, dev_data=None, n_iter=15,
          nsents=0, use_gpu=False,
          no_tagger=False, no_parser=False, no_entities=False):
    """
    Train a model. Expects data in spaCy's JSON format.
    """
    nsents = nsents or None
    cli_train(lang, output_dir, train_data, dev_data, n_iter, nsents,
              use_gpu, no_tagger, no_parser, no_entities)

if __name__ == '__main__':
    import plac
    import sys
    commands = {
        'train': train,
        'convert': convert,
        'download': download,
        'link': link,
        'info': info,
        'package': package,
    }
    if len(sys.argv) == 1:
        print("Available commands: %s" % ', '.join(sorted(commands)))
        sys.exit(1)
    command = sys.argv.pop(1)
    sys.argv[0] = 'spacy %s' % command
    if command in commands:
        plac.call(commands[command])
    else:
        print("Unknown command: %s. Available: %s" % (command, ', '.join(commands)))
        sys.exit(1)
<|endoftext|>


<commit_before>from PyQt4 import QtGui
from intro import IntroPage
from license import LicensePage
from template import TemplatePage
from load import LoadPage
from config import ConfigPage
from save import SavePage
from generate import GeneratePage
from ..template import TemplateCollection

class Wizard(QtGui.QWizard):
  def __init__(self, templates, parent=None):
    super(Wizard, self).__init__(parent)
    #self.setStyleSheet("QWizard { background: blue }; QWizardPage { background: yellow;}")
    self.templates = templates
    self.addPage(IntroPage())
    self.addPage(LicensePage())
    tmpl = TemplatePage(self.templates)
    self.addPage(tmpl)
    load = LoadPage(self.templates)
    self.addPage(load)
    conf = ConfigPage(self.templates)
    self.addPage(conf)
    save = SavePage(self.templates)
    self.addPage(save)
    gen  = GeneratePage(self.templates)
    self.addPage(gen)

def main(template, configuration):
    templates = TemplateCollection()
    if template:
      if template in templates.keys():
        templates.setTemplate(template)
        if configuration:
          if configuration in templates.listConfigurations():
            templates.loadConfiguration(configuration)
          else:
            print "Configuration %s for template %s not found" % (configuration, template)
        templates.generate()
      else:
        print "Template %s not found" % template
    else:
      app = QtGui.QApplication([])
      wizard = Wizard(templates)
      wizard.setWindowTitle("SoCRocket - Configuration Wizard")
      wizard.show()

      import sys
      sys.exit(wizard.exec_())
      QtGui.app = None

<commit_msg>Changing generator<commit_after>from PyQt4 import QtGui
from intro import IntroPage
from license import LicensePage
from template import TemplatePage
from load import LoadPage
from config import ConfigPage
from save import SavePage
from generate import GeneratePage
from ..template import TemplateCollection

class Wizard(QtGui.QWizard):
  def __init__(self, templates, parent=None):
    super(Wizard, self).__init__(parent)
    #self.setStyleSheet("QWizard { background: blue }; QWizardPage { background: yellow;}")
    self.templates = templates
    self.addPage(IntroPage())
    #self.addPage(LicensePage())
    tmpl = TemplatePage(self.templates)
    self.addPage(tmpl)
    load = LoadPage(self.templates)
    self.addPage(load)
    conf = ConfigPage(self.templates)
    self.addPage(conf)
    save = SavePage(self.templates)
    self.addPage(save)
    gen  = GeneratePage(self.templates)
    self.addPage(gen)

def main(template, configuration):
    templates = TemplateCollection()
    if template:
      if template in templates.keys():
        templates.setTemplate(template)
        if configuration:
          if configuration in templates.listConfigurations():
            templates.loadConfiguration(configuration)
          else:
            print "Configuration %s for template %s not found" % (configuration, template)
        templates.generate()
      else:
        print "Template %s not found" % template
    else:
      app = QtGui.QApplication([])
      wizard = Wizard(templates)
      wizard.setWindowTitle("SoCRocket - Configuration Wizard")
      wizard.show()

      import sys
      sys.exit(wizard.exec_())
      QtGui.app = None

<|endoftext|>


<commit_before># coding: utf8
from __future__ import absolute_import, unicode_literals
from contextlib import contextmanager
import shutil

from .tokenizer import Tokenizer
from .vocab import Vocab
from .tagger import Tagger
from .matcher import Matcher
from .lemmatizer import Lemmatizer
from .train import Trainer
from .syntax.parser import get_templates
from .syntax.nonproj import PseudoProjectivity
from .pipeline import DependencyParser, EntityRecognizer
from .syntax.arc_eager import ArcEager
from .syntax.ner import BiluoPushDown
from .compat import json_dumps
from .attrs import IS_STOP
from .lang.punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
from .lang.tokenizer_exceptions import TOKEN_MATCH
from .lang.tag_map import TAG_MAP
from . import attrs
from . import orth
from . import util

class BaseDefaults(object):
    @classmethod
    def create_lemmatizer(cls, nlp=None):
        return Lemmatizer(cls.lemma_index, cls.lemma_exc, cls.lemma_rules)

    @classmethod
    def create_vocab(cls, nlp=None):
        lemmatizer = cls.create_lemmatizer(nlp)
        if nlp is None or nlp.path is None:
            lex_attr_getters = dict(cls.lex_attr_getters)
            # This is very messy, but it's the minimal working fix to Issue #639.
            # This defaults stuff needs to be refactored (again)
            lex_attr_getters[IS_STOP] = lambda string: string.lower() in cls.stop_words
            vocab = Vocab(lex_attr_getters=lex_attr_getters, tag_map=cls.tag_map,
                         lemmatizer=lemmatizer)
        else:
            vocab = Vocab.load(nlp.path, lex_attr_getters=cls.lex_attr_getters,
                             tag_map=cls.tag_map, lemmatizer=lemmatizer)
        for tag_str, exc in cls.morph_rules.items():
            for orth_str, attrs in exc.items():
                vocab.morphology.add_special_case(tag_str, orth_str, attrs)
        return vocab

    @classmethod
    def add_vectors(cls, nlp=None):
        if nlp is None or nlp.path is None:
            return False
        else:
            vec_path = nlp.path / 'vocab' / 'vec.bin'
            if vec_path.exists():
                return lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)

    @classmethod
    def create_tokenizer(cls, nlp=None):
        rules = cls.tokenizer_exceptions
        if cls.token_match:
            token_match = cls.token_match
        if cls.prefixes:
            prefix_search  = util.compile_prefix_regex(cls.prefixes).search
        else:
            prefix_search = None
        if cls.suffixes:
            suffix_search  = util.compile_suffix_regex(cls.suffixes).search
        else:
            suffix_search = None
        if cls.infixes:
            infix_finditer = util.compile_infix_regex(cls.infixes).finditer
        else:
            infix_finditer = None
        vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
        return Tokenizer(vocab, rules=rules,
                         prefix_search=prefix_search, suffix_search=suffix_search,
                         infix_finditer=infix_finditer, token_match=token_match)

    @classmethod
    def create_tagger(cls, nlp=None):
        if nlp is None:
            return Tagger(cls.create_vocab(), features=cls.tagger_features)
        elif nlp.path is False:
            return Tagger(nlp.vocab, features=cls.tagger_features)
        elif nlp.path is None or not (nlp.path / 'pos').exists():
            return None
        else:
            return Tagger.load(nlp.path / 'pos', nlp.vocab)

    @classmethod
    def create_parser(cls, nlp=None, **cfg):
        if nlp is None:
            return DependencyParser(cls.create_vocab(), features=cls.parser_features,
                                    **cfg)
        elif nlp.path is False:
            return DependencyParser(nlp.vocab, features=cls.parser_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'deps').exists():
            return None
        else:
            return DependencyParser.load(nlp.path / 'deps', nlp.vocab, **cfg)

    @classmethod
    def create_entity(cls, nlp=None, **cfg):
        if nlp is None:
            return EntityRecognizer(cls.create_vocab(), features=cls.entity_features, **cfg)
        elif nlp.path is False:
            return EntityRecognizer(nlp.vocab, features=cls.entity_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'ner').exists():
            return None
        else:
            return EntityRecognizer.load(nlp.path / 'ner', nlp.vocab, **cfg)

    @classmethod
    def create_matcher(cls, nlp=None):
        if nlp is None:
            return Matcher(cls.create_vocab())
        elif nlp.path is False:
            return Matcher(nlp.vocab)
        elif nlp.path is None or not (nlp.path / 'vocab').exists():
            return None
        else:
            return Matcher.load(nlp.path / 'vocab', nlp.vocab)

    @classmethod
    def create_pipeline(self, nlp=None):
        pipeline = []
        if nlp is None:
            return []
        if nlp.tagger:
            pipeline.append(nlp.tagger)
        if nlp.parser:
            pipeline.append(nlp.parser)
            pipeline.append(PseudoProjectivity.deprojectivize)
        if nlp.entity:
            pipeline.append(nlp.entity)
        return pipeline

    token_match = TOKEN_MATCH
    prefixes = tuple(TOKENIZER_PREFIXES)
    suffixes = tuple(TOKENIZER_SUFFIXES)
    infixes = tuple(TOKENIZER_INFIXES)
    tag_map = dict(TAG_MAP)
    tokenizer_exceptions = {}
    parser_features = get_templates('parser')
    entity_features = get_templates('ner')
    tagger_features = Tagger.feature_templates # TODO -- fix this
    stop_words = set()
    lemma_rules = {}
    lemma_exc = {}
    lemma_index = {}
    morph_rules = {}

    lex_attr_getters = {
        attrs.LOWER: lambda string: string.lower(),
        attrs.NORM: lambda string: string,
        attrs.SHAPE: orth.word_shape,
        attrs.PREFIX: lambda string: string[0],
        attrs.SUFFIX: lambda string: string[-3:],
        attrs.CLUSTER: lambda string: 0,
        attrs.IS_ALPHA: orth.is_alpha,
        attrs.IS_ASCII: orth.is_ascii,
        attrs.IS_DIGIT: lambda string: string.isdigit(),
        attrs.IS_LOWER: orth.is_lower,
        attrs.IS_PUNCT: orth.is_punct,
        attrs.IS_SPACE: lambda string: string.isspace(),
        attrs.IS_TITLE: orth.is_title,
        attrs.IS_UPPER: orth.is_upper,
        attrs.IS_BRACKET: orth.is_bracket,
        attrs.IS_QUOTE: orth.is_quote,
        attrs.IS_LEFT_PUNCT: orth.is_left_punct,
        attrs.IS_RIGHT_PUNCT: orth.is_right_punct,
        attrs.LIKE_URL: orth.like_url,
        attrs.LIKE_NUM: orth.like_number,
        attrs.LIKE_EMAIL: orth.like_email,
        attrs.IS_STOP: lambda string: False,
        attrs.IS_OOV: lambda string: True
    }

class Language(object):
    """
    A text-processing pipeline. Usually you'll load this once per process, and
    pass the instance around your program.
    """
    Defaults = BaseDefaults
    lang = None

    @classmethod
    def setup_directory(cls, path, **configs):
        """
        Initialise a model directory.
        """
        for name, config in configs.items():
            directory = path / name
            if directory.exists():
                shutil.rmtree(str(directory))
            directory.mkdir()
            with (directory / 'config.json').open('w') as file_:
                data = json_dumps(config)
                file_.write(data)
        if not (path / 'vocab').exists():
            (path / 'vocab').mkdir()

    @classmethod
    @contextmanager
    def train(cls, path, gold_tuples, **configs):
        parser_cfg = configs.get('deps', {})
        if parser_cfg.get('pseudoprojective'):
            # preprocess training data here before ArcEager.get_labels() is called
            gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)

        for subdir in ('deps', 'ner', 'pos'):
            if subdir not in configs:
                configs[subdir] = {}
        if parser_cfg:
            configs['deps']['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
        if 'ner' in configs:
            configs['ner']['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)

        cls.setup_directory(path, **configs)

        self = cls(
                path=path,
                vocab=False,
                tokenizer=False,
                tagger=False,
                parser=False,
                entity=False,
                matcher=False,
                serializer=False,
                vectors=False,
                pipeline=False)

        self.vocab = self.Defaults.create_vocab(self)
        self.tokenizer = self.Defaults.create_tokenizer(self)
        self.tagger = self.Defaults.create_tagger(self)
        self.parser = self.Defaults.create_parser(self)
        self.entity = self.Defaults.create_entity(self)
        self.pipeline = self.Defaults.create_pipeline(self)
        yield Trainer(self, gold_tuples)
        self.end_training()
        self.save_to_directory(path)

    def __init__(self, **overrides):
        """
        Create or load the pipeline.

        Arguments:
            **overrides: Keyword arguments indicating which defaults to override.

        Returns:
            Language: The newly constructed object.
        """
        if 'data_dir' in overrides and 'path' not in overrides:
            raise ValueError("The argument 'data_dir' has been renamed to 'path'")
        path = util.ensure_path(overrides.get('path', True))
        if path is True:
            path = util.get_data_path() / self.lang
            if not path.exists() and 'path' not in overrides:
                path = None
        self.meta = overrides.get('meta', {})
        self.path = path

        self.vocab     = self.Defaults.create_vocab(self) \
                         if 'vocab' not in overrides \
                         else overrides['vocab']
        add_vectors    = self.Defaults.add_vectors(self) \
                         if 'add_vectors' not in overrides \
                         else overrides['add_vectors']
        if self.vocab and add_vectors:
            add_vectors(self.vocab)
        self.tokenizer = self.Defaults.create_tokenizer(self) \
                         if 'tokenizer' not in overrides \
                         else overrides['tokenizer']
        self.tagger    = self.Defaults.create_tagger(self) \
                         if 'tagger' not in overrides \
                         else overrides['tagger']
        self.parser    = self.Defaults.create_parser(self) \
                         if 'parser' not in overrides \
                         else overrides['parser']
        self.entity    = self.Defaults.create_entity(self) \
                         if 'entity' not in overrides \
                         else overrides['entity']
        self.matcher   = self.Defaults.create_matcher(self) \
                         if 'matcher' not in overrides \
                         else overrides['matcher']

        if 'make_doc' in overrides:
            self.make_doc = overrides['make_doc']
        elif 'create_make_doc' in overrides:
            self.make_doc = overrides['create_make_doc'](self)
        elif not hasattr(self, 'make_doc'):
            self.make_doc = lambda text: self.tokenizer(text)
        if 'pipeline' in overrides:
            self.pipeline = overrides['pipeline']
        elif 'create_pipeline' in overrides:
            self.pipeline = overrides['create_pipeline'](self)
        else:
            self.pipeline = [self.tagger, self.parser, self.matcher, self.entity]

    def __call__(self, text, tag=True, parse=True, entity=True):
        """
        Apply the pipeline to some text.  The text can span multiple sentences,
        and can contain arbtrary whitespace.  Alignment into the original string
        is preserved.

        Argsuments:
            text (unicode): The text to be processed.

        Returns:
            doc (Doc): A container for accessing the annotations.

        Example:
            >>> from spacy.en import English
            >>> nlp = English()
            >>> tokens = nlp('An example sentence. Another example sentence.')
            >>> tokens[0].orth_, tokens[0].head.tag_
            ('An', 'NN')
        """
        doc = self.make_doc(text)
        if self.entity and entity:
            # Add any of the entity labels already set, in case we don't have them.
            for token in doc:
                if token.ent_type != 0:
                    self.entity.add_label(token.ent_type)
        skip = {self.tagger: not tag, self.parser: not parse, self.entity: not entity}
        for proc in self.pipeline:
            if proc and not skip.get(proc):
                proc(doc)
        return doc

    def pipe(self, texts, tag=True, parse=True, entity=True, n_threads=2, batch_size=1000):
        """
        Process texts as a stream, and yield Doc objects in order.

        Supports GIL-free multi-threading.

        Arguments:
            texts (iterator)
            tag (bool)
            parse (bool)
            entity (bool)
        """
        skip = {self.tagger: not tag, self.parser: not parse, self.entity: not entity}
        stream = (self.make_doc(text) for text in texts)
        for proc in self.pipeline:
            if proc and not skip.get(proc):
                if hasattr(proc, 'pipe'):
                    stream = proc.pipe(stream, n_threads=n_threads, batch_size=batch_size)
                else:
                    stream = (proc(item) for item in stream)
        for doc in stream:
            yield doc

    def save_to_directory(self, path):
        """
        Save the Vocab, StringStore and pipeline to a directory.

        Arguments:
            path (string or pathlib path): Path to save the model.
        """
        configs = {
            'pos': self.tagger.cfg if self.tagger else {},
            'deps': self.parser.cfg if self.parser else {},
            'ner': self.entity.cfg if self.entity else {},
        }

        path = util.ensure_path(path)
        if not path.exists():
            path.mkdir()
        self.setup_directory(path, **configs)

        strings_loc = path / 'vocab' / 'strings.json'
        with strings_loc.open('w', encoding='utf8') as file_:
            self.vocab.strings.dump(file_)
        self.vocab.dump(path / 'vocab' / 'lexemes.bin')
        # TODO: Word vectors?
        if self.tagger:
            self.tagger.model.dump(str(path / 'pos' / 'model'))
        if self.parser:
            self.parser.model.dump(str(path / 'deps' / 'model'))
        if self.entity:
            self.entity.model.dump(str(path / 'ner' / 'model'))

    def end_training(self, path=None):
        if self.tagger:
            self.tagger.model.end_training()
        if self.parser:
            self.parser.model.end_training()
        if self.entity:
            self.entity.model.end_training()
        # NB: This is slightly different from before --- we no longer default
        # to taking nlp.path
        if path is not None:
            self.save_to_directory(path)
<commit_msg>Import LEX_ATTRS from lang.lex_attrs<commit_after># coding: utf8
from __future__ import absolute_import, unicode_literals
from contextlib import contextmanager
import shutil

from .tokenizer import Tokenizer
from .vocab import Vocab
from .tagger import Tagger
from .matcher import Matcher
from .lemmatizer import Lemmatizer
from .train import Trainer
from .syntax.parser import get_templates
from .syntax.nonproj import PseudoProjectivity
from .pipeline import DependencyParser, EntityRecognizer
from .syntax.arc_eager import ArcEager
from .syntax.ner import BiluoPushDown
from .compat import json_dumps
from .attrs import IS_STOP
from .lang.punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES
from .lang.tokenizer_exceptions import TOKEN_MATCH
from .lang.tag_map import TAG_MAP
from .lang.lex_attrs import LEX_ATTRS
from . import util

class BaseDefaults(object):
    @classmethod
    def create_lemmatizer(cls, nlp=None):
        return Lemmatizer(cls.lemma_index, cls.lemma_exc, cls.lemma_rules)

    @classmethod
    def create_vocab(cls, nlp=None):
        lemmatizer = cls.create_lemmatizer(nlp)
        if nlp is None or nlp.path is None:
            lex_attr_getters = dict(cls.lex_attr_getters)
            # This is very messy, but it's the minimal working fix to Issue #639.
            # This defaults stuff needs to be refactored (again)
            lex_attr_getters[IS_STOP] = lambda string: string.lower() in cls.stop_words
            vocab = Vocab(lex_attr_getters=lex_attr_getters, tag_map=cls.tag_map,
                         lemmatizer=lemmatizer)
        else:
            vocab = Vocab.load(nlp.path, lex_attr_getters=cls.lex_attr_getters,
                             tag_map=cls.tag_map, lemmatizer=lemmatizer)
        for tag_str, exc in cls.morph_rules.items():
            for orth_str, attrs in exc.items():
                vocab.morphology.add_special_case(tag_str, orth_str, attrs)
        return vocab

    @classmethod
    def add_vectors(cls, nlp=None):
        if nlp is None or nlp.path is None:
            return False
        else:
            vec_path = nlp.path / 'vocab' / 'vec.bin'
            if vec_path.exists():
                return lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)

    @classmethod
    def create_tokenizer(cls, nlp=None):
        rules = cls.tokenizer_exceptions
        if cls.token_match:
            token_match = cls.token_match
        if cls.prefixes:
            prefix_search  = util.compile_prefix_regex(cls.prefixes).search
        else:
            prefix_search = None
        if cls.suffixes:
            suffix_search  = util.compile_suffix_regex(cls.suffixes).search
        else:
            suffix_search = None
        if cls.infixes:
            infix_finditer = util.compile_infix_regex(cls.infixes).finditer
        else:
            infix_finditer = None
        vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp)
        return Tokenizer(vocab, rules=rules,
                         prefix_search=prefix_search, suffix_search=suffix_search,
                         infix_finditer=infix_finditer, token_match=token_match)

    @classmethod
    def create_tagger(cls, nlp=None):
        if nlp is None:
            return Tagger(cls.create_vocab(), features=cls.tagger_features)
        elif nlp.path is False:
            return Tagger(nlp.vocab, features=cls.tagger_features)
        elif nlp.path is None or not (nlp.path / 'pos').exists():
            return None
        else:
            return Tagger.load(nlp.path / 'pos', nlp.vocab)

    @classmethod
    def create_parser(cls, nlp=None, **cfg):
        if nlp is None:
            return DependencyParser(cls.create_vocab(), features=cls.parser_features,
                                    **cfg)
        elif nlp.path is False:
            return DependencyParser(nlp.vocab, features=cls.parser_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'deps').exists():
            return None
        else:
            return DependencyParser.load(nlp.path / 'deps', nlp.vocab, **cfg)

    @classmethod
    def create_entity(cls, nlp=None, **cfg):
        if nlp is None:
            return EntityRecognizer(cls.create_vocab(), features=cls.entity_features, **cfg)
        elif nlp.path is False:
            return EntityRecognizer(nlp.vocab, features=cls.entity_features, **cfg)
        elif nlp.path is None or not (nlp.path / 'ner').exists():
            return None
        else:
            return EntityRecognizer.load(nlp.path / 'ner', nlp.vocab, **cfg)

    @classmethod
    def create_matcher(cls, nlp=None):
        if nlp is None:
            return Matcher(cls.create_vocab())
        elif nlp.path is False:
            return Matcher(nlp.vocab)
        elif nlp.path is None or not (nlp.path / 'vocab').exists():
            return None
        else:
            return Matcher.load(nlp.path / 'vocab', nlp.vocab)

    @classmethod
    def create_pipeline(self, nlp=None):
        pipeline = []
        if nlp is None:
            return []
        if nlp.tagger:
            pipeline.append(nlp.tagger)
        if nlp.parser:
            pipeline.append(nlp.parser)
            pipeline.append(PseudoProjectivity.deprojectivize)
        if nlp.entity:
            pipeline.append(nlp.entity)
        return pipeline

    token_match = TOKEN_MATCH
    prefixes = tuple(TOKENIZER_PREFIXES)
    suffixes = tuple(TOKENIZER_SUFFIXES)
    infixes = tuple(TOKENIZER_INFIXES)
    tag_map = dict(TAG_MAP)
    tokenizer_exceptions = {}
    parser_features = get_templates('parser')
    entity_features = get_templates('ner')
    tagger_features = Tagger.feature_templates # TODO -- fix this
    stop_words = set()
    lemma_rules = {}
    lemma_exc = {}
    lemma_index = {}
    morph_rules = {}
    lex_attr_getters = LEX_ATTRS

class Language(object):
    """
    A text-processing pipeline. Usually you'll load this once per process, and
    pass the instance around your program.
    """
    Defaults = BaseDefaults
    lang = None

    @classmethod
    def setup_directory(cls, path, **configs):
        """
        Initialise a model directory.
        """
        for name, config in configs.items():
            directory = path / name
            if directory.exists():
                shutil.rmtree(str(directory))
            directory.mkdir()
            with (directory / 'config.json').open('w') as file_:
                data = json_dumps(config)
                file_.write(data)
        if not (path / 'vocab').exists():
            (path / 'vocab').mkdir()

    @classmethod
    @contextmanager
    def train(cls, path, gold_tuples, **configs):
        parser_cfg = configs.get('deps', {})
        if parser_cfg.get('pseudoprojective'):
            # preprocess training data here before ArcEager.get_labels() is called
            gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples)

        for subdir in ('deps', 'ner', 'pos'):
            if subdir not in configs:
                configs[subdir] = {}
        if parser_cfg:
            configs['deps']['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
        if 'ner' in configs:
            configs['ner']['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)

        cls.setup_directory(path, **configs)

        self = cls(
                path=path,
                vocab=False,
                tokenizer=False,
                tagger=False,
                parser=False,
                entity=False,
                matcher=False,
                serializer=False,
                vectors=False,
                pipeline=False)

        self.vocab = self.Defaults.create_vocab(self)
        self.tokenizer = self.Defaults.create_tokenizer(self)
        self.tagger = self.Defaults.create_tagger(self)
        self.parser = self.Defaults.create_parser(self)
        self.entity = self.Defaults.create_entity(self)
        self.pipeline = self.Defaults.create_pipeline(self)
        yield Trainer(self, gold_tuples)
        self.end_training()
        self.save_to_directory(path)

    def __init__(self, **overrides):
        """
        Create or load the pipeline.

        Arguments:
            **overrides: Keyword arguments indicating which defaults to override.

        Returns:
            Language: The newly constructed object.
        """
        if 'data_dir' in overrides and 'path' not in overrides:
            raise ValueError("The argument 'data_dir' has been renamed to 'path'")
        path = util.ensure_path(overrides.get('path', True))
        if path is True:
            path = util.get_data_path() / self.lang
            if not path.exists() and 'path' not in overrides:
                path = None
        self.meta = overrides.get('meta', {})
        self.path = path

        self.vocab     = self.Defaults.create_vocab(self) \
                         if 'vocab' not in overrides \
                         else overrides['vocab']
        add_vectors    = self.Defaults.add_vectors(self) \
                         if 'add_vectors' not in overrides \
                         else overrides['add_vectors']
        if self.vocab and add_vectors:
            add_vectors(self.vocab)
        self.tokenizer = self.Defaults.create_tokenizer(self) \
                         if 'tokenizer' not in overrides \
                         else overrides['tokenizer']
        self.tagger    = self.Defaults.create_tagger(self) \
                         if 'tagger' not in overrides \
                         else overrides['tagger']
        self.parser    = self.Defaults.create_parser(self) \
                         if 'parser' not in overrides \
                         else overrides['parser']
        self.entity    = self.Defaults.create_entity(self) \
                         if 'entity' not in overrides \
                         else overrides['entity']
        self.matcher   = self.Defaults.create_matcher(self) \
                         if 'matcher' not in overrides \
                         else overrides['matcher']

        if 'make_doc' in overrides:
            self.make_doc = overrides['make_doc']
        elif 'create_make_doc' in overrides:
            self.make_doc = overrides['create_make_doc'](self)
        elif not hasattr(self, 'make_doc'):
            self.make_doc = lambda text: self.tokenizer(text)
        if 'pipeline' in overrides:
            self.pipeline = overrides['pipeline']
        elif 'create_pipeline' in overrides:
            self.pipeline = overrides['create_pipeline'](self)
        else:
            self.pipeline = [self.tagger, self.parser, self.matcher, self.entity]

    def __call__(self, text, tag=True, parse=True, entity=True):
        """
        Apply the pipeline to some text.  The text can span multiple sentences,
        and can contain arbtrary whitespace.  Alignment into the original string
        is preserved.

        Argsuments:
            text (unicode): The text to be processed.

        Returns:
            doc (Doc): A container for accessing the annotations.

        Example:
            >>> from spacy.en import English
            >>> nlp = English()
            >>> tokens = nlp('An example sentence. Another example sentence.')
            >>> tokens[0].orth_, tokens[0].head.tag_
            ('An', 'NN')
        """
        doc = self.make_doc(text)
        if self.entity and entity:
            # Add any of the entity labels already set, in case we don't have them.
            for token in doc:
                if token.ent_type != 0:
                    self.entity.add_label(token.ent_type)
        skip = {self.tagger: not tag, self.parser: not parse, self.entity: not entity}
        for proc in self.pipeline:
            if proc and not skip.get(proc):
                proc(doc)
        return doc

    def pipe(self, texts, tag=True, parse=True, entity=True, n_threads=2, batch_size=1000):
        """
        Process texts as a stream, and yield Doc objects in order.

        Supports GIL-free multi-threading.

        Arguments:
            texts (iterator)
            tag (bool)
            parse (bool)
            entity (bool)
        """
        skip = {self.tagger: not tag, self.parser: not parse, self.entity: not entity}
        stream = (self.make_doc(text) for text in texts)
        for proc in self.pipeline:
            if proc and not skip.get(proc):
                if hasattr(proc, 'pipe'):
                    stream = proc.pipe(stream, n_threads=n_threads, batch_size=batch_size)
                else:
                    stream = (proc(item) for item in stream)
        for doc in stream:
            yield doc

    def save_to_directory(self, path):
        """
        Save the Vocab, StringStore and pipeline to a directory.

        Arguments:
            path (string or pathlib path): Path to save the model.
        """
        configs = {
            'pos': self.tagger.cfg if self.tagger else {},
            'deps': self.parser.cfg if self.parser else {},
            'ner': self.entity.cfg if self.entity else {},
        }

        path = util.ensure_path(path)
        if not path.exists():
            path.mkdir()
        self.setup_directory(path, **configs)

        strings_loc = path / 'vocab' / 'strings.json'
        with strings_loc.open('w', encoding='utf8') as file_:
            self.vocab.strings.dump(file_)
        self.vocab.dump(path / 'vocab' / 'lexemes.bin')
        # TODO: Word vectors?
        if self.tagger:
            self.tagger.model.dump(str(path / 'pos' / 'model'))
        if self.parser:
            self.parser.model.dump(str(path / 'deps' / 'model'))
        if self.entity:
            self.entity.model.dump(str(path / 'ner' / 'model'))

    def end_training(self, path=None):
        if self.tagger:
            self.tagger.model.end_training()
        if self.parser:
            self.parser.model.end_training()
        if self.entity:
            self.entity.model.end_training()
        # NB: This is slightly different from before --- we no longer default
        # to taking nlp.path
        if path is not None:
            self.save_to_directory(path)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

class Command(object):
    """
    Implements one command
    """

    def __init__(self, shell=None):
        self.shell = shell
        self.context = shell.context if shell else None

    def execute(self, arguments=None):
        """
        Executes this command

        :param arguments: The arguments for this command
        :type arguments: str or ``None``

        This function should report on progress by sending
        messages with one or multiple ``self.shell.say("Whatever response")``.

        """
        if self.information_message:
            self.shell.say(self.information_message)

    # verb or token for this command
    #
    keyword = None

    # basic information for this command
    #
    information_message = None

    # usage information for this command
    #
    usage_message = None

    # this command should be processed interactively
    #
    is_interactive = True

    # this command should be listed by 'help'
    #
    is_hidden = False
<commit_msg>streamline comments of base command<commit_after># -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

class Command(object):
    """
    Implements one command
    """

    def __init__(self, shell=None):
        self.shell = shell
        self.context = shell.context if shell else None

    def execute(self, arguments=None):
        """
        Executes this command

        :param arguments: The arguments for this command
        :type arguments: str or ``None``

        This function should report on progress by sending
        messages with one or multiple ``self.shell.say("Whatever response")``.

        """
        if self.information_message:
            self.shell.say(self.information_message)

    keyword = None      # verb or token for this command

    information_message = None    # basic information for this command

    usage_message = None    # usage information for this command

    is_interactive = True    # this command should be processed interactively

    is_hidden = False    # this command should be listed by 'help'

<|endoftext|>