<commit_before># Django settings for example project.
from os.path import join, dirname, realpath

SRC_DIR = dirname(dirname(realpath(__file__)))

# Add parent path,
# Allow starting the app without installing the module.
import sys
sys.path.insert(0, dirname(SRC_DIR))

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': SRC_DIR + '/example.db',
    }
}

TIME_ZONE = 'Europe/Amsterdam'
LANGUAGE_CODE = 'en'
SITE_ID = 1

USE_I18N = True
USE_L10N = True

MEDIA_ROOT = join(dirname(__file__), "media")
MEDIA_URL = '/media/'
STATIC_ROOT = join(dirname(__file__), "static")
STATIC_URL = '/static/'

STATICFILES_DIRS = ()
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '-#@bi6bue%#1j)6+4b&#i0g-*xro@%f@_#zwv=2-g_@n3n_kj5'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',   # Inserted language switcher, easy way to have multiple frontend languages.
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.request',
    'django.core.context_processors.static',
    'django.contrib.auth.context_processors.auth',
    'django.contrib.messages.context_processors.messages',
)

ROOT_URLCONF = 'example.urls'

TEMPLATE_DIRS = (
    join(dirname(__file__), "templates"),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',

    # Apps
    'article',
    'theme1',

    # Dependencies
    'parler',
)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse',
         }
     },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'filters': ['require_debug_false'],
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

PARLER_DEFAULT_LANGUAGE = 'en'

PARLER_LANGUAGES = {
    1: (
        {'code': 'en'},
        {'code': 'de'},
        {'code': 'fr'},
        {'code': 'nl'},
        {'code': 'es'},
    ),
    'default': {
        #'fallback': 'en',
    }
}
<commit_msg>django 1.7: silence system checks about tests<commit_after># Django settings for example project.
import django
from os.path import join, dirname, realpath

SRC_DIR = dirname(dirname(realpath(__file__)))

# Add parent path,
# Allow starting the app without installing the module.
import sys
sys.path.insert(0, dirname(SRC_DIR))

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': SRC_DIR + '/example.db',
    }
}

TIME_ZONE = 'Europe/Amsterdam'
LANGUAGE_CODE = 'en'
SITE_ID = 1

USE_I18N = True
USE_L10N = True

MEDIA_ROOT = join(dirname(__file__), "media")
MEDIA_URL = '/media/'
STATIC_ROOT = join(dirname(__file__), "static")
STATIC_URL = '/static/'

STATICFILES_DIRS = ()
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '-#@bi6bue%#1j)6+4b&#i0g-*xro@%f@_#zwv=2-g_@n3n_kj5'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',   # Inserted language switcher, easy way to have multiple frontend languages.
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.request',
    'django.core.context_processors.static',
    'django.contrib.auth.context_processors.auth',
    'django.contrib.messages.context_processors.messages',
)

ROOT_URLCONF = 'example.urls'

TEMPLATE_DIRS = (
    join(dirname(__file__), "templates"),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',

    # Apps
    'article',
    'theme1',

    # Dependencies
    'parler',
)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
         'require_debug_false': {
             '()': 'django.utils.log.RequireDebugFalse',
         }
     },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
            'filters': ['require_debug_false'],
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

if django.VERSION >= (1,7):
    TEST_RUNNER = 'django.test.runner.DiscoverRunner'  # silence system checks

PARLER_DEFAULT_LANGUAGE = 'en'

PARLER_LANGUAGES = {
    1: (
        {'code': 'en'},
        {'code': 'de'},
        {'code': 'fr'},
        {'code': 'nl'},
        {'code': 'es'},
    ),
    'default': {
        #'fallback': 'en',
    }
}
<|endoftext|>


<commit_before>import numpy as np
import json
import os
import imghdr
from datetime import datetime
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from prettytable import PrettyTable

from aovek.network.network import YOLO
from aovek.utils.image_processing import ImageProcessing

class Predict(ImageProcessing):

    def __init__(self, config):
        super().__init__(config)

        self.network = YOLO(config)
        self.network.load_model()

        self.grid_size = config['label_info']['grid_size']

        self.train_folder =\
            config['dataset']['cvpr10']['dataset_images']['train_folder']
        self.validation_folder =\
            config['dataset']['cvpr10']['dataset_images']['validation_folder']
        self.test_folder =\
            config['dataset']['cvpr10']['dataset_images']['test_folder']
        self.voc_folder = config['dataset']['voc']["folder"]

    def predict(self, image_file):
        image, original_size = self.process_image(image_file)
        original_image = self.load_image(image_file)

        predict = self.network.predict_boxes(image)
        predict = self.network.sess_run(predict)

        self.draw_rectangles(original_image, original_size, predict)

        return predict

    def predict_all_boxes(self, image_file):
        image, original_size = self.process_image(image_file)
        original_image = self.load_image(image_file)

        predict = self.network.predict(image)

        self.draw_rectangles(original_image, original_size,
                             predict, all_boxes=True)

        return predict

    def predict_video(self, video):
        predictions = self.network.predict_images(video)

        return predictions

    def make_predictions_for_datasets(self):
        start_time = datetime.now()

        self.make_predictions_for_dataset('Train', self.train_folder)
        self.make_predictions_for_dataset('Validation', self.validation_folder)
        self.make_predictions_for_dataset('Test', self.test_folder)
        self.make_predictions_for_dataset('VOC', self.voc_folder)

        end_time = datetime.now()
        full_time = end_time - start_time

        print('==========================================================')
        print('Time: {}'.format(full_time))
        print('==========================================================')

    def make_predictions_for_dataset(self, dataset, dataset_path):
        print('==========================================================')
        print(dataset)
        print('==========================================================')
        self.make_predictions_for_folder(dataset_path)

    def make_predictions_for_folder(self, path):
        for dirpath, dirnames, filenames in os.walk(path):
            image_files = map(lambda filename: os.path.join(dirpath, filename),
                              filenames)

            for image_file in image_files:
                image_type = imghdr.what(image_file)
                if not image_type:
                    continue

                prediction = self.predict(image_file)

                if np.sum(prediction) != 0:
                    print('.')

                for el in prediction:
                    if np.sum(el) != 0:
                            print(el)

    def draw_rectangles(self, image, original_size, lables, all_boxes=False):
        fig, ax = plt.subplots(1)

        ax.imshow(np.squeeze(image))

        for label in lables:
            if all_boxes is False:
                print(label[4])

            x = label[0] * original_size[0]
            y = label[1] * original_size[1]
            w = (label[2] - label[0]) * original_size[0]
            h = (label[3] - label[1]) * original_size[1]

            rect = Rectangle((x, y), w, h, linewidth=1,
                             edgecolor='r', facecolor='none')

            ax.add_patch(rect)

        plt.show()

    def draw_grid(self, image, predict):
        predict_table = PrettyTable()

        for p in predict:
            predict_table.add_row(p)
        print(predict)
        print(predict_table)

        fig, ax = plt.subplots(1)
        dx = int(self.image_size / self.grid_size)

        grid_color = [0, 0, 0]

        image[0][:, ::dx, :] = grid_color
        image[0][::dx, :, :] = grid_color
        ax.imshow(image[0])

        plt.show()

if __name__ == '__main__':
    with open('./config.json') as config_file:
        config = json.load(config_file)

    predict = Predict(config)

    predict.make_predictions_for_datasets()
<commit_msg>Correct mistake in dataset name<commit_after>import numpy as np
import json
import os
import imghdr
from datetime import datetime
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
from prettytable import PrettyTable

from aovek.network.network import YOLO
from aovek.utils.image_processing import ImageProcessing

class Predict(ImageProcessing):

    def __init__(self, config):
        super().__init__(config)

        self.network = YOLO(config)
        self.network.load_model()

        self.grid_size = config['label_info']['grid_size']

        self.train_folder =\
            config['dataset']['cvpt10']['dataset_images']['train_folder']
        self.validation_folder =\
            config['dataset']['cvpt10']['dataset_images']['validation_folder']
        self.test_folder =\
            config['dataset']['cvpt10']['dataset_images']['test_folder']
        self.voc_folder = config['dataset']['voc']["folder"]

    def predict(self, image_file):
        image, original_size = self.process_image(image_file)
        original_image = self.load_image(image_file)

        predict = self.network.predict_boxes(image)
        predict = self.network.sess_run(predict)

        self.draw_rectangles(original_image, original_size, predict)

        return predict

    def predict_all_boxes(self, image_file):
        image, original_size = self.process_image(image_file)
        original_image = self.load_image(image_file)

        predict = self.network.predict(image)

        self.draw_rectangles(original_image, original_size,
                             predict, all_boxes=True)

        return predict

    def predict_video(self, video):
        predictions = self.network.predict_images(video)

        return predictions

    def make_predictions_for_datasets(self):
        start_time = datetime.now()

        self.make_predictions_for_dataset('Train', self.train_folder)
        self.make_predictions_for_dataset('Validation', self.validation_folder)
        self.make_predictions_for_dataset('Test', self.test_folder)
        self.make_predictions_for_dataset('VOC', self.voc_folder)

        end_time = datetime.now()
        full_time = end_time - start_time

        print('==========================================================')
        print('Time: {}'.format(full_time))
        print('==========================================================')

    def make_predictions_for_dataset(self, dataset, dataset_path):
        print('==========================================================')
        print(dataset)
        print('==========================================================')
        self.make_predictions_for_folder(dataset_path)

    def make_predictions_for_folder(self, path):
        for dirpath, dirnames, filenames in os.walk(path):
            image_files = map(lambda filename: os.path.join(dirpath, filename),
                              filenames)

            for image_file in image_files:
                image_type = imghdr.what(image_file)
                if not image_type:
                    continue

                prediction = self.predict(image_file)

                if np.sum(prediction) != 0:
                    print('.')

                for el in prediction:
                    if np.sum(el) != 0:
                            print(el)

    def draw_rectangles(self, image, original_size, lables, all_boxes=False):
        fig, ax = plt.subplots(1)

        ax.imshow(np.squeeze(image))

        for label in lables:
            if all_boxes is False:
                print(label[4])

            x = label[0] * original_size[0]
            y = label[1] * original_size[1]
            w = (label[2] - label[0]) * original_size[0]
            h = (label[3] - label[1]) * original_size[1]

            rect = Rectangle((x, y), w, h, linewidth=1,
                             edgecolor='r', facecolor='none')

            ax.add_patch(rect)

        plt.show()

    def draw_grid(self, image, predict):
        predict_table = PrettyTable()

        for p in predict:
            predict_table.add_row(p)
        print(predict)
        print(predict_table)

        fig, ax = plt.subplots(1)
        dx = int(self.image_size / self.grid_size)

        grid_color = [0, 0, 0]

        image[0][:, ::dx, :] = grid_color
        image[0][::dx, :, :] = grid_color
        ax.imshow(image[0])

        plt.show()

if __name__ == '__main__':
    with open('./config.json') as config_file:
        config = json.load(config_file)

    predict = Predict(config)

    predict.make_predictions_for_datasets()
<|endoftext|>


<commit_before>"""Internal representation of email messages."""

from builtins import str
from builtins import set
from builtins import list
from builtins import dict
from builtins import object

import re
import email
import functools
import ipaddress
import email.utils
import html.parser
import collections
import email.header
import email.mime.base
import email.mime.text
import email.mime.multipart

from future.utils import PY3

import pad
import pad.context
from pad.received_parser import ReceivedParser

URL_RE = re.compile(r"""
(
    \b                      # the preceding character must not be alphanumeric
    (?:
        (?:
            (?:https? | ftp)  # capture the protocol
            ://               # skip the boilerplate
        )|
        (?= ftp\.[^\.\s<>"'\x7f-\xff] )|  # allow the protocol to be missing,
        (?= www\.[^\.\s<>"'\x7f-\xff] )   # but only if the rest of the url
                                          # starts with "www.x" or "ftp.x"
    )
    (?:[^\s<>"'\x7f-\xff]+)  # capture the guts
)
""", re.VERBOSE)

IPFRE = re.compile(r"[\[ \(]{1}[a-fA-F\d\.\:]{7,}?[\] \n;\)]{1}")

STRICT_CHARSETS = frozenset(("quopri-codec", "quopri", "quoted-printable",
                             "quotedprintable"))

class _ParseHTML(html.parser.HTMLParser):
    """Extract data from HTML parts."""

    def __init__(self, collector):
        try:
            html.parser.HTMLParser.__init__(self, convert_charrefs=False)
        except TypeError:
            # Python 2 does not have the convert_charrefs argument.
            html.parser.HTMLParser.__init__(self)
        self.reset()
        self.collector = collector

    def handle_data(self, data):
        """Keep track of the data."""
        data = data.strip()
        if data:
            self.collector.append(data)

class _Headers(collections.defaultdict):
    """Like a defaultdict that returns an empty list by default, but the
    keys are all case insensitive.
    """

    def __init__(self):
        collections.defaultdict.__init__(self, list)

    def get(self, k, d=None):
        return super(_Headers, self).get(k.lower(), d)

    def __setitem__(self, key, value):
        super(_Headers, self).__setitem__(key.lower(), value)

    def __getitem__(self, key):
        return super(_Headers, self).__getitem__(key.lower())

    def __contains__(self, key):
        return super(_Headers, self).__contains__(key.lower())

class _memoize(object):
    """Memoize the result of the function in a cache. Used to prevent
    superfluous parsing of headers.
    """

    def __init__(self, cache_name):
        self._cache_name = cache_name

    def __call__(self, func):
        """Check if the information is available in a cache, if not call the
        function and cache the result.
        """
        @functools.wraps(func)
        def wrapped_func(fself, name):
            cache = getattr(fself, self._cache_name)
            result = cache.get(name)
            if result is None:
                result = func(fself, name)
                cache[name] = result
            return result

        return wrapped_func

DEFAULT_SENDERH = (
    "X-Sender", "X-Envelope-From", "Envelope-Sender", "Return-Path", "From"
)

class Message(pad.context.MessageContext):
    """Internal representation of an email message. Used for rule matching."""

    def __init__(self, global_context, raw_msg):
        """Parse the message, extracts and decode all headers and all
        text parts.
        """
        super(Message, self).__init__(global_context)
        self.raw_msg = self.translate_line_breaks(raw_msg)
        self.msg = email.message_from_string(self.raw_msg)
        self.headers = _Headers()
        self.raw_headers = _Headers()
        self.addr_headers = _Headers()
        self.name_headers = _Headers()
        self.mime_headers = _Headers()
        self.received_headers = list()
        self.raw_mime_headers = _Headers()
        self.header_ips = _Headers()
        self.text = ""
        self.raw_text = ""
        self.uri_list = set()
        self.score = 0
        self.rules_checked = dict()
        self.interpolate_data = dict()
        self.plugin_tags = dict()
        # Data
        self.sender_address = ""
        self.hostname_with_ip = list()
        self.internal_relays = []
        self.external_relays = []
        self.last_internal_relay_index = 0
        self.last_trusted_relay_index = 0
        self.trusted_relays = []
        self.untrusted_relays = []
        self._parse_message()
        self._hook_parsed_metadata()

    def clear_matches(self):
        """Clear any already checked rules."""
        self.rules_checked = dict()
        self.score = 0

    @staticmethod
    def translate_line_breaks(text):
        """Convert any EOL style to Linux EOL."""
        text = text.replace("\r\n", "\n")
        return text.replace("\r", "\n")

    @staticmethod
    def normalize_html_part(payload):
        """Strip all HTML tags."""
        data = list()
        stripper = _ParseHTML(data)
        try:
            stripper.feed(payload)
        except (UnicodeDecodeError, html.parser.HTMLParseError):
            # We can't parse the HTML, so just strip it.  This is still
            # better than including generic HTML/CSS text.
            pass
        return data

    @staticmethod
    def _decode_header(header):
        """Decodes an email header and returns it as a string. Any  parts of
        the header that cannot be decoded are simply ignored.
        """
        parts = list()
        try:
            decoded_header = email.header.decode_header(header)
        except (ValueError, email.header.HeaderParseError):
            return

        for value, encoding in decoded_header:
            if encoding:
                try:
                    parts.append(value.decode(encoding, "ignore"))
                except (LookupError, UnicodeError, AssertionError):
                    continue
            else:
                if PY3:
                    parts.append(value)
                else:
                    parts.append(value.decode("utf-8", "ignore"))
        return "".join(parts)

    def get_raw_header(self, header_name):
        """Get a list of raw headers with this name."""
        # This is just for consistencies, the raw headers should have been
        # parsed together with the message.
        return self.raw_headers.get(header_name, list())

    @_memoize("headers")
    def get_decoded_header(self, header_name):
        """Get a list of decoded headers with this name."""
        values = list()
        for value in self.get_raw_header(header_name):
            values.append(self._decode_header(value))
        return values

    def get_untrusted_ips(self):
        """Returns the untrusted IPs based on the users trusted
        network settings.

        :return: A list of `ipaddress.ip_address`.
        """
        ips = [ip for ip in self.get_header_ips()
               if ip not in self.ctxt.networks.trusted]
        return ips

    def get_header_ips(self):
        values = list()
        for header in self.received_headers:
            values.append(ipaddress.ip_address(header["ip"]))
        return values

    @_memoize("addr_headers")
    def get_addr_header(self, header_name):
        """Get a list of the first addresses from this header."""
        values = list()
        for value in self.get_decoded_header(header_name):
            for dummy, addr in email.utils.getaddresses([value]):
                if addr:
                    values.append(addr)
                    break
        return values

    @_memoize("name_headers")
    def get_name_header(self, header_name):
        """Get a list of the first names from this header."""
        values = list()
        for value in self.get_decoded_header(header_name):
            for name, dummy in email.utils.getaddresses([value]):
                if name:
                    values.append(name)
                    break
        return values

    def get_raw_mime_header(self, header_name):
        """Get a list of raw MIME headers with this name."""
        # This is just for consistencies, the raw headers should have been
        # parsed together with the message.
        return self.raw_mime_headers.get(header_name, list())

    @_memoize("mime_headers")
    def get_decoded_mime_header(self, header_name):
        """Get a list of raw MIME headers with this name."""
        values = list()
        for value in self.get_raw_mime_header(header_name):
            values.append(self._decode_header(value))
        return values

    def iter_decoded_headers(self):
        """Iterate through all the decoded headers.

        Yields strings like "<header_name>: <header_value>"
        """
        for header_name in self.raw_headers:
            for value in self.get_decoded_header(header_name):
                yield "%s: %s" % (header_name, value)

    def _create_plugin_tags(self, header):
        for key, value in header.items():
            self.plugin_tags[key.upper()] = value

    def _parse_sender(self):
        """Extract the envelope sender from the message."""

        always_trust_envelope_from = self.ctxt.conf[
            'always_trust_envelope_sender']
        headers = self.ctxt.conf["envelope_sender_header"] or DEFAULT_SENDERH

        if self.external_relays:
            sender = self.external_relays[0].get("envfrom").strip()
            if sender:
                self.sender_address = sender
                return
        else:
            if self.trusted_relays and not always_trust_envelope_from:
                return

            for sender_header in headers:
                try:
                    sender = self.get_addr_header(sender_header)[0]
                except IndexError:
                    continue
                if sender:
                    self.sender_address = sender.strip()
                    self.ctxt.log.debug("Using %s as sender: %s",
                                        sender_header, sender)
                    return
        return

    def _parse_relays(self, relays):
        """Walks though a relays list to extract
        [un]trusted/internal/external relays"""
        is_trusted = True
        is_internal = True
        found_msa = False

        for position, relay in enumerate(relays):
            relay['msa'] = 0
            if relay['ip']:
                ip = ipaddress.ip_address(str(relay['ip']))
                in_internal = ip in self.ctxt.networks.internal
                in_trusted = ip in self.ctxt.networks.trusted
                in_msa = ip in self.ctxt.networks.msa
                has_auth = relay.get("auth", None)
                if is_trusted and not found_msa:
                    if self.ctxt.networks.configured:
                        if not in_trusted:
                            is_trusted = False
                            is_internal = False

                        if not in_internal:
                            is_internal = False

                        if has_auth and (is_internal or is_trusted) and in_msa:
                            relay['msa'] = 1
                            found_msa = True

                    elif not ip.is_private and not has_auth:
                            is_trusted = False
                            is_internal = False

                relay['intl'] = 1 if is_internal else 0
                if is_internal:
                    self.internal_relays.append(relay)
                    self.last_internal_relay_index = position
                else:
                    self.external_relays.append(relay)

                if is_trusted:
                    self.trusted_relays.append(relay)
                    self.last_trusted_relay_index = position
                else:
                    self.untrusted_relays.append(relay)
        tag_template = ("[ ip={ip} rdns={rdns} helo={helo} by={by} "
                        "ident={ident} intl={intl} id={id} auth={auth} "
                        "msa={msa} ]")

        relays_tags = {
            "RELAYSTRUSTED": " ".join([tag_template.format(**x)
                                       for x in self.trusted_relays]),
            "RELAYSUNTRUSTED": " ".join([tag_template.format(**x)
                                         for x in self.untrusted_relays]),
            "relaysinternal": " ".join([tag_template.format(**x)
                                        for x in self.internal_relays]),
            "relaysexternal": " ".join([tag_template.format(**x)
                                        for x in self.external_relays]),
            # "lastexternalip": self.external_relays[-1]['ip'],
            # "lastexternalrdns": self.external_relays[-1]['rdns'],
            # "lastexternalhelo": self.external_relays[-1]['helo']
        }

        self._create_plugin_tags(relays_tags)

    def _parse_message(self):
        """Parse the message."""
        self._hook_check_start()
        # Dump the message raw headers
        for name, raw_value in self.msg._headers:
            self.raw_headers[name].append(raw_value)

        # XXX This is strange, but it's what SA does.
        # The body starts with the Subject header(s)
        body = list(self.get_decoded_header("Subject"))
        raw_body = list()
        for payload, part in self._iter_parts(self.msg):
            # Extract any MIME headers
            for name, raw_value in part._headers:
                self.raw_mime_headers[name].append(raw_value)
            text = None
            if payload is not None:
                # this must be a text part
                self.uri_list.update(set(URL_RE.findall(payload)))
                if part.get_content_subtype() == "html":
                    text = self.normalize_html_part(payload.replace("\n", " "))
                    text = " ".join(text)
                    body.append(text)
                    raw_body.append(payload)
                else:
                    text = payload.replace("\n", " ")
                    body.append(text)
                    raw_body.append(payload)
            self._hook_extract_metadata(payload, text, part)
        self.text = " ".join(body)
        self.raw_text = "\n".join(raw_body)
        self._parse_sender()
        received_headers = self.get_decoded_header("Received")
        for header in self.ctxt.conf["originating_ip_headers"]:
            received_headers.extend(self.get_decoded_header(header))
        received_obj = ReceivedParser(received_headers,
                                      self.ctxt.conf["originating_ip_headers"])
        self.received_headers = received_obj.received
        self._parse_relays(self.received_headers)

        try:
            self._create_plugin_tags(self.received_headers[0])
        except IndexError:
            pass

        for header in self.received_headers:
            self.hostname_with_ip.append((header["rdns"], header["ip"]))

    @staticmethod
    def _iter_parts(msg):
        """Extract and decode the text parts from the parsed email message.
        For non-text parts the payload will be None.

        Yields (payload, part)
        """
        for part in msg.walk():
            if part.get_content_maintype() == "text":
                payload = part.get_payload(decode=True)

                charset = part.get_content_charset()
                errors = "ignore"
                if not charset:
                    charset = "ascii"
                elif charset.lower().replace("_", "-") in STRICT_CHARSETS:
                    errors = "strict"
                try:
                    payload = payload.decode(charset, errors)
                except (LookupError, UnicodeError, AssertionError):
                    try:
                        payload = payload.decode("ascii", "ignore")
                    except UnicodeError:
                        continue
                yield payload, part
            else:
                yield None, part
<commit_msg>Refs #40 Add tags for last external relay<commit_after>"""Internal representation of email messages."""

from builtins import str
from builtins import set
from builtins import list
from builtins import dict
from builtins import object

import re
import email
import functools
import ipaddress
import email.utils
import html.parser
import collections
import email.header
import email.mime.base
import email.mime.text
import email.mime.multipart

from future.utils import PY3

import pad
import pad.context
from pad.received_parser import ReceivedParser

URL_RE = re.compile(r"""
(
    \b                      # the preceding character must not be alphanumeric
    (?:
        (?:
            (?:https? | ftp)  # capture the protocol
            ://               # skip the boilerplate
        )|
        (?= ftp\.[^\.\s<>"'\x7f-\xff] )|  # allow the protocol to be missing,
        (?= www\.[^\.\s<>"'\x7f-\xff] )   # but only if the rest of the url
                                          # starts with "www.x" or "ftp.x"
    )
    (?:[^\s<>"'\x7f-\xff]+)  # capture the guts
)
""", re.VERBOSE)

IPFRE = re.compile(r"[\[ \(]{1}[a-fA-F\d\.\:]{7,}?[\] \n;\)]{1}")

STRICT_CHARSETS = frozenset(("quopri-codec", "quopri", "quoted-printable",
                             "quotedprintable"))

class _ParseHTML(html.parser.HTMLParser):
    """Extract data from HTML parts."""

    def __init__(self, collector):
        try:
            html.parser.HTMLParser.__init__(self, convert_charrefs=False)
        except TypeError:
            # Python 2 does not have the convert_charrefs argument.
            html.parser.HTMLParser.__init__(self)
        self.reset()
        self.collector = collector

    def handle_data(self, data):
        """Keep track of the data."""
        data = data.strip()
        if data:
            self.collector.append(data)

class _Headers(collections.defaultdict):
    """Like a defaultdict that returns an empty list by default, but the
    keys are all case insensitive.
    """

    def __init__(self):
        collections.defaultdict.__init__(self, list)

    def get(self, k, d=None):
        return super(_Headers, self).get(k.lower(), d)

    def __setitem__(self, key, value):
        super(_Headers, self).__setitem__(key.lower(), value)

    def __getitem__(self, key):
        return super(_Headers, self).__getitem__(key.lower())

    def __contains__(self, key):
        return super(_Headers, self).__contains__(key.lower())

class _memoize(object):
    """Memoize the result of the function in a cache. Used to prevent
    superfluous parsing of headers.
    """

    def __init__(self, cache_name):
        self._cache_name = cache_name

    def __call__(self, func):
        """Check if the information is available in a cache, if not call the
        function and cache the result.
        """
        @functools.wraps(func)
        def wrapped_func(fself, name):
            cache = getattr(fself, self._cache_name)
            result = cache.get(name)
            if result is None:
                result = func(fself, name)
                cache[name] = result
            return result

        return wrapped_func

DEFAULT_SENDERH = (
    "X-Sender", "X-Envelope-From", "Envelope-Sender", "Return-Path", "From"
)

class Message(pad.context.MessageContext):
    """Internal representation of an email message. Used for rule matching."""

    def __init__(self, global_context, raw_msg):
        """Parse the message, extracts and decode all headers and all
        text parts.
        """
        super(Message, self).__init__(global_context)
        self.raw_msg = self.translate_line_breaks(raw_msg)
        self.msg = email.message_from_string(self.raw_msg)
        self.headers = _Headers()
        self.raw_headers = _Headers()
        self.addr_headers = _Headers()
        self.name_headers = _Headers()
        self.mime_headers = _Headers()
        self.received_headers = list()
        self.raw_mime_headers = _Headers()
        self.header_ips = _Headers()
        self.text = ""
        self.raw_text = ""
        self.uri_list = set()
        self.score = 0
        self.rules_checked = dict()
        self.interpolate_data = dict()
        self.plugin_tags = dict()
        # Data
        self.sender_address = ""
        self.hostname_with_ip = list()
        self.internal_relays = []
        self.external_relays = []
        self.last_internal_relay_index = 0
        self.last_trusted_relay_index = 0
        self.trusted_relays = []
        self.untrusted_relays = []
        self._parse_message()
        self._hook_parsed_metadata()

    def clear_matches(self):
        """Clear any already checked rules."""
        self.rules_checked = dict()
        self.score = 0

    @staticmethod
    def translate_line_breaks(text):
        """Convert any EOL style to Linux EOL."""
        text = text.replace("\r\n", "\n")
        return text.replace("\r", "\n")

    @staticmethod
    def normalize_html_part(payload):
        """Strip all HTML tags."""
        data = list()
        stripper = _ParseHTML(data)
        try:
            stripper.feed(payload)
        except (UnicodeDecodeError, html.parser.HTMLParseError):
            # We can't parse the HTML, so just strip it.  This is still
            # better than including generic HTML/CSS text.
            pass
        return data

    @staticmethod
    def _decode_header(header):
        """Decodes an email header and returns it as a string. Any  parts of
        the header that cannot be decoded are simply ignored.
        """
        parts = list()
        try:
            decoded_header = email.header.decode_header(header)
        except (ValueError, email.header.HeaderParseError):
            return

        for value, encoding in decoded_header:
            if encoding:
                try:
                    parts.append(value.decode(encoding, "ignore"))
                except (LookupError, UnicodeError, AssertionError):
                    continue
            else:
                if PY3:
                    parts.append(value)
                else:
                    parts.append(value.decode("utf-8", "ignore"))
        return "".join(parts)

    def get_raw_header(self, header_name):
        """Get a list of raw headers with this name."""
        # This is just for consistencies, the raw headers should have been
        # parsed together with the message.
        return self.raw_headers.get(header_name, list())

    @_memoize("headers")
    def get_decoded_header(self, header_name):
        """Get a list of decoded headers with this name."""
        values = list()
        for value in self.get_raw_header(header_name):
            values.append(self._decode_header(value))
        return values

    def get_untrusted_ips(self):
        """Returns the untrusted IPs based on the users trusted
        network settings.

        :return: A list of `ipaddress.ip_address`.
        """
        ips = [ip for ip in self.get_header_ips()
               if ip not in self.ctxt.networks.trusted]
        return ips

    def get_header_ips(self):
        values = list()
        for header in self.received_headers:
            values.append(ipaddress.ip_address(header["ip"]))
        return values

    @_memoize("addr_headers")
    def get_addr_header(self, header_name):
        """Get a list of the first addresses from this header."""
        values = list()
        for value in self.get_decoded_header(header_name):
            for dummy, addr in email.utils.getaddresses([value]):
                if addr:
                    values.append(addr)
                    break
        return values

    @_memoize("name_headers")
    def get_name_header(self, header_name):
        """Get a list of the first names from this header."""
        values = list()
        for value in self.get_decoded_header(header_name):
            for name, dummy in email.utils.getaddresses([value]):
                if name:
                    values.append(name)
                    break
        return values

    def get_raw_mime_header(self, header_name):
        """Get a list of raw MIME headers with this name."""
        # This is just for consistencies, the raw headers should have been
        # parsed together with the message.
        return self.raw_mime_headers.get(header_name, list())

    @_memoize("mime_headers")
    def get_decoded_mime_header(self, header_name):
        """Get a list of raw MIME headers with this name."""
        values = list()
        for value in self.get_raw_mime_header(header_name):
            values.append(self._decode_header(value))
        return values

    def iter_decoded_headers(self):
        """Iterate through all the decoded headers.

        Yields strings like "<header_name>: <header_value>"
        """
        for header_name in self.raw_headers:
            for value in self.get_decoded_header(header_name):
                yield "%s: %s" % (header_name, value)

    def _create_plugin_tags(self, header):
        for key, value in header.items():
            self.plugin_tags[key.upper()] = value

    def _parse_sender(self):
        """Extract the envelope sender from the message."""

        always_trust_envelope_from = self.ctxt.conf[
            'always_trust_envelope_sender']
        headers = self.ctxt.conf["envelope_sender_header"] or DEFAULT_SENDERH

        if self.external_relays:
            sender = self.external_relays[0].get("envfrom").strip()
            if sender:
                self.sender_address = sender
                return
        else:
            if self.trusted_relays and not always_trust_envelope_from:
                return

            for sender_header in headers:
                try:
                    sender = self.get_addr_header(sender_header)[0]
                except IndexError:
                    continue
                if sender:
                    self.sender_address = sender.strip()
                    self.ctxt.log.debug("Using %s as sender: %s",
                                        sender_header, sender)
                    return
        return

    def _parse_relays(self, relays):
        """Walks though a relays list to extract
        [un]trusted/internal/external relays"""
        is_trusted = True
        is_internal = True
        found_msa = False

        for position, relay in enumerate(relays):
            relay['msa'] = 0
            if relay['ip']:
                ip = ipaddress.ip_address(str(relay['ip']))
                in_internal = ip in self.ctxt.networks.internal
                in_trusted = ip in self.ctxt.networks.trusted
                in_msa = ip in self.ctxt.networks.msa
                has_auth = relay.get("auth", None)
                if is_trusted and not found_msa:
                    if self.ctxt.networks.configured:
                        if not in_trusted:
                            is_trusted = False
                            is_internal = False

                        if not in_internal:
                            is_internal = False

                        if has_auth and (is_internal or is_trusted) and in_msa:
                            relay['msa'] = 1
                            found_msa = True

                    elif not ip.is_private and not has_auth:
                            is_trusted = False
                            is_internal = False

                relay['intl'] = 1 if is_internal else 0
                if is_internal:
                    self.internal_relays.append(relay)
                    self.last_internal_relay_index = position
                else:
                    self.external_relays.append(relay)

                if is_trusted:
                    self.trusted_relays.append(relay)
                    self.last_trusted_relay_index = position
                else:
                    self.untrusted_relays.append(relay)
        tag_template = ("[ ip={ip} rdns={rdns} helo={helo} by={by} "
                        "ident={ident} intl={intl} id={id} auth={auth} "
                        "msa={msa} ]")

        relays_tags = {
            "RELAYSTRUSTED": " ".join([tag_template.format(**x)
                                       for x in self.trusted_relays]),
            "RELAYSUNTRUSTED": " ".join([tag_template.format(**x)
                                         for x in self.untrusted_relays]),
            "RELAYSINTERNAL": " ".join([tag_template.format(**x)
                                        for x in self.internal_relays]),
            "RELAYSEXTERNAL": " ".join([tag_template.format(**x)
                                        for x in self.external_relays]),
        }
        if self.external_relays:
            relays_tags.update({
                "LASTEXTERNALIP": self.external_relays[-1]['ip'],
                "LASTEXTERNALRDNS": self.external_relays[-1]['rdns'],
                "LASTEXTERNALHELO": self.external_relays[-1]['helo']
            })

        self._create_plugin_tags(relays_tags)

    def _parse_message(self):
        """Parse the message."""
        self._hook_check_start()
        # Dump the message raw headers
        for name, raw_value in self.msg._headers:
            self.raw_headers[name].append(raw_value)

        # XXX This is strange, but it's what SA does.
        # The body starts with the Subject header(s)
        body = list(self.get_decoded_header("Subject"))
        raw_body = list()
        for payload, part in self._iter_parts(self.msg):
            # Extract any MIME headers
            for name, raw_value in part._headers:
                self.raw_mime_headers[name].append(raw_value)
            text = None
            if payload is not None:
                # this must be a text part
                self.uri_list.update(set(URL_RE.findall(payload)))
                if part.get_content_subtype() == "html":
                    text = self.normalize_html_part(payload.replace("\n", " "))
                    text = " ".join(text)
                    body.append(text)
                    raw_body.append(payload)
                else:
                    text = payload.replace("\n", " ")
                    body.append(text)
                    raw_body.append(payload)
            self._hook_extract_metadata(payload, text, part)
        self.text = " ".join(body)
        self.raw_text = "\n".join(raw_body)
        self._parse_sender()
        received_headers = self.get_decoded_header("Received")
        for header in self.ctxt.conf["originating_ip_headers"]:
            received_headers.extend(self.get_decoded_header(header))
        received_obj = ReceivedParser(received_headers,
                                      self.ctxt.conf["originating_ip_headers"])
        self.received_headers = received_obj.received
        self._parse_relays(self.received_headers)

        try:
            self._create_plugin_tags(self.received_headers[0])
        except IndexError:
            pass

        for header in self.received_headers:
            self.hostname_with_ip.append((header["rdns"], header["ip"]))

    @staticmethod
    def _iter_parts(msg):
        """Extract and decode the text parts from the parsed email message.
        For non-text parts the payload will be None.

        Yields (payload, part)
        """
        for part in msg.walk():
            if part.get_content_maintype() == "text":
                payload = part.get_payload(decode=True)

                charset = part.get_content_charset()
                errors = "ignore"
                if not charset:
                    charset = "ascii"
                elif charset.lower().replace("_", "-") in STRICT_CHARSETS:
                    errors = "strict"
                try:
                    payload = payload.decode(charset, errors)
                except (LookupError, UnicodeError, AssertionError):
                    try:
                        payload = payload.decode("ascii", "ignore")
                    except UnicodeError:
                        continue
                yield payload, part
            else:
                yield None, part
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import logging

import typing
from typing import Optional, Any
from typing import Text
from typing import Tuple

from rasa_nlu import utils, config
from rasa_nlu.components import ComponentBuilder
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.model import Interpreter
from rasa_nlu.model import Trainer
from rasa_nlu.training_data import load_data

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa_nlu.persistor import Persistor

def create_argument_parser():
    parser = argparse.ArgumentParser(
            description='train a custom language parser')

    parser.add_argument('-o', '--path',
                        default=None,
                        help="Path where model files will be saved")

    group = parser.add_mutually_exclusive_group(required=True)

    group.add_argument('-d', '--data',
                       default=None,
                       help="Location of the training data. For JSON and "
                            "markdown data, this can either be a single file "
                            "or a directory containing multiple training "
                            "data files.")

    group.add_argument('-u', '--url',
                       default=None,
                       help="URL of the server from which to receive the "
                            "training data")

    parser.add_argument('--token',
                        default=None,
                        help="Optional authentication string that might be "
                             "needed to retrieve training data from a URL. "
                             "The token gets appended to the query string "
                             "with `?token=<TOKEN>`.")

    parser.add_argument('-c', '--config',
                        required=True,
                        help="Rasa NLU configuration file")

    parser.add_argument('-t', '--num_threads',
                        default=None,
                        type=int,
                        help="Number of threads to use during model training")

    parser.add_argument('--project',
                        default=None,
                        help="Project this model belongs to.")

    parser.add_argument('--fixed_model_name',
                        help="If present, a model will always be persisted "
                             "in the specified directory instead of creating "
                             "a folder like 'model_20171020-160213'")

    parser.add_argument('--storage',
                        help='Set the remote location where models are stored. '
                             'E.g. on AWS. If nothing is configured, the '
                             'server will only serve the models that are '
                             'on disk in the configured `path`.')

    utils.add_logging_option_arguments(parser)
    return parser

class TrainingException(Exception):
    """Exception wrapping lower level exceptions that may happen while training

      Attributes:
          failed_target_project -- name of the failed project
          message -- explanation of why the request is invalid
      """

    def __init__(self, failed_target_project=None, exception=None):
        self.failed_target_project = failed_target_project
        if exception:
            self.message = exception.args[0]

    def __str__(self):
        return self.message

def create_persistor(persistor):
    # type: (Optional[Text]) -> Optional[Persistor]
    """Create a remote persistor to store the model if configured."""

    if persistor is not None:
        from rasa_nlu.persistor import get_persistor
        return get_persistor(persistor)
    else:
        return None

def do_train_in_worker(config,  # type: RasaNLUModelConfig
                       data,  # type: Text
                       path,  # type: Text
                       project=None,  # type: Optional[Text]
                       fixed_model_name=None,  # type: Optional[Text]
                       storage=None,  # type: Text
                       component_builder=None
                       # type: Optional[ComponentBuilder]
                       ):
    # type: (...) -> Text
    """Loads the trainer and the data and runs the training in a worker."""

    try:
        _, _, persisted_path = do_train(config, data, path, project,
                                        fixed_model_name, storage,
                                        component_builder)
        return persisted_path
    except Exception as e:
        logger.exception("Failed to train project '{}'.".format(project))
        raise TrainingException(project, e)

def do_train(cfg,  # type: RasaNLUModelConfig
             data=None,  # type: Text
             path=None,  # type: Text
             project=None,  # type: Optional[Text]
             fixed_model_name=None,  # type: Optional[Text]
             storage=None,  # type: Text
             component_builder=None,  # type: Optional[ComponentBuilder]
             token=None,  # type: Optional[Text]
             **kwargs  # type: Any
             ):
    # type: (...) -> Tuple[Trainer, Interpreter, Text]
    """Loads the trainer and the data and runs the training of the model."""

    # Ensure we are training a model that we can save in the end
    # WARN: there is still a race condition if a model with the same name is
    # trained in another subprocess
    trainer = Trainer(cfg, component_builder)
    persistor = create_persistor(storage)
    training_data = load_data(data, cfg.language, token)
    interpreter = trainer.train(training_data, **kwargs)

    if path:
        persisted_path = trainer.persist(path,
                                         persistor,
                                         project,
                                         fixed_model_name)
    else:
        persisted_path = None

    return trainer, interpreter, persisted_path

if __name__ == '__main__':
    cmdline_args = create_argument_parser().parse_args()

    utils.configure_colored_logging(cmdline_args.loglevel)

    do_train(config.load(cmdline_args.config),
             cmdline_args.data or cmdline_args.url,
             cmdline_args.path,
             cmdline_args.project,
             cmdline_args.fixed_model_name,
             cmdline_args.storage,
             num_threads=cmdline_args.num_threads,
             token=cmdline_args.token)
    logger.info("Finished training")
<commit_msg>fix default kwargs in do_train<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import logging

import typing
from typing import Optional, Any
from typing import Text
from typing import Tuple

from rasa_nlu import utils, config
from rasa_nlu.components import ComponentBuilder
from rasa_nlu.config import RasaNLUModelConfig
from rasa_nlu.model import Interpreter
from rasa_nlu.model import Trainer
from rasa_nlu.training_data import load_data

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa_nlu.persistor import Persistor

def create_argument_parser():
    parser = argparse.ArgumentParser(
            description='train a custom language parser')

    parser.add_argument('-o', '--path',
                        default=None,
                        help="Path where model files will be saved")

    group = parser.add_mutually_exclusive_group(required=True)

    group.add_argument('-d', '--data',
                       default=None,
                       help="Location of the training data. For JSON and "
                            "markdown data, this can either be a single file "
                            "or a directory containing multiple training "
                            "data files.")

    group.add_argument('-u', '--url',
                       default=None,
                       help="URL of the server from which to receive the "
                            "training data")

    parser.add_argument('--token',
                        default=None,
                        help="Optional authentication string that might be "
                             "needed to retrieve training data from a URL. "
                             "The token gets appended to the query string "
                             "with `?token=<TOKEN>`.")

    parser.add_argument('-c', '--config',
                        required=True,
                        help="Rasa NLU configuration file")

    parser.add_argument('-t', '--num_threads',
                        default=None,
                        type=int,
                        help="Number of threads to use during model training")

    parser.add_argument('--project',
                        default=None,
                        help="Project this model belongs to.")

    parser.add_argument('--fixed_model_name',
                        help="If present, a model will always be persisted "
                             "in the specified directory instead of creating "
                             "a folder like 'model_20171020-160213'")

    parser.add_argument('--storage',
                        help='Set the remote location where models are stored. '
                             'E.g. on AWS. If nothing is configured, the '
                             'server will only serve the models that are '
                             'on disk in the configured `path`.')

    utils.add_logging_option_arguments(parser)
    return parser

class TrainingException(Exception):
    """Exception wrapping lower level exceptions that may happen while training

      Attributes:
          failed_target_project -- name of the failed project
          message -- explanation of why the request is invalid
      """

    def __init__(self, failed_target_project=None, exception=None):
        self.failed_target_project = failed_target_project
        if exception:
            self.message = exception.args[0]

    def __str__(self):
        return self.message

def create_persistor(persistor):
    # type: (Optional[Text]) -> Optional[Persistor]
    """Create a remote persistor to store the model if configured."""

    if persistor is not None:
        from rasa_nlu.persistor import get_persistor
        return get_persistor(persistor)
    else:
        return None

def do_train_in_worker(config,  # type: RasaNLUModelConfig
                       data,  # type: Text
                       path,  # type: Text
                       project=None,  # type: Optional[Text]
                       fixed_model_name=None,  # type: Optional[Text]
                       storage=None,  # type: Text
                       component_builder=None
                       # type: Optional[ComponentBuilder]
                       ):
    # type: (...) -> Text
    """Loads the trainer and the data and runs the training in a worker."""

    try:
        _, _, persisted_path = do_train(config, data, path, project,
                                        fixed_model_name, storage,
                                        component_builder)
        return persisted_path
    except Exception as e:
        logger.exception("Failed to train project '{}'.".format(project))
        raise TrainingException(project, e)

def do_train(cfg,  # type: RasaNLUModelConfig
             data,  # type: Text
             path=None,  # type: Optional[Text]
             project=None,  # type: Optional[Text]
             fixed_model_name=None,  # type: Optional[Text]
             storage=None,  # type: Optional[Text]
             component_builder=None,  # type: Optional[ComponentBuilder]
             token=None,  # type: Optional[Text]
             **kwargs  # type: Any
             ):
    # type: (...) -> Tuple[Trainer, Interpreter, Text]
    """Loads the trainer and the data and runs the training of the model."""

    # Ensure we are training a model that we can save in the end
    # WARN: there is still a race condition if a model with the same name is
    # trained in another subprocess
    trainer = Trainer(cfg, component_builder)
    persistor = create_persistor(storage)
    training_data = load_data(data, cfg.language, token)
    interpreter = trainer.train(training_data, **kwargs)

    if path:
        persisted_path = trainer.persist(path,
                                         persistor,
                                         project,
                                         fixed_model_name)
    else:
        persisted_path = None

    return trainer, interpreter, persisted_path

if __name__ == '__main__':
    cmdline_args = create_argument_parser().parse_args()

    utils.configure_colored_logging(cmdline_args.loglevel)

    do_train(config.load(cmdline_args.config),
             cmdline_args.data or cmdline_args.url,
             cmdline_args.path,
             cmdline_args.project,
             cmdline_args.fixed_model_name,
             cmdline_args.storage,
             num_threads=cmdline_args.num_threads,
             token=cmdline_args.token)
    logger.info("Finished training")
<|endoftext|>


<commit_before>from random import uniform
from time import sleep
from .Commands import Kwargs
from selenium.webdriver.support.wait import WebDriverWait

# Generate a bunch of decorators for waiting up to 60 seconds.
for __ in range(1, 61):
    exec('''def waits{}(function):
    """
    A decorator for waiting {} second after function execution.  Great for waiting between actions.

    Args:
        function: function

    Returns:
        wait_decorator: function
    """

    def wait_decorator(*args, **kwargs):
        function_result = function(*args, **kwargs)
        sleep({})
        return function_result
    return wait_decorator'''.format(__, __, __))

def randomly_waits(function):
    """
    A decorator for waiting a random amount of time(0.1-3.01 seconds) after function execution.

    Args:
        function: function

    Returns:
        random_wait_decorator
    """

    def random_wait_decorator(*args, **kwargs):
        # Execute function and grab result
        function_result = function(*args, **kwargs)
        # Sleep
        sleep(uniform(0.99, 3.01))
        return function_result
    return random_wait_decorator

@randomly_waits
def human_fill(element, text):
    """
    Send keys to an element and wait a random amount of time afterwards.

    Args:
        element:
        text:

    Returns:

    """

    return element.send_keys(text)

@randomly_waits
def human_click(element):
    """
    Click on an element and wait a random amount of time afterwards.

    Args:
        element:

    Returns:

    """

    return element.click()

def has_kwargs(function):
    """
    Decorator for passing **kwargs with your *args through the use of
    the Kwargs object.  Note that only 1 Kwargs object can be passed
    with the *args to the decorated function.  Any additional Kwargs
    objects will be ignored.

    Args:
        function: function

    Returns:
        kwargsable: function
    """

    # Define a local function within has_kwargs
    def kwargsable(*args, **kwargs):
        args = list(args)
        # Extract kwargs from args.
        d_args = [thing for thing in args if type(thing) == Kwargs]

        # Check if default kwargs should be used.
        kwargs_len = len(kwargs)
        if kwargs_len == 0:
            # If there is a Kwargs instance in the list then process the kwargs
            try:
                d_args = d_args[0]  # Pop the Kwargs instance off the list
            except IndexError:
                # func is equal to some_controller_func
                return function(*args, **{})

        # Remove kwargs from args
        args = [item for item in args if type(item) != Kwargs]

        # Get kwargs from the dictionary args if default kwargs aren't used
        if kwargs_len == 0:
            kwargs = {k: v for (k, v) in d_args}
        # Execute the func, which is some_controller_func
        return function(*args, **kwargs)

    # Return the local callable function `_process`
    return kwargsable

class PageController(object):
    """
    Standard controller for controlling a page.
    """
    def __init__(self, page, wait=False):
        self.page = page
        self.wait = wait

    @randomly_waits
    def fill(self, element, text):
        """
        Send keys to an element, then randomly wait.

        Args:
            element:
            text:

        Returns:
            self
        """

        element.send_keys(text)
        return self

    @randomly_waits
    def click(self, element):
        """
        Click on an element, then randomly wait.

        Args:
            element:

        Returns:
            self
        """

        element.click()
        return self

class LoginPageController(PageController):
    """
    A generic controller for logging in to a webpage.  It uses a `Page` object and a
    `WebDriverWait` object to do the job.
    """
    def __init__(self, page, wait):
        super(LoginPageController, self).__init__(page, wait=wait)

    @randomly_waits
    def do_login(self, username, password, remember_me=False, stay_logged_in=False, wait_func=False):
        """
        Log in to a web page using the given `Page` object as a template.  The `username` and `password`
        attributes must be set on the `Page` object.  If you want to use the `remember_me` keyword arg,
        you must also have that attribute set on the `Page` object.

        Args:
            username: str
            password: str
            remember_me: bool
            stay_logged_in: bool
            wait_func: func, bool

        Returns:
            self
        """
        self.fill(self.page.username, username)
        self.fill(self.page.password, password)

        # Click remember me checkbox
        if remember_me:
            self.click(self.page.remember_me)

        # Stay logged in checkbox
        if stay_logged_in:
            self.click(self.page.stay_logged_in)

        # Click login button
        self.page.login_button.click()

        # Call the wait function.
        if wait_func:
            self.wait.until(wait_func)

        return self

class SearchPageController(PageController):
    """
    A generic controller for a simple search page.  Just pass in your `Page` object
    with a `search_input` and a `search_button` defined, along with a `WebDriverWait`
    object.  You can call the `perform_search` method and it will fill in the input
    using the `Page` object passed in, then click the search button using the `Page`
    object.  You can also pass in the `wait_func` keyword to set a wait function that
    should be called before returning control.
    """
    def __init__(self, page, wait):
        super(SearchPageController, self).__init__(page, wait=wait)

    def perform_search(self, term, wait_func=False):
        """
        Search for the given term using the page object and the wait object if it is
        set.

        Args:
            term:
            wait_func:

        Returns:
            self
        """
        self.fill(self.page.search_input, term)
        self.page.search_button.click()

        if wait_func:
            self.wait.until(wait_func)

        return self

class IndependentController(object):
    """
    The base class for a threaded controller setup.
    """
    def attach_driver(self, driver, timeout=30):
        """
        Drivers must be attached after the controller has been instantiated so each controller has
        its own driver.  This will also attach a WebDriverWait to the class instance.

        Args:
            driver: Selenium WebDriver
            timeout: int

        Returns:
            self
        """

        self.driver = driver
        self.wait = WebDriverWait(self.driver, timeout)
        return self

    def call(self, method_name, *args, **kwargs):
        """
        Call one of the controller's methods with the given *args or **kwargs

        Args:
            method_name:
            *args:
            **kwargs:

        Returns:
            method results
        """

        # Grab the method from self.
        method = getattr(self, method_name)

        # Determine how to call the method and return the results.
        if args and kwargs:
            return method(*args, **kwargs)
        elif args and not kwargs:
            return method(*args)
        elif kwargs and not args:
            return method(**kwargs)
        else:
            return method()
<commit_msg>Fixed generic controllers so they no longer need a wait object.<commit_after>from random import uniform
from time import sleep
from .Commands import Kwargs
from selenium.webdriver.support.wait import WebDriverWait

# Generate a bunch of decorators for waiting up to 60 seconds.
for __ in range(1, 61):
    exec('''def waits{}(function):
    """
    A decorator for waiting {} second after function execution.  Great for waiting between actions.

    Args:
        function: function

    Returns:
        wait_decorator: function
    """

    def wait_decorator(*args, **kwargs):
        function_result = function(*args, **kwargs)
        sleep({})
        return function_result
    return wait_decorator'''.format(__, __, __))

def randomly_waits(function):
    """
    A decorator for waiting a random amount of time(0.1-3.01 seconds) after function execution.

    Args:
        function: function

    Returns:
        random_wait_decorator
    """

    def random_wait_decorator(*args, **kwargs):
        # Execute function and grab result
        function_result = function(*args, **kwargs)
        # Sleep
        sleep(uniform(0.99, 3.01))
        return function_result
    return random_wait_decorator

@randomly_waits
def human_fill(element, text):
    """
    Send keys to an element and wait a random amount of time afterwards.

    Args:
        element:
        text:

    Returns:

    """

    return element.send_keys(text)

@randomly_waits
def human_click(element):
    """
    Click on an element and wait a random amount of time afterwards.

    Args:
        element:

    Returns:

    """

    return element.click()

def has_kwargs(function):
    """
    Decorator for passing **kwargs with your *args through the use of
    the Kwargs object.  Note that only 1 Kwargs object can be passed
    with the *args to the decorated function.  Any additional Kwargs
    objects will be ignored.

    Args:
        function: function

    Returns:
        kwargsable: function
    """

    # Define a local function within has_kwargs
    def kwargsable(*args, **kwargs):
        args = list(args)
        # Extract kwargs from args.
        d_args = [thing for thing in args if type(thing) == Kwargs]

        # Check if default kwargs should be used.
        kwargs_len = len(kwargs)
        if kwargs_len == 0:
            # If there is a Kwargs instance in the list then process the kwargs
            try:
                d_args = d_args[0]  # Pop the Kwargs instance off the list
            except IndexError:
                # func is equal to some_controller_func
                return function(*args, **{})

        # Remove kwargs from args
        args = [item for item in args if type(item) != Kwargs]

        # Get kwargs from the dictionary args if default kwargs aren't used
        if kwargs_len == 0:
            kwargs = {k: v for (k, v) in d_args}
        # Execute the func, which is some_controller_func
        return function(*args, **kwargs)

    # Return the local callable function `_process`
    return kwargsable

class PageController(object):
    """
    Standard controller for controlling a page.
    """
    def __init__(self, page):
        self.page = page

    @randomly_waits
    def fill(self, element, text):
        """
        Send keys to an element, then randomly wait.

        Args:
            element:
            text:

        Returns:
            self
        """

        element.send_keys(text)
        return self

    @randomly_waits
    def click(self, element):
        """
        Click on an element, then randomly wait.

        Args:
            element:

        Returns:
            self
        """

        element.click()
        return self

class LoginPageController(PageController):
    """
    A generic controller for logging in to a webpage.  It uses a `Page` object and a
    `WebDriverWait` object to do the job.
    """
    def __init__(self, page):
        super(LoginPageController, self).__init__(page)

    @randomly_waits
    def do_login(self, username, password, remember_me=False, stay_logged_in=False, wait_func=False):
        """
        Log in to a web page using the given `Page` object as a template.  The `username` and `password`
        attributes must be set on the `Page` object.  If you want to use the `remember_me` keyword arg,
        you must also have that attribute set on the `Page` object.

        Args:
            username: str
            password: str
            remember_me: bool
            stay_logged_in: bool
            wait_func: func, bool

        Returns:
            self
        """
        self.fill(self.page.username, username)
        self.fill(self.page.password, password)

        # Click remember me checkbox
        if remember_me:
            self.click(self.page.remember_me)

        # Stay logged in checkbox
        if stay_logged_in:
            self.click(self.page.stay_logged_in)

        # Click login button
        self.page.login_button.click()

        # Call the wait function.
        if wait_func:
            self.wait.until(wait_func)

        return self

class SearchPageController(PageController):
    """
    A generic controller for a simple search page.  Just pass in your `Page` object
    with a `search_input` and a `search_button` defined, along with a `WebDriverWait`
    object.  You can call the `perform_search` method and it will fill in the input
    using the `Page` object passed in, then click the search button using the `Page`
    object.  You can also pass in the `wait_func` keyword to set a wait function that
    should be called before returning control.
    """
    def __init__(self, page, wait):
        super(SearchPageController, self).__init__(page, wait=wait)

    def perform_search(self, term, wait_func=False):
        """
        Search for the given term using the page object and the wait object if it is
        set.

        Args:
            term:
            wait_func:

        Returns:
            self
        """
        self.fill(self.page.search_input, term)
        self.page.search_button.click()

        if wait_func:
            self.wait.until(wait_func)

        return self

class IndependentController(object):
    """
    The base class for a threaded controller setup.
    """
    def attach_driver(self, driver, timeout=30):
        """
        Drivers must be attached after the controller has been instantiated so each controller has
        its own driver.  This will also attach a WebDriverWait to the class instance.

        Args:
            driver: Selenium WebDriver
            timeout: int

        Returns:
            self
        """

        self.driver = driver
        self.wait = WebDriverWait(self.driver, timeout)
        return self

    def call(self, method_name, *args, **kwargs):
        """
        Call one of the controller's methods with the given *args or **kwargs

        Args:
            method_name:
            *args:
            **kwargs:

        Returns:
            method results
        """

        # Grab the method from self.
        method = getattr(self, method_name)

        # Determine how to call the method and return the results.
        if args and kwargs:
            return method(*args, **kwargs)
        elif args and not kwargs:
            return method(*args)
        elif kwargs and not args:
            return method(**kwargs)
        else:
            return method()
<|endoftext|>


<commit_before># Natural Language Toolkit
#
# Copyright (C) 2001 University of Pennsylvania
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

"""
The Natural Language Toolkit is a package intended to simplify the
task of programming natural language systems.  It is intended to be
used as a teaching tool, not as a basis for building production
systems.
"""

# Leave this out for now...
"""
The natural language toolkit is under active development.

Interfaces
==========

The Natural Language Toolkit is implemented as a set of interfaces and
classes.  Interfaces are a concept loosely borrowed from Java.  They
are essentially a specification of a set of methods.  Any class that
implements all of an interface's methods according to the interface's
specification are said to \"implement\" that interface.

In the context of this toolkit, an interface is implemented as a
class, all of whose methods simply raise AssertionError.  The
__doc__ strings of these methods, together with the methods'
declarations,  provide specifications for the methods.

Interface classes are named with a trailing \"I\", such as
C{TokenizerI} or C{EventI}.

Interface and Class Hierarchy
=============================

The classes defined by the Natural Language Toolkit can be divided
into two basic categories: Data classes; and Processing (or
Task-Oriented) Classes.

Data Classes
------------

Data classes are used to store several different types of information
that are relavant to natural language processing.  Data classes can
generally be grouped into small clusters, with minimal interaction
between the clusters.  The clusters that are currently defined by the
Natural Language Toolkit are listed below.  Under each cluster, the
top-level classes and interfaces contained in that cluster are given.

  - B{Sets}: Encodes the mathmatical notion of a \"finite set\".
    - Set: A finite set.

  - B{Tokens}: Encodes units of text such as words.
      - Location: A span over indices in a text
      - Type: A unit of text.  This is actually not implemented as a
        class or an interface -- (almost) anything can be a Type.
      - Token: An occurance of a unit of text.

  - B{Syntax Trees}: Encodes syntax trees.
      - Tree: A hierarchical structure spanning a text.
      - TreeToken: An occurance of a Tree.

  - B{Probability}: Encodes data structures associated with
    the mathmatical notion of probability, such as events and
    frequency distributions.
      - Sample: Encodes the mathmatical notion of a
        \"sample\".  This is actually not implemented as a class or
        an interface -- (almost) anything can be a Sample.
      - EventI: A (possibly infinite) set of samples.
      - FreqDistI: The frequency distribution of a collection of
        samples.
      - ProbDistI: A probability distribution, typically derived from
        a frequency distribution (e.g., using ELE).

Processing Classes
------------------

Processing classes are used to perform a variety of tasks that are
relavant to natural language processing.  Processing classes can
generally be grouped into small clusters, with minimial interaction
between the clusters.  Each cluster typically makes use of several
data-class clusters.  The processing clusters that are currently
defined by the Natural Language Toolkit are listed below.  Under each
cluster, the interfaces contained in that cluster are given.

  - B{Tokenizers}: Separate a string of text into a list of
    Tokens.
       - TokenizerI
  - B{Taggers}: Assign tags to each Token in a list of Tokens.
       - TaggerI
  - B{Parser}: Produce Trees that represent the internal structure of
    a text.

@author: Edward Loper
@version: 0.5
"""

"""
Open Questions
==============

The following is a list of currently unresolved questions, pertaining
to currently implemented interfaces and classes.
  - Terminology/Naming Questions
    - Is \"Token Type\" too easily confusable with the notion of
         type in python?  E.g., names like SimpleTokenType suggest
         that they are similar to StringType, IntType, etc. when they
         are very different.  I could use \"TokenTyp\" to distinguish,
         but this also seems somewhat confusing to the uninitiated.
  - Is the token/token type/text location system too complex?
       Often, one only cares about the token type.  E.g., a tokenizer
       could be defined as mapping string to list of TokenType, and a
       tagger as mapping list of SimpleTokenType to TaggedTokenType.
       But sometimes we really need to be able to distinguish tokens,
       not just token types.. e.g., to test the chunk parser from the
       chunk parsing problem set.
  - Should FreqDist.max() and FreqDist.cond_max() be merged, with
       the condition as an optional argument?  Same for
       FreqDist.freq() and FreqDist.cond_freq().
  - What cross-toolkit policies on how to use __str__
       and __repr__ should there be?
  - What cross-toolkit policies on __cmp__, __eq__, __gt__, etc.
       should there be?
"""

##//////////////////////////////////////////////////////
##  Meta-Data
##//////////////////////////////////////////////////////

# Version.  For each new release, the version number should be updated
# here and in the Epydoc comment (above).
__version__ = "0.5"

# Copyright notice
__copyright__ = """\
Copyright (C) 2001 University of Pennsylvania.

Distributed and Licensed under provisions of the IBM Common Public
License (Version 0.5), which is included by reference.  The IBM Common
Public License can be found in the file LICENSE.TXT in the
distribution."""
__licence__ = "IBM Common Public License (Version 0.5)"
# Description of the toolkit, keywords, and the project's primary URL.
__longdescr__ = """\
The Natural Langauge Toolkit is a Python package that simplifies the
construction of programs that process natural language; and defines
standard interfaces between the different components of an NLP system.
NLTK requires Python 2.1 or higher."""
__keywords__ = ['NLP', 'CL', 'natural language processing',
                'computational linguistics', 'parsing', 'tagging',
                'tokenizing', 'syntax', 'linguistics', 'language',
                'natural language']
__url__ = "http://nltk.sf.net/"

# Maintainer, contributors, etc.
__maintainer__ = "Edward Loper"
__maintainer_email__ = "edloper@gradient.cis.upenn.edu"
__author__ = __maintainer__
__author_email__ = __maintainer_email__
__contributors__ = """\
Edward Loper <edloper@gradient.cis.upenn.edu>
Steven Bird <sb@unagi.cis.upenn.edu>
Salim Zayats <szayats@seas.upenn.edu>
"""

# Used for epydoc sorting.  Also used for "from nltk import *"
__all__ = [
    # Base modules
    'token',
    'probability',
    'set',

    # Data types
    'tree',
    'cfg',
    'fsa',

    # Tasks
    'tagger',
    'parser',
    'classifier',

    # Visualization
    'draw',

    # Debugging/testing
    'chktype',
    'test',
    ]
<commit_msg>- Updated version number<commit_after># Natural Language Toolkit
#
# Copyright (C) 2001 University of Pennsylvania
# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
# URL: <http://nltk.sf.net>
# For license information, see LICENSE.TXT
#
# $Id$

"""
The Natural Language Toolkit is a package intended to simplify the
task of programming natural language systems.  It is intended to be
used as a teaching tool, not as a basis for building production
systems.

@author: Edward Loper
@version: 0.6
"""

# Leave this out for now... It's out of date.
"""
The natural language toolkit is under active development.

Interfaces
==========

The Natural Language Toolkit is implemented as a set of interfaces and
classes.  Interfaces are a concept loosely borrowed from Java.  They
are essentially a specification of a set of methods.  Any class that
implements all of an interface's methods according to the interface's
specification are said to \"implement\" that interface.

In the context of this toolkit, an interface is implemented as a
class, all of whose methods simply raise AssertionError.  The
__doc__ strings of these methods, together with the methods'
declarations,  provide specifications for the methods.

Interface classes are named with a trailing \"I\", such as
C{TokenizerI} or C{EventI}.

Interface and Class Hierarchy
=============================

The classes defined by the Natural Language Toolkit can be divided
into two basic categories: Data classes; and Processing (or
Task-Oriented) Classes.

Data Classes
------------

Data classes are used to store several different types of information
that are relavant to natural language processing.  Data classes can
generally be grouped into small clusters, with minimal interaction
between the clusters.  The clusters that are currently defined by the
Natural Language Toolkit are listed below.  Under each cluster, the
top-level classes and interfaces contained in that cluster are given.

  - B{Sets}: Encodes the mathmatical notion of a \"finite set\".
    - Set: A finite set.

  - B{Tokens}: Encodes units of text such as words.
      - Location: A span over indices in a text
      - Type: A unit of text.  This is actually not implemented as a
        class or an interface -- (almost) anything can be a Type.
      - Token: An occurance of a unit of text.

  - B{Syntax Trees}: Encodes syntax trees.
      - Tree: A hierarchical structure spanning a text.
      - TreeToken: An occurance of a Tree.

  - B{Probability}: Encodes data structures associated with
    the mathmatical notion of probability, such as events and
    frequency distributions.
      - Sample: Encodes the mathmatical notion of a
        \"sample\".  This is actually not implemented as a class or
        an interface -- (almost) anything can be a Sample.
      - EventI: A (possibly infinite) set of samples.
      - FreqDistI: The frequency distribution of a collection of
        samples.
      - ProbDistI: A probability distribution, typically derived from
        a frequency distribution (e.g., using ELE).

Processing Classes
------------------

Processing classes are used to perform a variety of tasks that are
relavant to natural language processing.  Processing classes can
generally be grouped into small clusters, with minimial interaction
between the clusters.  Each cluster typically makes use of several
data-class clusters.  The processing clusters that are currently
defined by the Natural Language Toolkit are listed below.  Under each
cluster, the interfaces contained in that cluster are given.

  - B{Tokenizers}: Separate a string of text into a list of
    Tokens.
       - TokenizerI
  - B{Taggers}: Assign tags to each Token in a list of Tokens.
       - TaggerI
  - B{Parser}: Produce Trees that represent the internal structure of
    a text.

@author: Edward Loper
@version: 0.6
"""

"""
Open Questions
==============

The following is a list of currently unresolved questions, pertaining
to currently implemented interfaces and classes.
  - Terminology/Naming Questions
    - Is \"Token Type\" too easily confusable with the notion of
         type in python?  E.g., names like SimpleTokenType suggest
         that they are similar to StringType, IntType, etc. when they
         are very different.  I could use \"TokenTyp\" to distinguish,
         but this also seems somewhat confusing to the uninitiated.
  - Is the token/token type/text location system too complex?
       Often, one only cares about the token type.  E.g., a tokenizer
       could be defined as mapping string to list of TokenType, and a
       tagger as mapping list of SimpleTokenType to TaggedTokenType.
       But sometimes we really need to be able to distinguish tokens,
       not just token types.. e.g., to test the chunk parser from the
       chunk parsing problem set.
  - Should FreqDist.max() and FreqDist.cond_max() be merged, with
       the condition as an optional argument?  Same for
       FreqDist.freq() and FreqDist.cond_freq().
  - What cross-toolkit policies on how to use __str__
       and __repr__ should there be?
  - What cross-toolkit policies on __cmp__, __eq__, __gt__, etc.
       should there be?
"""

##//////////////////////////////////////////////////////
##  Meta-Data
##//////////////////////////////////////////////////////

# Version.  For each new release, the version number should be updated
# here and in the Epydoc comment (above).
__version__ = "0.6"

# Copyright notice
__copyright__ = """\
Copyright (C) 2001 University of Pennsylvania.

Distributed and Licensed under provisions of the IBM Common Public
License (Version 0.5), which is included by reference.  The IBM Common
Public License can be found in the file LICENSE.TXT in the
distribution."""
__licence__ = "IBM Common Public License (Version 0.5)"
# Description of the toolkit, keywords, and the project's primary URL.
__longdescr__ = """\
The Natural Langauge Toolkit is a Python package that simplifies the
construction of programs that process natural language; and defines
standard interfaces between the different components of an NLP system.
NLTK requires Python 2.1 or higher."""
__keywords__ = ['NLP', 'CL', 'natural language processing',
                'computational linguistics', 'parsing', 'tagging',
                'tokenizing', 'syntax', 'linguistics', 'language',
                'natural language']
__url__ = "http://nltk.sf.net/"

# Maintainer, contributors, etc.
__maintainer__ = "Edward Loper"
__maintainer_email__ = "edloper@gradient.cis.upenn.edu"
__author__ = __maintainer__
__author_email__ = __maintainer_email__
__contributors__ = """\
Edward Loper <edloper@gradient.cis.upenn.edu>
Steven Bird <sb@unagi.cis.upenn.edu>
Salim Zayats <szayats@seas.upenn.edu>
"""

# Used for epydoc sorting.  Also used for "from nltk import *"
__all__ = [
    # Base modules
    'token',
    'probability',
    'set',

    # Data types
    'tree',
    'cfg',
    'fsa',

    # Tasks
    'tagger',
    'parser',
    'classifier',

    # Visualization
    'draw',

    # Debugging/testing
    'chktype',
    'test',
    ]
<|endoftext|>


<commit_before>from __future__ import print_function
import collections
import contextlib
import os
import re
import yaml

from rdopkg import exception
from rdopkg.utils import log
from rdopkg.utils.cmd import run

def confirm(msg, default_yes=True):
    if default_yes:
        options = '[Yn]'
    else:
        options = '[Ny]'
    print('')
    i = raw_input(log.term.important("%s %s " % (msg, options))).lower()
    if not i:
        if default_yes:
            return
        else:
            raise exception.UserAbort()
    if i != 'y' and i != 'yes':
        raise exception.UserAbort()
    print('')

def download_file(url):
    run('curl', '-L', '-f', '-O', url, direct=True)

def edit(path):
    editor = os.environ.get('EDITOR')
    if not editor:
        editor = 'vim'
        log.info("$EDITOR not set. Falling back to %s." % editor)
    try:
        r = run(editor, path, direct=True)
    except exception.CommandNotFound:
        raise exception.CommandNotFound(
            msg='Failed to find suitable text editor ({0}).  '
                'Please set $EDITOR environment variable.'.format(editor))
    return r.success

def ensure_dir(path):
    if os.path.exists(path):
        if not os.path.isdir(path):
            raise exception.NotADirectory(path=path)
    else:
        os.makedirs(path)

def ensure_new_file_dirs(path):
    if path[-1] == os.sep:
        raise exception.NotAFile(path=path)
    _dir = os.path.dirname(path)
    if _dir:
        ensure_dir(_dir)

def find_files(path, ext=None):
    paths = [os.path.join(path, f) for f in os.listdir(path)
             if os.path.isfile(os.path.join(path, f))]
    if ext:
        paths = [p for p in paths if p.endswith(ext)]
    return paths

def print_list(l, pre='- ', nl_before=False, nl_after=False):
    if nl_before:
        print("")
    for e in l:
        print('%s%s' % (pre, e))
    if nl_after:
        print("")

def action_required(msg):
    print("\n{t.important}Action required: {t.normal}{t.bold}{msg}{t.normal}"
          .format(t=log.term, msg=msg))
    print("\nOnce done, run `rdopkg -c` to continue.\n")

def is_same_hash(h1, h2):
    if h1 is None or h2 is None:
        return False
    if h1 == h2 or h1.startswith(h2) or h2.startswith(h1):
        return True
    return False

def strip_patch_subject(subj):
    return re.sub(r'^\[PATCH\]\s+', '', subj.strip())

def is_same_subject(s1, s2):
    if s1 is None or s2 is None:
        return False
    if strip_patch_subject(s1) == strip_patch_subject(s2):
        return True
    return False

@contextlib.contextmanager
def cdir(path):
    prev_cwd = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(prev_cwd)

@contextlib.contextmanager
def setenv(**env_dict):
    orig_env = os.environ.copy()
    for k, v in env_dict.items():
        os.environ[k] = str(v)
    try:
        yield
    finally:
        for k, v in env_dict.items():
            del os.environ[k]
        for k, v in orig_env.items():
            os.environ[k] = v

def print_keyval(key, val, kb=True, vb=False):
    if kb:
        fmt = '{t.bold}{key}{t.normal}: '
    else:
        fmt = '{key}'
    if vb:
        fmt += '{t.bold}{val}{t.bold}'
    else:
        fmt += '{val}'
    if not isinstance(
            val,
            basestring) and isinstance(
            val,
            collections.Iterable):
        vals = "\n" + "\n".join(map(lambda x: '- ' + str(x), val))
    else:
        vals = str(val)
    print(fmt.format(t=log.term, key=key, val=vals))

class DictPrinter(object):
    # Very simple and stupid dict printer.

    def __init__(self, header=None, first=None, last=None):
        self.first = first or []
        self.last = last or []
        self.header = header

    def __call__(self, d):
        dd = d.copy()
        # print header
        if self.header:
            hdr = dd.pop(self.header, '')
            print_keyval(self.header, hdr, kb=True, vb=True)
        # print first fields
        for key in self.first:
            if key not in dd:
                continue
            val = dd.pop(key)
            print_keyval(key, val)
        # filter out last fields
        last_items = []
        for key in self.last:
            if key not in dd:
                continue
            last_items.append((key, dd.pop(key)))
        if dd:
            print(yaml.dump(dd, default_flow_style=False).rstrip())
        for key, val in last_items:
            print_keyval(key, val)
<commit_msg>Avoid prompt on non interactive console<commit_after>from __future__ import print_function
import collections
import contextlib
import os
import re
import sys
import yaml

from rdopkg import exception
from rdopkg.utils import log
from rdopkg.utils.cmd import run

def confirm(msg, default_yes=True):
    if sys.stdout.isatty():
        if default_yes:
            options = '[Yn]'
        else:
            options = '[Ny]'
        print('')
        i = raw_input(log.term.important("%s %s " % (msg, options))).lower()
        if not i:
            if default_yes:
                return
            else:
                raise exception.UserAbort()
        if i != 'y' and i != 'yes':
            raise exception.UserAbort()
        print('')
    else:
        if default_yes:
            return
        else:
            raise exception.UserAbort()

def download_file(url):
    run('curl', '-L', '-f', '-O', url, direct=True)

def edit(path):
    editor = os.environ.get('EDITOR')
    if not editor:
        editor = 'vim'
        log.info("$EDITOR not set. Falling back to %s." % editor)
    try:
        r = run(editor, path, direct=True)
    except exception.CommandNotFound:
        raise exception.CommandNotFound(
            msg='Failed to find suitable text editor ({0}).  '
                'Please set $EDITOR environment variable.'.format(editor))
    return r.success

def ensure_dir(path):
    if os.path.exists(path):
        if not os.path.isdir(path):
            raise exception.NotADirectory(path=path)
    else:
        os.makedirs(path)

def ensure_new_file_dirs(path):
    if path[-1] == os.sep:
        raise exception.NotAFile(path=path)
    _dir = os.path.dirname(path)
    if _dir:
        ensure_dir(_dir)

def find_files(path, ext=None):
    paths = [os.path.join(path, f) for f in os.listdir(path)
             if os.path.isfile(os.path.join(path, f))]
    if ext:
        paths = [p for p in paths if p.endswith(ext)]
    return paths

def print_list(l, pre='- ', nl_before=False, nl_after=False):
    if nl_before:
        print("")
    for e in l:
        print('%s%s' % (pre, e))
    if nl_after:
        print("")

def action_required(msg):
    print("\n{t.important}Action required: {t.normal}{t.bold}{msg}{t.normal}"
          .format(t=log.term, msg=msg))
    print("\nOnce done, run `rdopkg -c` to continue.\n")

def is_same_hash(h1, h2):
    if h1 is None or h2 is None:
        return False
    if h1 == h2 or h1.startswith(h2) or h2.startswith(h1):
        return True
    return False

def strip_patch_subject(subj):
    return re.sub(r'^\[PATCH\]\s+', '', subj.strip())

def is_same_subject(s1, s2):
    if s1 is None or s2 is None:
        return False
    if strip_patch_subject(s1) == strip_patch_subject(s2):
        return True
    return False

@contextlib.contextmanager
def cdir(path):
    prev_cwd = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(prev_cwd)

@contextlib.contextmanager
def setenv(**env_dict):
    orig_env = os.environ.copy()
    for k, v in env_dict.items():
        os.environ[k] = str(v)
    try:
        yield
    finally:
        for k, v in env_dict.items():
            del os.environ[k]
        for k, v in orig_env.items():
            os.environ[k] = v

def print_keyval(key, val, kb=True, vb=False):
    if kb:
        fmt = '{t.bold}{key}{t.normal}: '
    else:
        fmt = '{key}'
    if vb:
        fmt += '{t.bold}{val}{t.bold}'
    else:
        fmt += '{val}'
    if not isinstance(
            val,
            basestring) and isinstance(
            val,
            collections.Iterable):
        vals = "\n" + "\n".join(map(lambda x: '- ' + str(x), val))
    else:
        vals = str(val)
    print(fmt.format(t=log.term, key=key, val=vals))

class DictPrinter(object):
    # Very simple and stupid dict printer.

    def __init__(self, header=None, first=None, last=None):
        self.first = first or []
        self.last = last or []
        self.header = header

    def __call__(self, d):
        dd = d.copy()
        # print header
        if self.header:
            hdr = dd.pop(self.header, '')
            print_keyval(self.header, hdr, kb=True, vb=True)
        # print first fields
        for key in self.first:
            if key not in dd:
                continue
            val = dd.pop(key)
            print_keyval(key, val)
        # filter out last fields
        last_items = []
        for key in self.last:
            if key not in dd:
                continue
            last_items.append((key, dd.pop(key)))
        if dd:
            print(yaml.dump(dd, default_flow_style=False).rstrip())
        for key, val in last_items:
            print_keyval(key, val)
<|endoftext|>