<commit_before>""" Any utility functions.

This module holds the owner data along with a handful of
command specific functions and helpers.
"""

import re
import shlex
from enum import Enum
from functools import wraps

import discord
import aiohttp

from pcbot import Config, config

owner_cfg = Config("owner")
member_mention_regex = re.compile(r"<@!?(?P<id>\d+)>")
channel_mention_regex = re.compile(r"<#(?P<id>\d+)>")
markdown_code_regex = re.compile(r"^(?P<capt>`*)(?:[a-z]+\n)?(?P<code>.+)(?P=capt)$", flags=re.DOTALL)
identifier_prefix = re.compile(r"[a-zA-Z_]")

class Annotate(Enum):
    """ Command annotation enum.
    Annotate a command argument with one of these to get the commented result. """
    Content = 1  # Return all the content after command and/or arguments with Message.content
    LowerContent = 2  # Same as above but returns the contents in lowercase
    CleanContent = 3  # Return all the content after command and/or arguments with Message.clean_content
    LowerCleanContent = 4  # Same as above but returns the contents in lowercase
    User = Member = 5  # Return a member (uses utils.find_member with steps=3)
    Channel = 6  # Return a channel (uses utils.find_channel with steps=3)
    Self = 7  # Used as a default for Member/Channel annotations and returns the message.author/message.channel
    Code = 8  # Get formatted code (like Content but extracts any code)

def int_range(f: int=None, t: int=None):
    """ Return a helper function for checking if a str converted to int is in the
    specified range, f (from) - t (to).

    If either f or t is None, they will be treated as -inf +inf respectively. """
    def wrapped(arg: str):
        # Convert to int and return None if unsuccessful
        try:
            num = int(arg)
        except ValueError:
            return None

        # Compare the lowest and highest numbers
        if (f and num < f) or (t and num > t):
            return None

        # The string given is converted to a number and fits the criteria
        return num

    return wrapped

def choice(*options, ignore_case: bool=True):
    """ Return a helper function for checking if the argument is either of the given
    options. """
    def wrapped(arg: str):
        # Compare lowercased version
        if ignore_case:
            return arg if arg.lower() in [s.lower() for s in options] else None

        return arg if arg in options else None

    return wrapped

def placeholder(_: str):
    """ Return False. Using this as a command argument annotation will always fail
    the command. Useful for groups. """
    return False

def format_usage(command):
    """ Format the usage string of the given command. Places any usage
    of a sub command on a newline.

    :param command: type plugins.Command """
    if command.hidden and command.parent is not None:
        return

    usage = [command.usage]
    for sub_command in command.sub_commands:
        # Recursively format the usage of the next sub commands
        formatted = format_usage(sub_command)

        if formatted:
            usage.append(formatted)

    return "\n".join(s for s in usage if s is not None) if usage else None

def format_help(command, no_subcommand=False):
    """ Format the help string of the given command as a message to be sent.

    :param command: type plugins.Command """
    usage = command.usage if no_subcommand else format_usage(command)

    # If there is no usage, the command isn't supposed to be displayed as such
    # Therefore, we switch to using the parent command instead
    if usage is None and command.parent is not None:
        return format_help(command.parent)

    desc = command.description

    # Notify the user when a command is owner specific
    if getattr(command.function, "__owner__", False):
        desc += "\n:information_source:`Only the bot owner can execute this command.`"

    # Format aliases
    alias_format = ""
    if command.aliases:
        # Don't add blank space unless necessary
        if not desc.strip().endswith("```"):
            alias_format += "\n"

        alias_format += "**Aliases**: ```{}```".format(
            ", ".join((config.command_prefix if identifier_prefix.match(alias[0]) and command.parent is None else "") +
                      alias for alias in command.aliases))

    return "**Usage**: ```{}```**Description**: {}{}".format(usage, desc, alias_format)

def is_owner(member):
    """ Return true if user/member is the assigned bot owner.

    :param member: discord.User, discord.Member or a str representing the member's ID. """
    if type(member) is not str:
        member = member.id

    if member == owner_cfg.data:
        return True

    return False

def owner(func):
    """ Decorator that runs the command only if the author is the owner. """
    @wraps(func)
    async def wrapped(client: discord.Client, message: discord.Message, *args, **kwargs):
        if is_owner(message.author):
            await func(client, message, *args, **kwargs)

    setattr(wrapped, "__owner__", True)
    return wrapped

def permission(*perms: str):
    """ Decorator that runs the command only if the author has the specified permissions.
    perms must be a string matching any property of discord.Permissions. """
    def decorator(func):
        @wraps(func)
        async def wrapped(client: discord.Client, message: discord.Message, *args, **kwargs):
            member_perms = message.author.permissions_in(message.channel)

            if all(getattr(member_perms, perm, False) for perm in perms):
                await func(client, message, *args, **kwargs)

        return wrapped
    return decorator

def role(*roles: str):
    """ Decorator that runs the command only if the author has the specified Roles.
    roles must be a string representing a role's name. """
    def decorator(func):
        @wraps(func)
        async def wrapped(client: discord.Client, message: discord.Message, *args, **kwargs):
            member_roles = [r.name for r in message.author.roles[1:]]

            if any(r in member_roles for r in roles):
                await func(client, message, *args, **kwargs)

        return wrapped
    return decorator

async def retrieve_headers(url, **params):
    """ Retrieve the headers from a URL.

    :param url: URL as str
    :param params: Any additional url parameters
    :return: Headers as a dict """
    async with aiohttp.ClientSession() as session:
        async with session.head(url, params=params) as response:
            return response.headers

async def download_file(url, **params):
    """ Download and return a byte-like object of a file.

    :param url: Download url as str
    :param params: Any additional url parameters
    :return: The byte-like file """
    async with aiohttp.ClientSession() as session:
        async with session.get(url, params=params) as response:
            return await response.read()

async def download_json(url, **params):
    """ Download and return a json file.

    :param url: Download url as str
    :param params: Any additional url parameters
    :return: A JSON representation of the downloaded file """
    async with aiohttp.ClientSession() as session:
        async with session.get(url, params=params) as response:
            try:
                return await response.json()
            except ValueError:
                return None

def find_member(server: discord.Server, name, steps=3, mention=True):
    """ Find any member by their name or a formatted mention.
    Steps define the depth at which to search. More steps equal
    less accurate checks.

    +--------+------------------+
    |  step  |     function     |
    +--------+------------------+
    |    0   | perform no check |
    |    1   |   name is equal  |
    |    2   | name starts with |
    |    3   |    name is in    |
    +--------+------------------+

    :param server: discord.Server to look through for members.
    :param name: display_name as a string or mention to find.
    :param steps: int from 0-3 to specify search depth.
    :param mention: check for mentions. """
    member = None

    # Return a member from mention
    found_mention = member_mention_regex.search(name)
    if found_mention and mention:
        member = server.get_member(found_mention.group("id"))

    name = name.lower()

    if not member:
        # Steps to check, higher values equal more fuzzy checks
        checks = [lambda m: m.display_name.lower() == name,
                  lambda m: m.display_name.lower().startswith(name),
                  lambda m: name in m.display_name.lower()]

        for i in range(steps if steps <= len(checks) else len(checks)):
            member = discord.utils.find(checks[i], server.members)

            if member:
                break

    # Return the found member or None
    return member

def find_channel(server: discord.Server, name, steps=3, mention=True):
    """ Find any channel by its name or a formatted mention.
        Steps define the depth at which to search. More steps equal
        less accurate checks.

        +--------+------------------+
        |  step  |     function     |
        +--------+------------------+
        |    0   | perform no check |
        |    1   |   name is equal  |
        |    2   | name starts with |
        |    3   |    name is in    |
        +--------+------------------+

        :param server: discord.Server to look through for channels.
        :param name: name as a string or mention to find.
        :param steps: int from 0-3 to specify search depth.
        :param mention: check for mentions. """
    channel = None

    # Return a member from mention
    found_mention = channel_mention_regex.search(name)
    if found_mention and mention:
        channel = server.get_channel(found_mention.group("id"))

    if not channel:
        # Steps to check, higher values equal more fuzzy checks
        checks = [lambda c: c.name.lower() == name.lower(),
                  lambda c: c.name.lower().startswith(name.lower()),
                  lambda c: name.lower() in c.name.lower()]

        for i in range(steps if steps <= len(checks) else len(checks)):
            channel = discord.utils.find(checks[i], server.channels)

            if channel:
                break

    # Return the found channel or None
    return channel

def get_member(client: discord.Client, member_id: str):
    """ Get a member from the specified ID. """
    for member in client.get_all_members():
        if member.id == member_id:
            return member

    return None

def format_exception(e):
    """ Returns a formatted string of Exception: e """
    return type(e).__name__ + ": " + str(e)

def format_syntax_error(e):
    """ Returns a formatted string of a SyntaxError.
    Stolen from https://github.com/Rapptz/RoboDanny/blob/master/cogs/repl.py#L24-L25 """
    return "{0.text}\n{1:>{0.offset}}\n{2}: {0}".format(e, "^", type(e).__name__)

def get_formatted_code(code):
    """ Format code from markdown format. This will filter out markdown code
    and give the executable python code, or return a string that would raise
    an error when it's executed by exec() or eval(). """
    match = markdown_code_regex.match(code)

    if match:
        code = match.group("code")

        if not code == "`":
            return code

    return "raise Exception(\"Could not format code.\")"

def format_objects(*objects: tuple, attr=None, dec: str= "", sep: str= ", "):
    """ Return a formatted string of objects (User, Member, Channel or Server) using
    the given decorator and the given separator.

    :param attr: The attribute to get from the member. """
    if not objects:
        return

    first_object = objects[0]
    if attr is None:
        if isinstance(first_object, discord.User):
            attr = "display_name"
        elif isinstance(first_object, discord.Channel):
            attr = "mention"
        elif isinstance(first_object, discord.Server):
            attr = "name"

    return sep.join(dec + getattr(m, attr) + dec for m in objects)

def format_channels(*members: discord.Member, attr="mention", dec: str = "`", sep: str = ", "):
    """ Return a formatted string of members (or member) using the given
    decorator and the given separator.

    :param attr: The attribute to get from the member. """
    return sep.join(dec + getattr(m, attr) + dec for m in members)

def split(string, maxsplit=-1):
    """ Split a string with shlex when possible, and add support for maxsplit. """
    if maxsplit == -1:
        try:
            split_object = shlex.shlex(string, posix=True)
            split_object.quotes = '"`'
            split_object.whitespace_split = True
            split_object.commenters = ""
            return list(split_object)
        except ValueError:
            return string.split()

    split_object = shlex.shlex(string, posix=True)
    split_object.quotes = '"`'
    split_object.whitespace_split = True
    split_object.commenters = ""
    maxsplit_object = []
    splits = 0

    while splits < maxsplit:
        maxsplit_object.append(next(split_object))

        splits += 1

    maxsplit_object.append(split_object.instream.read())

    return maxsplit_object
<commit_msg>Remove double newlines when formatting SyntaxError as this sometimes happens.<commit_after>""" Any utility functions.

This module holds the owner data along with a handful of
command specific functions and helpers.
"""

import re
import shlex
from enum import Enum
from functools import wraps

import discord
import aiohttp

from pcbot import Config, config

owner_cfg = Config("owner")
member_mention_regex = re.compile(r"<@!?(?P<id>\d+)>")
channel_mention_regex = re.compile(r"<#(?P<id>\d+)>")
markdown_code_regex = re.compile(r"^(?P<capt>`*)(?:[a-z]+\n)?(?P<code>.+)(?P=capt)$", flags=re.DOTALL)
identifier_prefix = re.compile(r"[a-zA-Z_]")

class Annotate(Enum):
    """ Command annotation enum.
    Annotate a command argument with one of these to get the commented result. """
    Content = 1  # Return all the content after command and/or arguments with Message.content
    LowerContent = 2  # Same as above but returns the contents in lowercase
    CleanContent = 3  # Return all the content after command and/or arguments with Message.clean_content
    LowerCleanContent = 4  # Same as above but returns the contents in lowercase
    User = Member = 5  # Return a member (uses utils.find_member with steps=3)
    Channel = 6  # Return a channel (uses utils.find_channel with steps=3)
    Self = 7  # Used as a default for Member/Channel annotations and returns the message.author/message.channel
    Code = 8  # Get formatted code (like Content but extracts any code)

def int_range(f: int=None, t: int=None):
    """ Return a helper function for checking if a str converted to int is in the
    specified range, f (from) - t (to).

    If either f or t is None, they will be treated as -inf +inf respectively. """
    def wrapped(arg: str):
        # Convert to int and return None if unsuccessful
        try:
            num = int(arg)
        except ValueError:
            return None

        # Compare the lowest and highest numbers
        if (f and num < f) or (t and num > t):
            return None

        # The string given is converted to a number and fits the criteria
        return num

    return wrapped

def choice(*options, ignore_case: bool=True):
    """ Return a helper function for checking if the argument is either of the given
    options. """
    def wrapped(arg: str):
        # Compare lowercased version
        if ignore_case:
            return arg if arg.lower() in [s.lower() for s in options] else None

        return arg if arg in options else None

    return wrapped

def placeholder(_: str):
    """ Return False. Using this as a command argument annotation will always fail
    the command. Useful for groups. """
    return False

def format_usage(command):
    """ Format the usage string of the given command. Places any usage
    of a sub command on a newline.

    :param command: type plugins.Command """
    if command.hidden and command.parent is not None:
        return

    usage = [command.usage]
    for sub_command in command.sub_commands:
        # Recursively format the usage of the next sub commands
        formatted = format_usage(sub_command)

        if formatted:
            usage.append(formatted)

    return "\n".join(s for s in usage if s is not None) if usage else None

def format_help(command, no_subcommand=False):
    """ Format the help string of the given command as a message to be sent.

    :param command: type plugins.Command """
    usage = command.usage if no_subcommand else format_usage(command)

    # If there is no usage, the command isn't supposed to be displayed as such
    # Therefore, we switch to using the parent command instead
    if usage is None and command.parent is not None:
        return format_help(command.parent)

    desc = command.description

    # Notify the user when a command is owner specific
    if getattr(command.function, "__owner__", False):
        desc += "\n:information_source:`Only the bot owner can execute this command.`"

    # Format aliases
    alias_format = ""
    if command.aliases:
        # Don't add blank space unless necessary
        if not desc.strip().endswith("```"):
            alias_format += "\n"

        alias_format += "**Aliases**: ```{}```".format(
            ", ".join((config.command_prefix if identifier_prefix.match(alias[0]) and command.parent is None else "") +
                      alias for alias in command.aliases))

    return "**Usage**: ```{}```**Description**: {}{}".format(usage, desc, alias_format)

def is_owner(member):
    """ Return true if user/member is the assigned bot owner.

    :param member: discord.User, discord.Member or a str representing the member's ID. """
    if type(member) is not str:
        member = member.id

    if member == owner_cfg.data:
        return True

    return False

def owner(func):
    """ Decorator that runs the command only if the author is the owner. """
    @wraps(func)
    async def wrapped(client: discord.Client, message: discord.Message, *args, **kwargs):
        if is_owner(message.author):
            await func(client, message, *args, **kwargs)

    setattr(wrapped, "__owner__", True)
    return wrapped

def permission(*perms: str):
    """ Decorator that runs the command only if the author has the specified permissions.
    perms must be a string matching any property of discord.Permissions. """
    def decorator(func):
        @wraps(func)
        async def wrapped(client: discord.Client, message: discord.Message, *args, **kwargs):
            member_perms = message.author.permissions_in(message.channel)

            if all(getattr(member_perms, perm, False) for perm in perms):
                await func(client, message, *args, **kwargs)

        return wrapped
    return decorator

def role(*roles: str):
    """ Decorator that runs the command only if the author has the specified Roles.
    roles must be a string representing a role's name. """
    def decorator(func):
        @wraps(func)
        async def wrapped(client: discord.Client, message: discord.Message, *args, **kwargs):
            member_roles = [r.name for r in message.author.roles[1:]]

            if any(r in member_roles for r in roles):
                await func(client, message, *args, **kwargs)

        return wrapped
    return decorator

async def retrieve_headers(url, **params):
    """ Retrieve the headers from a URL.

    :param url: URL as str
    :param params: Any additional url parameters
    :return: Headers as a dict """
    async with aiohttp.ClientSession() as session:
        async with session.head(url, params=params) as response:
            return response.headers

async def download_file(url, **params):
    """ Download and return a byte-like object of a file.

    :param url: Download url as str
    :param params: Any additional url parameters
    :return: The byte-like file """
    async with aiohttp.ClientSession() as session:
        async with session.get(url, params=params) as response:
            return await response.read()

async def download_json(url, **params):
    """ Download and return a json file.

    :param url: Download url as str
    :param params: Any additional url parameters
    :return: A JSON representation of the downloaded file """
    async with aiohttp.ClientSession() as session:
        async with session.get(url, params=params) as response:
            try:
                return await response.json()
            except ValueError:
                return None

def find_member(server: discord.Server, name, steps=3, mention=True):
    """ Find any member by their name or a formatted mention.
    Steps define the depth at which to search. More steps equal
    less accurate checks.

    +--------+------------------+
    |  step  |     function     |
    +--------+------------------+
    |    0   | perform no check |
    |    1   |   name is equal  |
    |    2   | name starts with |
    |    3   |    name is in    |
    +--------+------------------+

    :param server: discord.Server to look through for members.
    :param name: display_name as a string or mention to find.
    :param steps: int from 0-3 to specify search depth.
    :param mention: check for mentions. """
    member = None

    # Return a member from mention
    found_mention = member_mention_regex.search(name)
    if found_mention and mention:
        member = server.get_member(found_mention.group("id"))

    name = name.lower()

    if not member:
        # Steps to check, higher values equal more fuzzy checks
        checks = [lambda m: m.display_name.lower() == name,
                  lambda m: m.display_name.lower().startswith(name),
                  lambda m: name in m.display_name.lower()]

        for i in range(steps if steps <= len(checks) else len(checks)):
            member = discord.utils.find(checks[i], server.members)

            if member:
                break

    # Return the found member or None
    return member

def find_channel(server: discord.Server, name, steps=3, mention=True):
    """ Find any channel by its name or a formatted mention.
        Steps define the depth at which to search. More steps equal
        less accurate checks.

        +--------+------------------+
        |  step  |     function     |
        +--------+------------------+
        |    0   | perform no check |
        |    1   |   name is equal  |
        |    2   | name starts with |
        |    3   |    name is in    |
        +--------+------------------+

        :param server: discord.Server to look through for channels.
        :param name: name as a string or mention to find.
        :param steps: int from 0-3 to specify search depth.
        :param mention: check for mentions. """
    channel = None

    # Return a member from mention
    found_mention = channel_mention_regex.search(name)
    if found_mention and mention:
        channel = server.get_channel(found_mention.group("id"))

    if not channel:
        # Steps to check, higher values equal more fuzzy checks
        checks = [lambda c: c.name.lower() == name.lower(),
                  lambda c: c.name.lower().startswith(name.lower()),
                  lambda c: name.lower() in c.name.lower()]

        for i in range(steps if steps <= len(checks) else len(checks)):
            channel = discord.utils.find(checks[i], server.channels)

            if channel:
                break

    # Return the found channel or None
    return channel

def get_member(client: discord.Client, member_id: str):
    """ Get a member from the specified ID. """
    for member in client.get_all_members():
        if member.id == member_id:
            return member

    return None

def format_exception(e):
    """ Returns a formatted string of Exception: e """
    return type(e).__name__ + ": " + str(e)

def format_syntax_error(e):
    """ Returns a formatted string of a SyntaxError.
    Stolen from https://github.com/Rapptz/RoboDanny/blob/master/cogs/repl.py#L24-L25 """
    return "{0.text}\n{1:>{0.offset}}\n{2}: {0}".format(e, "^", type(e).__name__).replace("\n\n", "\n")

def get_formatted_code(code):
    """ Format code from markdown format. This will filter out markdown code
    and give the executable python code, or return a string that would raise
    an error when it's executed by exec() or eval(). """
    match = markdown_code_regex.match(code)

    if match:
        code = match.group("code")

        if not code == "`":
            return code

    return "raise Exception(\"Could not format code.\")"

def format_objects(*objects: tuple, attr=None, dec: str= "", sep: str= ", "):
    """ Return a formatted string of objects (User, Member, Channel or Server) using
    the given decorator and the given separator.

    :param attr: The attribute to get from the member. """
    if not objects:
        return

    first_object = objects[0]
    if attr is None:
        if isinstance(first_object, discord.User):
            attr = "display_name"
        elif isinstance(first_object, discord.Channel):
            attr = "mention"
        elif isinstance(first_object, discord.Server):
            attr = "name"

    return sep.join(dec + getattr(m, attr) + dec for m in objects)

def format_channels(*members: discord.Member, attr="mention", dec: str = "`", sep: str = ", "):
    """ Return a formatted string of members (or member) using the given
    decorator and the given separator.

    :param attr: The attribute to get from the member. """
    return sep.join(dec + getattr(m, attr) + dec for m in members)

def split(string, maxsplit=-1):
    """ Split a string with shlex when possible, and add support for maxsplit. """
    if maxsplit == -1:
        try:
            split_object = shlex.shlex(string, posix=True)
            split_object.quotes = '"`'
            split_object.whitespace_split = True
            split_object.commenters = ""
            return list(split_object)
        except ValueError:
            return string.split()

    split_object = shlex.shlex(string, posix=True)
    split_object.quotes = '"`'
    split_object.whitespace_split = True
    split_object.commenters = ""
    maxsplit_object = []
    splits = 0

    while splits < maxsplit:
        maxsplit_object.append(next(split_object))

        splits += 1

    maxsplit_object.append(split_object.instream.read())

    return maxsplit_object
<|endoftext|>


<commit_before>"""
Core models for active learning algorithms.
"""

import numpy as np
from sklearn.utils import check_array
from modAL.utils.validation import check_class_labels, check_class_proba
from modAL.uncertainty import classifier_uncertainty
from modAL.disagreement import vote_entropy
from modAL.query import max_uncertainty

class ActiveLearner:
    """
    This class is an abstract model of a general active learning algorithm.

    Parameters
    ----------
    predictor: scikit-learn estimator
        The estimator to be used in the active learning loop.

    uncertainty_measure: function
        Function providing the uncertainty measure, for instance
        modAL.uncertainty.classifier_uncertainty.

    query_strategy: function
        Function providing the query strategy for the active learning
        loop, for instance modAL.query.max_uncertainty.

    training_samples: None or numpy.ndarray of shape (n_samples, n_features)
        Initial training samples, if available.

    training_labels: None or numpy.ndarray of shape (n_samples, )
        Initial training labels corresponding to initial training samples

    fit_kwargs: keyword arguments for the fit method

    Attributes
    ----------
    predictor: scikit-learn estimator
        The estimator to be used in the active learning loop.

    uncertainty_measure: function
        Function providing the uncertainty measure, for instance
        modAL.uncertainty.classifier_uncertainty.

    query_strategy: function
        Function providing the query strategy for the active learning
        loop, for instance modAL.query.max_uncertainty.

    _training_samples: None numpy.ndarray of shape (n_samples, n_features)
        If the model hasn't been fitted yet: None
        If the model has been fitted already: numpy.ndarray containing the
        samples which the model has been trained on

    _training_labels: None or numpy.ndarray of shape (n_samples, )
        If the model hasn't been fitted yet: None
        If the model has been fitted already: numpy.ndarray containing the
        labels corresponding to _training_samples

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.datasets import load_iris
    >>> from sklearn.ensemble import RandomForestClassifier
    >>> from modAL.models import ActiveLearner
    >>>
    >>> iris = load_iris()
    >>> # give initial training examples
    >>> X_training = iris['data'][[0, 50, 100]]
    >>> y_training = iris['target'][[0, 50, 100]]
    >>>
    >>> # initialize active learner
    >>> learner = ActiveLearner(
    ...     predictor=RandomForestClassifier(),
    ...     X_initial=X_training, y_initial=y_training
    ... )
    >>>
    >>> # the active learning loop
    >>> n_queries = 20
    >>> for idx in range(n_queries):
    ...     query_idx, query_sample = learner.query(iris['data'])
    ...     learner.teach(
    ...         X=iris['data'][query_idx].reshape(1, -1),
    ...         y=iris['target'][query_idx].reshape(1, )
    ...     )

    """
    def __init__(
            self,
            predictor,                                           # scikit-learner estimator object
            uncertainty_measure=classifier_uncertainty,          # callable to measure uncertainty
            query_strategy=max_uncertainty, 		             # callable to query labels
            X_initial=None, y_initial=None,	                     # initial data if available
            **fit_kwargs                                         # keyword arguments for fitting the initial data
    ):
        assert callable(uncertainty_measure), 'utility_function must be callable'
        assert callable(query_strategy), 'query_function must be callable'

        self._predictor = predictor
        self.uncertainty_measure = uncertainty_measure
        self.query_strategy = query_strategy

        if type(X_initial) == type(None) and type(y_initial) == type(None):
            self._X_training = None
            self._y_training = None
        elif type(X_initial) != type(None) and type(y_initial) != type(None):
            self._X_training = check_array(X_initial)
            self._y_training = check_array(y_initial, ensure_2d=False)
            self.fit_to_known(**fit_kwargs)

    def add_training_data(self, X, y):
        """
        Adds the new data and label to the known data, but does
        not retrain the model. Used internally of in stream based
        active learning scenarios.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The new samples for which the labels are supplied
            by the expert.

        y: numpy.ndarray of shape (n_samples, )
            Labels corresponding to the new instances in X.

        Note
        ----
        If the classifier has been fitted, the features in X
        have to agree with the training samples which the
        classifier has seen.
        """

        X, y = check_array(X), check_array(y, ensure_2d=False)
        assert len(X) == len(y), 'the number of new data points and number of labels must match'

        if type(self._X_training) != type(None):
            try:
                self._X_training = np.vstack((self._X_training, X))
                self._y_training = np.concatenate((self._y_training, y))
            except ValueError:
                raise ValueError('the dimensions of the new training data and label must'
                                 'agree with the training data and labels provided so far')

        else:
            self._X_training = X
            self._y_training = y

    def calculate_uncertainty(self, X, **uncertainty_measure_kwargs):
        """
        This method calls the uncertainty measure function provided
        for ActiveLearner upon initialization on the samples passed
        to it. It is used to measure uncertainty for each data point.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples for which the uncertainty of prediction is
            to be calculated.

        uncertainty_measure_kwargs: keyword arguments
            Keyword arguments to be passed to the uncertainty measure
            function.

        Returns
        -------
        uncertainty: numpy.ndarray of shape (n_samples, )
            Contains the uncertainties for the predictions on sample X
        """
        check_array(X)
        return self.uncertainty_measure(self._predictor, X, **uncertainty_measure_kwargs)

    def fit_to_known(self, **fit_kwargs):
        """
        This method fits self.predictor to the training data and labels
        provided to it so far. Used internally or in stream based active
        learning scenarios.

        Parameters
        ----------
        fit_kwargs: keyword arguments
            Keyword arguments to be passed to the fit method of the predictor.

        """
        self._predictor.fit(self._X_training, self._y_training, **fit_kwargs)

    def fit(self, X, y, **fit_kwargs):
        """
        Interface for the fit method of the predictor. Fits the predictor
        to the supplied data, then stores it internally for the active
        learning loop.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples to be fitted.

        y: numpy.ndarray of shape (n_samples, )
            The corresponding labels.

        fit_kwargs: keyword arguments
            Keyword arguments to be passed to the fit method of the predictor.
        """
        self._predictor.fit(X, y, **fit_kwargs)
        self._X_training = X
        self._y_training = y

    def predict(self, X, **predict_kwargs):
        """
        Estimator predictions for X. Interface with the predict
        method of the estimator.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples to be predicted.

        Returns
        -------
        pred: numpy.ndarray of shape (n_samples, )
            Estimator predictions for X.
        """
        return self._predictor.predict(X, **predict_kwargs)

    def predict_proba(self, X, **predict_proba_kwargs):
        """
        Class probabilities if the predictor is a classifier.
        Interface with the predict_proba method of the classifier.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples for which the class probabilities are
            to be predicted.

        Returns
        -------
        proba: numpy.ndarray of shape (n_samples, n_classes)
            Class probabilities for X.
        """
        return self._predictor.predict_proba(X, **predict_proba_kwargs)

    def query(self, X_pool, n_instances=1, **uncertainty_measure_kwargs):
        """
        Finds the n_instances most informative point in the data provided, then
        returns the instances and its indices.

        Parameters
        ----------
        X_pool: numpy.ndarray of shape (n_samples, n_features)
            The pool of samples from which the query strategy should choose
            instances to request labels.

        n_instances: integer
            The number of instances chosen to be labelled by the oracle.

        uncertainty_measure_kwargs: keyword arguments
            Keyword arguments for the uncertainty measure function

        Returns
        -------
        query_idx: numpy.ndarray of shape (n_instances)
            The indices of the instances from X_pool chosen to be labelled.

        X_pool[query_idx]: numpy.ndarray of shape (n_instances, n_features)
            The instances from X_pool choosen to be labelled.
        """

        check_array(X_pool, ensure_2d=True)

        uncertainties = self.calculate_uncertainty(X_pool, **uncertainty_measure_kwargs)
        query_idx = self.query_strategy(uncertainties, n_instances)
        return query_idx, X_pool[query_idx]

    def score(self, X, y, **score_kwargs):
        """
        Interface for the score method
        :param X: np.ndarray of the instances to score
        :param y: np.ndarray of the labels
        :param score_kwargs: keyword arguments
        :return: output of the sklearn.base.ClassifierMixin.score method
        """
        return self._predictor.score(X, y, **score_kwargs)

    def teach(self, X, y, **fit_kwargs):
        """
        This function adds X and y to the known training data
        and retrains the predictor with the augmented dataset.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The new samples for which the labels are supplied
            by the expert.

        y: numpy.ndarray of shape (n_samples, )
            Labels corresponding to the new instances in X.

        fit_kwargs: keyword arguments
            Keyword arguments to be passed to the fit method
            of the predictor.
        """
        self.add_training_data(X, y)
        self.fit_to_known(**fit_kwargs)

class Committee:
    """
    This class is an abstract model of a committee-based active learning algorithm.
    """
    def __init__(
            self,
            learner_list,                                        # list of ActiveLearner objects
            disagreement_measure=vote_entropy,                   # callable to measure disagreement
            query_strategy=max_uncertainty                       # callable to query labels

    ):
        """
        :param learner_list: list of ActiveLearners
        """
        assert type(learner_list) == list, 'learners must be supplied in a list'

        self._learner_list = learner_list
        self.disagreement_measure = disagreement_measure
        self.query_strategy = query_strategy

        self._set_classes()

    def __len__(self):
        return len(self._learner_list)

    def _set_classes(self):
        """
        Checks the known class labels by each learner,
        merges the labels and returns a mapping which
        maps the learner's classes to the complete label
        list
        """

        # assemble the list of known classes from each learner
        self.classes_ = np.unique(
            np.concatenate(tuple(learner._predictor.classes_ for learner in self._learner_list), axis=0),
            axis=0
        )
        self.n_classes_ = len(self.classes_)

    def add_training_data(self, X, y):
        for learner in self._learner_list:
            learner.add_training_data(X, y)
        self._set_classes()

    def calculate_disagreement(self, X, **disagreement_measure_kwargs):
        return self.disagreement_measure(self, X, **disagreement_measure_kwargs)

    def calculate_uncertainty(self, X, **utility_function_kwargs):
        """
        Calculates the uncertainties for every learner in the Committee and returns it
        in the form of a numpy.ndarray
        :param X: numpy.ndarray, data points for which the utilities should be measures
        :return: numpy.ndarray of utilities
        """

        check_array(X, ensure_2d=True)
        uncertainties = np.zeros(shape=(X.shape[0], len(self._learner_list)))

        for learner_idx, learner in enumerate(self._learner_list):
            learner_utility = learner.calculate_uncertainty(X, **utility_function_kwargs)
            uncertainties[:, learner_idx] = learner_utility

        return uncertainties

    def fit_to_known(self, **fit_kwargs):
        for learner in self._learner_list:
            learner.fit_to_known(**fit_kwargs)

    def predict(self, X, **predict_proba_kwargs):
        """
        Predicts the class of the samples by picking
        the average least uncertain prediction.
        """
        # getting average certainties
        proba = self.predict_proba(X, **predict_proba_kwargs)
        # finding the sample-wise max probability
        max_proba_idx = np.argmax(proba, axis=1)
        # translating label indices to labels
        return self.classes_[max_proba_idx]

    def predict_proba(self, X, **predict_proba_kwargs):
        return np.mean(self.vote_proba(X, **predict_proba_kwargs), axis=1)

    def vote(self, X, **predict_kwargs):
        """
        Predicts the labels for the supplied data
        :param X: numpy.ndarray containing the instances to be predicted
        :param predict_kwargs: keyword arguments to be passed for the learners predict method
        :return: numpy.ndarray of shape (n_samples, 1) containing the predictions of all learners
        """

        check_array(X, ensure_2d=True)
        prediction = np.zeros(shape=(X.shape[0], len(self._learner_list)))

        for learner_idx, learner in enumerate(self._learner_list):
            prediction[:, learner_idx] = learner.predict(X, **predict_kwargs)

        return prediction

    def vote_proba(self, X, **predict_proba_kwargs):
        """
        Predicts the probabilities of the classes for each sample and each learner.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples to be predicted by all learners

        predict_proba_kwargs: dict of keyword arguments

        Returns
        -------
        proba: numpy.ndarray of shape (n_samples, n_learners, n_classes)
            Contains the probabilities of each class for each learner and
            each instance

        """

        check_array(X, ensure_2d=True)

        # get dimensions
        n_samples = X.shape[0]
        n_learners = len(self._learner_list)
        proba = np.zeros(shape=(n_samples, n_learners, self.n_classes_))

        # checking if the learners in the Committee know the same set of class labels
        if check_class_labels(*[learner.predictor for learner in self._learner_list]):
            # known class labels are the same for each learner
            # probability prediction is straightforward

            for learner_idx, learner in enumerate(self._learner_list):
                proba[:, learner_idx, :] = learner.predict_proba(X, **predict_proba_kwargs)

        else:
            for learner_idx, learner in enumerate(self._learner_list):
                proba[:, learner_idx, :] = check_class_proba(
                    proba=learner.predict_proba(X, **predict_proba_kwargs),
                    known_labels=learner.predictor.classes_,
                    all_labels=self.classes_
                )

        return proba

    def query(self, X_pool, n_instances=1, **disagreement_measure_kwargs):
        """
        Finds the most informative point in the data provided, then
        returns the instance and its index
        :param X_pool: numpy.ndarray, the pool from which the query is selected
        :return: tuple(query_idx, data[query_idx]), where query_idx is the index of the instance
                 to be queried
        """
        check_array(X_pool, ensure_2d=True)

        disagreement = self.calculate_disagreement(X_pool, **disagreement_measure_kwargs)
        query_idx = self.query_strategy(disagreement, n_instances)
        return query_idx, X_pool[query_idx]

    def teach(self, X, y, **fit_kwargs):
        self.add_training_data(X, y)
        self.fit_to_known(**fit_kwargs)

if __name__ == '__main__':
    import doctest
    doctest.testmod()<commit_msg>modAL.models some methods were set to private<commit_after>"""
Core models for active learning algorithms.
"""

import numpy as np
from sklearn.utils import check_array
from modAL.utils.validation import check_class_labels, check_class_proba
from modAL.uncertainty import classifier_uncertainty
from modAL.disagreement import vote_entropy
from modAL.query import max_uncertainty

class ActiveLearner:
    """
    This class is an abstract model of a general active learning algorithm.

    Parameters
    ----------
    predictor: scikit-learn estimator
        The estimator to be used in the active learning loop.

    uncertainty_measure: function
        Function providing the uncertainty measure, for instance
        modAL.uncertainty.classifier_uncertainty.

    query_strategy: function
        Function providing the query strategy for the active learning
        loop, for instance modAL.query.max_uncertainty.

    training_samples: None or numpy.ndarray of shape (n_samples, n_features)
        Initial training samples, if available.

    training_labels: None or numpy.ndarray of shape (n_samples, )
        Initial training labels corresponding to initial training samples

    fit_kwargs: keyword arguments for the fit method

    Attributes
    ----------
    predictor: scikit-learn estimator
        The estimator to be used in the active learning loop.

    uncertainty_measure: function
        Function providing the uncertainty measure, for instance
        modAL.uncertainty.classifier_uncertainty.

    query_strategy: function
        Function providing the query strategy for the active learning
        loop, for instance modAL.query.max_uncertainty.

    _training_samples: None numpy.ndarray of shape (n_samples, n_features)
        If the model hasn't been fitted yet: None
        If the model has been fitted already: numpy.ndarray containing the
        samples which the model has been trained on

    _training_labels: None or numpy.ndarray of shape (n_samples, )
        If the model hasn't been fitted yet: None
        If the model has been fitted already: numpy.ndarray containing the
        labels corresponding to _training_samples

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.datasets import load_iris
    >>> from sklearn.ensemble import RandomForestClassifier
    >>> from modAL.models import ActiveLearner
    >>>
    >>> iris = load_iris()
    >>> # give initial training examples
    >>> X_training = iris['data'][[0, 50, 100]]
    >>> y_training = iris['target'][[0, 50, 100]]
    >>>
    >>> # initialize active learner
    >>> learner = ActiveLearner(
    ...     predictor=RandomForestClassifier(),
    ...     X_initial=X_training, y_initial=y_training
    ... )
    >>>
    >>> # the active learning loop
    >>> n_queries = 20
    >>> for idx in range(n_queries):
    ...     query_idx, query_sample = learner.query(iris['data'])
    ...     learner.teach(
    ...         X=iris['data'][query_idx].reshape(1, -1),
    ...         y=iris['target'][query_idx].reshape(1, )
    ...     )

    """
    def __init__(
            self,
            predictor,                                           # scikit-learner estimator object
            uncertainty_measure=classifier_uncertainty,          # callable to measure uncertainty
            query_strategy=max_uncertainty, 		             # callable to query labels
            X_initial=None, y_initial=None,	                     # initial data if available
            **fit_kwargs                                         # keyword arguments for fitting the initial data
    ):
        assert callable(uncertainty_measure), 'utility_function must be callable'
        assert callable(query_strategy), 'query_function must be callable'

        self._predictor = predictor
        self.uncertainty_measure = uncertainty_measure
        self.query_strategy = query_strategy

        if type(X_initial) == type(None) and type(y_initial) == type(None):
            self._X_training = None
            self._y_training = None
        elif type(X_initial) != type(None) and type(y_initial) != type(None):
            self._X_training = check_array(X_initial)
            self._y_training = check_array(y_initial, ensure_2d=False)
            self._fit_to_known(**fit_kwargs)

    def _add_training_data(self, X, y):
        """
        Adds the new data and label to the known data, but does
        not retrain the model. Used internally of in stream based
        active learning scenarios.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The new samples for which the labels are supplied
            by the expert.

        y: numpy.ndarray of shape (n_samples, )
            Labels corresponding to the new instances in X.

        Note
        ----
        If the classifier has been fitted, the features in X
        have to agree with the training samples which the
        classifier has seen.
        """

        X, y = check_array(X), check_array(y, ensure_2d=False)
        assert len(X) == len(y), 'the number of new data points and number of labels must match'

        if type(self._X_training) != type(None):
            try:
                self._X_training = np.vstack((self._X_training, X))
                self._y_training = np.concatenate((self._y_training, y))
            except ValueError:
                raise ValueError('the dimensions of the new training data and label must'
                                 'agree with the training data and labels provided so far')

        else:
            self._X_training = X
            self._y_training = y

    def _fit_to_known(self, **fit_kwargs):
        """
        This method fits self.predictor to the training data and labels
        provided to it so far. Used internally or in stream based active
        learning scenarios.

        Parameters
        ----------
        fit_kwargs: keyword arguments
            Keyword arguments to be passed to the fit method of the predictor.

        """
        self._predictor.fit(self._X_training, self._y_training, **fit_kwargs)

    def _calculate_uncertainty(self, X, **uncertainty_measure_kwargs):
        """
        This method calls the uncertainty measure function provided
        for ActiveLearner upon initialization on the samples passed
        to it. It is used to measure uncertainty for each data point.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples for which the uncertainty of prediction is
            to be calculated.

        uncertainty_measure_kwargs: keyword arguments
            Keyword arguments to be passed to the uncertainty measure
            function.

        Returns
        -------
        uncertainty: numpy.ndarray of shape (n_samples, )
            Contains the uncertainties for the predictions on sample X
        """
        check_array(X)
        return self.uncertainty_measure(self._predictor, X, **uncertainty_measure_kwargs)

    def fit(self, X, y, **fit_kwargs):
        """
        Interface for the fit method of the predictor. Fits the predictor
        to the supplied data, then stores it internally for the active
        learning loop.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples to be fitted.

        y: numpy.ndarray of shape (n_samples, )
            The corresponding labels.

        fit_kwargs: keyword arguments
            Keyword arguments to be passed to the fit method of the predictor.
        """
        self._predictor.fit(X, y, **fit_kwargs)
        self._X_training = X
        self._y_training = y

    def predict(self, X, **predict_kwargs):
        """
        Estimator predictions for X. Interface with the predict
        method of the estimator.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples to be predicted.

        Returns
        -------
        pred: numpy.ndarray of shape (n_samples, )
            Estimator predictions for X.
        """
        return self._predictor.predict(X, **predict_kwargs)

    def predict_proba(self, X, **predict_proba_kwargs):
        """
        Class probabilities if the predictor is a classifier.
        Interface with the predict_proba method of the classifier.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples for which the class probabilities are
            to be predicted.

        Returns
        -------
        proba: numpy.ndarray of shape (n_samples, n_classes)
            Class probabilities for X.
        """
        return self._predictor.predict_proba(X, **predict_proba_kwargs)

    def query(self, X_pool, n_instances=1, **uncertainty_measure_kwargs):
        """
        Finds the n_instances most informative point in the data provided, then
        returns the instances and its indices.

        Parameters
        ----------
        X_pool: numpy.ndarray of shape (n_samples, n_features)
            The pool of samples from which the query strategy should choose
            instances to request labels.

        n_instances: integer
            The number of instances chosen to be labelled by the oracle.

        uncertainty_measure_kwargs: keyword arguments
            Keyword arguments for the uncertainty measure function

        Returns
        -------
        query_idx: numpy.ndarray of shape (n_instances)
            The indices of the instances from X_pool chosen to be labelled.

        X_pool[query_idx]: numpy.ndarray of shape (n_instances, n_features)
            The instances from X_pool choosen to be labelled.
        """

        check_array(X_pool, ensure_2d=True)

        uncertainties = self._calculate_uncertainty(X_pool, **uncertainty_measure_kwargs)
        query_idx = self.query_strategy(uncertainties, n_instances)
        return query_idx, X_pool[query_idx]

    def score(self, X, y, **score_kwargs):
        """
        Interface for the score method
        :param X: np.ndarray of the instances to score
        :param y: np.ndarray of the labels
        :param score_kwargs: keyword arguments
        :return: output of the sklearn.base.ClassifierMixin.score method
        """
        return self._predictor.score(X, y, **score_kwargs)

    def teach(self, X, y, **fit_kwargs):
        """
        This function adds X and y to the known training data
        and retrains the predictor with the augmented dataset.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The new samples for which the labels are supplied
            by the expert.

        y: numpy.ndarray of shape (n_samples, )
            Labels corresponding to the new instances in X.

        fit_kwargs: keyword arguments
            Keyword arguments to be passed to the fit method
            of the predictor.
        """
        self._add_training_data(X, y)
        self._fit_to_known(**fit_kwargs)

class Committee:
    """
    This class is an abstract model of a committee-based active learning algorithm.
    """
    def __init__(
            self,
            learner_list,                                        # list of ActiveLearner objects
            disagreement_measure=vote_entropy,                   # callable to measure disagreement
            query_strategy=max_uncertainty                       # callable to query labels

    ):
        """
        :param learner_list: list of ActiveLearners
        """
        assert type(learner_list) == list, 'learners must be supplied in a list'

        self._learner_list = learner_list
        self.disagreement_measure = disagreement_measure
        self.query_strategy = query_strategy

        self._set_classes()

    def __len__(self):
        return len(self._learner_list)

    def _set_classes(self):
        """
        Checks the known class labels by each learner,
        merges the labels and returns a mapping which
        maps the learner's classes to the complete label
        list
        """

        # assemble the list of known classes from each learner
        self.classes_ = np.unique(
            np.concatenate(tuple(learner._predictor.classes_ for learner in self._learner_list), axis=0),
            axis=0
        )
        self.n_classes_ = len(self.classes_)

    def _add_training_data(self, X, y):
        for learner in self._learner_list:
            learner._add_training_data(X, y)
        self._set_classes()

    def _fit_to_known(self, **fit_kwargs):
        for learner in self._learner_list:
            learner._fit_to_known(**fit_kwargs)

    def _calculate_disagreement(self, X, **disagreement_measure_kwargs):
        return self.disagreement_measure(self, X, **disagreement_measure_kwargs)

    def _calculate_uncertainty(self, X, **utility_function_kwargs):
        """
        Calculates the uncertainties for every learner in the Committee and returns it
        in the form of a numpy.ndarray
        :param X: numpy.ndarray, data points for which the utilities should be measures
        :return: numpy.ndarray of utilities
        """

        check_array(X, ensure_2d=True)
        uncertainties = np.zeros(shape=(X.shape[0], len(self._learner_list)))

        for learner_idx, learner in enumerate(self._learner_list):
            learner_utility = learner._calculate_uncertainty(X, **utility_function_kwargs)
            uncertainties[:, learner_idx] = learner_utility

        return uncertainties

    def predict(self, X, **predict_proba_kwargs):
        """
        Predicts the class of the samples by picking
        the average least uncertain prediction.
        """
        # getting average certainties
        proba = self.predict_proba(X, **predict_proba_kwargs)
        # finding the sample-wise max probability
        max_proba_idx = np.argmax(proba, axis=1)
        # translating label indices to labels
        return self.classes_[max_proba_idx]

    def predict_proba(self, X, **predict_proba_kwargs):
        return np.mean(self.vote_proba(X, **predict_proba_kwargs), axis=1)

    def vote(self, X, **predict_kwargs):
        """
        Predicts the labels for the supplied data
        :param X: numpy.ndarray containing the instances to be predicted
        :param predict_kwargs: keyword arguments to be passed for the learners predict method
        :return: numpy.ndarray of shape (n_samples, 1) containing the predictions of all learners
        """

        check_array(X, ensure_2d=True)
        prediction = np.zeros(shape=(X.shape[0], len(self._learner_list)))

        for learner_idx, learner in enumerate(self._learner_list):
            prediction[:, learner_idx] = learner.predict(X, **predict_kwargs)

        return prediction

    def vote_proba(self, X, **predict_proba_kwargs):
        """
        Predicts the probabilities of the classes for each sample and each learner.

        Parameters
        ----------
        X: numpy.ndarray of shape (n_samples, n_features)
            The samples to be predicted by all learners

        predict_proba_kwargs: dict of keyword arguments

        Returns
        -------
        proba: numpy.ndarray of shape (n_samples, n_learners, n_classes)
            Contains the probabilities of each class for each learner and
            each instance

        """

        check_array(X, ensure_2d=True)

        # get dimensions
        n_samples = X.shape[0]
        n_learners = len(self._learner_list)
        proba = np.zeros(shape=(n_samples, n_learners, self.n_classes_))

        # checking if the learners in the Committee know the same set of class labels
        if check_class_labels(*[learner.predictor for learner in self._learner_list]):
            # known class labels are the same for each learner
            # probability prediction is straightforward

            for learner_idx, learner in enumerate(self._learner_list):
                proba[:, learner_idx, :] = learner.predict_proba(X, **predict_proba_kwargs)

        else:
            for learner_idx, learner in enumerate(self._learner_list):
                proba[:, learner_idx, :] = check_class_proba(
                    proba=learner.predict_proba(X, **predict_proba_kwargs),
                    known_labels=learner.predictor.classes_,
                    all_labels=self.classes_
                )

        return proba

    def query(self, X_pool, n_instances=1, **disagreement_measure_kwargs):
        """
        Finds the most informative point in the data provided, then
        returns the instance and its index
        :param X_pool: numpy.ndarray, the pool from which the query is selected
        :return: tuple(query_idx, data[query_idx]), where query_idx is the index of the instance
                 to be queried
        """
        check_array(X_pool, ensure_2d=True)

        disagreement = self._calculate_disagreement(X_pool, **disagreement_measure_kwargs)
        query_idx = self.query_strategy(disagreement, n_instances)
        return query_idx, X_pool[query_idx]

    def teach(self, X, y, **fit_kwargs):
        self._add_training_data(X, y)
        self._fit_to_known(**fit_kwargs)

if __name__ == '__main__':
    import doctest
    doctest.testmod()<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import urllib
import os
import json

from UM.Signal import Signal, SignalEmitter
from UM.Resources import Resources
from UM.Logger import Logger
from UM.Preferences import Preferences

from UM.Settings.MachineDefinition import MachineDefinition
from UM.Settings.MachineInstance import MachineInstance
from UM.Settings.Profile import Profile
from UM.Settings.SettingsError import SettingsError

from UM.i18n import i18nCatalog
catalog = i18nCatalog("uranium")

class MachineManager(SignalEmitter):
    def __init__(self, app_name):
        super().__init__()

        self._application_name = app_name

        self._machine_definitions = []
        self._machine_instances = []
        self._profiles = []

        self._active_machine = None
        self._active_profile = None

        Preferences.getInstance().addPreference("machines/setting_visibility", "")
        Preferences.getInstance().addPreference("machines/active_instance", "")
        Preferences.getInstance().addPreference("machines/active_profile", "")

    def getApplicationName(self):
        return self._application_name

    machineDefinitionsChanged = Signal()

    def getMachineDefinitions(self, **kwargs):
        if kwargs.get("include_variants", True):
            return self._machine_definitions

        definitions = []
        variant_ids = []
        for definition in self._machine_definitions:
            if not definition.getVariantName() or definition.getId() not in variant_ids:
                definitions.append(definition)
                variant_ids.append(definition.getId())

        return definitions

    def getAllMachineVariants(self, machine_id):
        variants = []
        for definition in self._machine_definitions:
            if definition.getId() == machine_id:
                variants.append(definition)

        return variants

    def findMachineDefinition(self, machine_id, variant_name = None):
        for definition in self._machine_definitions:
            if definition.getId() == machine_id:
                if variant_name:
                    if definition.getVariantName() == variant_name:
                        return definition
                else:
                    return definition

        return None

    machineInstancesChanged = Signal()

    def getMachineInstance(self, index):
        return self._machine_instances[index]

    def getMachineInstances(self):
        return self._machine_instances

    def addMachineInstance(self, instance):
        if instance in self._machine_instances:
            return

        self._machine_instances.append(instance)
        self.machineInstancesChanged.emit()

    def removeMachineInstance(self, instance):
        if instance not in self._machine_instances:
            return

        self._machine_instances.remove(instance)

        try:
            path = Resources.getStoragePath(Resources.MachineInstances, urllib.parse.quote_plus(instance.getName()) + ".cfg")
            os.remove(path)
        except FileNotFoundError:
            pass

        self.machineInstancesChanged.emit()

    def findMachineInstance(self, name):
        for instance in self._machine_instances:
            if instance.getName() == name:
                return instance

        return None

    ##  Get the currently active machine instance
    #   \returns active_machine \type{MachineSettings}
    def getActiveMachineInstance(self):
        return self._active_machine

    ##  Set the currently active machine
    #   \param active_machine \type{MachineSettings}
    def setActiveMachineInstance(self, machine):
        if machine == self._active_machine:
            return

        setting_visibility = []
        if self._active_machine:
            setting_visibility = self._active_machine.getAllSettings(visible_only = True)
            setting_visibility = list(map(lambda s: s.getKey(), setting_visibility))

        self._active_machine = machine

        self._updateSettingVisibility(setting_visibility)

        Preferences.getInstance().setValue("machines/active_instance", self._active_machine.getName())

        self.activeMachineInstanceChanged.emit()

    activeMachineInstanceChanged = Signal()

    profilesChanged = Signal()

    def getProfiles(self):
        return self._profiles

    def addProfile(self, profile):
        if profile in self._profiles:
            return

        self._profiles.append(profile)
        self.profilesChanged.emit()

    def removeProfile(self, profile):
        if profile not in self._profiles:
            return

        self._profiles.remove(profile)
        self.profilesChanged.emit()

    def findProfile(self, name):
        for profile in self._profiles:
            if profile.getName() == name:
                return profile

        return None

    activeProfileChanged = Signal()

    def getActiveProfile(self):
        return self._active_profile

    def setActiveProfile(self, profile):
        if profile not in self._profiles or self._active_profile == profile:
            return

        self._active_profile = profile

        Preferences.getInstance().setValue("machines/active_profile", self._active_profile.getName())

        self.activeProfileChanged.emit()

    def loadAll(self):
        self.loadMachineDefinitions()
        self.loadMachineInstances()
        self.loadProfiles()
        self.loadVisibility()

    def loadMachineDefinitions(self):
        dirs = Resources.getAllPathsForType(Resources.MachineDefinitions)
        for dir in dirs:
            if not os.path.isdir(dir):
                continue

            for file_name in os.listdir(dir):
                data = None
                path = os.path.join(dir, file_name)

                if os.path.isdir(path):
                    continue

                definition = MachineDefinition(self)
                try:
                    definition.loadMetaData(path)
                except Exception as e:
                    Logger.log("e", "An error occurred loading Machine Definition %s: %s", path, str(e))
                    continue

                # Only add the definition if it did not exist yet. This prevents duplicates.
                if not self.findMachineDefinition(definition.getId(), definition.getVariantName()):
                    self._machine_definitions.append(definition)

        self.machineDefinitionsChanged.emit()

    def loadMachineInstances(self):
        dirs = Resources.getAllPathsForType(Resources.MachineInstances)
        for dir in dirs:
            if not os.path.isdir(dir):
                continue

            for file_name in os.listdir(dir):
                path = os.path.join(dir, file_name)

                if os.path.isdir(path):
                    continue

                instance = MachineInstance(self)
                try:
                    instance.loadFromFile(path)
                except Exception as e:
                    Logger.log("e", "An exception occurred loading Machine Instance: %s: %s", path, str(e))
                    continue

                if not self.findMachineInstance(instance.getName()):
                    self._machine_instances.append(instance)

        instance = self.findMachineInstance(Preferences.getInstance().getValue("machines/active_instance"))
        if instance:
            self.setActiveMachineInstance(instance)

        self.machineInstancesChanged.emit()

    def loadProfiles(self):
        dirs = Resources.getAllPathsForType(Resources.Profiles)
        for dir in dirs:
            if not os.path.isdir(dir):
                continue

            for file_name in os.listdir(dir):
                path = os.path.join(dir, file_name)

                if os.path.isdir(path):
                    continue

                profile = Profile()
                try:
                    profile.loadFromFile(path)
                except Exception as e:
                    Logger.log("e", "An exception occurred loading Profile %s: %s", path, str(e))
                    continue

                if not self.findProfile(profile.getName()):
                    self._profiles.append(profile)

        profile = self.findProfile(Preferences.getInstance().getValue("machines/active_profile"))
        if profile:
            self.setActiveProfile(profile)

    def loadVisibility(self):
        preference = Preferences.getInstance().getValue("general/setting_visibility")
        if not preference or not self._active_machine:
            return

        self._updateSettingVisibility(preference.split(","))

    def saveAll(self):
        self.saveMachineInstances()
        self.saveProfiles()
        self.saveVisibility()

    def saveMachineInstances(self):
        for instance in self._machine_instances:
            file_name = urllib.parse.quote_plus(instance.getName()) + ".cfg"
            instance.saveToFile(Resources.getStoragePath(Resources.MachineInstances, file_name))

    def saveProfiles(self):
        for profile in self._profiles:
            file_name = urllib.parse.quote_plus(profile.getName()) + ".cfg"
            profile.saveToFile(Resources.getStoragePath(Resources.Profiles, file_name))

    def saveVisibility(self):
        if not self._active_machine:
            Logger.log("w", "No active machine found when trying to save setting visibility")
            return

        visible_settings = self._active_machine.getAllSettings(visible_only = True)
        visible_settings = map(lambda s: s.getKey(), visible_settings)

        preference = ",".join(visible_settings)
        Preferences.getInstance().setValue("machines/setting_visibility", preference)

    def _updateSettingVisibility(self, visible_keys):
        if not visible_keys:
            return

        for setting in self._active_machine.getAllSettings():
            if setting.getKey() in visible_keys:
                setting.setVisible(True)
            else:
                setting.setVisible(False)
<commit_msg>Use the right setting for loading visibility<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import urllib
import os
import json

from UM.Signal import Signal, SignalEmitter
from UM.Resources import Resources
from UM.Logger import Logger
from UM.Preferences import Preferences

from UM.Settings.MachineDefinition import MachineDefinition
from UM.Settings.MachineInstance import MachineInstance
from UM.Settings.Profile import Profile
from UM.Settings.SettingsError import SettingsError

from UM.i18n import i18nCatalog
catalog = i18nCatalog("uranium")

class MachineManager(SignalEmitter):
    def __init__(self, app_name):
        super().__init__()

        self._application_name = app_name

        self._machine_definitions = []
        self._machine_instances = []
        self._profiles = []

        self._active_machine = None
        self._active_profile = None

        Preferences.getInstance().addPreference("machines/setting_visibility", "")
        Preferences.getInstance().addPreference("machines/active_instance", "")
        Preferences.getInstance().addPreference("machines/active_profile", "")

    def getApplicationName(self):
        return self._application_name

    machineDefinitionsChanged = Signal()

    def getMachineDefinitions(self, **kwargs):
        if kwargs.get("include_variants", True):
            return self._machine_definitions

        definitions = []
        variant_ids = []
        for definition in self._machine_definitions:
            if not definition.getVariantName() or definition.getId() not in variant_ids:
                definitions.append(definition)
                variant_ids.append(definition.getId())

        return definitions

    def getAllMachineVariants(self, machine_id):
        variants = []
        for definition in self._machine_definitions:
            if definition.getId() == machine_id:
                variants.append(definition)

        return variants

    def findMachineDefinition(self, machine_id, variant_name = None):
        for definition in self._machine_definitions:
            if definition.getId() == machine_id:
                if variant_name:
                    if definition.getVariantName() == variant_name:
                        return definition
                else:
                    return definition

        return None

    machineInstancesChanged = Signal()

    def getMachineInstance(self, index):
        return self._machine_instances[index]

    def getMachineInstances(self):
        return self._machine_instances

    def addMachineInstance(self, instance):
        if instance in self._machine_instances:
            return

        self._machine_instances.append(instance)
        self.machineInstancesChanged.emit()

    def removeMachineInstance(self, instance):
        if instance not in self._machine_instances:
            return

        self._machine_instances.remove(instance)

        try:
            path = Resources.getStoragePath(Resources.MachineInstances, urllib.parse.quote_plus(instance.getName()) + ".cfg")
            os.remove(path)
        except FileNotFoundError:
            pass

        self.machineInstancesChanged.emit()

    def findMachineInstance(self, name):
        for instance in self._machine_instances:
            if instance.getName() == name:
                return instance

        return None

    ##  Get the currently active machine instance
    #   \returns active_machine \type{MachineSettings}
    def getActiveMachineInstance(self):
        return self._active_machine

    ##  Set the currently active machine
    #   \param active_machine \type{MachineSettings}
    def setActiveMachineInstance(self, machine):
        if machine == self._active_machine:
            return

        setting_visibility = []
        if self._active_machine:
            setting_visibility = self._active_machine.getAllSettings(visible_only = True)
            setting_visibility = list(map(lambda s: s.getKey(), setting_visibility))

        self._active_machine = machine

        self._updateSettingVisibility(setting_visibility)

        Preferences.getInstance().setValue("machines/active_instance", self._active_machine.getName())

        self.activeMachineInstanceChanged.emit()

    activeMachineInstanceChanged = Signal()

    profilesChanged = Signal()

    def getProfiles(self):
        return self._profiles

    def addProfile(self, profile):
        if profile in self._profiles:
            return

        self._profiles.append(profile)
        self.profilesChanged.emit()

    def removeProfile(self, profile):
        if profile not in self._profiles:
            return

        self._profiles.remove(profile)
        self.profilesChanged.emit()

    def findProfile(self, name):
        for profile in self._profiles:
            if profile.getName() == name:
                return profile

        return None

    activeProfileChanged = Signal()

    def getActiveProfile(self):
        return self._active_profile

    def setActiveProfile(self, profile):
        if profile not in self._profiles or self._active_profile == profile:
            return

        self._active_profile = profile

        Preferences.getInstance().setValue("machines/active_profile", self._active_profile.getName())

        self.activeProfileChanged.emit()

    def loadAll(self):
        self.loadMachineDefinitions()
        self.loadMachineInstances()
        self.loadProfiles()
        self.loadVisibility()

    def loadMachineDefinitions(self):
        dirs = Resources.getAllPathsForType(Resources.MachineDefinitions)
        for dir in dirs:
            if not os.path.isdir(dir):
                continue

            for file_name in os.listdir(dir):
                data = None
                path = os.path.join(dir, file_name)

                if os.path.isdir(path):
                    continue

                definition = MachineDefinition(self)
                try:
                    definition.loadMetaData(path)
                except Exception as e:
                    Logger.log("e", "An error occurred loading Machine Definition %s: %s", path, str(e))
                    continue

                # Only add the definition if it did not exist yet. This prevents duplicates.
                if not self.findMachineDefinition(definition.getId(), definition.getVariantName()):
                    self._machine_definitions.append(definition)

        self.machineDefinitionsChanged.emit()

    def loadMachineInstances(self):
        dirs = Resources.getAllPathsForType(Resources.MachineInstances)
        for dir in dirs:
            if not os.path.isdir(dir):
                continue

            for file_name in os.listdir(dir):
                path = os.path.join(dir, file_name)

                if os.path.isdir(path):
                    continue

                instance = MachineInstance(self)
                try:
                    instance.loadFromFile(path)
                except Exception as e:
                    Logger.log("e", "An exception occurred loading Machine Instance: %s: %s", path, str(e))
                    continue

                if not self.findMachineInstance(instance.getName()):
                    self._machine_instances.append(instance)

        instance = self.findMachineInstance(Preferences.getInstance().getValue("machines/active_instance"))
        if instance:
            self.setActiveMachineInstance(instance)

        self.machineInstancesChanged.emit()

    def loadProfiles(self):
        dirs = Resources.getAllPathsForType(Resources.Profiles)
        for dir in dirs:
            if not os.path.isdir(dir):
                continue

            for file_name in os.listdir(dir):
                path = os.path.join(dir, file_name)

                if os.path.isdir(path):
                    continue

                profile = Profile()
                try:
                    profile.loadFromFile(path)
                except Exception as e:
                    Logger.log("e", "An exception occurred loading Profile %s: %s", path, str(e))
                    continue

                if not self.findProfile(profile.getName()):
                    self._profiles.append(profile)

        profile = self.findProfile(Preferences.getInstance().getValue("machines/active_profile"))
        if profile:
            self.setActiveProfile(profile)

    def loadVisibility(self):
        preference = Preferences.getInstance().getValue("machines/setting_visibility")
        if not preference or not self._active_machine:
            return

        self._updateSettingVisibility(preference.split(","))

    def saveAll(self):
        self.saveMachineInstances()
        self.saveProfiles()
        self.saveVisibility()

    def saveMachineInstances(self):
        for instance in self._machine_instances:
            file_name = urllib.parse.quote_plus(instance.getName()) + ".cfg"
            instance.saveToFile(Resources.getStoragePath(Resources.MachineInstances, file_name))

    def saveProfiles(self):
        for profile in self._profiles:
            file_name = urllib.parse.quote_plus(profile.getName()) + ".cfg"
            profile.saveToFile(Resources.getStoragePath(Resources.Profiles, file_name))

    def saveVisibility(self):
        if not self._active_machine:
            Logger.log("w", "No active machine found when trying to save setting visibility")
            return

        visible_settings = self._active_machine.getAllSettings(visible_only = True)
        visible_settings = map(lambda s: s.getKey(), visible_settings)

        preference = ",".join(visible_settings)
        Preferences.getInstance().setValue("machines/setting_visibility", preference)

    def _updateSettingVisibility(self, visible_keys):
        if not visible_keys:
            return

        for setting in self._active_machine.getAllSettings():
            if setting.getKey() in visible_keys:
                setting.setVisible(True)
            else:
                setting.setVisible(False)
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

import os
import sys
import math
import collections
from collections import OrderedDict
import re
from operator import add

import string
from string import digits
from string import punctuation

from pyspark import SparkContext

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: wordcount <file>", file=sys.stderr)
        exit(-1)
    sc = SparkContext(appName="PythonWordCount")
    line = sc.textFile("file:///mounted_volume/books/*.txt")
    lines = sc.textFile(sys.argv[1], 1)
    # counts = lines.flatMap(lambda x: x.split(' ')) \
    #               .map(lambda x: (x, 1)) \
    #               .reduceByKey(add)

    counts = sc.wholeTextFiles(sys.argv[1], 1)\
		.flatMap(lambda (name, content): map(lambda word: (word, name), content.split(' ')))\
		.map(lambda (word, name): ((word, name), 1))\
		.reduceByKey(lambda count1, count2: count1 + count2)\
        .map(lambda ((word, name), count): (word, name, count))

    output = counts.collect()
    ii = {}
    Results = {}
    fileList =[]

    for (word, name, count) in output:
        word = word.lower()
        i = name.find("/books/") + 7
        j = name.find(".txt")
        name = name[i:j]

        if word not in ii:
            ii[word] = {}
            ii[word][name] = count
        else:
            ii[word][name] = count

        #print("%s -> %s: %i" % (word.encode("utf-8"), name.encode("utf-8"), count))
        #if word not in ii:
        #   ii[word] = [(name, count)]
        #   # print("Added new %s" % (word.encode("utf-8")))
        #else:
        #    temp = ii[word]
        #    temp.append((name, count))
        #    ii[word] = temp
            # print("Added to %s" % (word.encode("utf-8")))

        if name not in fileList:
            fileList.append(name)

    numFiles = len(fileList)

    loop = True

    while loop:
        search = sys.argv[2]
        terms = search.split()

        for test in terms:
            if test == 'QUIT':
                loop = False

        for term in terms:                                                            #loop through keywords
            if term not in Results:                                                          #Only calculate if not already done
                if term in ii:                                                                 # Only calculate if keyword in any file
                    Results[term] = {}                                                               #Create Results dict
                    for fileName in fileList:                                                           #Loop through files
                        freq = 0                                                                            #initializ freq to zero
                        try:                                                                                #handle exceptions where keyword isnt in file
                            freq = ii[term][fileName]                                                      #calculate frequency of keyword in file
                        except KeyError:                                                                    # handle exceptions where keyword isnt in file
                            Results[term][fileName] = 0.000000                                           # handle exceptions where keyword isnt in file
                        if (freq > 0):                                                                      #calculate TF
                            tf = 1 + math.log(freq, 2)                                                          # as 1+log2(freq) or 0
                        else:                                                                                # depending on
                            tf = 0                                                                              # Frequency
                        IDF = math.log((float(numFiles) / len(ii[term])), 2)                           #calculate IDF = log2(N/n)
                        weight = tf * IDF                                                                   #calculate weight = TF * IDF
                        Results[term][fileName] = weight                                                 #add weight for keyword, file
                else:                                                                               #Handle cases where
                    for fileName in fileList:                                                           # keyword isnt in
                        if term not in Results:                                                          # any file
                            Results[term] = {fileName : 0.000000}                                             # if first time
                        else:                                                                               #
                            Results[term][fileName] = 0.000000                                               # if already in results for some file

        for searchTerm in Results:
            termResults = Results[term]
            sortedTermResults = sorted(termResults, key=lambda x: termResults[1])
            print (searchterm + "_____________________________")
            for doc in termResults.keys:
                print ("----Document: "+ doc)
                print ("------------Weight: "+termResults[doc])
    sc.stop()
<commit_msg>Cleaned up stuff and print results<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import print_function

import os
import sys
import math
import collections
from collections import OrderedDict
import re
from operator import add

import string
from string import digits
from string import punctuation

from pyspark import SparkContext

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: wordcount <file>", file=sys.stderr)
        exit(-1)
    sc = SparkContext(appName="PythonWordCount")
    line = sc.textFile("file:///mounted_volume/books/*.txt")
    lines = sc.textFile(sys.argv[1], 1)
    # counts = lines.flatMap(lambda x: x.split(' ')) \
    #               .map(lambda x: (x, 1)) \
    #               .reduceByKey(add)

    counts = sc.wholeTextFiles(sys.argv[1], 1)\
		.flatMap(lambda (name, content): map(lambda word: (word, name), content.split(' ')))\
		.map(lambda (word, name): ((word, name), 1))\
		.reduceByKey(lambda count1, count2: count1 + count2)\
        .map(lambda ((word, name), count): (word, name, count))

    output = counts.collect()
    ii = {}
    Results = {}
    fileList =[]

    for (word, name, count) in output:
        word = word.lower()
        i = name.find("/books/") + 7
        j = name.find(".txt")
        name = name[i:j]

        if word not in ii:
            print("Added %s to ii with word %s and count %i" % (name.encode("utf-8"), word.encode("utf-8"), count))
            ii[word] = {}
            ii[word][name] = count
        else:
            print("updated count for: %s to ii with word %s and to new count %i" % (name.encode("utf-8"), word.encode("utf-8"), count))
            ii[word][name] = count

        if name not in fileList:
            fileList.append(name)

    numFiles = len(fileList)

    search = sys.argv[2]
    terms = search.split()

    for term in terms:                                                            #loop through keywords
        if term not in Results:                                                          #Only calculate if not already done
            if term in ii:                                                                 # Only calculate if keyword in any file
                Results[term] = {}                                                               #Create Results dict
                for fileName in fileList:                                                           #Loop through files
                    freq = 0                                                                            #initializ freq to zero
                    try:                                                                                #handle exceptions where keyword isnt in file
                        freq = ii[term][fileName]                                                      #calculate frequency of keyword in file
                    except KeyError:                                                                    # handle exceptions where keyword isnt in file
                        Results[term][fileName] = 0.000000                                           # handle exceptions where keyword isnt in file
                    if (freq > 0):                                                                      #calculate TF
                        tf = 1 + math.log(freq, 2)                                                          # as 1+log2(freq) or 0
                    else:                                                                                # depending on
                        tf = 0                                                                              # Frequency
                    IDF = math.log((float(numFiles) / len(ii[term])), 2)                           #calculate IDF = log2(N/n)
                    weight = tf * IDF                                                                   #calculate weight = TF * IDF
                    Results[term][fileName] = weight                                                 #add weight for keyword, file
            else:                                                                               #Handle cases where
                for fileName in fileList:                                                           # keyword isnt in
                    if term not in Results:                                                          # any file
                        Results[term] = {fileName : 0.000000}                                             # if first time
                    else:                                                                               #
                        Results[term][fileName] = 0.000000                                               # if already in results for some file

    for searchTerm in Results.keys():
        termResults = Results[term]
        sortedTermResults = sorted(termResults, key=lambda x: termResults.values())
        print (searchTerm + "_____________________________")
        for doc in termResults.keys():
            print ("----Document: "+ doc)
            print ("------------Weight: "+str(termResults[doc]))
    sc.stop()
<|endoftext|>