<commit_before>#!/usr/bin/env python

"""models.py

Udacity conference server-side Python App Engine data & ProtoRPC models

$Id: models.py,v 1.1 2014/05/24 22:01:10 wesc Exp $

created/forked from conferences.py by wesc on 2014 may 24

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

import httplib
import endpoints
from protorpc import messages
from google.appengine.ext import ndb

class ConflictException(endpoints.ServiceException):
    """ConflictException -- exception mapped to HTTP 409 response"""
    http_status = httplib.CONFLICT

class Profile(ndb.Model):
    """Profile -- User profile object"""
    displayName = ndb.StringProperty()
    mainEmail = ndb.StringProperty()
    teeShirtSize = ndb.StringProperty(default='NOT_SPECIFIED')
    conferenceKeysToAttend = ndb.StringProperty(repeated=True)
    sessionWishlist = ndb.StringProperty(repeated=True)

class ProfileMiniForm(messages.Message):
    """ProfileMiniForm -- update Profile form message"""
    displayName = messages.StringField(1)
    teeShirtSize = messages.EnumField('TeeShirtSize', 2)

class ProfileForm(messages.Message):
    """ProfileForm -- Profile outbound form message"""
    displayName = messages.StringField(1)
    mainEmail = messages.StringField(2)
    teeShirtSize = messages.EnumField('TeeShirtSize', 3)
    conferenceKeysToAttend = messages.StringField(4, repeated=True)

class BooleanMessage(messages.Message):
    """BooleanMessage-- outbound Boolean value message"""
    data = messages.BooleanField(1)

class Conference(ndb.Model):
    """Conference -- Conference object"""
    name            = ndb.StringProperty(required=True)
    description     = ndb.StringProperty()
    organizerUserId = ndb.StringProperty()
    topics          = ndb.StringProperty(repeated=True)
    city            = ndb.StringProperty()
    startDate       = ndb.DateProperty()
    month           = ndb.IntegerProperty() # TODO: do we need for indexing like Java?
    endDate         = ndb.DateProperty()
    maxAttendees    = ndb.IntegerProperty()
    seatsAvailable  = ndb.IntegerProperty()

class ConferenceForm(messages.Message):
    """ConferenceForm -- Conference outbound form message"""
    name            = messages.StringField(1)
    description     = messages.StringField(2)
    organizerUserId = messages.StringField(3)
    topics          = messages.StringField(4, repeated=True)
    city            = messages.StringField(5)
    startDate       = messages.StringField(6) #DateTimeField()
    month           = messages.IntegerField(7)
    maxAttendees    = messages.IntegerField(8)
    seatsAvailable  = messages.IntegerField(9)
    endDate         = messages.StringField(10) #DateTimeField()
    websafeKey      = messages.StringField(11)
    organizerDisplayName = messages.StringField(12)

class Session(ndb.Model):
    """Session -- Session object"""
    name            = ndb.StringProperty(required=True)
    speaker         = ndb.KeyProperty(kind='Speaker')
    highlights      = ndb.StringProperty(repeated=True)
    duration        = ndb.IntegerProperty()
    typeOfSession   = ndb.StringProperty(choices=['workshop', 'keynotes', 'breakout'])  # list with possible types of sessions
    startDate       = ndb.DateProperty()
    startTime       = ndb.TimeProperty()
    conference = ndb.KeyProperty(kind='Conference')

    @classmethod
    def get_session_by_conferencekey(cls, confwebsafekey):
        return cls.query(cls.conference == confwebsafekey)

class SessionForm(messages.Message):
    """SessionForm -- Session outbound form message"""
    name            = messages.StringField(1)
    speaker         = messages.StringField(2)
    highlights      = messages.StringField(3, repeated=True)
    duration        = messages.IntegerField(4)
    typeOfSession   = messages.StringField(5)
    startDate       = messages.StringField(6) #DateField()
    startTime       = messages.StringField(7) #TimeField()
    sessionKey      = messages.StringField(8)

class Speaker(ndb.Model):
    """Speaker -- Speaker object"""
    name            = ndb.StringProperty(required=True)
    email           = ndb.StringProperty()

class SessionForms(messages.Message):
    """SessionForms -- multiple Sessions outbound form message"""
    items = messages.MessageField(SessionForm, 1, repeated=True)

class ConferenceForms(messages.Message):
    """ConferenceForms -- multiple Conference outbound form message"""
    items = messages.MessageField(ConferenceForm, 1, repeated=True)

class TeeShirtSize(messages.Enum):
    """TeeShirtSize -- t-shirt size enumeration value"""
    NOT_SPECIFIED = 1
    XS_M = 2
    XS_W = 3
    S_M = 4
    S_W = 5
    M_M = 6
    M_W = 7
    L_M = 8
    L_W = 9
    XL_M = 10
    XL_W = 11
    XXL_M = 12
    XXL_W = 13
    XXXL_M = 14
    XXXL_W = 15

class ConferenceQueryForm(messages.Message):
    """ConferenceQueryForm -- Conference query inbound form message"""
    field = messages.StringField(1)
    operator = messages.StringField(2)
    value = messages.StringField(3)

class ConferenceQueryForms(messages.Message):
    """ConferenceQueryForms -- multiple ConferenceQueryForm inbound form message"""
    filters = messages.MessageField(ConferenceQueryForm, 1, repeated=True)

class StringMessage(messages.Message):
    """StringMessage-- outbound (single) string message"""
    data = messages.StringField(1, required=True)

class MultiStringMessage(messages.Message):
    """StringMessage-- outbound (single) string message"""
    data = messages.StringField(1, repeated=True)
<commit_msg>Changes after review final<commit_after>#!/usr/bin/env python

"""models.py

Udacity conference server-side Python App Engine data & ProtoRPC models

$Id: models.py,v 1.1 2014/05/24 22:01:10 wesc Exp $

created/forked from conferences.py by wesc on 2014 may 24

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

import httplib
import endpoints
from protorpc import messages
from google.appengine.ext import ndb

class ConflictException(endpoints.ServiceException):
    """ConflictException -- exception mapped to HTTP 409 response"""
    http_status = httplib.CONFLICT

class Profile(ndb.Model):
    """Profile -- User profile object"""
    displayName = ndb.StringProperty()
    mainEmail = ndb.StringProperty()
    teeShirtSize = ndb.StringProperty(default='NOT_SPECIFIED')
    conferenceKeysToAttend = ndb.StringProperty(repeated=True)
    sessionWishlist = ndb.StringProperty(repeated=True)

class ProfileMiniForm(messages.Message):
    """ProfileMiniForm -- update Profile form message"""
    displayName = messages.StringField(1)
    teeShirtSize = messages.EnumField('TeeShirtSize', 2)

class ProfileForm(messages.Message):
    """ProfileForm -- Profile outbound form message"""
    displayName = messages.StringField(1)
    mainEmail = messages.StringField(2)
    teeShirtSize = messages.EnumField('TeeShirtSize', 3)
    conferenceKeysToAttend = messages.StringField(4, repeated=True)

class BooleanMessage(messages.Message):
    """BooleanMessage-- outbound Boolean value message"""
    data = messages.BooleanField(1)

class Conference(ndb.Model):
    """Conference -- Conference object"""
    name            = ndb.StringProperty(required=True)
    description     = ndb.StringProperty()
    organizerUserId = ndb.StringProperty()
    topics          = ndb.StringProperty(repeated=True)
    city            = ndb.StringProperty()
    startDate       = ndb.DateProperty()
    month           = ndb.IntegerProperty() # TODO: do we need for indexing like Java?
    endDate         = ndb.DateProperty()
    maxAttendees    = ndb.IntegerProperty()
    seatsAvailable  = ndb.IntegerProperty()

class ConferenceForm(messages.Message):
    """ConferenceForm -- Conference outbound form message"""
    name            = messages.StringField(1)
    description     = messages.StringField(2)
    organizerUserId = messages.StringField(3)
    topics          = messages.StringField(4, repeated=True)
    city            = messages.StringField(5)
    startDate       = messages.StringField(6) #DateTimeField()
    month           = messages.IntegerField(7)
    maxAttendees    = messages.IntegerField(8)
    seatsAvailable  = messages.IntegerField(9)
    endDate         = messages.StringField(10) #DateTimeField()
    websafeKey      = messages.StringField(11)
    organizerDisplayName = messages.StringField(12)

class Session(ndb.Model):
    """Session -- Session object"""
    name            = ndb.StringProperty(required=True)
    speaker         = ndb.StringProperty()
    highlights      = ndb.StringProperty(repeated=True)
    duration        = ndb.IntegerProperty()
    typeOfSession   = ndb.StringProperty(choices=['workshop', 'keynotes', 'breakout'])  # list with possible types of sessions
    startDate       = ndb.DateProperty()
    startTime       = ndb.TimeProperty()
    conference = ndb.KeyProperty(kind='Conference')

    @classmethod
    def get_session_by_conferencekey(cls, confwebsafekey):
        return cls.query(cls.conference == confwebsafekey)

class SessionForm(messages.Message):
    """SessionForm -- Session outbound form message"""
    name            = messages.StringField(1)
    speaker         = messages.StringField(2)
    highlights      = messages.StringField(3, repeated=True)
    duration        = messages.IntegerField(4)
    typeOfSession   = messages.StringField(5)
    startDate       = messages.StringField(6) #DateField()
    startTime       = messages.StringField(7) #TimeField()
    sessionKey      = messages.StringField(8)

class Speaker(ndb.Model):
    """Speaker -- Speaker object"""
    name            = ndb.StringProperty(required=True)
    email           = ndb.StringProperty()

class SessionForms(messages.Message):
    """SessionForms -- multiple Sessions outbound form message"""
    items = messages.MessageField(SessionForm, 1, repeated=True)

class ConferenceForms(messages.Message):
    """ConferenceForms -- multiple Conference outbound form message"""
    items = messages.MessageField(ConferenceForm, 1, repeated=True)

class TeeShirtSize(messages.Enum):
    """TeeShirtSize -- t-shirt size enumeration value"""
    NOT_SPECIFIED = 1
    XS_M = 2
    XS_W = 3
    S_M = 4
    S_W = 5
    M_M = 6
    M_W = 7
    L_M = 8
    L_W = 9
    XL_M = 10
    XL_W = 11
    XXL_M = 12
    XXL_W = 13
    XXXL_M = 14
    XXXL_W = 15

class ConferenceQueryForm(messages.Message):
    """ConferenceQueryForm -- Conference query inbound form message"""
    field = messages.StringField(1)
    operator = messages.StringField(2)
    value = messages.StringField(3)

class ConferenceQueryForms(messages.Message):
    """ConferenceQueryForms -- multiple ConferenceQueryForm inbound form message"""
    filters = messages.MessageField(ConferenceQueryForm, 1, repeated=True)

class StringMessage(messages.Message):
    """StringMessage-- outbound (single) string message"""
    data = messages.StringField(1, required=True)

class MultiStringMessage(messages.Message):
    """StringMessage-- outbound (single) string message"""
    data = messages.StringField(1, repeated=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Parse command line and call the appropriate functions."""
import argparse
import logging
import os
import sys

import zope.component
import zope.interface

from letsencrypt.client import CONFIG
from letsencrypt.client import client
from letsencrypt.client import display
from letsencrypt.client import interfaces
from letsencrypt.client import log
from letsencrypt.client import revoker
from letsencrypt.client.apache import configurator

def main():  # pylint: disable=too-many-statements
    """Command line argument parsing and main script execution."""
    if not os.geteuid() == 0:
        sys.exit(
            "{0}Root is required to run letsencrypt.  Please use sudo.{0}"
            .format(os.linesep))

    parser = argparse.ArgumentParser(
        description="An ACME client that can update Apache configurations.")

    parser.add_argument("-d", "--domains", dest="domains", metavar="DOMAIN",
                        nargs="+")
    parser.add_argument("-s", "--server", dest="server",
                        default=CONFIG.ACME_SERVER,
                        help="The ACME CA server. [%(default)s]")
    parser.add_argument("-p", "--privkey", dest="privkey", type=read_file,
                        help="Path to the private key file for certificate "
                             "generation.")
    parser.add_argument("-b", "--rollback", dest="rollback", type=int,
                        default=0, metavar="N",
                        help="Revert configuration N number of checkpoints.")
    parser.add_argument("-B", "--keysize", dest="key_size", type=int,
                        default=CONFIG.RSA_KEY_SIZE, metavar="N",
                        help="RSA key shall be sized N bits. [%(default)d]")
    parser.add_argument("-k", "--revoke", dest="revoke", action="store_true",
                        help="Revoke a certificate.")
    parser.add_argument("-v", "--view-config-changes",
                        dest="view_config_changes",
                        action="store_true",
                        help="View checkpoints and associated configuration "
                             "changes.")
    parser.add_argument("-r", "--redirect", dest="redirect",
                        action="store_const", const=True,
                        help="Automatically redirect all HTTP traffic to HTTPS "
                             "for the newly authenticated vhost.")
    parser.add_argument("-n", "--no-redirect", dest="redirect",
                        action="store_const", const=False,
                        help="Skip the HTTPS redirect question, allowing both "
                             "HTTP and HTTPS.")
    parser.add_argument("-e", "--agree-tos", dest="eula", action="store_true",
                        help="Skip the end user license agreement screen.")
    parser.add_argument("-t", "--text", dest="use_curses", action="store_false",
                        help="Use the text output instead of the curses UI.")
    parser.add_argument("--test", dest="test", action="store_true",
                        help="Run in test mode.")

    args = parser.parse_args()

    # Set up logging
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    if args.use_curses:
        logger.addHandler(log.DialogHandler())
        displayer = display.NcursesDisplay()
    else:
        displayer = display.FileDisplay(sys.stdout)
    zope.component.provideUtility(displayer)

    installer = determine_installer()

    if args.revoke:
        revoc = revoker.Revoker(args.server, installer)
        revoc.list_certs_keys()
        sys.exit()

    if args.rollback > 0:
        rollback(installer, args.rollback)
        sys.exit()

    if args.view_config_changes:
        view_config_changes(installer)
        sys.exit()

    if not args.eula:
        display_eula()

    # Use the same object if possible
    if interfaces.IAuthenticator.providedBy(installer):  # pylint: disable=no-member

        auth = installer
    else:
        auth = determine_authenticator()

    domains = choose_names(installer) if args.domains is None else args.domains

    # Prepare for init of Client
    if args.privkey is None:
        privkey = client.init_key(args.key_size)
    else:
        privkey = client.Client.Key(args.privkey[0], args.privkey[1])

    acme = client.Client(args.server, privkey, auth, installer)

    # Validate the key and csr
    client.validate_key_csr(privkey)

    cert_file, chain_file = acme.obtain_certificate(domains)
    acme.deploy_certificate(domains, privkey, cert_file, chain_file)
    acme.enhance_config(domains, args.redirect)

def display_eula():
    """Displays the end user agreement."""
    with open('EULA') as eula_file:
        if not zope.component.getUtility(interfaces.IDisplay).generic_yesno(
                eula_file.read(), "Agree", "Cancel"):
            sys.exit(0)

def choose_names(installer):
    """Display screen to select domains to validate.

    :param installer: An installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    """
    # This function adds all names
    # found within the config to self.names
    # Then filters them based on user selection
    code, names = zope.component.getUtility(
        interfaces.IDisplay).filter_names(get_all_names(installer))
    if code == display.OK and names:
        return names
    else:
        sys.exit(0)

def get_all_names(installer):
    """Return all valid names in the configuration.

    :param installer: An installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    """
    names = list(installer.get_all_names())
    client.sanity_check_names(names)

    if not names:
        logging.fatal("No domain names were found in your installation")
        logging.fatal("Either specify which names you would like "
                      "letsencrypt to validate or add server names "
                      "to your virtual hosts")
        sys.exit(1)

    return names

# This should be controlled by commandline parameters
def determine_authenticator():
    """Returns a valid IAuthenticator."""
    return configurator.ApacheConfigurator()

def determine_installer():
    """Returns a valid IInstaller."""
    return configurator.ApacheConfigurator()

def read_file(filename):
    """Returns the given file's contents with universal new line support.

    :param str filename: Filename

    :returns: A tuple of filename and its contents
    :rtype: tuple

    :raises argparse.ArgumentTypeError: File does not exist or is not readable.

    """
    try:
        return filename, open(filename, 'rU').read()
    except IOError as exc:
        raise argparse.ArgumentTypeError(exc.strerror)

def rollback(installer, checkpoints):
    """Revert configuration the specified number of checkpoints.

    :param installer: Installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    :param int checkpoints: Number of checkpoints to revert.

    """
    installer.rollback_checkpoints(checkpoints)
    installer.restart()

def view_config_changes(installer):
    """View checkpoints and associated configuration changes.

    :param installer: Installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    """
    installer.view_config_changes()

if __name__ == "__main__":
    main()
<commit_msg>whitespace<commit_after>#!/usr/bin/env python
"""Parse command line and call the appropriate functions."""
import argparse
import logging
import os
import sys

import zope.component
import zope.interface

from letsencrypt.client import CONFIG
from letsencrypt.client import client
from letsencrypt.client import display
from letsencrypt.client import interfaces
from letsencrypt.client import log
from letsencrypt.client import revoker
from letsencrypt.client.apache import configurator

def main():  # pylint: disable=too-many-statements
    """Command line argument parsing and main script execution."""
    if not os.geteuid() == 0:
        sys.exit(
            "{0}Root is required to run letsencrypt.  Please use sudo.{0}"
            .format(os.linesep))

    parser = argparse.ArgumentParser(
        description="An ACME client that can update Apache configurations.")

    parser.add_argument("-d", "--domains", dest="domains", metavar="DOMAIN",
                        nargs="+")
    parser.add_argument("-s", "--server", dest="server",
                        default=CONFIG.ACME_SERVER,
                        help="The ACME CA server. [%(default)s]")
    parser.add_argument("-p", "--privkey", dest="privkey", type=read_file,
                        help="Path to the private key file for certificate "
                             "generation.")
    parser.add_argument("-b", "--rollback", dest="rollback", type=int,
                        default=0, metavar="N",
                        help="Revert configuration N number of checkpoints.")
    parser.add_argument("-B", "--keysize", dest="key_size", type=int,
                        default=CONFIG.RSA_KEY_SIZE, metavar="N",
                        help="RSA key shall be sized N bits. [%(default)d]")
    parser.add_argument("-k", "--revoke", dest="revoke", action="store_true",
                        help="Revoke a certificate.")
    parser.add_argument("-v", "--view-config-changes",
                        dest="view_config_changes",
                        action="store_true",
                        help="View checkpoints and associated configuration "
                             "changes.")
    parser.add_argument("-r", "--redirect", dest="redirect",
                        action="store_const", const=True,
                        help="Automatically redirect all HTTP traffic to HTTPS "
                             "for the newly authenticated vhost.")
    parser.add_argument("-n", "--no-redirect", dest="redirect",
                        action="store_const", const=False,
                        help="Skip the HTTPS redirect question, allowing both "
                             "HTTP and HTTPS.")
    parser.add_argument("-e", "--agree-tos", dest="eula", action="store_true",
                        help="Skip the end user license agreement screen.")
    parser.add_argument("-t", "--text", dest="use_curses", action="store_false",
                        help="Use the text output instead of the curses UI.")
    parser.add_argument("--test", dest="test", action="store_true",
                        help="Run in test mode.")

    args = parser.parse_args()

    # Set up logging
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    if args.use_curses:
        logger.addHandler(log.DialogHandler())
        displayer = display.NcursesDisplay()
    else:
        displayer = display.FileDisplay(sys.stdout)
    zope.component.provideUtility(displayer)

    installer = determine_installer()

    if args.revoke:
        revoc = revoker.Revoker(args.server, installer)
        revoc.list_certs_keys()
        sys.exit()

    if args.rollback > 0:
        rollback(installer, args.rollback)
        sys.exit()

    if args.view_config_changes:
        view_config_changes(installer)
        sys.exit()

    if not args.eula:
        display_eula()

    # Use the same object if possible
    if interfaces.IAuthenticator.providedBy(installer):  # pylint: disable=no-member
        auth = installer
    else:
        auth = determine_authenticator()

    domains = choose_names(installer) if args.domains is None else args.domains

    # Prepare for init of Client
    if args.privkey is None:
        privkey = client.init_key(args.key_size)
    else:
        privkey = client.Client.Key(args.privkey[0], args.privkey[1])

    acme = client.Client(args.server, privkey, auth, installer)

    # Validate the key and csr
    client.validate_key_csr(privkey)

    cert_file, chain_file = acme.obtain_certificate(domains)
    acme.deploy_certificate(domains, privkey, cert_file, chain_file)
    acme.enhance_config(domains, args.redirect)

def display_eula():
    """Displays the end user agreement."""
    with open('EULA') as eula_file:
        if not zope.component.getUtility(interfaces.IDisplay).generic_yesno(
                eula_file.read(), "Agree", "Cancel"):
            sys.exit(0)

def choose_names(installer):
    """Display screen to select domains to validate.

    :param installer: An installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    """
    # This function adds all names
    # found within the config to self.names
    # Then filters them based on user selection
    code, names = zope.component.getUtility(
        interfaces.IDisplay).filter_names(get_all_names(installer))
    if code == display.OK and names:
        return names
    else:
        sys.exit(0)

def get_all_names(installer):
    """Return all valid names in the configuration.

    :param installer: An installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    """
    names = list(installer.get_all_names())
    client.sanity_check_names(names)

    if not names:
        logging.fatal("No domain names were found in your installation")
        logging.fatal("Either specify which names you would like "
                      "letsencrypt to validate or add server names "
                      "to your virtual hosts")
        sys.exit(1)

    return names

# This should be controlled by commandline parameters
def determine_authenticator():
    """Returns a valid IAuthenticator."""
    return configurator.ApacheConfigurator()

def determine_installer():
    """Returns a valid IInstaller."""
    return configurator.ApacheConfigurator()

def read_file(filename):
    """Returns the given file's contents with universal new line support.

    :param str filename: Filename

    :returns: A tuple of filename and its contents
    :rtype: tuple

    :raises argparse.ArgumentTypeError: File does not exist or is not readable.

    """
    try:
        return filename, open(filename, 'rU').read()
    except IOError as exc:
        raise argparse.ArgumentTypeError(exc.strerror)

def rollback(installer, checkpoints):
    """Revert configuration the specified number of checkpoints.

    :param installer: Installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    :param int checkpoints: Number of checkpoints to revert.

    """
    installer.rollback_checkpoints(checkpoints)
    installer.restart()

def view_config_changes(installer):
    """View checkpoints and associated configuration changes.

    :param installer: Installer object
    :type installer: :class:`letsencrypt.client.interfaces.IInstaller`

    """
    installer.view_config_changes()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
na = np.newaxis
from matplotlib import pyplot as plt
from matplotlib import cm
import scipy.special as special
import abc, copy
from warnings import warn

from abstractions import ModelGibbsSampling, ModelMeanField, ModelEM
from abstractions import Distribution, GibbsSampling, MeanField, Collapsed, MaxLikelihood
from distributions import Categorical, CategoricalAndConcentration
from internals.labels import Labels, CRPLabels
from pyhsmm.util.stats import getdatasize

class Mixture(ModelGibbsSampling, ModelMeanField, ModelEM):
    '''
    This class is for mixtures of other distributions.
    '''
    def __init__(self,components,alpha_0=None,a_0=None,b_0=None,weights=None):
        assert len(components) > 0
        assert (alpha_0 is not None) ^ (a_0 is not None and b_0 is not None)

        self.components = components

        if alpha_0 is not None:
            self.weights = Categorical(alpha_0=alpha_0,K=len(components),weights=weights)
        else:
            self.weights = CategoricalAndConcentration(a_0=a_0,b_0=b_0,K=len(components),weights=weights)

        self.labels_list = []

    def add_data(self,data):
        # TODO get rid of this asarray...
        self.labels_list.append(Labels(data=np.asarray(data),
            components=self.components,weights=self.weights))

    def generate(self,N,keep=True):
        templabels = Labels(components=self.components,weights=self.weights,N=N) # this samples labels

        out = np.empty(self.components[0].rvs(N).shape)
        counts = np.bincount(templabels.z,minlength=len(self.components))
        for idx,(c,count) in enumerate(zip(self.components,counts)):
            out[templabels.z == idx,...] = c.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

    def _log_likelihoods(self,x):
        x = np.asarray(x)
        K = len(self.components)
        vals = np.empty((x.shape[0],K))
        for idx, c in enumerate(self.components):
            vals[:,idx] = c.log_likelihood(x)
        vals += self.weights.log_likelihood(np.arange(K))
        assert not np.isnan(vals).any()
        return np.logaddexp.reduce(vals,axis=1)

    def log_likelihood(self,x):
        return self._log_likelihoods(x).sum()

    ### Gibbs sampling

    def resample_model(self,temp=None):
        assert all(isinstance(c,GibbsSampling) for c in self.components), \
                'Components must implement GibbsSampling'

        for l in self.labels_list:
            l.resample(temp=temp)

        for idx, c in enumerate(self.components):
            c.resample(data=[(l.data[l.z == idx] if (l.z == idx).any() else []) for l in self.labels_list]) # numpy issue #2587, np.array([]).reshape((0,2))[[]]

        self.weights.resample([l.z for l in self.labels_list])

    def copy_sample(self):
        new = copy.copy(self)
        new.obs_distns = [o.copy_sample() for o in self.obs_distns]
        new.weights = self.weights.copy_sample()
        new.labels_list = [l.copy_sample() for l in self.labels_list]
        return new

    ### Mean Field

    def meanfield_coordinate_descent_step(self):
        assert all(isinstance(c,MeanField) for c in self.components), \
                'Components must implement MeanField'
        assert len(self.labels_list) > 0, 'Must have data to run MeanField'

        ### update sweep!
        # ask labels to get weights over z, stored as l.r
        for l in self.labels_list:
            l.meanfieldupdate()

        # pass the weights to pi
        K = len(self.components)
        self.weights.meanfieldupdate(np.arange(K),[l.r for l in self.labels_list])

        # pass the weights to the components
        for idx, c in enumerate(self.components):
            c.meanfieldupdate([l.data for l in self.labels_list],
                    [l.r[:,idx] for l in self.labels_list])

        ### get vlb!
        vlb = 0.

        # get labels terms
        vlb += sum(l.get_vlb() for l in self.labels_list)

        # get pi term
        vlb += self.weights.get_vlb()

        # get components terms
        vlb += sum(c.get_vlb() for c in self.components)

        # finally, need the evidence term in the vlb
        for l in self.labels_list:
            vlb += np.sum([r.dot(c.expected_log_likelihood(l.data))
                                for c,r in zip(self.components, l.r.T)])

        # add in symmetry factor (if we're actually symmetric)
        if len(set(self.weights.weights)) == 1 and \
                len(set(type(c) for c in self.components)) == 1:
            vlb += special.gammaln(len(self.components)+1)

        return vlb

    ### EM

    def EM_step(self):
        assert all(isinstance(c,MaxLikelihood) for c in self.components), \
                'Components must implement MaxLikelihood'
        assert len(self.labels_list) > 0, 'Must have data to run EM'

        ## E step
        for l in self.labels_list:
            l.E_step()

        ## M step
        # component parameters
        for idx, c in enumerate(self.components):
            c.max_likelihood([l.data for l in self.labels_list],
                    [l.expectations[:,idx] for l in self.labels_list])

        # mixture weights
        self.weights.max_likelihood(np.arange(len(self.components)),
                [l.expectations for l in self.labels_list])

    def num_parameters(self):
        # NOTE: scikit.learn's gmm.py doesn't count the weights in the number of
        # parameters, but I don't know why they wouldn't. Some convention?
        return sum(c.num_parameters() for c in self.components) + self.weights.num_parameters()

    def BIC(self,data=None):
        '''
        BIC on the passed data.
        If passed data is None (default), calculates BIC on the model's assigned data.
        '''
        # NOTE: in principle this method computes the BIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        if data is None:
            assert len(self.labels_list) > 0, \
                    "If not passing in data, the class must already have it. Use the method add_data()"
            return -2*sum(self.log_likelihood(l.data) for l in self.labels_list) + \
                        self.num_parameters() * np.log(sum(l.data.shape[0] for l in self.labels_list))
        else:
            return -2*self.log_likelihood(data) + self.num_parameters() * np.log(data.shape[0])

    def AIC(self):
        # NOTE: in principle this method computes the AIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        assert len(self.labels_list) > 0, 'Must have data to get AIC'
        return 2*self.num_parameters() - 2*sum(self.log_likelihood(l.data) for l in self.labels_list)

    ### Misc.

    def plot(self,color=None,legend=True):
        cmap = cm.get_cmap()

        if len(self.labels_list) > 0:
            label_colors = {}

            used_labels = reduce(set.union,[set(l.z) for l in self.labels_list],set([]))
            num_labels = len(used_labels)
            num_subfig_rows = len(self.labels_list)

            for idx,label in enumerate(used_labels):
                label_colors[label] = idx/(num_labels-1 if num_labels > 1 else 1) \
                        if color is None else color

            for subfigidx,l in enumerate(self.labels_list):
                # plot the current observation distributions (and obs. if given)
                plt.subplot(num_subfig_rows,1,1+subfigidx)
                for label, o in enumerate(self.components):
                    if label in l.z:
                        o.plot(color=cmap(label_colors[label]),
                                data=(l.data[l.z == label] if l.data is not None else None),
                                label='%d' % label)

            if legend:
                plt.legend(
                        [plt.Rectangle((0,0),1,1,fc=cmap(c))
                            for i,c in label_colors.iteritems() if i in used_labels],
                        [i for i in label_colors if i in used_labels],
                        loc='best'
                        )

        else:
            top10 = np.array(self.components)[np.argsort(self.weights.weights)][-1:-11:-1]
            colors = [cmap(x) for x in np.linspace(0,1,len(top10))] if color is None \
                    else [color]*len(top10)
            for i,(o,c) in enumerate(zip(top10,colors)):
                o.plot(color=c,label='%d' % i)

    def to_json_dict(self):
        assert len(self.labels_list) == 1
        data = self.labels_list[0].data
        z = self.labels_list[0].z
        assert data.ndim == 2 and data.shape[1] == 2

        return  {
                    'points':[{'x':x,'y':y,'label':int(label)} for x,y,label in zip(data[:,0],data[:,1],z)],
                    'ellipses':[dict(c.to_json_dict().items() + [('label',i)])
                        for i,c in enumerate(self.components) if i in z]
                }

    def predictive_likelihoods(self,test_data,forecast_horizons):
        likes = self._log_likelihoods(test_data)
        return [likes[k:] for k in forecast_horizons]

    def block_predictive_likelihoods(self,test_data,blocklens):
        csums = np.cumsum(self._log_likelihoods(test_data))
        outs = []
        for k in blocklens:
            outs.append(csums[k:] - csums[:-k])
        return outs

class MixtureDistribution(Mixture, GibbsSampling, Distribution):
    '''
    This makes a Mixture act like a Distribution for use in other compound models
    '''

    def log_likelihood(self,x):
        return self._log_likelihoods(x)

    def resample(self,data,niter=25,temp=None):
        # doesn't keep a reference to the data like a model would
        assert isinstance(data,list) or isinstance(data,np.ndarray)

        if getdatasize(data) > 0:
            if not isinstance(data,np.ndarray):
                data = np.concatenate(data)

            self.add_data(data)

            for itr in range(niter):
                self.resample_model(temp=temp)

            self.labels_list.pop()
        else:
            self.resample_model(temp=temp)

    def max_likelihood(self,data,weights=None):
        if weights is not None:
            raise NotImplementedError
        assert isinstance(data,list) or isinstance(data,np.ndarray)
        if isinstance(data,np.ndarray):
            data = [np.asarray(data)]
        else:
            data = map(lambda x: np.asarray(x), data)

        if getdatasize(data) > 0:
            for d in data:
                self.add_data(d)

            prev_like = sum(self.log_likelihood(d) for d in data)
            for itr in range(100):
                self.EM_step()
                new_like = sum(self.log_likelihood(d) for d in data)
                if new_like <= prev_like + 0.1:
                    break
                else:
                    prev_like = new_like

            for d in data:
                self.labels_list.pop()

    def plot(self,data=[],color='b',label='',plot_params=True):
        if not isinstance(data,list):
            data = [data]
        for d in data:
            self.add_data(d)

        for l in self.labels_list:
            l.E_step() # sets l.z to MAP estimates
            for label, o in enumerate(self.components):
                if label in l.z:
                    o.plot(color=color,label=label,
                            data=l.data[l.z == label] if l.data is not None else None)

        for d in data:
            self.labels_list.pop()

class CollapsedMixture(ModelGibbsSampling):
    __metaclass__ = abc.ABCMeta
    def _get_counts(self,k):
        return sum(l._get_counts(k) for l in self.labels_list)

    def _get_data_withlabel(self,k):
        return [l._get_data_withlabel(k) for l in self.labels_list]

    def _get_occupied(self):
        return reduce(set.union,(l._get_occupied() for l in self.labels_list),set([]))

    def plot(self):
        plt.figure()
        cmap = cm.get_cmap()
        used_labels = self._get_occupied()
        num_labels = len(used_labels)

        label_colors = {}
        for idx,label in enumerate(used_labels):
            label_colors[label] = idx/(num_labels-1. if num_labels > 1 else 1.)

        for subfigidx,l in enumerate(self.labels_list):
            plt.subplot(len(self.labels_list),1,1+subfigidx)
            # TODO assuming data is 2D
            for label in used_labels:
                if label in l.z:
                    plt.plot(l.data[l.z==label,0],l.data[l.z==label,1],
                            color=cmap(label_colors[label]),ls='None',marker='x')

class CRPMixture(CollapsedMixture):
    def __init__(self,alpha_0,obs_distn):
        assert isinstance(obs_distn,Collapsed)
        self.obs_distn = obs_distn
        self.alpha_0 = alpha_0

        self.labels_list = []

    def add_data(self,data):
        assert len(self.labels_list) == 0
        self.labels_list.append(CRPLabels(model=self,data=np.asarray(data),
            alpha_0=self.alpha_0,obs_distn=self.obs_distn))

    def resample_model(self):
        for l in self.labels_list:
            l.resample()

    def generate(self,N,keep=True):
        warn('not fully implemented')
        # TODO only works if there's no other data in the model; o/w need to add
        # existing data to obs resample. should be an easy update.
        # templabels needs to pay attention to its own counts as well as model
        # counts
        assert len(self.labels_list) == 0

        templabels = CRPLabels(model=self,alpha_0=self.alpha_0,obs_distn=self.obs_distn,N=N)

        counts = np.bincount(templabels.z)
        out = np.empty(self.obs_distn.rvs(N).shape)
        for idx, count in enumerate(counts):
            self.obs_distn.resample()
            out[templabels.z == idx,...] = self.obs_distn.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

<commit_msg>obs_distns -> componenets<commit_after>from __future__ import division
import numpy as np
na = np.newaxis
from matplotlib import pyplot as plt
from matplotlib import cm
import scipy.special as special
import abc, copy
from warnings import warn

from abstractions import ModelGibbsSampling, ModelMeanField, ModelEM
from abstractions import Distribution, GibbsSampling, MeanField, Collapsed, MaxLikelihood
from distributions import Categorical, CategoricalAndConcentration
from internals.labels import Labels, CRPLabels
from pyhsmm.util.stats import getdatasize

class Mixture(ModelGibbsSampling, ModelMeanField, ModelEM):
    '''
    This class is for mixtures of other distributions.
    '''
    def __init__(self,components,alpha_0=None,a_0=None,b_0=None,weights=None):
        assert len(components) > 0
        assert (alpha_0 is not None) ^ (a_0 is not None and b_0 is not None)

        self.components = components

        if alpha_0 is not None:
            self.weights = Categorical(alpha_0=alpha_0,K=len(components),weights=weights)
        else:
            self.weights = CategoricalAndConcentration(a_0=a_0,b_0=b_0,K=len(components),weights=weights)

        self.labels_list = []

    def add_data(self,data):
        # TODO get rid of this asarray...
        self.labels_list.append(Labels(data=np.asarray(data),
            components=self.components,weights=self.weights))

    def generate(self,N,keep=True):
        templabels = Labels(components=self.components,weights=self.weights,N=N) # this samples labels

        out = np.empty(self.components[0].rvs(N).shape)
        counts = np.bincount(templabels.z,minlength=len(self.components))
        for idx,(c,count) in enumerate(zip(self.components,counts)):
            out[templabels.z == idx,...] = c.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

    def _log_likelihoods(self,x):
        x = np.asarray(x)
        K = len(self.components)
        vals = np.empty((x.shape[0],K))
        for idx, c in enumerate(self.components):
            vals[:,idx] = c.log_likelihood(x)
        vals += self.weights.log_likelihood(np.arange(K))
        assert not np.isnan(vals).any()
        return np.logaddexp.reduce(vals,axis=1)

    def log_likelihood(self,x):
        return self._log_likelihoods(x).sum()

    ### Gibbs sampling

    def resample_model(self,temp=None):
        assert all(isinstance(c,GibbsSampling) for c in self.components), \
                'Components must implement GibbsSampling'

        for l in self.labels_list:
            l.resample(temp=temp)

        for idx, c in enumerate(self.components):
            c.resample(data=[(l.data[l.z == idx] if (l.z == idx).any() else []) for l in self.labels_list]) # numpy issue #2587, np.array([]).reshape((0,2))[[]]

        self.weights.resample([l.z for l in self.labels_list])

    def copy_sample(self):
        new = copy.copy(self)
        new.components = [c.copy_sample() for c in self.components]
        new.weights = self.weights.copy_sample()
        new.labels_list = [l.copy_sample() for l in self.labels_list]
        return new

    ### Mean Field

    def meanfield_coordinate_descent_step(self):
        assert all(isinstance(c,MeanField) for c in self.components), \
                'Components must implement MeanField'
        assert len(self.labels_list) > 0, 'Must have data to run MeanField'

        ### update sweep!
        # ask labels to get weights over z, stored as l.r
        for l in self.labels_list:
            l.meanfieldupdate()

        # pass the weights to pi
        K = len(self.components)
        self.weights.meanfieldupdate(np.arange(K),[l.r for l in self.labels_list])

        # pass the weights to the components
        for idx, c in enumerate(self.components):
            c.meanfieldupdate([l.data for l in self.labels_list],
                    [l.r[:,idx] for l in self.labels_list])

        ### get vlb!
        vlb = 0.

        # get labels terms
        vlb += sum(l.get_vlb() for l in self.labels_list)

        # get pi term
        vlb += self.weights.get_vlb()

        # get components terms
        vlb += sum(c.get_vlb() for c in self.components)

        # finally, need the evidence term in the vlb
        for l in self.labels_list:
            vlb += np.sum([r.dot(c.expected_log_likelihood(l.data))
                                for c,r in zip(self.components, l.r.T)])

        # add in symmetry factor (if we're actually symmetric)
        if len(set(self.weights.weights)) == 1 and \
                len(set(type(c) for c in self.components)) == 1:
            vlb += special.gammaln(len(self.components)+1)

        return vlb

    ### EM

    def EM_step(self):
        assert all(isinstance(c,MaxLikelihood) for c in self.components), \
                'Components must implement MaxLikelihood'
        assert len(self.labels_list) > 0, 'Must have data to run EM'

        ## E step
        for l in self.labels_list:
            l.E_step()

        ## M step
        # component parameters
        for idx, c in enumerate(self.components):
            c.max_likelihood([l.data for l in self.labels_list],
                    [l.expectations[:,idx] for l in self.labels_list])

        # mixture weights
        self.weights.max_likelihood(np.arange(len(self.components)),
                [l.expectations for l in self.labels_list])

    def num_parameters(self):
        # NOTE: scikit.learn's gmm.py doesn't count the weights in the number of
        # parameters, but I don't know why they wouldn't. Some convention?
        return sum(c.num_parameters() for c in self.components) + self.weights.num_parameters()

    def BIC(self,data=None):
        '''
        BIC on the passed data.
        If passed data is None (default), calculates BIC on the model's assigned data.
        '''
        # NOTE: in principle this method computes the BIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        if data is None:
            assert len(self.labels_list) > 0, \
                    "If not passing in data, the class must already have it. Use the method add_data()"
            return -2*sum(self.log_likelihood(l.data) for l in self.labels_list) + \
                        self.num_parameters() * np.log(sum(l.data.shape[0] for l in self.labels_list))
        else:
            return -2*self.log_likelihood(data) + self.num_parameters() * np.log(data.shape[0])

    def AIC(self):
        # NOTE: in principle this method computes the AIC only after finding the
        # maximum likelihood parameters (or, of course, an EM fixed-point as an
        # approximation!)
        assert len(self.labels_list) > 0, 'Must have data to get AIC'
        return 2*self.num_parameters() - 2*sum(self.log_likelihood(l.data) for l in self.labels_list)

    ### Misc.

    def plot(self,color=None,legend=True):
        cmap = cm.get_cmap()

        if len(self.labels_list) > 0:
            label_colors = {}

            used_labels = reduce(set.union,[set(l.z) for l in self.labels_list],set([]))
            num_labels = len(used_labels)
            num_subfig_rows = len(self.labels_list)

            for idx,label in enumerate(used_labels):
                label_colors[label] = idx/(num_labels-1 if num_labels > 1 else 1) \
                        if color is None else color

            for subfigidx,l in enumerate(self.labels_list):
                # plot the current observation distributions (and obs. if given)
                plt.subplot(num_subfig_rows,1,1+subfigidx)
                for label, o in enumerate(self.components):
                    if label in l.z:
                        o.plot(color=cmap(label_colors[label]),
                                data=(l.data[l.z == label] if l.data is not None else None),
                                label='%d' % label)

            if legend:
                plt.legend(
                        [plt.Rectangle((0,0),1,1,fc=cmap(c))
                            for i,c in label_colors.iteritems() if i in used_labels],
                        [i for i in label_colors if i in used_labels],
                        loc='best'
                        )

        else:
            top10 = np.array(self.components)[np.argsort(self.weights.weights)][-1:-11:-1]
            colors = [cmap(x) for x in np.linspace(0,1,len(top10))] if color is None \
                    else [color]*len(top10)
            for i,(o,c) in enumerate(zip(top10,colors)):
                o.plot(color=c,label='%d' % i)

    def to_json_dict(self):
        assert len(self.labels_list) == 1
        data = self.labels_list[0].data
        z = self.labels_list[0].z
        assert data.ndim == 2 and data.shape[1] == 2

        return  {
                    'points':[{'x':x,'y':y,'label':int(label)} for x,y,label in zip(data[:,0],data[:,1],z)],
                    'ellipses':[dict(c.to_json_dict().items() + [('label',i)])
                        for i,c in enumerate(self.components) if i in z]
                }

    def predictive_likelihoods(self,test_data,forecast_horizons):
        likes = self._log_likelihoods(test_data)
        return [likes[k:] for k in forecast_horizons]

    def block_predictive_likelihoods(self,test_data,blocklens):
        csums = np.cumsum(self._log_likelihoods(test_data))
        outs = []
        for k in blocklens:
            outs.append(csums[k:] - csums[:-k])
        return outs

class MixtureDistribution(Mixture, GibbsSampling, Distribution):
    '''
    This makes a Mixture act like a Distribution for use in other compound models
    '''

    def log_likelihood(self,x):
        return self._log_likelihoods(x)

    def resample(self,data,niter=25,temp=None):
        # doesn't keep a reference to the data like a model would
        assert isinstance(data,list) or isinstance(data,np.ndarray)

        if getdatasize(data) > 0:
            if not isinstance(data,np.ndarray):
                data = np.concatenate(data)

            self.add_data(data)

            for itr in range(niter):
                self.resample_model(temp=temp)

            self.labels_list.pop()
        else:
            self.resample_model(temp=temp)

    def max_likelihood(self,data,weights=None):
        if weights is not None:
            raise NotImplementedError
        assert isinstance(data,list) or isinstance(data,np.ndarray)
        if isinstance(data,np.ndarray):
            data = [np.asarray(data)]
        else:
            data = map(lambda x: np.asarray(x), data)

        if getdatasize(data) > 0:
            for d in data:
                self.add_data(d)

            prev_like = sum(self.log_likelihood(d) for d in data)
            for itr in range(100):
                self.EM_step()
                new_like = sum(self.log_likelihood(d) for d in data)
                if new_like <= prev_like + 0.1:
                    break
                else:
                    prev_like = new_like

            for d in data:
                self.labels_list.pop()

    def plot(self,data=[],color='b',label='',plot_params=True):
        if not isinstance(data,list):
            data = [data]
        for d in data:
            self.add_data(d)

        for l in self.labels_list:
            l.E_step() # sets l.z to MAP estimates
            for label, o in enumerate(self.components):
                if label in l.z:
                    o.plot(color=color,label=label,
                            data=l.data[l.z == label] if l.data is not None else None)

        for d in data:
            self.labels_list.pop()

class CollapsedMixture(ModelGibbsSampling):
    __metaclass__ = abc.ABCMeta
    def _get_counts(self,k):
        return sum(l._get_counts(k) for l in self.labels_list)

    def _get_data_withlabel(self,k):
        return [l._get_data_withlabel(k) for l in self.labels_list]

    def _get_occupied(self):
        return reduce(set.union,(l._get_occupied() for l in self.labels_list),set([]))

    def plot(self):
        plt.figure()
        cmap = cm.get_cmap()
        used_labels = self._get_occupied()
        num_labels = len(used_labels)

        label_colors = {}
        for idx,label in enumerate(used_labels):
            label_colors[label] = idx/(num_labels-1. if num_labels > 1 else 1.)

        for subfigidx,l in enumerate(self.labels_list):
            plt.subplot(len(self.labels_list),1,1+subfigidx)
            # TODO assuming data is 2D
            for label in used_labels:
                if label in l.z:
                    plt.plot(l.data[l.z==label,0],l.data[l.z==label,1],
                            color=cmap(label_colors[label]),ls='None',marker='x')

class CRPMixture(CollapsedMixture):
    def __init__(self,alpha_0,obs_distn):
        assert isinstance(obs_distn,Collapsed)
        self.obs_distn = obs_distn
        self.alpha_0 = alpha_0

        self.labels_list = []

    def add_data(self,data):
        assert len(self.labels_list) == 0
        self.labels_list.append(CRPLabels(model=self,data=np.asarray(data),
            alpha_0=self.alpha_0,obs_distn=self.obs_distn))

    def resample_model(self):
        for l in self.labels_list:
            l.resample()

    def generate(self,N,keep=True):
        warn('not fully implemented')
        # TODO only works if there's no other data in the model; o/w need to add
        # existing data to obs resample. should be an easy update.
        # templabels needs to pay attention to its own counts as well as model
        # counts
        assert len(self.labels_list) == 0

        templabels = CRPLabels(model=self,alpha_0=self.alpha_0,obs_distn=self.obs_distn,N=N)

        counts = np.bincount(templabels.z)
        out = np.empty(self.obs_distn.rvs(N).shape)
        for idx, count in enumerate(counts):
            self.obs_distn.resample()
            out[templabels.z == idx,...] = self.obs_distn.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z

<|endoftext|>


<commit_before>from django.test import mock, TestCase

from .. import image_data

class TestImageFile(TestCase):

    def setUp(self):
        self.image = mock.MagicMock()
        self.image.name = 'gallery/foo.jpg'
        self.image_file = image_data.ImageFile(
            self.image,
            100,
            50,
            'bar'
        )

    def tearDown(self):
        self.image_file.delete()

    def test_init(self):
        self.assertEqual(self.image_file.name, 'foo.jpg')
        self.assertEqual(self.image_file.size[0], 100)
        self.assertEqual(self.image_file.size[1], 50)

    def test_set_name_with_path(self):
        self.image_file._set_name('gallery/bar.jpg')
        self.assertEqual(self.image_file.name, 'bar.jpg')

    def test_set_name_without_path(self):
        self.image_file._set_name('bar.jpg')
        self.assertEqual(self.image_file.name, 'bar.jpg')

    def test_create_filename_with_path(self):
        name = self.image_file._create_filename('gallery/foo.jpg')
        self.assertEqual(name, 'gallery/foo_bar.jpg')

    def test_create_filename_without_path(self):
        name = self.image_file._create_filename('foo.jpg')
        self.assertEqual(name, 'foo_bar.jpg')

    def test_filename_property(self):
        mock_func = mock.MagicMock(return_value='foo')
        self.image_file._create_filename = mock_func
        self.assertEqual(self.image_file.filename, 'foo')
        mock_func.assert_called_with(self.image_file.name)

    @mock.patch('gallery.utils.create_path', return_value='foo')
    def test_path_property(self, create_path):
        with mock.patch('gallery.image_data.ImageFile.filename',
                new_callable=mock.PropertyMock(return_value='bar')):
            self.assertEqual(self.image_file.path, 'foo')
            create_path.assert_called_with('bar')

    @mock.patch('gallery.utils.create_url', return_value='foo')
    def test_url_property(self, create_url):
        with mock.patch('gallery.image_data.ImageFile.filename',
                new_callable=mock.PropertyMock(return_value='bar')):
            self.assertEqual(self.image_file.url, 'foo')
            create_url.assert_called_with('bar')

    def test_change_ext(self):
        self.image_file._change_ext('bar.png')
        self.assertEqual(self.image_file.name, 'foo.png')

    #def test_save_
<commit_msg>Completed tests of ImageFile<commit_after>from django.test import mock, TestCase

from .. import image_data

class TestImageFile(TestCase):

    def setUp(self):
        self.image = mock.MagicMock()
        self.image.name = 'gallery/foo.jpg'
        self.image_file = image_data.ImageFile(
            self.image,
            100,
            50,
            'bar'
        )

    def tearDown(self):
        self.image_file.delete()

    def test_init(self):
        self.assertEqual(self.image_file.name, 'foo.jpg')
        self.assertEqual(self.image_file.word, 'bar')
        self.assertEqual(self.image_file.size[0], 100)
        self.assertEqual(self.image_file.size[1], 50)

    def test_set_name_with_path(self):
        self.image_file._set_name('gallery/bar.jpg')
        self.assertEqual(self.image_file.name, 'bar.jpg')

    def test_set_name_without_path(self):
        self.image_file._set_name('bar.jpg')
        self.assertEqual(self.image_file.name, 'bar.jpg')

    def test_create_filename_with_path(self):
        name = self.image_file._create_filename('gallery/foo.jpg')
        self.assertEqual(name, 'gallery/foo_bar.jpg')

    def test_create_filename_without_path(self):
        name = self.image_file._create_filename('foo.jpg')
        self.assertEqual(name, 'foo_bar.jpg')

    def test_filename_property(self):
        mock_func = mock.MagicMock(return_value='foo')
        self.image_file._create_filename = mock_func
        self.assertEqual(self.image_file.filename, 'foo')
        mock_func.assert_called_with(self.image_file.name)

    @mock.patch('gallery.utils.create_path', return_value='foo')
    def test_path_property(self, create_path):
        with mock.patch('gallery.image_data.ImageFile.filename',
                new_callable=mock.PropertyMock(return_value='bar')):
            self.assertEqual(self.image_file.path, 'foo')
            create_path.assert_called_with('bar')

    @mock.patch('gallery.utils.create_url', return_value='foo')
    def test_url_property(self, create_url):
        with mock.patch('gallery.image_data.ImageFile.filename',
                new_callable=mock.PropertyMock(return_value='bar')):
            self.assertEqual(self.image_file.url, 'foo')
            create_url.assert_called_with('bar')

    def test_change_ext(self):
        self.image_file._change_ext('bar.png')
        self.assertEqual(self.image_file.name, 'foo.png')

    @mock.patch('gallery.utils.get_ext', return_value='.jpg')
    def test_save_new_image(self, get_ext):
        self.image.name = 'foo.jpg'
        image_file = mock.MagicMock()
        image_file.name = self.image.name
        image_data.ImageFile.save(image_file, self.image, 'bar', '')
        image_file._set_name.assert_called_with('')
        image_file.delete.assert_not_called()
        get_ext.assert_called_with('foo.jpg')
        self.assertEqual(image_file.name, 'bar.jpg')
        image_file._rename_file.assert_not_called()
        image_file._change_ext.assert_not_called()
        image_file._create_image.assert_called_with(self.image)

    @mock.patch('gallery.utils.get_ext', return_value='.jpg')
    def test_save_image_with_new_file(self, get_ext):
        self.image.name = 'foo.jpg'
        image_file = mock.MagicMock()
        image_file.name = 'gallery/bar.jpg'
        image_data.ImageFile.save(image_file, self.image, '', self.image.name)
        image_file._set_name.assert_called_with(self.image.name)
        image_file.delete.assert_called()
        get_ext.assert_not_called()
        self.assertEqual(image_file.name, 'gallery/bar.jpg')
        image_file._rename_file.assert_not_called()
        image_file._change_ext.assert_called_with(self.image.name)
        image_file._create_image.assert_called_with(self.image)

    @mock.patch('gallery.utils.get_ext', return_value='.jpg')
    def test_save_image_change_content_object(self, get_ext):
        image_file = mock.MagicMock()
        image_file.name = 'gallery/foo.jpg'
        image_data.ImageFile.save(image_file, self.image, 'bar', self.image.name)
        image_file._set_name.assert_called_with(self.image.name)
        image_file.delete.assert_not_called()
        get_ext.assert_called_with(self.image.name)
        self.assertEqual(image_file.name, 'bar.jpg')
        image_file._rename_file.assert_called_with('bar.jpg')
        image_file._change_ext.assert_not_called()
        image_file._create_image.assert_not_called()

    @mock.patch('gallery.utils.get_ext', return_value='.jpg')
    def test_save_image_with_new_file_change_content_object(self, get_ext):
        self.image.name = 'foo.jpg'
        image_file = mock.MagicMock()
        image_file.name = 'gallery/bar.jpg'
        image_data.ImageFile.save(image_file, self.image, 'baz', self.image.name)
        image_file._set_name.assert_called_with(self.image.name)
        image_file.delete.assert_called()
        get_ext.assert_not_called()
        self.assertEqual(image_file.name, 'baz.jpg')
        image_file._rename_file.assert_not_called()
        image_file._change_ext.assert_not_called()
        image_file._create_image.assert_called_with(self.image)

    @mock.patch('os.rename')
    @mock.patch('gallery.utils.create_path', return_value='gallery/baz_bar.jpg')
    def test_rename_file(self, create_path, rename):
        mock_func = mock.MagicMock(return_value='baz_bar.jpg')
        self.image_file._create_filename = mock_func
        with mock.patch('gallery.image_data.ImageFile.path',
                new_callable=mock.PropertyMock(return_value='gallery/foo_bar.jpg')):
            self.image_file._rename_file('baz.jpg')
        mock_func.assert_called_with('baz.jpg')
        create_path.assert_called_with('baz_bar.jpg')
        rename.assert_called_with('gallery/foo_bar.jpg', 'gallery/baz_bar.jpg')

    @mock.patch('os.remove')
    def test_delete_existing_file(self, remove):
        with mock.patch('gallery.image_data.ImageFile.path',
                new_callable=mock.PropertyMock(return_value='foo.jpg')):
            self.image_file.delete()
        remove.assert_called_with('foo.jpg')

    @mock.patch('os.remove', side_effect=FileNotFoundError)
    def test_delete_not_existing_file(self, remove):
        with mock.patch('gallery.image_data.ImageFile.path',
                new_callable=mock.PropertyMock(return_value='foo.jpg')):
            self.image_file.delete()
        remove.assert_called_with('foo.jpg')

    def test_create_filename_with_path(self):
        name = self.image_file._create_filename('gallery/foo.jpg')
        self.assertEqual(name, 'gallery/foo_bar.jpg')

    def test_create_filename_without_path(self):
        name = self.image_file._create_filename('foo.jpg')
        self.assertEqual(name, 'foo_bar.jpg')

    @mock.patch('gallery.utils.image_resize')
    def test_create_image(self, image_resize):
        image_file = mock.MagicMock()
        image_file.path = 'foo.jpg'
        image_file.size = (100, 50)
        image_data.ImageFile._create_image(image_file, self.image)
        image_resize.assert_called_with(self.image, 'foo.jpg', (100, 50))
<|endoftext|>


<commit_before>from typing import List, Tuple, Optional, NamedTuple
from plotly.offline import plot
import pandas as pd
import numpy as np
import plotly.graph_objs as go

from lexos.helpers.error_messages import EMPTY_DTM_MESSAGE
from lexos.models.base_model import BaseModel
from lexos.models.matrix_model import MatrixModel
from lexos.receivers.matrix_receiver import MatrixReceiver, IdTempLabelMap

class StatsTestOptions(NamedTuple):
    """A typed tuple to hold test options."""
    doc_term_matrix: pd.DataFrame
    id_temp_label_map: IdTempLabelMap

class CorpusStats(NamedTuple):
    """A typed tuple to represent statistics of the whole corpus."""
    mean: float  # Average size of all files.
    median: float  # Median (second quartile) of all file sizes.
    # File anomaly found using standard error.
    anomaly_se: List[Optional[Tuple[str, str]]]
    # File anomaly found using interquartile range.
    anomaly_iqr: List[Optional[Tuple[str, str]]]
    std_deviation: float  # Standard deviation of all file sizes.
    first_quartile: float  # First quartile of all file sizes.
    third_quartile: float  # Third quartile of all file sizes.
    inter_quartile_range: float  # Interquartile range.

class StatsModel(BaseModel):
    def __init__(self, test_options: Optional[StatsTestOptions] = None):
        """This is the class to generate statistics of the input file.

        :param test_options: the input used in testing to override the
                             dynamically loaded option
        """
        super().__init__()
        if test_options is not None:
            self._test_dtm = test_options.doc_term_matrix
            self._test_id_temp_label_map = test_options.id_temp_label_map
        else:
            self._test_dtm = None
            self._test_id_temp_label_map = None

    @property
    def _doc_term_matrix(self) -> pd.DataFrame:
        """:return: the document term matrix"""
        return self._test_dtm if self._test_dtm is not None \
            else MatrixModel().get_matrix()

    @property
    def _id_temp_label_map(self) -> IdTempLabelMap:
        """:return: a map takes an id to temp labels"""
        return self._test_id_temp_label_map \
            if self._test_id_temp_label_map is not None \
            else MatrixModel().get_id_temp_label_map()

    def get_corpus_info(self) -> CorpusStats:
        """Converts word lists completely to statistic.

        :return: a typed tuple that holds all statistic of the entire corpus.
        """

        # Check if empty corpus is given.
        assert not self._doc_term_matrix.empty, EMPTY_DTM_MESSAGE

        # Get file names.
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # Get the file count sums by sum the column.
        file_sizes = self._doc_term_matrix.sum(1)
        # Get the average file word counts.
        mean = file_sizes.mean(0)
        # Get the standard deviation of the file word counts.
        std_deviation = file_sizes.std(0)
        # Get the median of the file word counts.
        median = file_sizes.mean(0)
        # Get the iqr of the file word counts.
        first_quartile = file_sizes.quantile(0.25)
        third_quartile = file_sizes.quantile(0.75)
        iqr = third_quartile - first_quartile

        # Standard error analysis: assume file sizes are normally distributed;
        # we detect anomaly by finding files with sizes that are more than two
        # standard deviation away from the mean. In another word, we find files
        # with sizes that are not in the major 95% range.

        anomaly_se = [
            ("small", label)
            if file_sizes[count] < mean - 2 * std_deviation
            else ("large", label)
            if file_sizes[count] > mean + 2 * std_deviation
            else None
            for count, label in enumerate(labels)]

        # Interquartile range analysis: We detect anomaly by finding files with
        # sizes that are either 1.5 interquartile ranges above third quartile
        # or 1.5 interquartile ranges below first quartile.

        anomaly_iqr = [
            ("small", label)
            if file_sizes[count] < first_quartile - 1.5 * iqr
            else ("large", label)
            if file_sizes[count] > third_quartile + 1.5 * iqr
            else None
            for count, label in enumerate(labels)]

        return CorpusStats(mean=mean,
                           median=median,
                           anomaly_se=anomaly_se,
                           anomaly_iqr=anomaly_iqr,
                           std_deviation=std_deviation,
                           first_quartile=first_quartile,
                           third_quartile=third_quartile,
                           inter_quartile_range=iqr)

    def get_file_info(self) -> str:
        """Get statistics of each file.

        :return: A HTML table converted from a pandas data frame.
        """
        # Check if empty corpus is given.
        assert not self._doc_term_matrix.empty, EMPTY_DTM_MESSAGE

        # Get file names.
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # Get proper name headers.
        token_type = \
            MatrixReceiver().options_from_front_end().token_option.token_type

        token_name = "Terms" if token_type == "word" else "Characters"

        file_stats = pd.DataFrame(
            columns=["Documents",
                     f"Number of {token_name} occuring once",
                     f"Total number of {token_name}",
                     f"Average number of {token_name}",
                     f"Distinct number of {token_name}"])

        file_stats["Documents"] = labels
        file_stats[f"Number of {token_name} occuring once"] = \
            self._doc_term_matrix.eq(1).sum(axis=1).values
        file_stats[f"Total number of {token_name}"] = \
            self._doc_term_matrix.sum(axis=1).values
        file_stats[f"Distinct number of {token_name}"] = \
            self._doc_term_matrix.ne(0).sum(axis=1).values
        file_stats[f"Average number of {token_name}"] = \
            file_stats[f"Total number of {token_name}"] / \
            file_stats[f"Distinct number of {token_name}"]

        return file_stats.round(4).to_html(
            index=False,
            classes="table table-striped table-bordered"
        )

    def get_box_plot(self) -> str:
        """Get box plot for the entire corpus.

        :return: A plotly object that contains the box plot.
        """
        # Get file names.
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # Set up the box plot.
        box_plot = go.Box(x0=0.5,  # Initial position of the box plot
                          y=self._doc_term_matrix.sum(1).values,
                          name="Corpus Box Plot",
                          hoverinfo="y",
                          marker=dict(color='rgb(10, 140, 200)'))

        # Set up the points.
        scatter_plot = go.Scatter(
            # Get random x values with the range.
            x=[np.random.uniform(-0.3, 0) for _, _ in enumerate(labels)],
            y=self._doc_term_matrix.sum(1).values,
            name="Corpus Scatter Plot",
            hoverinfo="text",
            mode="markers",
            text=labels)

        # Set up the plot data set.
        data = [scatter_plot, box_plot]

        # Hide information on x-axis as we do not really need any of those.
        layout = go.Layout(
            xaxis=dict(
                autorange=True,
                showgrid=False,
                zeroline=False,
                autotick=False,
                showline=False,
                showticklabels=False
            )
        )

        # Return plotly object as a div.
        return plot(go.Figure(data=data, layout=layout),
                    show_link=False,
                    output_type="div")
<commit_msg>Remove unused data<commit_after>from typing import List, Tuple, Optional, NamedTuple
from plotly.offline import plot
import pandas as pd
import numpy as np
import plotly.graph_objs as go

from lexos.helpers.error_messages import EMPTY_DTM_MESSAGE
from lexos.models.base_model import BaseModel
from lexos.models.matrix_model import MatrixModel
from lexos.receivers.matrix_receiver import MatrixReceiver, IdTempLabelMap

class StatsTestOptions(NamedTuple):
    """A typed tuple to hold test options."""
    doc_term_matrix: pd.DataFrame
    id_temp_label_map: IdTempLabelMap

class CorpusStats(NamedTuple):
    """A typed tuple to represent statistics of the whole corpus."""
    mean: float  # Average size of all files.
    # File anomaly found using standard error.
    anomaly_se: List[Optional[Tuple[str, str]]]
    # File anomaly found using interquartile range.
    anomaly_iqr: List[Optional[Tuple[str, str]]]
    std_deviation: float  # Standard deviation of all file sizes.
    inter_quartile_range: float  # Interquartile range.

class StatsModel(BaseModel):
    def __init__(self, test_options: Optional[StatsTestOptions] = None):
        """This is the class to generate statistics of the input file.

        :param test_options: the input used in testing to override the
                             dynamically loaded option
        """
        super().__init__()
        if test_options is not None:
            self._test_dtm = test_options.doc_term_matrix
            self._test_id_temp_label_map = test_options.id_temp_label_map
        else:
            self._test_dtm = None
            self._test_id_temp_label_map = None

    @property
    def _doc_term_matrix(self) -> pd.DataFrame:
        """:return: the document term matrix"""
        return self._test_dtm if self._test_dtm is not None \
            else MatrixModel().get_matrix()

    @property
    def _id_temp_label_map(self) -> IdTempLabelMap:
        """:return: a map takes an id to temp labels"""
        return self._test_id_temp_label_map \
            if self._test_id_temp_label_map is not None \
            else MatrixModel().get_id_temp_label_map()

    def get_corpus_info(self) -> CorpusStats:
        """Converts word lists completely to statistic.

        :return: a typed tuple that holds all statistic of the entire corpus.
        """

        # Check if empty corpus is given.
        assert not self._doc_term_matrix.empty, EMPTY_DTM_MESSAGE

        # Get file names.
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # Get the file count sums by sum the column.
        file_sizes = self._doc_term_matrix.sum(1)
        # Get the average file word counts.
        mean = file_sizes.mean(0)
        # Get the standard deviation of the file word counts.
        std_deviation = file_sizes.std(0)
        # Get the median of the file word counts.
        median = file_sizes.mean(0)
        # Get the iqr of the file word counts.
        first_quartile = file_sizes.quantile(0.25)
        third_quartile = file_sizes.quantile(0.75)
        iqr = third_quartile - first_quartile

        # Standard error analysis: assume file sizes are normally distributed;
        # we detect anomaly by finding files with sizes that are more than two
        # standard deviation away from the mean. In another word, we find files
        # with sizes that are not in the major 95% range.

        anomaly_se = [
            ("small", label)
            if file_sizes[count] < mean - 2 * std_deviation
            else ("large", label)
            if file_sizes[count] > mean + 2 * std_deviation
            else None
            for count, label in enumerate(labels)]

        # Interquartile range analysis: We detect anomaly by finding files with
        # sizes that are either 1.5 interquartile ranges above third quartile
        # or 1.5 interquartile ranges below first quartile.

        anomaly_iqr = [
            ("small", label)
            if file_sizes[count] < first_quartile - 1.5 * iqr
            else ("large", label)
            if file_sizes[count] > third_quartile + 1.5 * iqr
            else None
            for count, label in enumerate(labels)]

        return CorpusStats(mean=mean,
                           anomaly_se=anomaly_se,
                           anomaly_iqr=anomaly_iqr,
                           std_deviation=std_deviation,
                           inter_quartile_range=iqr)

    def get_file_info(self) -> str:
        """Get statistics of each file.

        :return: A HTML table converted from a pandas data frame.
        """
        # Check if empty corpus is given.
        assert not self._doc_term_matrix.empty, EMPTY_DTM_MESSAGE

        # Get file names.
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # Get proper name headers.
        token_type = \
            MatrixReceiver().options_from_front_end().token_option.token_type

        token_name = "Terms" if token_type == "word" else "Characters"

        file_stats = pd.DataFrame(
            columns=["Documents",
                     f"Number of {token_name} occuring once",
                     f"Total number of {token_name}",
                     f"Average number of {token_name}",
                     f"Distinct number of {token_name}"])

        file_stats["Documents"] = labels
        file_stats[f"Number of {token_name} occuring once"] = \
            self._doc_term_matrix.eq(1).sum(axis=1).values
        file_stats[f"Total number of {token_name}"] = \
            self._doc_term_matrix.sum(axis=1).values
        file_stats[f"Distinct number of {token_name}"] = \
            self._doc_term_matrix.ne(0).sum(axis=1).values
        file_stats[f"Average number of {token_name}"] = \
            file_stats[f"Total number of {token_name}"] / \
            file_stats[f"Distinct number of {token_name}"]

        return file_stats.round(4).to_html(
            index=False,
            classes="table table-striped table-bordered"
        )

    def get_box_plot(self) -> str:
        """Get box plot for the entire corpus.

        :return: A plotly object that contains the box plot.
        """
        # Get file names.
        labels = [self._id_temp_label_map[file_id]
                  for file_id in self._doc_term_matrix.index.values]

        # Set up the box plot.
        box_plot = go.Box(x0=0.5,  # Initial position of the box plot
                          y=self._doc_term_matrix.sum(1).values,
                          name="Corpus Box Plot",
                          hoverinfo="y",
                          marker=dict(color='rgb(10, 140, 200)'))

        # Set up the points.
        scatter_plot = go.Scatter(
            # Get random x values with the range.
            x=[np.random.uniform(-0.3, 0) for _, _ in enumerate(labels)],
            y=self._doc_term_matrix.sum(1).values,
            name="Corpus Scatter Plot",
            hoverinfo="text",
            mode="markers",
            text=labels)

        # Set up the plot data set.
        data = [scatter_plot, box_plot]

        # Hide information on x-axis as we do not really need any of those.
        layout = go.Layout(
            xaxis=dict(
                autorange=True,
                showgrid=False,
                zeroline=False,
                autotick=False,
                showline=False,
                showticklabels=False
            )
        )

        # Return plotly object as a div.
        return plot(go.Figure(data=data, layout=layout),
                    show_link=False,
                    output_type="div")
<|endoftext|>


<commit_before>"""Test the checkers"""
from nose.tools import ok_, eq_

from preflyt import check

CHECKERS = [
    {"checker": "env", "name": "USER"}
]

BAD_CHECKERS = [
    {"checker": "env", "name": "USER1231342dhkfgjhk2394dv09324jk12039csdfg01231"}
]

def test_everything():
    """Test the check method."""
    good, results = check(CHECKERS)
    ok_(good, results)
    eq_(len(results), 1)
    for field_name in ('check', 'success', 'message'):
        ok_(field_name in results[0])

def test_everything_failure():
    """Test the check method."""
    good, results = check(BAD_CHECKERS)
    ok_(not good, results)
    eq_(len(results), 1)
    for field_name in ('check', 'success', 'message'):
        ok_(field_name in results[0])
<commit_msg>Update environment test to have cross platform support<commit_after>"""Test the checkers"""
from nose.tools import ok_, eq_

from preflyt import check

CHECKERS = [
    {"checker": "env", "name": "PATH"}
]

BAD_CHECKERS = [
    {"checker": "env", "name": "PATH1231342dhkfgjhk2394dv09324jk12039csdfg01231"}
]

def test_everything():
    """Test the check method."""
    good, results = check(CHECKERS)
    ok_(good, results)
    eq_(len(results), 1)
    for field_name in ('check', 'success', 'message'):
        ok_(field_name in results[0])

def test_everything_failure():
    """Test the check method."""
    good, results = check(BAD_CHECKERS)
    ok_(not good, results)
    eq_(len(results), 1)
    for field_name in ('check', 'success', 'message'):
        ok_(field_name in results[0])
<|endoftext|>


<commit_before>import ipaddress
import six
import sqlalchemy as sa
from sqlalchemy_utils import IPAddressType
from tests import TestCase

class TestIPAddressType(TestCase):
    def create_models(self):
        class Visitor(self.Base):
            __tablename__ = 'document'
            id = sa.Column(sa.Integer, primary_key=True)
            ip_address = sa.Column(IPAddressType)

            def __repr__(self):
                return 'Visitor(%r)' % self.id

        self.Visitor = Visitor

    def test_parameter_processing(self):
        visitor = self.Visitor(
            ip_address=ipaddress.ip_address(u'111.111.111.111')
        )

        self.session.add(visitor)
        self.session.commit()

        visitor = self.session.query(self.Visitor).first()
        assert six.text_type(visitor.ip_address) == u'111.111.111.111'
<commit_msg>Removed test dependency for ipaddress package<commit_after>import six
import sqlalchemy as sa
from sqlalchemy_utils import IPAddressType
from tests import TestCase

class TestIPAddressType(TestCase):
    def create_models(self):
        class Visitor(self.Base):
            __tablename__ = 'document'
            id = sa.Column(sa.Integer, primary_key=True)
            ip_address = sa.Column(IPAddressType)

            def __repr__(self):
                return 'Visitor(%r)' % self.id

        self.Visitor = Visitor

    def test_parameter_processing(self):
        visitor = self.Visitor(
            ip_address=u'111.111.111.111'
        )

        self.session.add(visitor)
        self.session.commit()

        visitor = self.session.query(self.Visitor).first()
        assert six.text_type(visitor.ip_address) == u'111.111.111.111'
<|endoftext|>


<commit_before>from flask import session, render_template, Blueprint

from lexos.helpers import constants as constants
from lexos.managers import session_manager as session_manager
from lexos.models.filemanager_model import FileManagerModel
from lexos.models.kmeans_model import KMeansModel
from lexos.views.base_view import detect_active_docs

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
k_means_blueprint = Blueprint('k_means', __name__)

# Tells Flask to load this function when someone is at '/kmeans'
@k_means_blueprint.route("/kmeans", methods=["GET"])
def k_means():
    """Handles the functionality on the K Means page.

    It analyzes the various texts and displays the class label of the files.
    :return: a response object (often a render_template call) to flask and
    eventually to the browser.
    """
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    # Set default number of clusters to be half of the number of documents.
    default_k = int(num_active_docs / 2)
    # Get file labels.
    labels = FileManagerModel().load_file_manager().get_active_labels_with_id()
    # Fill the default options.
    if 'analyoption' not in session:
        session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
    if 'kmeanoption' not in session:
        session['kmeanoption'] = constants.DEFAULT_KMEAN_OPTIONS
        session['kmeanoption']['nclusters'] = default_k
    return render_template(
        'kmeans.html',
        labels=labels,
        numActiveDocs=num_active_docs)

@k_means_blueprint.route("/KMeansPlot", methods=['POST'])
def k_means_plot():
    session_manager.cache_analysis_option()
    session_manager.cache_k_mean_option()
    return KMeansModel().get_plot()

@k_means_blueprint.route("/KMeansTable", methods=['POST'])
def k_means_table():
    session_manager.cache_analysis_option()
    session_manager.cache_k_mean_option()
    return KMeansModel().get_table_result()
<commit_msg>Always cache nclusters<commit_after>from flask import session, render_template, Blueprint

from lexos.helpers import constants as constants
from lexos.managers import session_manager as session_manager
from lexos.models.filemanager_model import FileManagerModel
from lexos.models.kmeans_model import KMeansModel
from lexos.views.base_view import detect_active_docs

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
k_means_blueprint = Blueprint('k_means', __name__)

# Tells Flask to load this function when someone is at '/kmeans'
@k_means_blueprint.route("/kmeans", methods=["GET"])
def k_means():
    """Handles the functionality on the K Means page.

    It analyzes the various texts and displays the class label of the files.
    :return: a response object (often a render_template call) to flask and
    eventually to the browser.
    """
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    # Set default number of clusters to be half of the number of documents.
    default_k = int(num_active_docs / 2)
    # Get file labels.
    labels = FileManagerModel().load_file_manager().get_active_labels_with_id()
    # Fill the default options.
    if 'analyoption' not in session:
        session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
    if 'kmeanoption' not in session:
        session['kmeanoption'] = constants.DEFAULT_KMEAN_OPTIONS
    # Always update the nclusters.
    session['kmeanoption']['nclusters'] = default_k
    return render_template(
        'kmeans.html',
        labels=labels,
        numActiveDocs=num_active_docs)

@k_means_blueprint.route("/KMeansPlot", methods=['POST'])
def k_means_plot():
    session_manager.cache_analysis_option()
    session_manager.cache_k_mean_option()
    return KMeansModel().get_plot()

@k_means_blueprint.route("/KMeansTable", methods=['POST'])
def k_means_table():
    session_manager.cache_analysis_option()
    session_manager.cache_k_mean_option()
    return KMeansModel().get_table_result()
<|endoftext|>


<commit_before>import six
import sqlalchemy as sa
from sqlalchemy_utils import IPAddressType
from tests import TestCase

class TestIPAddressType(TestCase):
    def create_models(self):
        class Visitor(self.Base):
            __tablename__ = 'document'
            id = sa.Column(sa.Integer, primary_key=True)
            ip_address = sa.Column(IPAddressType)

            def __repr__(self):
                return 'Visitor(%r)' % self.id

        self.Visitor = Visitor

    def test_parameter_processing(self):
        visitor = self.Visitor(
            ip_address=u'111.111.111.111'
        )

        self.session.add(visitor)
        self.session.commit()

        visitor = self.session.query(self.Visitor).first()
        assert six.text_type(visitor.ip_address) == u'111.111.111.111'
<commit_msg>Ipaddress tests marked as xfailed if ipaddress package not installed<commit_after>from pytest import mark
import six
import sqlalchemy as sa
from sqlalchemy_utils.types import ip_address
from tests import TestCase

@mark.xfail('ip_address.ipaddress is None')
class TestIPAddressType(TestCase):
    def create_models(self):
        class Visitor(self.Base):
            __tablename__ = 'document'
            id = sa.Column(sa.Integer, primary_key=True)
            ip_address = sa.Column(ip_address.IPAddressType)

            def __repr__(self):
                return 'Visitor(%r)' % self.id

        self.Visitor = Visitor

    def test_parameter_processing(self):
        visitor = self.Visitor(
            ip_address=u'111.111.111.111'
        )

        self.session.add(visitor)
        self.session.commit()

        visitor = self.session.query(self.Visitor).first()
        assert six.text_type(visitor.ip_address) == u'111.111.111.111'
<|endoftext|>


<commit_before>from django.conf import settings
from django.db import models
from mercadopago import MP

class PreferenceManager(models.Manager):
    """
    Wraps mercadopago in a very simple interface that creates and manages
    django objects.
    """
    mp = None

    def connect(self):
        if not self.mp:
            self.mp = MP(
                settings.MERCADOPAGO_CLIENT_ID,
                settings.MERCADOPAGO_CLIENT_SECRET,
            )

    def create(self, title, price, reference):
        # TODO: validate that reference is unused
        self.connect()
        preference_request = {
            'items': [
                {
                    'title': title,
                    'quantity': 1,
                    'currency_id': 'ARS',
                    # In case we get something like Decimal:
                    'unit_price': float(price),
                }
            ],
            'external_reference': reference
        }

        preference_result = self.mp.create_preference(preference_request)

        preference = Preference(
            mp_id=preference_result['response']['id'],
            payment_url=preference_result['response']['init_point'],
            sandbox_url=preference_result['response']['sandbox_init_point'],
            reference=reference,
        )

        preference.save()
        return preference

class Preference(models.Model):
    """
    An MP payment preference.
    Price and other data is send to MP and not stored locally - it's assumed
    it's part of the model that relates to this one.
    """

    # Doc says it's a UUID. It's not.
    mp_id = models.CharField(max_length=46)

    payment_url = models.URLField()
    sandbox_url = models.URLField()
    reference = models.TextField(unique=True)

    objects = PreferenceManager()

    def __str__(self):
        return self.mp_id

class Payment(models.Model):
    """
    A payment received, related to a preference.
    """

    mp_id = models.IntegerField()

    preference = models.ForeignKey(
        Preference,
        related_name='payments',
    )
    status = models.CharField(max_length=16)
    status_detail = models.CharField(max_length=16)

    created = models.DateTimeField()

    def __str__(self):
        return self.mp_id
<commit_msg>Prefix an identifier to preferences<commit_after>from django.conf import settings
from django.db import models
from mercadopago import MP

class PreferenceManager(models.Manager):
    """
    Wraps mercadopago in a very simple interface that creates and manages
    django objects.
    """
    mp = None

    def connect(self):
        if not self.mp:
            self.mp = MP(
                settings.MERCADOPAGO_CLIENT_ID,
                settings.MERCADOPAGO_CLIENT_SECRET,
            )

    def create(self, title, price, reference):
        # TODO: validate that reference is unused
        self.connect()
        preference_request = {
            'items': [
                {
                    'title': title,
                    'quantity': 1,
                    'currency_id': 'ARS',
                    # In case we get something like Decimal:
                    'unit_price': float(price),
                }
            ],
            'external_reference': reference
        }

        preference_result = self.mp.create_preference(preference_request)

        preference = Preference(
            mp_id=preference_result['response']['id'],
            payment_url=preference_result['response']['init_point'],
            sandbox_url=preference_result['response']['sandbox_init_point'],
            # TODO: Make prefix configurable?
            reference='django_mercadopago_{}'.format(reference),
        )

        preference.save()
        return preference

class Preference(models.Model):
    """
    An MP payment preference.
    Price and other data is send to MP and not stored locally - it's assumed
    it's part of the model that relates to this one.
    """

    # Doc says it's a UUID. It's not.
    mp_id = models.CharField(max_length=46)

    payment_url = models.URLField()
    sandbox_url = models.URLField()
    reference = models.TextField(unique=True)

    objects = PreferenceManager()

    def __str__(self):
        return self.mp_id

class Payment(models.Model):
    """
    A payment received, related to a preference.
    """

    mp_id = models.IntegerField()

    preference = models.ForeignKey(
        Preference,
        related_name='payments',
    )
    status = models.CharField(max_length=16)
    status_detail = models.CharField(max_length=16)

    created = models.DateTimeField()

    def __str__(self):
        return self.mp_id
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function

import unittest

from django.conf import settings
# Configure settings before attempting to import modules depending on them.
settings.configure(
    ALLOWED_HOSTS=["*"],
    LP_CONTENT_PROVIDER_KEY='merchant-key',
    LP_SECRET='merchant-super-secret',
)

from django.test.client import RequestFactory

from laterpay import signing

from djlaterpay.middleware import LPTokenMiddleware

class RequestMock:

    def __init__(self, path):
        self.path = path

class MiddlewareTest(unittest.TestCase):

    def setUp(self):
        self.request_factory = RequestFactory()

    def test_exempt_paths(self):
        paths = ('/one', '/two')
        LPTokenMiddleware.add_exempt_paths(*paths)

        for path in paths:
            request = RequestMock(path)
            m = LPTokenMiddleware()
            self.assertEqual(m.process_request(request), None)

    def _test_lptoken_validation(self, data):
        request = self.request_factory.get('/end', data)
        middleware = LPTokenMiddleware()
        middleware.process_request(request)
        return request

    def test_lptoken_validation_no_hmac(self):
        request = self._test_lptoken_validation({
            'lptoken': 'tokentoken',
            'ts': '141500500',
        })
        self.assertEqual(request.laterpay.lptoken, None)

    def test_lptoken_validation_no_timestamp(self):
        request = self._test_lptoken_validation({
            'lptoken': 'tokentoken',
            'hmac': 'a-fake-signature',
        })
        self.assertEqual(request.laterpay.lptoken, None)

    def test_lptoken_validation_ok(self):
        params = {
            'lptoken': 'tokentoken',
            'ts': '141500500',
        }
        signature = signing.sign(
            secret='merchant-super-secret',
            params=params,
            url='http://testserver/end',
            method='GET',
        )
        data = params.copy()
        data['hmac'] = signature

        request = self._test_lptoken_validation(data)

        self.assertEqual(request.laterpay.lptoken, 'tokentoken')
<commit_msg>Removed RequestMock in favour of RequestFactory.<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function

import unittest

from django.conf import settings
# Configure settings before attempting to import modules depending on them.
settings.configure(
    ALLOWED_HOSTS=["*"],
    LP_CONTENT_PROVIDER_KEY='merchant-key',
    LP_SECRET='merchant-super-secret',
)

from django.test.client import RequestFactory

from laterpay import signing

from djlaterpay.middleware import LPTokenMiddleware

class MiddlewareTest(unittest.TestCase):

    def setUp(self):
        self.request_factory = RequestFactory()

    def test_exempt_paths(self):
        paths = ('/one', '/two')
        LPTokenMiddleware.add_exempt_paths(*paths)

        for path in paths:
            request = self.request_factory.get(path)
            m = LPTokenMiddleware()
            self.assertEqual(m.process_request(request), None)

    def _test_lptoken_validation(self, data):
        request = self.request_factory.get('/end', data)
        middleware = LPTokenMiddleware()
        middleware.process_request(request)
        return request

    def test_lptoken_validation_no_hmac(self):
        request = self._test_lptoken_validation({
            'lptoken': 'tokentoken',
            'ts': '141500500',
        })
        self.assertEqual(request.laterpay.lptoken, None)

    def test_lptoken_validation_no_timestamp(self):
        request = self._test_lptoken_validation({
            'lptoken': 'tokentoken',
            'hmac': 'a-fake-signature',
        })
        self.assertEqual(request.laterpay.lptoken, None)

    def test_lptoken_validation_ok(self):
        params = {
            'lptoken': 'tokentoken',
            'ts': '141500500',
        }
        signature = signing.sign(
            secret='merchant-super-secret',
            params=params,
            url='http://testserver/end',
            method='GET',
        )
        data = params.copy()
        data['hmac'] = signature

        request = self._test_lptoken_validation(data)

        self.assertEqual(request.laterpay.lptoken, 'tokentoken')
<|endoftext|>