<commit_before># -*- coding: utf-8 -*-
#
# Copyright © 2014  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions
# of the GNU General Public License v.2, or (at your option) any later
# version.  This program is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY expressed or implied, including the
# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE.  See the GNU General Public License for more details.  You
# should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Any Red Hat trademarks that are incorporated in the source
# code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission
# of Red Hat, Inc.
#

'''
MirrorManager2 internal api.
'''

import datetime

import sqlalchemy

from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.exc import SQLAlchemyError

from mirrormanager2.lib import model

def create_session(db_url, debug=False, pool_recycle=3600):
    ''' Create the Session object to use to query the database.

    :arg db_url: URL used to connect to the database. The URL contains
    information with regards to the database engine, the host to connect
    to, the user and password and the database name.
      ie: <engine>://<user>:<password>@<host>/<dbname>
    :kwarg debug: a boolean specifying wether we should have the verbose
        output of sqlalchemy or not.
    :return a Session that can be used to query the database.

    '''
    engine = sqlalchemy.create_engine(
        db_url, echo=debug, pool_recycle=pool_recycle)
    scopedsession = scoped_session(sessionmaker(bind=engine))
    return scopedsession

def get_site(session, site_id):
    ''' Return a specified Site via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Site
    ).filter(
        model.Site.id == site_id
    )

    return query.first()

def get_host(session, host_id):
    ''' Return a specified Host via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Host
    ).filter(
        model.Host.id == host_id
    )

    return query.first()

def get_host_acl_ip(session, host_acl_ip_id):
    ''' Return a specified HostAclIp via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostAclIp
    ).filter(
        model.HostAclIp.id == host_acl_ip_id
    )

    return query.first()

def get_host_netblock(session, host_netblock_id):
    ''' Return a specified HostNetblock via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostNetblock
    ).filter(
        model.HostNetblock.id == host_netblock_id
    )

    return query.first()

def get_host_peer_asn(session, host_asn_id):
    ''' Return a specified HostPeerAsn via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostPeerAsn
    ).filter(
        model.HostPeerAsn.id == host_asn_id
    )

    return query.first()

def get_host_country(session, host_country_id):
    ''' Return a specified HostCountry via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostCountry
    ).filter(
        model.HostCountry.id == host_country_id
    )

    return query.first()

def get_host_category(session, host_category_id):
    ''' Return a specified HostCategory via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostCategory
    ).filter(
        model.HostCategory.id == host_category_id
    )

    return query.first()

def get_host_category_url(session, host_category_url_id):
    ''' Return a specified HostCategoryUrl via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostCategoryUrl
    ).filter(
        model.HostCategoryUrl.id == host_category_url_id
    )

    return query.first()

def get_country_by_name(session, country_code):
    ''' Return a specified Country via its two letter code.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Country
    ).filter(
        model.Country.code == country_code
    )

    return query.first()

def get_user_by_username(session, username):
    ''' Return a specified User via its username.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.User
    ).filter(
        model.User.username == username
    )

    return query.first()

def get_user_by_email(session, email):
    ''' Return a specified User via its email address.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.User
    ).filter(
        model.User.email_address == email
    )

    return query.first()

def get_version_by_name_version(session, p_name, p_version):
    ''' Return a specified Version given the Product name and Version name
    provided.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Version
    ).filter(
        model.Version.product_id == model.Product.id
    ).filter(
        model.Product.name == p_name
    ).filter(
        model.Version.name == p_version
    )

    return query.first()

def get_arch_by_name(session, arch_name):
    ''' Return a Arch specified via name.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Arch
    ).filter(
        model.Arch.name == arch_name
    )

    return query.first()

def get_categories(session):
    ''' Return the list of all the categories in the database.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Category
    )

    return query.all()

def get_products(session):
    ''' Return the list of all the products in the database.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Product
    ).order_by(
        model.Product.name
    )

    return query.all()

def get_arches(session):
    ''' Return the list of all the arch in the database.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Arch
    ).order_by(
        model.Arch.name
    )

    return query.all()

def add_admin_to_site(session, site, admin):
    ''' Add an admin to the specified site.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.SiteAdmin
    ).filter(
        model.SiteAdmin.site_id == site_id
    )

    admins = [sa.username for sa in query.all()]

    if admin in admins:
        return '%s was already listed as an admin' % admin
    else:
        sa = model.SiteAdmin(site_id=site_id, username=admin)
        session.add(sa)
        session.flush()
        return '%s added as an admin' % admin

def get_mirrors(
        session, private=None, internet2=None, internet2_clients=None,
        asn_clients=None, admin_active=None, user_active=None, urls=None,
        last_crawl_duration=False, last_checked_in=False, last_crawled=False,
        site_private=None, site_admin_active=None, site_user_active=None,
        up2date=None, host_category_url_private=None,
        version_id=None, arch_id=None):
    ''' Retrieve the mirrors based on the criteria specified.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Host
    ).order_by(
        model.Host.country, model.Host.name
    )

    if private is not None:
        query = query.filter(model.Host.private == private)
    if internet2 is not None:
        query = query.filter(model.Host.internet2 == internet2)
    if internet2_clients is not None:
        query = query.filter(model.Host.internet2_clients == internet2_clients)
    if asn_clients is not None:
        query = query.filter(model.Host.asn_clients == asn_clients)
    if admin_active is not None:
        query = query.filter(model.Host.admin_active == admin_active)
    if user_active is not None:
        query = query.filter(model.Host.user_active == user_active)

    if host_category_url_private is not None:
        query = query.filter(
            model.HostCategory.host_id == model.Host.id
        ).filter(
            model.HostCategoryUrl.host_category_id == model.HostCategory.id
        ).filter(
            model.HostCategoryUrl.private == host_category_url_private
        )

    if last_crawl_duration is True:
        query = query.filter(model.Host.last_crawl_duration > 0)
    if last_crawled is True:
        query = query.filter(model.Host.last_crawled != None)
    if last_checked_in is True:
        query = query.filter(model.Host.last_checked_in != None)

    if site_private is not None:
        query = query.filter(
            model.Host.site_id == model.Site.id
        ).filter(
            model.Site.private == site_private
        )
    if site_user_active is not None:
        query = query.filter(
            model.Host.site_id == model.Site.id
        ).filter(
            model.Site.user_active == site_user_active
        )
    if site_admin_active is not None:
        query = query.filter(
            model.Host.site_id == model.Site.id
        ).filter(
            model.Site.admin_active == site_admin_active
        )

    if up2date is not None:
        query = query.filter(
            model.Host.id == model.HostCategory.host_id
        ).filter(
            model.HostCategory.id == model.HostCategoryDir.host_category_id
        ).filter(
            model.HostCategoryDir.up2date == up2date
        )

    if version_id is not None:
        query = query.filter(
            model.Host.id == model.HostCategory.host_id
        ).filter(
            model.HostCategory.category_id == model.Category.id
        ).filter(
            model.Category.id == model.Repository.category_id
        ).filter(
            model.Repository.version_id == version_id
        )

    if arch_id is not None:
        query = query.filter(
            model.Host.id == model.HostCategory.host_id
        ).filter(
            model.HostCategory.category_id == model.Category.id
        ).filter(
            model.Category.id == model.Repository.category_id
        ).filter(
            model.Repository.arch_id == arch_id
        )

    return query.all()
<commit_msg>Add method to retrieve a specific method via its confirmation token<commit_after># -*- coding: utf-8 -*-
#
# Copyright © 2014  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions
# of the GNU General Public License v.2, or (at your option) any later
# version.  This program is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY expressed or implied, including the
# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE.  See the GNU General Public License for more details.  You
# should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Any Red Hat trademarks that are incorporated in the source
# code or documentation are not subject to the GNU General Public
# License and may only be used or replicated with the express permission
# of Red Hat, Inc.
#

'''
MirrorManager2 internal api.
'''

import datetime

import sqlalchemy

from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.exc import SQLAlchemyError

from mirrormanager2.lib import model

def create_session(db_url, debug=False, pool_recycle=3600):
    ''' Create the Session object to use to query the database.

    :arg db_url: URL used to connect to the database. The URL contains
    information with regards to the database engine, the host to connect
    to, the user and password and the database name.
      ie: <engine>://<user>:<password>@<host>/<dbname>
    :kwarg debug: a boolean specifying wether we should have the verbose
        output of sqlalchemy or not.
    :return a Session that can be used to query the database.

    '''
    engine = sqlalchemy.create_engine(
        db_url, echo=debug, pool_recycle=pool_recycle)
    scopedsession = scoped_session(sessionmaker(bind=engine))
    return scopedsession

def get_site(session, site_id):
    ''' Return a specified Site via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Site
    ).filter(
        model.Site.id == site_id
    )

    return query.first()

def get_host(session, host_id):
    ''' Return a specified Host via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Host
    ).filter(
        model.Host.id == host_id
    )

    return query.first()

def get_host_acl_ip(session, host_acl_ip_id):
    ''' Return a specified HostAclIp via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostAclIp
    ).filter(
        model.HostAclIp.id == host_acl_ip_id
    )

    return query.first()

def get_host_netblock(session, host_netblock_id):
    ''' Return a specified HostNetblock via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostNetblock
    ).filter(
        model.HostNetblock.id == host_netblock_id
    )

    return query.first()

def get_host_peer_asn(session, host_asn_id):
    ''' Return a specified HostPeerAsn via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostPeerAsn
    ).filter(
        model.HostPeerAsn.id == host_asn_id
    )

    return query.first()

def get_host_country(session, host_country_id):
    ''' Return a specified HostCountry via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostCountry
    ).filter(
        model.HostCountry.id == host_country_id
    )

    return query.first()

def get_host_category(session, host_category_id):
    ''' Return a specified HostCategory via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostCategory
    ).filter(
        model.HostCategory.id == host_category_id
    )

    return query.first()

def get_host_category_url(session, host_category_url_id):
    ''' Return a specified HostCategoryUrl via its identifier.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.HostCategoryUrl
    ).filter(
        model.HostCategoryUrl.id == host_category_url_id
    )

    return query.first()

def get_country_by_name(session, country_code):
    ''' Return a specified Country via its two letter code.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Country
    ).filter(
        model.Country.code == country_code
    )

    return query.first()

def get_user_by_username(session, username):
    ''' Return a specified User via its username.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.User
    ).filter(
        model.User.username == username
    )

    return query.first()

def get_user_by_email(session, email):
    ''' Return a specified User via its email address.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.User
    ).filter(
        model.User.email_address == email
    )

    return query.first()

def get_user_by_token(session, token):
    ''' Return a specified User via its token.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.User
    ).filter(
        model.User.token == token
    )

    return query.first()

def get_version_by_name_version(session, p_name, p_version):
    ''' Return a specified Version given the Product name and Version name
    provided.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Version
    ).filter(
        model.Version.product_id == model.Product.id
    ).filter(
        model.Product.name == p_name
    ).filter(
        model.Version.name == p_version
    )

    return query.first()

def get_arch_by_name(session, arch_name):
    ''' Return a Arch specified via name.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Arch
    ).filter(
        model.Arch.name == arch_name
    )

    return query.first()

def get_categories(session):
    ''' Return the list of all the categories in the database.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Category
    )

    return query.all()

def get_products(session):
    ''' Return the list of all the products in the database.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Product
    ).order_by(
        model.Product.name
    )

    return query.all()

def get_arches(session):
    ''' Return the list of all the arch in the database.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Arch
    ).order_by(
        model.Arch.name
    )

    return query.all()

def add_admin_to_site(session, site, admin):
    ''' Add an admin to the specified site.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.SiteAdmin
    ).filter(
        model.SiteAdmin.site_id == site_id
    )

    admins = [sa.username for sa in query.all()]

    if admin in admins:
        return '%s was already listed as an admin' % admin
    else:
        sa = model.SiteAdmin(site_id=site_id, username=admin)
        session.add(sa)
        session.flush()
        return '%s added as an admin' % admin

def get_mirrors(
        session, private=None, internet2=None, internet2_clients=None,
        asn_clients=None, admin_active=None, user_active=None, urls=None,
        last_crawl_duration=False, last_checked_in=False, last_crawled=False,
        site_private=None, site_admin_active=None, site_user_active=None,
        up2date=None, host_category_url_private=None,
        version_id=None, arch_id=None):
    ''' Retrieve the mirrors based on the criteria specified.

    :arg session: the session with which to connect to the database.

    '''
    query = session.query(
        model.Host
    ).order_by(
        model.Host.country, model.Host.name
    )

    if private is not None:
        query = query.filter(model.Host.private == private)
    if internet2 is not None:
        query = query.filter(model.Host.internet2 == internet2)
    if internet2_clients is not None:
        query = query.filter(model.Host.internet2_clients == internet2_clients)
    if asn_clients is not None:
        query = query.filter(model.Host.asn_clients == asn_clients)
    if admin_active is not None:
        query = query.filter(model.Host.admin_active == admin_active)
    if user_active is not None:
        query = query.filter(model.Host.user_active == user_active)

    if host_category_url_private is not None:
        query = query.filter(
            model.HostCategory.host_id == model.Host.id
        ).filter(
            model.HostCategoryUrl.host_category_id == model.HostCategory.id
        ).filter(
            model.HostCategoryUrl.private == host_category_url_private
        )

    if last_crawl_duration is True:
        query = query.filter(model.Host.last_crawl_duration > 0)
    if last_crawled is True:
        query = query.filter(model.Host.last_crawled != None)
    if last_checked_in is True:
        query = query.filter(model.Host.last_checked_in != None)

    if site_private is not None:
        query = query.filter(
            model.Host.site_id == model.Site.id
        ).filter(
            model.Site.private == site_private
        )
    if site_user_active is not None:
        query = query.filter(
            model.Host.site_id == model.Site.id
        ).filter(
            model.Site.user_active == site_user_active
        )
    if site_admin_active is not None:
        query = query.filter(
            model.Host.site_id == model.Site.id
        ).filter(
            model.Site.admin_active == site_admin_active
        )

    if up2date is not None:
        query = query.filter(
            model.Host.id == model.HostCategory.host_id
        ).filter(
            model.HostCategory.id == model.HostCategoryDir.host_category_id
        ).filter(
            model.HostCategoryDir.up2date == up2date
        )

    if version_id is not None:
        query = query.filter(
            model.Host.id == model.HostCategory.host_id
        ).filter(
            model.HostCategory.category_id == model.Category.id
        ).filter(
            model.Category.id == model.Repository.category_id
        ).filter(
            model.Repository.version_id == version_id
        )

    if arch_id is not None:
        query = query.filter(
            model.Host.id == model.HostCategory.host_id
        ).filter(
            model.HostCategory.category_id == model.Category.id
        ).filter(
            model.Category.id == model.Repository.category_id
        ).filter(
            model.Repository.arch_id == arch_id
        )

    return query.all()
<|endoftext|>


<commit_before>#! /usr/bin/env python

import numpy as np

from ..core.utils import as_id_array

def count_repeated_values(x):
    """Count how many times in an array values repeat and where they appear.

    Return a list of length *n* that gives the values and indices of repeated
    values. The first element of the list will be the values and indices of
    all values that appear once or the first time repeated values appear. The
    next element, values that repeat twice or more, and so on. Thus, the
    length of the returned list will be the maximum number that any value is
    repeated in *x*.

    Parameters
    ----------
    x : array_like
        Input array to count repeated values.

    Returns
    -------
    list of tuple
        List of tuples of (*repeated_values*, *indices*).

    Examples
    --------

    For an array that contains no repeated values, this function just returns
    a copy of *x*, and the indices to each element.

    >>> import numpy as np
    >>> from landlab.utils.count_repeats import count_repeated_values
    >>> counts = count_repeated_values(np.array([20, 30, 40], dtype=np.int))
    >>> len(counts)
    1
    >>> counts[0]
    (array([20, 30, 40]), array([0, 1, 2]))

    If *x* contains a repeated value, the first element contains all unique
    values along with their indices. For repeated values, return indices to
    their first occurance. The second element contains values and indices to
    values occuring two or more times.

    >>> counts = count_repeated_values(np.array([20, 30, 40, 30, 30], dtype=np.int))
    >>> len(counts)
    3
    >>> counts[0]
    (array([20, 30, 40]), array([0, 1, 2]))
    >>> counts[0][0].dtype
    dtype('int64')
    >>> counts[0][0].dtype.itemsize
    8
    >>> counts[0][1].dtype
    dtype('int64')
    >>> counts[0][1].dtype.itemsize
    8
    >>> counts[1]
    (array([30]), array([3]))
    >>> counts[2]
    (array([30]), array([4]))
    """
    counts = []

    (unique_values, unique_inds) = np.unique(x, return_index=True)
    #if len(unique_values) > 0:
    #    x_inds = np.arange(len(x), dtype=np.int)
    #    #counts.append((unique_values, unique_inds.astype(np.int, copy=True)))
    #    #counts.append((unique_values, as_id_array(unique_inds)))
    #    counts.append((unique_values, x_inds[unique_ids]))

    #    while 1:
    #        x = np.delete(x, unique_inds)
    #        x_inds = np.delete(x_inds, unique_inds)
    #        (unique_values, unique_inds) = np.unique(x, return_index=True)

    #        if len(unique_values) > 0:
    #            counts.append((unique_values, x_inds[unique_inds]))
    #        else:
    #            break

    x_inds = np.arange(len(x), dtype=np.int)
    while len(unique_values) > 0:
        counts.append((unique_values, x_inds[unique_inds]))
        x = np.delete(x, unique_inds)
        x_inds = np.delete(x_inds, unique_inds)
        (unique_values, unique_inds) = np.unique(x, return_index=True)

    return counts
<commit_msg>Removed doctest to check for array type.<commit_after>#! /usr/bin/env python

import numpy as np

from ..core.utils import as_id_array

def count_repeated_values(x):
    """Count how many times in an array values repeat and where they appear.

    Return a list of length *n* that gives the values and indices of repeated
    values. The first element of the list will be the values and indices of
    all values that appear once or the first time repeated values appear. The
    next element, values that repeat twice or more, and so on. Thus, the
    length of the returned list will be the maximum number that any value is
    repeated in *x*.

    Parameters
    ----------
    x : array_like
        Input array to count repeated values.

    Returns
    -------
    list of tuple
        List of tuples of (*repeated_values*, *indices*).

    Examples
    --------

    For an array that contains no repeated values, this function just returns
    a copy of *x*, and the indices to each element.

    >>> import numpy as np
    >>> from landlab.utils.count_repeats import count_repeated_values
    >>> counts = count_repeated_values(np.array([20, 30, 40], dtype=np.int))
    >>> len(counts)
    1
    >>> counts[0]
    (array([20, 30, 40]), array([0, 1, 2]))

    If *x* contains a repeated value, the first element contains all unique
    values along with their indices. For repeated values, return indices to
    their first occurance. The second element contains values and indices to
    values occuring two or more times.

    >>> counts = count_repeated_values(np.array([20, 30, 40, 30, 30], dtype=np.int))
    >>> len(counts)
    3
    >>> counts[0]
    (array([20, 30, 40]), array([0, 1, 2]))
    >>> counts[1]
    (array([30]), array([3]))
    >>> counts[2]
    (array([30]), array([4]))
    """
    counts = []

    (unique_values, unique_inds) = np.unique(x, return_index=True)
    x_inds = np.arange(len(x), dtype=np.int)
    while len(unique_values) > 0:
        counts.append((unique_values, x_inds[unique_inds]))
        x = np.delete(x, unique_inds)
        x_inds = np.delete(x_inds, unique_inds)
        (unique_values, unique_inds) = np.unique(x, return_index=True)

    return counts
<|endoftext|>


<commit_before># coding=utf-8
"""
Imports the proper readline for the platform and provides utility functions for it
"""
from enum import Enum
import sys

# Prefer statically linked gnureadline if available (for macOS compatibility due to issues with libedit)
try:
    # noinspection PyPackageRequirements
    import gnureadline as readline
except ImportError:
    # Try to import readline, but allow failure for convenience in Windows unit testing
    # Note: If this actually fails, you should install readline on Linux or Mac or pyreadline on Windows
    try:
        # noinspection PyUnresolvedReferences
        import readline
    except ImportError:  # pragma: no cover
        pass

class RlType(Enum):
    """Readline library types we recognize"""
    GNU = 1
    PYREADLINE = 2
    NONE = 3

# Check what implementation of readline we are using
rl_type = RlType.NONE

# Tells if the terminal we are running in supports vt100 control characters
vt100_support = False

# Explanation for why readline wasn't loaded
_rl_warn_reason = ''

# The order of this check matters since importing pyreadline will also show readline in the modules list
if 'pyreadline' in sys.modules:
    rl_type = RlType.PYREADLINE

    from ctypes import byref
    from ctypes.wintypes import DWORD, HANDLE
    import atexit

    # Check if we are running in a terminal
    if sys.stdout.isatty():  # pragma: no cover
        # noinspection PyPep8Naming,PyUnresolvedReferences
        def enable_win_vt100(handle: HANDLE) -> bool:
            """
            Enables VT100 character sequences in a Windows console
            This only works on Windows 10 and up
            :param handle: the handle on which to enable vt100
            :return: True if vt100 characters are enabled for the handle
            """
            ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004

            # Get the current mode for this handle in the console
            cur_mode = DWORD(0)
            readline.rl.console.GetConsoleMode(handle, byref(cur_mode))

            retVal = False

            # Check if ENABLE_VIRTUAL_TERMINAL_PROCESSING is already enabled
            if (cur_mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING) != 0:
                retVal = True

            elif readline.rl.console.SetConsoleMode(handle, cur_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING):
                # Restore the original mode when we exit
                atexit.register(readline.rl.console.SetConsoleMode, handle, cur_mode)
                retVal = True

            return retVal

        # Enable VT100 sequences for stdout and stderr
        STD_OUT_HANDLE = -11
        STD_ERROR_HANDLE = -12
        # noinspection PyUnresolvedReferences
        vt100_stdout_support = enable_win_vt100(readline.rl.console.GetStdHandle(STD_OUT_HANDLE))
        # noinspection PyUnresolvedReferences
        vt100_stderr_support = enable_win_vt100(readline.rl.console.GetStdHandle(STD_ERROR_HANDLE))
        vt100_support = vt100_stdout_support and vt100_stderr_support

    ############################################################################################################
    # pyreadline is incomplete in terms of the Python readline API. Add the missing functions we need.
    ############################################################################################################
    # readline.redisplay()
    try:
        getattr(readline, 'redisplay')
    except AttributeError:
        # noinspection PyProtectedMember,PyUnresolvedReferences
        readline.redisplay = readline.rl.mode._update_line

    # readline.remove_history_item()
    try:
        getattr(readline, 'remove_history_item')
    except AttributeError:
        # noinspection PyProtectedMember,PyUnresolvedReferences
        def pyreadline_remove_history_item(pos: int) -> None:
            """
            An implementation of remove_history_item() for pyreadline
            :param pos: The 0-based position in history to remove
            """
            # Save of the current location of the history cursor
            saved_cursor = readline.rl.mode._history.history_cursor

            # Delete the history item
            del(readline.rl.mode._history.history[pos])

            # Update the cursor if needed
            if saved_cursor > pos:
                readline.rl.mode._history.history_cursor -= 1

        readline.remove_history_item = pyreadline_remove_history_item

elif 'gnureadline' in sys.modules or 'readline' in sys.modules:
    # We don't support libedit
    if 'libedit' not in readline.__doc__:
        try:
            # Load the readline lib so we can access members of it
            import ctypes
            readline_lib = ctypes.CDLL(readline.__file__)
        except AttributeError:  # pragma: no cover
            _rl_warn_reason = ("this application is running in a non-standard Python environment in\n"
                               "which readline is not loaded dynamically from a shared library file.")
        else:
            rl_type = RlType.GNU
            if sys.stdout.isatty():
                vt100_support = True

# Check if readline was loaded
if rl_type == RlType.NONE:  # pragma: no cover
    if not _rl_warn_reason:
        _rl_warn_reason = ("no supported version of readline was found. To resolve this, install\n"
                           "pyreadline on Windows or gnureadline on Mac.")
    rl_warning = ("Readline features including tab completion have been disabled because\n"
                  + _rl_warn_reason + '\n\n')
else:
    rl_warning = ''

# noinspection PyProtectedMember,PyUnresolvedReferences
def rl_force_redisplay() -> None:  # pragma: no cover
    """
    Causes readline to display the prompt and input text wherever the cursor is and start
    reading input from this location. This is the proper way to restore the input line after
    printing to the screen
    """
    if not sys.stdout.isatty():
        return

    if rl_type == RlType.GNU:
        readline_lib.rl_forced_update_display()

        # After manually updating the display, readline asks that rl_display_fixed be set to 1 for efficiency
        display_fixed = ctypes.c_int.in_dll(readline_lib, "rl_display_fixed")
        display_fixed.value = 1

    elif rl_type == RlType.PYREADLINE:
        # Call _print_prompt() first to set the new location of the prompt
        readline.rl.mode._print_prompt()
        readline.rl.mode._update_line()

# noinspection PyProtectedMember, PyUnresolvedReferences
def rl_get_point() -> int:  # pragma: no cover
    """
    Returns the offset of the current cursor position in rl_line_buffer
    """
    if rl_type == RlType.GNU:
        return ctypes.c_int.in_dll(readline_lib, "rl_point").value

    elif rl_type == RlType.PYREADLINE:
        return readline.rl.mode.l_buffer.point

    else:
        return 0

# noinspection PyProtectedMember, PyUnresolvedReferences
def rl_set_prompt(prompt: str) -> None:  # pragma: no cover
    """
    Sets readline's prompt
    :param prompt: the new prompt value
    """
    safe_prompt = rl_make_safe_prompt(prompt)

    if rl_type == RlType.GNU:
        encoded_prompt = bytes(safe_prompt, encoding='utf-8')
        readline_lib.rl_set_prompt(encoded_prompt)

    elif rl_type == RlType.PYREADLINE:
        readline.rl._set_prompt(safe_prompt)

def rl_make_safe_prompt(prompt: str) -> str:  # pragma: no cover
    """Overcome bug in GNU Readline in relation to calculation of prompt length in presence of ANSI escape codes.

    :param prompt: original prompt
    :return: prompt safe to pass to GNU Readline
    """
    if rl_type == RlType.GNU:
        # start code to tell GNU Readline about beginning of invisible characters
        start = "\x01"

        # end code to tell GNU Readline about end of invisible characters
        end = "\x02"

        escaped = False
        result = ""

        for c in prompt:
            if c == "\x1b" and not escaped:
                result += start + c
                escaped = True
            elif c.isalpha() and escaped:
                result += c + end
                escaped = False
            else:
                result += c

        return result

    else:
        return prompt
<commit_msg>Simplified line of code<commit_after># coding=utf-8
"""
Imports the proper readline for the platform and provides utility functions for it
"""
from enum import Enum
import sys

# Prefer statically linked gnureadline if available (for macOS compatibility due to issues with libedit)
try:
    # noinspection PyPackageRequirements
    import gnureadline as readline
except ImportError:
    # Try to import readline, but allow failure for convenience in Windows unit testing
    # Note: If this actually fails, you should install readline on Linux or Mac or pyreadline on Windows
    try:
        # noinspection PyUnresolvedReferences
        import readline
    except ImportError:  # pragma: no cover
        pass

class RlType(Enum):
    """Readline library types we recognize"""
    GNU = 1
    PYREADLINE = 2
    NONE = 3

# Check what implementation of readline we are using
rl_type = RlType.NONE

# Tells if the terminal we are running in supports vt100 control characters
vt100_support = False

# Explanation for why readline wasn't loaded
_rl_warn_reason = ''

# The order of this check matters since importing pyreadline will also show readline in the modules list
if 'pyreadline' in sys.modules:
    rl_type = RlType.PYREADLINE

    from ctypes import byref
    from ctypes.wintypes import DWORD, HANDLE
    import atexit

    # Check if we are running in a terminal
    if sys.stdout.isatty():  # pragma: no cover
        # noinspection PyPep8Naming,PyUnresolvedReferences
        def enable_win_vt100(handle: HANDLE) -> bool:
            """
            Enables VT100 character sequences in a Windows console
            This only works on Windows 10 and up
            :param handle: the handle on which to enable vt100
            :return: True if vt100 characters are enabled for the handle
            """
            ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004

            # Get the current mode for this handle in the console
            cur_mode = DWORD(0)
            readline.rl.console.GetConsoleMode(handle, byref(cur_mode))

            retVal = False

            # Check if ENABLE_VIRTUAL_TERMINAL_PROCESSING is already enabled
            if (cur_mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING) != 0:
                retVal = True

            elif readline.rl.console.SetConsoleMode(handle, cur_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING):
                # Restore the original mode when we exit
                atexit.register(readline.rl.console.SetConsoleMode, handle, cur_mode)
                retVal = True

            return retVal

        # Enable VT100 sequences for stdout and stderr
        STD_OUT_HANDLE = -11
        STD_ERROR_HANDLE = -12
        # noinspection PyUnresolvedReferences
        vt100_stdout_support = enable_win_vt100(readline.rl.console.GetStdHandle(STD_OUT_HANDLE))
        # noinspection PyUnresolvedReferences
        vt100_stderr_support = enable_win_vt100(readline.rl.console.GetStdHandle(STD_ERROR_HANDLE))
        vt100_support = vt100_stdout_support and vt100_stderr_support

    ############################################################################################################
    # pyreadline is incomplete in terms of the Python readline API. Add the missing functions we need.
    ############################################################################################################
    # readline.redisplay()
    try:
        getattr(readline, 'redisplay')
    except AttributeError:
        # noinspection PyProtectedMember,PyUnresolvedReferences
        readline.redisplay = readline.rl.mode._update_line

    # readline.remove_history_item()
    try:
        getattr(readline, 'remove_history_item')
    except AttributeError:
        # noinspection PyProtectedMember,PyUnresolvedReferences
        def pyreadline_remove_history_item(pos: int) -> None:
            """
            An implementation of remove_history_item() for pyreadline
            :param pos: The 0-based position in history to remove
            """
            # Save of the current location of the history cursor
            saved_cursor = readline.rl.mode._history.history_cursor

            # Delete the history item
            del(readline.rl.mode._history.history[pos])

            # Update the cursor if needed
            if saved_cursor > pos:
                readline.rl.mode._history.history_cursor -= 1

        readline.remove_history_item = pyreadline_remove_history_item

elif 'gnureadline' in sys.modules or 'readline' in sys.modules:
    # We don't support libedit
    if 'libedit' not in readline.__doc__:
        try:
            # Load the readline lib so we can access members of it
            import ctypes
            readline_lib = ctypes.CDLL(readline.__file__)
        except AttributeError:  # pragma: no cover
            _rl_warn_reason = ("this application is running in a non-standard Python environment in\n"
                               "which readline is not loaded dynamically from a shared library file.")
        else:
            rl_type = RlType.GNU
            vt100_support = sys.stdout.isatty()

# Check if readline was loaded
if rl_type == RlType.NONE:  # pragma: no cover
    if not _rl_warn_reason:
        _rl_warn_reason = ("no supported version of readline was found. To resolve this, install\n"
                           "pyreadline on Windows or gnureadline on Mac.")
    rl_warning = ("Readline features including tab completion have been disabled because\n"
                  + _rl_warn_reason + '\n\n')
else:
    rl_warning = ''

# noinspection PyProtectedMember,PyUnresolvedReferences
def rl_force_redisplay() -> None:  # pragma: no cover
    """
    Causes readline to display the prompt and input text wherever the cursor is and start
    reading input from this location. This is the proper way to restore the input line after
    printing to the screen
    """
    if not sys.stdout.isatty():
        return

    if rl_type == RlType.GNU:
        readline_lib.rl_forced_update_display()

        # After manually updating the display, readline asks that rl_display_fixed be set to 1 for efficiency
        display_fixed = ctypes.c_int.in_dll(readline_lib, "rl_display_fixed")
        display_fixed.value = 1

    elif rl_type == RlType.PYREADLINE:
        # Call _print_prompt() first to set the new location of the prompt
        readline.rl.mode._print_prompt()
        readline.rl.mode._update_line()

# noinspection PyProtectedMember, PyUnresolvedReferences
def rl_get_point() -> int:  # pragma: no cover
    """
    Returns the offset of the current cursor position in rl_line_buffer
    """
    if rl_type == RlType.GNU:
        return ctypes.c_int.in_dll(readline_lib, "rl_point").value

    elif rl_type == RlType.PYREADLINE:
        return readline.rl.mode.l_buffer.point

    else:
        return 0

# noinspection PyProtectedMember, PyUnresolvedReferences
def rl_set_prompt(prompt: str) -> None:  # pragma: no cover
    """
    Sets readline's prompt
    :param prompt: the new prompt value
    """
    safe_prompt = rl_make_safe_prompt(prompt)

    if rl_type == RlType.GNU:
        encoded_prompt = bytes(safe_prompt, encoding='utf-8')
        readline_lib.rl_set_prompt(encoded_prompt)

    elif rl_type == RlType.PYREADLINE:
        readline.rl._set_prompt(safe_prompt)

def rl_make_safe_prompt(prompt: str) -> str:  # pragma: no cover
    """Overcome bug in GNU Readline in relation to calculation of prompt length in presence of ANSI escape codes.

    :param prompt: original prompt
    :return: prompt safe to pass to GNU Readline
    """
    if rl_type == RlType.GNU:
        # start code to tell GNU Readline about beginning of invisible characters
        start = "\x01"

        # end code to tell GNU Readline about end of invisible characters
        end = "\x02"

        escaped = False
        result = ""

        for c in prompt:
            if c == "\x1b" and not escaped:
                result += start + c
                escaped = True
            elif c.isalpha() and escaped:
                result += c + end
                escaped = False
            else:
                result += c

        return result

    else:
        return prompt
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.9.5 on 2017-05-09 09:24
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('attempts', '0007_auto_20161004_0927'),
    ]

    operations = [
        migrations.AddField(
            model_name='attempt',
            name='submission_date',
            field=models.DateTimeField(null=True),
        ),
        migrations.AddField(
            model_name='historicalattempt',
            name='submission_date',
            field=models.DateTimeField(null=True),
        ),
        migrations.RunSQL(
            'UPDATE attempts_historicalattempt SET submission_date = history_date'
        ),
        migrations.RunSQL(
            '''UPDATE attempts_attempt
                  SET submission_date = subquery.submission_date
                 FROM (
                    SELECT user_id, part_id, max(history_date) AS submission_date
                      FROM attempts_historicalattempt
                     GROUP BY user_id, part_id
                 ) AS subquery
                WHERE attempts_attempt.user_id = subquery.user_id
                  AND attempts_attempt.part_id = subquery.part_id
            '''
        ),
        migrations.AlterField(
            model_name='attempt',
            name='submission_date',
            field=models.DateTimeField(auto_now=True),
        ),
        migrations.AlterField(
            model_name='historicalattempt',
            name='submission_date',
            field=models.DateTimeField(blank=True, editable=False),
        ),
    ]
<commit_msg>Revert "Revert "Make migration SQLite compatible""<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.9.5 on 2017-05-09 09:24
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('attempts', '0007_auto_20161004_0927'),
    ]

    operations = [
        migrations.AddField(
            model_name='attempt',
            name='submission_date',
            field=models.DateTimeField(null=True),
        ),
        migrations.AddField(
            model_name='historicalattempt',
            name='submission_date',
            field=models.DateTimeField(null=True),
        ),
        migrations.RunSQL(
            'UPDATE attempts_historicalattempt SET submission_date = history_date'
        ),
        migrations.RunSQL(
            '''UPDATE attempts_attempt
                  SET submission_date = (
                     SELECT max(history_date)
                      FROM attempts_historicalattempt
                     WHERE attempts_attempt.user_id = user_id
                       AND attempts_attempt.part_id = part_id
                 )
            '''
        ),
        migrations.AlterField(
            model_name='attempt',
            name='submission_date',
            field=models.DateTimeField(auto_now=True),
        ),
        migrations.AlterField(
            model_name='historicalattempt',
            name='submission_date',
            field=models.DateTimeField(blank=True, editable=False),
        ),
    ]
<|endoftext|>


<commit_before>"""
Team Challenge: Predicting Turn Types
Authors: Tong, Will, and Ryan
"""

import os
import re
import sys
import csv
import argparse
from pprint import pprint
from time import time

import logging
from operator import itemgetter
import numpy as np

from sklearn.preprocessing import LabelEncoder
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.naive_bayes import MultinomialNB
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.cross_validation import KFold

def read_csv(path):
    output = []
    with open(path, 'rb') as f:
        reader = csv.reader(f, delimiter=",")
        for row in reader:
            output.append(row)
    return output

def read_dir_sk(path):
    """
    Takes a path to a directory of csv data files, parses them individually,
    Returns an array of dicts, array of qa labels, and an array of em labels
    """
    X = []
    y_qa = []
    y_em = []
    for root, subdirs, files in os.walk(path):
        for f in files:
            if f.endswith(".csv"):
                a_file_path = os.path.join(root, f)
                csv = read_csv(a_file_path)

                for row in csv:
                    y_qa.append(row[3])
                    y_em.append(row[4])
                    # remove asterisk *
                    # remove entire repair turns <>
                    # remove brackets around words []
                    text = re.sub(r'(<|>|\[|\]|\*)', '', row[5])
                    X.append(text)
    return X, y_qa, y_em

def read_dir_dict(path):
    """
    Takes a path to a directory of csv data files, parses them individually,
    Returns an array of dicts
    """
    output = []
    for root, subdirs, files in os.walk(path):
        for f in files:
            if f.endswith(".csv"):
                a_file_path = os.path.join(root, f)
                csv = read_csv(a_file_path)

                for row in csv:
                    example = { "subjectID": row[0],
                                "imageID": row[1],
                                "questionID": row[2],
                                "Q/A": row[3],
                                "E/M": row[4],
                                "text": row[5] }
                    output.append(example)
    return output

def q_features(data):
    questionWords = ["who", "what", "where", "when", "why", "[why]", "how", "and"]
    ansWords = ["sp", "well", "uh", "hm", "it", "mm"]

    totalCount = 0
    qCount = 0
    aCount = 0
    for d in data:
        totalCount += 1
        for q in questionWords:
            if d["text"].startswith(q):
                qCount += 1
                break
        for a in ansWords:
            if d["text"].startswith(a):
                aCount += 1
                break

    print("Total: ", totalCount, "Q: ", qCount, "A: ", aCount)
    print("Ratio: ", (qCount + aCount)/totalCount)

def encode_labels(y):
    """
    Takes a list of labels and converts them to numpy compatible classes
    Returns (the converted labels, and the encoder)
    """
    le = LabelEncoder()
    y = le.fit_transform(y)

    # example of how to get the labels back:
    # print le.inverse_transform([1, 0, 1, 0, 1, 0, 1, 0])
    return y, le

def grid_search_pipeline(pipe, params, cv, data, targets):
    """
    For a given pipeline, parameters, cross validator, dataset, and target labels
    Run a grid search using the given pipeline and cross validator over the set
    of parameters, printing the progress as it goes.
    Returns the results
    """

    grid_search = GridSearchCV(pipe, params, cv=cv, verbose=1)

    print("Performing grid search...")
    print("pipe:", [name for name, _ in pipe.steps])
    print("params:")
    pprint(params)
    print
    t0 = time()
    grid_search.fit(data, targets)
    print("done in %0.3fs" % (time() - t0))

    return grid_search

def tfidf_mnb_pipeline(data, targets, num_images=11):
    """
    A simple pipeline using tfidf vectorizer, multinomial naive bayes, and KFold
    cross validation such that on each run one image file is left out (set
    num_images differently if working on a smaller training set)
    Returns the grid_search results, pipeline, and parameters used
    """
    pipe = Pipeline([
        ("vect", TfidfVectorizer(stop_words="english")),
        ("clf", MultinomialNB())
    ])

    params = {
        #"vect__max_df": (0.5, 0.75, 1.0),
        #"vect__max_features": (None, 5000, 10000, 50000),
        "vect__use_idf": (True, False),
        #"vect__analyzer": ("word", "char"),
        "vect__ngram_range": ((1,2), (1,3), (1,1)),
        "vect__norm": ("l1", "l2"),
        "clf__alpha": (0.001, 0.00001, 0.000001)
    }

    cv = KFold(len(targets), num_images)

    grid_search = grid_search_pipeline(pipe, params, cv, data, targets)

    #return grid_search.best_estimator_
    return grid_search, pipe, params

def report_grid_search(grid_search, pipe, params):
    """
    Report the results of a given grid search
    """
    print("Best score: %0.3f" % grid_search.best_score_)
    print("Best params set:")
    best_params = grid_search.best_estimator_.get_params()
    for param_name in sorted(params.keys()):
        print("\t%s: %r" % (param_name, best_params[param_name]))

    report_grid_scores(grid_search.grid_scores_)

def report_grid_scores(grid_scores, n_top=10):
    """
    Helper function to report score performance of the top n classifier / params
    """
    top_scores = sorted(grid_scores, key=itemgetter(1), reverse=True)[:n_top]
    for i, score in enumerate(top_scores):
        print("{2}. Mean validation score: {0:.3f} (std: {1:.3f})".format(
              score.mean_validation_score,
              np.std(score.cv_validation_scores),
              i + 1 ))
        print("Parameters: {0}".format(score.parameters))
        print("")

def main(args):
    if (args.data):
        # Our features and two sets of labels
        X, y_qa, y_em = read_dir_sk(args.data)

        y_qa, le_qa = encode_labels(y_qa)
        y_em, le_em = encode_labels(y_em)

        print("--- Q/A ---")
        qa_grid_search, qa_pipe, qa_params = tfidf_mnb_pipeline(X, y_qa)
        report_grid_search(qa_grid_search, qa_pipe, qa_params)
        best_qa_clf = qa_grid_search.best_estimator_

        print
        print("--- E/M ---")
        em_grid_search, em_pipe, em_params = tfidf_mnb_pipeline(X, y_em)
        report_grid_search(em_grid_search, em_pipe, em_params)
        best_em_clf = em_grid_search.best_estimator_

    elif (args.test and args.train):
        train_X, train_y_qa, train_y_em = read_dir_sk(args.train)
        test_X, test_y_qa, test_y_em = read_dir_sk(args.test)

        # how many documents are in the training set?
        len_img_train = int(float(len(train_y_qa))/40.0)
        print(len_img_train)

        print("--- Q/A ---")
        qa_grid_search, qa_pipe, qa_params = tfidf_mnb_pipeline(train_X,
                                                                train_y_qa,
                                                                len_img_train)
        report_grid_search(qa_grid_search, qa_pipe, qa_params)
        best_qa_clf = qa_grid_search.best_estimator_

        print("Q/A performance on the left out dataset: {0}".format(
            best_qa_clf.score(test_X, test_y_qa)))
        qa_predictions = best_qa_clf.predict(test_X)
        print(qa_predictions)

        print
        print("--- E/M ---")
        em_grid_search, em_pipe, em_params = tfidf_mnb_pipeline(train_X,
                                                                train_y_em,
                                                                len_img_train)
        report_grid_search(em_grid_search, em_pipe, em_params)
        best_em_clf = em_grid_search.best_estimator_

        print("E/M performance on the left out dataset: {0}".format(
            best_em_clf.score(test_X, test_y_em)))
        em_predictions = best_em_clf.predict(test_X)
        print(em_predictions)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    # If no explicit training data, algorithm splits it itself
    parser.add_argument( "-d", "--data", help="pass a folder path to the data")

    # Otherwise
    parser.add_argument( "-t", "--train", help="pass a folder path to the training data")
    parser.add_argument( "-s", "--test", help="pass a folder path to the testing data")

    args = parser.parse_args()
    main(args)
<commit_msg>Add the number of documents in the test set to a variable, len_img_test<commit_after>"""
Team Challenge: Predicting Turn Types
Authors: Tong, Will, and Ryan
"""

import os
import re
import sys
import csv
import argparse
from pprint import pprint
from time import time

import logging
from operator import itemgetter
import numpy as np

from sklearn.preprocessing import LabelEncoder
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.naive_bayes import MultinomialNB
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.cross_validation import KFold

def read_csv(path):
    output = []
    with open(path, 'rb') as f:
        reader = csv.reader(f, delimiter=",")
        for row in reader:
            output.append(row)
    return output

def read_dir_sk(path):
    """
    Takes a path to a directory of csv data files, parses them individually,
    Returns an array of dicts, array of qa labels, and an array of em labels
    """
    X = []
    y_qa = []
    y_em = []
    for root, subdirs, files in os.walk(path):
        for f in files:
            if f.endswith(".csv"):
                a_file_path = os.path.join(root, f)
                csv = read_csv(a_file_path)

                for row in csv:
                    y_qa.append(row[3])
                    y_em.append(row[4])
                    # remove asterisk *
                    # remove entire repair turns <>
                    # remove brackets around words []
                    text = re.sub(r'(<|>|\[|\]|\*)', '', row[5])
                    X.append(text)
    return X, y_qa, y_em

def read_dir_dict(path):
    """
    Takes a path to a directory of csv data files, parses them individually,
    Returns an array of dicts
    """
    output = []
    for root, subdirs, files in os.walk(path):
        for f in files:
            if f.endswith(".csv"):
                a_file_path = os.path.join(root, f)
                csv = read_csv(a_file_path)

                for row in csv:
                    example = { "subjectID": row[0],
                                "imageID": row[1],
                                "questionID": row[2],
                                "Q/A": row[3],
                                "E/M": row[4],
                                "text": row[5] }
                    output.append(example)
    return output

def q_features(data):
    questionWords = ["who", "what", "where", "when", "why", "[why]", "how", "and"]
    ansWords = ["sp", "well", "uh", "hm", "it", "mm"]

    totalCount = 0
    qCount = 0
    aCount = 0
    for d in data:
        totalCount += 1
        for q in questionWords:
            if d["text"].startswith(q):
                qCount += 1
                break
        for a in ansWords:
            if d["text"].startswith(a):
                aCount += 1
                break

    print("Total: ", totalCount, "Q: ", qCount, "A: ", aCount)
    print("Ratio: ", (qCount + aCount)/totalCount)

def encode_labels(y):
    """
    Takes a list of labels and converts them to numpy compatible classes
    Returns (the converted labels, and the encoder)
    """
    le = LabelEncoder()
    y = le.fit_transform(y)

    # example of how to get the labels back:
    # print le.inverse_transform([1, 0, 1, 0, 1, 0, 1, 0])
    return y, le

def grid_search_pipeline(pipe, params, cv, data, targets):
    """
    For a given pipeline, parameters, cross validator, dataset, and target labels
    Run a grid search using the given pipeline and cross validator over the set
    of parameters, printing the progress as it goes.
    Returns the results
    """

    grid_search = GridSearchCV(pipe, params, cv=cv, verbose=1)

    print("Performing grid search...")
    print("pipe:", [name for name, _ in pipe.steps])
    print("params:")
    pprint(params)
    print
    t0 = time()
    grid_search.fit(data, targets)
    print("done in %0.3fs" % (time() - t0))

    return grid_search

def tfidf_mnb_pipeline(data, targets, num_images=11):
    """
    A simple pipeline using tfidf vectorizer, multinomial naive bayes, and KFold
    cross validation such that on each run one image file is left out (set
    num_images differently if working on a smaller training set)
    Returns the grid_search results, pipeline, and parameters used
    """
    pipe = Pipeline([
        ("vect", TfidfVectorizer(stop_words="english")),
        ("clf", MultinomialNB())
    ])

    params = {
        #"vect__max_df": (0.5, 0.75, 1.0),
        #"vect__max_features": (None, 5000, 10000, 50000),
        "vect__use_idf": (True, False),
        #"vect__analyzer": ("word", "char"),
        "vect__ngram_range": ((1,2), (1,3), (1,1)),
        "vect__norm": ("l1", "l2"),
        "clf__alpha": (0.001, 0.00001, 0.000001)
    }

    cv = KFold(len(targets), num_images)

    grid_search = grid_search_pipeline(pipe, params, cv, data, targets)

    #return grid_search.best_estimator_
    return grid_search, pipe, params

def report_grid_search(grid_search, pipe, params):
    """
    Report the results of a given grid search
    """
    print("Best score: %0.3f" % grid_search.best_score_)
    print("Best params set:")
    best_params = grid_search.best_estimator_.get_params()
    for param_name in sorted(params.keys()):
        print("\t%s: %r" % (param_name, best_params[param_name]))

    report_grid_scores(grid_search.grid_scores_)

def report_grid_scores(grid_scores, n_top=10):
    """
    Helper function to report score performance of the top n classifier / params
    """
    top_scores = sorted(grid_scores, key=itemgetter(1), reverse=True)[:n_top]
    for i, score in enumerate(top_scores):
        print("{2}. Mean validation score: {0:.3f} (std: {1:.3f})".format(
              score.mean_validation_score,
              np.std(score.cv_validation_scores),
              i + 1 ))
        print("Parameters: {0}".format(score.parameters))
        print("")

def main(args):
    if (args.data):
        # Our features and two sets of labels
        X, y_qa, y_em = read_dir_sk(args.data)

        y_qa, le_qa = encode_labels(y_qa)
        y_em, le_em = encode_labels(y_em)

        print("--- Q/A ---")
        qa_grid_search, qa_pipe, qa_params = tfidf_mnb_pipeline(X, y_qa)
        report_grid_search(qa_grid_search, qa_pipe, qa_params)
        best_qa_clf = qa_grid_search.best_estimator_

        print
        print("--- E/M ---")
        em_grid_search, em_pipe, em_params = tfidf_mnb_pipeline(X, y_em)
        report_grid_search(em_grid_search, em_pipe, em_params)
        best_em_clf = em_grid_search.best_estimator_

    elif (args.test and args.train):
        train_X, train_y_qa, train_y_em = read_dir_sk(args.train)
        test_X, test_y_qa, test_y_em = read_dir_sk(args.test)

        # how many documents are in the training set?
        len_img_train = int(float(len(train_y_qa))/40.0)
        len_img_test = int(float(len(test_y_qa))/40.0)
        print(len_img_train, len_img_test)

        print("--- Q/A ---")
        qa_grid_search, qa_pipe, qa_params = tfidf_mnb_pipeline(train_X,
                                                                train_y_qa,
                                                                len_img_train)
        report_grid_search(qa_grid_search, qa_pipe, qa_params)
        best_qa_clf = qa_grid_search.best_estimator_

        print("Q/A performance on the left out dataset: {0}".format(
            best_qa_clf.score(test_X, test_y_qa)))
        qa_predictions = best_qa_clf.predict(test_X)
        print(qa_predictions)

        print
        print("--- E/M ---")
        em_grid_search, em_pipe, em_params = tfidf_mnb_pipeline(train_X,
                                                                train_y_em,
                                                                len_img_train)
        report_grid_search(em_grid_search, em_pipe, em_params)
        best_em_clf = em_grid_search.best_estimator_

        print("E/M performance on the left out dataset: {0}".format(
            best_em_clf.score(test_X, test_y_em)))
        em_predictions = best_em_clf.predict(test_X)
        print(em_predictions)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    # If no explicit training data, algorithm splits it itself
    parser.add_argument( "-d", "--data", help="pass a folder path to the data")

    # Otherwise
    parser.add_argument( "-t", "--train", help="pass a folder path to the training data")
    parser.add_argument( "-s", "--test", help="pass a folder path to the testing data")

    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>from django.utils.translation import ugettext_noop
from django.utils.translation import ugettext as _
from corehq.apps.reports.standard import CustomProjectReport
from corehq.apps.reports.generic import GenericTabularReport
from corehq.apps.reports.datatables import DataTablesColumn, DataTablesHeader
from casexml.apps.case.models import CommCareCase
from corehq.apps.sms.models import ExpectedCallbackEventLog, CALLBACK_PENDING, CALLBACK_RECEIVED, CALLBACK_MISSED
from datetime import datetime, timedelta
from dimagi.utils.timezones import utils as tz_utils
from dimagi.utils.parsing import json_format_datetime
from corehq.apps.reports.util import format_datatables_data
import pytz
from dateutil.parser import parse

class MissedCallbackReport(CustomProjectReport, GenericTabularReport):
    name = ugettext_noop("Missed Callbacks")
    slug = "missed_callbacks"
    description = ugettext_noop("Summarizes two weeks of SMS / Callback interactions for all participants.")
    hide_filters = True
    flush_layout = True

    def get_past_two_weeks(self):
        if self.request.couch_user.is_commcare_user():
            time_zone = self.request.couch_user.get_time_zone()
        else:
            time_zone = None

        now = datetime.utcnow()
        if time_zone is None:
            local_datetime = now
        else:
            local_datetime = tz_utils.adjust_datetime_to_timezone(now, pytz.utc.zone, pytz.timezone(time_zone).zone)

        return [(local_datetime + timedelta(days = x)).date() for x in range(-14, 0)]

    @property
    def headers(self):
        args = [
            DataTablesColumn(_("Participant ID")),
            DataTablesColumn(_("Total No Response")),
            DataTablesColumn(_("Total Indicated")),
            DataTablesColumn(_("Total Pending")),
        ]
        args += [DataTablesColumn(date.strftime("%Y-%m-%d")) for date in self.get_past_two_weeks()]
        return DataTablesHeader(*args)

    @property
    def rows(self):
        group_id = None
        if self.request.couch_user.is_commcare_user():
            group_ids = self.request.couch_user.get_group_ids()
            if len(group_ids) > 0:
                group_id = group_ids[0]

        cases = CommCareCase.view("hqcase/types_by_domain",
                                  key=[self.domain, "participant"],
                                  reduce=False,
                                  include_docs=True).all()

        data = {}

        for case in cases:
            # If a site coordinator is viewing the report, only show participants from that site (group)
            if group_id is None or group_id == case.owner_id:
                data[case._id] = {
                    "name" : case.name,
                    "time_zone" : pytz.timezone(case.get_case_property("time_zone")),
                    "dates" : [None for x in range(14)],
                }

        dates = self.get_past_two_weeks()
        start_date = dates[0] - timedelta(days=1)
        end_date = dates[-1] + timedelta(days=2)
        date_strings = [date.strftime("%Y-%m-%d") for date in dates]

        start_utc_timestamp = json_format_datetime(start_date)
        end_utc_timestamp = json_format_datetime(end_date)

        expected_callback_events = ExpectedCallbackEventLog.view("sms/expected_callback_event",
                                                                 startkey=[self.domain, start_utc_timestamp],
                                                                 endkey=[self.domain, end_utc_timestamp],
                                                                 include_docs=True).all()

        for event in expected_callback_events:
            if event.couch_recipient in data:
                event_date = tz_utils.adjust_datetime_to_timezone(event.date, pytz.utc.zone, data[event.couch_recipient]["time_zone"].zone).date()
                event_date = event_date.strftime("%Y-%m-%d")
                if event_date in date_strings:
                    data[event.couch_recipient]["dates"][date_strings.index(event_date)] = event.status

        result = []
        for case_id, data_dict in data.items():
            row = [
                self._fmt(data_dict["name"]),
                None,
                None,
                None,
            ]

            total_no_response = 0
            total_indicated = 0
            total_pending = 0

            for date_status in data_dict["dates"]:
                if date_status == CALLBACK_PENDING:
                    total_indicated += 1
                    total_pending += 1
                    row.append(self._fmt(_("pending")))
                elif date_status == CALLBACK_RECEIVED:
                    total_indicated += 1
                    row.append(self._fmt(_("OK")))
                elif date_status == CALLBACK_MISSED:
                    total_indicated += 1
                    total_no_response += 1
                    row.append(self._fmt_highlight(_("No Response")))
                else:
                    row.append(self._fmt(_("not indicated")))

            if total_no_response > 0:
                row[1] = self._fmt_highlight(total_no_response)
            else:
                row[1] = self._fmt(total_no_response)
            row[2] = self._fmt(total_indicated)
            row[3] = self._fmt(total_pending)

            result.append(row)

        return result

    def _fmt(self, value):
        return self.table_cell(value, '
%s
' % value)

    def _fmt_highlight(self, value):
        return self.table_cell(value, '
%s
' % value)

<commit_msg>Tweaks a5288 report<commit_after>from django.utils.translation import ugettext_noop
from django.utils.translation import ugettext as _
from corehq.apps.reports.standard import CustomProjectReport
from corehq.apps.reports.generic import GenericTabularReport
from corehq.apps.reports.datatables import DataTablesColumn, DataTablesHeader
from casexml.apps.case.models import CommCareCase
from corehq.apps.sms.models import ExpectedCallbackEventLog, CALLBACK_PENDING, CALLBACK_RECEIVED, CALLBACK_MISSED
from datetime import datetime, timedelta
from dimagi.utils.timezones import utils as tz_utils
from dimagi.utils.parsing import json_format_datetime
from corehq.apps.reports.util import format_datatables_data
import pytz
from dateutil.parser import parse

class MissedCallbackReport(CustomProjectReport, GenericTabularReport):
    name = ugettext_noop("Missed Callbacks")
    slug = "missed_callbacks"
    description = ugettext_noop("Summarizes two weeks of SMS / Callback interactions for all participants.")
    hide_filters = True
    flush_layout = True

    def get_past_two_weeks(self):
        now = datetime.utcnow()
        local_datetime = tz_utils.adjust_datetime_to_timezone(now, pytz.utc.zone, self.timezone.zone)
        return [(local_datetime + timedelta(days = x)).date() for x in range(-14, 0)]

    @property
    def headers(self):
        args = [
            DataTablesColumn(_("Participant ID")),
            DataTablesColumn(_("Total No Response")),
            DataTablesColumn(_("Total Indicated")),
            DataTablesColumn(_("Total Pending")),
        ]
        args += [DataTablesColumn(date.strftime("%Y-%m-%d")) for date in self.get_past_two_weeks()]
        return DataTablesHeader(*args)

    @property
    def rows(self):
        group_id = None
        if self.request.couch_user.is_commcare_user():
            group_ids = self.request.couch_user.get_group_ids()
            if len(group_ids) > 0:
                group_id = group_ids[0]

        cases = CommCareCase.view("hqcase/types_by_domain",
                                  key=[self.domain, "participant"],
                                  reduce=False,
                                  include_docs=True).all()

        data = {}

        for case in cases:
            # If a site coordinator is viewing the report, only show participants from that site (group)
            if group_id is None or group_id == case.owner_id:
                data[case._id] = {
                    "name" : case.name,
                    "time_zone" : case.get_case_property("time_zone"),
                    "dates" : [None for x in range(14)],
                }

        dates = self.get_past_two_weeks()
        date_strings = [date.strftime("%Y-%m-%d") for date in dates]

        start_date = dates[0] - timedelta(days=1)
        end_date = dates[-1] + timedelta(days=2)

        start_utc_timestamp = json_format_datetime(start_date)
        end_utc_timestamp = json_format_datetime(end_date)

        expected_callback_events = ExpectedCallbackEventLog.view("sms/expected_callback_event",
                                                                 startkey=[self.domain, start_utc_timestamp],
                                                                 endkey=[self.domain, end_utc_timestamp],
                                                                 include_docs=True).all()

        for event in expected_callback_events:
            if event.couch_recipient in data:
                event_date = tz_utils.adjust_datetime_to_timezone(event.date, pytz.utc.zone, data[event.couch_recipient]["time_zone"]).date()
                event_date = event_date.strftime("%Y-%m-%d")
                if event_date in date_strings:
                    data[event.couch_recipient]["dates"][date_strings.index(event_date)] = event.status

        result = []
        for case_id, data_dict in data.items():
            row = [
                self._fmt(data_dict["name"]),
                None,
                None,
                None,
            ]

            total_no_response = 0
            total_indicated = 0
            total_pending = 0

            for date_status in data_dict["dates"]:
                if date_status == CALLBACK_PENDING:
                    total_indicated += 1
                    total_pending += 1
                    row.append(self._fmt(_("pending")))
                elif date_status == CALLBACK_RECEIVED:
                    total_indicated += 1
                    row.append(self._fmt(_("OK")))
                elif date_status == CALLBACK_MISSED:
                    total_indicated += 1
                    total_no_response += 1
                    row.append(self._fmt_highlight(_("No Response")))
                else:
                    row.append(self._fmt(_("not indicated")))

            if total_no_response > 0:
                row[1] = self._fmt_highlight(total_no_response)
            else:
                row[1] = self._fmt(total_no_response)
            row[2] = self._fmt(total_indicated)
            row[3] = self._fmt(total_pending)

            result.append(row)

        return result

    def _fmt(self, value):
        return self.table_cell(value, '
%s
' % value)

    def _fmt_highlight(self, value):
        return self.table_cell(value, '
%s
' % value)

<|endoftext|>


<commit_before>"""
Created on 21 Jun 2018

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

import optparse

# --------------------------------------------------------------------------------------------------------------------

class CmdMPL115A2Conf(object):
    """
    unix command line handler
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self):
        self.__parser = optparse.OptionParser(usage="%prog [{ -a ALTITUDE | -d }] [-v]", version="%prog 1.0")

        # optional...
        self.__parser.add_option("--altitude", "-a", type="string", nargs=1, action="store", dest="altitude",
                                 help="altitude in metres or 'auto' for GPS altitude")

        self.__parser.add_option("--delete", "-d", action="store_true", dest="delete", default=False,
                                 help="delete the MPL115A2 configuration")

        self.__parser.add_option("--verbose", "-v", action="store_true", dest="verbose", default=False,
                                 help="report narrative to stderr")

        self.__opts, self.__args = self.__parser.parse_args()

    # ----------------------------------------------------------------------------------------------------------------

    def is_valid(self):
        if self.set() and self.delete:
            return False

        if self.altitude is None or self.altitude == 'auto':
            return True

        try:
            int(self.altitude)
        except ValueError:
            return False

        return True

    def set(self):
        return self.__opts.altitude is not None

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def altitude(self):
        try:
            return int(self.__opts.altitude)
        except (TypeError, ValueError):
            return self.__opts.altitude

    @property
    def delete(self):
        return self.__opts.delete

    @property
    def verbose(self):
        return self.__opts.verbose

    # ----------------------------------------------------------------------------------------------------------------

    def print_help(self, file):
        self.__parser.print_help(file)

    def __str__(self, *args, **kwargs):
        return "CmdMPL115A2Conf:{altitude:%s, delete:%s, verbose:%s}" % (self.altitude, self.delete, self.verbose)
<commit_msg>enabled null altitudes for mpl115a2_conf.py top-level script.<commit_after>"""
Created on 21 Jun 2018

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

import optparse

# --------------------------------------------------------------------------------------------------------------------

class CmdMPL115A2Conf(object):
    """
    unix command line handler
    """

    # ----------------------------------------------------------------------------------------------------------------

    def __init__(self):
        self.__parser = optparse.OptionParser(usage="%prog [{ -s [-a ALTITUDE] | -d }] [-v]", version="%prog 1.0")

        # optional...
        self.__parser.add_option("--set", "-s", action="store_true", dest="set", default=False,
                                 help="create or update an MPL115A2 configuration")

        self.__parser.add_option("--altitude", "-a", type="string", nargs=1, action="store", dest="altitude",
                                 help="altitude in metres or 'GPS' for GPS altitude")

        self.__parser.add_option("--delete", "-d", action="store_true", dest="delete", default=False,
                                 help="delete the MPL115A2 configuration")

        self.__parser.add_option("--verbose", "-v", action="store_true", dest="verbose", default=False,
                                 help="report narrative to stderr")

        self.__opts, self.__args = self.__parser.parse_args()

    # ----------------------------------------------------------------------------------------------------------------

    def is_valid(self):
        if self.set() and self.delete:
            return False

        if not self.set() and self.altitude is not None:
            return False

        if self.altitude is None or self.altitude == 'GPS':
            return True

        try:
            int(self.altitude)
        except ValueError:
            return False

        return True

    def set(self):
        return self.__opts.set

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def altitude(self):
        try:
            return int(self.__opts.altitude)
        except (TypeError, ValueError):
            return self.__opts.altitude

    @property
    def delete(self):
        return self.__opts.delete

    @property
    def verbose(self):
        return self.__opts.verbose

    # ----------------------------------------------------------------------------------------------------------------

    def print_help(self, file):
        self.__parser.print_help(file)

    def __str__(self, *args, **kwargs):
        return "CmdMPL115A2Conf:{set:%s, altitude:%s, delete:%s, verbose:%s}" % \
               (self.set(), self.altitude, self.delete, self.verbose)
<|endoftext|>