<commit_before>from __future__ import division
import sys, os;  sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..')))

import sessionmanager
from models import db_session, FolderMeta, UIDValidity, MessageMeta, MessagePart
from sqlalchemy import distinct, func
import sqlalchemy.exc

from encoding import EncodingError
from server.util import chunk, partition
import logging as log

def refresh_crispin(email):
    return sessionmanager.get_crispin_from_email(email)

def load_validity_cache(crispin_client, email):
    # in practice UIDVALIDITY and HIGHESTMODSEQ are always positive
    # integers with gmail, but let's not take chances on our default.
    defaults = dict(UIDVALIDITY=float('-inf'), HIGHESTMODSEQ=float('-inf'))
    # populated cache looks like:
    # {'Inbox': {'UIDVALIDITY': 123, 'HIGHESTMODSEQ': 456}}
    cache_validity = dict([(folder, defaults.copy())
        for folder in crispin_client.sync_folders])
    for folder, uid_validity, highestmodseq in db_session.query(
            UIDValidity.folder_name,
            UIDValidity.uid_validity,
            UIDValidity.highestmodseq).filter_by(
                    g_email=email, folder_name=folder):
        cache_validity[folder] = dict(UIDVALIDITY=uid_validity,
                HIGHESTMODSEQ=highestmodseq)

    return cache_validity

def check_uidvalidity(crispin_client):
    if not uidvalidity_valid(crispin_client):
        log.info("UIDVALIDITY for {0} has changed; resyncing UIDs".format(
            crispin_client.selected_folder_name))
        resync_uids(crispin_client)

def uidvalidity_valid(crispin_client):
    """ Validate UIDVALIDITY on currently selected folder. """
    try:
        cached_validity = db_session.query(UIDValidity.uid_validity).filter_by(
                g_email=crispin_client.email_address,
                folder_name=crispin_client.selected_folder_name).one()[0]
    except sqlalchemy.orm.exc.NoResultFound:
        # No entry? No problem!
        return True
    assert type(cached_validity) == type(crispin_client.selected_uidvalidity)
    return crispin_client.selected_uidvalidity >= cached_validity

def resync_uids(crispin_client):
    """ Call this when UIDVALIDITY is invalid to fix up the database.

    What happens here is we fetch new UIDs from the IMAP server and match
    them with X-GM-MSGIDs and sub in the new UIDs for the old. No messages
    are re-downloaded.
    """
    raise Exception("Unimplemented")

def delete_messages(uids, folder):
    # delete these UIDs from this folder
    fm_query = db_session.query(FolderMeta).filter(
            FolderMeta.msg_uid.in_(uids),
            FolderMeta.folder_name==folder)
    # g_msgids = [fm.g_msgid for fm in fm_query]
    fm_query.delete(synchronize_session='fetch')

    # XXX TODO not sure if there's a good non-expensive way to find
    # dangling messages; we may want to have a different worker deal
    # with this
    # if the g_msgid is now dangling, delete the message meta and parts as
    # well
    # dangling_g_msgids = [g_msgid for g_msgid, count in
    # db_session.query(FolderMeta.g_msgid,
    #     func.count(FolderMeta)).group_by(FolderMeta.g_msgid)\
    #         .filter(FolderMeta.g_msgid.in_(g_msgids)) if count == 0]
    # #
    # db_session.query(MessageMeta).filter(
    #         MessageMeta.g_msgids.in_(dangling_g_msgids)).delete()
    # db_session.query(MessagePart).filter(
    #         MessagePart.g_msgids.in_(dangling_g_msgids)).delete()
    # XXX also delete message parts from the block store
    db_session.commit()

def remove_deleted_messages(crispin_client):
    """ Works as follows:
        1. do a LIST on the current folder to see what messages are on the server
        2. compare to message uids stored locally
        3. purge messages we have locally but not on the server. ignore
            messages we have on the server that aren't local.
    """
    server_uids = crispin_client.all_uids()
    local_uids = [uid for uid, in
            db_session.query(FolderMeta.msg_uid).filter_by(
                folder_name=crispin_client.selected_folder_name)]
    if len(server_uids) > 0 and len(local_uids) > 0:
        assert type(server_uids[0]) == type(local_uids[0])

    to_delete = set(local_uids).difference(set(server_uids))
    if to_delete:
        delete_messages(to_delete, crispin_client.selected_folder_name)

def new_or_updated(uids, folder):
    local_uids = set([unicode(uid) for uid, in \
            db_session.query(FolderMeta.msg_uid).filter(
            FolderMeta.folder_name==folder,
            FolderMeta.msg_uid.in_(uids))])
    return partition(lambda x: x not in local_uids, uids)

def incremental_sync(user_email_address):
    """ Poll this every N seconds for active (logged-in) users and every
        N minutes for logged-out users. It checks for changed message metadata
        and new messages using CONDSTORE / HIGHESTMODSEQ and also checks for
        deleted messages.

        We may also wish to frob update frequencies based on which folder
        a user has visible in the UI as well.
    """
    crispin_client = refresh_crispin(user_email_address)
    cache_validity = load_validity_cache(crispin_client, user_email_address)
    needs_update = []
    for folder in crispin_client.sync_folders:
        # eventually we might want to be holding a cache of this stuff from any
        # SELECT calls that have already happened, to save on a status call.
        # but status is fast, so maybe not.
        status = crispin_client.imap_server.folder_status(folder,
                ('UIDVALIDITY', 'HIGHESTMODSEQ'))
        cached_highestmodseq = cache_validity[folder]['HIGHESTMODSEQ']
        if status['HIGHESTMODSEQ'] > cached_highestmodseq:
            needs_update.append((folder, cached_highestmodseq))

    for folder, highestmodseq in needs_update:
        highestmodseq_update(folder, crispin_client)

    return 0

def update_cached_highestmodseq(folder, crispin_client, cached_validity=None):
    if cached_validity is None:
        cached_validity = db_session.query(UIDValidity).filter_by(
                g_email=crispin_client.email_address, folder_name=folder).one()
    cached_validity.highestmodseq = crispin_client.selected_highestmodseq
    db_session.add(cached_validity)

def highestmodseq_update(folder, crispin_client, cached_validity=None):
    crispin_client.select_folder(folder)
    check_uidvalidity(crispin_client)
    uids = crispin_client.imap_server.search(
            ['NOT DELETED', 'MODSEQ {0}'.format(
                crispin_client.selected_highestmodseq)])
    log.info("Starting highestmodseq update on {0} (current HIGHESTMODSEQ: {1})".format(folder, crispin_client.selected_highestmodseq))
    if uids:
        new, updated = new_or_updated(uids, folder)
        for uids in chunk(new, crispin_client.CHUNK_SIZE):
            new_messagemeta, new_messagepart, new_foldermeta = safe_download(new,
                    folder, crispin_client)
            db_session.add_all(new_foldermeta)
            db_session.add_all(new_messagemeta)
            db_session.add_all(new_messagepart)
            safe_commit()
        # bigger chunk because the data being fetched here is very small
        for uids in chunk(updated, 5*crispin_client.CHUNK_SIZE):
            update_metadata(updated, crispin_client)
            safe_commit()
    remove_deleted_messages(crispin_client)
    # not sure if this one is actually needed - does delete() automatically
    # commit?
    safe_commit()

    update_cached_highestmodseq(folder, crispin_client, cached_validity)
    db_session.commit()

def safe_download(uids, folder, crispin_client):
    try:
        new_messagemeta, new_messagepart, new_foldermeta = \
                crispin_client.fetch_uids(uids)
    except EncodingError, e:
        raise
    # XXX make this catch more specific
    except Exception, e:
        log.error("Crispin fetch failure: %s. Reconnecting..." % e)
        crispin_client = refresh_crispin(crispin_client.email_address)
        new_messagemeta, new_messagepart, new_foldermeta = \
                crispin_client.fetch_uids(uids)

    return new_messagemeta, new_messagepart, new_foldermeta

def safe_commit():
    try:
        db_session.commit()
    except sqlalchemy.exc.SQLAlchemyError, e:
        log.error(e.orig.args)
    except Exception, e:
        log.error("Unknown exception: %s" % e)

def update_metadata(uids, crispin_client):
    """ Update flags (the only metadata that can change). """
    new_metadata = crispin_client.fetch_metadata(uids)
    for fm in db_session.query(FolderMeta).filter(
            FolderMeta.msg_uid.in_(uids),
            FolderMeta.folder_name==crispin_client.selected_folder_name):
        if fm.flags != new_metadata[fm.uid]:
            fm.flags = new_metadata[fm.uid]
            db_session.add(fm)

def initial_sync(user_email_address):
    """ Downloads entire messages and
    (1) creates the metadata database
    (2) stores message parts to the block store
    """
    crispin_client = refresh_crispin(user_email_address)

    log.info('Syncing mail for {0}'.format(user_email_address))

    # message download for messages from sync_folders is prioritized before
    # AllMail in the order of appearance in this list

    for folder in crispin_client.sync_folders:
        # for each folder, compare what's on the server to what we have.
        # this allows restarts of the initial sync script in the case of
        # total failure.
        crispin_client.select_folder(folder)
        check_uidvalidity(crispin_client)
        server_uids = crispin_client.all_uids()
        server_g_msgids = crispin_client.fetch_g_msgids(server_uids)
        g_msgids = set([g_msgid for g_msgid, in
            db_session.query(distinct(FolderMeta.g_msgid))])

        log.info("Found {0} UIDs for folder {1}".format(
            len(server_uids), folder))
        existing_uids = [uid for uid, in
                db_session.query(FolderMeta.msg_uid).filter_by(
                    g_email=user_email_address, folder_name=folder)]
        log.info("Already have {0} items".format(len(existing_uids)))
        warn_uids = set(existing_uids).difference(set(server_uids))
        unknown_uids = set(server_uids).difference(set(existing_uids))

        if warn_uids:
            delete_messages(warn_uids, folder)
            log.info("Deleted the following UIDs that no longer exist on the server: {0}".format(' '.join(sorted(warn_uids, key=int))))

        full_download, foldermeta_only = partition(
                lambda uid: server_g_msgids[uid] in g_msgids,
                sorted(unknown_uids, key=int))

        log.info("{0} uids left to fetch".format(len(full_download)))

        log.info("skipping {0} uids downloaded via other folders".format(
            len(foldermeta_only)))
        if len(foldermeta_only) > 0:
            db_session.add_all(
                    [crispin_client.make_fm(server_g_msgids[uid], folder,
                        uid) for uid in foldermeta_only])
            db_session.commit()

        total_messages = len(existing_uids)

        log.info("Starting sync for {0} with chunks of size {1}".format(
            folder, crispin_client.CHUNK_SIZE))
        for uids in chunk(full_download, crispin_client.CHUNK_SIZE):
            new_messagemeta, new_messagepart, new_foldermeta = \
                    safe_download(uids, folder, crispin_client)
            db_session.add_all(new_foldermeta)
            db_session.add_all(new_messagemeta)
            db_session.add_all(new_messagepart)

            safe_commit()

            total_messages += len(uids)

            log.info("Synced %i of %i (%.4f%%)" % (total_messages,
                                                   len(server_uids),
                                                    total_messages / len(server_uids) * 100))

        # transaction commit
        cached_validity = db_session.query(UIDValidity).filter_by(
                g_email=user_email_address, folder_name=folder).one()
        if not cached_validity:
            db_session.add(UIDValidity(
                g_email=user_email_address, folder_name=folder,
                uid_validity=crispin_client.selected_uidvalidity,
                highestmodseq=crispin_client.selected_highestmodseq))
            db_session.commit()
        elif cached_validity.highestmodseq < crispin_client.selected_highestmodseq:
            # if we've done a restart on the initial sync, we may have already
            # saved a UIDValidity row for any given folder, so we need to
            # update it instead. BUT, we first need to check for updated
            # metadata since the recorded HIGHESTMODSEQ. (Yes, some messages
            # here may have already been downloaded in the UID query above;
            # the modseq update will properly skip those messages that already
            # exist locally.)
            highestmodseq_update(folder, crispin_client, cached_validity)
        else:
            # nothing to do here
            pass

        log.info("Saved all messages and metadata on {0} to UIDVALIDITY {1} / HIGHESTMODSEQ {2}".format(folder, crispin_client.selected_uidvalidity,
            crispin_client.selected_highestmodseq))

    log.info("Finished.")

    crispin_client.user_obj.initial_sync_done = True
    db_session.add(crispin_client.user_obj)
    db_session.commit()

    return 0
<commit_msg>And again.<commit_after>from __future__ import division
import sys, os;  sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..')))

import sessionmanager
from models import db_session, FolderMeta, UIDValidity, MessageMeta, MessagePart
from sqlalchemy import distinct, func
import sqlalchemy.exc

from encoding import EncodingError
from server.util import chunk, partition
import logging as log

def refresh_crispin(email):
    return sessionmanager.get_crispin_from_email(email)

def load_validity_cache(crispin_client, email):
    # in practice UIDVALIDITY and HIGHESTMODSEQ are always positive
    # integers with gmail, but let's not take chances on our default.
    defaults = dict(UIDVALIDITY=float('-inf'), HIGHESTMODSEQ=float('-inf'))
    # populated cache looks like:
    # {'Inbox': {'UIDVALIDITY': 123, 'HIGHESTMODSEQ': 456}}
    cache_validity = dict([(folder, defaults.copy())
        for folder in crispin_client.sync_folders])
    for folder, uid_validity, highestmodseq in db_session.query(
            UIDValidity.folder_name,
            UIDValidity.uid_validity,
            UIDValidity.highestmodseq).filter_by(
                    g_email=email, folder_name=folder):
        cache_validity[folder] = dict(UIDVALIDITY=uid_validity,
                HIGHESTMODSEQ=highestmodseq)

    return cache_validity

def check_uidvalidity(crispin_client):
    if not uidvalidity_valid(crispin_client):
        log.info("UIDVALIDITY for {0} has changed; resyncing UIDs".format(
            crispin_client.selected_folder_name))
        resync_uids(crispin_client)

def uidvalidity_valid(crispin_client):
    """ Validate UIDVALIDITY on currently selected folder. """
    try:
        cached_validity = db_session.query(UIDValidity.uid_validity).filter_by(
                g_email=crispin_client.email_address,
                folder_name=crispin_client.selected_folder_name).one()[0]
    except sqlalchemy.orm.exc.NoResultFound:
        # No entry? No problem!
        return True
    assert type(cached_validity) == type(crispin_client.selected_uidvalidity)
    return crispin_client.selected_uidvalidity >= cached_validity

def resync_uids(crispin_client):
    """ Call this when UIDVALIDITY is invalid to fix up the database.

    What happens here is we fetch new UIDs from the IMAP server and match
    them with X-GM-MSGIDs and sub in the new UIDs for the old. No messages
    are re-downloaded.
    """
    raise Exception("Unimplemented")

def delete_messages(uids, folder):
    # delete these UIDs from this folder
    fm_query = db_session.query(FolderMeta).filter(
            FolderMeta.msg_uid.in_(uids),
            FolderMeta.folder_name==folder)
    # g_msgids = [fm.g_msgid for fm in fm_query]
    fm_query.delete(synchronize_session='fetch')

    # XXX TODO not sure if there's a good non-expensive way to find
    # dangling messages; we may want to have a different worker deal
    # with this
    # if the g_msgid is now dangling, delete the message meta and parts as
    # well
    # dangling_g_msgids = [g_msgid for g_msgid, count in
    # db_session.query(FolderMeta.g_msgid,
    #     func.count(FolderMeta)).group_by(FolderMeta.g_msgid)\
    #         .filter(FolderMeta.g_msgid.in_(g_msgids)) if count == 0]
    # #
    # db_session.query(MessageMeta).filter(
    #         MessageMeta.g_msgids.in_(dangling_g_msgids)).delete()
    # db_session.query(MessagePart).filter(
    #         MessagePart.g_msgids.in_(dangling_g_msgids)).delete()
    # XXX also delete message parts from the block store
    db_session.commit()

def remove_deleted_messages(crispin_client):
    """ Works as follows:
        1. do a LIST on the current folder to see what messages are on the server
        2. compare to message uids stored locally
        3. purge messages we have locally but not on the server. ignore
            messages we have on the server that aren't local.
    """
    server_uids = crispin_client.all_uids()
    local_uids = [uid for uid, in
            db_session.query(FolderMeta.msg_uid).filter_by(
                folder_name=crispin_client.selected_folder_name)]
    if len(server_uids) > 0 and len(local_uids) > 0:
        assert type(server_uids[0]) == type(local_uids[0])

    to_delete = set(local_uids).difference(set(server_uids))
    if to_delete:
        delete_messages(to_delete, crispin_client.selected_folder_name)

def new_or_updated(uids, folder):
    local_uids = set([unicode(uid) for uid, in \
            db_session.query(FolderMeta.msg_uid).filter(
            FolderMeta.folder_name==folder,
            FolderMeta.msg_uid.in_(uids))])
    return partition(lambda x: x not in local_uids, uids)

def incremental_sync(user_email_address):
    """ Poll this every N seconds for active (logged-in) users and every
        N minutes for logged-out users. It checks for changed message metadata
        and new messages using CONDSTORE / HIGHESTMODSEQ and also checks for
        deleted messages.

        We may also wish to frob update frequencies based on which folder
        a user has visible in the UI as well.
    """
    crispin_client = refresh_crispin(user_email_address)
    cache_validity = load_validity_cache(crispin_client, user_email_address)
    needs_update = []
    for folder in crispin_client.sync_folders:
        # eventually we might want to be holding a cache of this stuff from any
        # SELECT calls that have already happened, to save on a status call.
        # but status is fast, so maybe not.
        status = crispin_client.imap_server.folder_status(folder,
                ('UIDVALIDITY', 'HIGHESTMODSEQ'))
        cached_highestmodseq = cache_validity[folder]['HIGHESTMODSEQ']
        if status['HIGHESTMODSEQ'] > cached_highestmodseq:
            needs_update.append((folder, cached_highestmodseq))

    for folder, highestmodseq in needs_update:
        highestmodseq_update(folder, crispin_client)

    return 0

def update_cached_highestmodseq(folder, crispin_client, cached_validity=None):
    if cached_validity is None:
        cached_validity = db_session.query(UIDValidity).filter_by(
                g_email=crispin_client.email_address, folder_name=folder).one()
    cached_validity.highestmodseq = crispin_client.selected_highestmodseq
    db_session.add(cached_validity)

def highestmodseq_update(folder, crispin_client, cached_validity=None):
    crispin_client.select_folder(folder)
    check_uidvalidity(crispin_client)
    uids = crispin_client.imap_server.search(
            ['NOT DELETED', 'MODSEQ {0}'.format(
                crispin_client.selected_highestmodseq)])
    log.info("Starting highestmodseq update on {0} (current HIGHESTMODSEQ: {1})".format(folder, crispin_client.selected_highestmodseq))
    if uids:
        new, updated = new_or_updated(uids, folder)
        for uids in chunk(new, crispin_client.CHUNK_SIZE):
            new_messagemeta, new_messagepart, new_foldermeta = safe_download(new,
                    folder, crispin_client)
            db_session.add_all(new_foldermeta)
            db_session.add_all(new_messagemeta)
            db_session.add_all(new_messagepart)
            safe_commit()
        # bigger chunk because the data being fetched here is very small
        for uids in chunk(updated, 5*crispin_client.CHUNK_SIZE):
            update_metadata(updated, crispin_client)
            safe_commit()
    remove_deleted_messages(crispin_client)
    # not sure if this one is actually needed - does delete() automatically
    # commit?
    safe_commit()

    update_cached_highestmodseq(folder, crispin_client, cached_validity)
    db_session.commit()

def safe_download(uids, folder, crispin_client):
    try:
        new_messagemeta, new_messagepart, new_foldermeta = \
                crispin_client.fetch_uids(uids)
    except EncodingError, e:
        raise
    # XXX make this catch more specific
    except Exception, e:
        log.error("Crispin fetch failure: %s. Reconnecting..." % e)
        crispin_client = refresh_crispin(crispin_client.email_address)
        new_messagemeta, new_messagepart, new_foldermeta = \
                crispin_client.fetch_uids(uids)

    return new_messagemeta, new_messagepart, new_foldermeta

def safe_commit():
    try:
        db_session.commit()
    except sqlalchemy.exc.SQLAlchemyError, e:
        log.error(e.orig.args)
    except Exception, e:
        log.error("Unknown exception: %s" % e)

def update_metadata(uids, crispin_client):
    """ Update flags (the only metadata that can change). """
    new_metadata = crispin_client.fetch_metadata(uids)
    for fm in db_session.query(FolderMeta).filter(
            FolderMeta.msg_uid.in_(uids),
            FolderMeta.folder_name==crispin_client.selected_folder_name):
        if fm.flags != new_metadata[fm.uid]:
            fm.flags = new_metadata[fm.uid]
            db_session.add(fm)

def initial_sync(user_email_address):
    """ Downloads entire messages and
    (1) creates the metadata database
    (2) stores message parts to the block store
    """
    crispin_client = refresh_crispin(user_email_address)

    log.info('Syncing mail for {0}'.format(user_email_address))

    # message download for messages from sync_folders is prioritized before
    # AllMail in the order of appearance in this list

    for folder in crispin_client.sync_folders:
        # for each folder, compare what's on the server to what we have.
        # this allows restarts of the initial sync script in the case of
        # total failure.
        crispin_client.select_folder(folder)
        check_uidvalidity(crispin_client)
        server_uids = crispin_client.all_uids()
        server_g_msgids = crispin_client.fetch_g_msgids(server_uids)
        g_msgids = set([g_msgid for g_msgid, in
            db_session.query(distinct(FolderMeta.g_msgid))])

        log.info("Found {0} UIDs for folder {1}".format(
            len(server_uids), folder))
        existing_uids = [uid for uid, in
                db_session.query(FolderMeta.msg_uid).filter_by(
                    g_email=user_email_address, folder_name=folder)]
        log.info("Already have {0} items".format(len(existing_uids)))
        warn_uids = set(existing_uids).difference(set(server_uids))
        unknown_uids = set(server_uids).difference(set(existing_uids))

        if warn_uids:
            delete_messages(warn_uids, folder)
            log.info("Deleted the following UIDs that no longer exist on the server: {0}".format(' '.join(sorted(warn_uids, key=int))))

        full_download, foldermeta_only = partition(
                lambda uid: server_g_msgids[uid] in g_msgids,
                sorted(unknown_uids, key=int))

        log.info("{0} uids left to fetch".format(len(full_download)))

        log.info("skipping {0} uids downloaded via other folders".format(
            len(foldermeta_only)))
        if len(foldermeta_only) > 0:
            db_session.add_all(
                    [crispin_client.make_fm(server_g_msgids[uid], folder,
                        uid) for uid in foldermeta_only])
            db_session.commit()

        total_messages = len(existing_uids)

        log.info("Starting sync for {0} with chunks of size {1}".format(
            folder, crispin_client.CHUNK_SIZE))
        for uids in chunk(full_download, crispin_client.CHUNK_SIZE):
            new_messagemeta, new_messagepart, new_foldermeta = \
                    safe_download(uids, folder, crispin_client)
            db_session.add_all(new_foldermeta)
            db_session.add_all(new_messagemeta)
            db_session.add_all(new_messagepart)

            safe_commit()

            total_messages += len(uids)

            log.info("Synced %i of %i (%.4f%%)" % (total_messages,
                                                   len(server_uids),
                                                    total_messages / len(server_uids) * 100))

        # transaction commit
        try:
            cached_validity = db_session.query(UIDValidity).filter_by(
                    g_email=user_email_address, folder_name=folder).one()
            if cached_validity.highestmodseq < crispin_client.selected_highestmodseq:
                # if we've done a restart on the initial sync, we may have already
                # saved a UIDValidity row for any given folder, so we need to
                # update it instead. BUT, we first need to check for updated
                # metadata since the recorded HIGHESTMODSEQ. (Yes, some messages
                # here may have already been downloaded in the UID query above;
                # the modseq update will properly skip those messages that already
                # exist locally.)
                highestmodseq_update(folder, crispin_client, cached_validity)
            else:
                # nothing to do here
                pass
        except sqlalchemy.orm.exc.NoResultFound:
            db_session.add(UIDValidity(
                g_email=user_email_address, folder_name=folder,
                uid_validity=crispin_client.selected_uidvalidity,
                highestmodseq=crispin_client.selected_highestmodseq))
            db_session.commit()
        log.info("Saved all messages and metadata on {0} to UIDVALIDITY {1} / HIGHESTMODSEQ {2}".format(folder, crispin_client.selected_uidvalidity,
            crispin_client.selected_highestmodseq))

    log.info("Finished.")

    crispin_client.user_obj.initial_sync_done = True
    db_session.add(crispin_client.user_obj)
    db_session.commit()

    return 0
<|endoftext|>


<commit_before>import numpy as np
from cvxopt import matrix, solvers
import cdd
import shapely.geometry as geom

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

def cross_m(vec):
  return np.array([[0, -vec.item(2), vec.item(1)],
                   [vec.item(2), 0, -vec.item(0)],
                   [-vec.item(1), vec.item(0), 0]])

def normalize(vec):
  return vec/(np.linalg.norm(vec))

def area_convex(hrep):
  gen = np.array(cdd.Polyhedron(hrep).get_generators())
  #If the polygon is empty or degenerate, return 0
  if gen.shape[0] < 3:
    return 0
  p = np.vstack([gen, gen[0, :]])
  x, y = p[:, 1], p[:, 2]
  poly = geom.Polygon(zip(x, y))
  return poly.convex_hull.area

def convex_hull(x_p, y_p):
  poly = geom.Polygon(zip(x_p, y_p))
  x_c, y_c = poly.convex_hull.exterior.xy
  return x_c, y_c

#A contact should contain :
# - r : position world
# - n : normal to teh surface
# - mu : friction coefficient

class Contact():
  def __init__(self, mu, pos, normal):
    self.n = normal
    self.r = pos
    self.mu = mu

  def check_size(self):
    assert self.n.shape == (3, 1),\
        "Normal is {} shape instead of (3,1))".format(self.n.shape)
    assert self.r.shape == (3, 1),\
        "Position is {} shape instead of (3,1))".format(self.r.shape)

  def __str__(self):
    lines = ["Contact at :", str(self.r.T), "With normal :", str(self.n.T)]
    return "\n".join(lines)

class SteppingException(Exception):
  def __init__(self, m):
    self.message = m

  def __str__(self):
    return self.message

#Algorithm to compute stability polygon according to
#Bretl et al. "Testing static equilibrium of legged robots"
# You need to first set some contacts and a robot mass
# Then call compute with desired precision.

class StabilityPolygon():
  def __init__(self, robotMass, gravity=-9.81):
    solvers.options['show_progress'] = False
    self.contacts = []
    self.mass = robotMass
    self.gravity = np.array([[0, 0, gravity]]).T
    self.proj = np.array([[1, 0, 0], [0, 1, 0]])
    self.inner = []
    self.outer = []

  def nrVars(self):
    return self.size_x() + self.size_z()

  def size_x(self):
    return 3*len(self.contacts)

  def size_z(self):
    return 2

  def addContact(self, contact):
    self.contacts.append(contact)

  def reset(self):
    self.contacts = []

  def make_problem(self):
    A_s = []
    B_s = []
    u_s = []

    for c in self.contacts:
      A_s.append(np.vstack([np.eye(3), cross_m(c.r)]))
      B_s.append(np.eye(3) - np.dot(c.n, c.n.T))
      u_s.append(c.mu*c.n)

    self.A1 = np.hstack(A_s)
    self.A2 = np.vstack([np.zeros((3, 2)),
                         -cross_m(self.mass*self.gravity).dot(self.proj.T)])
    self.t = np.vstack([-self.mass*self.gravity, np.array([[0], [0], [0]])])

    self.B_s = B_s
    self.u_s = u_s

  def check_sizes(self):
    assert(self.A1.shape[1] + self.A2.shape[1] == self.nrVars())

  def solve(self, a):
    self.sol = self.socp(a, self.A1, self.A2, self.t, self.B_s, self.u_s)
    if self.sol['status'] == 'optimal':
      vec = np.array(self.sol['x'])
      self.com = vec[-2:]
      self.forces = vec[:-2].reshape((3, len(self.contacts))).T
      return True
    return False

  def socp(self, a, A1, A2, t, B_s, u_s):
    #Max a^T z ~ min -a^T z
    c = matrix(np.vstack([np.zeros((self.size_x(), 1)), -a]))

    A = matrix(np.hstack([A1, A2]))

    #For G : compute all cones
    G = []
    H = []
    for i, (b, u) in enumerate(zip(B_s, u_s)):
      block = np.hstack([-np.vstack([u.T, b]),
                        np.zeros((4, 2))])
      g = np.hstack([np.zeros((4, 3*i)),
                     block,
                     np.zeros((4, 3*(len(B_s)-1-i)))])
      G.append(g)
      H.append(np.zeros((4, 1)))

    sol = solvers.socp(c, Gq=map(matrix, G), hq=map(matrix, H),
                       A=A, b=matrix(t))
    return sol

  def init_algo(self):
    self.make_problem()
    self.directions = []
    self.points = []
    self.offsets = []
    self.inner = None
    self.outer = None
    #Search in three "random" directions
    directions = map(normalize, [np.array([[0, 1]]).T,
                                 np.array([[1, 0]]).T,
                                 np.array([[-1, -1]]).T])
    rdirs = []

    for d in directions:
      try:
        self.step(d)
      except SteppingException as e:
        rdir = np.random.random((2, 1))
        print str(e), " Will try in a random direction {}".format(rdir)
        rdirs.append(rdir)

    for d in rdirs:
      self.step(d)

  def step(self, d):
    if self.solve(d):
      self.directions.append(d.T)
      self.points.append(self.com)
      self.offsets.append(d.T.dot(self.com))
    else:
      m = ["Failed to step in direction {}".format(d.T),
           "Terminated in {} state".format(self.sol['status'])]
      raise SteppingException('\n'.join(m))

  def build_polys(self):
    if self.outer is None:
      A = np.vstack(self.directions)
      b = np.vstack(self.offsets)
      self.outer = cdd.Matrix(np.hstack([b, -A]))
      self.outer.rep_type = cdd.RepType.INEQUALITY
    else:
      self.outer.extend(np.hstack((self.offsets[-1], -self.directions[-1])))
      self.outer.canonicalize()

    if self.inner is None:
      A = np.vstack([p.T for p in self.points])
      b = np.ones((len(self.points), 1))
      self.inner = cdd.Matrix(np.hstack((b, A)))
      self.inner.rep_type = cdd.RepType.GENERATOR
    else:
      self.inner.extend(np.hstack(([[1]], self.points[-1].T)))
      self.inner.canonicalize()

  def find_direction(self):
    self.build_polys()
    out_area = area_convex(self.outer)

    areas = []
    ineq = np.array(cdd.Polyhedron(self.inner).get_inequalities())

    for line in ineq:
      A_e = self.outer.copy()
      A_e.extend(cdd.Matrix(line.reshape(1, line.size)))
      A_e.canonicalize()
      areas.append(out_area - area_convex(A_e))

    i, a = max(enumerate(areas), key=lambda x: x[1])
    return -ineq[i, 1:]

  def next_edge(self, plot=False, record=False, number=0):
    d = normalize(self.find_direction().reshape((2, 1)))
    self.step(d)

    if plot or record:
      self.plot()
      if plot:
        self.show()
      if record:
        filename = "stability_{0:04d}".format(number)
        plt.savefig(filename)
        plt.close()

  def iterBound(self, nr_edges_init, error_init, prec):
    c = float(343)/float(243)
    return nr_edges_init*(np.sqrt(c*error_init/prec) - 1)

  def compute(self, epsilon=1e-4, plot_init=False,
              plot_step=False,
              record_anim=False,
              plot_final=True):
    self.make_problem()
    self.init_algo()
    self.build_polys()

    if plot_init:
      self.plot()
      self.show()

    error = area_convex(self.outer) - area_convex(self.inner)

    iterBound = self.iterBound(3, error, epsilon)

    print "This should take {} iterations".format(np.ceil(iterBound))

    nrSteps = 0
    while(error > epsilon):
      try:
        self.next_edge(plot_step, record_anim, nrSteps)
      except SteppingException as e:
        print "Failure detected... Aborting"
        print e.message
        break
      error = area_convex(self.outer) - area_convex(self.inner)
      nrSteps += 1

    print "NrIter : {} | Remainder : {}".format(nrSteps, error)

    if plot_final:
      self.plot()
      self.show()

  def polygon(self, centroid=None):
    gen = np.array(cdd.Polyhedron(self.inner).get_generators())
    p = np.vstack([gen, gen[0, :]])
    if centroid is None:
      x, y = p[:, 1], p[:, 2]
    else:
      x, y = p[:, 1] + centroid.item(0), p[:, 2] + centroid.item(1)
    return geom.Polygon(zip(x, y)).convex_hull

  def reset_fig(self):
    fig = plt.figure()
    self.ax = fig.add_subplot('111', aspect='equal', projection='3d')
    self.ax.set_xlim([-0.5, 1])
    self.ax.set_ylim([-0.5, 1])
    self.ax.set_zlim([-0.5, 1])

  def plot(self):
    self.reset_fig()
    self.plot_contacts()
    self.plot_solution()
    self.plot_polygons()

  def show(self):
    plt.show()

  def plot_contacts(self):
    X, Y, Z, U, V, W = [], [], [], [], [], []
    positions = np.hstack([c.r for c in self.contacts])
    normals = np.hstack([c.n for c in self.contacts])
    X, Y, Z = positions[0, :], positions[1, :], positions[2, :]
    U, V, W = normals[0, :], normals[1, :], normals[2, :]
    self.ax.quiver(X, Y, Z, U, V, W, color='black', linestyles='dashed')

  def plot_solution(self):
    com_pos = self.com
    forces = self.forces

    x, y = com_pos.item(0), com_pos.item(1)
    self.ax.plot([x], [y], linestyle="none",
                 marker='o', alpha=0.6,
                 markersize=10, markerfacecolor='black')

    positions = np.hstack([c.r for c in self.contacts])

    for pos, force in zip(positions.T, forces.T):
      X, Y, Z = pos[0], pos[1], pos[2]
      U, V, W = -force[0], -force[1], -force[2]
      l = np.linalg.norm(force)/(self.mass*abs(self.gravity.item(2)))
      self.ax.quiver(X, Y, Z, U, V, W, color='r', length=l)

  def plot_direction(self, d):
    self.ax.plot([0, d.item(0)], [0, d.item(1)], marker='d')

  def plot_polygons(self):
    self.plot_polygon(self.inner, 'x')
    self.plot_polygon(self.outer, 'o')

  def plot_polygon(self, poly, m):
    gen = np.array(cdd.Polyhedron(poly).get_generators())
    p = np.vstack([gen, gen[0, :]])
    x, y = p[:, 1], p[:, 2]
    poly = geom.Polygon(zip(x, y))
    x, y = poly.convex_hull.exterior.xy
    self.ax.plot(x, y, marker=m)
<commit_msg>Corrected a display bug : forces were incorrectly projected<commit_after>import numpy as np
from cvxopt import matrix, solvers
import cdd
import shapely.geometry as geom

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

def cross_m(vec):
  return np.array([[0, -vec.item(2), vec.item(1)],
                   [vec.item(2), 0, -vec.item(0)],
                   [-vec.item(1), vec.item(0), 0]])

def normalize(vec):
  return vec/(np.linalg.norm(vec))

def area_convex(hrep):
  gen = np.array(cdd.Polyhedron(hrep).get_generators())
  #If the polygon is empty or degenerate, return 0
  if gen.shape[0] < 3:
    return 0
  p = np.vstack([gen, gen[0, :]])
  x, y = p[:, 1], p[:, 2]
  poly = geom.Polygon(zip(x, y))
  return poly.convex_hull.area

def convex_hull(x_p, y_p):
  poly = geom.Polygon(zip(x_p, y_p))
  x_c, y_c = poly.convex_hull.exterior.xy
  return x_c, y_c

#A contact should contain :
# - r : position world
# - n : normal to teh surface
# - mu : friction coefficient

class Contact():
  def __init__(self, mu, pos, normal):
    self.n = normal
    self.r = pos
    self.mu = mu

  def check_size(self):
    assert self.n.shape == (3, 1),\
        "Normal is {} shape instead of (3,1))".format(self.n.shape)
    assert self.r.shape == (3, 1),\
        "Position is {} shape instead of (3,1))".format(self.r.shape)

  def __str__(self):
    lines = ["Contact at :", str(self.r.T), "With normal :", str(self.n.T)]
    return "\n".join(lines)

class SteppingException(Exception):
  def __init__(self, m):
    self.message = m

  def __str__(self):
    return self.message

#Algorithm to compute stability polygon according to
#Bretl et al. "Testing static equilibrium of legged robots"
# You need to first set some contacts and a robot mass
# Then call compute with desired precision.

class StabilityPolygon():
  def __init__(self, robotMass, gravity=-9.81):
    solvers.options['show_progress'] = False
    self.contacts = []
    self.mass = robotMass
    self.gravity = np.array([[0, 0, gravity]]).T
    self.proj = np.array([[1, 0, 0], [0, 1, 0]])
    self.inner = []
    self.outer = []

  def nrVars(self):
    return self.size_x() + self.size_z()

  def size_x(self):
    return 3*len(self.contacts)

  def size_z(self):
    return 2

  def addContact(self, contact):
    self.contacts.append(contact)

  def reset(self):
    self.contacts = []

  def make_problem(self):
    A_s = []
    B_s = []
    u_s = []

    for c in self.contacts:
      A_s.append(np.vstack([np.eye(3), cross_m(c.r)]))
      B_s.append(np.eye(3) - np.dot(c.n, c.n.T))
      u_s.append(c.mu*c.n)

    self.A1 = np.hstack(A_s)
    self.A2 = np.vstack([np.zeros((3, 2)),
                         -cross_m(self.mass*self.gravity).dot(self.proj.T)])
    self.t = np.vstack([-self.mass*self.gravity, np.array([[0], [0], [0]])])

    self.B_s = B_s
    self.u_s = u_s

  def check_sizes(self):
    assert(self.A1.shape[1] + self.A2.shape[1] == self.nrVars())

  def solve(self, a):
    self.sol = self.socp(a, self.A1, self.A2, self.t, self.B_s, self.u_s)
    if self.sol['status'] == 'optimal':
      vec = np.array(self.sol['x'])
      self.com = vec[-2:]
      self.forces = vec[:-2].reshape((len(self.contacts), 3)).T
      return True
    return False

  def socp(self, a, A1, A2, t, B_s, u_s):
    #Max a^T z ~ min -a^T z
    c = matrix(np.vstack([np.zeros((self.size_x(), 1)), -a]))

    A = matrix(np.hstack([A1, A2]))

    #For G : compute all cones
    G = []
    H = []
    for i, (b, u) in enumerate(zip(B_s, u_s)):
      block = np.hstack([-np.vstack([u.T, b]),
                        np.zeros((4, 2))])
      g = np.hstack([np.zeros((4, 3*i)),
                     block,
                     np.zeros((4, 3*(len(B_s)-1-i)))])
      G.append(g)
      H.append(np.zeros((4, 1)))

    sol = solvers.socp(c, Gq=map(matrix, G), hq=map(matrix, H),
                       A=A, b=matrix(t))
    return sol

  def init_algo(self):
    self.make_problem()
    self.directions = []
    self.points = []
    self.offsets = []
    self.inner = None
    self.outer = None
    #Search in three "random" directions
    directions = map(normalize, [np.array([[0, 1]]).T,
                                 np.array([[1, 0]]).T,
                                 np.array([[-1, -1]]).T])
    rdirs = []

    for d in directions:
      try:
        self.step(d)
      except SteppingException as e:
        rdir = np.random.random((2, 1))
        print str(e), " Will try in a random direction {}".format(rdir)
        rdirs.append(rdir)

    for d in rdirs:
      self.step(d)

  def step(self, d):
    if self.solve(d):
      self.directions.append(d.T)
      self.points.append(self.com)
      self.offsets.append(d.T.dot(self.com))
    else:
      m = ["Failed to step in direction {}".format(d.T),
           "Terminated in {} state".format(self.sol['status'])]
      raise SteppingException('\n'.join(m))

  def build_polys(self):
    if self.outer is None:
      A = np.vstack(self.directions)
      b = np.vstack(self.offsets)
      self.outer = cdd.Matrix(np.hstack([b, -A]))
      self.outer.rep_type = cdd.RepType.INEQUALITY
    else:
      self.outer.extend(np.hstack((self.offsets[-1], -self.directions[-1])))
      self.outer.canonicalize()

    if self.inner is None:
      A = np.vstack([p.T for p in self.points])
      b = np.ones((len(self.points), 1))
      self.inner = cdd.Matrix(np.hstack((b, A)))
      self.inner.rep_type = cdd.RepType.GENERATOR
    else:
      self.inner.extend(np.hstack(([[1]], self.points[-1].T)))
      self.inner.canonicalize()

  def find_direction(self):
    self.build_polys()
    out_area = area_convex(self.outer)

    areas = []
    ineq = np.array(cdd.Polyhedron(self.inner).get_inequalities())

    for line in ineq:
      A_e = self.outer.copy()
      A_e.extend(cdd.Matrix(line.reshape(1, line.size)))
      A_e.canonicalize()
      areas.append(out_area - area_convex(A_e))

    i, a = max(enumerate(areas), key=lambda x: x[1])
    return -ineq[i, 1:]

  def next_edge(self, plot=False, record=False, number=0):
    d = normalize(self.find_direction().reshape((2, 1)))
    self.step(d)

    if plot or record:
      self.plot()
      if plot:
        self.show()
      if record:
        filename = "stability_{0:04d}".format(number)
        plt.savefig(filename)
        plt.close()

  def iterBound(self, nr_edges_init, error_init, prec):
    c = float(343)/float(243)
    return nr_edges_init*(np.sqrt(c*error_init/prec) - 1)

  def compute(self, epsilon=1e-4, plot_init=False,
              plot_step=False,
              record_anim=False,
              plot_final=True):
    self.make_problem()
    self.init_algo()
    self.build_polys()

    if plot_init:
      self.plot()
      self.show()

    error = area_convex(self.outer) - area_convex(self.inner)

    iterBound = self.iterBound(3, error, epsilon)

    print "This should take {} iterations".format(np.ceil(iterBound))

    nrSteps = 0
    while(error > epsilon):
      try:
        self.next_edge(plot_step, record_anim, nrSteps)
      except SteppingException as e:
        print "Failure detected... Aborting"
        print e.message
        break
      error = area_convex(self.outer) - area_convex(self.inner)
      nrSteps += 1

    print "NrIter : {} | Remainder : {}".format(nrSteps, error)

    if plot_final:
      self.plot()
      self.show()

  def polygon(self, centroid=None):
    gen = np.array(cdd.Polyhedron(self.inner).get_generators())
    p = np.vstack([gen, gen[0, :]])
    if centroid is None:
      x, y = p[:, 1], p[:, 2]
    else:
      x, y = p[:, 1] + centroid.item(0), p[:, 2] + centroid.item(1)
    return geom.Polygon(zip(x, y)).convex_hull

  def reset_fig(self):
    fig = plt.figure()
    self.ax = fig.add_subplot('111', aspect='equal', projection='3d')
    self.ax.set_xlim([-0.5, 1])
    self.ax.set_ylim([-0.5, 1])
    self.ax.set_zlim([-0.5, 1])

  def plot(self):
    self.reset_fig()
    self.plot_contacts()
    self.plot_solution()
    self.plot_polygons()

  def show(self):
    plt.show()

  def plot_contacts(self):
    X, Y, Z, U, V, W = [], [], [], [], [], []
    positions = np.hstack([c.r for c in self.contacts])
    normals = np.hstack([c.n for c in self.contacts])
    X, Y, Z = positions[0, :], positions[1, :], positions[2, :]
    U, V, W = normals[0, :], normals[1, :], normals[2, :]
    self.ax.quiver(X, Y, Z, U, V, W, color='black', linestyles='dashed')

  def plot_solution(self):
    com_pos = self.com
    forces = self.forces

    x, y = com_pos.item(0), com_pos.item(1)
    self.ax.plot([x], [y], linestyle="none",
                 marker='o', alpha=0.6,
                 markersize=10, markerfacecolor='black')

    positions = np.hstack([c.r for c in self.contacts])

    for pos, force in zip(positions.T, forces.T):
      X, Y, Z = pos[0], pos[1], pos[2]
      U, V, W = -force[0], -force[1], -force[2]
      l = np.linalg.norm(force)/(self.mass*abs(self.gravity.item(2)))
      self.ax.quiver(X, Y, Z, U, V, W, color='r', length=l)

  def plot_direction(self, d):
    self.ax.plot([0, d.item(0)], [0, d.item(1)], marker='d')

  def plot_polygons(self):
    self.plot_polygon(self.inner, 'x')
    self.plot_polygon(self.outer, 'o')

  def plot_polygon(self, poly, m):
    gen = np.array(cdd.Polyhedron(poly).get_generators())
    p = np.vstack([gen, gen[0, :]])
    x, y = p[:, 1], p[:, 2]
    poly = geom.Polygon(zip(x, y))
    x, y = poly.convex_hull.exterior.xy
    self.ax.plot(x, y, marker=m)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import ast
import copy
import jinja2
import numpy as np
import six

from collections import Mapping, defaultdict

from polyaxon_schemas.exceptions import PolyaxonfileError
from polyaxon_schemas.polyaxonfile.utils import deep_update
from polyaxon_schemas.utils import to_list

class Parser(object):
    """Parses the Polyaxonfile."""

    env = jinja2.Environment()

    @classmethod
    def get_headers(cls, spec, data):
        parsed_data = {
            section: data[section] for section in spec.HEADER_SECTIONS
            if data.get(section)
        }
        return parsed_data

    @classmethod
    def parse(cls, spec, data, matrix_declarations=None):
        declarations = copy.copy(data.get(spec.DECLARATIONS, {}))
        matrix_declarations = copy.copy(matrix_declarations)
        if matrix_declarations:
            declarations = deep_update(matrix_declarations, declarations)

        parsed_data = {
            spec.VERSION: data[spec.VERSION],
            spec.KIND: data[spec.KIND],
        }

        if declarations:
            declarations = cls.parse_expression(spec, declarations, declarations)
            parsed_data[spec.DECLARATIONS] = declarations

        if spec.ENVIRONMENT in data:
            parsed_data[spec.ENVIRONMENT] = cls.parse_expression(
                spec, data[spec.ENVIRONMENT], declarations)

        if spec.SETTINGS in data:
            parsed_data[spec.SETTINGS] = cls.parse_expression(
                spec, data[spec.SETTINGS], declarations)

        if spec.RUN_EXEC in data:
            parsed_data[spec.RUN_EXEC] = cls.parse_expression(
                spec, data[spec.RUN_EXEC], declarations, True, False)

        for section in spec.GRAPH_SECTIONS:
            if section in data:
                parsed_data[section] = cls.parse_expression(
                    spec, data[section], declarations, True, True)

        return parsed_data

    @classmethod
    def parse_expression(cls,
                         spec,
                         expression,
                         declarations,
                         check_operators=False,
                         check_graph=False):
        if isinstance(expression, (int, float, complex, type(None), np.integer, np.floating)):
            return expression
        if isinstance(expression, Mapping):
            if len(expression) == 1:
                old_key, value = list(six.iteritems(expression))[0]
                # always parse the keys, they must be base object or evaluate to base objects
                key = cls.parse_expression(spec, old_key, declarations)
                if check_operators and cls.is_operator(spec, key):
                    return cls._parse_operator(spec, {key: value}, declarations)
                if check_graph and key in ['graph', 'encoder', 'decoder']:
                    return {key: cls._parse_graph(spec, value, declarations)}
                if check_graph and key == 'feature_processors':  # noqa, no-else-return
                    return {
                        key: {
                            cls.parse_expression(spec, f_key, declarations):
                                cls._parse_graph(spec, f_vlaue, declarations)
                            for f_key, f_vlaue in six.iteritems(value)
                        }
                    }
                else:
                    return {
                        key: cls.parse_expression(
                            spec, value, declarations, check_operators, check_graph)
                    }

            new_expression = {}
            for k, v in six.iteritems(expression):
                new_expression.update(
                    cls.parse_expression(spec, {k: v}, declarations, check_operators, check_graph))
            return new_expression

        if isinstance(expression, list):
            return list(cls.parse_expression(spec, v, declarations, check_operators, check_graph)
                        for v in expression)
        if isinstance(expression, tuple):
            return tuple(cls.parse_expression(spec, v, declarations, check_operators, check_graph)
                         for v in expression)
        if isinstance(expression, six.string_types):
            return cls._evaluate_expression(
                spec, expression, declarations, check_operators, check_graph)

    @classmethod
    def _evaluate_expression(cls, spec, expression, declarations, check_operators, check_graph):
        result = cls.env.from_string(expression).render(**declarations)
        if result == expression:
            try:
                return ast.literal_eval(result)
            except (ValueError, SyntaxError):
                pass
            return result
        return cls.parse_expression(spec, result, declarations, check_operators, check_graph)

    @classmethod
    def _parse_operator(cls, spec, expression, declarations):
        k, v = list(six.iteritems(expression))[0]
        op = spec.OPERATORS[k].from_dict(v)
        return op.parse(spec=spec, parser=cls, declarations=declarations)

    @staticmethod
    def is_operator(spec, key):
        return key in spec.OPERATORS

    @classmethod
    def _parse_graph(cls, spec, graph, declarations):  # noqa, too-many-branches
        input_layers = to_list(graph['input_layers'])
        layer_names = set(input_layers)
        tags = {}
        layers = []
        outputs = []
        layers_counters = defaultdict(int)
        unused_layers = set(input_layers)

        if not isinstance(graph['layers'], list):
            raise PolyaxonfileError("Graph definition expects a list of layer definitions.")

        def add_tag(tag, layer_value):
            if tag in tags:
                tags[tag] = to_list(tags[tag])
                tags[tag].append(layer_value['name'])
            else:
                tags[tag] = layer_value['name']

        def get_layer_name(layer_value, layer_type):
            if 'name' not in layer_value:
                layers_counters[layer_type] += 1
                return '{}_{}'.format(layer_type, layers_counters[layer_type])

            return layer_value['name']

        layers_declarations = {}
        layers_declarations.update(declarations)

        last_layer = None
        first_layer = True
        for layer_expression in graph['layers']:
            parsed_layer = cls.parse_expression(spec, layer_expression, layers_declarations, True)
            # Gather all tags from the layers
            parsed_layer = to_list(parsed_layer)
            for layer in parsed_layer:
                if not layer:
                    continue

                layer_type, layer_value = list(six.iteritems(layer))[0]

                if layer_value is None:
                    layer_value = {}
                # Check that the layer has a name otherwise generate one
                name = get_layer_name(layer_value, layer_type)
                if name not in layer_names:
                    layer_names.add(name)
                    layer_value['name'] = name
                else:
                    raise PolyaxonfileError(
                        "The name `{}` is used 2 times in the graph. "
                        "All layer names should be unique. "
                        "If you need to reference a layer in a for loop "
                        "think about using `tags`".format(name))

                for tag in to_list(layer_value.get('tags', [])):
                    add_tag(tag, layer_value)

                # Check if the layer is an output
                if layer_value.get('is_output', False) is True:
                    outputs.append(layer_value['name'])
                else:
                    # Add the layer to unused
                    unused_layers.add(layer_value['name'])

                # Check the layers inputs
                if not layer_value.get('inbound_nodes'):
                    if last_layer is not None:
                        layer_value['inbound_nodes'] = [
                            last_layer['name']]  # noqa, unsubscriptable-object
                    if first_layer and len(input_layers) == 1:
                        layer_value['inbound_nodes'] = input_layers
                    if first_layer and len(input_layers) > 1:
                        raise PolyaxonfileError("The first layer must indicate which input to use,"
                                                "You have {} layers: {}".format(len(input_layers),
                                                                                input_layers))

                first_layer = False
                for input_layer in layer_value.get('inbound_nodes', []):
                    if input_layer not in layer_names:
                        raise PolyaxonfileError(
                            "The layer `{}` has a non existing "
                            "inbound node `{}`".format(layer_value['name'], input_layer))
                    if input_layer in unused_layers:
                        unused_layers.remove(input_layer)

                # Add layer
                layers.append({layer_type: layer_value})

                # Update layers_declarations
                layers_declarations['tags'] = tags

                # Update last_layer
                last_layer = layer_value

        # Add last layer as output
        if last_layer:
            if last_layer['name'] not in outputs:
                outputs.append(last_layer['name'])

            # Remove last layer from unused layers
            if last_layer['name'] in unused_layers:
                unused_layers.remove(last_layer['name'])

        # Check if some layers are unused
        if unused_layers:
            raise PolyaxonfileError(
                "These layers `{}` were declared but are not used.".format(unused_layers))

        return {
            'input_layers': to_list(graph['input_layers']),
            'layers': layers,
            'output_layers': outputs
        }
<commit_msg>Convert numpy types to base types<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import ast
import copy
import jinja2
import numpy as np
import six

from collections import Mapping, defaultdict

from polyaxon_schemas.exceptions import PolyaxonfileError
from polyaxon_schemas.polyaxonfile.utils import deep_update
from polyaxon_schemas.utils import to_list

class Parser(object):
    """Parses the Polyaxonfile."""

    env = jinja2.Environment()

    @classmethod
    def get_headers(cls, spec, data):
        parsed_data = {
            section: data[section] for section in spec.HEADER_SECTIONS
            if data.get(section)
        }
        return parsed_data

    @classmethod
    def parse(cls, spec, data, matrix_declarations=None):
        declarations = copy.copy(data.get(spec.DECLARATIONS, {}))
        matrix_declarations = copy.copy(matrix_declarations)
        if matrix_declarations:
            declarations = deep_update(matrix_declarations, declarations)

        parsed_data = {
            spec.VERSION: data[spec.VERSION],
            spec.KIND: data[spec.KIND],
        }

        if declarations:
            declarations = cls.parse_expression(spec, declarations, declarations)
            parsed_data[spec.DECLARATIONS] = declarations

        if spec.ENVIRONMENT in data:
            parsed_data[spec.ENVIRONMENT] = cls.parse_expression(
                spec, data[spec.ENVIRONMENT], declarations)

        if spec.SETTINGS in data:
            parsed_data[spec.SETTINGS] = cls.parse_expression(
                spec, data[spec.SETTINGS], declarations)

        if spec.RUN_EXEC in data:
            parsed_data[spec.RUN_EXEC] = cls.parse_expression(
                spec, data[spec.RUN_EXEC], declarations, True, False)

        for section in spec.GRAPH_SECTIONS:
            if section in data:
                parsed_data[section] = cls.parse_expression(
                    spec, data[section], declarations, True, True)

        return parsed_data

    @classmethod
    def parse_expression(cls,
                         spec,
                         expression,
                         declarations,
                         check_operators=False,
                         check_graph=False):
        if isinstance(expression, (int, float, complex, type(None))):
            return expression
        if isinstance(expression, np.integer):
            return int(expression)
        if isinstance(expression, np.floating):
            return float(expression)
        if isinstance(expression, Mapping):
            if len(expression) == 1:
                old_key, value = list(six.iteritems(expression))[0]
                # always parse the keys, they must be base object or evaluate to base objects
                key = cls.parse_expression(spec, old_key, declarations)
                if check_operators and cls.is_operator(spec, key):
                    return cls._parse_operator(spec, {key: value}, declarations)
                if check_graph and key in ['graph', 'encoder', 'decoder']:
                    return {key: cls._parse_graph(spec, value, declarations)}
                if check_graph and key == 'feature_processors':  # noqa, no-else-return
                    return {
                        key: {
                            cls.parse_expression(spec, f_key, declarations):
                                cls._parse_graph(spec, f_vlaue, declarations)
                            for f_key, f_vlaue in six.iteritems(value)
                        }
                    }
                else:
                    return {
                        key: cls.parse_expression(
                            spec, value, declarations, check_operators, check_graph)
                    }

            new_expression = {}
            for k, v in six.iteritems(expression):
                new_expression.update(
                    cls.parse_expression(spec, {k: v}, declarations, check_operators, check_graph))
            return new_expression

        if isinstance(expression, list):
            return list(cls.parse_expression(spec, v, declarations, check_operators, check_graph)
                        for v in expression)
        if isinstance(expression, tuple):
            return tuple(cls.parse_expression(spec, v, declarations, check_operators, check_graph)
                         for v in expression)
        if isinstance(expression, six.string_types):
            return cls._evaluate_expression(
                spec, expression, declarations, check_operators, check_graph)

    @classmethod
    def _evaluate_expression(cls, spec, expression, declarations, check_operators, check_graph):
        result = cls.env.from_string(expression).render(**declarations)
        if result == expression:
            try:
                return ast.literal_eval(result)
            except (ValueError, SyntaxError):
                pass
            return result
        return cls.parse_expression(spec, result, declarations, check_operators, check_graph)

    @classmethod
    def _parse_operator(cls, spec, expression, declarations):
        k, v = list(six.iteritems(expression))[0]
        op = spec.OPERATORS[k].from_dict(v)
        return op.parse(spec=spec, parser=cls, declarations=declarations)

    @staticmethod
    def is_operator(spec, key):
        return key in spec.OPERATORS

    @classmethod
    def _parse_graph(cls, spec, graph, declarations):  # noqa, too-many-branches
        input_layers = to_list(graph['input_layers'])
        layer_names = set(input_layers)
        tags = {}
        layers = []
        outputs = []
        layers_counters = defaultdict(int)
        unused_layers = set(input_layers)

        if not isinstance(graph['layers'], list):
            raise PolyaxonfileError("Graph definition expects a list of layer definitions.")

        def add_tag(tag, layer_value):
            if tag in tags:
                tags[tag] = to_list(tags[tag])
                tags[tag].append(layer_value['name'])
            else:
                tags[tag] = layer_value['name']

        def get_layer_name(layer_value, layer_type):
            if 'name' not in layer_value:
                layers_counters[layer_type] += 1
                return '{}_{}'.format(layer_type, layers_counters[layer_type])

            return layer_value['name']

        layers_declarations = {}
        layers_declarations.update(declarations)

        last_layer = None
        first_layer = True
        for layer_expression in graph['layers']:
            parsed_layer = cls.parse_expression(spec, layer_expression, layers_declarations, True)
            # Gather all tags from the layers
            parsed_layer = to_list(parsed_layer)
            for layer in parsed_layer:
                if not layer:
                    continue

                layer_type, layer_value = list(six.iteritems(layer))[0]

                if layer_value is None:
                    layer_value = {}
                # Check that the layer has a name otherwise generate one
                name = get_layer_name(layer_value, layer_type)
                if name not in layer_names:
                    layer_names.add(name)
                    layer_value['name'] = name
                else:
                    raise PolyaxonfileError(
                        "The name `{}` is used 2 times in the graph. "
                        "All layer names should be unique. "
                        "If you need to reference a layer in a for loop "
                        "think about using `tags`".format(name))

                for tag in to_list(layer_value.get('tags', [])):
                    add_tag(tag, layer_value)

                # Check if the layer is an output
                if layer_value.get('is_output', False) is True:
                    outputs.append(layer_value['name'])
                else:
                    # Add the layer to unused
                    unused_layers.add(layer_value['name'])

                # Check the layers inputs
                if not layer_value.get('inbound_nodes'):
                    if last_layer is not None:
                        layer_value['inbound_nodes'] = [
                            last_layer['name']]  # noqa, unsubscriptable-object
                    if first_layer and len(input_layers) == 1:
                        layer_value['inbound_nodes'] = input_layers
                    if first_layer and len(input_layers) > 1:
                        raise PolyaxonfileError("The first layer must indicate which input to use,"
                                                "You have {} layers: {}".format(len(input_layers),
                                                                                input_layers))

                first_layer = False
                for input_layer in layer_value.get('inbound_nodes', []):
                    if input_layer not in layer_names:
                        raise PolyaxonfileError(
                            "The layer `{}` has a non existing "
                            "inbound node `{}`".format(layer_value['name'], input_layer))
                    if input_layer in unused_layers:
                        unused_layers.remove(input_layer)

                # Add layer
                layers.append({layer_type: layer_value})

                # Update layers_declarations
                layers_declarations['tags'] = tags

                # Update last_layer
                last_layer = layer_value

        # Add last layer as output
        if last_layer:
            if last_layer['name'] not in outputs:
                outputs.append(last_layer['name'])

            # Remove last layer from unused layers
            if last_layer['name'] in unused_layers:
                unused_layers.remove(last_layer['name'])

        # Check if some layers are unused
        if unused_layers:
            raise PolyaxonfileError(
                "These layers `{}` were declared but are not used.".format(unused_layers))

        return {
            'input_layers': to_list(graph['input_layers']),
            'layers': layers,
            'output_layers': outputs
        }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from math import *

##Directions:
##[{'p1':Point1, 'p2':Point2, 'len1':Line1, ('len2':Line2, 'angle':angle)}, ...]

class solver():
    def __init__(self, Directions=list()):
        #Cosine Theorem
        self.CosineTheoremAngle = lambda a, b, c: acos((b**2+c**2-a**2)/(2*b*c))
        self.CosineTheoremSide = lambda alpha, b, c: b**2+c**2-2*b*c*cos(alpha)
        self.Directions = Directions

    def set(self, Directions): self.Directions = Directions

    def answer(self):
        answer = self.Iterator() if self.Parser() else None
        self.Directions.clear()
        return answer

    def Parser(self):
        for e in self.Directions:
            pos = self.Directions.index(e)
            if e.get('p1', False) is False: return False
            if e.get('p2', False) is False: return False
            if e.get('len1', False) is False: return False
            if e.get('len2', False) is False and e.get('angle', False) is False: return False
            if e.get('len2', False) is False: self.Directions[pos]['Type'] = 'PLAP'
            elif e.get('angle', False) is False: self.Directions[pos]['Type'] = 'PLLP'
        return True

    def Iterator(self):
        results = list()
        for e in self.Directions:
            p1 = results[e['p1']] if type(e['p1'])==int else e['p1']
            p2 = results[e['p2']] if type(e['p2'])==int else e['p2']
            if e['Type']=='PLAP': results.append(self.PLAP(p1, e['len1'], e['angle'], p2))
            elif e['Type']=='PLLP': results.append(self.PLLP(p1, e['len1'], e['len2'], p2))
        return results

    def PLAP(self, p1, line1, angle, p2, other=False):
        x1 = p1[0]
        y1 = p1[1]
        len1 = float(line1)
        angle2 = radians(float(angle))
        angle1 = self.m(p1, p2)
        if other:
            cx = x1+len1*cos(angle1-angle2)
            cy = y1+len1*sin(angle1-angle2)
        else:
            cx = x1+len1*cos(angle1+angle2)
            cy = y1+len1*sin(angle1+angle2)
        return cx, cy

    def PLLP(self, p1, line1, line2, p2, other=False):
        x1 = p1[0]
        y1 = p1[1]
        x2 = p2[0]
        y2 = p2[1]
        len1 = float(line1)
        len2 = float(line2)
        diffAB = sqrt((x1-x2)**2+(y2-y1)**2)
        angle1 = self.m(p1, p2)
        angle2 = self.CosineTheoremAngle(len2, len1, diffAB)
        print(degrees(angle2))
        print(degrees(angle1))
        if other:
            cx = x1-len1*cos(angle1+angle2)
            cy = y1+len1*sin(angle1+angle2)
        else:
            cx = x1+len1*cos(angle1-angle2)
            cy = y1-len1*sin(angle1-angle2)
        return cx, cy

    def m(self, p1, p2):
        x1 = p1[0]
        y1 = p1[1]
        x2 = p2[0]
        y2 = p2[1]
        x = abs(x2-x1)
        y = abs(y2-y1)
        diffAB = sqrt(x**2+y**2)
        return self.CosineTheoremAngle(y, x, diffAB)

if __name__=='__main__':
    #Test
    s = solver([
        {'p1':(-60, 0), 'p2':(0, 0), 'len1':30, 'angle':50}, #C
        {'p1':0, 'p2':(0, 0), 'len1':50, 'len2':60}, #D
        {'p1':1, 'p2':0, 'len1':50, 'len2':50}, #E
        ])
    print(s.answer())

    ##cx= -40.716371709403816 cy= 22.98133329356934
    ##dx= -6.698073034033397 dy= 59.62495968661744
    ##ex= -55.44153371488418 ey= 70.76385733649067
<commit_msg>Fix functions (Done).<commit_after># -*- coding: utf-8 -*-
from math import *

##Directions:
##[{'p1':Point1, 'p2':Point2, 'len1':Line1, ('len2':Line2, 'angle':angle)}, ...]

class solver():
    def __init__(self, Directions=list()):
        #Cosine Theorem
        self.CosineTheoremAngle = lambda a, b, c: acos((b**2+c**2-a**2)/(2*b*c))
        self.CosineTheoremSide = lambda alpha, b, c: b**2+c**2-2*b*c*cos(alpha)
        self.Directions = Directions

    def set(self, Directions): self.Directions = Directions

    def answer(self):
        answer = self.Iterator() if self.Parser() else None
        self.Directions.clear()
        return answer

    def Parser(self):
        for e in self.Directions:
            pos = self.Directions.index(e)
            if e.get('p1', False) is False: return False
            if e.get('p2', False) is False: return False
            if e.get('len1', False) is False: return False
            if e.get('len2', False) is False and e.get('angle', False) is False: return False
            if e.get('len2', False) is False: self.Directions[pos]['Type'] = 'PLAP'
            elif e.get('angle', False) is False: self.Directions[pos]['Type'] = 'PLLP'
        return True

    def Iterator(self):
        results = list()
        for e in self.Directions:
            p1 = results[e['p1']] if type(e['p1'])==int else e['p1']
            p2 = results[e['p2']] if type(e['p2'])==int else e['p2']
            #Direction of the point
            other = e.get('other', False)
            ##True: Toward len1.
            ##False: Toward len2.
            if e['Type']=='PLAP': results.append(self.PLAP(p1, e['len1'], e['angle'], p2, other))
            elif e['Type']=='PLLP': results.append(self.PLLP(p1, e['len1'], e['len2'], p2, other))
        return results

    def PLAP(self, p1, line1, angle, p2, other=False):
        x1 = p1[0]
        y1 = p1[1]
        len1 = float(line1)
        angle2 = radians(float(angle))
        angle1 = self.m(p1, p2)
        if other:
            cx = x1+len1*cos(angle1-angle2)
            cy = y1+len1*sin(angle1-angle2)
        else:
            cx = x1+len1*cos(angle1+angle2)
            cy = y1+len1*sin(angle1+angle2)
        return cx, cy

    def PLLP(self, p1, line1, line2, p2, other=False):
        x1 = p1[0]
        y1 = p1[1]
        x2 = p2[0]
        y2 = p2[1]
        len1 = float(line1)
        len2 = float(line2)
        diffAB = sqrt((x1-x2)**2+(y2-y1)**2)
        angle1 = self.m(p1, p2)
        angle2 = self.CosineTheoremAngle(len2, len1, diffAB)
        if other:
            cx = x1+len1*cos(angle1+angle2)
            cy = y1+len1*sin(angle1+angle2)
        else:
            cx = x1+len1*cos(angle1-angle2)
            cy = y1-len1*sin(angle1-angle2)
        return cx, cy

    def m(self, p1, p2):
        x1 = p1[0]
        y1 = p1[1]
        x2 = p2[0]
        y2 = p2[1]
        x = abs(x2-x1)
        y = abs(y2-y1)
        diffAB = sqrt(x**2+y**2)
        return self.CosineTheoremAngle(y, x, diffAB)

if __name__=='__main__':
    #Test
    s = solver([
        {'p1':(-60, 0), 'p2':(0, 0), 'len1':30, 'angle':50}, #C
        {'p1':0, 'p2':(0, 0), 'len1':50, 'len2':60}, #D
        {'p1':0, 'p2':1, 'len1':50, 'len2':50, 'other':True}, #E
        ])
    print("C={}\nD={}\nE={}".format(*s.answer()))

    ##cx= -40.716371709403816 cy= 22.98133329356934
    ##dx= -6.698073034033397 dy= 59.62495968661744
    ##ex= -55.44153371488418 ey= 70.76385733649067
<|endoftext|>


<commit_before>import DeepFried2 as df
from collections import OrderedDict as _OrderedDict
from itertools import chain as _chain

class Container(df.Module):

    def __init__(self, *modules):
        df.Module.__init__(self)

        self.modules = []
        self.add(*modules)

    def evaluate(self):
        df.Module.evaluate(self)
        for module in self.modules:
            module.evaluate()

    def training(self):
        df.Module.training(self)
        for module in self.modules:
            module.training()

    def parameters(self, *a, **kw):
        params = _chain.from_iterable(m.parameters(*a, **kw) for m in self.modules)

        # We actually need to remove duplicates from the list of parameters
        # (and their corresponding gradients) in order to support reusing
        # the same layer at multiple places in the graph,
        # e.g. do weight sharing.
        return list(_OrderedDict.fromkeys(params).keys())

    def get_stat_updates(self):
        return _chain.from_iterable(m.get_stat_updates() for m in self.modules)

    def add(self, *modules):
        for m in modules:
            assert isinstance(m, df.Module), "`{}`s can only contain objects subtyping `df.Module`. You tried to add the following `{}`: {}".format(df.utils.typename(self), df.utils.typename(m), m)
        self.modules += modules

    def __getitem__(self, key):
        if isinstance(key, slice):
            return type(self)(*df.utils.aslist(self.modules[key]))
        elif isinstance(key, (list, tuple)):
            return type(self)(*[self.modules[k] for k in key])
        else:
            return self.modules[key]

    def __getstate__(self):
        return [m.__getstate__() for m in self.modules]

    def __setstate__(self, state):
        for m, s in zip(self.modules, state):
            m.__setstate__(s)
<commit_msg>Fix container bug (iterator -> list)<commit_after>import DeepFried2 as df
from collections import OrderedDict as _OrderedDict
from itertools import chain as _chain

class Container(df.Module):

    def __init__(self, *modules):
        df.Module.__init__(self)

        self.modules = []
        self.add(*modules)

    def evaluate(self):
        df.Module.evaluate(self)
        for module in self.modules:
            module.evaluate()

    def training(self):
        df.Module.training(self)
        for module in self.modules:
            module.training()

    def parameters(self, *a, **kw):
        params = list(_chain.from_iterable(m.parameters(*a, **kw) for m in self.modules))

        # We actually need to remove duplicates from the list of parameters
        # (and their corresponding gradients) in order to support reusing
        # the same layer at multiple places in the graph,
        # e.g. do weight sharing.
        return list(_OrderedDict.fromkeys(params).keys())

    def get_stat_updates(self):
        return list(_chain.from_iterable(m.get_stat_updates() for m in self.modules))

    def add(self, *modules):
        for m in modules:
            assert isinstance(m, df.Module), "`{}`s can only contain objects subtyping `df.Module`. You tried to add the following `{}`: {}".format(df.utils.typename(self), df.utils.typename(m), m)
        self.modules += modules

    def __getitem__(self, key):
        if isinstance(key, slice):
            return type(self)(*df.utils.aslist(self.modules[key]))
        elif isinstance(key, (list, tuple)):
            return type(self)(*[self.modules[k] for k in key])
        else:
            return self.modules[key]

    def __getstate__(self):
        return [m.__getstate__() for m in self.modules]

    def __setstate__(self, state):
        for m, s in zip(self.modules, state):
            m.__setstate__(s)
<|endoftext|>


<commit_before>"""
Module for standardizing Chinese text
"""

from unicodedata import category

import mafan

def standardize(text):
    """
    Standardize a line of Chinese text
    :param text: unicode string of Chinese
    :return: standardize form of input line
    """
    stripped = strip(text)
    trad = make_traditional(stripped)
    return trad

def make_traditional(text):
    """
    Makes Chinese text Traditional Chinese
    :param text: unicode string of Chinese
    :return: unicode string of Traditional Chinese
    """
    if not mafan.is_traditional(text):
        trad = mafan.tradify(text)
    else:
        trad = text
    return trad

def is_traditional(text):
    """
    Checks if a unicode string is Traditional Chinese
    :param text: unicode string
    :return: boolean
    """
    return mafan.is_traditional(text)

def is_simplified(text):
    return mafan.is_simplified(text)

def strip(text):
    """
    Strip punctuation and symbols from text
    Thanks to http://stackoverflow.com/a/11066579/2701544
    :param text: input unicode string
    :return: unicode string without punctuation or symbols
    """
    punctuation_cats = {'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po'}
    symbol_cats = {'Sc', 'Sk', 'Sm', 'So'}
    separator_cats = {'Zi', 'Zp', 'Zs'}
    remove_cats = punctuation_cats | symbol_cats | separator_cats
    gen = (x for x in text
           if category(x) not in remove_cats)
    return u''.join(gen)

<commit_msg>avoid changes<commit_after>"""
Module for standardizing Chinese text
"""

from unicodedata import category

import mafan

def standardize(text):
    """
    Standardize a line of Chinese text
    :param text: unicode string of Chinese
    :return: standardize form of input line
    """
    stripped = strip(text)
    trad = make_traditional(stripped)
    return trad

def make_traditional(text):
    """
    Makes Chinese text Traditional Chinese
    :param text: unicode string of Chinese
    :return: unicode string of Traditional Chinese
    """
    if not mafan.is_traditional(text):
        trad = mafan.tradify(text)
    else:
        trad = text
    return trad

def is_traditional(text):
    """
    Checks if a unicode string is Traditional Chinese
    :param text: unicode string
    :return: boolean
    """
    return mafan.is_traditional(text)

def is_simplified(text):
    return mafan.is_simplified(text)

def strip(text):
    """
    Strip punctuation and symbols from text
    Thanks to http://stackoverflow.com/a/11066579/2701544
    :param text: input unicode string
    :return: unicode string without punctuation or symbols
    """
    gen = chunk_gen(text)
    return u''.join(gen)

def chunk_gen(text, sub=' '):
    punctuation_cats = {'Pc', 'Pd', 'Ps', 'Pe', 'Pi', 'Pf', 'Po'}
    symbol_cats = {'Sc', 'Sk', 'Sm', 'So'}
    separator_cats = {'Zi', 'Zp', 'Zs'}
    sub_cats = punctuation_cats | symbol_cats | separator_cats

    for char in text:
        if category(char) in sub_cats:
            yield char
        else:
            yield sub
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

SOURCE_DIR = os.path.dirname(__file__)
LATEX_DIR = os.path.join(SOURCE_DIR, '_latex')
PREAMBLE_FILE = os.path.join(LATEX_DIR, 'preamble.tex')
TITLE_PAGE_FILE = os.path.join(LATEX_DIR, 'title_page.tex')

BUILD_TYPE = os.getenv("BUILD_TYPE")

# -- Project information -----------------------------------------------------

if BUILD_TYPE == 'oneapi':
    project = u'Intel® oneAPI Threading Building Blocks (oneTBB)'
else:
    project = u'oneAPI Threading Building Blocks (oneTBB)'
copyright = u'2021, Intel Corporation'
author = u'Intel'

# The short X.Y version
version = u''
# The full version, including alpha/beta/rc tags
release = u''

# -- General configuration ---------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.imgmath',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinx.ext.githubpages'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
#master_doc = 'main/title_main'
master_doc = 'index'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None

# Syntax highlighting for the :: directive
highlight_language = 'cpp'

if BUILD_TYPE == 'oneapi':
    rst_prolog = """
.. |full_name| replace:: Intel\ |reg|\  oneAPI Threading Building Blocks (oneTBB)
.. |short_name| replace:: oneTBB
.. |product| replace:: oneTBB
.. |reg| unicode:: U+000AE
.. |copy| unicode:: U+000A9
.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
    """
else:
    rst_prolog = """
.. |full_name| replace:: oneAPI Threading Building Blocks (oneTBB)
.. |short_name| replace:: oneTBB
.. |product| replace:: oneTBB
.. |reg| unicode:: U+000AE
.. |copy| unicode:: U+000A9
.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
    """

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
if BUILD_TYPE == 'oneapi':
    html_theme = 'sphinx_rtd_theme'
else:
    html_theme = 'sphinx_book_theme'
    html_theme_options = {
    'repository_url': 'https://github.com/oneapi-src/oneTBB',
    'path_to_docs': 'doc/main',
    'use_issues_button': True,
    'use_edit_page_button': True,
    'repository_branch': 'master',
    'extra_footer': '
Cookies
'
    }

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

if BUILD_TYPE == 'oneapi':
    html_context = {
        'css_files': [
            '_static/theme_overrides.css',  # override wide tables in RTD theme
        ],
    }
else:
    html_js_files = ['custom.js']
    html_logo = '_static/oneAPI-rgb-rev-100.png'
    html_favicon = '_static/favicons.png'

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself.  Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}

# -- Options for HTMLHelp output ---------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'sphinx-infodevdoc'

# -- Options for LaTeX output ------------------------------------------------

#latex_engine = 'xelatex'
#PDF_TITLE = 'Information Development Template'
#
#with open(PREAMBLE_FILE, 'r', encoding='utf-8') as f:
#    PREAMBLE = f.read()
#
#with open(TITLE_PAGE_FILE, 'r', encoding='utf-8') as f:
#    TITLE_PAGE = f.read().replace('<PDF_TITLE>', PDF_TITLE)
#
#
#latex_elements = {
#    # The paper size ('letterpaper' or 'a4paper').
#    #
#    'extraclassoptions': 'openany,oneside',
#    'babel' : '\\usepackage[english]{babel}',
#    'papersize': 'a4paper',
#    'releasename':" ",
#    # Sonny, Lenny, Glenn, Conny, Rejne, Bjarne and Bjornstrup
#    # 'fncychap': '\\usepackage[Lenny]{fncychap}',
#    'fncychap':  '',
#    #'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}',
#
#    'figure_align':'htbp',
#    # The font size ('10pt', '11pt' or '12pt').
#    #
#    'pointsize': '12pt',
#
#    # Additional stuff for the LaTeX preamble.
#    #
#    'preamble': PREAMBLE,
#
#    'maketitle': TITLE_PAGE,
#    # Latex figure (float) alignment
#    #
#    # 'figure_align': 'htbp',
#    'sphinxsetup': \
#        'hmargin={0.7in,0.7in}, vmargin={1in,1in}, \
#        verbatimwithframe=true, \
#        TitleColor={rgb}{0,0.686,0.941}, \
#        HeaderFamily=\\rmfamily\\bfseries, \
#        InnerLinkColor={rgb}{0,0.686,0.941}, \
#        OuterLinkColor={rgb}{0,0.686,0.941}',
#
#    'tableofcontents':' '
#}
#
#latex_logo = '_latex/intel_logo.png'
## Grouping the document tree into LaTeX files. List of tuples
## (source start file, target name, title,
##  author, documentclass [howto, manual, or own class]).
#latex_documents = [
#    (master_doc, 'sphinx-infodev.tex', u'sphinx-infodev Documentation',
#     u'Intel', 'manual'),
#]

#breathe_projects = {                                       #todd-mod
#    project: "../doxygen/xml"
#}
#breathe_default_project = project

# Setup the exhale extension
#exhale_args = {                                            #todd-mod
#    # These arguments are required
#    "containmentFolder":     "./api",
#    "rootFileName":          "library_root.rst",
#    "rootFileTitle":         "Library API",
#    "doxygenStripFromPath":  "..",
#    "fullApiSubSectionTitle": 'Full API'
#}

# -- Options for manual page output ------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
     [author], 1)
]

# -- Options for Texinfo output ----------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
     author, 'sphinx-infodev', 'One line description of project.',
     'Miscellaneous'),
]

# -- Options for Epub output -------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# -- Extension configuration -------------------------------------------------

# -- Options for intersphinx extension ---------------------------------------

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}

# -- Options for todo extension ----------------------------------------------

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
<commit_msg>Update conf.py (#597)<commit_after># -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
# This file does only contain a selection of the most common options. For a
# full list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

SOURCE_DIR = os.path.dirname(__file__)
LATEX_DIR = os.path.join(SOURCE_DIR, '_latex')
PREAMBLE_FILE = os.path.join(LATEX_DIR, 'preamble.tex')
TITLE_PAGE_FILE = os.path.join(LATEX_DIR, 'title_page.tex')

BUILD_TYPE = os.getenv("BUILD_TYPE")

# -- Project information -----------------------------------------------------

if BUILD_TYPE == 'oneapi' or BUILD_TYPE == 'dita':
    project = u'Intel® oneAPI Threading Building Blocks (oneTBB)'
else:
    project = u'oneAPI Threading Building Blocks (oneTBB)'
copyright = u'2021, Intel Corporation'
author = u'Intel'

# The short X.Y version
version = u''
# The full version, including alpha/beta/rc tags
release = u''

# -- General configuration ---------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.imgmath',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinx.ext.githubpages'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
#master_doc = 'main/title_main'
master_doc = 'index'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None

# Syntax highlighting for the :: directive
highlight_language = 'cpp'

if BUILD_TYPE == 'oneapi' or BUILD_TYPE == 'dita':
    rst_prolog = """
.. |full_name| replace:: Intel\ |reg|\  oneAPI Threading Building Blocks (oneTBB)
.. |short_name| replace:: oneTBB
.. |product| replace:: oneTBB
.. |reg| unicode:: U+000AE
.. |copy| unicode:: U+000A9
.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
    """
else:
    rst_prolog = """
.. |full_name| replace:: oneAPI Threading Building Blocks (oneTBB)
.. |short_name| replace:: oneTBB
.. |product| replace:: oneTBB
.. |reg| unicode:: U+000AE
.. |copy| unicode:: U+000A9
.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
    """

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
if BUILD_TYPE == 'oneapi' or BUILD_TYPE == 'dita':
    html_theme = 'sphinx_rtd_theme'
else:
    html_theme = 'sphinx_book_theme'
    html_theme_options = {
    'repository_url': 'https://github.com/oneapi-src/oneTBB',
    'path_to_docs': 'doc/main',
    'use_issues_button': True,
    'use_edit_page_button': True,
    'repository_branch': 'master',
    'extra_footer': '
Cookies
'
    }

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

if BUILD_TYPE == 'oneapi'  or BUILD_TYPE == 'dita':
    html_context = {
        'css_files': [
            '_static/theme_overrides.css',  # override wide tables in RTD theme
        ],
    }
else:
    html_js_files = ['custom.js']
    html_logo = '_static/oneAPI-rgb-rev-100.png'
    html_favicon = '_static/favicons.png'

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# The default sidebars (for documents that don't match any pattern) are
# defined by theme itself.  Builtin themes are using these templates by
# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
# 'searchbox.html']``.
#
# html_sidebars = {}

# -- Options for HTMLHelp output ---------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'sphinx-infodevdoc'

# -- Options for LaTeX output ------------------------------------------------

#latex_engine = 'xelatex'
#PDF_TITLE = 'Information Development Template'
#
#with open(PREAMBLE_FILE, 'r', encoding='utf-8') as f:
#    PREAMBLE = f.read()
#
#with open(TITLE_PAGE_FILE, 'r', encoding='utf-8') as f:
#    TITLE_PAGE = f.read().replace('<PDF_TITLE>', PDF_TITLE)
#
#
#latex_elements = {
#    # The paper size ('letterpaper' or 'a4paper').
#    #
#    'extraclassoptions': 'openany,oneside',
#    'babel' : '\\usepackage[english]{babel}',
#    'papersize': 'a4paper',
#    'releasename':" ",
#    # Sonny, Lenny, Glenn, Conny, Rejne, Bjarne and Bjornstrup
#    # 'fncychap': '\\usepackage[Lenny]{fncychap}',
#    'fncychap':  '',
#    #'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}',
#
#    'figure_align':'htbp',
#    # The font size ('10pt', '11pt' or '12pt').
#    #
#    'pointsize': '12pt',
#
#    # Additional stuff for the LaTeX preamble.
#    #
#    'preamble': PREAMBLE,
#
#    'maketitle': TITLE_PAGE,
#    # Latex figure (float) alignment
#    #
#    # 'figure_align': 'htbp',
#    'sphinxsetup': \
#        'hmargin={0.7in,0.7in}, vmargin={1in,1in}, \
#        verbatimwithframe=true, \
#        TitleColor={rgb}{0,0.686,0.941}, \
#        HeaderFamily=\\rmfamily\\bfseries, \
#        InnerLinkColor={rgb}{0,0.686,0.941}, \
#        OuterLinkColor={rgb}{0,0.686,0.941}',
#
#    'tableofcontents':' '
#}
#
#latex_logo = '_latex/intel_logo.png'
## Grouping the document tree into LaTeX files. List of tuples
## (source start file, target name, title,
##  author, documentclass [howto, manual, or own class]).
#latex_documents = [
#    (master_doc, 'sphinx-infodev.tex', u'sphinx-infodev Documentation',
#     u'Intel', 'manual'),
#]

#breathe_projects = {                                       #todd-mod
#    project: "../doxygen/xml"
#}
#breathe_default_project = project

# Setup the exhale extension
#exhale_args = {                                            #todd-mod
#    # These arguments are required
#    "containmentFolder":     "./api",
#    "rootFileName":          "library_root.rst",
#    "rootFileTitle":         "Library API",
#    "doxygenStripFromPath":  "..",
#    "fullApiSubSectionTitle": 'Full API'
#}

# -- Options for manual page output ------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
     [author], 1)
]

# -- Options for Texinfo output ----------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
     author, 'sphinx-infodev', 'One line description of project.',
     'Miscellaneous'),
]

# -- Options for Epub output -------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# -- Extension configuration -------------------------------------------------

# -- Options for intersphinx extension ---------------------------------------

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}

# -- Options for todo extension ----------------------------------------------

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True
<|endoftext|>