<commit_before>from __future__ import absolute_import
from __future__ import print_function

from collections import namedtuple as _namedtuple

"""
Token sequences alignement and diffing based on the longest common substrings of
"high tokens". This essentially a non-optimal and reasonably fast single local
sequence alignment between two sequences of integers/token ids.

Based on and heavily modified from Python's difflib.py from the 3.X tip:
https://hg.python.org/cpython/raw-file/0a69b1e8b7fe/Lib/difflib.py

license: PSF. See seq.ABOUT file for details.
"""

Match = _namedtuple('Match', 'a b size')

def find_longest_match(a, b, alo, ahi, blo, bhi, b2j, len_junk, matchables):
    """
    Find longest matching block of a and b in a[alo:ahi] and b[blo:bhi].

    `b2j` is a mapping of b high token ids -> list of position in b
    `len_junk` is such that token ids smaller than `len_junk` are treated as junk.
    `matchables` is a set of matchable positions. Positions absent from this set are ignored.

    Return (i,j,k) tuple such that a[i:i+k] is equal to b[j:j+k], where
        alo <= i <= i+k <= ahi
        blo <= j <= j+k <= bhi
    and for all (i',j',k') matchable token positions meeting those conditions,
        k >= k'
        i <= i'
        and if i == i', j <= j'

    In other words, of all maximal matching blocks, return one that starts
    earliest in a, and of all those maximal matching blocks that start earliest
    in a, return the one that starts earliest in b.

    First the longest matching block (aka contiguous substring) is determined
    where no junk element appears in the block.  Then that block is extended as
    far as possible by matching other tokens including junk on both sides.  So
    the resulting block never matches on junk.

    If no blocks match, return (alo, blo, 0).
    """

    besti, bestj, bestsize = alo, blo, 0
    b2j_get = b2j.get

    # find longest matchable junk-free match
    # during an iteration of the loop, j2len[j] = length of longest
    # junk-free match ending with a[i-1] and b[j]
    j2len = {}
    nothing = []
    for i in range(alo, ahi):
        if not i in matchables:
            continue
        # look at all instances of a[i] in b; note that because
        # b2j has no junk token, the loop is skipped if a[i] is junk
        j2lenget = j2len.get
        newj2len = {}
        for j in b2j_get(a[i], nothing):
            # a[i] matches b[j]
            if j < blo:
                continue
            if j >= bhi:
                break
            k = newj2len[j] = j2lenget(j - 1, 0) + 1
            if k > bestsize:
                besti, bestj, bestsize = i - k + 1, j - k + 1, k
        j2len = newj2len

    # Extend the best by non-junk tokens on each end.
    while (besti > alo and bestj > blo
           and b[bestj - 1] >= len_junk
           and (besti - 1) in matchables
           and a[besti - 1] == b[bestj - 1]):
        besti, bestj, bestsize = besti - 1, bestj - 1, bestsize + 1

    while (besti + bestsize < ahi and bestj + bestsize < bhi
        and b[bestj + bestsize] >= len_junk
        and (besti + bestsize) in matchables
        and a[besti + bestsize] == b[bestj + bestsize]):
        bestsize += 1

    if bestsize:
        # if have a non-empty match, append any matching tokens on each end.
        while (besti > alo and bestj > blo
            and (besti - 1) in matchables
            and a[besti - 1] == b[bestj - 1]):
            besti, bestj, bestsize = besti - 1, bestj - 1, bestsize + 1

        while (besti + bestsize < ahi and bestj + bestsize < bhi
            and (besti + bestsize) in matchables
            and a[besti + bestsize] == b[bestj + bestsize]):
            bestsize = bestsize + 1

    return Match(besti, bestj, bestsize)

def match_blocks(a, b, starta, lena, b2j, len_junk, matchables=frozenset()):
    """
    Return list of matching block Match triples describing matching subsequences
    of a in b starting at the `starta` a position for the `lena` a length.

    `b2j` is a mapping of b high token ids -> list of position in b
    `len_junk` is such that token ids smaller than `len_junk` are treated as junk.
    `matchables` is a set of matchable positions. Positions absent from this set are ignored.

    Each triple is of the form (i, j, n), and means that a[i:i+n] == b[j:j+n].
    The triples are monotonically increasing in i and in j.  It is also
    guaranteed that adjacent triples never describe adjacent equal blocks.
    """

    # This non-recursive algorithm is using a list as a queue of blocks we still
    # need to look at and append partial results to matching_blocks in a loop.
    # The matches are sorted at the end.

    queue = [(starta, lena, 0, len(b))]
    queue_append = queue.append
    queue_pop = queue.pop
    matching_blocks = []
    matching_blocks_append = matching_blocks.append
    while queue:
        alo, ahi, blo, bhi = queue_pop()
        i, j, k = x = find_longest_match(a, b, alo, ahi, blo, bhi, b2j, len_junk, matchables)
        # a[alo:i] vs b[blo:j] unknown
        # a[i:i+k] same as b[j:j+k]
        # a[i+k:ahi] vs b[j+k:bhi] unknown
        if k:  # if k is 0, there was no matching block
            matching_blocks_append(x)
            if alo < i and blo < j:
                queue_append((alo, i, blo, j))
            if i + k < ahi and j + k < bhi:
                queue_append((i + k, ahi, j + k, bhi))

    matching_blocks.sort()

    # collapse adjacent blocks
    i1 = j1 = k1 = 0
    non_adjacent = []
    for i2, j2, k2 in matching_blocks:
        # Is this block adjacent to i1, j1, k1?
        if i1 + k1 == i2 and j1 + k1 == j2:
            # Yes, so collapse them -- this just increases the length of the first
            # block by the length of the second, and the first block so lengthened
            # remains the block to compare against.
            k1 += k2
        else:
            # Not adjacent: keep it unless this is the first block (k1==0 means it's
            # the dummy we started with), and make the second block the new block to
            # compare against.
            if k1:
                non_adjacent.append((i1, j1, k1))
            i1, j1, k1 = i2, j2, k2
    if k1:
        non_adjacent.append((i1, j1, k1))

    return map(Match._make, non_adjacent)
<commit_msg>Micro optimization to test set containment last.<commit_after>from __future__ import absolute_import
from __future__ import print_function

from collections import namedtuple as _namedtuple

"""
Token sequences alignement and diffing based on the longest common substrings of
"high tokens". This essentially a non-optimal and reasonably fast single local
sequence alignment between two sequences of integers/token ids.

Based on and heavily modified from Python's difflib.py from the 3.X tip:
https://hg.python.org/cpython/raw-file/0a69b1e8b7fe/Lib/difflib.py

license: PSF. See seq.ABOUT file for details.
"""

Match = _namedtuple('Match', 'a b size')

def find_longest_match(a, b, alo, ahi, blo, bhi, b2j, len_junk, matchables):
    """
    Find longest matching block of a and b in a[alo:ahi] and b[blo:bhi].

    `b2j` is a mapping of b high token ids -> list of position in b
    `len_junk` is such that token ids smaller than `len_junk` are treated as junk.
    `matchables` is a set of matchable positions. Positions absent from this set are ignored.

    Return (i,j,k) tuple such that a[i:i+k] is equal to b[j:j+k], where
        alo <= i <= i+k <= ahi
        blo <= j <= j+k <= bhi
    and for all (i',j',k') matchable token positions meeting those conditions,
        k >= k'
        i <= i'
        and if i == i', j <= j'

    In other words, of all maximal matching blocks, return one that starts
    earliest in a, and of all those maximal matching blocks that start earliest
    in a, return the one that starts earliest in b.

    First the longest matching block (aka contiguous substring) is determined
    where no junk element appears in the block.  Then that block is extended as
    far as possible by matching other tokens including junk on both sides.  So
    the resulting block never matches on junk.

    If no blocks match, return (alo, blo, 0).
    """

    besti, bestj, bestsize = alo, blo, 0
    b2j_get = b2j.get

    # find longest matchable junk-free match
    # during an iteration of the loop, j2len[j] = length of longest
    # junk-free match ending with a[i-1] and b[j]
    j2len = {}
    j2lenget = j2len.get
    nothing = []
    for i in range(alo, ahi):
        if not i in matchables:
            continue
        # look at all instances of a[i] in b; note that because
        # b2j has no junk token, the loop is skipped if a[i] is junk
        newj2len = {}
        for j in b2j_get(a[i], nothing):
            # a[i] matches b[j]
            if j < blo:
                continue
            if j >= bhi:
                break
            k = newj2len[j] = j2lenget(j - 1, 0) + 1
            if k > bestsize:
                besti, bestj, bestsize = i - k + 1, j - k + 1, k
        j2len = newj2len
        j2lenget = j2len.get

    # Extend the best by non-junk tokens on each end.
    while (besti > alo and bestj > blo
        and b[bestj - 1] >= len_junk
        and a[besti - 1] == b[bestj - 1]
        and (besti - 1) in matchables
        ):
        besti, bestj, bestsize = besti - 1, bestj - 1, bestsize + 1

    while (besti + bestsize < ahi and bestj + bestsize < bhi
        and b[bestj + bestsize] >= len_junk
        and a[besti + bestsize] == b[bestj + bestsize]
        and (besti + bestsize) in matchables
        ):
        bestsize += 1

    if bestsize:
        # if have a non-empty match, append any matching tokens on each end.
        while (besti > alo and bestj > blo
            and a[besti - 1] == b[bestj - 1]
            and (besti - 1) in matchables
            ):
            besti, bestj, bestsize = besti - 1, bestj - 1, bestsize + 1

        while (besti + bestsize < ahi and bestj + bestsize < bhi
            and a[besti + bestsize] == b[bestj + bestsize]
            and (besti + bestsize) in matchables
            ):
            bestsize = bestsize + 1

    return Match(besti, bestj, bestsize)

def match_blocks(a, b, starta, lena, b2j, len_junk, matchables=frozenset()):
    """
    Return list of matching block Match triples describing matching subsequences
    of a in b starting at the `starta` a position for the `lena` a length.

    `b2j` is a mapping of b high token ids -> list of position in b
    `len_junk` is such that token ids smaller than `len_junk` are treated as junk.
    `matchables` is a set of matchable positions. Positions absent from this set are ignored.

    Each triple is of the form (i, j, n), and means that a[i:i+n] == b[j:j+n].
    The triples are monotonically increasing in i and in j.  It is also
    guaranteed that adjacent triples never describe adjacent equal blocks.
    """

    # This non-recursive algorithm is using a list as a queue of blocks we still
    # need to look at and append partial results to matching_blocks in a loop.
    # The matches are sorted at the end.

    queue = [(starta, lena, 0, len(b))]
    queue_append = queue.append
    queue_pop = queue.pop
    matching_blocks = []
    matching_blocks_append = matching_blocks.append
    while queue:
        alo, ahi, blo, bhi = queue_pop()
        i, j, k = x = find_longest_match(a, b, alo, ahi, blo, bhi, b2j, len_junk, matchables)
        # a[alo:i] vs b[blo:j] unknown
        # a[i:i+k] same as b[j:j+k]
        # a[i+k:ahi] vs b[j+k:bhi] unknown
        if k:  # if k is 0, there was no matching block
            matching_blocks_append(x)
            if alo < i and blo < j:
                queue_append((alo, i, blo, j))
            if i + k < ahi and j + k < bhi:
                queue_append((i + k, ahi, j + k, bhi))

    matching_blocks.sort()

    # collapse adjacent blocks
    i1 = j1 = k1 = 0
    non_adjacent = []
    for i2, j2, k2 in matching_blocks:
        # Is this block adjacent to i1, j1, k1?
        if i1 + k1 == i2 and j1 + k1 == j2:
            # Yes, so collapse them -- this just increases the length of the first
            # block by the length of the second, and the first block so lengthened
            # remains the block to compare against.
            k1 += k2
        else:
            # Not adjacent: keep it unless this is the first block (k1==0 means it's
            # the dummy we started with), and make the second block the new block to
            # compare against.
            if k1:
                non_adjacent.append((i1, j1, k1))
            i1, j1, k1 = i2, j2, k2
    if k1:
        non_adjacent.append((i1, j1, k1))

    return map(Match._make, non_adjacent)
<|endoftext|>


<commit_before>import time

import pandas as pd
import numpy as np

import tensorflow as tf
from tensorflow.contrib.distributions import percentile

def do_clipped_factorization(
        counts_df,
        rank=3,
        clip_percentile=99.9,
        learning_rate=1.0,
        minibatch_size=1024 * 32,
        patience=5,
        max_epochs=1000,
        normalize_to_reads_per_million=True,
        log_every_seconds=10):
    """
    Implements the factorization:

    X = AB
        where X is (n x s), A is (n x r), and B is (r x s)

    by minimizing the clipped loss:

        ||minimum(X - AB, percentile(X - AB, q)||_2 + unclipped_term

    The minimum is taken elementwise, and ||M||_2 is the Frobenius norm.
    q is a parameter giving the percentile to clip at, default value 99.9. This
    truncation makes the factorization robust to outliers, which are our hits.

    If the above is optimized without an `unclipped_term`, a few phage clones
    may have all of their residuals above the truncation threshold. Once this
    happens they will likely stay stuck there since they do not contribute to
    the gradient. The `unclipped_term` fixes this by providing a small nudge
    toward smaller errors without truncation.
    """

    # Non-tf setup
    if normalize_to_reads_per_million:
        observed = (counts_df * 1e6 / counts_df.sum(0)).astype("float32")
    else:
        observed = counts_df.astype("float32")
    (n, s) = observed.shape
    if len(counts_df) < minibatch_size:
        minibatch_size = len(counts_df)

    # Placeholders
    target = tf.placeholder(
        name="target", dtype="float32", shape=[None, s])
    minibatch_indices = tf.placeholder(
        name="minibatch_indices", dtype="int32")

    # Variables
    a = tf.Variable(
        np.random.rand(n, rank), name="A", dtype="float32")
    b = tf.Variable(
        np.random.rand(rank, s), name="B", dtype="float32")
    truncate_threshold = tf.Variable(observed.max().max())

    # Derived quantities
    reconstruction = tf.matmul(
        tf.gather(a, minibatch_indices), b)
    differences = target - reconstruction

    # unclipped_term is based only on the minimum unclipped error for each
    # clone. The intuition is that we know for every clone at least one sample
    # must be a non-hit (e.g. a beads only sample), and so should be well modeled
    # by the background process.
    unclipped_term = tf.reduce_min(
        tf.pow(differences, 2), axis=1)
    loss = (
        tf.reduce_mean(
            tf.pow(tf.minimum(differences, truncate_threshold), 2))
        + tf.reduce_mean(unclipped_term) / s)

    update_truncate_value = truncate_threshold.assign(
        percentile(differences, clip_percentile))

    # Training
    train_step = tf.train.AdamOptimizer(
        learning_rate=learning_rate).minimize(loss)

    init = tf.global_variables_initializer()

    best_cost_value = None
    last_log_at = 0
    with tf.Session() as session:
        session.run(init)
        all_indices = np.arange(observed.shape[0], dtype=int)

        for i in range(max_epochs):
            indices = np.array(list(range(observed.shape[0])))
            np.random.shuffle(indices)
            for minibatch_indices_value in np.array_split(indices, int(
                            len(indices) / minibatch_size)):
                minibatch_indices_value = minibatch_indices_value[:minibatch_size]
                if len(minibatch_indices_value) == minibatch_size:
                    feed_dict = {
                        target: observed.values[minibatch_indices_value],
                        minibatch_indices: minibatch_indices_value
                    }
                    session.run(train_step, feed_dict=feed_dict)

            feed_dict = {
                target: observed,
                minibatch_indices: all_indices,
            }
            (truncate_threshold_value, cost_value) = session.run(
                [update_truncate_value, loss], feed_dict=feed_dict)

            # Update best epoch
            if best_cost_value is None or cost_value < best_cost_value:
                best_cost_value = cost_value
                best_epoch = i
                (best_a, best_b) = session.run([a, b], feed_dict=feed_dict)

            # Log
            if log_every_seconds and time.time() - last_log_at > log_every_seconds:
                print("[Epoch %5d] %f, truncating at %f%s" % (
                    i,
                    cost_value,
                    truncate_threshold_value,
                    ' [new best]' if i == best_epoch else ''))

            # Stop criterion
            if i - best_epoch > patience:
                print("Early stopping at epoch %d." % i)
                break

    background_names = ["_background_%d" % i for i in range(rank)]
    best_a = pd.DataFrame(
        best_a,
        index=observed.index,
        columns=background_names)
    best_b = pd.DataFrame(
        best_b,
        index=background_names,
        columns=observed.columns)

    results = observed - np.matmul(best_a, best_b)
    for name in background_names:
        results[name] = best_a[name]
        results.loc[name] = best_b.loc[name]

    return results

<commit_msg>Better docstring<commit_after>import time

import pandas as pd
import numpy as np

import tensorflow as tf
from tensorflow.contrib.distributions import percentile

def do_clipped_factorization(
        counts_df,
        rank=3,
        clip_percentile=99.9,
        learning_rate=1.0,
        minibatch_size=1024 * 32,
        patience=5,
        max_epochs=1000,
        normalize_to_reads_per_million=True,
        log_every_seconds=10):
    """
    Attempt to detect and correct for clone and sample batch effects by
    subtracting off a learned low-rank reconstruction of the counts matrix.

    The return value is the clones x samples matrix of residuals after
    correcting for batch effects, with a few additional rows and columns giving
    the learned background effects.

    Implements the factorization:

    X = AB
        where X is (clones x samples), A is (clones x rank), and B is
        (rank x samples)

    by minimizing the "clipped" loss:

        ||minimum(X - AB, percentile(X - AB, clip_percentile)||_2 + unclipped

    The minimum is taken elementwise, and ||...||_2 is the Frobenius norm.
    clip_percentile is a parameter giving the percentile to clip at. The
    clipping makes the factorization robust to outliers, some of which are
    likely phip-seq hits.

    If the above is optimized without an `unclipped` term, a few phage clones
    may have all of their residuals above the truncation threshold. Once this
    happens they will likely stay stuck there since they do not contribute to
    the gradient. The unclipped term fixes this by providing a small nudge
    toward smaller errors without truncation.

    Note that "beads-only" samples are not treated in any special way here.

    Parameters
    ----------
    counts_df : pandas.DataFrame
        Matrix of read counts (clones x samples)
    rank : int
        Rank of low-dimensional background effect matrices A and B
    clip_percentile : float
        Elements with reconstruction errors above this percentile do not
        contribute to the gradient. Aim for a lower-bound on the fraction
        of entries you expect NOT to be hits.
    learning_rate : float
        Stochastic gradient descent (SGD) optimizer learning rate
    minibatch_size : int
        Number of rows per SGD minibatch
    patience : int
        Number of epochs without improvement in training loss to tolerate before
        stopping
    max_epochs : int
        Maximum number of epochs
    normalize_to_reads_per_million : boolean
        Before computing factorization, first divide each column by the total
        number of reads for that sample and multiple by 1 million.
    log_every_seconds : float
        Seconds to wait before printing another optimization status update

    Returns
    -------
    pandas.DataFrame : residuals after correcting for batch effects

    In addition to the clones x samples residuals, rows and columns named
    "_background_0", "_background_1", ... giving the learned background vectors
    are also included.
    """

    # Non-tf setup
    if normalize_to_reads_per_million:
        observed = (counts_df * 1e6 / counts_df.sum(0)).astype("float32")
    else:
        observed = counts_df.astype("float32")
    (n, s) = observed.shape
    if len(counts_df) < minibatch_size:
        minibatch_size = len(counts_df)

    # Placeholders
    target = tf.placeholder(
        name="target", dtype="float32", shape=[None, s])
    minibatch_indices = tf.placeholder(
        name="minibatch_indices", dtype="int32")

    # Variables
    a = tf.Variable(
        np.random.rand(n, rank), name="A", dtype="float32")
    b = tf.Variable(
        np.random.rand(rank, s), name="B", dtype="float32")
    truncate_threshold = tf.Variable(observed.max().max())

    # Derived quantities
    reconstruction = tf.matmul(
        tf.gather(a, minibatch_indices), b)
    differences = target - reconstruction

    # unclipped_term is based only on the minimum unclipped error for each
    # clone. The intuition is that we know for every clone at least one sample
    # must be a non-hit (e.g. a beads only sample), and so should be well modeled
    # by the background process.
    unclipped_term = tf.reduce_min(
        tf.pow(differences, 2), axis=1)
    loss = (
        tf.reduce_mean(
            tf.pow(tf.minimum(differences, truncate_threshold), 2))
        + tf.reduce_mean(unclipped_term) / s)

    update_truncate_value = truncate_threshold.assign(
        percentile(differences, clip_percentile))

    # Training
    train_step = tf.train.AdamOptimizer(
        learning_rate=learning_rate).minimize(loss)

    init = tf.global_variables_initializer()

    best_cost_value = None
    last_log_at = 0
    with tf.Session() as session:
        session.run(init)
        all_indices = np.arange(observed.shape[0], dtype=int)

        for i in range(max_epochs):
            indices = np.array(list(range(observed.shape[0])))
            np.random.shuffle(indices)
            for minibatch_indices_value in np.array_split(indices, int(
                            len(indices) / minibatch_size)):
                minibatch_indices_value = minibatch_indices_value[:minibatch_size]
                if len(minibatch_indices_value) == minibatch_size:
                    feed_dict = {
                        target: observed.values[minibatch_indices_value],
                        minibatch_indices: minibatch_indices_value
                    }
                    session.run(train_step, feed_dict=feed_dict)

            feed_dict = {
                target: observed,
                minibatch_indices: all_indices,
            }
            (truncate_threshold_value, cost_value) = session.run(
                [update_truncate_value, loss], feed_dict=feed_dict)

            # Update best epoch
            if best_cost_value is None or cost_value < best_cost_value:
                best_cost_value = cost_value
                best_epoch = i
                (best_a, best_b) = session.run([a, b], feed_dict=feed_dict)

            # Log
            if log_every_seconds and time.time() - last_log_at > log_every_seconds:
                print("[Epoch %5d] %f, truncating at %f%s" % (
                    i,
                    cost_value,
                    truncate_threshold_value,
                    ' [new best]' if i == best_epoch else ''))

            # Stop criterion
            if i - best_epoch > patience:
                print("Early stopping at epoch %d." % i)
                break

    background_names = ["_background_%d" % i for i in range(rank)]
    best_a = pd.DataFrame(
        best_a,
        index=observed.index,
        columns=background_names)
    best_b = pd.DataFrame(
        best_b,
        index=background_names,
        columns=observed.columns)

    results = observed - np.matmul(best_a, best_b)
    for name in background_names:
        results[name] = best_a[name]
        results.loc[name] = best_b.loc[name]

    return results

<|endoftext|>


<commit_before>from nose.tools import eq_, ok_, raises

from flask.ext import wtf
from flask.ext.admin.contrib.sqlamodel import ModelView

from . import setup

class CustomModelView(ModelView):
    def __init__(self, model, session,
                 name=None, category=None, endpoint=None, url=None,
                 **kwargs):
        for k, v in kwargs.iteritems():
            setattr(self, k, v)

        super(CustomModelView, self).__init__(model, session,
                                              name, category,
                                              endpoint, url)

def create_models(db):
    class Model1(db.Model):
        def __init__(self, test1=None, test2=None, test3=None, test4=None):
            self.test1 = test1
            self.test2 = test2
            self.test3 = test3
            self.test4 = test4

        id = db.Column(db.Integer, primary_key=True)
        test1 = db.Column(db.String(20))
        test2 = db.Column(db.Unicode(20))
        test3 = db.Column(db.Text)
        test4 = db.Column(db.UnicodeText)

    class Model2(db.Model):
        id = db.Column(db.Integer, primary_key=True)
        int_field = db.Column(db.Integer)
        bool_field = db.Column(db.Boolean)

    db.create_all()

    return Model1, Model2

def test_model():
    app, db, admin = setup()
    Model1, Model2 = create_models(db)
    db.create_all()

    view = CustomModelView(Model1, db.session)
    admin.add_view(view)

    eq_(view.model, Model1)
    eq_(view.name, 'Model1')
    eq_(view.endpoint, 'model1view')

    eq_(view._primary_key, 'id')

    ok_('test1' in view._sortable_columns)
    ok_('test2' in view._sortable_columns)
    ok_('test3' in view._sortable_columns)
    ok_('test4' in view._sortable_columns)

    ok_(view._create_form_class is not None)
    ok_(view._edit_form_class is not None)
    eq_(view._search_supported, False)
    eq_(view._filters, None)

    # Verify form
    eq_(view._create_form_class.test1.field_class, wtf.TextField)
    eq_(view._create_form_class.test2.field_class, wtf.TextField)
    eq_(view._create_form_class.test3.field_class, wtf.TextAreaField)
    eq_(view._create_form_class.test4.field_class, wtf.TextAreaField)

    # Make some test clients
    client = app.test_client()

    rv = client.get('/admin/model1view/')
    eq_(rv.status_code, 200)

    rv = client.get('/admin/model1view/new/')
    eq_(rv.status_code, 200)

    rv = client.post('/admin/model1view/new/',
                     data=dict(test1='test1large', test2='test2'))
    eq_(rv.status_code, 302)

    model = db.session.query(Model1).first()
    eq_(model.test1, 'test1large')
    eq_(model.test2, 'test2')
    eq_(model.test3, '')
    eq_(model.test4, '')

    rv = client.get('/admin/model1view/')
    eq_(rv.status_code, 200)
    ok_('test1large' in rv.data)

    url = '/admin/model1view/edit/?id=%s' % model.id
    rv = client.get(url)
    eq_(rv.status_code, 200)

    rv = client.post(url,
                     data=dict(test1='test1small', test2='test2large'))
    eq_(rv.status_code, 302)

    model = db.session.query(Model1).first()
    eq_(model.test1, 'test1small')
    eq_(model.test2, 'test2large')
    eq_(model.test3, '')
    eq_(model.test4, '')

    url = '/admin/model1view/delete/?id=%s' % model.id
    rv = client.post(url)
    eq_(rv.status_code, 302)
    eq_(db.session.query(Model1).count(), 0)

@raises(Exception)
def test_no_pk():
    app, db, admin = setup()

    class Model(db.Model):
        test = db.Column(db.Integer)

    view = CustomModelView(Model)
    admin.add_view(view)

def test_list_columns():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           list_columns=['test1', 'test3'],
                           rename_columns=dict(test1='Column1'))
    admin.add_view(view)

    eq_(len(view._list_columns), 2)
    eq_(view._list_columns, [('test1', 'Column1'), ('test3', 'Test3')])

    client = app.test_client()

    rv = client.get('/admin/model1view/')
    ok_('Column1' in rv.data)
    ok_('Test2' not in rv.data)

def test_exclude_columns():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           excluded_list_columns=['test2', 'test4'])
    admin.add_view(view)

    eq_(view._list_columns, [('test1', 'Test1'), ('test3', 'Test3')])

    client = app.test_client()

    rv = client.get('/admin/model1view/')
    ok_('Test1' in rv.data)
    ok_('Test2' not in rv.data)

def test_searchable_columns():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           searchable_columns=['test1', 'test2'])
    admin.add_view(view)

    eq_(view._search_supported, True)
    eq_(len(view._search_fields), 2)
    ok_(isinstance(view._search_fields[0], db.Column))
    ok_(isinstance(view._search_fields[1], db.Column))
    eq_(view._search_fields[0].name, 'test1')
    eq_(view._search_fields[1].name, 'test2')

    db.session.add(Model1('model1'))
    db.session.add(Model1('model2'))
    db.session.commit()

    client = app.test_client()

    rv = client.get('/admin/model1view/?search=model1')
    ok_('model1' in rv.data)
    ok_('model2' not in rv.data)

def test_column_filters():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           column_filters=['test1'])
    admin.add_view(view)

    eq_(len(view._filters), 4)

    eq_(view._filter_dict, {'Test1': [(0, 'equals'),
                                      (1, 'not equal'),
                                      (2, 'contains'),
                                      (3, 'not contains')]})

    db.session.add(Model1('model1'))
    db.session.add(Model1('model2'))
    db.session.add(Model1('model3'))
    db.session.add(Model1('model4'))
    db.session.commit()

    client = app.test_client()

    rv = client.get('/admin/model1view/?flt0_0=model1')
    eq_(rv.status_code, 200)
    ok_('model1' in rv.data)
    ok_('model2' not in rv.data)

    rv = client.get('/admin/model1view/?flt0_5=model1')
    eq_(rv.status_code, 200)
    ok_('model1' in rv.data)
    ok_('model2' in rv.data)

    # Test different filter types
    view = CustomModelView(Model2, db.session,
                           column_filters=['int_field'])
    admin.add_view(view)

    eq_(view._filter_dict, {'Int Field': [(0, 'equals'), (1, 'not equal'),
                                          (2, 'greater than'), (3, 'smaller than')]})

def test_url_args():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           page_size=2,
                           searchable_columns=['test1'],
                           column_filters=['test1'])
    admin.add_view(view)

    db.session.add(Model1('data1'))
    db.session.add(Model1('data2'))
    db.session.add(Model1('data3'))
    db.session.add(Model1('data4'))
    db.session.commit()

    client = app.test_client()

    rv = client.get('/admin/model1view/')
    ok_('data1' in rv.data)
    ok_('data3' not in rv.data)

    # page
    rv = client.get('/admin/model1view/?page=1')
    ok_('data1' not in rv.data)
    ok_('data3' in rv.data)

    # sort
    rv = client.get('/admin/model1view/?sort=0&desc=1')
    ok_('data1' not in rv.data)
    ok_('data3' in rv.data)
    ok_('data4' in rv.data)

    # search
    rv = client.get('/admin/model1view/?search=data1')
    ok_('data1' in rv.data)
    ok_('data2' not in rv.data)

    rv = client.get('/admin/model1view/?search=^data1')
    ok_('data2' not in rv.data)

    # like
    rv = client.get('/admin/model1view/?flt0=0&flt0v=data1')
    ok_('data1' in rv.data)

    # not like
    rv = client.get('/admin/model1view/?flt0=1&flt0v=data1')
    ok_('data2' in rv.data)

def test_non_int_pk():
    app, db, admin = setup()

    class Model(db.Model):
        id = db.Column(db.String, primary_key=True)
        test = db.Column(db.String)

    db.create_all()

    view = CustomModelView(Model, db.session, form_columns=['id', 'test'])
    admin.add_view(view)

    client = app.test_client()

    rv = client.get('/admin/modelview/')
    eq_(rv.status_code, 200)

    rv = client.post('/admin/modelview/new/',
                     data=dict(id='test1', test='test2'))
    eq_(rv.status_code, 302)

    rv = client.get('/admin/modelview/')
    eq_(rv.status_code, 200)
    ok_('test1' in rv.data)

    rv = client.get('/admin/modelview/edit/?id=test1')
    eq_(rv.status_code, 200)
    ok_('test2' in rv.data)

def test_form():
    # TODO: form_columns
    # TODO: excluded_form_columns
    # TODO: form_args
    pass

def test_form_override():
    app, db, admin = setup()

    class Model(db.Model):
        id = db.Column(db.String, primary_key=True)
        test = db.Column(db.String)

    db.create_all()

    view1 = CustomModelView(Model, db.session, endpoint='view1')
    view2 = CustomModelView(Model, db.session, endpoint='view2', form_overrides=dict(test=wtf.FileField))
    admin.add_view(view1)
    admin.add_view(view2)

    eq_(view1._create_form_class.test.field_class, wtf.TextField)
    eq_(view2._create_form_class.test.field_class, wtf.FileField)

def test_relations():
    # TODO: test relations
    pass

def test_on_model_change_delete():
    app, db, admin = setup()
    Model1, _ = create_models(db)
    db.create_all()

    class ModelView(CustomModelView):
        def on_model_change(self, form, model):
            model.test1 = model.test1.upper()

        def on_model_delete(self, model):
            self.deleted = True

    view = ModelView(Model1, db.session)
    admin.add_view(view)

    client = app.test_client()

    client.post('/admin/model1view/new/',
                     data=dict(test1='test1large', test2='test2'))

    model = db.session.query(Model1).first()
    eq_(model.test1, 'TEST1LARGE')

    url = '/admin/model1view/edit/?id=%s' % model.id
    client.post(url, data=dict(test1='test1small', test2='test2large'))

    model = db.session.query(Model1).first()
    eq_(model.test1, 'TEST1SMALL')

    url = '/admin/model1view/delete/?id=%s' % model.id
    client.post(url)
    ok_(view.deleted)

<commit_msg>Add test for bulk-deletion<commit_after>from nose.tools import eq_, ok_, raises

from flask.ext import wtf
from flask.ext.admin.contrib.sqlamodel import ModelView

from . import setup

class CustomModelView(ModelView):
    def __init__(self, model, session,
                 name=None, category=None, endpoint=None, url=None,
                 **kwargs):
        for k, v in kwargs.iteritems():
            setattr(self, k, v)

        super(CustomModelView, self).__init__(model, session,
                                              name, category,
                                              endpoint, url)

def create_models(db):
    class Model1(db.Model):
        def __init__(self, test1=None, test2=None, test3=None, test4=None):
            self.test1 = test1
            self.test2 = test2
            self.test3 = test3
            self.test4 = test4

        id = db.Column(db.Integer, primary_key=True)
        test1 = db.Column(db.String(20))
        test2 = db.Column(db.Unicode(20))
        test3 = db.Column(db.Text)
        test4 = db.Column(db.UnicodeText)

    class Model2(db.Model):
        id = db.Column(db.Integer, primary_key=True)
        int_field = db.Column(db.Integer)
        bool_field = db.Column(db.Boolean)

    db.create_all()

    return Model1, Model2

def test_model():
    app, db, admin = setup()
    Model1, Model2 = create_models(db)
    db.create_all()

    view = CustomModelView(Model1, db.session)
    admin.add_view(view)

    eq_(view.model, Model1)
    eq_(view.name, 'Model1')
    eq_(view.endpoint, 'model1view')

    eq_(view._primary_key, 'id')

    ok_('test1' in view._sortable_columns)
    ok_('test2' in view._sortable_columns)
    ok_('test3' in view._sortable_columns)
    ok_('test4' in view._sortable_columns)

    ok_(view._create_form_class is not None)
    ok_(view._edit_form_class is not None)
    eq_(view._search_supported, False)
    eq_(view._filters, None)

    # Verify form
    eq_(view._create_form_class.test1.field_class, wtf.TextField)
    eq_(view._create_form_class.test2.field_class, wtf.TextField)
    eq_(view._create_form_class.test3.field_class, wtf.TextAreaField)
    eq_(view._create_form_class.test4.field_class, wtf.TextAreaField)

    # Make some test clients
    client = app.test_client()

    rv = client.get('/admin/model1view/')
    eq_(rv.status_code, 200)

    rv = client.get('/admin/model1view/new/')
    eq_(rv.status_code, 200)

    rv = client.post('/admin/model1view/new/',
                     data=dict(test1='test1large', test2='test2'))
    eq_(rv.status_code, 302)

    model = db.session.query(Model1).first()
    eq_(model.test1, 'test1large')
    eq_(model.test2, 'test2')
    eq_(model.test3, '')
    eq_(model.test4, '')

    rv = client.get('/admin/model1view/')
    eq_(rv.status_code, 200)
    ok_('test1large' in rv.data)

    url = '/admin/model1view/edit/?id=%s' % model.id
    rv = client.get(url)
    eq_(rv.status_code, 200)

    rv = client.post(url,
                     data=dict(test1='test1small', test2='test2large'))
    eq_(rv.status_code, 302)

    model = db.session.query(Model1).first()
    eq_(model.test1, 'test1small')
    eq_(model.test2, 'test2large')
    eq_(model.test3, '')
    eq_(model.test4, '')

    url = '/admin/model1view/delete/?id=%s' % model.id
    rv = client.post(url)
    eq_(rv.status_code, 302)
    eq_(db.session.query(Model1).count(), 0)

@raises(Exception)
def test_no_pk():
    app, db, admin = setup()

    class Model(db.Model):
        test = db.Column(db.Integer)

    view = CustomModelView(Model)
    admin.add_view(view)

def test_list_columns():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           list_columns=['test1', 'test3'],
                           rename_columns=dict(test1='Column1'))
    admin.add_view(view)

    eq_(len(view._list_columns), 2)
    eq_(view._list_columns, [('test1', 'Column1'), ('test3', 'Test3')])

    client = app.test_client()

    rv = client.get('/admin/model1view/')
    ok_('Column1' in rv.data)
    ok_('Test2' not in rv.data)

def test_exclude_columns():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           excluded_list_columns=['test2', 'test4'])
    admin.add_view(view)

    eq_(view._list_columns, [('test1', 'Test1'), ('test3', 'Test3')])

    client = app.test_client()

    rv = client.get('/admin/model1view/')
    ok_('Test1' in rv.data)
    ok_('Test2' not in rv.data)

def test_searchable_columns():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           searchable_columns=['test1', 'test2'])
    admin.add_view(view)

    eq_(view._search_supported, True)
    eq_(len(view._search_fields), 2)
    ok_(isinstance(view._search_fields[0], db.Column))
    ok_(isinstance(view._search_fields[1], db.Column))
    eq_(view._search_fields[0].name, 'test1')
    eq_(view._search_fields[1].name, 'test2')

    db.session.add(Model1('model1'))
    db.session.add(Model1('model2'))
    db.session.commit()

    client = app.test_client()

    rv = client.get('/admin/model1view/?search=model1')
    ok_('model1' in rv.data)
    ok_('model2' not in rv.data)

def test_column_filters():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           column_filters=['test1'])
    admin.add_view(view)

    eq_(len(view._filters), 4)

    eq_(view._filter_dict, {'Test1': [(0, 'equals'),
                                      (1, 'not equal'),
                                      (2, 'contains'),
                                      (3, 'not contains')]})

    db.session.add(Model1('model1'))
    db.session.add(Model1('model2'))
    db.session.add(Model1('model3'))
    db.session.add(Model1('model4'))
    db.session.commit()

    client = app.test_client()

    rv = client.get('/admin/model1view/?flt0_0=model1')
    eq_(rv.status_code, 200)
    ok_('model1' in rv.data)
    ok_('model2' not in rv.data)

    rv = client.get('/admin/model1view/?flt0_5=model1')
    eq_(rv.status_code, 200)
    ok_('model1' in rv.data)
    ok_('model2' in rv.data)

    # Test different filter types
    view = CustomModelView(Model2, db.session,
                           column_filters=['int_field'])
    admin.add_view(view)

    eq_(view._filter_dict, {'Int Field': [(0, 'equals'), (1, 'not equal'),
                                          (2, 'greater than'), (3, 'smaller than')]})

def test_url_args():
    app, db, admin = setup()

    Model1, Model2 = create_models(db)

    view = CustomModelView(Model1, db.session,
                           page_size=2,
                           searchable_columns=['test1'],
                           column_filters=['test1'])
    admin.add_view(view)

    db.session.add(Model1('data1'))
    db.session.add(Model1('data2'))
    db.session.add(Model1('data3'))
    db.session.add(Model1('data4'))
    db.session.commit()

    client = app.test_client()

    rv = client.get('/admin/model1view/')
    ok_('data1' in rv.data)
    ok_('data3' not in rv.data)

    # page
    rv = client.get('/admin/model1view/?page=1')
    ok_('data1' not in rv.data)
    ok_('data3' in rv.data)

    # sort
    rv = client.get('/admin/model1view/?sort=0&desc=1')
    ok_('data1' not in rv.data)
    ok_('data3' in rv.data)
    ok_('data4' in rv.data)

    # search
    rv = client.get('/admin/model1view/?search=data1')
    ok_('data1' in rv.data)
    ok_('data2' not in rv.data)

    rv = client.get('/admin/model1view/?search=^data1')
    ok_('data2' not in rv.data)

    # like
    rv = client.get('/admin/model1view/?flt0=0&flt0v=data1')
    ok_('data1' in rv.data)

    # not like
    rv = client.get('/admin/model1view/?flt0=1&flt0v=data1')
    ok_('data2' in rv.data)

def test_non_int_pk():
    app, db, admin = setup()

    class Model(db.Model):
        id = db.Column(db.String, primary_key=True)
        test = db.Column(db.String)

    db.create_all()

    view = CustomModelView(Model, db.session, form_columns=['id', 'test'])
    admin.add_view(view)

    client = app.test_client()

    rv = client.get('/admin/modelview/')
    eq_(rv.status_code, 200)

    rv = client.post('/admin/modelview/new/',
                     data=dict(id='test1', test='test2'))
    eq_(rv.status_code, 302)

    rv = client.get('/admin/modelview/')
    eq_(rv.status_code, 200)
    ok_('test1' in rv.data)

    rv = client.get('/admin/modelview/edit/?id=test1')
    eq_(rv.status_code, 200)
    ok_('test2' in rv.data)

def test_form():
    # TODO: form_columns
    # TODO: excluded_form_columns
    # TODO: form_args
    pass

def test_form_override():
    app, db, admin = setup()

    class Model(db.Model):
        id = db.Column(db.String, primary_key=True)
        test = db.Column(db.String)

    db.create_all()

    view1 = CustomModelView(Model, db.session, endpoint='view1')
    view2 = CustomModelView(Model, db.session, endpoint='view2', form_overrides=dict(test=wtf.FileField))
    admin.add_view(view1)
    admin.add_view(view2)

    eq_(view1._create_form_class.test.field_class, wtf.TextField)
    eq_(view2._create_form_class.test.field_class, wtf.FileField)

def test_relations():
    # TODO: test relations
    pass

def test_on_model_change_delete():
    app, db, admin = setup()
    Model1, _ = create_models(db)
    db.create_all()

    class ModelView(CustomModelView):
        def on_model_change(self, form, model):
            model.test1 = model.test1.upper()

        def on_model_delete(self, model):
            self.deleted = True

    view = ModelView(Model1, db.session)
    admin.add_view(view)

    client = app.test_client()

    client.post('/admin/model1view/new/',
                     data=dict(test1='test1large', test2='test2'))

    model = db.session.query(Model1).first()
    eq_(model.test1, 'TEST1LARGE')

    url = '/admin/model1view/edit/?id=%s' % model.id
    client.post(url, data=dict(test1='test1small', test2='test2large'))

    model = db.session.query(Model1).first()
    eq_(model.test1, 'TEST1SMALL')

    url = '/admin/model1view/delete/?id=%s' % model.id
    client.post(url)
    ok_(view.deleted)

def test_multiple_delete():
    app, db, admin = setup()
    M1, _ = create_models(db)

    db.session.add_all([M1('a'), M1('b'), M1('c')])
    db.session.commit()
    eq_(M1.query.count(), 3)

    view = ModelView(M1, db.session)
    admin.add_view(view)

    client = app.test_client()

    rv = client.post('/admin/model1view/action/', data=dict(action='delete', rowid=[1,2,3]))
    eq_(rv.status_code, 302)
    eq_(M1.query.count(), 0)
<|endoftext|>


<commit_before># Copyright (C) 2018 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy as np
from phonopy.harmonic.dynamical_matrix import get_dynamical_matrix
from phonopy.harmonic.dynmat_to_fc import (
    get_commensurate_points_in_integers, DynmatToForceConstants,
    categorize_commensurate_points)
from phonopy.units import VaspToTHz, THzToEv, Kb, Hbar, AMU, EV, Angstrom, THz

def bose_einstein_dist(x, t):
    return 1.0 / (np.exp(THzToEv * x / (Kb * t)) - 1)

class RandomDisplacements(object):
    """Generate random displacements by Canonical ensenmble.

    Note
    ----
    Phonon frequencies are used to calculate phonon occupation number,
    for which phonon frequencies have to be given in THz. Therefore unit
    conversion factor has to be specified at the initialization.

    Imaginary phonon modes are treated so as to have their absolute phonon
    frequencies |omega| and phonon modes having |omega| < cutoff_frequency
    are ignored.

    Attributes
    ----------
    u : ndarray
        Random atomic displacements generated by canonical distribution of
        harmonic oscillator. The unit of distance is Angstrom.
        shape=(number_of_snapshots, supercell_atoms, 3)
        dtype='double', order='C'
    qpoints : ndarray
        Commensurate q-points corresponding to the supercell matrix but not
        all. Only half of the commensurate q-points that are not on the BZ
        boundary and Gamma-points are only taken, because of the symmetry
        of dynamical matrix: omega_q = omega_-q and e_q = e_-q^*.
    frequencies : ndarray
        Phonon frequencies at commensurate q-points as explained above
        qpoints attribute. Both of getter and setter are implemented.
        The aim of this is to modify random displacements by modifying
        frequencies by users.
        shape=(len(qpoints), num_band), dtype='double', order='C'
        where num_band is 3 * number of atoms in primitive cell.
    force_constants : ndarray
        Force constants calculated from phonon frequencies and eigenvectors
        at commensurate q-points as given above qpoints attribute. By this,
        phonon can be calculated with modified phonon frequencies. To
        calculate force constants, run_d2f has to be executed. For example,

            rd = RandomDisplacements(supercell, primitive, force_constants)
            freqs = rd.frequencies
            ... modify freqs by users
            rd.frequencies = freqs
            rd.run(500)  # To get random displacements
            rd.run_d2f()
            fc = rd.force_constants  # To draw phonons with modified freqs

        shape=(superell_atoms, supercell_atoms, 3, 3)
        dtype='double', order='C'

    """

    def __init__(self,
                 supercell,
                 primitive,
                 force_constants,
                 dist_func=None,
                 cutoff_frequency=None,
                 factor=VaspToTHz):
        """

        Parameters
        ----------
        supercell : Supercell
            Supercell.
        primitive : Primitive
            Primitive cell
        force_constants : array_like
            Force constants matrix. See the details at docstring of
            DynamialMatrix.
        dist_func : str or None
            Harmonic oscillator distribution function either by 'quantum'
            or 'classical'. The starndard deviation of normal distribution
            is determined following the choice. Default is None, corresponding
            to 'quantum'.
        cutoff_frequency : float
            Lowest phonon frequency below which frequency the phonon mode
            is treated specially. See _get_sigma. Default is None, which
            means 0.01.
        factor : float
            Phonon frequency unit conversion factor to THz

        """

        if cutoff_frequency is None or cutoff_frequency < 0:
            self._cutoff_frequency = 0.01
        else:
            self._cutoff_frequency = cutoff_frequency
        self._factor = factor
        self._T = None
        self.u = None

        if dist_func is None or dist_func == 'quantum':
            self._dist_func = 'quantum'
        elif dist_func == 'classical':
            self._dist_func = 'classical'
        else:
            raise RuntimeError(
                "Either 'quantum' or 'classical' is required.")

        self._unit_conversion = (Hbar * EV / AMU / THz
                                 / (2 * np.pi) / Angstrom ** 2)
        self._unit_conversion_classical = (
            Kb * EV / AMU / (THz * (2 * np.pi)) ** 2 / Angstrom ** 2)

        # Dynamical matrix without NAC because of commensurate points only
        self._dynmat = get_dynamical_matrix(
            force_constants, supercell, primitive)

        self._setup_sampling_qpoints(supercell.cell, primitive.cell)

        s2p = primitive.s2p_map
        p2p = primitive.p2p_map
        self._s2pp = [p2p[i] for i in s2p]
        # Transformation matrix of scaled supercell positions to primitive
        self._tmat = np.dot(supercell.cell, np.linalg.inv(primitive.cell)).T

        self._eigvals_ii = []
        self._eigvecs_ii = []
        self._phase_ii = []
        self._eigvals_ij = []
        self._eigvecs_ij = []
        self._phase_ij = []
        self._prepare()

        # This is set when running run_d2f.
        # The aim is to produce force constants from modified frequencies.
        self._force_constants = None

    def run(self, T, number_of_snapshots=1, random_seed=None, randn=None):
        """

        Parameters
        ----------
        T : float
            Temperature in Kelvin.
        number_of_snapshots : int
            Number of snapshots to be generated.
        random_seed : int or None, optional
            Random seed passed to np.random.seed. Default is None. Integer
            number has to be positive.
        randn : tuple
            (randn_ii, randn_ij).
            Used for testing purpose for the fixed random numbers of
            np.random.normal that can depends on system.

        """

        np.random.seed(seed=random_seed)

        N = len(self._comm_points)

        # This randn is used only for testing purpose.
        if randn is None:
            randn_ii = None
            randn_ij = None
        else:
            randn_ii = randn[0]
            randn_ij = randn[1]

        u_ii = self._solve_ii(T, number_of_snapshots, randn=randn_ii)
        if self._ij:
            u_ij = self._solve_ij(T, number_of_snapshots, randn=randn_ij)
        else:
            u_ij = 0

        mass = self._dynmat.supercell.masses.reshape(-1, 1)
        u = np.array((u_ii + u_ij) / np.sqrt(mass * N),
                     dtype='double', order='C')
        self.u = u

    @property
    def frequencies(self):
        if self._ij:
            eigvals = np.vstack((self._eigvals_ii, self._eigvals_ij))
        else:
            eigvals = self._eigvals_ii
        freqs = np.sqrt(np.abs(eigvals)) * np.sign(eigvals) * self._factor
        return np.array(freqs, dtype='double', order='C')

    @frequencies.setter
    def frequencies(self, freqs):
        eigvals = (freqs / self._factor) ** 2
        if len(eigvals) != len(self._eigvals_ii) + len(self._eigvals_ij):
            raise RuntimeError("Dimension of frequencies is wrong.")

        self._eigvals_ii = eigvals[:len(self._eigvals_ii)]
        self._eigvals_ij = eigvals[len(self._eigvals_ii):]

    @property
    def qpoints(self):
        N = len(self._comm_points)
        return self._comm_points[self._ii + self._ij] / float(N)

    @property
    def force_constants(self):
        return self._force_constants

    def run_d2f(self):
        N = len(self._comm_points)
        if self._ij:
            eigvals = np.vstack(
                (self._eigvals_ii, self._eigvals_ij, self._eigvals_ij))
            eigvecs = np.vstack(
                (self._eigvecs_ii, self._eigvecs_ij, self._eigvecs_ij))
            eigvecs[-len(self._ij):] = eigvecs[-len(self._ij):].conj()
            qpoints = self._comm_points[self._ii + self._ij * 2] / float(N)
            qpoints[-len(self._ij):] = -qpoints[-len(self._ij):]
        else:
            eigvals = self._eigvals_ii
            eigvecs = self._eigvecs_ii
            qpoints = self._comm_points[self._ii] / float(N)

        d2f = DynmatToForceConstants(self._dynmat.primitive,
                                     self._dynmat.supercell)
        d2f.commensurate_points = qpoints
        d2f.create_dynamical_matrices(eigvals, eigvecs)
        d2f.run()
        self._force_constants = d2f.force_constants

    def _prepare(self):
        spos = np.dot(self._dynmat.supercell.scaled_positions, self._tmat.T)
        ppos = self._dynmat.primitive.scaled_positions
        lpos = spos - ppos[self._s2pp]
        N = len(self._comm_points)
        for q in self._comm_points[self._ii] / float(N):
            self._dynmat.run(q)
            dm = self._C_to_D(self._dynmat.dynamical_matrix, ppos, q)
            self._phase_ii.append(
                np.cos(2 * np.pi * np.dot(lpos, q)).reshape(-1, 1))
            eigvals, eigvecs = np.linalg.eigh(dm)
            self._eigvals_ii.append(eigvals)
            self._eigvecs_ii.append(eigvecs)

        if self._ij:
            for q in self._comm_points[self._ij] / float(N):
                self._dynmat.run(q)
                dm = self._dynmat.dynamical_matrix
                eigvals, eigvecs = np.linalg.eigh(dm)
                self._eigvals_ij.append(eigvals.real)
                self._eigvecs_ij.append(eigvecs)
                self._phase_ij.append(
                    np.exp(2j * np.pi * np.dot(spos, q)).reshape(-1, 1))

    def _C_to_D(self, dm, ppos, q):
        """Transform C-type dynamical matrix to D-type

        Taking real part is valid only when q is at Gamma or on BZ boundary,
        i.e., q=G-q and q in BZ are assumed.

        D(q) = (D(q) + D(G-q)) / 2 -> real matrix.

        """

        V = np.repeat(2j * np.pi * np.dot(ppos, q), 3)
        dm = ((V * (V.conj() * dm).T).T).real  # C-type to D-type
        return dm

    def _setup_sampling_qpoints(self, slat, plat):
        smat = np.rint(np.dot(slat, np.linalg.inv(plat)).T).astype(int)
        self._comm_points = get_commensurate_points_in_integers(smat)
        self._ii, self._ij = categorize_commensurate_points(self._comm_points)

    def _solve_ii(self, T, number_of_snapshots, randn=None):
        """

        randn parameter is used for the test.

        """
        natom = len(self._dynmat.supercell)
        u = np.zeros((number_of_snapshots, natom, 3), dtype='double')

        shape = (len(self._eigvals_ii), number_of_snapshots,
                 len(self._eigvals_ii[0]))
        if randn is None:
            _randn = np.random.normal(size=shape)
        else:
            _randn = randn
        sigmas = self._get_sigma(self._eigvals_ii, T)
        for norm_dist, sigma, eigvecs, phase in zip(
                _randn, sigmas, self._eigvecs_ii, self._phase_ii):
            u_red = np.dot(norm_dist * sigma, eigvecs.T).reshape(
                number_of_snapshots, -1, 3)[:, self._s2pp, :]
            u += u_red * phase

        return u

    def _solve_ij(self, T, number_of_snapshots, randn=None):
        """

        randn parameter is used for the test.

        """
        natom = len(self._dynmat.supercell)
        u = np.zeros((number_of_snapshots, natom, 3), dtype='double')
        shape = (len(self._eigvals_ij), 2, number_of_snapshots,
                 len(self._eigvals_ij[0]))
        if randn is None:
            _randn = np.random.normal(size=shape)
        else:
            _randn = randn
        sigmas = self._get_sigma(self._eigvals_ij, T)
        for norm_dist, sigma, eigvecs, phase in zip(
                _randn, sigmas, self._eigvecs_ij, self._phase_ij):
            u_red = np.dot(norm_dist * sigma, eigvecs.T).reshape(
                2, number_of_snapshots, -1, 3)[:, :, self._s2pp, :]
            u += (u_red[0] * phase).real
            u -= (u_red[1] * phase).imag

        return u * np.sqrt(2)

    def _get_sigma(self, eigvals, T):
        """Returns sigma in sqrt(AMU).Angstrom unit"""
        freqs = np.sqrt(np.abs(eigvals)) * self._factor
        conditions = freqs > self._cutoff_frequency
        freqs = np.where(conditions, freqs, 1)
        if self._dist_func == 'classical':
            sigma = np.where(
                conditions,
                np.sqrt(T * self._unit_conversion_classical) / freqs,
                0)
        else:
            n = np.where(conditions, bose_einstein_dist(freqs, T), 0)
            sigma = np.where(
                conditions,
                np.sqrt(self._unit_conversion / freqs * (0.5 + n)),
                0)
        return sigma
<commit_msg>Still bug fix of finite temperature random displacements<commit_after># Copyright (C) 2018 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import numpy as np
from phonopy.harmonic.dynamical_matrix import get_dynamical_matrix
from phonopy.harmonic.dynmat_to_fc import (
    get_commensurate_points_in_integers, DynmatToForceConstants,
    categorize_commensurate_points)
from phonopy.units import VaspToTHz, THzToEv, Kb, Hbar, AMU, EV, Angstrom, THz

def bose_einstein_dist(x, t):
    return 1.0 / (np.exp(THzToEv * x / (Kb * t)) - 1)

class RandomDisplacements(object):
    """Generate random displacements by Canonical ensenmble.

    Note
    ----
    Phonon frequencies are used to calculate phonon occupation number,
    for which phonon frequencies have to be given in THz. Therefore unit
    conversion factor has to be specified at the initialization.

    Imaginary phonon modes are treated so as to have their absolute phonon
    frequencies |omega| and phonon modes having |omega| < cutoff_frequency
    are ignored.

    Attributes
    ----------
    u : ndarray
        Random atomic displacements generated by canonical distribution of
        harmonic oscillator. The unit of distance is Angstrom.
        shape=(number_of_snapshots, supercell_atoms, 3)
        dtype='double', order='C'
    qpoints : ndarray
        Commensurate q-points corresponding to the supercell matrix but not
        all. Only half of the commensurate q-points that are not on the BZ
        boundary and Gamma-points are only taken, because of the symmetry
        of dynamical matrix: omega_q = omega_-q and e_q = e_-q^*.
    frequencies : ndarray
        Phonon frequencies at commensurate q-points as explained above
        qpoints attribute. Both of getter and setter are implemented.
        The aim of this is to modify random displacements by modifying
        frequencies by users.
        shape=(len(qpoints), num_band), dtype='double', order='C'
        where num_band is 3 * number of atoms in primitive cell.
    force_constants : ndarray
        Force constants calculated from phonon frequencies and eigenvectors
        at commensurate q-points as given above qpoints attribute. By this,
        phonon can be calculated with modified phonon frequencies. To
        calculate force constants, run_d2f has to be executed. For example,

            rd = RandomDisplacements(supercell, primitive, force_constants)
            freqs = rd.frequencies
            ... modify freqs by users
            rd.frequencies = freqs
            rd.run(500)  # To get random displacements
            rd.run_d2f()
            fc = rd.force_constants  # To draw phonons with modified freqs

        shape=(superell_atoms, supercell_atoms, 3, 3)
        dtype='double', order='C'

    """

    def __init__(self,
                 supercell,
                 primitive,
                 force_constants,
                 dist_func=None,
                 cutoff_frequency=None,
                 factor=VaspToTHz):
        """

        Parameters
        ----------
        supercell : Supercell
            Supercell.
        primitive : Primitive
            Primitive cell
        force_constants : array_like
            Force constants matrix. See the details at docstring of
            DynamialMatrix.
        dist_func : str or None
            Harmonic oscillator distribution function either by 'quantum'
            or 'classical'. The starndard deviation of normal distribution
            is determined following the choice. Default is None, corresponding
            to 'quantum'.
        cutoff_frequency : float
            Lowest phonon frequency below which frequency the phonon mode
            is treated specially. See _get_sigma. Default is None, which
            means 0.01.
        factor : float
            Phonon frequency unit conversion factor to THz

        """

        if cutoff_frequency is None or cutoff_frequency < 0:
            self._cutoff_frequency = 0.01
        else:
            self._cutoff_frequency = cutoff_frequency
        self._factor = factor
        self._T = None
        self.u = None

        if dist_func is None or dist_func == 'quantum':
            self._dist_func = 'quantum'
        elif dist_func == 'classical':
            self._dist_func = 'classical'
        else:
            raise RuntimeError(
                "Either 'quantum' or 'classical' is required.")

        self._unit_conversion = (Hbar * EV / AMU / THz
                                 / (2 * np.pi) / Angstrom ** 2)
        self._unit_conversion_classical = (
            Kb * EV / AMU / (THz * (2 * np.pi)) ** 2 / Angstrom ** 2)

        # Dynamical matrix without NAC because of commensurate points only
        self._dynmat = get_dynamical_matrix(
            force_constants, supercell, primitive)

        self._setup_sampling_qpoints(supercell.cell, primitive.cell)

        s2p = primitive.s2p_map
        p2p = primitive.p2p_map
        self._s2pp = [p2p[i] for i in s2p]
        # Transformation matrix of scaled supercell positions to primitive
        self._tmat = np.dot(supercell.cell, np.linalg.inv(primitive.cell)).T

        self._eigvals_ii = []
        self._eigvecs_ii = []
        self._phase_ii = []
        self._eigvals_ij = []
        self._eigvecs_ij = []
        self._phase_ij = []
        self._prepare()

        # This is set when running run_d2f.
        # The aim is to produce force constants from modified frequencies.
        self._force_constants = None

    def run(self, T, number_of_snapshots=1, random_seed=None, randn=None):
        """

        Parameters
        ----------
        T : float
            Temperature in Kelvin.
        number_of_snapshots : int
            Number of snapshots to be generated.
        random_seed : int or None, optional
            Random seed passed to np.random.seed. Default is None. Integer
            number has to be positive.
        randn : tuple
            (randn_ii, randn_ij).
            Used for testing purpose for the fixed random numbers of
            np.random.normal that can depends on system.

        """

        np.random.seed(seed=random_seed)

        N = len(self._comm_points)

        # This randn is used only for testing purpose.
        if randn is None:
            randn_ii = None
            randn_ij = None
        else:
            randn_ii = randn[0]
            randn_ij = randn[1]

        u_ii = self._solve_ii(T, number_of_snapshots, randn=randn_ii)
        if self._ij:
            u_ij = self._solve_ij(T, number_of_snapshots, randn=randn_ij)
        else:
            u_ij = 0

        mass = self._dynmat.supercell.masses.reshape(-1, 1)
        u = np.array((u_ii + u_ij) / np.sqrt(mass * N),
                     dtype='double', order='C')
        self.u = u

    @property
    def frequencies(self):
        if self._ij:
            eigvals = np.vstack((self._eigvals_ii, self._eigvals_ij))
        else:
            eigvals = self._eigvals_ii
        freqs = np.sqrt(np.abs(eigvals)) * np.sign(eigvals) * self._factor
        return np.array(freqs, dtype='double', order='C')

    @frequencies.setter
    def frequencies(self, freqs):
        eigvals = (freqs / self._factor) ** 2
        if len(eigvals) != len(self._eigvals_ii) + len(self._eigvals_ij):
            raise RuntimeError("Dimension of frequencies is wrong.")

        self._eigvals_ii = eigvals[:len(self._eigvals_ii)]
        self._eigvals_ij = eigvals[len(self._eigvals_ii):]

    @property
    def qpoints(self):
        N = len(self._comm_points)
        return self._comm_points[self._ii + self._ij] / float(N)

    @property
    def force_constants(self):
        return self._force_constants

    def run_d2f(self):
        N = len(self._comm_points)
        if self._ij:
            eigvals = np.vstack(
                (self._eigvals_ii, self._eigvals_ij, self._eigvals_ij))
            eigvecs = np.vstack(
                (self._eigvecs_ii, self._eigvecs_ij, self._eigvecs_ij))
            eigvecs[-len(self._ij):] = eigvecs[-len(self._ij):].conj()
            qpoints = self._comm_points[self._ii + self._ij * 2] / float(N)
            qpoints[-len(self._ij):] = -qpoints[-len(self._ij):]
        else:
            eigvals = self._eigvals_ii
            eigvecs = self._eigvecs_ii
            qpoints = self._comm_points[self._ii] / float(N)

        d2f = DynmatToForceConstants(self._dynmat.primitive,
                                     self._dynmat.supercell)
        d2f.commensurate_points = qpoints
        d2f.create_dynamical_matrices(eigvals, eigvecs)
        d2f.run()
        self._force_constants = d2f.force_constants

    def _prepare(self):
        spos = np.dot(self._dynmat.supercell.scaled_positions, self._tmat.T)
        ppos = self._dynmat.primitive.scaled_positions
        lpos = spos - ppos[self._s2pp]
        N = len(self._comm_points)
        for q in self._comm_points[self._ii] / float(N):
            self._dynmat.run(q)
            dm = self._C_to_D(self._dynmat.dynamical_matrix, ppos, q)
            self._phase_ii.append(
                np.cos(2 * np.pi * np.dot(lpos, q)).reshape(-1, 1))
            eigvals, eigvecs = np.linalg.eigh(dm)
            self._eigvals_ii.append(eigvals)
            self._eigvecs_ii.append(eigvecs)

        if self._ij:
            for q in self._comm_points[self._ij] / float(N):
                self._dynmat.run(q)
                dm = self._dynmat.dynamical_matrix
                eigvals, eigvecs = np.linalg.eigh(dm)
                self._eigvals_ij.append(eigvals.real)
                self._eigvecs_ij.append(eigvecs)
                self._phase_ij.append(
                    np.exp(2j * np.pi * np.dot(spos, q)).reshape(-1, 1))

    def _C_to_D(self, dm, ppos, q):
        """Transform C-type dynamical matrix to D-type

        Taking real part is valid only when q is at Gamma or on BZ boundary,
        i.e., q=G-q and q in BZ are assumed.

        D(q) = (D(q) + D(G-q)) / 2 -> real matrix.

        """

        V = np.repeat(np.exp(2j * np.pi * np.dot(ppos, q)), 3)
        dm = ((V * (V.conj() * dm).T).T).real  # C-type to D-type
        return dm

    def _setup_sampling_qpoints(self, slat, plat):
        smat = np.rint(np.dot(slat, np.linalg.inv(plat)).T).astype(int)
        self._comm_points = get_commensurate_points_in_integers(smat)
        self._ii, self._ij = categorize_commensurate_points(self._comm_points)

    def _solve_ii(self, T, number_of_snapshots, randn=None):
        """

        randn parameter is used for the test.

        """
        natom = len(self._dynmat.supercell)
        u = np.zeros((number_of_snapshots, natom, 3), dtype='double')

        shape = (len(self._eigvals_ii), number_of_snapshots,
                 len(self._eigvals_ii[0]))
        if randn is None:
            _randn = np.random.normal(size=shape)
        else:
            _randn = randn
        sigmas = self._get_sigma(self._eigvals_ii, T)
        for norm_dist, sigma, eigvecs, phase in zip(
                _randn, sigmas, self._eigvecs_ii, self._phase_ii):
            u_red = np.dot(norm_dist * sigma, eigvecs.T).reshape(
                number_of_snapshots, -1, 3)[:, self._s2pp, :]
            u += u_red * phase

        return u

    def _solve_ij(self, T, number_of_snapshots, randn=None):
        """

        randn parameter is used for the test.

        """
        natom = len(self._dynmat.supercell)
        u = np.zeros((number_of_snapshots, natom, 3), dtype='double')
        shape = (len(self._eigvals_ij), 2, number_of_snapshots,
                 len(self._eigvals_ij[0]))
        if randn is None:
            _randn = np.random.normal(size=shape)
        else:
            _randn = randn
        sigmas = self._get_sigma(self._eigvals_ij, T)
        for norm_dist, sigma, eigvecs, phase in zip(
                _randn, sigmas, self._eigvecs_ij, self._phase_ij):
            u_red = np.dot(norm_dist * sigma, eigvecs.T).reshape(
                2, number_of_snapshots, -1, 3)[:, :, self._s2pp, :]
            u += (u_red[0] * phase).real
            u -= (u_red[1] * phase).imag

        return u * np.sqrt(2)

    def _get_sigma(self, eigvals, T):
        """Returns sigma in sqrt(AMU).Angstrom unit"""
        freqs = np.sqrt(np.abs(eigvals)) * self._factor
        conditions = freqs > self._cutoff_frequency
        freqs = np.where(conditions, freqs, 1)
        if self._dist_func == 'classical':
            sigma = np.where(
                conditions,
                np.sqrt(T * self._unit_conversion_classical) / freqs,
                0)
        else:
            n = np.where(conditions, bose_einstein_dist(freqs, T), 0)
            sigma = np.where(
                conditions,
                np.sqrt(self._unit_conversion / freqs * (0.5 + n)),
                0)
        return sigma
<|endoftext|>


<commit_before>import shlex
import logging
from pi_director.models.models import (
    DBSession,
    RasPi,
    Tags
    )
from sqlalchemy import desc, and_

def get_pis():
    PiList=DBSession.query(RasPi).filter(RasPi.uuid!="default").order_by(desc(RasPi.lastseen)).all()
    return PiList

def get_tagged_pis(tags):
    splitter = shlex.shlex(tags)
    splitter.whitespace += ','
    splitter.whitespace += ';'
    splitter.whitespace_split = True
    taglist = list(splitter)

    taglist_str = ""
    for tag in taglist:
        taglist_str += "\'{tag}\',".format(tag=tag)
    taglist_str = taglist_str.rstrip(taglist_str[-1:])

    tagged_pis = []

    if (len(taglist) > 1):
        query_str = """
    SELECT t1.uuid,count(*) AS ct
      FROM Tags t1
      JOIN Tags t2 ON t1.tag!=t2.tag AND t1.uuid==t2.uuid
     WHERE 1
       AND t1.tag IN ({taglist})
       AND t2.tag IN ({taglist})
  GROUP BY t1.uuid
    HAVING ct >= {args}
        """.format(taglist=taglist_str, args=len(taglist))

        result = DBSession.get_bind().execute(query_str)

        for row in result:
            tagged_pis.append(row[0])
    else:
        PisWithTags = DBSession.query(Tags).filter(Tags.tag.in_(taglist)).all()
        for pi in PisWithTags:
            tagged_pis.append(pi.uuid)

    PiList = DBSession.query(MyModel).filter(MyModel.uuid.in_(tagged_pis)).order_by(desc(MyModel.lastseen)).all()
    return PiList

def get_pi_info(uid):
    tags = []
    row = DBSession.query(RasPi).filter(RasPi.uuid==uid).first()
    if row is None:
        row = RasPi()
        row.uuid = uid
        row.url = "http://www.stackexchange.com"
        row.landscape = True
        row.lastseen = datetime.now()
        row.description = ""
        DBSession.add(row)
        DBSession.flush()
    else:
        try:
            tagset = DBSession.query(Tags).filter(Tags.uuid==uid).all()
            for tag in tagset:
                tags.append(tag.tag)
        except Exception:
            pass

    rowdict = row.get_dict()
    return rowdict

def get_pi_cmd_info(uid):
    tags = []
    row = DBSession.query(RasPi).filter(RasPi.uuid == uid).first()
    if row is None:
        return []
    else:
        try:
            tagset = DBSession.query(Tags).filter(Tags.uuid==uid).all()
            for tag in tagset:
                tags.append(tag.tag)
        except Exception:
            pass

    rowdict = row.get_dict()
    return rowdict
<commit_msg>add tags to return of get_pi_info and get_pi_cmd_info<commit_after>import shlex
import logging
from pi_director.models.models import (
    DBSession,
    RasPi,
    Tags
    )
from sqlalchemy import desc, and_

def get_pis():
    PiList=DBSession.query(RasPi).filter(RasPi.uuid!="default").order_by(desc(RasPi.lastseen)).all()
    return PiList

def get_tagged_pis(tags):
    splitter = shlex.shlex(tags)
    splitter.whitespace += ','
    splitter.whitespace += ';'
    splitter.whitespace_split = True
    taglist = list(splitter)

    taglist_str = ""
    for tag in taglist:
        taglist_str += "\'{tag}\',".format(tag=tag)
    taglist_str = taglist_str.rstrip(taglist_str[-1:])

    tagged_pis = []

    if (len(taglist) > 1):
        query_str = """
    SELECT t1.uuid,count(*) AS ct
      FROM Tags t1
      JOIN Tags t2 ON t1.tag!=t2.tag AND t1.uuid==t2.uuid
     WHERE 1
       AND t1.tag IN ({taglist})
       AND t2.tag IN ({taglist})
  GROUP BY t1.uuid
    HAVING ct >= {args}
        """.format(taglist=taglist_str, args=len(taglist))

        result = DBSession.get_bind().execute(query_str)

        for row in result:
            tagged_pis.append(row[0])
    else:
        PisWithTags = DBSession.query(Tags).filter(Tags.tag.in_(taglist)).all()
        for pi in PisWithTags:
            tagged_pis.append(pi.uuid)

    PiList = DBSession.query(MyModel).filter(MyModel.uuid.in_(tagged_pis)).order_by(desc(MyModel.lastseen)).all()
    return PiList

def get_pi_info(uid):
    tags = []
    row = DBSession.query(RasPi).filter(RasPi.uuid==uid).first()
    if row is None:
        row = RasPi()
        row.uuid = uid
        row.url = "http://www.stackexchange.com"
        row.landscape = True
        row.lastseen = datetime.now()
        row.description = ""
        DBSession.add(row)
        DBSession.flush()
    else:
        try:
            tagset = DBSession.query(Tags).filter(Tags.uuid==uid).all()
            for tag in tagset:
                tags.append(tag.tag)
        except Exception:
            pass

    rowdict = row.get_dict()
    rowdict['tags']=tags
    return rowdict

def get_pi_cmd_info(uid):
    tags = []
    row = DBSession.query(RasPi).filter(RasPi.uuid == uid).first()
    if row is None:
        return []
    else:
        try:
            tagset = DBSession.query(Tags).filter(Tags.uuid==uid).all()
            for tag in tagset:
                tags.append(tag.tag)
        except Exception:
            pass

    rowdict = row.get_dict()
    rowdict['tags']=tags
    return rowdict
<|endoftext|>


<commit_before>import pytest
from snovault.schema_utils import load_schema
pytestmark = [pytest.mark.setone, pytest.mark.working, pytest.mark.schema]

@pytest.fixture
def biosample_cc_w_diff(testapp, de_term, lab, award):
    item = {
        "culture_start_date": "2018-01-01",
        "differentiation_state": "Differentiated to definitive endoderm demonstrated by decreased Oct4 expression and increased Sox17 expression",
        "differentiation_tissue": de_term['@id'],
        "in_vitro_differentiated": "Yes",
        'award': award['@id'],
        'lab': lab['@id']
    }
    return testapp.post_json('/biosample_cell_culture', item).json['@graph'][0]

@pytest.fixture
def biosample_1(testapp, human_biosource, lab, award):
    item = {
        'description': "GM12878 prepared for Hi-C",
        'biosource': [human_biosource['@id'], ],
        'award': award['@id'],
        'lab': lab['@id'],
    }
    return testapp.post_json('/biosample', item).json['@graph'][0]

@pytest.fixture
def biosample_w_mod(testapp, biosample_1, mod_w_target):
    return testapp.patch_json(biosample_1['@id'], {'modifications': [mod_w_target['@id']]}).json['@graph'][0]

@pytest.fixture
def biosample_w_treatment(testapp, biosample_1, rnai):
    return testapp.patch_json(biosample_1['@id'], {'treatments': [rnai['@id']]}).json['@graph'][0]

@pytest.fixture
def biosample_relation(derived_from):
    return {"biosample_relation": [{"relationship_type": "derived from",
            "biosample": derived_from['@id']}]}

def test_biosample_has_display_title(testapp, biosample_1):
    # accession fallback used for display title here
    assert biosample_1['display_title'] == biosample_1['accession']

# data from test/datafixtures
def test_update_biosample_relation(testapp, human_biosample, biosample_1):
    patch_res = testapp.patch_json(human_biosample['@id'], biosample_relation(biosample_1))
    res = testapp.get(biosample_1['@id'])
    # expected relation: 'biosample': human_biosample['@id'],
    #                    'relationship_type': 'parent of'
    assert res.json['biosample_relation'][0]['biosample']['@id'] == human_biosample['@id']
    assert res.json['biosample_relation'][0]['relationship_type'] == 'parent of'

def test_biosample_calculated_properties(testapp, biosample_1, ):
    """
    Test to ensure the calculated properties are in result returned from testapp
    """
    res = testapp.get(biosample_1['@id']).json
    assert 'modifications_summary' in res
    assert 'modifications_summary_short' in res
    assert 'treatments_summary' in res
    assert 'biosource_summary' in res

def test_biosample_biosource_summary_one_biosource(testapp, biosample_1, human_biosource):
    assert biosample_1['biosource_summary'] == human_biosource['biosource_name']

def test_biosample_biosource_summary_two_biosource(testapp, biosample_1, human_biosource, lung_biosource):
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [human_biosource['@id'], lung_biosource['@id']]}).json['@graph'][0]
    assert human_biosource['biosource_name'] in res['biosource_summary']
    assert lung_biosource['biosource_name'] in res['biosource_summary']
    assert ' and ' in res['biosource_summary']

def test_biosample_biosource_summary_w_differentiation(testapp, biosample_1, human_biosource, biosample_cc_w_diff, de_term):
    res = testapp.patch_json(biosample_1['@id'], {'cell_culture_details': [biosample_cc_w_diff['@id']]}).json['@graph'][0]
    assert human_biosource['biosource_name'] in res['biosource_summary']
    assert ' differentiated to ' in res['biosource_summary']
    assert de_term['display_title'] in res['biosource_summary']

def test_biosample_sample_type_w_differentiation(testapp, biosample_1, biosample_cc_w_diff):
    res = testapp.patch_json(biosample_1['@id'], {'cell_culture_details': [biosample_cc_w_diff['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'in vitro differentiated cells'

def test_biosample_sample_type_immortalized_wo_differentiation(testapp, biosample_1, biosample_cc_wo_diff):
    res = testapp.patch_json(biosample_1['@id'], {'cell_culture_details': [biosample_cc_wo_diff['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'immortalized cells'

def test_biosample_sample_type_bs_stem_cell_line(testapp, biosample_1, human_biosource):
    bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': 'stem cell derived cell line'}).json['@graph'][0]
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'stem cells'

def test_biosample_sample_type_bs_multicellular(testapp, biosample_1, human_biosource):
    bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': 'multicellular organism'}).json['@graph'][0]
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'whole organisms'

def test_biosample_sample_type_bs_tissue(testapp, biosample_1, human_biosource):
    bty = 'tissue'
    bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': bty}).json['@graph'][0]
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == bty

def test_biosample_sample_type_bs_lines_and_to_pluralize(testapp, biosample_1, human_biosource):
    types = {
        "primary cell": "primary cells",
        "primary cell line": "primary cells",
        "immortalized cell line": "immortalized cells",
        "stem cell": "stem cells",
        "induced pluripotent stem cell": "induced pluripotent stem cells"
    }
    for bty, bsty in types.items():
        bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': bty}).json['@graph'][0]
        res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
        assert res['biosample_type'] == bsty

def test_biosample_sample_type_bs_multiple_same_type(testapp, biosample_1, human_biosource, GM12878_biosource):
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [human_biosource['@id'], GM12878_biosource['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'immortalized cells'

def test_biosample_sample_type_bs_multiple_diff_types(testapp, biosample_1, human_biosource, lung_biosource):
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [human_biosource['@id'], lung_biosource['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'mixed sample'

def test_biosample_modifications_summaries(biosample_w_mod):
    assert biosample_w_mod['modifications_summary'] == 'Crispr for RAD21 gene'
    assert biosample_w_mod['modifications_summary_short'] == 'RAD21 Crispr'

def test_biosample_modifications_summaries_no_mods(biosample_1):
    assert biosample_1.get('modifications_summary') == 'None'
    assert biosample_1.get('modifications_summary_short') == 'None'

def test_biosample_treatments_summary(biosample_w_treatment):
    assert biosample_w_treatment.get('treatments_summary') == 'shRNA treatment'

def test_biosample_treatments_summary_no_treatment(biosample_1):
    assert biosample_1.get('treatments_summary') == 'None'
<commit_msg>fixed biosample test fixture with differentiated cell culture<commit_after>import pytest
from snovault.schema_utils import load_schema
pytestmark = [pytest.mark.setone, pytest.mark.working, pytest.mark.schema]

@pytest.fixture
def biosample_cc_w_diff(testapp, de_term, lab, award):
    item = {
        "culture_start_date": "2018-01-01",
        "differentiation_state": "Differentiated to definitive endoderm demonstrated by decreased Oct4 expression and increased Sox17 expression",
        "tissue": de_term['@id'],
        "in_vitro_differentiated": "Yes",
        'award': award['@id'],
        'lab': lab['@id']
    }
    return testapp.post_json('/biosample_cell_culture', item).json['@graph'][0]

@pytest.fixture
def biosample_1(testapp, human_biosource, lab, award):
    item = {
        'description': "GM12878 prepared for Hi-C",
        'biosource': [human_biosource['@id'], ],
        'award': award['@id'],
        'lab': lab['@id'],
    }
    return testapp.post_json('/biosample', item).json['@graph'][0]

@pytest.fixture
def biosample_w_mod(testapp, biosample_1, mod_w_target):
    return testapp.patch_json(biosample_1['@id'], {'modifications': [mod_w_target['@id']]}).json['@graph'][0]

@pytest.fixture
def biosample_w_treatment(testapp, biosample_1, rnai):
    return testapp.patch_json(biosample_1['@id'], {'treatments': [rnai['@id']]}).json['@graph'][0]

@pytest.fixture
def biosample_relation(derived_from):
    return {"biosample_relation": [{"relationship_type": "derived from",
            "biosample": derived_from['@id']}]}

def test_biosample_has_display_title(testapp, biosample_1):
    # accession fallback used for display title here
    assert biosample_1['display_title'] == biosample_1['accession']

# data from test/datafixtures
def test_update_biosample_relation(testapp, human_biosample, biosample_1):
    patch_res = testapp.patch_json(human_biosample['@id'], biosample_relation(biosample_1))
    res = testapp.get(biosample_1['@id'])
    # expected relation: 'biosample': human_biosample['@id'],
    #                    'relationship_type': 'parent of'
    assert res.json['biosample_relation'][0]['biosample']['@id'] == human_biosample['@id']
    assert res.json['biosample_relation'][0]['relationship_type'] == 'parent of'

def test_biosample_calculated_properties(testapp, biosample_1, ):
    """
    Test to ensure the calculated properties are in result returned from testapp
    """
    res = testapp.get(biosample_1['@id']).json
    assert 'modifications_summary' in res
    assert 'modifications_summary_short' in res
    assert 'treatments_summary' in res
    assert 'biosource_summary' in res

def test_biosample_biosource_summary_one_biosource(testapp, biosample_1, human_biosource):
    assert biosample_1['biosource_summary'] == human_biosource['biosource_name']

def test_biosample_biosource_summary_two_biosource(testapp, biosample_1, human_biosource, lung_biosource):
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [human_biosource['@id'], lung_biosource['@id']]}).json['@graph'][0]
    assert human_biosource['biosource_name'] in res['biosource_summary']
    assert lung_biosource['biosource_name'] in res['biosource_summary']
    assert ' and ' in res['biosource_summary']

def test_biosample_biosource_summary_w_differentiation(testapp, biosample_1, human_biosource, biosample_cc_w_diff, de_term):
    res = testapp.patch_json(biosample_1['@id'], {'cell_culture_details': [biosample_cc_w_diff['@id']]}).json['@graph'][0]
    assert human_biosource['biosource_name'] in res['biosource_summary']
    assert ' differentiated to ' in res['biosource_summary']
    assert de_term['display_title'] in res['biosource_summary']

def test_biosample_sample_type_w_differentiation(testapp, biosample_1, biosample_cc_w_diff):
    res = testapp.patch_json(biosample_1['@id'], {'cell_culture_details': [biosample_cc_w_diff['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'in vitro differentiated cells'

def test_biosample_sample_type_immortalized_wo_differentiation(testapp, biosample_1, biosample_cc_wo_diff):
    res = testapp.patch_json(biosample_1['@id'], {'cell_culture_details': [biosample_cc_wo_diff['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'immortalized cells'

def test_biosample_sample_type_bs_stem_cell_line(testapp, biosample_1, human_biosource):
    bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': 'stem cell derived cell line'}).json['@graph'][0]
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'stem cells'

def test_biosample_sample_type_bs_multicellular(testapp, biosample_1, human_biosource):
    bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': 'multicellular organism'}).json['@graph'][0]
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'whole organisms'

def test_biosample_sample_type_bs_tissue(testapp, biosample_1, human_biosource):
    bty = 'tissue'
    bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': bty}).json['@graph'][0]
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == bty

def test_biosample_sample_type_bs_lines_and_to_pluralize(testapp, biosample_1, human_biosource):
    types = {
        "primary cell": "primary cells",
        "primary cell line": "primary cells",
        "immortalized cell line": "immortalized cells",
        "stem cell": "stem cells",
        "induced pluripotent stem cell": "induced pluripotent stem cells"
    }
    for bty, bsty in types.items():
        bsres = testapp.patch_json(human_biosource['@id'], {'biosource_type': bty}).json['@graph'][0]
        res = testapp.patch_json(biosample_1['@id'], {'biosource': [bsres['@id']]}).json['@graph'][0]
        assert res['biosample_type'] == bsty

def test_biosample_sample_type_bs_multiple_same_type(testapp, biosample_1, human_biosource, GM12878_biosource):
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [human_biosource['@id'], GM12878_biosource['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'immortalized cells'

def test_biosample_sample_type_bs_multiple_diff_types(testapp, biosample_1, human_biosource, lung_biosource):
    res = testapp.patch_json(biosample_1['@id'], {'biosource': [human_biosource['@id'], lung_biosource['@id']]}).json['@graph'][0]
    assert res['biosample_type'] == 'mixed sample'

def test_biosample_modifications_summaries(biosample_w_mod):
    assert biosample_w_mod['modifications_summary'] == 'Crispr for RAD21 gene'
    assert biosample_w_mod['modifications_summary_short'] == 'RAD21 Crispr'

def test_biosample_modifications_summaries_no_mods(biosample_1):
    assert biosample_1.get('modifications_summary') == 'None'
    assert biosample_1.get('modifications_summary_short') == 'None'

def test_biosample_treatments_summary(biosample_w_treatment):
    assert biosample_w_treatment.get('treatments_summary') == 'shRNA treatment'

def test_biosample_treatments_summary_no_treatment(biosample_1):
    assert biosample_1.get('treatments_summary') == 'None'
<|endoftext|>