<commit_before>#!/usr/bin/env python

"""Do the final prediction of binding site given all features."""

from __future__ import print_function, absolute_import
import pickle
import os
import sys
import optparse
import cryptosite.config

if sys.version_info[0] == 2:
    load_py2_pickle = pickle.load
else:
    def load_py2_pickle(fh):
        return pickle.load(fh, encoding='latin1')

def get_matrix(inputdata, model='linear'):

    Res = {'CYS': (0, 0, 1, 0, 0), 'ASP': (0, 0, 0, 1, 1),
           'SER': (0, 1, 1, 1, 1), 'GLN': (0, 0, 1, 0, 1),
           'LYS': (0, 1, 0, 1, 1), 'ILE': (0, 1, 0, 0, 1),
           'PRO': (0, 1, 1, 1, 0), 'THR': (1, 0, 0, 0, 0),
           'PHE': (0, 1, 1, 0, 1), 'ALA': (0, 0, 0, 0, 0),
           'GLY': (0, 0, 1, 1, 1), 'HIS': (0, 1, 0, 0, 0),
           'GLU': (0, 0, 1, 1, 0), 'LEU': (0, 1, 0, 1, 0),
           'ARG': (0, 0, 0, 0, 1), 'TRP': (1, 0, 0, 0, 1),
           'VAL': (1, 0, 0, 1, 1), 'ASN': (0, 0, 0, 1, 0),
           'TYR': (1, 0, 0, 1, 0), 'MET': (0, 1, 1, 0, 0)}
    Res = Res.keys()
    SSE = ['B', 'E', 'G', 'H', 'I', 'S', 'T', 'U']

    data = open(inputdata)
    D = data.readlines()
    data.close()

    Header = D[0].strip().split()

    if model == 'poly':
        # the bottom visited for poly SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'),
                   Header.index('D2S')]
        visited += [Header.index('SQCn'), Header.index('PCKn'),
                    Header.index('Hn')]
        visited += [Header.index('CN5_std_450'), Header.index('CN5_std_300'),
                    Header.index('CN5_std_350')]
        visited += [Header.index('CNC'), Header.index('PRT_std_450'),
                    Header.index('CN5_mean_500')]
        visited += [Header.index('Bn'), Header.index('CHRn'),
                    Header.index('In')]
        visited += [Header.index('CNC_std_300'), Header.index('CNS_300'),
                    Header.index('SAS14_std_400')]
        visited += [Header.index('SASn')]

    elif model == 'linear':
        # for linear SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'),
                   Header.index('CN5_std_450')]
        visited += [Header.index('D2S'), Header.index('CNS_300'),
                    Header.index('Hn')]
        visited += [Header.index('CN5_mean_450'), Header.index('CN5_std_300'),
                    Header.index('SQCn')]
        visited += [Header.index('CNC_std_350'), Header.index('CNCn'),
                    Header.index('CVX_mean_450')]
        visited += [Header.index('In')]

    elif model == 'final':
        visited = [Header.index('CNC_mean_'), Header.index('SQC'),
                   Header.index('PTM')]

    else:
        print('Wrong model: ', model)
        exit()

    M = []
    Indeces, cnt = {}, 0
    for d in D[1:]:
        d = d.strip().split('\t')
        LA = []
        for hd in range(len(Header)):
            if hd not in visited:
                pass
            else:
                if hd == 1:
                    L = [0.] * len(Res)
                    L[Res.index(d[1])] = 1.
                    LA += L
                elif hd in range(4, 8):
                    LA += [float(d[hd])]
                elif hd == 8:
                    s = [0.] * len(SSE)
                    s[SSE.index(d[8])] = 1.
                    LA += s
                else:
                    LA += [float(d[hd])]
        LA += [float(d[-1])]

        M.append([d[0]] + LA)

        Indeces[cnt] = tuple(d[:3])
        cnt += 1
    return M, [Header[j] for j in sorted(visited)], Indeces

def predict(inputdata, model='linear'):
    import numpy as np
    import sklearn
    from sklearn.metrics import confusion_matrix

    print('Reading in the data ...')
    M, Header, Indeces = get_matrix(inputdata, model)

    print('Processing the data for model %s ...' % model.upper())
    pdb = inputdata.split('.')[0]
    print(pdb)
    NewIndeces, newcnt = {}, 0
    X_learn, Y_learn = [], []

    for r, m in enumerate(M):
        if len(np.argwhere(np.isnan(np.array(m[1:-1])))) > 0:
            raise ValueError(r, m[0])

        X_learn.append(np.array(m[1:-1]))
        Y_learn.append(m[-1])
        NewIndeces[newcnt] = Indeces[r]
        newcnt += 1

    X_learn = np.array(X_learn)
    X_learn = np.vstack((X_learn[:, 0], X_learn[:, 2], X_learn[:, 1])).T

    scaler_pkl = {'linear': 'LinearScaler_Final.pkl',
                  'poly': 'PolyScaler_Final.pkl',
                  'final': 'Scaler_Final_Final.pkl'}[model]
    outmodel_pkl = {'linear': 'LinearSVC_FinalModel.pkl',
                    'poly': 'PolySVC_FinalModel.pkl',
                    'final': 'SVM_Final_Final.pkl'}[model]

    print('Scaling ...')
    with open(os.path.join(cryptosite.config.datadir, scaler_pkl), 'rb') as fh:
        scaler = load_py2_pickle(fh)
    X_learn = scaler.transform(X_learn)

    with open(os.path.join(cryptosite.config.datadir,
                           outmodel_pkl), 'rb') as fh:
        learner = load_py2_pickle(fh)

    print('Predicting ...')

    # Set _gamma explicitly (earlier versions of cryptosite relied on a hacked
    # local copy of sklearn that did this)
    learner._gamma = 1.0 / X_learn.shape[1]

    # Our pickles are for sklearn 0.12. If using 0.14.1,
    # add necessary attributes
    if sklearn.__version__ == '0.14.1':
        learner._impl = learner.impl
        learner._label = learner.classes_ = learner.class_weight_label_

    Y_pred = learner.predict(X_learn)
    CM = confusion_matrix(Y_learn, Y_pred)
    print()
    print("Confusion matrix for: ", pdb)
    print(CM)
    print()

    # output
    Y_pred_prob = learner.predict_proba(X_learn)
    Y_PRED_PROB_ALL = list(Y_pred_prob[:, 1])

    suffix = {'linear': 'lin', 'poly': 'pol', 'final': 'pol'}[model]
    outn = open(pdb + '.%s.pred' % suffix, 'w')

    print('Writing output files ...')
    outn.write('\t'.join(['PDBID', 'Res', 'ResID'] + Header +
                         ['CryptositeValue']) + '\n')
    for x in range(len(Y_PRED_PROB_ALL)):
        outn.write('\t'.join(list(NewIndeces[x]) +
                             [str(i) for i in X_learn[x]] +
                             [str(Y_PRED_PROB_ALL[x])]) + '\n')
    outn.close()

    write_pdb(pdb, model)

    print('Done!')

def write_pdb(pdb, model='linear'):
    suffix = {'linear': 'lin', 'poly': 'pol', 'final': 'pol'}[model]

    with open(pdb + '.%s.pred' % suffix) as data:
        D = data.readlines()
    out = open('%s.%s.pred.pdb' % (pdb, suffix), 'w')

    Data = {}
    for d in D:
        d = d.strip().split()
        Data[(d[1], d[2])] = ('0.0', d[-1])

    with open('%s_mdl.pdb' % pdb.split('/')[-1]) as data:
        D = data.readlines()

    for d in D:
        if 'ATOM' == d[:4]:
            p = (d[17:20], str(int(d[22:26])))

            try:
                pred = Data[p]
            except KeyError:
                pred = ('0.0', '0.0')

            v = '%.2f' % (float(pred[1]) * 100)
            v = (6 - len(v)) * ' ' + v
            line = d[:56] + pred[0] + '0' + v + '\n'
            out.write(line)

        else:
            out.write(d)
    out.close()

def parse_args():
    usage = """%prog [opts] <model_name>

Do the final prediction of binding site given all features.

<model_name> should be the name of the model. The model's 3D structure,
<model_name>_mdl.pdb, and the features file, <model_name>.features,
are read in from the current directory.

Two files are generated on successful prediction:
<model_name>.pol.pred: a simple tab-separated file listing the value of
              the CryptoSite score for each residue.
<model_name>.pol.pred.pdb: a PDB file with the CryptoSite score in the
              occupancy column, for visualization.
"""
    parser = optparse.OptionParser(usage)
    opts, args = parser.parse_args()
    if len(args) != 1:
        parser.error("incorrect number of arguments")
    return args[0]

def main():
    model_name = parse_args()
    predict(model_name + '.features', model='final')

if __name__ == '__main__':
    main()
<commit_msg>Make pickle.load easier to mock<commit_after>#!/usr/bin/env python

"""Do the final prediction of binding site given all features."""

from __future__ import print_function, absolute_import
import pickle
import os
import sys
import optparse
import cryptosite.config

if sys.version_info[0] == 2:
    def load_py2_pickle(fh):
        return pickle.load(fh)
else:
    def load_py2_pickle(fh):
        return pickle.load(fh, encoding='latin1')

def get_matrix(inputdata, model='linear'):

    Res = {'CYS': (0, 0, 1, 0, 0), 'ASP': (0, 0, 0, 1, 1),
           'SER': (0, 1, 1, 1, 1), 'GLN': (0, 0, 1, 0, 1),
           'LYS': (0, 1, 0, 1, 1), 'ILE': (0, 1, 0, 0, 1),
           'PRO': (0, 1, 1, 1, 0), 'THR': (1, 0, 0, 0, 0),
           'PHE': (0, 1, 1, 0, 1), 'ALA': (0, 0, 0, 0, 0),
           'GLY': (0, 0, 1, 1, 1), 'HIS': (0, 1, 0, 0, 0),
           'GLU': (0, 0, 1, 1, 0), 'LEU': (0, 1, 0, 1, 0),
           'ARG': (0, 0, 0, 0, 1), 'TRP': (1, 0, 0, 0, 1),
           'VAL': (1, 0, 0, 1, 1), 'ASN': (0, 0, 0, 1, 0),
           'TYR': (1, 0, 0, 1, 0), 'MET': (0, 1, 1, 0, 0)}
    Res = Res.keys()
    SSE = ['B', 'E', 'G', 'H', 'I', 'S', 'T', 'U']

    data = open(inputdata)
    D = data.readlines()
    data.close()

    Header = D[0].strip().split()

    if model == 'poly':
        # the bottom visited for poly SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'),
                   Header.index('D2S')]
        visited += [Header.index('SQCn'), Header.index('PCKn'),
                    Header.index('Hn')]
        visited += [Header.index('CN5_std_450'), Header.index('CN5_std_300'),
                    Header.index('CN5_std_350')]
        visited += [Header.index('CNC'), Header.index('PRT_std_450'),
                    Header.index('CN5_mean_500')]
        visited += [Header.index('Bn'), Header.index('CHRn'),
                    Header.index('In')]
        visited += [Header.index('CNC_std_300'), Header.index('CNS_300'),
                    Header.index('SAS14_std_400')]
        visited += [Header.index('SASn')]

    elif model == 'linear':
        # for linear SVM
        visited = [Header.index('CNC_mean_300'), Header.index('SQC'),
                   Header.index('CN5_std_450')]
        visited += [Header.index('D2S'), Header.index('CNS_300'),
                    Header.index('Hn')]
        visited += [Header.index('CN5_mean_450'), Header.index('CN5_std_300'),
                    Header.index('SQCn')]
        visited += [Header.index('CNC_std_350'), Header.index('CNCn'),
                    Header.index('CVX_mean_450')]
        visited += [Header.index('In')]

    elif model == 'final':
        visited = [Header.index('CNC_mean_'), Header.index('SQC'),
                   Header.index('PTM')]

    else:
        print('Wrong model: ', model)
        exit()

    M = []
    Indeces, cnt = {}, 0
    for d in D[1:]:
        d = d.strip().split('\t')
        LA = []
        for hd in range(len(Header)):
            if hd not in visited:
                pass
            else:
                if hd == 1:
                    L = [0.] * len(Res)
                    L[Res.index(d[1])] = 1.
                    LA += L
                elif hd in range(4, 8):
                    LA += [float(d[hd])]
                elif hd == 8:
                    s = [0.] * len(SSE)
                    s[SSE.index(d[8])] = 1.
                    LA += s
                else:
                    LA += [float(d[hd])]
        LA += [float(d[-1])]

        M.append([d[0]] + LA)

        Indeces[cnt] = tuple(d[:3])
        cnt += 1
    return M, [Header[j] for j in sorted(visited)], Indeces

def predict(inputdata, model='linear'):
    import numpy as np
    import sklearn
    from sklearn.metrics import confusion_matrix

    print('Reading in the data ...')
    M, Header, Indeces = get_matrix(inputdata, model)

    print('Processing the data for model %s ...' % model.upper())
    pdb = inputdata.split('.')[0]
    print(pdb)
    NewIndeces, newcnt = {}, 0
    X_learn, Y_learn = [], []

    for r, m in enumerate(M):
        if len(np.argwhere(np.isnan(np.array(m[1:-1])))) > 0:
            raise ValueError(r, m[0])

        X_learn.append(np.array(m[1:-1]))
        Y_learn.append(m[-1])
        NewIndeces[newcnt] = Indeces[r]
        newcnt += 1

    X_learn = np.array(X_learn)
    X_learn = np.vstack((X_learn[:, 0], X_learn[:, 2], X_learn[:, 1])).T

    scaler_pkl = {'linear': 'LinearScaler_Final.pkl',
                  'poly': 'PolyScaler_Final.pkl',
                  'final': 'Scaler_Final_Final.pkl'}[model]
    outmodel_pkl = {'linear': 'LinearSVC_FinalModel.pkl',
                    'poly': 'PolySVC_FinalModel.pkl',
                    'final': 'SVM_Final_Final.pkl'}[model]

    print('Scaling ...')
    with open(os.path.join(cryptosite.config.datadir, scaler_pkl), 'rb') as fh:
        scaler = load_py2_pickle(fh)
    X_learn = scaler.transform(X_learn)

    with open(os.path.join(cryptosite.config.datadir,
                           outmodel_pkl), 'rb') as fh:
        learner = load_py2_pickle(fh)

    print('Predicting ...')

    # Set _gamma explicitly (earlier versions of cryptosite relied on a hacked
    # local copy of sklearn that did this)
    learner._gamma = 1.0 / X_learn.shape[1]

    # Our pickles are for sklearn 0.12. If using 0.14.1,
    # add necessary attributes
    if sklearn.__version__ == '0.14.1':
        learner._impl = learner.impl
        learner._label = learner.classes_ = learner.class_weight_label_

    Y_pred = learner.predict(X_learn)
    CM = confusion_matrix(Y_learn, Y_pred)
    print()
    print("Confusion matrix for: ", pdb)
    print(CM)
    print()

    # output
    Y_pred_prob = learner.predict_proba(X_learn)
    Y_PRED_PROB_ALL = list(Y_pred_prob[:, 1])

    suffix = {'linear': 'lin', 'poly': 'pol', 'final': 'pol'}[model]
    outn = open(pdb + '.%s.pred' % suffix, 'w')

    print('Writing output files ...')
    outn.write('\t'.join(['PDBID', 'Res', 'ResID'] + Header +
                         ['CryptositeValue']) + '\n')
    for x in range(len(Y_PRED_PROB_ALL)):
        outn.write('\t'.join(list(NewIndeces[x]) +
                             [str(i) for i in X_learn[x]] +
                             [str(Y_PRED_PROB_ALL[x])]) + '\n')
    outn.close()

    write_pdb(pdb, model)

    print('Done!')

def write_pdb(pdb, model='linear'):
    suffix = {'linear': 'lin', 'poly': 'pol', 'final': 'pol'}[model]

    with open(pdb + '.%s.pred' % suffix) as data:
        D = data.readlines()
    out = open('%s.%s.pred.pdb' % (pdb, suffix), 'w')

    Data = {}
    for d in D:
        d = d.strip().split()
        Data[(d[1], d[2])] = ('0.0', d[-1])

    with open('%s_mdl.pdb' % pdb.split('/')[-1]) as data:
        D = data.readlines()

    for d in D:
        if 'ATOM' == d[:4]:
            p = (d[17:20], str(int(d[22:26])))

            try:
                pred = Data[p]
            except KeyError:
                pred = ('0.0', '0.0')

            v = '%.2f' % (float(pred[1]) * 100)
            v = (6 - len(v)) * ' ' + v
            line = d[:56] + pred[0] + '0' + v + '\n'
            out.write(line)

        else:
            out.write(d)
    out.close()

def parse_args():
    usage = """%prog [opts] <model_name>

Do the final prediction of binding site given all features.

<model_name> should be the name of the model. The model's 3D structure,
<model_name>_mdl.pdb, and the features file, <model_name>.features,
are read in from the current directory.

Two files are generated on successful prediction:
<model_name>.pol.pred: a simple tab-separated file listing the value of
              the CryptoSite score for each residue.
<model_name>.pol.pred.pdb: a PDB file with the CryptoSite score in the
              occupancy column, for visualization.
"""
    parser = optparse.OptionParser(usage)
    opts, args = parser.parse_args()
    if len(args) != 1:
        parser.error("incorrect number of arguments")
    return args[0]

def main():
    model_name = parse_args()
    predict(model_name + '.features', model='final')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import argparse
import numpy as np

import chainer
from chainer import iterators

import chainermn

from chainercv.datasets import cityscapes_semantic_segmentation_label_names
from chainercv.datasets import CityscapesSemanticSegmentationDataset
from chainercv.evaluations import calc_semantic_segmentation_confusion
from chainercv.evaluations import calc_semantic_segmentation_iou
from chainercv.experimental.links import PSPNetResNet101
from chainercv.utils import apply_to_iterator
from chainercv.utils import ProgressHook

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--model', choices=('pspnet_resnet101',),
        default='pspnet_resnet101')
    parser.add_argument('--pretrained-model')
    args = parser.parse_args()

    comm = chainermn.create_communicator()
    device = comm.intra_rank

    if args.model == 'pspnet_resnet101':
        if args.pretrained_model:
            model = PSPNetResNet101(
                n_class=len(cityscapes_semantic_segmentation_label_names),
                pretrained_model=args.pretrained_model, input_size=(713, 713)
            )
        else:
            model = PSPNetResNet101(pretrained_model='cityscapes')

    chainer.cuda.get_device_from_id(device).use()
    model.to_gpu()

    dataset = CityscapesSemanticSegmentationDataset(
        split='val', label_resolution='fine')

    if comm.rank == 0:
        indices = np.arange(len(dataset))
    else:
        indices = None
    indices = chainermn.scatter_dataset(indices, comm)
    dataset = dataset.slice[indices]

    it = iterators.SerialIterator(
        dataset, 1, repeat=False, shuffle=False)

    in_values, out_values, rest_values = apply_to_iterator(
        model.predict, it, hook=ProgressHook(len(dataset)))
    # Delete an iterator of images to save memory usage.
    del in_values
    pred_labels, = out_values
    gt_labels, = rest_values

    confusion = calc_semantic_segmentation_confusion(pred_labels, gt_labels)
    confusion = comm.allreduce(confusion)

    if comm.rank == 0:
        iou = calc_semantic_segmentation_iou(confusion)
        pixel_accuracy = np.diag(confusion).sum() / confusion.sum()
        class_accuracy = np.diag(confusion) / np.sum(confusion, axis=1)

        for iu, label_name in zip(
                iou, cityscapes_semantic_segmentation_label_names):
            print('{:>23} : {:.4f}'.format(label_name, iu))
        print('=' * 34)
        print('{:>23} : {:.4f}'.format('mean IoU', np.nanmean(iou)))
        print('{:>23} : {:.4f}'.format(
            'Class average accuracy', np.nanmean(class_accuracy)))
        print('{:>23} : {:.4f}'.format(
            'Global average accuracy', np.nanmean(pixel_accuracy)))

if __name__ == '__main__':
    main()
<commit_msg>remove mean<commit_after>import argparse
import numpy as np

import chainer
from chainer import iterators

import chainermn

from chainercv.datasets import cityscapes_semantic_segmentation_label_names
from chainercv.datasets import CityscapesSemanticSegmentationDataset
from chainercv.evaluations import calc_semantic_segmentation_confusion
from chainercv.evaluations import calc_semantic_segmentation_iou
from chainercv.experimental.links import PSPNetResNet101
from chainercv.utils import apply_to_iterator
from chainercv.utils import ProgressHook

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--model', choices=('pspnet_resnet101',),
        default='pspnet_resnet101')
    parser.add_argument('--pretrained-model')
    args = parser.parse_args()

    comm = chainermn.create_communicator()
    device = comm.intra_rank

    if args.model == 'pspnet_resnet101':
        if args.pretrained_model:
            model = PSPNetResNet101(
                n_class=len(cityscapes_semantic_segmentation_label_names),
                pretrained_model=args.pretrained_model, input_size=(713, 713)
            )
        else:
            model = PSPNetResNet101(pretrained_model='cityscapes')

    chainer.cuda.get_device_from_id(device).use()
    model.to_gpu()

    dataset = CityscapesSemanticSegmentationDataset(
        split='val', label_resolution='fine')

    if comm.rank == 0:
        indices = np.arange(len(dataset))
    else:
        indices = None
    indices = chainermn.scatter_dataset(indices, comm)
    dataset = dataset.slice[indices]

    it = iterators.SerialIterator(
        dataset, 1, repeat=False, shuffle=False)

    in_values, out_values, rest_values = apply_to_iterator(
        model.predict, it, hook=ProgressHook(len(dataset)))
    # Delete an iterator of images to save memory usage.
    del in_values
    pred_labels, = out_values
    gt_labels, = rest_values

    confusion = calc_semantic_segmentation_confusion(pred_labels, gt_labels)
    confusion = comm.allreduce(confusion)

    if comm.rank == 0:
        iou = calc_semantic_segmentation_iou(confusion)
        pixel_accuracy = np.diag(confusion).sum() / confusion.sum()
        class_accuracy = np.diag(confusion) / np.sum(confusion, axis=1)

        for iu, label_name in zip(
                iou, cityscapes_semantic_segmentation_label_names):
            print('{:>23} : {:.4f}'.format(label_name, iu))
        print('=' * 34)
        print('{:>23} : {:.4f}'.format('mean IoU', np.nanmean(iou)))
        print('{:>23} : {:.4f}'.format(
            'Class average accuracy', np.nanmean(class_accuracy)))
        print('{:>23} : {:.4f}'.format(
            'Global average accuracy', pixel_accuracy))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

import python_utils.evaluate_detection as eval
import python_utils.general_utils as g_utils
import datasets
import datasets.nyud2_voc
import os
import datasets.imdb
import xml.dom.minidom as minidom
import numpy as np
import scipy.sparse
import scipy.io as sio
import utils.cython_bbox
import cPickle
import subprocess
from IPython.core.debugger import Tracer

class nyud2_voc(datasets.imdb):
    def __init__(self, image_set, year, devkit_path=None, image_type = 'images'):
        datasets.imdb.__init__(self, 'nyud2_' + image_type + '_' + year + '_' + image_set)
        self._year = year
        self._devkit_path = self._get_default_path() if devkit_path is None \
                            else devkit_path
        self._data_path = os.path.join(self._devkit_path, 'data')

        self._classes = ('__background__', # always index 0
            'bathtub', 'bed', 'bookshelf', 'box', 'chair', 'counter', 'desk',
            'door', 'dresser', 'garbage-bin', 'lamp', 'monitor', 'night-stand',
            'pillow', 'sink', 'sofa', 'table', 'television', 'toilet');

        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
        self._image_type = image_type;
        self._image_set = image_set;
        self._image_ext = '.png'
        self._image_index = self._load_image_set_index()
        # Default to roidb handler
        self._roidb_handler = self.mcg_roidb

        # PASCAL specific config options
        self.config = {'cleanup'  : True,
                       'use_salt' : True,
                       'top_k'    : 2000}

        assert os.path.exists(self._devkit_path), \
                'VOCdevkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), \
                'Path does not exist: {}'.format(self._data_path)

    def image_path_at(self, i):
        """
        Return the absolute path to image i in the image sequence.
        """
        return self.image_path_from_index(self._image_index[i])

    def image_path_from_index(self, index):
        """
        Construct an image path from the image's "index" identifier.
        """
        image_path = []
        image_type_list = self._image_type.split('+')
        for typ in image_type_list:
            image_path.append(os.path.join(self._data_path, typ, index + self._image_ext))
            assert os.path.exists(image_path[-1]), 'Path does not exist: {}'.format(image_path)
        return image_path

    def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # Example path to image set file:
        # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt
        image_set_file = os.path.join(self._devkit_path, 'benchmarkData', 'metadata',
                                      'nyusplits.mat')
        assert os.path.exists(image_set_file), \
                'Path does not exist: {}'.format(image_set_file)
        raw_data = sio.loadmat(image_set_file)[self._image_set].ravel()
        image_index = ['img_{:4d}'.format(i) for i in raw_data]

        return image_index

    def _get_default_path(self):
        """
        Return the default path where PASCAL VOC is expected to be installed.
        """
        return os.path.join('/work4', 'sgupta', 'datasets', 'nyud2')

    def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = [self._load_nyud2_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote gt roidb to {}'.format(cache_file)

        return gt_roidb

    def mcg_roidb(self):
        """
        Return the database of selective search regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path,
                                  self.name + '_mcg_roidb.pkl')

        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        if True:
            gt_roidb = self.gt_roidb()
            ss_roidb = self._load_mcg_roidb(gt_roidb)
            roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        else:
            roidb = self._load_mcg_roidb(None)
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote mcg roidb to {}'.format(cache_file)

        return roidb

    def _load_mcg_roidb(self, gt_roidb):
        filename = os.path.abspath(os.path.join(self.cache_path, '..',
                                                'mcg_data', 'nyud2_2015' + '.mat'))
        assert os.path.exists(filename), \
               'Selective search data not found at: {}'.format(filename)
        boxes = sio.loadmat(filename)['bboxes'].ravel()
        imnames = sio.loadmat(filename)['imnames'].ravel()
        imnames = [str(x[0]) for x in imnames]

        box_list = []
        for i in xrange(len(self._image_index)):
            ind = np.where(self._image_index[i] == np.array(imnames))[0]
            assert(len(ind) == 1)
            box_list.append(boxes[ind[0]][:, (1, 0, 3, 2)] - 1)

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def _load_nyud2_annotation(self, index):
        """
        Load image and bounding boxes info from XML file in the PASCAL VOC
        format.
        """
        filename = os.path.join(self._devkit_path, 'benchmarkData', \
            'gt_box_cache_dir', index + '.mat')
        print 'Loading: {}'.format(filename)
        raw_data = sio.loadmat(filename)
        objs = raw_data['rec']['objects'][0][0][0]

        # Select object we care about
        objs = [obj for obj in objs if self._class_to_ind.get(str(obj['class'][0])) is not None]

        num_objs = len(objs)

        boxes = np.zeros((num_objs, 4), dtype=np.uint16)
        gt_classes = np.zeros((num_objs), dtype=np.int32)
        overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)

        # Load object bounding boxes into a data frame.
        for ix, obj in enumerate(objs):
            # Make pixel indexes 0-based
            cls = self._class_to_ind.get(str(obj['class'][0]))
            boxes[ix, :] = obj['bbox'][0] - 1
            gt_classes[ix] = cls
            overlaps[ix, cls] = 1.0

        overlaps = scipy.sparse.csr_matrix(overlaps)

        return {'boxes' : boxes,
                'gt_classes': gt_classes,
                'gt_overlaps' : overlaps,
                'flipped' : False}

    def evaluate_detections(self, all_boxes, output_dir, det_salt = '', eval_salt = '', overlap_thresh = 0.5):
      num_classes = self.num_classes
      num_images = self.num_images
      gt_roidb = self.gt_roidb()
      ap = [[]]; prec = [[]]; rec = [[]]
      ap_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.txt')
      with open(ap_file, 'wt') as f:
          for i in xrange(1, self.num_classes):
              dt = []; gt = [];
              # Prepare the output
              for j in xrange(0,num_images):
                  bs = all_boxes[i][j]
                  if len(bs) == 0:
                    bb = np.zeros((0,4)).astype(np.float32)
                    sc = np.zeros((0,1)).astype(np.float32)
                  else:
                    bb = bs[:,:4].reshape(bs.shape[0],4)
                    sc = bs[:,4].reshape(bs.shape[0],1)
                  dtI = dict({'sc': sc, 'boxInfo': bb})
                  dt.append(dtI)

              # Prepare the annotations
              for j in xrange(0,num_images):
                  cls_ind = np.where(gt_roidb[j]['gt_classes'] == i)[0]
                  bb = gt_roidb[j]['boxes'][cls_ind,:]
                  diff = np.zeros((len(cls_ind),1)).astype(np.bool)
                  gt.append(dict({'diff': diff, 'boxInfo': bb}))
              bOpts = dict({'minoverlap': overlap_thresh})
              ap_i, rec_i, prec_i = eval.inst_image(dt, gt, bOpts)
              ap.append(ap_i[0]); prec.append(prec_i); rec.append(rec_i)
              ap_str = '{:20s}:{:10f}'.format(self.classes[i], ap_i[0]*100)
              f.write(ap_str + '\n')
              print ap_str
          ap_str = '{:20s}:{:10f}'.format('mean', np.mean(ap[1:])*100)
          f.write(ap_str + '\n')
          print ap_str

      eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.pkl')
      g_utils.save_variables(eval_file, [ap, prec, rec, self._classes, self._class_to_ind], \
          ['ap', 'prec', 'rec', 'classes', 'class_to_ind'], overwrite = True)
      eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.mat')
      g_utils.scio.savemat(eval_file, {'ap': ap, 'prec': prec, 'rec': rec, 'classes': self._classes}, do_compression = True);

      return ap, prec, rec, self._classes, self._class_to_ind

if __name__ == '__main__':
    d = datasets.nyud2_voc('trainval', '2007')
    # res = d.roidb
    from IPython import embed; embed()
<commit_msg>fix paths<commit_after># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

import python_utils.evaluate_detection as eval
import python_utils.general_utils as g_utils
import datasets
import datasets.nyud2_voc
import os
import datasets.imdb
import xml.dom.minidom as minidom
import numpy as np
import scipy.sparse
import scipy.io as sio
import utils.cython_bbox
import cPickle
import subprocess
from IPython.core.debugger import Tracer

class nyud2_voc(datasets.imdb):
    def __init__(self, image_set, year, devkit_path=None, image_type = 'images'):
        datasets.imdb.__init__(self, 'nyud2_' + image_type + '_' + year + '_' + image_set)
        self._year = year
        self._devkit_path = self._get_default_path() if devkit_path is None \
                            else devkit_path
        self._data_path = os.path.join(self._devkit_path, 'data')

        self._classes = ('__background__', # always index 0
            'bathtub', 'bed', 'bookshelf', 'box', 'chair', 'counter', 'desk',
            'door', 'dresser', 'garbage-bin', 'lamp', 'monitor', 'night-stand',
            'pillow', 'sink', 'sofa', 'table', 'television', 'toilet');

        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
        self._image_type = image_type;
        self._image_set = image_set;
        self._image_ext = '.png'
        self._image_index = self._load_image_set_index()
        # Default to roidb handler
        self._roidb_handler = self.mcg_roidb

        # PASCAL specific config options
        self.config = {'cleanup'  : True,
                       'use_salt' : True,
                       'top_k'    : 2000}

        assert os.path.exists(self._devkit_path), \
                'VOCdevkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), \
                'Path does not exist: {}'.format(self._data_path)

    def image_path_at(self, i):
        """
        Return the absolute path to image i in the image sequence.
        """
        return self.image_path_from_index(self._image_index[i])

    def image_path_from_index(self, index):
        """
        Construct an image path from the image's "index" identifier.
        """
        image_path = []
        image_type_list = self._image_type.split('+')
        for typ in image_type_list:
            image_path.append(os.path.join(self._data_path, typ, index + self._image_ext))
            assert os.path.exists(image_path[-1]), 'Path does not exist: {}'.format(image_path)
        return image_path

    def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # Example path to image set file:
        # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt
        image_set_file = os.path.join(self._devkit_path, 'benchmarkData', 'metadata',
                                      'nyusplits.mat')
        assert os.path.exists(image_set_file), \
                'Path does not exist: {}'.format(image_set_file)
        raw_data = sio.loadmat(image_set_file)[self._image_set].ravel()
        image_index = ['img_{:4d}'.format(i) for i in raw_data]

        return image_index

    def _get_default_path(self):
        """
        Return the default path where PASCAL VOC is expected to be installed.
        """
        return os.path.join('data', 'nyud2')

    def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = [self._load_nyud2_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote gt roidb to {}'.format(cache_file)

        return gt_roidb

    def mcg_roidb(self):
        """
        Return the database of selective search regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path,
                                  self.name + '_mcg_roidb.pkl')

        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        if True:
            gt_roidb = self.gt_roidb()
            ss_roidb = self._load_mcg_roidb(gt_roidb)
            roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        else:
            roidb = self._load_mcg_roidb(None)
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote mcg roidb to {}'.format(cache_file)

        return roidb

    def _load_mcg_roidb(self, gt_roidb):
        filename = os.path.abspath(os.path.join('data', 'nyud2_mcg_boxes' + '.mat'))
        assert os.path.exists(filename), \
               'Selective search data not found at: {}'.format(filename)
        boxes = sio.loadmat(filename)['bboxes'].ravel()
        imnames = sio.loadmat(filename)['imnames'].ravel()
        imnames = [str(x[0]) for x in imnames]

        box_list = []
        for i in xrange(len(self._image_index)):
            ind = np.where(self._image_index[i] == np.array(imnames))[0]
            assert(len(ind) == 1)
            box_list.append(boxes[ind[0]][:, (1, 0, 3, 2)] - 1)

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def _load_nyud2_annotation(self, index):
        """
        Load image and bounding boxes info from XML file in the PASCAL VOC
        format.
        """
        filename = os.path.join(self._devkit_path, 'benchmarkData', \
            'gt_box_cache_dir', index + '.mat')
        print 'Loading: {}'.format(filename)
        raw_data = sio.loadmat(filename)
        objs = raw_data['rec']['objects'][0][0][0]

        # Select object we care about
        objs = [obj for obj in objs if self._class_to_ind.get(str(obj['class'][0])) is not None]

        num_objs = len(objs)

        boxes = np.zeros((num_objs, 4), dtype=np.uint16)
        gt_classes = np.zeros((num_objs), dtype=np.int32)
        overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)

        # Load object bounding boxes into a data frame.
        for ix, obj in enumerate(objs):
            # Make pixel indexes 0-based
            cls = self._class_to_ind.get(str(obj['class'][0]))
            boxes[ix, :] = obj['bbox'][0] - 1
            gt_classes[ix] = cls
            overlaps[ix, cls] = 1.0

        overlaps = scipy.sparse.csr_matrix(overlaps)

        return {'boxes' : boxes,
                'gt_classes': gt_classes,
                'gt_overlaps' : overlaps,
                'flipped' : False}

    def evaluate_detections(self, all_boxes, output_dir, det_salt = '', eval_salt = '', overlap_thresh = 0.5):
      num_classes = self.num_classes
      num_images = self.num_images
      gt_roidb = self.gt_roidb()
      ap = [[]]; prec = [[]]; rec = [[]]
      ap_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.txt')
      with open(ap_file, 'wt') as f:
          for i in xrange(1, self.num_classes):
              dt = []; gt = [];
              # Prepare the output
              for j in xrange(0,num_images):
                  bs = all_boxes[i][j]
                  if len(bs) == 0:
                    bb = np.zeros((0,4)).astype(np.float32)
                    sc = np.zeros((0,1)).astype(np.float32)
                  else:
                    bb = bs[:,:4].reshape(bs.shape[0],4)
                    sc = bs[:,4].reshape(bs.shape[0],1)
                  dtI = dict({'sc': sc, 'boxInfo': bb})
                  dt.append(dtI)

              # Prepare the annotations
              for j in xrange(0,num_images):
                  cls_ind = np.where(gt_roidb[j]['gt_classes'] == i)[0]
                  bb = gt_roidb[j]['boxes'][cls_ind,:]
                  diff = np.zeros((len(cls_ind),1)).astype(np.bool)
                  gt.append(dict({'diff': diff, 'boxInfo': bb}))
              bOpts = dict({'minoverlap': overlap_thresh})
              ap_i, rec_i, prec_i = eval.inst_image(dt, gt, bOpts)
              ap.append(ap_i[0]); prec.append(prec_i); rec.append(rec_i)
              ap_str = '{:20s}:{:10f}'.format(self.classes[i], ap_i[0]*100)
              f.write(ap_str + '\n')
              print ap_str
          ap_str = '{:20s}:{:10f}'.format('mean', np.mean(ap[1:])*100)
          f.write(ap_str + '\n')
          print ap_str

      eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.pkl')
      g_utils.save_variables(eval_file, [ap, prec, rec, self._classes, self._class_to_ind], \
          ['ap', 'prec', 'rec', 'classes', 'class_to_ind'], overwrite = True)
      eval_file = os.path.join(output_dir, 'eval' + det_salt + eval_salt + '.mat')
      g_utils.scio.savemat(eval_file, {'ap': ap, 'prec': prec, 'rec': rec, 'classes': self._classes}, do_compression = True);

      return ap, prec, rec, self._classes, self._class_to_ind

if __name__ == '__main__':
    d = datasets.nyud2_voc('trainval', '2007')
    # res = d.roidb
    from IPython import embed; embed()
<|endoftext|>


<commit_before>from provdbconnector.db_adapters.baseadapter import BaseAdapter, DbDocument, DbBundle, DbRecord, DbRelation, METADATA_KEY_IDENTIFIER, METADATA_KEY_PROV_TYPE
from provdbconnector.exceptions.database import InvalidOptionsException, NotFoundException
from provdbconnector.utils.serializer import encode_dict_values_to_primitive,split_into_formal_and_other_attributes,merge_record
from prov.constants import PROV_ASSOCIATION, PROV_TYPE, PROV_MENTION
from uuid import uuid4

import logging

log = logging.getLogger(__name__)

class SimpleInMemoryAdapter(BaseAdapter):

    document_bundle_ids = dict()

    bundles = dict()  # dict for all bundles including record and relation information

    all_nodes = dict()  # separate dict for records only (to get them by id)

    all_relations = dict()

    all_relations_id_map = dict()

    def __init__(self, *args):
        super(SimpleInMemoryAdapter, self).__init__()
        pass

    def connect(self, authentication_info):
        if authentication_info is not None:
            raise InvalidOptionsException()

        return True

    def save_record(self, attributes, metadata):
        #because it is in memory, we should copy the dicts to prevent others from modify the data
        attributes = attributes.copy()
        metadata = metadata.copy()

        # save all record information and return record id as string

        identifier = metadata[METADATA_KEY_IDENTIFIER]
        if str(identifier)in self.all_nodes:
            #try to merge nodes
            (old_attributes,old_metadata) = self.all_nodes[str(identifier)]
            (merged_attributes,merged_metadata) = merge_record(old_attributes,old_metadata, attributes,metadata)

            self.all_nodes.update({str(identifier): (merged_attributes,merged_metadata)})

        else:

            #encode your variables, based on your database architecture
            # (in this case it is not really necessary but for demonstration propose I saved the encoded vars )
            #attr = encode_dict_values_to_primitive(attributes)
            #meta = encode_dict_values_to_primitive(metadata)

            self.all_nodes.update({str(identifier): (attributes,metadata)})

        return str(identifier)

    def save_relation(self,  from_node,  to_node, attributes, metadata):
        # save all relation information and return the relation id as string

        # because it is in memory, we should copy the dicts to prevent others from modify the data
        attributes = attributes.copy()
        metadata = metadata.copy()

        #add dict if it is the first relation
        if str(from_node) not in self.all_relations:
            self.all_relations.update({str(from_node): dict()})

        # ===============
        # MERGE RELATION
        # ===============
        new_relation_formal_attributes = split_into_formal_and_other_attributes(attributes, metadata)

        #check that the from node already has some relations
        if str(from_node) in self.all_relations:

            #for each relation with the origin "from_node"
            for (relation_id,(to_identifier,old_attributes,old_metadata)) in self.all_relations[str(from_node)].items():
                #check if connecton is to the same identifier
                if str(to_node) == to_identifier and metadata[METADATA_KEY_PROV_TYPE] == old_metadata[METADATA_KEY_PROV_TYPE]:
                    # okay got potential duplicate... lets check the formal attributes
                    old_relation_formal_attributes = split_into_formal_and_other_attributes(old_attributes, old_metadata)

                    if old_relation_formal_attributes.formal == new_relation_formal_attributes.formal:
                        # got duplicate
                        (merged_attributes, merged_metadata) = merge_record(old_attributes,old_metadata,attributes,metadata)
                        self.all_relations[str(from_node)].update({relation_id: (to_identifier,merged_attributes,merged_metadata)})
                        return relation_id

        # ===============
        # CREATRE NEW RELATION
        # ===============

        id = str(uuid4())

        relations = self.all_relations[str(from_node)]
        relations.update({id: (str(to_node), attributes,metadata)})

        #self.all_records.update({new_rel_id: db_relations})

        return id

    def get_record(self, record_id):
        if record_id not in self.all_nodes:
            raise NotFoundException()

        (attributes, metadata)= self.all_nodes.get(record_id)
        attributes = encode_dict_values_to_primitive(attributes)
        metadata = encode_dict_values_to_primitive(metadata)
        db_record = DbRecord(attributes, metadata)

        return db_record

    def get_relation(self, relation_id):

        for (from_uri, relations) in self.all_relations.items():
            if relation_id in relations:

                (to_uri,  attributes, metadata) = relations[relation_id]

                attributes = encode_dict_values_to_primitive(attributes)
                metadata = encode_dict_values_to_primitive(metadata)

                return DbRelation(attributes, metadata)
        raise NotFoundException("could't find the relation with id {}".format(relation_id))

    def get_bundle(self, bundle_id):
        bundle = self.bundles.get(bundle_id)
        if bundle is None:
            raise NotFoundException()

        records = list()
        for record_id in bundle.records:
            record = self.all_records.get(record_id)
            records.append(record)

        return DbBundle(records, bundle.bundle_record)

    def delete_relation(self, relation_id):

        for (from_key, relations) in self.all_relations.items():
            if relation_id in relations:
                del relations[relation_id]
                break

        return True

    def delete_record(self, record_id):

        if record_id not in self.all_nodes:
            raise NotFoundException()

        del self.all_nodes[record_id]

        return True

    def get_records_tail(self, properties_dict=dict(), metadata_dict=dict(), depth=None):
        #only transform dict into list
        return list(self._get_records_tail_internal(properties_dict,metadata_dict).values())

    def _get_records_tail_internal(self, properties_dict=dict(), metadata_dict=dict(), max_depth=None, current_depth=0, result_records = None):
        origin_records = self.get_records_by_filter(properties_dict, metadata_dict)

        if result_records is None:
            result_records = dict()
        if current_depth == max_depth:
            return dict()
        for record in origin_records:
            from_identifier = record.metadata[METADATA_KEY_IDENTIFIER]
            if from_identifier  in self.all_relations:
                for (relation_id,(to_identifier, attributes, metadata)) in self.all_relations[from_identifier].items():
                    #find to node
                    if to_identifier not in result_records:
                        (to_attributes, to_metadata)= self.all_nodes[to_identifier]
                        result_records.update({to_identifier: DbRecord(to_attributes, to_metadata)})

                        # add other nodes recursive
                        result_records.update(self._get_records_tail_internal(metadata_dict={METADATA_KEY_IDENTIFIER: to_identifier},current_depth=current_depth+1,result_records=result_records))

                    # add relation to result
                    result_records.update({relation_id: DbRelation(attributes, metadata)})

        return result_records

    def delete_records_by_filter(self, properties_dict=dict(), metadata_dict=dict()):
        #erase all if no filter set
        if len(properties_dict) == 0 and len(metadata_dict) == 0:
            del self.all_nodes
            self.all_nodes = dict()
            return True

        #erase only matching nodes
        records_to_delete = self.get_records_by_filter(properties_dict,metadata_dict)

        for record in records_to_delete:
            if not isinstance(record, DbRecord):
                continue
            identifier = record.metadata[METADATA_KEY_IDENTIFIER]

            if identifier not in self.all_nodes:
                raise  NotFoundException("We cant find the id ")
            del self.all_nodes[identifier]

        return True

    def get_bundle_records(self, bundle_identifier):

        bundle_records = dict()

        #get all nodes for the bundle
        for (from_identifier, relations) in self.all_relations.items():
            # search in all relations for a relation that points to the bundle and also fulfill the other constraints
            for (relation_id, (to_identifier,attributes,metadata)) in relations.items():

                if to_identifier == str(bundle_identifier):
                    #got potential bundle association, check prov:type to be sure

                    if metadata[METADATA_KEY_PROV_TYPE] == PROV_ASSOCIATION and str(attributes[PROV_TYPE]) == "prov:bundleAssociation":
                        (attributes, metadata) = self.all_nodes[from_identifier]
                        bundle_records.update({from_identifier: DbRecord(attributes,metadata)})

        #search for all relations between the bundle nodes
        for from_identifier in bundle_records.copy().keys():
            if from_identifier in self.all_relations:
                for (relation_id,  (to_identifier, attributes, metadata)) in self.all_relations[from_identifier].items():

                    # If the target of the relation is also in the bundle the relation belongs to the bundle
                    if to_identifier in bundle_records:
                        bundle_records.update({relation_id: DbRelation(attributes, metadata)})

                    elif metadata[METADATA_KEY_PROV_TYPE] == PROV_MENTION:
                        # prov mentions used to connect between bundles , see w3c bundle links
                            bundle_records.update({relation_id: DbRelation(attributes, metadata)})

        return list(bundle_records.values())

    def _check_attribute_metadata_filter(self, properties_filter, metadata_filter, attributes,metadata):

        match = True

        meta_encoded = encode_dict_values_to_primitive(metadata)
        attr_encoded = encode_dict_values_to_primitive(attributes)

        filter_meta_encoded = encode_dict_values_to_primitive(metadata_filter)
        filter_attr_encoded = encode_dict_values_to_primitive(properties_filter)

        # check properties
        for (key, value) in filter_attr_encoded.items():
            if key not in attr_encoded:
                return False

            elif attr_encoded[key] != value:
                return False

        # check metadata
        for (key, value) in filter_meta_encoded.items():
            if key not in meta_encoded:
                return False

            elif meta_encoded[key] != value:
                return False

        return True

    def get_records_by_filter(self, properties_dict=dict(), metadata_dict=dict()):

        return_records = list()
        return_keys = set()
        properties_filter_dict = encode_dict_values_to_primitive(properties_dict.copy())
        metadata_filter_dict = encode_dict_values_to_primitive(metadata_dict.copy())
        for (identifier,(attributes, metadata)) in self.all_nodes.items():

            if self._check_attribute_metadata_filter(properties_filter=properties_dict,
                                                     metadata_filter=metadata_dict,
                                                     metadata=metadata,
                                                     attributes=attributes):

                meta_encoded = encode_dict_values_to_primitive(metadata)
                attr_encoded = encode_dict_values_to_primitive(attributes)

                return_records.append(DbRecord(attr_encoded,meta_encoded))
                return_keys.add(identifier)

            else:
                #not match so dont add
                pass

        #get relations
        for (from_id,relations) in self.all_relations.items():

            if from_id in return_keys:

                for (relation_id, (to_id, attributes, metadata)) in relations.items():

                    attributes = encode_dict_values_to_primitive(attributes)
                    metadata = encode_dict_values_to_primitive(metadata)

                    return_records.append(DbRelation(attributes,metadata))

        return  return_records<commit_msg>removed unused attr.<commit_after>from provdbconnector.db_adapters.baseadapter import BaseAdapter, DbDocument, DbBundle, DbRecord, DbRelation, METADATA_KEY_IDENTIFIER, METADATA_KEY_PROV_TYPE
from provdbconnector.exceptions.database import InvalidOptionsException, NotFoundException
from provdbconnector.utils.serializer import encode_dict_values_to_primitive,split_into_formal_and_other_attributes,merge_record
from prov.constants import PROV_ASSOCIATION, PROV_TYPE, PROV_MENTION
from uuid import uuid4

import logging

log = logging.getLogger(__name__)

class SimpleInMemoryAdapter(BaseAdapter):

    document_bundle_ids = dict()

    bundles = dict()  # dict for all bundles including record and relation information

    all_nodes = dict()  # separate dict for records only (to get them by id)

    all_relations = dict()

    def __init__(self, *args):
        super(SimpleInMemoryAdapter, self).__init__()
        pass

    def connect(self, authentication_info):
        if authentication_info is not None:
            raise InvalidOptionsException()

        return True

    def save_record(self, attributes, metadata):
        #because it is in memory, we should copy the dicts to prevent others from modify the data
        attributes = attributes.copy()
        metadata = metadata.copy()

        # save all record information and return record id as string

        identifier = metadata[METADATA_KEY_IDENTIFIER]
        if str(identifier)in self.all_nodes:
            #try to merge nodes
            (old_attributes,old_metadata) = self.all_nodes[str(identifier)]
            (merged_attributes,merged_metadata) = merge_record(old_attributes,old_metadata, attributes,metadata)

            self.all_nodes.update({str(identifier): (merged_attributes,merged_metadata)})

        else:

            #encode your variables, based on your database architecture
            # (in this case it is not really necessary but for demonstration propose I saved the encoded vars )
            #attr = encode_dict_values_to_primitive(attributes)
            #meta = encode_dict_values_to_primitive(metadata)

            self.all_nodes.update({str(identifier): (attributes,metadata)})

        return str(identifier)

    def save_relation(self,  from_node,  to_node, attributes, metadata):
        # save all relation information and return the relation id as string

        # because it is in memory, we should copy the dicts to prevent others from modify the data
        attributes = attributes.copy()
        metadata = metadata.copy()

        #add dict if it is the first relation
        if str(from_node) not in self.all_relations:
            self.all_relations.update({str(from_node): dict()})

        # ===============
        # MERGE RELATION
        # ===============
        new_relation_formal_attributes = split_into_formal_and_other_attributes(attributes, metadata)

        #check that the from node already has some relations
        if str(from_node) in self.all_relations:

            #for each relation with the origin "from_node"
            for (relation_id,(to_identifier,old_attributes,old_metadata)) in self.all_relations[str(from_node)].items():
                #check if connecton is to the same identifier
                if str(to_node) == to_identifier and metadata[METADATA_KEY_PROV_TYPE] == old_metadata[METADATA_KEY_PROV_TYPE]:
                    # okay got potential duplicate... lets check the formal attributes
                    old_relation_formal_attributes = split_into_formal_and_other_attributes(old_attributes, old_metadata)

                    if old_relation_formal_attributes.formal == new_relation_formal_attributes.formal:
                        # got duplicate
                        (merged_attributes, merged_metadata) = merge_record(old_attributes,old_metadata,attributes,metadata)
                        self.all_relations[str(from_node)].update({relation_id: (to_identifier,merged_attributes,merged_metadata)})
                        return relation_id

        # ===============
        # CREATRE NEW RELATION
        # ===============

        id = str(uuid4())

        relations = self.all_relations[str(from_node)]
        relations.update({id: (str(to_node), attributes,metadata)})

        #self.all_records.update({new_rel_id: db_relations})

        return id

    def get_record(self, record_id):
        if record_id not in self.all_nodes:
            raise NotFoundException()

        (attributes, metadata)= self.all_nodes.get(record_id)
        attributes = encode_dict_values_to_primitive(attributes)
        metadata = encode_dict_values_to_primitive(metadata)
        db_record = DbRecord(attributes, metadata)

        return db_record

    def get_relation(self, relation_id):

        for (from_uri, relations) in self.all_relations.items():
            if relation_id in relations:

                (to_uri,  attributes, metadata) = relations[relation_id]

                attributes = encode_dict_values_to_primitive(attributes)
                metadata = encode_dict_values_to_primitive(metadata)

                return DbRelation(attributes, metadata)
        raise NotFoundException("could't find the relation with id {}".format(relation_id))

    def get_bundle(self, bundle_id):
        bundle = self.bundles.get(bundle_id)
        if bundle is None:
            raise NotFoundException()

        records = list()
        for record_id in bundle.records:
            record = self.all_records.get(record_id)
            records.append(record)

        return DbBundle(records, bundle.bundle_record)

    def delete_relation(self, relation_id):

        for (from_key, relations) in self.all_relations.items():
            if relation_id in relations:
                del relations[relation_id]
                break

        return True

    def delete_record(self, record_id):

        if record_id not in self.all_nodes:
            raise NotFoundException()

        del self.all_nodes[record_id]

        return True

    def get_records_tail(self, properties_dict=dict(), metadata_dict=dict(), depth=None):
        #only transform dict into list
        return list(self._get_records_tail_internal(properties_dict,metadata_dict).values())

    def _get_records_tail_internal(self, properties_dict=dict(), metadata_dict=dict(), max_depth=None, current_depth=0, result_records = None):
        origin_records = self.get_records_by_filter(properties_dict, metadata_dict)

        if result_records is None:
            result_records = dict()
        if current_depth == max_depth:
            return dict()
        for record in origin_records:
            from_identifier = record.metadata[METADATA_KEY_IDENTIFIER]
            if from_identifier  in self.all_relations:
                for (relation_id,(to_identifier, attributes, metadata)) in self.all_relations[from_identifier].items():
                    #find to node
                    if to_identifier not in result_records:
                        (to_attributes, to_metadata)= self.all_nodes[to_identifier]
                        result_records.update({to_identifier: DbRecord(to_attributes, to_metadata)})

                        # add other nodes recursive
                        result_records.update(self._get_records_tail_internal(metadata_dict={METADATA_KEY_IDENTIFIER: to_identifier},current_depth=current_depth+1,result_records=result_records))

                    # add relation to result
                    result_records.update({relation_id: DbRelation(attributes, metadata)})

        return result_records

    def delete_records_by_filter(self, properties_dict=dict(), metadata_dict=dict()):
        #erase all if no filter set
        if len(properties_dict) == 0 and len(metadata_dict) == 0:
            del self.all_nodes
            self.all_nodes = dict()
            return True

        #erase only matching nodes
        records_to_delete = self.get_records_by_filter(properties_dict,metadata_dict)

        for record in records_to_delete:
            if not isinstance(record, DbRecord):
                continue
            identifier = record.metadata[METADATA_KEY_IDENTIFIER]

            if identifier not in self.all_nodes:
                raise  NotFoundException("We cant find the id ")
            del self.all_nodes[identifier]

        return True

    def get_bundle_records(self, bundle_identifier):

        bundle_records = dict()

        #get all nodes for the bundle
        for (from_identifier, relations) in self.all_relations.items():
            # search in all relations for a relation that points to the bundle and also fulfill the other constraints
            for (relation_id, (to_identifier,attributes,metadata)) in relations.items():

                if to_identifier == str(bundle_identifier):
                    #got potential bundle association, check prov:type to be sure

                    if metadata[METADATA_KEY_PROV_TYPE] == PROV_ASSOCIATION and str(attributes[PROV_TYPE]) == "prov:bundleAssociation":
                        (attributes, metadata) = self.all_nodes[from_identifier]
                        bundle_records.update({from_identifier: DbRecord(attributes,metadata)})

        #search for all relations between the bundle nodes
        for from_identifier in bundle_records.copy().keys():
            if from_identifier in self.all_relations:
                for (relation_id,  (to_identifier, attributes, metadata)) in self.all_relations[from_identifier].items():

                    # If the target of the relation is also in the bundle the relation belongs to the bundle
                    if to_identifier in bundle_records:
                        bundle_records.update({relation_id: DbRelation(attributes, metadata)})

                    elif metadata[METADATA_KEY_PROV_TYPE] == PROV_MENTION:
                        # prov mentions used to connect between bundles , see w3c bundle links
                            bundle_records.update({relation_id: DbRelation(attributes, metadata)})

        return list(bundle_records.values())

    def _check_attribute_metadata_filter(self, properties_filter, metadata_filter, attributes,metadata):

        match = True

        meta_encoded = encode_dict_values_to_primitive(metadata)
        attr_encoded = encode_dict_values_to_primitive(attributes)

        filter_meta_encoded = encode_dict_values_to_primitive(metadata_filter)
        filter_attr_encoded = encode_dict_values_to_primitive(properties_filter)

        # check properties
        for (key, value) in filter_attr_encoded.items():
            if key not in attr_encoded:
                return False

            elif attr_encoded[key] != value:
                return False

        # check metadata
        for (key, value) in filter_meta_encoded.items():
            if key not in meta_encoded:
                return False

            elif meta_encoded[key] != value:
                return False

        return True

    def get_records_by_filter(self, properties_dict=dict(), metadata_dict=dict()):

        return_records = list()
        return_keys = set()
        properties_filter_dict = encode_dict_values_to_primitive(properties_dict.copy())
        metadata_filter_dict = encode_dict_values_to_primitive(metadata_dict.copy())
        for (identifier,(attributes, metadata)) in self.all_nodes.items():

            if self._check_attribute_metadata_filter(properties_filter=properties_dict,
                                                     metadata_filter=metadata_dict,
                                                     metadata=metadata,
                                                     attributes=attributes):

                meta_encoded = encode_dict_values_to_primitive(metadata)
                attr_encoded = encode_dict_values_to_primitive(attributes)

                return_records.append(DbRecord(attr_encoded,meta_encoded))
                return_keys.add(identifier)

            else:
                #not match so dont add
                pass

        #get relations
        for (from_id,relations) in self.all_relations.items():

            if from_id in return_keys:

                for (relation_id, (to_id, attributes, metadata)) in relations.items():

                    attributes = encode_dict_values_to_primitive(attributes)
                    metadata = encode_dict_values_to_primitive(metadata)

                    return_records.append(DbRelation(attributes,metadata))

        return  return_records<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright 2008-2010 WebDriver committers
# Copyright 2008-2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import tempfile
import time
import shutil
import unittest
from selenium.webdriver.common.exceptions import NoSuchElementException
from selenium.webdriver.common.exceptions import NoSuchFrameException

def not_available_on_remote(func):
    def testMethod(self):
        print self.driver
        if type(self.driver) == 'remote':
            return lambda x: None
        else:
            return func(self)
    return testMethod

class ImplicitWaitTest(unittest.TestCase):

    def testShouldImplicitlyWaitForASingleElement(self):
        self._loadPage("dynamic")
        add = self.driver.find_element_by_id("adder")
        self.driver.implicitly_wait(3)
        add.click();
        self.driver.find_element_by_id("box0")  # All is well if this doesn't throw.

    def testShouldStillFailToFindAnElementWhenImplicitWaitsAreEnabled(self):
        self._loadPage("dynamic")
        self.driver.implicitly_wait(0.5)
        try:
            self.driver.find_element_by_id("box0")
            self.fail("Expected NoSuchElementException to have been thrown")
        except NoSuchElementException, e:
            pass
        except Exception, e:
            self.fail("Expected NoSuchElementException but got " + str(e))

    def testShouldReturnAfterFirstAttemptToFindOneAfterDisablingImplicitWaits(self):
        self._loadPage("dynamic")
        self.driver.implicitly_wait(3)
        self.driver.implicitly_wait(0)
        try:
            self.driver.find_element_by_id("box0")
            self.fail("Expected NoSuchElementException to have been thrown")
        except NoSuchElementException, e:
            pass
        except Exception, e:
            self.fail("Expected NoSuchElementException but got " + str(e))

    def testShouldImplicitlyWaitUntilAtLeastOneElementIsFoundWhenSearchingForMany(self):
        self._loadPage("dynamic")
        add = self.driver.find_element_by_id("adder")

        self.driver.implicitly_wait(2)
        add.click();
        add.click();

        elements = self.driver.find_elements_by_class_name("redbox")
        self.assertTrue(len(elements) >= 1)

    def testShouldStillFailToFindAnElemenstWhenImplicitWaitsAreEnabled(self):
        self._loadPage("dynamic")

        self.driver.implicitly_wait(0.5)
        elements = self.driver.find_elements_by_class_name("redbox")
        self.assertEqual(0, len(elements))

    def testShouldReturnAfterFirstAttemptToFindManyAfterDisablingImplicitWaits(self):
        self._loadPage("dynamic")
        add = self.driver.find_element_by_id("adder")
        self.driver.implicitly_wait(1.1)
        self.driver.implicitly_wait(0)
        add.click()
        elements = self.driver.find_elements_by_class_name("redbox")
        self.assertEqual(0, len(elements))

    def _pageURL(self, name):
        return "http://localhost:2310/common/%s.html" % name

    def _loadSimplePage(self):
        self._loadPage("simpleTest")

    def _loadPage(self, name):
        self.driver.get(self._pageURL(name))
<commit_msg>DavidBurns fix to webserver details in implicit tests<commit_after>#!/usr/bin/python

# Copyright 2008-2010 WebDriver committers
# Copyright 2008-2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import tempfile
import time
import shutil
import unittest
from selenium.webdriver.common.exceptions import NoSuchElementException
from selenium.webdriver.common.exceptions import NoSuchFrameException

def not_available_on_remote(func):
    def testMethod(self):
        print self.driver
        if type(self.driver) == 'remote':
            return lambda x: None
        else:
            return func(self)
    return testMethod

class ImplicitWaitTest(unittest.TestCase):

    def testShouldImplicitlyWaitForASingleElement(self):
        self._loadPage("dynamic")
        add = self.driver.find_element_by_id("adder")
        self.driver.implicitly_wait(3)
        add.click();
        self.driver.find_element_by_id("box0")  # All is well if this doesn't throw.

    def testShouldStillFailToFindAnElementWhenImplicitWaitsAreEnabled(self):
        self._loadPage("dynamic")
        self.driver.implicitly_wait(0.5)
        try:
            self.driver.find_element_by_id("box0")
            self.fail("Expected NoSuchElementException to have been thrown")
        except NoSuchElementException, e:
            pass
        except Exception, e:
            self.fail("Expected NoSuchElementException but got " + str(e))

    def testShouldReturnAfterFirstAttemptToFindOneAfterDisablingImplicitWaits(self):
        self._loadPage("dynamic")
        self.driver.implicitly_wait(3)
        self.driver.implicitly_wait(0)
        try:
            self.driver.find_element_by_id("box0")
            self.fail("Expected NoSuchElementException to have been thrown")
        except NoSuchElementException, e:
            pass
        except Exception, e:
            self.fail("Expected NoSuchElementException but got " + str(e))

    def testShouldImplicitlyWaitUntilAtLeastOneElementIsFoundWhenSearchingForMany(self):
        self._loadPage("dynamic")
        add = self.driver.find_element_by_id("adder")

        self.driver.implicitly_wait(2)
        add.click();
        add.click();

        elements = self.driver.find_elements_by_class_name("redbox")
        self.assertTrue(len(elements) >= 1)

    def testShouldStillFailToFindAnElemenstWhenImplicitWaitsAreEnabled(self):
        self._loadPage("dynamic")

        self.driver.implicitly_wait(0.5)
        elements = self.driver.find_elements_by_class_name("redbox")
        self.assertEqual(0, len(elements))

    def testShouldReturnAfterFirstAttemptToFindManyAfterDisablingImplicitWaits(self):
        self._loadPage("dynamic")
        add = self.driver.find_element_by_id("adder")
        self.driver.implicitly_wait(1.1)
        self.driver.implicitly_wait(0)
        add.click()
        elements = self.driver.find_elements_by_class_name("redbox")
        self.assertEqual(0, len(elements))

    def _pageURL(self, name):
        return "http://localhost:%d/%s.html" % (self.webserver.port, name)

    def _loadSimplePage(self):
        self._loadPage("simpleTest")

    def _loadPage(self, name):
        self.driver.get(self._pageURL(name))
<|endoftext|>


<commit_before>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np

import paddle.fluid.core as core
from op_test import OpTest
import paddle.fluid as fluid

SIGMOID_THRESHOLD_MIN = -40.0
SIGMOID_THRESHOLD_MAX = 13.0
EXP_MAX_INPUT = 40.0

def lstm_naive(
        input,
        w, ):
    seq_len, batch_size, hidden_size = input.shape

    offset = 0
    wi = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    wf = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    wc = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    wo = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    ri = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    rf = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    rc = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    ro = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size

    bi_1 = w[offset:offset + hidden_size]
    offset += hidden_size
    bf_1 = w[offset:offset + hidden_size]
    offset += hidden_size
    bc_1 = w[offset:offset + hidden_size]
    offset += hidden_size
    bo_1 = w[offset:offset + hidden_size]
    offset += hidden_size

    bi_2 = w[offset:offset + hidden_size]
    offset += hidden_size
    bf_2 = w[offset:offset + hidden_size]
    offset += hidden_size
    bc_2 = w[offset:offset + hidden_size]
    offset += hidden_size
    bo_2 = w[offset:offset + hidden_size]

    def sigmoid(x):
        y = np.copy(x)
        y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN
        y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX
        return 1. / (1. + np.exp(-y))

    def tanh(x):
        y = -2. * x
        y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT
        return (2. / (1. + np.exp(y))) - 1.

    output = []
    pre_h = np.zeros((batch_size, hidden_size), dtype=input.dtype)
    pre_c = np.zeros((batch_size, hidden_size), dtype=input.dtype)

    for i in range(seq_len):
        emb_1 = input[i]

        input_gate = sigmoid(
            np.matmul(emb_1, wi) + np.matmul(pre_h, ri) + bi_1 + bi_2)
        forget_gate = sigmoid(
            np.matmul(emb_1, wf) + np.matmul(pre_h, rf) + bf_1 + bf_2)
        output_gate = sigmoid(
            np.matmul(emb_1, wo) + np.matmul(pre_h, ro) + bo_1 + bo_2)
        c_t_temp = tanh(
            np.matmul(emb_1, wc) + np.matmul(pre_h, rc) + bc_1 + bc_2)
        new_c = input_gate * c_t_temp + forget_gate * pre_c
        new_h = output_gate * tanh(new_c)

        pre_h = new_h
        pre_c = new_c

        output.append(new_h)

    output = np.concatenate(output, -1)
    output = output.reshape((batch_size, -1, hidden_size))

    output = output.transpose((1, 0, 2))

    return output, pre_h, pre_c

class TestCUDNNLstmOp(OpTest):
    def setUp(self):
        self.op_type = "cudnn_lstm"
        self.dtype = np.float32

        num_steps = 50
        batch_size = 20
        hidden_size = 200

        input_weight_size = (hidden_size * hidden_size) * 4
        hidden_weight_size = (hidden_size * hidden_size) * 4
        weight_size = input_weight_size + hidden_weight_size
        weight_size += hidden_size * 8

        input = np.random.uniform(
            low=-0.1, high=0.1, size=(num_steps, batch_size,
                                      hidden_size)).astype(self.dtype)
        flat_w = np.random.uniform(
            low=-0.1, high=0.1, size=(weight_size)).astype(self.dtype)

        output, last_hidden, last_cell = lstm_naive(input, flat_w)

        init_h = np.zeros((batch_size, hidden_size), dtype=np.float32)
        init_c = np.zeros((batch_size, hidden_size), dtype=np.float32)
        scope = core.Scope()
        program = fluid.Program()
        block = program.global_block()

        cache_temp = block.create_var(
            name="Cache",
            persistable=True,
            type=core.VarDesc.VarType.RAW,
            stop_gradient=True)
        self.inputs = {
            'Input': OpTest.np_dtype_to_fluid_dtype(input),
            'W': OpTest.np_dtype_to_fluid_dtype(flat_w),
            'InitH': OpTest.np_dtype_to_fluid_dtype(init_h),
            'InitC': OpTest.np_dtype_to_fluid_dtype(init_c),
        }
        self.cache_name_list = ['Cache']
        self.attrs = {
            'max_len': num_steps,
            'dropout_prob': 0.0,
            'is_bidirec': False,
            'input_size': hidden_size,
            'hidden_size': hidden_size,
            'num_layers': 1,
        }
        self.outputs = {
            'Out': output,
            "last_h": last_hidden,
            'last_c': last_cell
        }

    def test_output_with_place(self):
        place = core.CUDAPlace(0)
        self.check_output_with_place(place, atol=1e-5)

if __name__ == '__main__':
    unittest.main()
<commit_msg>fix cpu test; test=develop<commit_after>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np

import paddle.fluid.core as core
from op_test import OpTest
import paddle.fluid as fluid

SIGMOID_THRESHOLD_MIN = -40.0
SIGMOID_THRESHOLD_MAX = 13.0
EXP_MAX_INPUT = 40.0

def lstm_naive(
        input,
        w, ):
    seq_len, batch_size, hidden_size = input.shape

    offset = 0
    wi = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    wf = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    wc = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    wo = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    ri = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    rf = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    rc = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size
    ro = w[offset:offset + hidden_size * hidden_size].reshape(
        (hidden_size, hidden_size)).transpose()
    offset += hidden_size * hidden_size

    bi_1 = w[offset:offset + hidden_size]
    offset += hidden_size
    bf_1 = w[offset:offset + hidden_size]
    offset += hidden_size
    bc_1 = w[offset:offset + hidden_size]
    offset += hidden_size
    bo_1 = w[offset:offset + hidden_size]
    offset += hidden_size

    bi_2 = w[offset:offset + hidden_size]
    offset += hidden_size
    bf_2 = w[offset:offset + hidden_size]
    offset += hidden_size
    bc_2 = w[offset:offset + hidden_size]
    offset += hidden_size
    bo_2 = w[offset:offset + hidden_size]

    def sigmoid(x):
        y = np.copy(x)
        y[x < SIGMOID_THRESHOLD_MIN] = SIGMOID_THRESHOLD_MIN
        y[x > SIGMOID_THRESHOLD_MAX] = SIGMOID_THRESHOLD_MAX
        return 1. / (1. + np.exp(-y))

    def tanh(x):
        y = -2. * x
        y[y > EXP_MAX_INPUT] = EXP_MAX_INPUT
        return (2. / (1. + np.exp(y))) - 1.

    output = []
    pre_h = np.zeros((batch_size, hidden_size), dtype=input.dtype)
    pre_c = np.zeros((batch_size, hidden_size), dtype=input.dtype)

    for i in range(seq_len):
        emb_1 = input[i]

        input_gate = sigmoid(
            np.matmul(emb_1, wi) + np.matmul(pre_h, ri) + bi_1 + bi_2)
        forget_gate = sigmoid(
            np.matmul(emb_1, wf) + np.matmul(pre_h, rf) + bf_1 + bf_2)
        output_gate = sigmoid(
            np.matmul(emb_1, wo) + np.matmul(pre_h, ro) + bo_1 + bo_2)
        c_t_temp = tanh(
            np.matmul(emb_1, wc) + np.matmul(pre_h, rc) + bc_1 + bc_2)
        new_c = input_gate * c_t_temp + forget_gate * pre_c
        new_h = output_gate * tanh(new_c)

        pre_h = new_h
        pre_c = new_c

        output.append(new_h)

    output = np.concatenate(output, -1)
    output = output.reshape((batch_size, -1, hidden_size))

    output = output.transpose((1, 0, 2))

    return output, pre_h, pre_c

class TestCUDNNLstmOp(OpTest):
    def setUp(self):
        self.op_type = "cudnn_lstm"
        self.dtype = np.float32

        num_steps = 50
        batch_size = 20
        hidden_size = 200

        input_weight_size = (hidden_size * hidden_size) * 4
        hidden_weight_size = (hidden_size * hidden_size) * 4
        weight_size = input_weight_size + hidden_weight_size
        weight_size += hidden_size * 8

        input = np.random.uniform(
            low=-0.1, high=0.1, size=(num_steps, batch_size,
                                      hidden_size)).astype(self.dtype)
        flat_w = np.random.uniform(
            low=-0.1, high=0.1, size=(weight_size)).astype(self.dtype)

        output, last_hidden, last_cell = lstm_naive(input, flat_w)

        init_h = np.zeros((batch_size, hidden_size), dtype=np.float32)
        init_c = np.zeros((batch_size, hidden_size), dtype=np.float32)
        scope = core.Scope()
        program = fluid.Program()
        block = program.global_block()

        cache_temp = block.create_var(
            name="Cache",
            persistable=True,
            type=core.VarDesc.VarType.RAW,
            stop_gradient=True)
        self.inputs = {
            'Input': OpTest.np_dtype_to_fluid_dtype(input),
            'W': OpTest.np_dtype_to_fluid_dtype(flat_w),
            'InitH': OpTest.np_dtype_to_fluid_dtype(init_h),
            'InitC': OpTest.np_dtype_to_fluid_dtype(init_c),
        }
        self.cache_name_list = ['Cache']
        self.attrs = {
            'max_len': num_steps,
            'dropout_prob': 0.0,
            'is_bidirec': False,
            'input_size': hidden_size,
            'hidden_size': hidden_size,
            'num_layers': 1,
        }
        self.outputs = {
            'Out': output,
            "last_h": last_hidden,
            'last_c': last_cell
        }

    def test_output_with_place(self):
        if not self.testcuda():
            pass
        place = core.CUDAPlace(0)
        self.check_output_with_place(place, atol=1e-5)

    def testcuda(self):
        return core.is_compiled_with_cuda()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from .BaggedClassifier import BaggedClassifier
from .Classifier import Classifier, load_classifier

__all__ = ['BaggedClassifier', 'Classifier', 'load_classifier']<commit_msg>Fixes import issue with BaggedClassifier<commit_after>from .Classifier import Classifier, load_classifier
from .BaggedClassifier import BaggedClassifier

__all__ = ['BaggedClassifier', 'Classifier', 'load_classifier']<|endoftext|>


<commit_before># lint_ignore=E501
config = {
    "shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/mobile/android/locales/all-locales",
    "products": {
        "installer": {
            "product-name": "Fennec-%(version)s",
            "ssl-only": False,
            "add-locales": True,
            "paths": {
                "android": {
                    "path": "/mobile/releases/%(version)s/android/:lang/fennec-%(version)s.:lang.android-arm.apk",
                    "bouncer-platform": "android",
                },
                "android-x86": {
                    "path": "/mobile/releases/%(version)s/android-x86/:lang/fennec-%(version)s.:lang.android-x86.apk",
                    "bouncer-platform": "android-x86",
                },
            },
        },
    },
}<commit_msg>Bug 1098359 - bouncer checks failing for fennec r=bhearsum<commit_after># lint_ignore=E501
config = {
    "shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/mobile/android/locales/all-locales",
    "products": {
        "installer": {
            "product-name": "Fennec-%(version)s",
            "ssl-only": False,
            "add-locales": True,
            "paths": {
                "android": {
                    "path": "/mobile/releases/%(version)s/android/:lang/fennec-%(version)s.:lang.android-arm.apk",
                    "bouncer-platform": "android",
                },
            },
        },
    },
}<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RRpostgresql(RPackage):
    """Database interface and PostgreSQL driver for R This package provides a
    Database Interface (DBI) compliant driver for R to access PostgreSQL
    database systems. In order to build and install this package from source,
    PostgreSQL itself must be present your system to provide PostgreSQL
    functionality via its libraries and header files. These files are provided
    as postgresql-devel package under some Linux distributions. On Microsoft
    Windows system the attached libpq library source will be used. A wiki and
    issue tracking system for the package are available at Google Code at
    https://code.google.com/p/rpostgresql/."""

    homepage = "https://code.google.com/p/rpostgresql/"
    url      = "https://cloud.r-project.org/src/contrib/RPostgreSQL_0.4-1.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/RPostgreSQL"

    version('0.6-2', sha256='080118647208bfa2621bcaac0d324891cc513e07618fa22e3c50ec2050e1b0d5')
    version('0.4-1', sha256='6292e37fa841670a3fb1a0950ceb83d15beb4631c3c532c8ce279d1c0d10bf79')

    depends_on('r@2.9.0:', type=('build', 'run'))
    depends_on('r-dbi@0.3:', type=('build', 'run'))
    depends_on('postgresql')
<commit_msg>r-rpostgresql: update config.guess reusing logic from autotools (#18917)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RRpostgresql(RPackage):
    """Database interface and PostgreSQL driver for R This package provides a
    Database Interface (DBI) compliant driver for R to access PostgreSQL
    database systems. In order to build and install this package from source,
    PostgreSQL itself must be present your system to provide PostgreSQL
    functionality via its libraries and header files. These files are provided
    as postgresql-devel package under some Linux distributions. On Microsoft
    Windows system the attached libpq library source will be used. A wiki and
    issue tracking system for the package are available at Google Code at
    https://code.google.com/p/rpostgresql/."""

    homepage = "https://code.google.com/p/rpostgresql/"
    url      = "https://cloud.r-project.org/src/contrib/RPostgreSQL_0.4-1.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/RPostgreSQL"

    version('0.6-2', sha256='080118647208bfa2621bcaac0d324891cc513e07618fa22e3c50ec2050e1b0d5')
    version('0.4-1', sha256='6292e37fa841670a3fb1a0950ceb83d15beb4631c3c532c8ce279d1c0d10bf79')

    depends_on('r@2.9.0:', type=('build', 'run'))
    depends_on('r-dbi@0.3:', type=('build', 'run'))
    depends_on('postgresql')

    depends_on('automake', type='build')

    patch_config_files = True

    @run_before('install')
    def patch_config_guess(self):
        AutotoolsPackage._do_patch_config_files(self)
<|endoftext|>


<commit_before>from smif import cli

from argparse import ArgumentParser

from smif.data_layer.file import (CSVDataStore, FileMetadataStore,
                                  ParquetDataStore, YamlConfigStore)

arguments = ['prepare-convert', 'energy_water_cp_cr']

parser = cli.parse_arguments()
args = parser.parse_args(arguments)

print('Now calling function {}'.format(args.func.__name__))

args.func(args)
<commit_msg>remove script test_convert.py used for testing<commit_after><|endoftext|>