<commit_before>"""This module tests POST requests to the Cloud Record endpoint."""

import glob
import logging
import os
import shutil

from django.core.urlresolvers import reverse
from django.test import Client, TestCase
from mock import Mock
from rest_framework.request import Request
from rest_framework.test import APIRequestFactory

from api.views.CloudRecordView import CloudRecordView

QPATH_TEST = '/tmp/django-test/'

class CloudRecordTest(TestCase):
    """Tests POST requests to the Cloud Record endpoint."""

    def setUp(self):
        """Prevent logging from appearing in test output."""
        logging.disable(logging.CRITICAL)

    def test_cloud_record_post_provider_banned(self):
        """Test that a banned provider on the provider list cannot POST."""
        example_dn = "/C=XX/O=XX/OU=XX/L=XX/CN=allowed_host.test"
        with self.settings(BANNED_FROM_POST=[example_dn]):

            # Mock the functionality of the provider list
            # Used in the underlying POST handling method
            CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)

            # Make (and check) the POST request
            self._check_record_post(MESSAGE, 403, dn=example_dn)

    def test_cloud_record_post_provider_special(self):
        """
        Test that a provider granted POST rights can, indeed, POST.

        Even if they aren't on the provider list.
        """
        example_dn = "/C=XX/O=XX/OU=XX/L=XX/CN=special_host.test"
        with self.settings(ALLOWED_TO_POST=[example_dn]):

            # Mock the functionality of the provider list
            # Used in the underlying POST handling method
            CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)

            # Make (and check) the POST request
            self._check_record_post(MESSAGE, 202, dn=example_dn)

    def test_cloud_record_post_provider_fail(self):
        """Test what happens if we fail to retrieve the providers."""
        # Mock the functionality of the provider list
        # Used in the underlying POST handling method
        # Shouldn't allow any POSTs,
        # i.e. we have failed to retrieve the providers list
        CloudRecordView._get_provider_list = Mock(return_value={})

        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 403)

    def test_cloud_record_post_403(self):
        """Test unknown provider POST request returns a 403 code."""
        # Mock the functionality of the provider list
        # Used in the underlying POST handling method
        # Allows only allowed_host.test to POST
        CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)

        example_dn = "/C=XX/O=XX/OU=XX/L=XX/CN=prohibited_host.test"

        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 403, dn=example_dn)

    def test_cloud_record_post_401(self):
        """Test certificate-less POST request returns a 401 code."""
        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 401, dn=None)

    def test_cloud_record_post_202(self):
        """Test POST request for content equality and a 202 return code."""
        # Mock the functionality of the provider list.
        # Used in the underlying POST handling method so that the
        # default dn used in _check_record_post is authorized
        # to make POST request.
        CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)
        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 202)

    def tearDown(self):
        """Delete any messages under QPATH and re-enable logging.INFO."""
        self._delete_messages(QPATH_TEST)
        logging.disable(logging.NOTSET)

    def _delete_messages(self, message_path):
        """Delete any messages under message_path."""
        if os.path.exists(message_path):
            shutil.rmtree(message_path)

    def _saved_messages(self, message_path):
        """Return a list of messages under message_path."""
        return glob.glob(message_path)

    def _check_record_post(self, message, expected_status,
                          dn='/C=XX/O=XX/OU=XX/L=XX/CN=allowed_host.test',
                          empaid='Test Process'):
        """
        Helper method to make a POST request.

        This method makes a POST request that is authenticated by
        the supplied dn and identified (in the queue logs) by empaid.
        It checks the message as receieved is the same as
        the message saved.

        By default,
         - dn is /C=XX/O=XX/OU=XX/L=XX/CN=allowed_host.test
         - empaid is 'Test Process'

        These defaults are set so that the majority of tests can use this
        method with minimal parameters in the method call, but that edge
        cases can still be tested using this method.

        i.e. a successful POST need only call
          self._check_record_post(MESSAGE, 202)
        """
        # This avoids the test writing to the QPATH
        # set in apel_rest/settings.py
        with self.settings(QPATH=QPATH_TEST):

            # Make the POST request
            test_client = Client()
            url = reverse('CloudRecordView')

            if dn is not None:
                # Include the in dn SSL_CLIENT_S_DN so the
                # request is able to be authenticated
                response = test_client.post(url,
                                            message,
                                            content_type="text/plain",
                                            HTTP_EMPA_ID=empaid,
                                            SSL_CLIENT_S_DN=dn)
            else:
                # Omit the SSL_CLIENT_S_DN header so the request is
                # made without any certificate (and not authenticated)
                response = test_client.post(url,
                                            message,
                                            content_type="text/plain",
                                            HTTP_EMPA_ID=empaid)

            # check the expected response code has been received
            self.assertEqual(response.status_code, expected_status)

            # If the message was expected to be successfull, check for
            # equality between the sent message and the saved message
            if expected_status == 202:
                # get saved messages under QPATH_TEST
                messages = self._saved_messages('%s*/*/*/body' % QPATH_TEST)

                # check one and only one message body saved
                self.assertEqual(len(messages), 1)

                # get message content
                # can unpack sequence because we have asserted length 1
                [message_path] = messages
                with file(message_path) as message_file:
                    message_content = message_file.read()

                # check saved message content
                self.assertEqual(message, message_content)

PROVIDERS = {'total_rows': 735,
             'offset': 695,
             'rows': [
                 {'id': '1',
                  'key': ['service'],
                  'value':{
                      'sitename': 'TEST2',
                      'provider_id': 'TEST2',
                      'type': 'cloud'}},
                 {'id': '2',
                  'key': ['service'],
                  'value':{
                      'sitename': 'TEST',
                      'provider_id': 'TEST',
                      'hostname': 'allowed_host.test',
                      'type': 'cloud'}}]}

MESSAGE = """APEL-cloud-message: v0.2
VMUUID: TestVM1 2013-02-25 17:37:27+00:00
SiteName: CESGA
MachineName: one-2421
LocalUserId: 19
LocalGroupId: 101
GlobalUserName: NULL
FQAN: NULL
Status: completed
StartTime: 1361813847
EndTime: 1361813870
SuspendDuration: NULL
WallDuration: NULL
CpuDuration: NULL
CpuCount: 1
NetworkType: NULL
NetworkInbound: 0
NetworkOutbound: 0
Memory: 1000
Disk: NULL
StorageRecordId: NULL
ImageId: NULL
CloudType: OpenNebula
%%
VMUUID: TestVM1 2015-06-25 17:37:27+00:00
SiteName: CESGA
MachineName: one-2422
LocalUserId: 13
LocalGroupId: 131
GlobalUserName: NULL
FQAN: NULL
Status: completed
StartTime: 1361413847
EndTime: 1361811870
SuspendDuration: NULL
WallDuration: NULL
CpuDuration: NULL
CpuCount: 1
NetworkType: NULL
NetworkInbound: 0
NetworkOutbound: 0
Memory: 1000
Disk: NULL
StorageRecordId: NULL
ImageId: NULL
CloudType: OpenNebula
%%"""
<commit_msg>Reword comment in _check_record_post<commit_after>"""This module tests POST requests to the Cloud Record endpoint."""

import glob
import logging
import os
import shutil

from django.core.urlresolvers import reverse
from django.test import Client, TestCase
from mock import Mock
from rest_framework.request import Request
from rest_framework.test import APIRequestFactory

from api.views.CloudRecordView import CloudRecordView

QPATH_TEST = '/tmp/django-test/'

class CloudRecordTest(TestCase):
    """Tests POST requests to the Cloud Record endpoint."""

    def setUp(self):
        """Prevent logging from appearing in test output."""
        logging.disable(logging.CRITICAL)

    def test_cloud_record_post_provider_banned(self):
        """Test that a banned provider on the provider list cannot POST."""
        example_dn = "/C=XX/O=XX/OU=XX/L=XX/CN=allowed_host.test"
        with self.settings(BANNED_FROM_POST=[example_dn]):

            # Mock the functionality of the provider list
            # Used in the underlying POST handling method
            CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)

            # Make (and check) the POST request
            self._check_record_post(MESSAGE, 403, dn=example_dn)

    def test_cloud_record_post_provider_special(self):
        """
        Test that a provider granted POST rights can, indeed, POST.

        Even if they aren't on the provider list.
        """
        example_dn = "/C=XX/O=XX/OU=XX/L=XX/CN=special_host.test"
        with self.settings(ALLOWED_TO_POST=[example_dn]):

            # Mock the functionality of the provider list
            # Used in the underlying POST handling method
            CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)

            # Make (and check) the POST request
            self._check_record_post(MESSAGE, 202, dn=example_dn)

    def test_cloud_record_post_provider_fail(self):
        """Test what happens if we fail to retrieve the providers."""
        # Mock the functionality of the provider list
        # Used in the underlying POST handling method
        # Shouldn't allow any POSTs,
        # i.e. we have failed to retrieve the providers list
        CloudRecordView._get_provider_list = Mock(return_value={})

        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 403)

    def test_cloud_record_post_403(self):
        """Test unknown provider POST request returns a 403 code."""
        # Mock the functionality of the provider list
        # Used in the underlying POST handling method
        # Allows only allowed_host.test to POST
        CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)

        example_dn = "/C=XX/O=XX/OU=XX/L=XX/CN=prohibited_host.test"

        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 403, dn=example_dn)

    def test_cloud_record_post_401(self):
        """Test certificate-less POST request returns a 401 code."""
        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 401, dn=None)

    def test_cloud_record_post_202(self):
        """Test POST request for content equality and a 202 return code."""
        # Mock the functionality of the provider list.
        # Used in the underlying POST handling method so that the
        # default dn used in _check_record_post is authorized
        # to make POST request.
        CloudRecordView._get_provider_list = Mock(return_value=PROVIDERS)
        # Make (and check) the POST request
        self._check_record_post(MESSAGE, 202)

    def tearDown(self):
        """Delete any messages under QPATH and re-enable logging.INFO."""
        self._delete_messages(QPATH_TEST)
        logging.disable(logging.NOTSET)

    def _delete_messages(self, message_path):
        """Delete any messages under message_path."""
        if os.path.exists(message_path):
            shutil.rmtree(message_path)

    def _saved_messages(self, message_path):
        """Return a list of messages under message_path."""
        return glob.glob(message_path)

    def _check_record_post(self, message, expected_status,
                          dn='/C=XX/O=XX/OU=XX/L=XX/CN=allowed_host.test',
                          empaid='Test Process'):
        """
        Helper method to make a POST request.

        This method makes a POST request that is authenticated by
        the supplied dn and identified (in the queue logs) by empaid.
        It checks the message as receieved is the same as
        the message saved.

        By default,
         - dn is /C=XX/O=XX/OU=XX/L=XX/CN=allowed_host.test
         - empaid is 'Test Process'

        These defaults are set so that the majority of tests can use this
        method with minimal parameters in the method call, but that edge
        cases can still be tested using this method.

        i.e. a successful POST need only call
          self._check_record_post(MESSAGE, 202)
        """
        # This avoids the test writing to the QPATH
        # set in apel_rest/settings.py
        with self.settings(QPATH=QPATH_TEST):

            # Make the POST request
            test_client = Client()
            url = reverse('CloudRecordView')

            if dn is not None:
                # Include the dn in SSL_CLIENT_S_DN so the
                # request is able to be authenticated
                response = test_client.post(url,
                                            message,
                                            content_type="text/plain",
                                            HTTP_EMPA_ID=empaid,
                                            SSL_CLIENT_S_DN=dn)
            else:
                # Omit the SSL_CLIENT_S_DN header so the request is
                # made without any certificate (and not authenticated)
                response = test_client.post(url,
                                            message,
                                            content_type="text/plain",
                                            HTTP_EMPA_ID=empaid)

            # check the expected response code has been received
            self.assertEqual(response.status_code, expected_status)

            # If the message was expected to be successfull, check for
            # equality between the sent message and the saved message
            if expected_status == 202:
                # get saved messages under QPATH_TEST
                messages = self._saved_messages('%s*/*/*/body' % QPATH_TEST)

                # check one and only one message body saved
                self.assertEqual(len(messages), 1)

                # get message content
                # can unpack sequence because we have asserted length 1
                [message_path] = messages
                with file(message_path) as message_file:
                    message_content = message_file.read()

                # check saved message content
                self.assertEqual(message, message_content)

PROVIDERS = {'total_rows': 735,
             'offset': 695,
             'rows': [
                 {'id': '1',
                  'key': ['service'],
                  'value':{
                      'sitename': 'TEST2',
                      'provider_id': 'TEST2',
                      'type': 'cloud'}},
                 {'id': '2',
                  'key': ['service'],
                  'value':{
                      'sitename': 'TEST',
                      'provider_id': 'TEST',
                      'hostname': 'allowed_host.test',
                      'type': 'cloud'}}]}

MESSAGE = """APEL-cloud-message: v0.2
VMUUID: TestVM1 2013-02-25 17:37:27+00:00
SiteName: CESGA
MachineName: one-2421
LocalUserId: 19
LocalGroupId: 101
GlobalUserName: NULL
FQAN: NULL
Status: completed
StartTime: 1361813847
EndTime: 1361813870
SuspendDuration: NULL
WallDuration: NULL
CpuDuration: NULL
CpuCount: 1
NetworkType: NULL
NetworkInbound: 0
NetworkOutbound: 0
Memory: 1000
Disk: NULL
StorageRecordId: NULL
ImageId: NULL
CloudType: OpenNebula
%%
VMUUID: TestVM1 2015-06-25 17:37:27+00:00
SiteName: CESGA
MachineName: one-2422
LocalUserId: 13
LocalGroupId: 131
GlobalUserName: NULL
FQAN: NULL
Status: completed
StartTime: 1361413847
EndTime: 1361811870
SuspendDuration: NULL
WallDuration: NULL
CpuDuration: NULL
CpuCount: 1
NetworkType: NULL
NetworkInbound: 0
NetworkOutbound: 0
Memory: 1000
Disk: NULL
StorageRecordId: NULL
ImageId: NULL
CloudType: OpenNebula
%%"""
<|endoftext|>


<commit_before># Author: xiaotaw@qq.com (Any bug report is welcome)
# Time: Aug 2016
# Addr: Shenzhen
# Description: define functions and parameters related to input data

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy
import cPickle
import random # for random.shuffle()

import tensorflow as tf

data_dir = "data_files"
mgfp_dir = "mgfp_files"

record_bytes = 2048

def dense_to_one_hot(labels_dense, num_classes=2, dtype=numpy.int):
  """Convert class labels from scalars to one-hot vectors.
  """
  num_labels = labels_dense.shape[0]
  index_offset = numpy.arange(num_labels) * num_classes
  labels_one_hot = numpy.zeros((num_labels, num_classes))
  labels_one_hot.flat[index_offset + labels_dense.ravel().astype(dtype)] = 1
  # print(labels_one_hot)
  return labels_one_hot

class DataSet(object):
  """dataset class, contains compds and labels,
     and also provides method to generate a next batch of data.
  """
  def __init__(self, compds, labels, to_one_hot=True, dtype=numpy.float32, norm=False):
    assert compds.shape[0] == labels.shape[0], "shape don't match"
    self.compds = compds.astype(dtype)
    if to_one_hot:
      self.labels = dense_to_one_hot(labels)
    else:
      self.labels = labels
    self.size = compds.shape[0]
    self.begin = 0
    self.end = 0
    if norm:
      self.normalize()

  def generate_batch(self, batch_size):
    assert self.compds.shape[0] == self.labels.shape[0], "shape don't match"
    assert batch_size <= self.size, "too big batch_size"
    self.end = self.begin + batch_size
    if self.end > self.size:
      self.shuffle()
      self.begin = 0
      self.end = batch_size
    compds_batch = self.compds[self.begin: self.end]
    labels_batch = self.labels[self.begin: self.end]
    self.begin = self.end
    return compds_batch, labels_batch

  def shuffle(self):
    perm = range(self.size)
    random.shuffle(perm)
    self.compds = self.compds[perm]
    self.labels = self.labels[perm]

  def normalize(self):
    self.compds = self.compds / self.compds.max(axis=1).reshape([self.compds.shape[0],1])

def dense_to_one_hot(labels_dense, num_classes=2, dtype=numpy.int):
  """Convert class labels from scalars to one-hot vectors.
  """
  num_labels = labels_dense.shape[0]
  index_offset = numpy.arange(num_labels) * num_classes
  labels_one_hot = numpy.zeros((num_labels, num_classes))
  labels_one_hot.flat[index_offset + labels_dense.ravel().astype(dtype)] = 1
  # print(labels_one_hot)
  return labels_one_hot

def get_inputs_by_cpickle(pkl_filename, clip=True):
  """read dataset from file according to target name
     note: target.pkl are 2048 length mgfp code files, which were generated by 'cPickle.dump'.
  Args:
    target: a string of the protein kinase target name
  """
  with open(pkl_filename, "rb") as pkl_file:
    data = cPickle.load(pkl_file)
    if clip:
      numpy.clip(data, 0, 1, out=data)
      return DataSet(data[:, 1:], data[:, :1], to_one_hot=True)
    else:
      return DataSet(data[:, 1:], data[:, :1], to_one_hot=True, norm=True)

""" not used
def get_compds_by_filequeue(batch_size):
  """"""
  target_list = ["cdk2", "egfr_erbB1", "gsk3b", "hgfr", "map_k_p38a",
                 "tpk_lck", "tpk_src", "vegfr2", "pubchem_neg_sample"]
  reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
  fq = {}
  value = {}
  records = {}
  for t in target_list:
    fq[t] = tf.train.string_input_producer(["data_files/" + t + ".bin"])
    _, value[t] = reader.read(fq[t])
    records[t] = tf.decode_raw(value[t], tf.uint8)

  compds = tf.train.batch(records, batch_size=batch_size, num_threads=1, capacity=1000 + 3 * batch_size, shapes=[(2048,)] * len(records))

  return compds
"""

<commit_msg>create pos/neg dataset class, using h5py files as input, split data into train and cv<commit_after># Author: xiaotaw@qq.com (Any bug report is welcome)
# Time: Aug 2016
# Addr: Shenzhen
# Description: define functions and parameters related to input data

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import h5py
import time
import random

import numpy as np

from scipy import sparse

vec_len = 2039
data_dir = "../data_files"
h5_dir = os.path.join(data_dir, "h5_files")

def dense_to_one_hot(labels_dense, num_classes=2, dtype=np.int):
  """Convert class labels from scalars to one-hot vectors.
  """
  num_labels = labels_dense.shape[0]
  index_offset = np.arange(num_labels) * num_classes
  labels_one_hot = np.zeros((num_labels, num_classes))
  labels_one_hot.flat[index_offset + labels_dense.ravel().astype(dtype)] = 1
  return labels_one_hot

class Dataset(object):
  def __init__(self, size, is_shuffle=False, fold=10):

    self.size = size
    self.perm = np.array(range(self.size))
    if is_shuffle:
      random.shuffle(self.perm)

    self.train_size = int(self.size * (1.0 - 1.0 / fold))
    self.train_perm = self.perm[range(self.train_size)]
    self.train_begin = 0
    self.train_end = 0

    self.test_perm = self.perm[range(self.train_size, self.size)]

  def generate_perm_for_train_batch(self, batch_size):
    self.train_begin = self.train_end
    self.train_end += batch_size
    if self.train_end > self.train_size:
      random.shuffle(self.train_perm)
      self.train_begin = 0
      self.train_end = batch_size
    perm = self.train_perm[self.train_begin: self.train_end]
    return perm

class PosDataset(Dataset):
  def __init__(self, target, dtype=np.float32):
    # open h5 file
    self.h5_fn = os.path.join(h5_dir, target + ".h5")
    self.h5 = h5py.File(self.h5_fn, "r")
    # read ids
    self.ids = self.h5["chembl_id"].value
    # read 3 fp, and stack as feauture
    ap = sparse.csr_matrix((self.h5["ap"]["data"], self.h5["ap"]["indices"], self.h5["ap"]["indptr"]), shape=[len(self.h5["ap"]["indptr"]) - 1, vec_len])
    mg = sparse.csr_matrix((self.h5["mg"]["data"], self.h5["mg"]["indices"], self.h5["mg"]["indptr"]), shape=[len(self.h5["mg"]["indptr"]) - 1, vec_len])
    tt = sparse.csr_matrix((self.h5["tt"]["data"], self.h5["tt"]["indices"], self.h5["tt"]["indptr"]), shape=[len(self.h5["tt"]["indptr"]) - 1, vec_len])
    self.features = sparse.hstack([ap, mg, tt]).toarray()
    # label
    labels = self.h5["label"].value
    self.labels = dense_to_one_hot(labels)
    # year
    if "year" in self.h5.keys():
      self.years = self.h5["year"].value
    else:
      self.years = None
    # close h5 file
    self.h5.close()
    # dtype
    self.dtype = dtype
    # pre_process
    #self.features = np.log10(1.0 + self.features).astype(self.dtype)
    self.features = np.clip(self.features, 0, 1).astype(self.dtype)
    #
    Dataset.__init__(self, self.features.shape[0])

  def next_train_batch(self, batch_size):
    perm = self.generate_perm_for_train_batch(batch_size)
    return self.features[perm], self.labels[perm]

class NegDataset(Dataset):
  def __init__(self, target_list, dtype=np.float32):
    # open h5 file
    self.h5_fn = os.path.join(h5_dir, "pubchem_neg_sample.h5")
    self.h5 = h5py.File(self.h5_fn, "r")
    # read ids
    self.ids = self.h5["chembl_id"].value
    # read 3 fp, and stack as feauture
    ap = sparse.csr_matrix((self.h5["ap"]["data"], self.h5["ap"]["indices"], self.h5["ap"]["indptr"]), shape=[len(self.h5["ap"]["indptr"]) - 1, vec_len])
    mg = sparse.csr_matrix((self.h5["mg"]["data"], self.h5["mg"]["indices"], self.h5["mg"]["indptr"]), shape=[len(self.h5["mg"]["indptr"]) - 1, vec_len])
    tt = sparse.csr_matrix((self.h5["tt"]["data"], self.h5["tt"]["indices"], self.h5["tt"]["indptr"]), shape=[len(self.h5["tt"]["indptr"]) - 1, vec_len])
    self.features = sparse.hstack([ap, mg, tt]).toarray()
    # label(mask)
    self.mask_dict = {}
    for target in target_list:
      #mask = self.h5["mask"][target].value
      mask = self.h5["cliped_mask"][target].value
      self.mask_dict[target] = dense_to_one_hot(mask)
    # close h5 file
    self.h5.close()
    # dtype
    self.dtype = dtype
    # pre_process
    #self.features = np.log10(1.0 + self.features).astype(self.dtype)
    self.features = np.clip(self.features, 0, 1).astype(self.dtype)
    #
    Dataset.__init__(self, self.features.shape[0])

  def next_train_batch(self, target, batch_size):
    perm = self.generate_perm_for_train_batch(batch_size)
    return self.features[perm], self.mask_dict[target][perm]

class Datasets(object):
  """dataset class, contains compds and labels,
     and also provides method to generate a next batch of data.
  """
  def __init__(self, target_list):

    self.neg = NegDataset(target_list)

    # read pos sample
    # and generate mask for neg sample(it's very complex, and I am not going to tell you the details)
    self.pos = {}
    for target in target_list:
      self.pos[target] = PosDataset(target)

  def next_train_batch(self, target, pos_batch_size, neg_batch_size):
    pos_feature_batch, pos_label_batch = self.pos[target].next_train_batch(pos_batch_size)
    neg_feature_batch, neg_label_batch = self.neg.next_train_batch(target, neg_batch_size)
    return np.vstack([pos_feature_batch, neg_feature_batch]), np.vstack([pos_label_batch, neg_label_batch])

"""
import pk_input as pki
target_list = ["cdk2", "egfr_erbB1", "gsk3b", "hgfr", "map_k_p38a", "tpk_lck", "tpk_src", "vegfr2"]
d = pki.Datasets(target_list)

for step in range(20 * 500):
  for target in target_list:
    compds_batch, labels_batch = d.next_train_batch(target, 128, 128)
    if np.isnan(compds_batch).sum() > 0 or (step % 500) == 0:
      print("%7d %3.2f %3.2f %3.2f %3.2f" % (step, compds_batch.min(), compds_batch.max(), labels_batch.min(), labels_batch.max()))

"""

if __name__ == "__main__":
  target_list = ["cdk2", "egfr_erbB1", "gsk3b", "hgfr", "map_k_p38a", "tpk_lck", "tpk_src", "vegfr2"]

<|endoftext|>


<commit_before>#!/usr/bin/python
# Author: xiaotaw@qq.com (Any bug report is welcome)
# Time: Aug 2016
# Addr: Shenzhen
# Description: train pk model

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import time
import datetime
import math
import numpy
import random
import tensorflow as tf

import pk_input
import pk_model

def train(target_list, train_from = 0):

  # model parameters
  batch_size = 128
  ckpt_dir = "ckpt_files"
  log_dir = "log_files"
  max_step = 5000
  keep_prob = 0.8
  start_learning_rate = 0.05
  decay_step = 5000
  decay_rate = 0.7

  # train log file
  log_path = os.path.join(log_dir, "pk_train.log")
  logfile = open(log_path, 'w')
  logfile.write("train starts at: %s\n" % datetime.datetime.now())

  # get input dataset
  train_dataset_dict = dict()
  test_dataset_dict = dict()
  for target in target_list:
    train_dataset_dict[target] = pk_input.get_inputs_by_cpickle("data_files/pkl_files/" + target + "_train.pkl", clip=True)
    test_dataset_dict[target] = pk_input.get_inputs_by_cpickle("data_files/pkl_files/" + target + "_test.pkl", clip=True)

  neg_dataset = pk_input.get_inputs_by_cpickle("data_files/pkl_files/pubchem_neg_sample.pkl", clip=True)

  # train the model
  with tf.Graph().as_default(), tf.device("/gpu:0"):

    # exponential decay learning rate
    global_step = tf.Variable(train_from, trainable=False)
    learning_rate = tf.train.exponential_decay(start_learning_rate, global_step, decay_step, decay_rate)

    # build the model
    input_placeholder = tf.placeholder(tf.float32, shape = (None, 8192))
    label_placeholder = tf.placeholder(tf.float32, shape = (None, 2))
    # build the "Tree" with a mutual "Term" and several "Branches"
    base = pk_model.term(input_placeholder)
    softmax_dict = dict()
    wd_loss_dict = dict()
    x_entropy_dict = dict()
    loss_dict = dict()
    accuracy_dict = dict()
    train_op_dict = dict()
    for target in target_list:
      # compute softmax
      softmax_dict[target] = pk_model.branch(target, base)
      # compute loss.
      wd_loss_dict[target] = tf.add_n(tf.get_collection("term_wd_loss") + tf.get_collection(target+"_wd_loss"))
      x_entropy_dict[target] = pk_model.x_entropy(softmax_dict[target], label_placeholder, target)
      loss_dict[target]  = tf.add(wd_loss_dict[target], x_entropy_dict[target])
      # compute accuracy
      accuracy_dict[target] = pk_model.accuracy(softmax_dict[target], label_placeholder, target)
      # train op
      train_op_dict[target] = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_dict[target], global_step=global_step)

    # create a saver.
    saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=None)

    # start running operations on the Graph.
    config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
    config.gpu_options.per_process_gpu_memory_fraction = 0.5
    sess = tf.Session(config=config)

    # initialize all variables at first.
    sess.run(tf.initialize_all_variables())

    # define train batch
    perm = range(batch_size * 2)
    compds_batch = numpy.zeros([batch_size * 2, 8192])
    labels_batch = numpy.zeros([batch_size * 2, 2])

    # train with max step
    print("  step g_step wdloss x_loss learn_rate    TP    FN    TN    FP    SEN    SPE    ACC    MCC t1-t0 t2-t1  target")
    for step in xrange(0, max_step):
      for target in target_list:
        t0 = time.time()
        compds_batch[: batch_size], labels_batch[: batch_size] = neg_dataset.generate_batch(batch_size)
        compds_batch[batch_size: ], labels_batch[batch_size: ] = train_dataset_dict[target].generate_batch(batch_size)
        random.shuffle(perm)
        compds_batch = compds_batch[perm]
        labels_batch = labels_batch[perm]

        _ = sess.run(train_op_dict[target], feed_dict={input_placeholder: compds_batch, label_placeholder: labels_batch})

        t1 = float(time.time())

        # compute performance
        if step % 100 ==0 or (step + 1) == max_step:
          LV, XLV, LR, ACC, prediction, label_dense = sess.run(
            [wd_loss_dict[target],
             x_entropy_dict[target],
             learning_rate,
             accuracy_dict[target],
             tf.argmax(softmax_dict[target], 1),
             tf.argmax(labels_batch, 1)],
            feed_dict = {
              input_placeholder: compds_batch,
              label_placeholder: labels_batch,
            }
          )

          TP, TN, FP, FN, SEN, SPE, MCC = pk_model.compute_performance(label_dense, prediction)

          t2 = float(time.time())

          # print to file and screen
          format_str = "%6d %6d %6.3f %6.3f %10.3f %5d %5d %5d %5d %6.3f %6.3f %6.3f %6.3f %5.3f %5.3f %s"
          logfile.write(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))
          logfile.write('\n')
          print(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))

        # save the model checkpoint periodically.
        if step % 1000 == 0 or (step + 1) == max_step:
          checkpoint_path = os.path.join(ckpt_dir, 'model.ckpt')
          saver.save(sess, checkpoint_path, global_step=global_step, write_meta_graph=False)

    for target in target_list:
      # the whole train
      compds_batch = numpy.vstack([train_dataset_dict[target].compds, neg_dataset.compds])
      labels_batch = numpy.vstack([train_dataset_dict[target].labels, neg_dataset.labels])
      LV, XLV, LR, ACC, prediction, label_dense = sess.run(
        [wd_loss_dict[target],
         x_entropy_dict[target],
         learning_rate,
         accuracy_dict[target],
         tf.argmax(softmax_dict[target], 1),
         tf.argmax(labels_batch, 1)],
        feed_dict = {
          input_placeholder: compds_batch,
          label_placeholder: labels_batch,
        }
      )

      TP, TN, FP, FN, SEN, SPE, MCC = pk_model.compute_performance(label_dense, prediction)

      # print to file and screen
      format_str = "%6d %6d %6.3f %6.3f %10.3f %5d %5d %5d %5d %6.3f %6.3f %6.3f %6.3f %5.3f %5.3f %s"
      logfile.write(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))
      logfile.write('\n')
      print(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))

    for target in target_list:
      # the whole test
      LV, XLV, LR, ACC, prediction, label_dense = sess.run(
        [wd_loss_dict[target],
         x_entropy_dict[target],
         learning_rate,
         accuracy_dict[target],
         tf.argmax(softmax_dict[target], 1),
         tf.argmax(test_dataset_dict[target].labels, 1)],
        feed_dict = {
          input_placeholder: test_dataset_dict[target].compds,
          label_placeholder: test_dataset_dict[target].labels,
        }
      )

      TP, TN, FP, FN, SEN, SPE, MCC = pk_model.compute_performance(label_dense, prediction)

      # print to file and screen
      format_str = "%6d %6d %6.3f %6.3f %10.3f %5d %5d %5d %5d %6.3f %6.3f %6.3f %6.3f %5.3f %5.3f %s"
      logfile.write(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))
      logfile.write('\n')
      print(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))

  logfile.write("train ends at: %s\n" % datetime.datetime.now())
  logfile.close()

if __name__ == "__main__":

  target_list = ["cdk2", "egfr_erbB1", "gsk3b", "hgfr",
                 "map_k_p38a", "tpk_lck", "tpk_src", "vegfr2"]

  train(target_list)

<commit_msg>train the model using the class Datasets from pk_input<commit_after>#!/usr/bin/python
# Author: xiaotaw@qq.com (Any bug report is welcome)
# Time: Aug 2016
# Addr: Shenzhen
# Description: train pk model

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import time
import datetime
import math
import numpy
import random
import tensorflow as tf

import pk_input as pki
import pk_model

def train(target_list, train_from = 0):

  # dataset
  d = pki.Datasets(target_list)

  # model parameters
  pos_batch_size = 256
  neg_batch_size = pos_batch_size * 10 # the mean number of neg sample is 25.23 times as many as pos's.
  ckpt_dir = "ckpt_files/%d_%d" % (pos_batch_size, neg_batch_size)
  if not os.path.exists(ckpt_dir):
    os.makedirs(ckpt_dir)
  log_dir = "log_files"
  if not os.path.exists(log_dir):
    os.mkdir(log_dir)
  step_per_epoch = d.neg.size / pos_batch_size
  max_step = int(5.0 * step_per_epoch)
  keep_prob = 0.8
  start_learning_rate = 0.05
  decay_step = step_per_epoch * 8
  decay_rate = 0.7

  input_vec_len = 6117

  # train log file
  log_path = os.path.join(log_dir, "pk_train_%d_%d.log" % (pos_batch_size, neg_batch_size))
  logfile = open(log_path, 'w')
  logfile.write("train starts at: %s\n" % datetime.datetime.now())

  # train the model
  with tf.Graph().as_default(), tf.device("/gpu:2"):

    # exponential decay learning rate
    global_step = tf.Variable(train_from, trainable=False)
    learning_rate = tf.train.exponential_decay(start_learning_rate, global_step, decay_step, decay_rate)

    # build the model
    input_placeholder = tf.placeholder(tf.float32, shape = (None, input_vec_len))
    label_placeholder = tf.placeholder(tf.float32, shape = (None, 2))
    # build the "Tree" with a mutual "Term" and several "Branches"
    base = pk_model.term(input_placeholder)
    softmax_dict = dict()
    wd_loss_dict = dict()
    x_entropy_dict = dict()
    loss_dict = dict()
    accuracy_dict = dict()
    train_op_dict = dict()
    for target in target_list:
      # compute softmax
      softmax_dict[target] = pk_model.branch(target, base)
      # compute loss.
      wd_loss_dict[target] = tf.add_n(tf.get_collection("term_wd_loss") + tf.get_collection(target+"_wd_loss"))
      x_entropy_dict[target] = pk_model.x_entropy(softmax_dict[target], label_placeholder, target)
      loss_dict[target]  = tf.add(wd_loss_dict[target], x_entropy_dict[target])
      # compute accuracy
      accuracy_dict[target] = pk_model.accuracy(softmax_dict[target], label_placeholder, target)
      # train op
      train_op_dict[target] = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_dict[target], global_step=global_step)

    # create a saver.
    saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=None)

    # start running operations on the Graph.
    config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
    config.gpu_options.per_process_gpu_memory_fraction = 0.5
    sess = tf.Session(config=config)

    # initialize all variables at first.
    sess.run(tf.initialize_all_variables())
    if train_from != 0:
      saver.restore(sess, "ckpt_files/256_256/pk.ckpt-" + str(train_from))

    # train with max step
    print("  step g_step wdloss x_loss learn_rate    TP    FN    TN    FP    SEN    SPE    ACC    MCC t1-t0 t2-t1 t3-t2  target")
    for step in xrange(max_step):
      for target in target_list:
        t0 = time.time()
        compds_batch, labels_batch = d.next_train_batch(target, pos_batch_size, neg_batch_size)

        t1 = float(time.time())
        _ = sess.run(train_op_dict[target], feed_dict={input_placeholder: compds_batch, label_placeholder: labels_batch})

        t2 = float(time.time())
        # compute performance
        if step % 100 ==0 or (step + 1) == max_step:
          LV, XLV, LR, ACC, prediction, label_dense = sess.run(
            [wd_loss_dict[target],
             x_entropy_dict[target],
             learning_rate,
             accuracy_dict[target],
             tf.argmax(softmax_dict[target], 1),
             tf.argmax(labels_batch, 1)],
            feed_dict = {
              input_placeholder: compds_batch,
              label_placeholder: labels_batch,
            }
          )

          TP, TN, FP, FN, SEN, SPE, MCC = pk_model.compute_performance(label_dense, prediction)

          t3 = float(time.time())

          # print to file and screen
          format_str = "%6d %6d %6.3f %6.3f %10.9f %5d %5d %5d %5d %6.3f %6.3f %6.3f %6.3f %5.3f %5.3f %5.3f %s"
          logfile.write(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, t3-t2, target))
          logfile.write('\n')
          print(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, t3-t2, target))

        # save the model checkpoint periodically.
        if step % 1000 == 0 or (step + 1) == max_step:
          checkpoint_path = os.path.join(ckpt_dir, 'pk.ckpt')
          saver.save(sess, checkpoint_path, global_step=global_step, write_meta_graph=False)

    for target in target_list:
      # the whole train
      compds_batch = numpy.vstack([d.pos[target].features[d.pos[target].train_perm], d.neg.features[d.neg.train_perm]])
      labels_batch = numpy.vstack([d.pos[target].labels[d.pos[target].train_perm], d.neg.mask_dict[target][d.neg.train_perm]])
      LV, XLV, LR, ACC, prediction, label_dense = sess.run(
        [wd_loss_dict[target],
         x_entropy_dict[target],
         learning_rate,
         accuracy_dict[target],
         tf.argmax(softmax_dict[target], 1),
         tf.argmax(labels_batch, 1)],
        feed_dict = {
          input_placeholder: compds_batch,
          label_placeholder: labels_batch,
        }
      )

      TP, TN, FP, FN, SEN, SPE, MCC = pk_model.compute_performance(label_dense, prediction)

      # print to file and screen
      format_str = "%6d %6d %6.3f %6.3f %10.3f %5d %5d %5d %5d %6.3f %6.3f %6.3f %6.3f %5.3f %5.3f %s"
      logfile.write(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))
      logfile.write('\n')
      print(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))

    for target in target_list:
      # the whole test
      compds_batch = numpy.vstack([d.pos[target].features[d.pos[target].test_perm], d.neg.features[d.neg.test_perm]])
      labels_batch = numpy.vstack([d.pos[target].labels[d.pos[target].test_perm], d.neg.mask_dict[target][d.neg.test_perm]])
      LV, XLV, LR, ACC, prediction, label_dense = sess.run(
        [wd_loss_dict[target],
         x_entropy_dict[target],
         learning_rate,
         accuracy_dict[target],
         tf.argmax(softmax_dict[target], 1),
         tf.argmax(labels_batch, 1)],
        feed_dict = {
          input_placeholder: compds_batch,
          label_placeholder: labels_batch,
        }
      )

      TP, TN, FP, FN, SEN, SPE, MCC = pk_model.compute_performance(label_dense, prediction)

      # print to file and screen
      format_str = "%6d %6d %6.3f %6.3f %10.3f %5d %5d %5d %5d %6.3f %6.3f %6.3f %6.3f %5.3f %5.3f %s"
      logfile.write(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))
      logfile.write('\n')
      print(format_str % (step, global_step.eval(sess), LV, XLV, LR, TP, FN, TN, FP, SEN, SPE, ACC, MCC, t1-t0, t2-t1, target))

  logfile.write("train ends at: %s\n" % datetime.datetime.now())
  logfile.close()

if __name__ == "__main__":

  target_list = ["cdk2", "egfr_erbB1", "gsk3b", "hgfr",
                 "map_k_p38a", "tpk_lck", "tpk_src", "vegfr2"]

  train(target_list, train_from=11824)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Markdown reader unittest
"""

from __future__ import unicode_literals

import unittest
import peanut.reader as pr

class TestMarkdown(unittest.TestCase):

    def test_parser(self):
        md_reader = pr.MarkdownReader()
        res = md_reader.read('drafts/example.md')
        self.assertEqual(res['publish'], True)
        self.assertEqual(res['top'], False)
        self.assertEqual(res['title'], 'This is my title')
        self.assertEqual(res['html'], '
这行是中文
')
        self.assertEqual(res['slug'], 'example.md')
        self.assertIn('example', res['slug'])

    def test_get_reader(self):
        reader = None
        for name in ('a.md', 'a.MD', 'a.Markdown', 'a.markdown'):
            r = pr.reader_for_file(name)
            if not reader:
                reader = r
            self.assertIs(r, reader)

        r2 = pr.reader_for_file('b.unknow')
        self.assertIsNone(r2)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Update test cases<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Markdown reader unittest
"""

from __future__ import unicode_literals

import unittest
import peanut.reader as pr

class TestMarkdown(unittest.TestCase):

    def test_parser(self):
        md_reader = pr.MarkdownReader()
        res = md_reader.read('drafts/example.md')
        self.assertEqual(res['publish'], True)
        self.assertEqual(res['top'], False)
        self.assertEqual(res['title'], 'This is my title')
        self.assertEqual(res['html'], '
这行是中文
')
        self.assertEqual(res['slug'], 'example.md')
        self.assertIn('example', res['slug'])

    def test_get_reader(self):
        reader = None
        for name in ('a.md', 'a.MD', 'a.Markdown', 'a.markdown'):
            r = pr.reader_for_file(name)
            if not reader:
                reader = r
            self.assertIs(r, reader)

        r2 = pr.reader_for_file('b.unknow')
        self.assertIsNone(r2)

    def test_meta_parser(self):
        self.assertEqual(pr.parser_single('hello'), 'hello')
        self.assertEqual(pr.parser_single(['hello', 'world']), 'hello')

        self.assertListEqual(pr.parser_list('hello'), ['hello'])
        self.assertListEqual(pr.parser_list(['hello', 'world']),
                ['hello', 'world'])

        for value in ['Yes', 'yes', 'true', 'True', True]:
            self.assertTrue(pr.parser_bool(value))
        for value in ['No', 'no', 'false', 'False', False]:
            self.assertFalse(pr.parser_bool(value))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Test the "pipeline.py" restricted parallel programming library.
"""
from __future__ import division, absolute_import, print_function

import six

from test._common import unittest
from beets.util import pipeline

# Some simple pipeline stages for testing.
def _produce(num=5):
    for i in range(num):
        yield i

def _work():
    i = None
    while True:
        i = yield i
        i *= 2

def _consume(l):
    while True:
        i = yield
        l.append(i)

# A worker that raises an exception.
class TestException(Exception):
    pass

def _exc_work(num=3):
    i = None
    while True:
        i = yield i
        if i == num:
            raise TestException()
        i *= 2

# A worker that yields a bubble.
def _bub_work(num=3):
    i = None
    while True:
        i = yield i
        if i == num:
            i = pipeline.BUBBLE
        else:
            i *= 2

# Yet another worker that yields multiple messages.
def _multi_work():
    i = None
    while True:
        i = yield i
        i = pipeline.multiple([i, -i])

class SimplePipelineTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((_produce(), _work(), _consume(self.l)))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 2, 4, 6, 8])

    def test_run_parallel(self):
        self.pl.run_parallel()
        self.assertEqual(self.l, [0, 2, 4, 6, 8])

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _work()))
        self.assertEqual(list(pl.pull()), [0, 2, 4, 6, 8])

    def test_pull_chain(self):
        pl = pipeline.Pipeline((_produce(), _work()))
        pl2 = pipeline.Pipeline((pl.pull(), _work()))
        self.assertEqual(list(pl2.pull()), [0, 4, 8, 12, 16])

class ParallelStageTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((
            _produce(), (_work(), _work()), _consume(self.l)
        ))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 2, 4, 6, 8])

    def test_run_parallel(self):
        self.pl.run_parallel()
        # Order possibly not preserved; use set equality.
        self.assertEqual(set(self.l), set([0, 2, 4, 6, 8]))

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), (_work(), _work())))
        self.assertEqual(list(pl.pull()), [0, 2, 4, 6, 8])

class ExceptionTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((_produce(), _exc_work(),
                                     _consume(self.l)))

    def test_run_sequential(self):
        self.assertRaises(TestException, self.pl.run_sequential)

    def test_run_parallel(self):
        self.assertRaises(TestException, self.pl.run_parallel)

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _exc_work()))
        pull = pl.pull()
        for i in range(3):
            next(pull)
        self.assertRaises(TestException, pull.next)

class ParallelExceptionTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((
            _produce(), (_exc_work(), _exc_work()), _consume(self.l)
        ))

    def test_run_parallel(self):
        self.assertRaises(TestException, self.pl.run_parallel)

class ConstrainedThreadedPipelineTest(unittest.TestCase):
    def test_constrained(self):
        l = []
        # Do a "significant" amount of work...
        pl = pipeline.Pipeline((_produce(1000), _work(), _consume(l)))
        # ... with only a single queue slot.
        pl.run_parallel(1)
        self.assertEqual(l, [i * 2 for i in range(1000)])

    @unittest.skipIf(six.PY3, u'freezes the test suite in py3')
    def test_constrained_exception(self):
        # Raise an exception in a constrained pipeline.
        l = []
        pl = pipeline.Pipeline((_produce(1000), _exc_work(), _consume(l)))
        self.assertRaises(TestException, pl.run_parallel, 1)

    def test_constrained_parallel(self):
        l = []
        pl = pipeline.Pipeline((
            _produce(1000), (_work(), _work()), _consume(l)
        ))
        pl.run_parallel(1)
        self.assertEqual(set(l), set(i * 2 for i in range(1000)))

class BubbleTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((_produce(), _bub_work(),
                                     _consume(self.l)))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 2, 4, 8])

    def test_run_parallel(self):
        self.pl.run_parallel()
        self.assertEqual(self.l, [0, 2, 4, 8])

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _bub_work()))
        self.assertEqual(list(pl.pull()), [0, 2, 4, 8])

class MultiMessageTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((
            _produce(), _multi_work(), _consume(self.l)
        ))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])

    def test_run_parallel(self):
        self.pl.run_parallel()
        self.assertEqual(self.l, [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _multi_work()))
        self.assertEqual(list(pl.pull()), [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])

class StageDecoratorTest(unittest.TestCase):

    def test_stage_decorator(self):
        @pipeline.stage
        def add(n, i):
            return i + n

        pl = pipeline.Pipeline([
            iter([1, 2, 3]),
            add(2)
        ])
        self.assertEqual(list(pl.pull()), [3, 4, 5])

    def test_mutator_stage_decorator(self):
        @pipeline.mutator_stage
        def setkey(key, item):
            item[key] = True

        pl = pipeline.Pipeline([
            iter([{'x': False}, {'a': False}]),
            setkey('x'),
        ])
        self.assertEqual(list(pl.pull()),
                         [{'x': True}, {'a': False, 'x': True}])

def suite():
    return unittest.TestLoader().loadTestsFromName(__name__)

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<commit_msg>check pull.__next__ on py3 in pipeline tests<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Test the "pipeline.py" restricted parallel programming library.
"""
from __future__ import division, absolute_import, print_function

import six

from test._common import unittest
from beets.util import pipeline

# Some simple pipeline stages for testing.
def _produce(num=5):
    for i in range(num):
        yield i

def _work():
    i = None
    while True:
        i = yield i
        i *= 2

def _consume(l):
    while True:
        i = yield
        l.append(i)

# A worker that raises an exception.
class TestException(Exception):
    pass

def _exc_work(num=3):
    i = None
    while True:
        i = yield i
        if i == num:
            raise TestException()
        i *= 2

# A worker that yields a bubble.
def _bub_work(num=3):
    i = None
    while True:
        i = yield i
        if i == num:
            i = pipeline.BUBBLE
        else:
            i *= 2

# Yet another worker that yields multiple messages.
def _multi_work():
    i = None
    while True:
        i = yield i
        i = pipeline.multiple([i, -i])

class SimplePipelineTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((_produce(), _work(), _consume(self.l)))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 2, 4, 6, 8])

    def test_run_parallel(self):
        self.pl.run_parallel()
        self.assertEqual(self.l, [0, 2, 4, 6, 8])

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _work()))
        self.assertEqual(list(pl.pull()), [0, 2, 4, 6, 8])

    def test_pull_chain(self):
        pl = pipeline.Pipeline((_produce(), _work()))
        pl2 = pipeline.Pipeline((pl.pull(), _work()))
        self.assertEqual(list(pl2.pull()), [0, 4, 8, 12, 16])

class ParallelStageTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((
            _produce(), (_work(), _work()), _consume(self.l)
        ))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 2, 4, 6, 8])

    def test_run_parallel(self):
        self.pl.run_parallel()
        # Order possibly not preserved; use set equality.
        self.assertEqual(set(self.l), set([0, 2, 4, 6, 8]))

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), (_work(), _work())))
        self.assertEqual(list(pl.pull()), [0, 2, 4, 6, 8])

class ExceptionTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((_produce(), _exc_work(),
                                     _consume(self.l)))

    def test_run_sequential(self):
        self.assertRaises(TestException, self.pl.run_sequential)

    def test_run_parallel(self):
        self.assertRaises(TestException, self.pl.run_parallel)

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _exc_work()))
        pull = pl.pull()
        for i in range(3):
            next(pull)
        if six.PY2:
            self.assertRaises(TestException, pull.next)
        else:
            self.assertRaises(TestException, pull.__next__)

class ParallelExceptionTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((
            _produce(), (_exc_work(), _exc_work()), _consume(self.l)
        ))

    def test_run_parallel(self):
        self.assertRaises(TestException, self.pl.run_parallel)

class ConstrainedThreadedPipelineTest(unittest.TestCase):
    def test_constrained(self):
        l = []
        # Do a "significant" amount of work...
        pl = pipeline.Pipeline((_produce(1000), _work(), _consume(l)))
        # ... with only a single queue slot.
        pl.run_parallel(1)
        self.assertEqual(l, [i * 2 for i in range(1000)])

    @unittest.skipIf(six.PY3, u'freezes the test suite in py3')
    def test_constrained_exception(self):
        # Raise an exception in a constrained pipeline.
        l = []
        pl = pipeline.Pipeline((_produce(1000), _exc_work(), _consume(l)))
        self.assertRaises(TestException, pl.run_parallel, 1)

    def test_constrained_parallel(self):
        l = []
        pl = pipeline.Pipeline((
            _produce(1000), (_work(), _work()), _consume(l)
        ))
        pl.run_parallel(1)
        self.assertEqual(set(l), set(i * 2 for i in range(1000)))

class BubbleTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((_produce(), _bub_work(),
                                     _consume(self.l)))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 2, 4, 8])

    def test_run_parallel(self):
        self.pl.run_parallel()
        self.assertEqual(self.l, [0, 2, 4, 8])

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _bub_work()))
        self.assertEqual(list(pl.pull()), [0, 2, 4, 8])

class MultiMessageTest(unittest.TestCase):
    def setUp(self):
        self.l = []
        self.pl = pipeline.Pipeline((
            _produce(), _multi_work(), _consume(self.l)
        ))

    def test_run_sequential(self):
        self.pl.run_sequential()
        self.assertEqual(self.l, [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])

    def test_run_parallel(self):
        self.pl.run_parallel()
        self.assertEqual(self.l, [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])

    def test_pull(self):
        pl = pipeline.Pipeline((_produce(), _multi_work()))
        self.assertEqual(list(pl.pull()), [0, 0, 1, -1, 2, -2, 3, -3, 4, -4])

class StageDecoratorTest(unittest.TestCase):

    def test_stage_decorator(self):
        @pipeline.stage
        def add(n, i):
            return i + n

        pl = pipeline.Pipeline([
            iter([1, 2, 3]),
            add(2)
        ])
        self.assertEqual(list(pl.pull()), [3, 4, 5])

    def test_mutator_stage_decorator(self):
        @pipeline.mutator_stage
        def setkey(key, item):
            item[key] = True

        pl = pipeline.Pipeline([
            iter([{'x': False}, {'a': False}]),
            setkey('x'),
        ])
        self.assertEqual(list(pl.pull()),
                         [{'x': True}, {'a': False, 'x': True}])

def suite():
    return unittest.TestLoader().loadTestsFromName(__name__)

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<|endoftext|>


<commit_before>import numpy as np
from cereal import log
from common.filter_simple import FirstOrderFilter
from common.numpy_fast import interp
from common.realtime import DT_MDL
from selfdrive.hardware import EON, TICI
from selfdrive.swaglog import cloudlog

TRAJECTORY_SIZE = 33
# camera offset is meters from center car to camera
if EON:
  CAMERA_OFFSET = 0.06
  PATH_OFFSET = 0.0
elif TICI:
  CAMERA_OFFSET = -0.04
  PATH_OFFSET = -0.04
else:
  CAMERA_OFFSET = 0.0
  PATH_OFFSET = 0.0

class LanePlanner:
  def __init__(self, wide_camera=False):
    self.ll_t = np.zeros((TRAJECTORY_SIZE,))
    self.ll_x = np.zeros((TRAJECTORY_SIZE,))
    self.lll_y = np.zeros((TRAJECTORY_SIZE,))
    self.rll_y = np.zeros((TRAJECTORY_SIZE,))
    self.lane_width_estimate = FirstOrderFilter(3.7, 9.95, DT_MDL)
    self.lane_width_certainty = FirstOrderFilter(1.0, 0.95, DT_MDL)
    self.lane_width = 3.7

    self.lll_prob = 0.
    self.rll_prob = 0.
    self.d_prob = 0.

    self.lll_std = 0.
    self.rll_std = 0.

    self.l_lane_change_prob = 0.
    self.r_lane_change_prob = 0.

    self.camera_offset = -CAMERA_OFFSET if wide_camera else CAMERA_OFFSET
    self.path_offset = -PATH_OFFSET if wide_camera else PATH_OFFSET

  def parse_model(self, md):
    if len(md.laneLines) == 4 and len(md.laneLines[0].t) == TRAJECTORY_SIZE:
      self.ll_t = (np.array(md.laneLines[1].t) + np.array(md.laneLines[2].t))/2
      # left and right ll x is the same
      self.ll_x = md.laneLines[1].x
      # only offset left and right lane lines; offsetting path does not make sense
      self.lll_y = np.array(md.laneLines[1].y) - self.camera_offset
      self.rll_y = np.array(md.laneLines[2].y) - self.camera_offset
      self.lll_prob = md.laneLineProbs[1]
      self.rll_prob = md.laneLineProbs[2]
      self.lll_std = md.laneLineStds[1]
      self.rll_std = md.laneLineStds[2]

    if len(md.meta.desireState):
      self.l_lane_change_prob = md.meta.desireState[log.LateralPlan.Desire.laneChangeLeft]
      self.r_lane_change_prob = md.meta.desireState[log.LateralPlan.Desire.laneChangeRight]

  def get_d_path(self, v_ego, path_t, path_xyz):
    # Reduce reliance on lanelines that are too far apart or
    # will be in a few seconds
    path_xyz[:, 1] -= self.path_offset
    l_prob, r_prob = self.lll_prob, self.rll_prob
    width_pts = self.rll_y - self.lll_y
    prob_mods = []
    for t_check in [0.0, 1.5, 3.0]:
      width_at_t = interp(t_check * (v_ego + 7), self.ll_x, width_pts)
      prob_mods.append(interp(width_at_t, [4.0, 5.0], [1.0, 0.0]))
    mod = min(prob_mods)
    l_prob *= mod
    r_prob *= mod

    # Reduce reliance on uncertain lanelines
    l_std_mod = interp(self.lll_std, [.15, .3], [1.0, 0.0])
    r_std_mod = interp(self.rll_std, [.15, .3], [1.0, 0.0])
    l_prob *= l_std_mod
    r_prob *= r_std_mod

    # Find current lanewidth
    self.lane_width_certainty.update(l_prob * r_prob)
    current_lane_width = abs(self.rll_y[0] - self.lll_y[0])
    self.lane_width_estimate.update(current_lane_width)
    speed_lane_width = interp(v_ego, [0., 31.], [2.8, 3.5])
    self.lane_width = self.lane_width_certainty.x * self.lane_width_estimate.x + \
                      (1 - self.lane_width_certainty.x) * speed_lane_width

    clipped_lane_width = min(4.0, self.lane_width)
    path_from_left_lane = self.lll_y + clipped_lane_width / 2.0
    path_from_right_lane = self.rll_y - clipped_lane_width / 2.0

    self.d_prob = l_prob + r_prob - l_prob * r_prob
    lane_path_y = (l_prob * path_from_left_lane + r_prob * path_from_right_lane) / (l_prob + r_prob + 0.0001)
    safe_idxs = np.isfinite(self.ll_t)
    if safe_idxs[0]:
      lane_path_y_interp = np.interp(path_t, self.ll_t[safe_idxs], lane_path_y[safe_idxs])
      path_xyz[:,1] = self.d_prob * lane_path_y_interp + (1.0 - self.d_prob) * path_xyz[:,1]
    else:
      cloudlog.warning("Lateral mpc - NaNs in laneline times, ignoring")
    return path_xyz
<commit_msg>lane_planner.py/parse_model: follow capnp best practices (#23368)<commit_after>import numpy as np
from cereal import log
from common.filter_simple import FirstOrderFilter
from common.numpy_fast import interp
from common.realtime import DT_MDL
from selfdrive.hardware import EON, TICI
from selfdrive.swaglog import cloudlog

TRAJECTORY_SIZE = 33
# camera offset is meters from center car to camera
if EON:
  CAMERA_OFFSET = 0.06
  PATH_OFFSET = 0.0
elif TICI:
  CAMERA_OFFSET = -0.04
  PATH_OFFSET = -0.04
else:
  CAMERA_OFFSET = 0.0
  PATH_OFFSET = 0.0

class LanePlanner:
  def __init__(self, wide_camera=False):
    self.ll_t = np.zeros((TRAJECTORY_SIZE,))
    self.ll_x = np.zeros((TRAJECTORY_SIZE,))
    self.lll_y = np.zeros((TRAJECTORY_SIZE,))
    self.rll_y = np.zeros((TRAJECTORY_SIZE,))
    self.lane_width_estimate = FirstOrderFilter(3.7, 9.95, DT_MDL)
    self.lane_width_certainty = FirstOrderFilter(1.0, 0.95, DT_MDL)
    self.lane_width = 3.7

    self.lll_prob = 0.
    self.rll_prob = 0.
    self.d_prob = 0.

    self.lll_std = 0.
    self.rll_std = 0.

    self.l_lane_change_prob = 0.
    self.r_lane_change_prob = 0.

    self.camera_offset = -CAMERA_OFFSET if wide_camera else CAMERA_OFFSET
    self.path_offset = -PATH_OFFSET if wide_camera else PATH_OFFSET

  def parse_model(self, md):
    lane_lines = md.laneLines
    if len(lane_lines) == 4 and len(lane_lines[0].t) == TRAJECTORY_SIZE:
      self.ll_t = (np.array(lane_lines[1].t) + np.array(lane_lines[2].t))/2
      # left and right ll x is the same
      self.ll_x = lane_lines[1].x
      # only offset left and right lane lines; offsetting path does not make sense
      self.lll_y = np.array(lane_lines[1].y) - self.camera_offset
      self.rll_y = np.array(lane_lines[2].y) - self.camera_offset
      self.lll_prob = md.laneLineProbs[1]
      self.rll_prob = md.laneLineProbs[2]
      self.lll_std = md.laneLineStds[1]
      self.rll_std = md.laneLineStds[2]

    desire_state = md.meta.desireState
    if len(desire_state):
      self.l_lane_change_prob = desire_state[log.LateralPlan.Desire.laneChangeLeft]
      self.r_lane_change_prob = desire_state[log.LateralPlan.Desire.laneChangeRight]

  def get_d_path(self, v_ego, path_t, path_xyz):
    # Reduce reliance on lanelines that are too far apart or
    # will be in a few seconds
    path_xyz[:, 1] -= self.path_offset
    l_prob, r_prob = self.lll_prob, self.rll_prob
    width_pts = self.rll_y - self.lll_y
    prob_mods = []
    for t_check in [0.0, 1.5, 3.0]:
      width_at_t = interp(t_check * (v_ego + 7), self.ll_x, width_pts)
      prob_mods.append(interp(width_at_t, [4.0, 5.0], [1.0, 0.0]))
    mod = min(prob_mods)
    l_prob *= mod
    r_prob *= mod

    # Reduce reliance on uncertain lanelines
    l_std_mod = interp(self.lll_std, [.15, .3], [1.0, 0.0])
    r_std_mod = interp(self.rll_std, [.15, .3], [1.0, 0.0])
    l_prob *= l_std_mod
    r_prob *= r_std_mod

    # Find current lanewidth
    self.lane_width_certainty.update(l_prob * r_prob)
    current_lane_width = abs(self.rll_y[0] - self.lll_y[0])
    self.lane_width_estimate.update(current_lane_width)
    speed_lane_width = interp(v_ego, [0., 31.], [2.8, 3.5])
    self.lane_width = self.lane_width_certainty.x * self.lane_width_estimate.x + \
                      (1 - self.lane_width_certainty.x) * speed_lane_width

    clipped_lane_width = min(4.0, self.lane_width)
    path_from_left_lane = self.lll_y + clipped_lane_width / 2.0
    path_from_right_lane = self.rll_y - clipped_lane_width / 2.0

    self.d_prob = l_prob + r_prob - l_prob * r_prob
    lane_path_y = (l_prob * path_from_left_lane + r_prob * path_from_right_lane) / (l_prob + r_prob + 0.0001)
    safe_idxs = np.isfinite(self.ll_t)
    if safe_idxs[0]:
      lane_path_y_interp = np.interp(path_t, self.ll_t[safe_idxs], lane_path_y[safe_idxs])
      path_xyz[:,1] = self.d_prob * lane_path_y_interp + (1.0 - self.d_prob) * path_xyz[:,1]
    else:
      cloudlog.warning("Lateral mpc - NaNs in laneline times, ignoring")
    return path_xyz
<|endoftext|>


<commit_before>'''
'''

import multiprocessing
from functools import partial
import Mekong_loss
import subprocess
import os
import glob
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def main ():

    # List of tiles that could be run. This list is only used to create the FIA region tiles if they don't already exist.
    tile_id_list = uu.tile_list_s3(cn.WHRC_biomass_2000_unmasked_dir)
    # tile_id_list = ['50N_130W'] # test tiles
    print tile_id_list
    print "There are {} tiles to process".format(str(len(tile_id_list))) + "\n"

    # By definition, this script is for US-specific removals
    sensit_type = 'Mekong_loss'

    # # Downloads the Mekong loss folder. Each year of loss has its own raster
    # uu.s3_folder_download(cn.Mekong_loss_raw_dir, '.', sensit_type)
    #
    # count = multiprocessing.cpu_count()
    #
    # # The list of all annual loss rasters
    # annual_loss_list = glob.glob('Loss_20*tif')
    # print annual_loss_list
    #
    # print "Creating first year of loss Hansen tiles for Mekong region..."
    # # Recodes raw loss rasters with their loss year (for model years only)
    pool = multiprocessing.Pool(count/2)
    # pool.map(Mekong_loss.recode_tiles, annual_loss_list)
    #
    # # Makes a single raster of all first loss year pixels in the Mekong (i.e. where loss occurred in multiple years,
    # # the earlier loss gets)
    # print "Merging all loss years within model range..."
    loss_composite = "Mekong_loss_2001_2015.tif"
    # cmd = ['gdal_merge.py', '-o', loss_composite, '-co', 'COMPRESS=LZW', '-a_nodata', '0', '-ot', 'Byte',
    #        "Mekong_loss_recoded_2015.tif", "Mekong_loss_recoded_2014.tif", "Mekong_loss_recoded_2013.tif",
    #        "Mekong_loss_recoded_2012.tif", "Mekong_loss_recoded_2011.tif", "Mekong_loss_recoded_2010.tif",
    #        "Mekong_loss_recoded_2009.tif", "Mekong_loss_recoded_2008.tif", "Mekong_loss_recoded_2007.tif",
    #        "Mekong_loss_recoded_2006.tif", "Mekong_loss_recoded_2005.tif", "Mekong_loss_recoded_2004.tif",
    #        "Mekong_loss_recoded_2003.tif", "Mekong_loss_recoded_2002.tif", "Mekong_loss_recoded_2001.tif"]
    # subprocess.check_call(cmd)

    # Creates Hansen tiles out of the composite Mekong loss
    source_raster = loss_composite
    out_pattern = cn.pattern_Mekong_loss_processed
    dt = 'Byte'
    pool.map(partial(uu.mp_warp_to_Hansen, source_raster,out_pattern, dt=dt), tile_id_list)

    # Only uploads tiles that actually have Mekong loss in them
    uu.check_and_upload(tile_id_list, cn.Mekong_loss_processed_dir, cn.pattern_Mekong_loss_processed)

if __name__ == '__main__':
    main()<commit_msg>Creating first year of loss Hansen loss v2.0 for Mekong for Mekong_loss sensitivity analysis.<commit_after>'''
'''

import multiprocessing
from functools import partial
import Mekong_loss
import subprocess
import os
import glob
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def main ():

    # List of tiles that could be run. This list is only used to create the FIA region tiles if they don't already exist.
    tile_id_list = uu.tile_list_s3(cn.WHRC_biomass_2000_unmasked_dir)
    # tile_id_list = ['50N_130W'] # test tiles
    print tile_id_list
    print "There are {} tiles to process".format(str(len(tile_id_list))) + "\n"

    # By definition, this script is for US-specific removals
    sensit_type = 'Mekong_loss'

    # # Downloads the Mekong loss folder. Each year of loss has its own raster
    # uu.s3_folder_download(cn.Mekong_loss_raw_dir, '.', sensit_type)
    #
    count = multiprocessing.cpu_count()
    #
    # # The list of all annual loss rasters
    # annual_loss_list = glob.glob('Loss_20*tif')
    # print annual_loss_list
    #
    # print "Creating first year of loss Hansen tiles for Mekong region..."
    # # Recodes raw loss rasters with their loss year (for model years only)
    pool = multiprocessing.Pool(count/2)
    # pool.map(Mekong_loss.recode_tiles, annual_loss_list)
    #
    # # Makes a single raster of all first loss year pixels in the Mekong (i.e. where loss occurred in multiple years,
    # # the earlier loss gets)
    # print "Merging all loss years within model range..."
    loss_composite = "Mekong_loss_2001_2015.tif"
    # cmd = ['gdal_merge.py', '-o', loss_composite, '-co', 'COMPRESS=LZW', '-a_nodata', '0', '-ot', 'Byte',
    #        "Mekong_loss_recoded_2015.tif", "Mekong_loss_recoded_2014.tif", "Mekong_loss_recoded_2013.tif",
    #        "Mekong_loss_recoded_2012.tif", "Mekong_loss_recoded_2011.tif", "Mekong_loss_recoded_2010.tif",
    #        "Mekong_loss_recoded_2009.tif", "Mekong_loss_recoded_2008.tif", "Mekong_loss_recoded_2007.tif",
    #        "Mekong_loss_recoded_2006.tif", "Mekong_loss_recoded_2005.tif", "Mekong_loss_recoded_2004.tif",
    #        "Mekong_loss_recoded_2003.tif", "Mekong_loss_recoded_2002.tif", "Mekong_loss_recoded_2001.tif"]
    # subprocess.check_call(cmd)

    # Creates Hansen tiles out of the composite Mekong loss
    source_raster = loss_composite
    out_pattern = cn.pattern_Mekong_loss_processed
    dt = 'Byte'
    pool.map(partial(uu.mp_warp_to_Hansen, source_raster,out_pattern, dt=dt), tile_id_list)

    # Only uploads tiles that actually have Mekong loss in them
    uu.check_and_upload(tile_id_list, cn.Mekong_loss_processed_dir, cn.pattern_Mekong_loss_processed)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright 2015, Craig Tracey <craigtracey@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import subprocess
import sys

from novaclient.v1_1.client import Client as nova_client

CREDS = {
    'username': os.environ['OS_USERNAME'],
    'api_key': os.environ['OS_PASSWORD'],
    'project_id': os.environ['OS_TENANT_NAME'],
    'auth_url': os.environ['OS_AUTH_URL'],
    'region_name': 'RegionOne',
}

def _get_hostname():
    return subprocess.check_output(['hostname', '-f']).strip()

def _get_local_active_instance_volumes():
    hostname = _get_hostname()

    local_attached_vols = []
    nova = nova_client(**CREDS)
    search_opts = {
        'all_tenants': True
    }
    for server in nova.servers.list(search_opts=search_opts):
        if getattr(server, "OS-EXT-STS:vm_state") != "active":
            continue
        server_hv = \
            getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname')
        attached_vols = \
            getattr(server, 'os-extended-volumes:volumes_attached')
        if server_hv == hostname and len(attached_vols):
            local_attached_vols.append([ getattr(server, 'id'), attached_vols ])
    return local_attached_vols

def _get_active_iscsi_volumes():
    volumes = set()
    try:
        out = subprocess.check_output(['iscsiadm', '-m', 'session'])
        for line in out.strip().split('\n'):
            parts = line.split(':')
            volumes.add(parts[-1][7:43])  # remove 'volume-'
    except Exception, e:
        # iscsiadm returns 21 if there are no active sessions.
        if e.returncode == 21:
            pass
        else:
            raise
    return list(volumes)

def main():
    failure = False
    try:
        expected_vols = _get_local_active_instance_volumes()
        connected_vols = _get_active_iscsi_volumes()

        for vm_and_vols in expected_vols:
            if not vm_and_vols[1][0]['id'] in connected_vols:
                print "The instance %s has the volume %s attached, " \
                      "but that volume does not have an iscsi session" \
                      % (vm_and_vols[0], vm_and_vols[1][0]['id'],)
    except Exception as e:
        print "Could not run check: %s" % e
        sys.exit(2)
    sys.exit(failure)

if __name__ == '__main__':
    main()
<commit_msg>cinder-session-show-vm: use print()<commit_after>#!/usr/bin/python
#
# Copyright 2015, Craig Tracey <craigtracey@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import subprocess
import sys

from novaclient.v1_1.client import Client as nova_client

CREDS = {
    'username': os.environ['OS_USERNAME'],
    'api_key': os.environ['OS_PASSWORD'],
    'project_id': os.environ['OS_TENANT_NAME'],
    'auth_url': os.environ['OS_AUTH_URL'],
    'region_name': 'RegionOne',
}

def _get_hostname():
    return subprocess.check_output(['hostname', '-f']).strip()

def _get_local_active_instance_volumes():
    hostname = _get_hostname()

    local_attached_vols = []
    nova = nova_client(**CREDS)
    search_opts = {
        'all_tenants': True
    }
    for server in nova.servers.list(search_opts=search_opts):
        if getattr(server, "OS-EXT-STS:vm_state") != "active":
            continue
        server_hv = \
            getattr(server, 'OS-EXT-SRV-ATTR:hypervisor_hostname')
        attached_vols = \
            getattr(server, 'os-extended-volumes:volumes_attached')
        if server_hv == hostname and len(attached_vols):
            local_attached_vols.append([ getattr(server, 'id'), attached_vols ])
    return local_attached_vols

def _get_active_iscsi_volumes():
    volumes = set()
    try:
        out = subprocess.check_output(['iscsiadm', '-m', 'session'])
        for line in out.strip().split('\n'):
            parts = line.split(':')
            volumes.add(parts[-1][7:43])  # remove 'volume-'
    except Exception, e:
        # iscsiadm returns 21 if there are no active sessions.
        if e.returncode == 21:
            pass
        else:
            raise
    return list(volumes)

def main():
    failure = False
    try:
        expected_vols = _get_local_active_instance_volumes()
        connected_vols = _get_active_iscsi_volumes()

        for vm_and_vols in expected_vols:
            if not vm_and_vols[1][0]['id'] in connected_vols:
                print("The instance %s has the volume %s attached, " \
                      "but that volume does not have an iscsi session" \
                      % (vm_and_vols[0], vm_and_vols[1][0]['id'],))
    except Exception as e:
        print("Could not run check: %s" % e)
        sys.exit(2)
    sys.exit(failure)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Copyright (C) 2016  Genome Research Ltd.

Author: Irina Colgiu <ic4@sanger.ac.uk>

This program is part of serapis

serapis is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

This file has been created on Aug 01, 2016.
"""

from serapis.storage.irods.api import IrodsBasicAPI, CollectionAPI, DataObjectAPI, MetadataAPI
from serapis.storage.irods import exceptions

import typing
import subprocess

class ICmdsBasicAPI(IrodsBasicAPI):

    @classmethod
    def _build_icmd_args(cls, cmd_name: str, args_list: typing.List, options=None):
        cmd_list = [cmd_name]
        if options:
            cmd_list.extend(options)
        cmd_list.extend(args_list)
        return cmd_list

    @classmethod
    def _run_icmd(cls, cmd_args):
        child_proc = subprocess.Popen(cmd_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        (out, err) = child_proc.communicate()
        if err:
            raise exceptions.IrodsException(err, out, cmd=str(cmd_args))

    @classmethod
    def remove(cls, path):
        raise NotImplementedError()

class ICmdsCollectionAPI(CollectionAPI):

    @classmethod
    def create(cls, path):
        cmd_args = cls._build_icmd_args('imkdir', [path])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iMkDirException(error=e.err, output=e.out, cmd=e.cmd)

    # TODO: distinguish somehow between different types of iRODS errors
    @classmethod
    def remove(cls, path):
        cmd_args = cls._build_icmd_args('irm', [path], ["-r"])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iRMException(error=e.err, output=e.out, cmd=e.cmd)

class ICmdsDataObjectAPI(DataObjectAPI):

    @classmethod
    def upload(cls, src_path, dest_path):
        # TODO: check if it is ever the case to give it extra options
        cmd_args = cls._build_icmd_args('iput', [src_path, dest_path], options=["-K"])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iPutException(error=e.err, output=e.out, cmd=cmd_args)

    @classmethod
    def copy(cls, src_path, dest_path):
        cmd_args = cls._build_icmd_args('icp', [src_path, dest_path], options=[])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iPutException(error=e.err, output=e.out, cmd=cmd_args)

    @classmethod
    def move(cls, src_path, dest_path):
        cmd_args = cls._build_icmd_args('imv', [src_path, dest_path], options=[])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iPutException(error=e.err, output=e.out, cmd=cmd_args)

    @classmethod
    def remove(cls, path):
        cmd_args = cls._build_icmd_args('irm', [path])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iRMException(error=e.err, output=e.out, cmd=e.cmd)

    @classmethod
    def recalculate_checksums(cls, path):
        """
        This method re-calculates the checksums on all replicas and stores them within iRODS.
        It doesn't, however, display any "conclusion" after recalculating the checksums,
        so one needs to check manually on the result, whether after checksuming
        the checksums are still the same across all replicas.
        :return:
        """
        cmd_args = cls._build_icmd_args('ichksum', [path], ['-a', '-K'])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iRMException(error=e.err, output=e.out, cmd=e.cmd)

<commit_msg>Changed inheritance<commit_after>"""
Copyright (C) 2016  Genome Research Ltd.

Author: Irina Colgiu <ic4@sanger.ac.uk>

This program is part of serapis

serapis is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

This file has been created on Aug 01, 2016.
"""

from serapis.storage.irods.api import IrodsBasicAPI, CollectionAPI, DataObjectAPI, MetadataAPI
from serapis.storage.irods import exceptions

import typing
import subprocess

class ICmdsBasicAPI(IrodsBasicAPI):

    @classmethod
    def _build_icmd_args(cls, cmd_name: str, args_list: typing.List, options=None):
        cmd_list = [cmd_name]
        if options:
            cmd_list.extend(options)
        cmd_list.extend(args_list)
        return cmd_list

    @classmethod
    def _run_icmd(cls, cmd_args):
        child_proc = subprocess.Popen(cmd_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
        (out, err) = child_proc.communicate()
        if err:
            raise exceptions.IrodsException(err, out, cmd=str(cmd_args))

    @classmethod
    def remove(cls, path):
        raise NotImplementedError()

class ICmdsCollectionAPI(IrodsBasicAPI):

    @classmethod
    def create(cls, path):
        cmd_args = cls._build_icmd_args('imkdir', [path])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iMkDirException(error=e.err, output=e.out, cmd=e.cmd)

    # TODO: distinguish somehow between different types of iRODS errors
    @classmethod
    def remove(cls, path):
        cmd_args = cls._build_icmd_args('irm', [path], ["-r"])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iRMException(error=e.err, output=e.out, cmd=e.cmd)

class ICmdsDataObjectAPI(IrodsBasicAPI):

    @classmethod
    def upload(cls, src_path, dest_path):
        # TODO: check if it is ever the case to give it extra options
        cmd_args = cls._build_icmd_args('iput', [src_path, dest_path], options=["-K"])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iPutException(error=e.err, output=e.out, cmd=cmd_args)

    @classmethod
    def copy(cls, src_path, dest_path):
        cmd_args = cls._build_icmd_args('icp', [src_path, dest_path], options=[])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iPutException(error=e.err, output=e.out, cmd=cmd_args)

    @classmethod
    def move(cls, src_path, dest_path):
        cmd_args = cls._build_icmd_args('imv', [src_path, dest_path], options=[])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iPutException(error=e.err, output=e.out, cmd=cmd_args)

    @classmethod
    def remove(cls, path):
        cmd_args = cls._build_icmd_args('irm', [path])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iRMException(error=e.err, output=e.out, cmd=e.cmd)

    @classmethod
    def recalculate_checksums(cls, path):
        """
        This method re-calculates the checksums on all replicas and stores them within iRODS.
        It doesn't, however, display any "conclusion" after recalculating the checksums,
        so one needs to check manually on the result, whether after checksuming
        the checksums are still the same across all replicas.
        :return:
        """
        cmd_args = cls._build_icmd_args('ichksum', [path], ['-a', '-K'])
        try:
            cls._run_icmd(cmd_args)
        except exceptions.iRODSException as e:
            raise exceptions.iRMException(error=e.err, output=e.out, cmd=e.cmd)

<|endoftext|>


<commit_before>#!/usr/bin/env python2.5
#
# Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for students in GCI to upload their forms.
"""

__authors__ = [
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from django import forms
from django.utils.translation import ugettext

from google.appengine.dist import httplib
from google.appengine.ext import blobstore

from soc.views.helper import blobstore as bs_helper
from soc.views.helper import url_patterns

from soc.modules.gci.models.profile import GCIStudentInfo
from soc.modules.gci.views import forms as gci_forms
from soc.modules.gci.views.base import RequestHandler
from soc.modules.gci.views.helper.url_patterns import url

DEF_NO_UPLOAD_MSG = ugettext('Please choose at least one file to upload.')

class UploadForm(gci_forms.GCIModelForm):
  """Django form to upload student forms
  """

  class Meta:
    model = GCIStudentInfo
    css_prefix = 'gci_student_forms'
    fields = ['consent_form', 'student_id_form']

  consent_form = forms.FileField(required=False)
  student_id_form = forms.FileField(required=False)

  def __init__(self, r, *args, **kwargs):
    """Initializes the FileFields.
    """
    super(UploadForm, self).__init__(*args, **kwargs)

    if self.instance:
      self.fields['consent_form']._file = self.instance.consent_form
      download_url = '%s?consent_form' %r.program().urlOf('gci_student_form_upload')
      self.fields['consent_form']._link = download_url

      self.fields['student_id_form']._file = self.instance.student_id_form
      download_url = '%s?student_id_form' %r.program().urlOf('gci_student_form_upload')
      self.fields['student_id_form']._link = download_url

  def clean(self):
    """Ensure that at least one of the fields has data.
    """
    cleaned_data = self.cleaned_data

    consent_form = cleaned_data.get('consent_form')
    student_id_form = cleaned_data.get('student_id_form')

    if not (consent_form or student_id_form):
      raise gci_forms.ValidationError(DEF_NO_UPLOAD_MSG)

    return cleaned_data

class StudentFormUpload(RequestHandler):
  """View for uploading your student forms.
  """

  def djangoURLPatterns(self):
    """The URL pattern for the view.
    """
    return [
        url(r'student/forms/%s$' % url_patterns.PROGRAM, self,
            name='gci_student_form_upload')]

  def checkAccess(self):
    """Denies access if you are not a student.
    """
    self.check.isActiveStudent()

  def templatePath(self):
    """Returns the path to the template.
    """
    return 'v2/modules/gci/student_forms/base.html'

  def get(self):
    """Handles download of the forms otherwise resumes normal rendering.
    """
    if 'consent_form' in self.data.GET:
      download = self.data.student_info.consent_form
    elif 'student_id_form' in self.data.GET:
      download = self.data.student_info.student_id_form
    else:
      return super(StudentFormUpload, self).get()

    # download has been requested
    if not download:
      self.error(httplib.NOT_FOUND, 'File not found')

    self.response = bs_helper.send_blob(download, save_as=True)

  def context(self):
    """Handler for default HTTP GET request.
    """
    context = {
        'page_name': 'Student form upload'
        }

    if self.data.POST:
      upload_form = UploadForm(self.redirect, self.data.POST,
                               instance=self.data.student_info)
    else:
      upload_form = UploadForm(self.redirect, instance=self.data.student_info)

    context['form'] = upload_form
    url = self.redirect.program().urlOf('gci_student_form_upload')
    context['upload_url'] = blobstore.create_upload_url(url)

    return context

  def post(self):
    """Handles POST requests for the bulk create page.
    """
    form = UploadForm(
        self.redirect,data=self.data.POST, instance=self.data.student_info,
        files=self.data.request.file_uploads)

    if not form.is_valid():
      # we are not storing this form, remove the uploaded blobs from the cloud
      for file in self.data.request.file_uploads.itervalues():
        file.delete()

      return self.get()

    # delete existing data
    cleaned_data = form.cleaned_data
    for field_name in self.data.request.file_uploads.keys():
      if field_name in cleaned_data:
        existing = getattr(self.data.student_info, field_name)
        if existing:
          existing.delete()

    form.save()

    self.redirect.program().to('gci_student_form_upload')

class StudentFormDownload(RequestHandler):
  """View for uploading your student forms.
  """

  def djangoURLPatterns(self):
    """The URL pattern for the view.
    """
    return [
        url(r'student/forms/%s$' % url_patterns.PROFILE, self,
            name='gci_student_form_download')]

  def checkAccess(self):
    """Denies access if you are not a host.
    """
    self.check.isHost()
    self.mutator.studentFromKwargs()

  def get(self):
    """Allows hosts to download the student forms.
    """

    download = None
    if 'consent_form' in self.data.GET:
      download = self.data.student_info.consent_form
    elif 'student_id_form' in self.data.GET:
      download = self.data.student_info.student_id_form

    # download has been requested
    if not download:
      self.error(httplib.NOT_FOUND, 'File not found')

    self.response = bs_helper.send_blob(download, save_as=True)
<commit_msg>Raise error when host download does not request a file.<commit_after>#!/usr/bin/env python2.5
#
# Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for students in GCI to upload their forms.
"""

__authors__ = [
  '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from django import forms
from django.utils.translation import ugettext

from google.appengine.dist import httplib
from google.appengine.ext import blobstore

from soc.views.helper import blobstore as bs_helper
from soc.views.helper import url_patterns

from soc.modules.gci.models.profile import GCIStudentInfo
from soc.modules.gci.views import forms as gci_forms
from soc.modules.gci.views.base import RequestHandler
from soc.modules.gci.views.helper.url_patterns import url

DEF_NO_UPLOAD_MSG = ugettext('Please choose at least one file to upload.')

class UploadForm(gci_forms.GCIModelForm):
  """Django form to upload student forms
  """

  class Meta:
    model = GCIStudentInfo
    css_prefix = 'gci_student_forms'
    fields = ['consent_form', 'student_id_form']

  consent_form = forms.FileField(required=False)
  student_id_form = forms.FileField(required=False)

  def __init__(self, r, *args, **kwargs):
    """Initializes the FileFields.
    """
    super(UploadForm, self).__init__(*args, **kwargs)

    if self.instance:
      self.fields['consent_form']._file = self.instance.consent_form
      download_url = '%s?consent_form' %r.program().urlOf('gci_student_form_upload')
      self.fields['consent_form']._link = download_url

      self.fields['student_id_form']._file = self.instance.student_id_form
      download_url = '%s?student_id_form' %r.program().urlOf('gci_student_form_upload')
      self.fields['student_id_form']._link = download_url

  def clean(self):
    """Ensure that at least one of the fields has data.
    """
    cleaned_data = self.cleaned_data

    consent_form = cleaned_data.get('consent_form')
    student_id_form = cleaned_data.get('student_id_form')

    if not (consent_form or student_id_form):
      raise gci_forms.ValidationError(DEF_NO_UPLOAD_MSG)

    return cleaned_data

class StudentFormUpload(RequestHandler):
  """View for uploading your student forms.
  """

  def djangoURLPatterns(self):
    """The URL pattern for the view.
    """
    return [
        url(r'student/forms/%s$' % url_patterns.PROGRAM, self,
            name='gci_student_form_upload')]

  def checkAccess(self):
    """Denies access if you are not a student.
    """
    self.check.isActiveStudent()

  def templatePath(self):
    """Returns the path to the template.
    """
    return 'v2/modules/gci/student_forms/base.html'

  def get(self):
    """Handles download of the forms otherwise resumes normal rendering.
    """
    if 'consent_form' in self.data.GET:
      download = self.data.student_info.consent_form
    elif 'student_id_form' in self.data.GET:
      download = self.data.student_info.student_id_form
    else:
      return super(StudentFormUpload, self).get()

    # download has been requested
    if not download:
      self.error(httplib.NOT_FOUND, 'File not found')

    self.response = bs_helper.send_blob(download, save_as=True)

  def context(self):
    """Handler for default HTTP GET request.
    """
    context = {
        'page_name': 'Student form upload'
        }

    if self.data.POST:
      upload_form = UploadForm(self.redirect, self.data.POST,
                               instance=self.data.student_info)
    else:
      upload_form = UploadForm(self.redirect, instance=self.data.student_info)

    context['form'] = upload_form
    url = self.redirect.program().urlOf('gci_student_form_upload')
    context['upload_url'] = blobstore.create_upload_url(url)

    return context

  def post(self):
    """Handles POST requests for the bulk create page.
    """
    form = UploadForm(
        self.redirect,data=self.data.POST, instance=self.data.student_info,
        files=self.data.request.file_uploads)

    if not form.is_valid():
      # we are not storing this form, remove the uploaded blobs from the cloud
      for file in self.data.request.file_uploads.itervalues():
        file.delete()

      return self.get()

    # delete existing data
    cleaned_data = form.cleaned_data
    for field_name in self.data.request.file_uploads.keys():
      if field_name in cleaned_data:
        existing = getattr(self.data.student_info, field_name)
        if existing:
          existing.delete()

    form.save()

    self.redirect.program().to('gci_student_form_upload')

class StudentFormDownload(RequestHandler):
  """View for uploading your student forms.
  """

  def djangoURLPatterns(self):
    """The URL pattern for the view.
    """
    return [
        url(r'student/forms/%s$' % url_patterns.PROFILE, self,
            name='gci_student_form_download')]

  def checkAccess(self):
    """Denies access if you are not a host.
    """
    self.check.isHost()
    self.mutator.studentFromKwargs()

  def get(self):
    """Allows hosts to download the student forms.
    """
    download = None
    if 'consent_form' in self.data.GET:
      download = self.data.student_info.consent_form
    elif 'student_id_form' in self.data.GET:
      download = self.data.student_info.student_id_form
    else:
      return self.error(httplib.BAD_REQUEST, 'No file requested')

    # download has been requested
    if not download:
      self.error(httplib.NOT_FOUND, 'File not found')

    self.response = bs_helper.send_blob(download, save_as=True)
<|endoftext|>