<commit_before># -*- encoding: utf-8 -*-
import time
import numpy as np
import utils.logger
import tensorflow as tf
from utils import checkpoint_utils
from utils.decorators import only_on_train
from actor_learner import ActorLearner, ONE_LIFE_GAMES
from networks.policy_v_network import PolicyValueNetwork

logger = utils.logger.getLogger('policy_based_actor_learner')

class BaseA3CLearner(ActorLearner):
    def __init__(self, args):
        super(BaseA3CLearner, self).__init__(args)
        self.td_lambda = args.td_lambda

        # Shared mem vars
        self.learning_vars = args.learning_vars
        self.beta = args.entropy_regularisation_strength
        self.q_target_update_steps = args.q_target_update_steps

    def sample_policy_action(self, probs, temperature=0.5):
        probs = probs - np.finfo(np.float32).epsneg

        histogram = np.random.multinomial(1, probs)
        action_index = int(np.nonzero(histogram)[0])
        return action_index

    def compute_gae(self, rewards, values, next_val):
        values = values + [next_val]

        adv_batch = list()
        for i in range(len(rewards)):
            gae = 0.0
            for j in range(i, len(rewards)):
                TD_i = rewards[j] + self.gamma*values[j+1] - values[j]
                gae += TD_i * (self.gamma*self.td_lambda)**(j - i)

            adv_batch.append(gae)

        return adv_batch

    def prepare_state(self, state, mean_entropy, mean_value, episode_start_step, total_episode_reward,
                      steps_at_last_reward, sel_actions, episode_over):
        # prevent the agent from getting stuck
        reset_game = False

        if (self.local_step - steps_at_last_reward > 5000
            or (self.emulator.get_lives() == 0
                and self.emulator.game not in ONE_LIFE_GAMES)):

            steps_at_last_reward = self.local_step
            episode_over = True
            reset_game = True

        # Start a new game on reaching terminal state
        if episode_over:
            elapsed_time = time.time() - self.start_time
            steps_per_sec = self.global_step.value() / elapsed_time
            perf = "{:.0f}".format(steps_per_sec)
            logger.info("T{} / EPISODE {} / STEP {}k / REWARD {} / {} STEPS/s, Actions {}".format(
                self.actor_id,
                self.local_episode,
                self.global_step.value()/1000,
                total_episode_reward,
                perf,
                sel_actions))

            self.log_summary(total_episode_reward, mean_value, mean_entropy)

            self.reset_hidden_state()
            self.local_episode += 1
            episode_start_step = self.local_step
            steps_at_last_reward = self.local_step
            total_episode_reward = 0.0
            mean_entropy = 0.0
            mean_value = 0.0

            if reset_game or self.emulator.game in ONE_LIFE_GAMES:
                state = self.emulator.get_initial_state()

        return state, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward

class A3CLearner(BaseA3CLearner):
    def __init__(self, args):
        super(A3CLearner, self).__init__(args)

        conf_learning = {'name': 'local_learning_{}'.format(self.actor_id),
                         'input_shape': self.input_shape,
                         'num_act': self.num_actions,
                         'args': args}

        self.local_network = PolicyValueNetwork(conf_learning)
        self.reset_hidden_state()

        if self.is_master():
            var_list = self.local_network.params
            self.saver = tf.train.Saver(var_list=var_list, max_to_keep=3,
                                        keep_checkpoint_every_n_hours=2)

    def choose_next_action(self, state):
        network_output_v, network_output_pi = self.session.run(
                [self.local_network.output_layer_v,
                 self.local_network.output_layer_pi],
                feed_dict={self.local_network.input_ph: [state]})

        network_output_pi = network_output_pi.reshape(-1)
        network_output_v = np.asscalar(network_output_v)

        action_index = self.sample_policy_action(network_output_pi)
        new_action = np.zeros([self.num_actions])
        new_action[action_index] = 1

        return new_action, network_output_v, network_output_pi

    def train(self):
        """ Main actor learner loop for advantage actor critic learning. """
        logger.debug("Actor {} resuming at Step {}".format(self.actor_id,
            self.global_step.value()))

        s = self.emulator.get_initial_state()
        steps_at_last_reward = self.local_step
        total_episode_reward = 0.0
        episode_start_step = 0

        while (self.global_step.value() < self.max_global_steps):
            # Sync local learning net with shared mem
            self.sync_net_with_shared_memory(self.local_network, self.learning_vars)
            self.save_vars()

            local_step_start = self.local_step
            reset_game = False
            episode_over = False

            rewards   = list()
            states    = list()
            actions   = list()
            values    = list()
            s_batch   = list()
            a_batch   = list()
            y_batch   = list()
            adv_batch = list()

            while not (episode_over
                or (self.local_step - local_step_start
                    == self.max_local_steps)):

                # Choose next action and execute it
                a, readout_v_t, readout_pi_t = self.choose_next_action(s)

                if self.is_master() and (self.local_step % 100 == 0):
                    logger.debug("pi={}, V={}".format(readout_pi_t, readout_v_t))

                new_s, reward, episode_over = self.emulator.next(a)
                if reward != 0.0:
                    steps_at_last_reward = self.local_step

                total_episode_reward += reward
                # Rescale or clip immediate reward
                reward = self.rescale_reward(reward)

                rewards.append(reward)
                states.append(s)
                actions.append(a)
                values.append(readout_v_t)

                s = new_s
                self.local_step += 1
                self.global_step.increment()

            # Calculate the value offered by critic in the new state.
            if episode_over:
                R = 0
            else:
                R = self.session.run(
                    self.local_network.output_layer_v,
                    feed_dict={self.local_network.input_ph:[new_s]})[0][0]

            sel_actions = []
            for i in reversed(xrange(len(states))):
                R = rewards[i] + self.gamma * R

                y_batch.append(R)
                a_batch.append(actions[i])
                s_batch.append(states[i])
                adv_batch.append(R - values[i])

                sel_actions.append(np.argmax(actions[i]))

            # Compute gradients on the local policy/V network and apply them to shared memory
            feed_dict={
                self.local_network.input_ph: s_batch,
                self.local_network.critic_target_ph: y_batch,
                self.local_network.selected_action_ph: a_batch,
                self.local_network.adv_actor_ph: adv_batch,
            }
            grads, entropy = self.session.run(
                [self.local_network.get_gradients, self.local_network.entropy],
                feed_dict=feed_dict)

            self.apply_gradients_to_shared_memory_vars(grads)

            delta_old = local_step_start - episode_start_step
            delta_new = self.local_step -  local_step_start

            s, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward = self.prepare_state(
                s, entropy, np.array(values).mean(), episode_start_step, total_episode_reward, steps_at_last_reward, sel_actions, episode_over)

class A3CLSTMLearner(BaseA3CLearner):
    def __init__(self, args):
        super(A3CLSTMLearner, self).__init__(args)

        conf_learning = {'name': 'local_learning_{}'.format(self.actor_id),
                         'input_shape': self.input_shape,
                         'num_act': self.num_actions,
                         'args': args}

        self.local_network = PolicyValueNetwork(conf_learning)
        self.reset_hidden_state()

        if self.is_master():
            var_list = self.local_network.params
            self.saver = tf.train.Saver(var_list=var_list, max_to_keep=3,
                                        keep_checkpoint_every_n_hours=2)

    def reset_hidden_state(self):
        self.lstm_state_out = np.zeros([1, 2*self.local_network.hidden_state_size])

    def choose_next_action(self, state):
        network_output_v, network_output_pi, self.lstm_state_out = self.session.run(
            [
                self.local_network.output_layer_v,
                self.local_network.output_layer_pi,
                self.local_network.lstm_state,
            ],
            feed_dict={
                self.local_network.input_ph: [state],
                self.local_network.step_size: [1],
                self.local_network.initial_lstm_state: self.lstm_state_out,
            })

        network_output_pi = network_output_pi.reshape(-1)
        network_output_v = np.asscalar(network_output_v)

        action_index = self.sample_policy_action(network_output_pi)
        new_action = np.zeros([self.num_actions])
        new_action[action_index] = 1

        return new_action, network_output_v, network_output_pi

    def _run(self):
        if not self.is_train:
            return self.test()

        """ Main actor learner loop for advantage actor critic learning. """
        logger.debug("Actor {} resuming at Step {}".format(self.actor_id,
            self.global_step.value()))

        s = self.emulator.get_initial_state()
        steps_at_last_reward = self.local_step
        total_episode_reward = 0.0
        episode_start_step = 0
        mean_entropy = 0.0

        while (self.global_step.value() < self.max_global_steps):
            # Sync local learning net with shared mem
            self.sync_net_with_shared_memory(self.local_network, self.learning_vars)
            self.save_vars()

            local_step_start = self.local_step
            local_lstm_state = np.copy(self.lstm_state_out)
            mean_value = 0.0
            reset_game = False
            episode_over = False

            rewards   = list()
            states    = list()
            actions   = list()
            values    = list()
            s_batch   = list()
            a_batch   = list()
            y_batch   = list()
            adv_batch = list()

            while not (episode_over
                or (self.local_step - local_step_start
                    == self.max_local_steps)):

                # Choose next action and execute it
                a, readout_v_t, readout_pi_t = self.choose_next_action(s)

                assert not np.allclose(local_lstm_state, self.lstm_state_out)

                if self.is_master() and (self.local_step % 100 == 0):
                    logger.debug("pi={}, V={}".format(readout_pi_t, readout_v_t))

                new_s, reward, episode_over = self.emulator.next(a)
                if reward != 0.0:
                    steps_at_last_reward = self.local_step

                total_episode_reward += reward
                # Rescale or clip immediate reward
                reward = self.rescale_reward(reward)

                rewards.append(reward)
                states.append(s)
                actions.append(a)
                values.append(readout_v_t)

                s = new_s
                self.local_step += 1
                self.global_step.increment()

            # Calculate the value offered by critic in the new state.
            if episode_over:
                R = 0
            else:
                # compute with repsect to target network
                prev_lstm_state_out = self.lstm_state_out
                R = self.session.run(
                    self.local_network.output_layer_v,
                    feed_dict={
                        self.local_network.input_ph:[new_s],
                        self.local_network.step_size: [1],
                        self.local_network.initial_lstm_state: self.lstm_state_out,
                    }
                )[0][0]
                assert np.allclose(prev_lstm_state_out, self.lstm_state_out)

            sel_actions = []
            for i in reversed(xrange(len(states))):
                R = rewards[i] + self.gamma * R

                y_batch.append(R)
                a_batch.append(actions[i])
                s_batch.append(states[i])
                adv_batch.append(R - values[i])

                sel_actions.append(np.argmax(actions[i]))

            # reverse everything so that the LSTM inputs are time-ordered
            y_batch.reverse()
            a_batch.reverse()
            s_batch.reverse()
            adv_batch.reverse()
            sel_actions.reverse()

            # Compute gradients on the local policy/V network and apply them to shared memory
            feed_dict={
                self.local_network.input_ph: s_batch,
                self.local_network.critic_target_ph: y_batch,
                self.local_network.selected_action_ph: a_batch,
                self.local_network.adv_actor_ph: adv_batch,
                self.local_network.step_size : [len(s_batch)],
                self.local_network.initial_lstm_state: local_lstm_state,
            }
            grads, entropy = self.session.run(
                [self.local_network.get_gradients, self.local_network.entropy],
                feed_dict=feed_dict)

            self.apply_gradients_to_shared_memory_vars(grads)

            delta_old = local_step_start - episode_start_step
            delta_new = self.local_step -  local_step_start

            s, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward = self.prepare_state(
                s, entropy, np.array(values).mean(), episode_start_step, total_episode_reward, steps_at_last_reward, sel_actions, episode_over)

<commit_msg>rename _run to train in A3CLSTMLearner<commit_after># -*- encoding: utf-8 -*-
import time
import numpy as np
import utils.logger
import tensorflow as tf
from utils import checkpoint_utils
from utils.decorators import only_on_train
from actor_learner import ActorLearner, ONE_LIFE_GAMES
from networks.policy_v_network import PolicyValueNetwork

logger = utils.logger.getLogger('policy_based_actor_learner')

class BaseA3CLearner(ActorLearner):
    def __init__(self, args):
        super(BaseA3CLearner, self).__init__(args)
        self.td_lambda = args.td_lambda

        # Shared mem vars
        self.learning_vars = args.learning_vars
        self.beta = args.entropy_regularisation_strength
        self.q_target_update_steps = args.q_target_update_steps

    def sample_policy_action(self, probs, temperature=0.5):
        probs = probs - np.finfo(np.float32).epsneg

        histogram = np.random.multinomial(1, probs)
        action_index = int(np.nonzero(histogram)[0])
        return action_index

    def compute_gae(self, rewards, values, next_val):
        values = values + [next_val]

        adv_batch = list()
        for i in range(len(rewards)):
            gae = 0.0
            for j in range(i, len(rewards)):
                TD_i = rewards[j] + self.gamma*values[j+1] - values[j]
                gae += TD_i * (self.gamma*self.td_lambda)**(j - i)

            adv_batch.append(gae)

        return adv_batch

    def prepare_state(self, state, mean_entropy, mean_value, episode_start_step, total_episode_reward,
                      steps_at_last_reward, sel_actions, episode_over):
        # prevent the agent from getting stuck
        reset_game = False

        if (self.local_step - steps_at_last_reward > 5000
            or (self.emulator.get_lives() == 0
                and self.emulator.game not in ONE_LIFE_GAMES)):

            steps_at_last_reward = self.local_step
            episode_over = True
            reset_game = True

        # Start a new game on reaching terminal state
        if episode_over:
            elapsed_time = time.time() - self.start_time
            steps_per_sec = self.global_step.value() / elapsed_time
            perf = "{:.0f}".format(steps_per_sec)
            logger.info("T{} / EPISODE {} / STEP {}k / REWARD {} / {} STEPS/s, Actions {}".format(
                self.actor_id,
                self.local_episode,
                self.global_step.value()/1000,
                total_episode_reward,
                perf,
                sel_actions))

            self.log_summary(total_episode_reward, mean_value, mean_entropy)

            self.reset_hidden_state()
            self.local_episode += 1
            episode_start_step = self.local_step
            steps_at_last_reward = self.local_step
            total_episode_reward = 0.0
            mean_entropy = 0.0
            mean_value = 0.0

            if reset_game or self.emulator.game in ONE_LIFE_GAMES:
                state = self.emulator.get_initial_state()

        return state, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward

class A3CLearner(BaseA3CLearner):
    def __init__(self, args):
        super(A3CLearner, self).__init__(args)

        conf_learning = {'name': 'local_learning_{}'.format(self.actor_id),
                         'input_shape': self.input_shape,
                         'num_act': self.num_actions,
                         'args': args}

        self.local_network = PolicyValueNetwork(conf_learning)
        self.reset_hidden_state()

        if self.is_master():
            var_list = self.local_network.params
            self.saver = tf.train.Saver(var_list=var_list, max_to_keep=3,
                                        keep_checkpoint_every_n_hours=2)

    def choose_next_action(self, state):
        network_output_v, network_output_pi = self.session.run(
                [self.local_network.output_layer_v,
                 self.local_network.output_layer_pi],
                feed_dict={self.local_network.input_ph: [state]})

        network_output_pi = network_output_pi.reshape(-1)
        network_output_v = np.asscalar(network_output_v)

        action_index = self.sample_policy_action(network_output_pi)
        new_action = np.zeros([self.num_actions])
        new_action[action_index] = 1

        return new_action, network_output_v, network_output_pi

    def train(self):
        """ Main actor learner loop for advantage actor critic learning. """
        logger.debug("Actor {} resuming at Step {}".format(self.actor_id,
            self.global_step.value()))

        s = self.emulator.get_initial_state()
        steps_at_last_reward = self.local_step
        total_episode_reward = 0.0
        episode_start_step = 0

        while (self.global_step.value() < self.max_global_steps):
            # Sync local learning net with shared mem
            self.sync_net_with_shared_memory(self.local_network, self.learning_vars)
            self.save_vars()

            local_step_start = self.local_step
            reset_game = False
            episode_over = False

            rewards   = list()
            states    = list()
            actions   = list()
            values    = list()
            s_batch   = list()
            a_batch   = list()
            y_batch   = list()
            adv_batch = list()

            while not (episode_over
                or (self.local_step - local_step_start
                    == self.max_local_steps)):

                # Choose next action and execute it
                a, readout_v_t, readout_pi_t = self.choose_next_action(s)

                if self.is_master() and (self.local_step % 100 == 0):
                    logger.debug("pi={}, V={}".format(readout_pi_t, readout_v_t))

                new_s, reward, episode_over = self.emulator.next(a)
                if reward != 0.0:
                    steps_at_last_reward = self.local_step

                total_episode_reward += reward
                # Rescale or clip immediate reward
                reward = self.rescale_reward(reward)

                rewards.append(reward)
                states.append(s)
                actions.append(a)
                values.append(readout_v_t)

                s = new_s
                self.local_step += 1
                self.global_step.increment()

            # Calculate the value offered by critic in the new state.
            if episode_over:
                R = 0
            else:
                R = self.session.run(
                    self.local_network.output_layer_v,
                    feed_dict={self.local_network.input_ph:[new_s]})[0][0]

            sel_actions = []
            for i in reversed(xrange(len(states))):
                R = rewards[i] + self.gamma * R

                y_batch.append(R)
                a_batch.append(actions[i])
                s_batch.append(states[i])
                adv_batch.append(R - values[i])

                sel_actions.append(np.argmax(actions[i]))

            # Compute gradients on the local policy/V network and apply them to shared memory
            feed_dict={
                self.local_network.input_ph: s_batch,
                self.local_network.critic_target_ph: y_batch,
                self.local_network.selected_action_ph: a_batch,
                self.local_network.adv_actor_ph: adv_batch,
            }
            grads, entropy = self.session.run(
                [self.local_network.get_gradients, self.local_network.entropy],
                feed_dict=feed_dict)

            self.apply_gradients_to_shared_memory_vars(grads)

            delta_old = local_step_start - episode_start_step
            delta_new = self.local_step -  local_step_start

            s, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward = self.prepare_state(
                s, entropy, np.array(values).mean(), episode_start_step, total_episode_reward, steps_at_last_reward, sel_actions, episode_over)

class A3CLSTMLearner(BaseA3CLearner):
    def __init__(self, args):
        super(A3CLSTMLearner, self).__init__(args)

        conf_learning = {'name': 'local_learning_{}'.format(self.actor_id),
                         'input_shape': self.input_shape,
                         'num_act': self.num_actions,
                         'args': args}

        self.local_network = PolicyValueNetwork(conf_learning)
        self.reset_hidden_state()

        if self.is_master():
            var_list = self.local_network.params
            self.saver = tf.train.Saver(var_list=var_list, max_to_keep=3,
                                        keep_checkpoint_every_n_hours=2)

    def reset_hidden_state(self):
        self.lstm_state_out = np.zeros([1, 2*self.local_network.hidden_state_size])

    def choose_next_action(self, state):
        network_output_v, network_output_pi, self.lstm_state_out = self.session.run(
            [
                self.local_network.output_layer_v,
                self.local_network.output_layer_pi,
                self.local_network.lstm_state,
            ],
            feed_dict={
                self.local_network.input_ph: [state],
                self.local_network.step_size: [1],
                self.local_network.initial_lstm_state: self.lstm_state_out,
            })

        network_output_pi = network_output_pi.reshape(-1)
        network_output_v = np.asscalar(network_output_v)

        action_index = self.sample_policy_action(network_output_pi)
        new_action = np.zeros([self.num_actions])
        new_action[action_index] = 1

        return new_action, network_output_v, network_output_pi

    def train(self):
        """ Main actor learner loop for advantage actor critic learning. """
        logger.debug("Actor {} resuming at Step {}".format(self.actor_id,
            self.global_step.value()))

        s = self.emulator.get_initial_state()
        steps_at_last_reward = self.local_step
        total_episode_reward = 0.0
        episode_start_step = 0
        mean_entropy = 0.0

        while (self.global_step.value() < self.max_global_steps):
            # Sync local learning net with shared mem
            self.sync_net_with_shared_memory(self.local_network, self.learning_vars)
            self.save_vars()

            local_step_start = self.local_step
            local_lstm_state = np.copy(self.lstm_state_out)
            mean_value = 0.0
            reset_game = False
            episode_over = False

            rewards   = list()
            states    = list()
            actions   = list()
            values    = list()
            s_batch   = list()
            a_batch   = list()
            y_batch   = list()
            adv_batch = list()

            while not (episode_over
                or (self.local_step - local_step_start
                    == self.max_local_steps)):

                # Choose next action and execute it
                a, readout_v_t, readout_pi_t = self.choose_next_action(s)

                assert not np.allclose(local_lstm_state, self.lstm_state_out)

                if self.is_master() and (self.local_step % 100 == 0):
                    logger.debug("pi={}, V={}".format(readout_pi_t, readout_v_t))

                new_s, reward, episode_over = self.emulator.next(a)
                if reward != 0.0:
                    steps_at_last_reward = self.local_step

                total_episode_reward += reward
                # Rescale or clip immediate reward
                reward = self.rescale_reward(reward)

                rewards.append(reward)
                states.append(s)
                actions.append(a)
                values.append(readout_v_t)

                s = new_s
                self.local_step += 1
                self.global_step.increment()

            # Calculate the value offered by critic in the new state.
            if episode_over:
                R = 0
            else:
                # compute with repsect to target network
                prev_lstm_state_out = self.lstm_state_out
                R = self.session.run(
                    self.local_network.output_layer_v,
                    feed_dict={
                        self.local_network.input_ph:[new_s],
                        self.local_network.step_size: [1],
                        self.local_network.initial_lstm_state: self.lstm_state_out,
                    }
                )[0][0]
                assert np.allclose(prev_lstm_state_out, self.lstm_state_out)

            sel_actions = []
            for i in reversed(xrange(len(states))):
                R = rewards[i] + self.gamma * R

                y_batch.append(R)
                a_batch.append(actions[i])
                s_batch.append(states[i])
                adv_batch.append(R - values[i])

                sel_actions.append(np.argmax(actions[i]))

            # reverse everything so that the LSTM inputs are time-ordered
            y_batch.reverse()
            a_batch.reverse()
            s_batch.reverse()
            adv_batch.reverse()
            sel_actions.reverse()

            # Compute gradients on the local policy/V network and apply them to shared memory
            feed_dict={
                self.local_network.input_ph: s_batch,
                self.local_network.critic_target_ph: y_batch,
                self.local_network.selected_action_ph: a_batch,
                self.local_network.adv_actor_ph: adv_batch,
                self.local_network.step_size : [len(s_batch)],
                self.local_network.initial_lstm_state: local_lstm_state,
            }
            grads, entropy = self.session.run(
                [self.local_network.get_gradients, self.local_network.entropy],
                feed_dict=feed_dict)

            self.apply_gradients_to_shared_memory_vars(grads)

            delta_old = local_step_start - episode_start_step
            delta_new = self.local_step -  local_step_start

            s, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward = self.prepare_state(
                s, entropy, np.array(values).mean(), episode_start_step, total_episode_reward, steps_at_last_reward, sel_actions, episode_over)

<|endoftext|>


<commit_before># coding: utf-8
#
# Copyright 2014 The Oppia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Interface for storage model switching."""

import feconf
import utils

# Valid model names.
NAMES = utils.create_enum(
    'activity', 'audit', 'base_model', 'classifier', 'collection', 'config',
    'email', 'exploration', 'feedback', 'file', 'job', 'question',
    'recommendations', 'statistics', 'user')

class _Platform(object):
    @classmethod
    def import_models(cls):
        raise NotImplementedError

class _Gae(_Platform):
    """Provides platform-specific imports related to GAE (Google App Engine).
    """
    @classmethod
    def import_models(cls, model_names):
        """Imports and returns the storage modules listed in model_names.

        Args:
            model_names: list(str). List of storage module names.

        Returns:
            tuple(module): Tuple of storage modules.

        Raises:
            Exception: Invalid model name.
        """
        returned_models = []
        for name in model_names:
            if name == NAMES.activity:
                from core.storage.activity import gae_models as activity_models
                returned_models.append(activity_models)
            elif name == NAMES.audit:
                from core.storage.audit import gae_models as audit_models
                returned_models.append(audit_models)
            elif name == NAMES.base_model:
                from core.storage.base_model import gae_models as base_models
                returned_models.append(base_models)
            elif name == NAMES.classifier:
                from core.storage.classifier import gae_models as classifier_data_models # pylint: disable=line-too-long
                returned_models.append(classifier_data_models)
            elif name == NAMES.collection:
                from core.storage.collection import gae_models as collection_models # pylint: disable=line-too-long
                returned_models.append(collection_models)
            elif name == NAMES.config:
                from core.storage.config import gae_models as config_models
                returned_models.append(config_models)
            elif name == NAMES.email:
                from core.storage.email import gae_models as email_models
                returned_models.append(email_models)
            elif name == NAMES.exploration:
                from core.storage.exploration import gae_models as exp_models
                returned_models.append(exp_models)
            elif name == NAMES.feedback:
                from core.storage.feedback import gae_models as feedback_models
                returned_models.append(feedback_models)
            elif name == NAMES.file:
                from core.storage.file import gae_models as file_models
                returned_models.append(file_models)
            elif name == NAMES.job:
                from core.storage.job import gae_models as job_models
                returned_models.append(job_models)
            elif name == NAMES.question:
                from core.storage.question import gae_models as question_models
                returned_models.append(question_models)
            elif name == NAMES.recommendations:
                from core.storage.recommendations import gae_models as recommendations_models # pylint: disable=line-too-long
                returned_models.append(recommendations_models)
            elif name == NAMES.statistics:
                from core.storage.statistics import gae_models as statistics_models # pylint: disable=line-too-long
                returned_models.append(statistics_models)
            elif name == NAMES.user:
                from core.storage.user import gae_models as user_models
                returned_models.append(user_models)
            else:
                raise Exception('Invalid model name: %s' % name)

        return tuple(returned_models)

    @classmethod
    def import_transaction_services(cls):
        """Imports and returns gae_transaction_services module.

        Returns:
            module. The gae_transaction_services module.
        """
        from core.platform.transactions import gae_transaction_services
        return gae_transaction_services

    @classmethod
    def import_current_user_services(cls):
        """Imports and returns gae_current_user_services module.

        Returns:
            module. The gae_current_user_services module.
        """
        from core.platform.users import gae_current_user_services
        return gae_current_user_services

    @classmethod
    def import_datastore_services(cls):
        """Imports and returns gae_datastore_services module.

        Returns:
            module. The gae_datastore_services module.
        """
        from core.platform.datastore import gae_datastore_services
        return gae_datastore_services

    @classmethod
    def import_app_identity_services(cls):
        """Imports and returns gae_app_identity_services module.

        Returns:
            module. The gae_app_identity_services module.
        """
        from core.platform.app_identity import gae_app_identity_services
        return gae_app_identity_services

    @classmethod
    def import_email_services(cls):
        """Imports and returns the email services module specified in feconf.py.

        Returns:
            module. The email_services module to use, based on the feconf.py
            setting.

        Raises:
            Exception: feconf.EMAIL_SERVICE_PROVIDER does not correspond
            to a valid email_services module.
        """
        if feconf.EMAIL_SERVICE_PROVIDER == feconf.EMAIL_SERVICE_PROVIDER_GAE:
            from core.platform.email import gae_email_services
            return gae_email_services
        elif (feconf.EMAIL_SERVICE_PROVIDER ==
              feconf.EMAIL_SERVICE_PROVIDER_MAILGUN):
            from core.platform.email import mailgun_email_services
            return mailgun_email_services
        else:
            raise Exception(
                ('Invalid email service provider: %s'
                 % feconf.EMAIL_SERVICE_PROVIDER))

    @classmethod
    def import_memcache_services(cls):
        """Imports and returns gae_memcache_services.

        Returns:
            module. The gae_memcache_services module.
        """
        from core.platform.memcache import gae_memcache_services
        return gae_memcache_services

    @classmethod
    def import_taskqueue_services(cls):
        """Imports and returns gae_taskqueue_services module.

        Returns:
            module. The gae_taskqueue_services module.
        """
        from core.platform.taskqueue import gae_taskqueue_services
        return gae_taskqueue_services

    @classmethod
    def import_search_services(cls):
        """Imports and returns gae_search_services module.

        Returns:
            module. The gae_search_services module.
        """
        from core.platform.search import gae_search_services
        return gae_search_services

    NAME = 'gae'

class Registry(object):
    """Platform-agnostic interface for retrieving platform-specific modules.
    """

    # Maps platform names to the corresponding module registry classes.
    _PLATFORM_MAPPING = {
        _Gae.NAME: _Gae,
    }

    @classmethod
    def _get(cls):
        """Returns the appropriate interface class for platform-specific
        imports.

        Returns:
            class: The corresponding platform-specific interface class.
        """
        return cls._PLATFORM_MAPPING.get(feconf.PLATFORM)

    @classmethod
    def import_models(cls, model_names):
        """Imports and returns the storage modules listed in model_names.

        Args:
            model_names: list(str). List of storage module names.

        Returns:
            list(module). The corresponding storage-layer modules.
        """
        return cls._get().import_models(model_names)

    @classmethod
    def import_current_user_services(cls):
        """Imports and returns current_user_services module.

        Returns:
            module. The current_user_services module.
        """
        return cls._get().import_current_user_services()

    @classmethod
    def import_datastore_services(cls):
        """Imports and returns datastore_services module.

        Returns:
            module. The datastore_services module.
        """
        return cls._get().import_datastore_services()

    @classmethod
    def import_transaction_services(cls):
        """Imports and returns transaction_services module.

        Returns:
            module. The transaction_services module.
        """
        return cls._get().import_transaction_services()

    @classmethod
    def import_app_identity_services(cls):
        """Imports and returns app_identity_services module.

        Returns:
            module. The app_identity_services module.
        """
        return cls._get().import_app_identity_services()

    @classmethod
    def import_email_services(cls):
        """Imports and returns email_services module.

        Returns:
            module. The email_services module.
        """
        return cls._get().import_email_services()

    @classmethod
    def import_memcache_services(cls):
        """Imports and returns memcache_services module.

        Returns:
            module. The memcache_services module.
        """
        return cls._get().import_memcache_services()

    @classmethod
    def import_taskqueue_services(cls):
        """Imports and returns taskqueue_services module.

        Returns:
            module. The taskqueue_services module.
        """
        return cls._get().import_taskqueue_services()

    @classmethod
    def import_search_services(cls):
        """Imports and returns search_services module.

        Returns:
            module. The search_services module.
        """
        return cls._get().import_search_services()
<commit_msg>Fix part of #4374: Documented core.platform.models (#4596)<commit_after># coding: utf-8
#
# Copyright 2014 The Oppia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Interface for storage model switching."""

import feconf
import utils

# Valid model names.
NAMES = utils.create_enum(
    'activity', 'audit', 'base_model', 'classifier', 'collection', 'config',
    'email', 'exploration', 'feedback', 'file', 'job', 'question',
    'recommendations', 'statistics', 'user')

class _Platform(object):
    """A base class for platform-specific imports related to GAE."""

    @classmethod
    def import_models(cls):
        """An abstract method that should be implemented on inherited
        classes.

        Raises:
            NotImplementedError: The method is not overwritten in derived
                classes.
        """
        raise NotImplementedError

class _Gae(_Platform):
    """Provides platform-specific imports related to GAE (Google App Engine).
    """

    @classmethod
    def import_models(cls, model_names):
        """Imports and returns the storage modules listed in model_names.

        Args:
            model_names: list(str). List of storage module names.

        Returns:
            tuple(module): Tuple of storage modules.

        Raises:
            Exception: Invalid model name.
        """
        returned_models = []
        for name in model_names:
            if name == NAMES.activity:
                from core.storage.activity import gae_models as activity_models
                returned_models.append(activity_models)
            elif name == NAMES.audit:
                from core.storage.audit import gae_models as audit_models
                returned_models.append(audit_models)
            elif name == NAMES.base_model:
                from core.storage.base_model import gae_models as base_models
                returned_models.append(base_models)
            elif name == NAMES.classifier:
                from core.storage.classifier import gae_models as classifier_data_models # pylint: disable=line-too-long
                returned_models.append(classifier_data_models)
            elif name == NAMES.collection:
                from core.storage.collection import gae_models as collection_models # pylint: disable=line-too-long
                returned_models.append(collection_models)
            elif name == NAMES.config:
                from core.storage.config import gae_models as config_models
                returned_models.append(config_models)
            elif name == NAMES.email:
                from core.storage.email import gae_models as email_models
                returned_models.append(email_models)
            elif name == NAMES.exploration:
                from core.storage.exploration import gae_models as exp_models
                returned_models.append(exp_models)
            elif name == NAMES.feedback:
                from core.storage.feedback import gae_models as feedback_models
                returned_models.append(feedback_models)
            elif name == NAMES.file:
                from core.storage.file import gae_models as file_models
                returned_models.append(file_models)
            elif name == NAMES.job:
                from core.storage.job import gae_models as job_models
                returned_models.append(job_models)
            elif name == NAMES.question:
                from core.storage.question import gae_models as question_models
                returned_models.append(question_models)
            elif name == NAMES.recommendations:
                from core.storage.recommendations import gae_models as recommendations_models # pylint: disable=line-too-long
                returned_models.append(recommendations_models)
            elif name == NAMES.statistics:
                from core.storage.statistics import gae_models as statistics_models # pylint: disable=line-too-long
                returned_models.append(statistics_models)
            elif name == NAMES.user:
                from core.storage.user import gae_models as user_models
                returned_models.append(user_models)
            else:
                raise Exception('Invalid model name: %s' % name)

        return tuple(returned_models)

    @classmethod
    def import_transaction_services(cls):
        """Imports and returns gae_transaction_services module.

        Returns:
            module. The gae_transaction_services module.
        """
        from core.platform.transactions import gae_transaction_services
        return gae_transaction_services

    @classmethod
    def import_current_user_services(cls):
        """Imports and returns gae_current_user_services module.

        Returns:
            module. The gae_current_user_services module.
        """
        from core.platform.users import gae_current_user_services
        return gae_current_user_services

    @classmethod
    def import_datastore_services(cls):
        """Imports and returns gae_datastore_services module.

        Returns:
            module. The gae_datastore_services module.
        """
        from core.platform.datastore import gae_datastore_services
        return gae_datastore_services

    @classmethod
    def import_app_identity_services(cls):
        """Imports and returns gae_app_identity_services module.

        Returns:
            module. The gae_app_identity_services module.
        """
        from core.platform.app_identity import gae_app_identity_services
        return gae_app_identity_services

    @classmethod
    def import_email_services(cls):
        """Imports and returns the email services module specified in feconf.py.

        Returns:
            module. The email_services module to use, based on the feconf.py
            setting.

        Raises:
            Exception: feconf.EMAIL_SERVICE_PROVIDER does not correspond
            to a valid email_services module.
        """
        if feconf.EMAIL_SERVICE_PROVIDER == feconf.EMAIL_SERVICE_PROVIDER_GAE:
            from core.platform.email import gae_email_services
            return gae_email_services
        elif (feconf.EMAIL_SERVICE_PROVIDER ==
              feconf.EMAIL_SERVICE_PROVIDER_MAILGUN):
            from core.platform.email import mailgun_email_services
            return mailgun_email_services
        else:
            raise Exception(
                ('Invalid email service provider: %s'
                 % feconf.EMAIL_SERVICE_PROVIDER))

    @classmethod
    def import_memcache_services(cls):
        """Imports and returns gae_memcache_services.

        Returns:
            module. The gae_memcache_services module.
        """
        from core.platform.memcache import gae_memcache_services
        return gae_memcache_services

    @classmethod
    def import_taskqueue_services(cls):
        """Imports and returns gae_taskqueue_services module.

        Returns:
            module. The gae_taskqueue_services module.
        """
        from core.platform.taskqueue import gae_taskqueue_services
        return gae_taskqueue_services

    @classmethod
    def import_search_services(cls):
        """Imports and returns gae_search_services module.

        Returns:
            module. The gae_search_services module.
        """
        from core.platform.search import gae_search_services
        return gae_search_services

    NAME = 'gae'

class Registry(object):
    """Platform-agnostic interface for retrieving platform-specific modules.
    """

    # Maps platform names to the corresponding module registry classes.
    _PLATFORM_MAPPING = {
        _Gae.NAME: _Gae,
    }

    @classmethod
    def _get(cls):
        """Returns the appropriate interface class for platform-specific
        imports.

        Returns:
            class: The corresponding platform-specific interface class.
        """
        return cls._PLATFORM_MAPPING.get(feconf.PLATFORM)

    @classmethod
    def import_models(cls, model_names):
        """Imports and returns the storage modules listed in model_names.

        Args:
            model_names: list(str). List of storage module names.

        Returns:
            list(module). The corresponding storage-layer modules.
        """
        return cls._get().import_models(model_names)

    @classmethod
    def import_current_user_services(cls):
        """Imports and returns current_user_services module.

        Returns:
            module. The current_user_services module.
        """
        return cls._get().import_current_user_services()

    @classmethod
    def import_datastore_services(cls):
        """Imports and returns datastore_services module.

        Returns:
            module. The datastore_services module.
        """
        return cls._get().import_datastore_services()

    @classmethod
    def import_transaction_services(cls):
        """Imports and returns transaction_services module.

        Returns:
            module. The transaction_services module.
        """
        return cls._get().import_transaction_services()

    @classmethod
    def import_app_identity_services(cls):
        """Imports and returns app_identity_services module.

        Returns:
            module. The app_identity_services module.
        """
        return cls._get().import_app_identity_services()

    @classmethod
    def import_email_services(cls):
        """Imports and returns email_services module.

        Returns:
            module. The email_services module.
        """
        return cls._get().import_email_services()

    @classmethod
    def import_memcache_services(cls):
        """Imports and returns memcache_services module.

        Returns:
            module. The memcache_services module.
        """
        return cls._get().import_memcache_services()

    @classmethod
    def import_taskqueue_services(cls):
        """Imports and returns taskqueue_services module.

        Returns:
            module. The taskqueue_services module.
        """
        return cls._get().import_taskqueue_services()

    @classmethod
    def import_search_services(cls):
        """Imports and returns search_services module.

        Returns:
            module. The search_services module.
        """
        return cls._get().import_search_services()
<|endoftext|>


<commit_before>"""Abstract collection for experiment and integration of all experiment types."""

from snovault import (
    abstract_collection,
    calculated_property,
    collection,
    load_schema,
)
# from pyramid.security import Authenticated
from .base import (
    Item
    # paths_filtered_by_status,
)

@abstract_collection(
    name='experiments',
    unique_key='accession',
    properties={
        'title': "Experiments",
        'description': 'Listing of all types of experiments.',
    })
class Experiment(Item):
    """The main expeperiment class."""

    base_types = ['Experiment'] + Item.base_types
    embedded = ["protocol", "protocol_variation", "lab", "award", "biosample",
                "biosample.biosource", "biosample.modifications",
                "biosample.treatments", "biosample.biosource.individual.organism"]
    name_key = 'accession'

    def _update(self, properties, sheets=None):
        # update self first to ensure 'experiment_relation' are stored in self.properties
        super(Experiment, self)._update(properties, sheets)
        DicRefRelation = {
            "controlled by": "control for",
            "derived from": "source for",
            "control for": "controlled by",
            "source for": "derived from"
        }
        acc = str(self.uuid)
        if 'experiment_relation' in properties.keys():
            for relation in properties["experiment_relation"]:
                switch = relation["relationship_type"]
                rev_switch = DicRefRelation[switch]
                related_exp = relation["experiment"]
                relationship_entry = {"relationship_type": rev_switch, "experiment": acc}
                rel_dic = {'experiment_relation': [relationship_entry, ]}

                target_exp = self.collection.get(related_exp)
                # case one we don't have relations
                if 'experiment_relation' not in target_exp.properties.keys():
                    target_exp.properties.update(rel_dic)
                    target_exp.update(target_exp.properties)
                else:
                    # case two we have relations but not the one we need
                    for target_relation in target_exp.properties['experiment_relation']:
                        if target_relation['experiment'] == acc:
                            break
                    else:
                        # make data for new experiemnt_relation
                        target_exp.properties['experiment_relation'].append(relationship_entry)
                        target_exp.update(target_exp.properties)

        # this part is for experiment_set
        if 'experiment_sets' in properties.keys():
            for exp_set in properties['experiment_sets']:
                target_exp_set = self.collection.get(exp_set)
                # look at the experiments inside the set
                if acc in target_exp_set.properties["experiments_in_set"]:
                    break
                else:
                    # incase it is not in the list of files
                    target_exp_set.properties["experiments_in_set"].append(acc)
                    target_exp_set.update(target_exp_set.properties)

@collection(
    name='experiment-sets',
    unique_key='uuid',
    properties={
        'title': 'Experiment Sets',
        'description': 'Listing Experiment Sets',
    })
class ExperimentSet(Item):
    """The experiment class for Hi-C experiments."""

    item_type = 'experiment_set'
    schema = load_schema('encoded:schemas/experiment_set.json')
    name_key = "uuid"
    embedded = ["experiments_in_set", "experiments_in_set.protocol", "experiments_in_set.protocol_variation",
                "experiments_in_set.lab", "experiments_in_set.award", "experiments_in_set.biosample",
                "experiments_in_set.biosample.biosource", "experiments_in_set.biosample.modifications",
                "experiments_in_set.biosample.treatments", "experiments_in_set.biosample.biosource.individual.organism",
                "experiments_in_set.files", "experiments_in_set.filesets", "experiments_in_set.filesets.files_in_set", "experiments_in_set.digestion_enzyme"]

    def _update(self, properties, sheets=None):
        # update self first
        super(ExperimentSet, self)._update(properties, sheets)
        esacc = str(self.uuid)
        if "experiments_in_set" in properties.keys():
            for each_exp in properties["experiments_in_set"]:
                target_exp = self.collection.get(each_exp)
                # are there any experiment sets in the experiment
                if 'experiment_sets' not in target_exp.properties.keys():
                    target_exp.properties.update({'experiment_sets': [esacc, ]})
                    target_exp.update(target_exp.properties)
                else:
                    # incase file already has the fileset_type
                    if esacc in target_exp.properties['experiment_sets']:
                        break
                    else:
                        target_exp.properties['experiment_sets'].append(esacc)
                        target_exp.update(target_exp.properties)

@collection(
    name='experiments-hic',
    unique_key='accession',
    properties={
        'title': 'Experiments Hi-C',
        'description': 'Listing Hi-C Experiments',
    })
class ExperimentHiC(Experiment):
    """The experiment class for Hi-C experiments."""

    item_type = 'experiment_hic'
    schema = load_schema('encoded:schemas/experiment_hic.json')
    embedded = Experiment.embedded + ["digestion_enzyme", "submitted_by"]

    @calculated_property(schema={
        "title": "Experiment summary",
        "description": "Summary of the experiment, including type, enzyme and biosource.",
        "type": "string",
    })
    def experiment_summary(self, request, experiment_type='Undefined', digestion_enzyme=None, biosample=None):
        sum_str = experiment_type
        if biosample:
            biosamp_props = request.embed(biosample, '@@object')
            biosource = biosamp_props['biosource_summary']
            sum_str += (' on ' + biosource)
        if digestion_enzyme:
            de_props = request.embed(digestion_enzyme, '@@object')
            de_name = de_props['name']
            sum_str += (' with ' + de_name)
        return sum_str

    def generate_mapid(self, experiment_type, num):
        delim = '_'
        mapid = str(type(self).__name__)
        mapid = mapid + delim + ''.join(experiment_type.split())
        return mapid + delim + str(num)

    @calculated_property(schema={
        "title": "SOP map",
        "description": "The mapping of fields default values from SOP",
        "type": "object",
        "linkTo": "SopMap"
    })
    def sop_mapping(self, request, experiment_type):
        maps = []
        suffnum = 1
        mapid = self.generate_mapid(experiment_type, suffnum)
        print(mapid)
        #bs = '4DNBS1234567'
        while(True):
            m = self.collection.get(mapid)
            #m = self.collection.get(bs)
            print(m)
            if not m:
                break
            maps.append(m)
            suffnum += 1
            mapid = self.generate_mapid(experiment_type, suffnum)

        if len(maps) > 0:
            lmap = maps[-1]
            print(lmap)
            return request.embed('/sop_maps/' + lmap['@id'] + '/', '@@object')
        else:
            return None

@collection(
    name='experiments-capture-hic',
    unique_key='accession',
    properties={
        'title': 'Experiments Capture Hi-C',
        'description': 'Listing Capture Hi-C Experiments',
    })
class ExperimentCaptureC(Experiment):
    """The experiment class for Capture Hi-C experiments."""
    item_type = 'experiment_capture_c'
    schema = load_schema('encoded:schemas/experiment_capture_c.json')
    embedded = Experiment.embedded + ["digestion_enzyme", "submitted_by"]

    @calculated_property(schema={
        "title": "Experiment summary",
        "description": "Summary of the experiment, including type, enzyme and biosource.",
        "type": "string",
    })
    def experiment_summary(self, request, experiment_type='Undefined', digestion_enzyme=None, biosample=None):
        sum_str = experiment_type
        if biosample:
            biosamp_props = request.embed(biosample, '@@object')
            biosource = biosamp_props['biosource_summary']
            sum_str += (' on ' + biosource)
        if digestion_enzyme:
            de_props = request.embed(digestion_enzyme, '@@object')
            de_name = de_props['name']
            sum_str += (' with ' + de_name)
        return sum_str
<commit_msg>FF-357 #comment Tried moving around method in experiment.py - get still not working.<commit_after>"""Abstract collection for experiment and integration of all experiment types."""

from snovault import (
    abstract_collection,
    calculated_property,
    collection,
    load_schema,
)
# from pyramid.security import Authenticated
from .base import (
    Item
    # paths_filtered_by_status,
)

@abstract_collection(
    name='experiments',
    unique_key='accession',
    properties={
        'title': "Experiments",
        'description': 'Listing of all types of experiments.',
    })
class Experiment(Item):
    """The main expeperiment class."""

    base_types = ['Experiment'] + Item.base_types
    embedded = ["protocol", "protocol_variation", "lab", "award", "biosample",
                "biosample.biosource", "biosample.modifications",
                "biosample.treatments", "biosample.biosource.individual.organism"]
    name_key = 'accession'

    def _update(self, properties, sheets=None):
        # update self first to ensure 'experiment_relation' are stored in self.properties
        super(Experiment, self)._update(properties, sheets)
        DicRefRelation = {
            "controlled by": "control for",
            "derived from": "source for",
            "control for": "controlled by",
            "source for": "derived from"
        }
        acc = str(self.uuid)
        if 'experiment_relation' in properties.keys():
            for relation in properties["experiment_relation"]:
                switch = relation["relationship_type"]
                rev_switch = DicRefRelation[switch]
                related_exp = relation["experiment"]
                relationship_entry = {"relationship_type": rev_switch, "experiment": acc}
                rel_dic = {'experiment_relation': [relationship_entry, ]}

                target_exp = self.collection.get(related_exp)
                # case one we don't have relations
                if 'experiment_relation' not in target_exp.properties.keys():
                    target_exp.properties.update(rel_dic)
                    target_exp.update(target_exp.properties)
                else:
                    # case two we have relations but not the one we need
                    for target_relation in target_exp.properties['experiment_relation']:
                        if target_relation['experiment'] == acc:
                            break
                    else:
                        # make data for new experiemnt_relation
                        target_exp.properties['experiment_relation'].append(relationship_entry)
                        target_exp.update(target_exp.properties)

        # this part is for experiment_set
        if 'experiment_sets' in properties.keys():
            for exp_set in properties['experiment_sets']:
                target_exp_set = self.collection.get(exp_set)
                # look at the experiments inside the set
                if acc in target_exp_set.properties["experiments_in_set"]:
                    break
                else:
                    # incase it is not in the list of files
                    target_exp_set.properties["experiments_in_set"].append(acc)
                    target_exp_set.update(target_exp_set.properties)

@collection(
    name='experiment-sets',
    unique_key='uuid',
    properties={
        'title': 'Experiment Sets',
        'description': 'Listing Experiment Sets',
    })
class ExperimentSet(Item):
    """The experiment class for Hi-C experiments."""

    item_type = 'experiment_set'
    schema = load_schema('encoded:schemas/experiment_set.json')
    name_key = "uuid"
    embedded = ["experiments_in_set", "experiments_in_set.protocol", "experiments_in_set.protocol_variation",
                "experiments_in_set.lab", "experiments_in_set.award", "experiments_in_set.biosample",
                "experiments_in_set.biosample.biosource", "experiments_in_set.biosample.modifications",
                "experiments_in_set.biosample.treatments", "experiments_in_set.biosample.biosource.individual.organism",
                "experiments_in_set.files", "experiments_in_set.filesets", "experiments_in_set.filesets.files_in_set", "experiments_in_set.digestion_enzyme"]

    def _update(self, properties, sheets=None):
        # update self first
        super(ExperimentSet, self)._update(properties, sheets)
        esacc = str(self.uuid)
        if "experiments_in_set" in properties.keys():
            for each_exp in properties["experiments_in_set"]:
                target_exp = self.collection.get(each_exp)
                # are there any experiment sets in the experiment
                if 'experiment_sets' not in target_exp.properties.keys():
                    target_exp.properties.update({'experiment_sets': [esacc, ]})
                    target_exp.update(target_exp.properties)
                else:
                    # incase file already has the fileset_type
                    if esacc in target_exp.properties['experiment_sets']:
                        break
                    else:
                        target_exp.properties['experiment_sets'].append(esacc)
                        target_exp.update(target_exp.properties)

@collection(
    name='experiments-hic',
    unique_key='accession',
    properties={
        'title': 'Experiments Hi-C',
        'description': 'Listing Hi-C Experiments',
    })
class ExperimentHiC(Experiment):
    """The experiment class for Hi-C experiments."""

    item_type = 'experiment_hic'
    schema = load_schema('encoded:schemas/experiment_hic.json')
    embedded = Experiment.embedded + ["digestion_enzyme", "submitted_by"]

    @calculated_property(schema={
        "title": "Experiment summary",
        "description": "Summary of the experiment, including type, enzyme and biosource.",
        "type": "string",
    })
    def experiment_summary(self, request, experiment_type='Undefined', digestion_enzyme=None, biosample=None):
        sum_str = experiment_type
        if biosample:
            biosamp_props = request.embed(biosample, '@@object')
            biosource = biosamp_props['biosource_summary']
            sum_str += (' on ' + biosource)
        if digestion_enzyme:
            de_props = request.embed(digestion_enzyme, '@@object')
            de_name = de_props['name']
            sum_str += (' with ' + de_name)
        return sum_str

    def generate_mapid(self, experiment_type, num):
        delim = '_'
        mapid = str(type(self).__name__)
        mapid = mapid + delim + ''.join(experiment_type.split())
        return mapid + delim + str(num)

    @calculated_property(schema={
        "title": "SOP map",
        "description": "The mapping of fields default values from SOP",
        "type": "object",
        "linkTo": "SopMap"
    })
    def sop_mapping(self, request, experiment_type):
        maps = []
        suffnum = 1
        mapid = self.generate_mapid(experiment_type, suffnum)
        print(mapid)
        #bs = '4DNBS1234567'
        while(True):
            m = self.collection.get('ExperimentHiC_insituHi-C_1')
            #m = self.collection.get(bs)
            print(m)
            if not m:
                break
            maps.append(m)
            suffnum += 1
            mapid = self.generate_mapid(experiment_type, suffnum)

        if len(maps) > 0:
            lmap = maps[-1]
            print(lmap)
            return request.embed('/sop_maps/' + lmap['@id'] + '/', '@@object')
        else:
            return None

@collection(
    name='experiments-capture-hic',
    unique_key='accession',
    properties={
        'title': 'Experiments Capture Hi-C',
        'description': 'Listing Capture Hi-C Experiments',
    })
class ExperimentCaptureC(Experiment):
    """The experiment class for Capture Hi-C experiments."""
    item_type = 'experiment_capture_c'
    schema = load_schema('encoded:schemas/experiment_capture_c.json')
    embedded = Experiment.embedded + ["digestion_enzyme", "submitted_by"]

    @calculated_property(schema={
        "title": "Experiment summary",
        "description": "Summary of the experiment, including type, enzyme and biosource.",
        "type": "string",
    })
    def experiment_summary(self, request, experiment_type='Undefined', digestion_enzyme=None, biosample=None):
        sum_str = experiment_type
        if biosample:
            biosamp_props = request.embed(biosample, '@@object')
            biosource = biosamp_props['biosource_summary']
            sum_str += (' on ' + biosource)
        if digestion_enzyme:
            de_props = request.embed(digestion_enzyme, '@@object')
            de_name = de_props['name']
            sum_str += (' with ' + de_name)
        return sum_str
<|endoftext|>


<commit_before>import nose.tools

import ckan.tests.factories as factories
import ckan.tests.helpers as helpers
import ckan.plugins.toolkit as toolkit
import ckanext.mapactionimporter.tests.helpers as custom_helpers

assert_equals = nose.tools.assert_equals
assert_true = nose.tools.assert_true
assert_regexp_matches = nose.tools.assert_regexp_matches

class TestDataPackageController(
    custom_helpers.FunctionalTestBaseClass):

    def test_import_zipfile(self):
        slug = 'ma001-aptivate-example'
        user = factories.User()
        env = {'REMOTE_USER': user['name'].encode('ascii')}
        url = toolkit.url_for('import_mapactionzip')
        response = self.app.post(
            url,
            extra_environ=env,
            upload_files=[(
                'upload',
                 custom_helpers.get_test_file().name,
            )],
        )

        # Should redirect to dataset's page
        assert_equals(response.status_int, 302)
        assert_regexp_matches(
            response.headers['Location'],
            '/dataset/%s' % slug)

        # Should create the dataset
        dataset = helpers.call_action('package_show', id=slug)
        assert_equals(dataset['name'], slug)

<commit_msg>Fixed PEP8 warnings<commit_after>import nose.tools

import ckan.tests.factories as factories
import ckan.tests.helpers as helpers
import ckan.plugins.toolkit as toolkit
import ckanext.mapactionimporter.tests.helpers as custom_helpers

assert_equals = nose.tools.assert_equals
assert_true = nose.tools.assert_true
assert_regexp_matches = nose.tools.assert_regexp_matches

class TestDataPackageController(custom_helpers.FunctionalTestBaseClass):

    def test_import_zipfile(self):
        slug = 'ma001-aptivate-example'
        user = factories.User()
        env = {'REMOTE_USER': user['name'].encode('ascii')}
        url = toolkit.url_for('import_mapactionzip')
        response = self.app.post(
            url,
            extra_environ=env,
            upload_files=[(
                'upload',
                custom_helpers.get_test_file().name,
            )],
        )

        # Should redirect to dataset's page
        assert_equals(response.status_int, 302)
        assert_regexp_matches(
            response.headers['Location'],
            '/dataset/%s' % slug)

        # Should create the dataset
        dataset = helpers.call_action('package_show', id=slug)
        assert_equals(dataset['name'], slug)
<|endoftext|>


<commit_before>from django.contrib import admin
from reversion.admin import VersionAdmin
from ajax_select.admin import AjaxSelectAdmin
from ajax_select import make_ajax_form

# from cms.admin import JSONEditor

from django.forms.widgets import Textarea
try:
    from django.forms.util import flatatt
except ImportError:
    from django.forms.utils import flatatt
from django.utils.safestring import mark_safe
from django.conf import settings
import json

from jsonfield import JSONField
from .models import Discipline, TrainingTerms, Semester, Variant, Technology, Diagram

class JSONEditor(Textarea):
    class Media:
        js = (
            getattr(settings, "JQUERY", settings.STATIC_URL + 'jquery-3.1.1.min.js'),
            getattr(settings, "JQUERY_UI", settings.STATIC_URL + 'jquery-ui.min.js'),
            getattr(settings, "JQUERY_EVENT_DRAG", settings.STATIC_URL + 'jquery.event.drag.js'),

            getattr(settings, "SLICK_JS_CORE", settings.STATIC_URL+'slickgrid/slick.core.js'),
            getattr(settings, "SLICK_JS_FORMATTERS", settings.STATIC_URL + 'slickgrid/slick.formatters.js'),
            getattr(settings, "SLICK_JS_EDITOR", settings.STATIC_URL + 'slickgrid/slick.editor.js'),
            getattr(settings, "SLICK_JS_GRID", settings.STATIC_URL + 'slickgrid/slick.grid.js'),
        )
        css = {'all': (getattr(settings, "SLICK_CSS", settings.STATIC_URL+'slickgrid/slick.grid.css'),)}

    def render(self, name, value, attrs=None):
        try:
            value = json.loads(value)
        except TypeError:
            pass
        input_attrs = {'hidden': True}
        input_attrs.update(attrs)
        if 'class' not in input_attrs:
            input_attrs['class'] = 'for_jsoneditor'
        else:
            input_attrs['class'] += ' for_jsoneditor'
        r = super(JSONEditor, self).render(name, value, input_attrs)
        div_attrs = {}
        div_attrs.update(attrs)
        div_attrs.update({'id': (attrs['id']+'_jsoneditor')})
        final_attrs = self.build_attrs(div_attrs, extra_attrs={"name": name})
        r += '''

        ''' % {
            'attrs': flatatt(final_attrs),
            'name': name
        }
        return mark_safe(r)

@admin.register(Discipline)
class DisciplineAdmin(VersionAdmin):
    fields = ("title", "description", "module", "period",  "labor", "form", "status", "archived", "results", "results_text")
    list_display = ("id", "title", "description", "module", "period", "labor", "form", "status", "archived")
    filter_horizontal = ("results",)
    list_filter = ("archived", "created", "updated", "status", "form")
    search_fields = ['title', 'module__title', "module__uni_number"]

@admin.register(TrainingTerms)
class TrainingTermsAdmin(VersionAdmin):
    list_display = ("title", "limit")

@admin.register(Semester)
class SemesterAdmin(VersionAdmin):
    list_display = ("__str__", )
    search_fields = ("discipline__title", "discipline__module__title", "program__title")
    form = make_ajax_form(Semester, {'discipline': 'discipline'})

@admin.register(Variant)
class VariantAdmin(VersionAdmin, AjaxSelectAdmin):
    list_display = ("discipline", "program", "course", "semester", "parity", "link", "status", "archived")
    list_filter = ("parity", "archived", "created", "updated", "status")
    search_fields = ("discipline__title", "program__title", "course__title", "parity", "link")
    form = make_ajax_form(Variant, {'discipline': 'discipline', "semester": "semester"})

@admin.register(Technology)
class TechnologyAdmin(VersionAdmin):
    list_display = ("title", "description", "contact_work_category", "get_color")

@admin.register(Diagram)
class DiagramAdmin(VersionAdmin):
    list_display = ("title", "diagram")
    formfield_overrides = {
        JSONField: {'widget': JSONEditor},
    }
<commit_msg>slickgrid#5<commit_after>from django.contrib import admin
from reversion.admin import VersionAdmin
from ajax_select.admin import AjaxSelectAdmin
from ajax_select import make_ajax_form

# from cms.admin import JSONEditor

from django.forms.widgets import Textarea
try:
    from django.forms.util import flatatt
except ImportError:
    from django.forms.utils import flatatt
from django.utils.safestring import mark_safe
from django.conf import settings
import json

from jsonfield import JSONField
from .models import Discipline, TrainingTerms, Semester, Variant, Technology, Diagram

class JSONEditor(Textarea):
    class Media:
        js = (
            getattr(settings, "JQUERY", settings.STATIC_URL + 'jquery-3.1.1.min.js'),
            getattr(settings, "JQUERY_UI", settings.STATIC_URL + 'jquery-ui.min.js'),
            getattr(settings, "JQUERY_EVENT_DRAG", settings.STATIC_URL + 'jquery.event.drag.js'),

            getattr(settings, "SLICK_JS_CORE", settings.STATIC_URL+'slickgrid/slick.core.js'),
            getattr(settings, "SLICK_JS_FORMATTERS", settings.STATIC_URL + 'slickgrid/slick.formatters.js'),
            getattr(settings, "SLICK_JS_EDITORS", settings.STATIC_URL + 'slickgrid/slick.editors.js'),
            getattr(settings, "SLICK_JS_GRID", settings.STATIC_URL + 'slickgrid/slick.grid.js'),
        )
        css = {'all': (getattr(settings, "SLICK_CSS", settings.STATIC_URL+'slickgrid/slick.grid.css'),)}

    def render(self, name, value, attrs=None):
        try:
            value = json.loads(value)
        except TypeError:
            pass
        input_attrs = {'hidden': True}
        input_attrs.update(attrs)
        if 'class' not in input_attrs:
            input_attrs['class'] = 'for_jsoneditor'
        else:
            input_attrs['class'] += ' for_jsoneditor'
        r = super(JSONEditor, self).render(name, value, input_attrs)
        div_attrs = {}
        div_attrs.update(attrs)
        div_attrs.update({'id': (attrs['id']+'_jsoneditor')})
        final_attrs = self.build_attrs(div_attrs, extra_attrs={"name": name})
        r += '''

        ''' % {
            'attrs': flatatt(final_attrs),
            'name': name
        }
        return mark_safe(r)

@admin.register(Discipline)
class DisciplineAdmin(VersionAdmin):
    fields = ("title", "description", "module", "period",  "labor", "form", "status", "archived", "results", "results_text")
    list_display = ("id", "title", "description", "module", "period", "labor", "form", "status", "archived")
    filter_horizontal = ("results",)
    list_filter = ("archived", "created", "updated", "status", "form")
    search_fields = ['title', 'module__title', "module__uni_number"]

@admin.register(TrainingTerms)
class TrainingTermsAdmin(VersionAdmin):
    list_display = ("title", "limit")

@admin.register(Semester)
class SemesterAdmin(VersionAdmin):
    list_display = ("__str__", )
    search_fields = ("discipline__title", "discipline__module__title", "program__title")
    form = make_ajax_form(Semester, {'discipline': 'discipline'})

@admin.register(Variant)
class VariantAdmin(VersionAdmin, AjaxSelectAdmin):
    list_display = ("discipline", "program", "course", "semester", "parity", "link", "status", "archived")
    list_filter = ("parity", "archived", "created", "updated", "status")
    search_fields = ("discipline__title", "program__title", "course__title", "parity", "link")
    form = make_ajax_form(Variant, {'discipline': 'discipline', "semester": "semester"})

@admin.register(Technology)
class TechnologyAdmin(VersionAdmin):
    list_display = ("title", "description", "contact_work_category", "get_color")

@admin.register(Diagram)
class DiagramAdmin(VersionAdmin):
    list_display = ("title", "diagram")
    formfield_overrides = {
        JSONField: {'widget': JSONEditor},
    }
<|endoftext|>


<commit_before>"""Support for Synology DSM binary sensors."""
from __future__ import annotations

from typing import Any

from homeassistant.components.binary_sensor import BinarySensorEntity
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import CONF_DISKS
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator

from . import SynoApi, SynologyDSMBaseEntity, SynologyDSMDeviceEntity
from .const import (
    COORDINATOR_CENTRAL,
    DOMAIN,
    SECURITY_BINARY_SENSORS,
    STORAGE_DISK_BINARY_SENSORS,
    SYNO_API,
    UPGRADE_BINARY_SENSORS,
    SynologyDSMBinarySensorEntityDescription,
)

async def async_setup_entry(
    hass: HomeAssistant, entry: ConfigEntry, async_add_entities: AddEntitiesCallback
) -> None:
    """Set up the Synology NAS binary sensor."""

    data = hass.data[DOMAIN][entry.unique_id]
    api: SynoApi = data[SYNO_API]
    coordinator = data[COORDINATOR_CENTRAL]

    entities: list[
        SynoDSMSecurityBinarySensor
        | SynoDSMUpgradeBinarySensor
        | SynoDSMStorageBinarySensor
    ] = [
        SynoDSMSecurityBinarySensor(api, coordinator, description)
        for description in SECURITY_BINARY_SENSORS
    ]

    entities.extend(
        [
            SynoDSMUpgradeBinarySensor(api, coordinator, description)
            for description in UPGRADE_BINARY_SENSORS
        ]
    )

    # Handle all disks
    if api.storage.disks_ids:
        entities.extend(
            [
                SynoDSMStorageBinarySensor(api, coordinator, description, disk)
                for disk in entry.data.get(CONF_DISKS, api.storage.disks_ids)
                for description in STORAGE_DISK_BINARY_SENSORS
            ]
        )

    async_add_entities(entities)

class SynoDSMBinarySensor(SynologyDSMBaseEntity, BinarySensorEntity):
    """Mixin for binary sensor specific attributes."""

    entity_description: SynologyDSMBinarySensorEntityDescription

    def __init__(
        self,
        api: SynoApi,
        coordinator: DataUpdateCoordinator[dict[str, dict[str, Any]]],
        description: SynologyDSMBinarySensorEntityDescription,
    ) -> None:
        """Initialize the Synology DSM binary_sensor entity."""
        super().__init__(api, coordinator, description)

class SynoDSMSecurityBinarySensor(SynoDSMBinarySensor):
    """Representation a Synology Security binary sensor."""

    @property
    def is_on(self) -> bool:
        """Return the state."""
        return getattr(self._api.security, self.entity_description.key) != "safe"  # type: ignore[no-any-return]

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        return bool(self._api.security)

    @property
    def extra_state_attributes(self) -> dict[str, str]:
        """Return security checks details."""
        return self._api.security.status_by_check  # type: ignore[no-any-return]

class SynoDSMStorageBinarySensor(SynologyDSMDeviceEntity, SynoDSMBinarySensor):
    """Representation a Synology Storage binary sensor."""

    entity_description: SynologyDSMBinarySensorEntityDescription

    def __init__(
        self,
        api: SynoApi,
        coordinator: DataUpdateCoordinator[dict[str, dict[str, Any]]],
        description: SynologyDSMBinarySensorEntityDescription,
        device_id: str | None = None,
    ) -> None:
        """Initialize the Synology DSM storage binary_sensor entity."""
        super().__init__(api, coordinator, description, device_id)

    @property
    def is_on(self) -> bool:
        """Return the state."""
        return bool(
            getattr(self._api.storage, self.entity_description.key)(self._device_id)
        )

class SynoDSMUpgradeBinarySensor(SynoDSMBinarySensor):
    """Representation a Synology Upgrade binary sensor."""

    @property
    def is_on(self) -> bool:
        """Return the state."""
        return bool(getattr(self._api.upgrade, self.entity_description.key))

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        return bool(self._api.upgrade)
<commit_msg>Add current and latest firmware info to Synology_dsm (#56460)<commit_after>"""Support for Synology DSM binary sensors."""
from __future__ import annotations

from collections.abc import Mapping
from typing import Any

from homeassistant.components.binary_sensor import BinarySensorEntity
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import CONF_DISKS
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator

from . import SynoApi, SynologyDSMBaseEntity, SynologyDSMDeviceEntity
from .const import (
    COORDINATOR_CENTRAL,
    DOMAIN,
    SECURITY_BINARY_SENSORS,
    STORAGE_DISK_BINARY_SENSORS,
    SYNO_API,
    UPGRADE_BINARY_SENSORS,
    SynologyDSMBinarySensorEntityDescription,
)

async def async_setup_entry(
    hass: HomeAssistant, entry: ConfigEntry, async_add_entities: AddEntitiesCallback
) -> None:
    """Set up the Synology NAS binary sensor."""

    data = hass.data[DOMAIN][entry.unique_id]
    api: SynoApi = data[SYNO_API]
    coordinator = data[COORDINATOR_CENTRAL]

    entities: list[
        SynoDSMSecurityBinarySensor
        | SynoDSMUpgradeBinarySensor
        | SynoDSMStorageBinarySensor
    ] = [
        SynoDSMSecurityBinarySensor(api, coordinator, description)
        for description in SECURITY_BINARY_SENSORS
    ]

    entities.extend(
        [
            SynoDSMUpgradeBinarySensor(api, coordinator, description)
            for description in UPGRADE_BINARY_SENSORS
        ]
    )

    # Handle all disks
    if api.storage.disks_ids:
        entities.extend(
            [
                SynoDSMStorageBinarySensor(api, coordinator, description, disk)
                for disk in entry.data.get(CONF_DISKS, api.storage.disks_ids)
                for description in STORAGE_DISK_BINARY_SENSORS
            ]
        )

    async_add_entities(entities)

class SynoDSMBinarySensor(SynologyDSMBaseEntity, BinarySensorEntity):
    """Mixin for binary sensor specific attributes."""

    entity_description: SynologyDSMBinarySensorEntityDescription

    def __init__(
        self,
        api: SynoApi,
        coordinator: DataUpdateCoordinator[dict[str, dict[str, Any]]],
        description: SynologyDSMBinarySensorEntityDescription,
    ) -> None:
        """Initialize the Synology DSM binary_sensor entity."""
        super().__init__(api, coordinator, description)

class SynoDSMSecurityBinarySensor(SynoDSMBinarySensor):
    """Representation a Synology Security binary sensor."""

    @property
    def is_on(self) -> bool:
        """Return the state."""
        return getattr(self._api.security, self.entity_description.key) != "safe"  # type: ignore[no-any-return]

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        return bool(self._api.security)

    @property
    def extra_state_attributes(self) -> dict[str, str]:
        """Return security checks details."""
        return self._api.security.status_by_check  # type: ignore[no-any-return]

class SynoDSMStorageBinarySensor(SynologyDSMDeviceEntity, SynoDSMBinarySensor):
    """Representation a Synology Storage binary sensor."""

    entity_description: SynologyDSMBinarySensorEntityDescription

    def __init__(
        self,
        api: SynoApi,
        coordinator: DataUpdateCoordinator[dict[str, dict[str, Any]]],
        description: SynologyDSMBinarySensorEntityDescription,
        device_id: str | None = None,
    ) -> None:
        """Initialize the Synology DSM storage binary_sensor entity."""
        super().__init__(api, coordinator, description, device_id)

    @property
    def is_on(self) -> bool:
        """Return the state."""
        return bool(
            getattr(self._api.storage, self.entity_description.key)(self._device_id)
        )

class SynoDSMUpgradeBinarySensor(SynoDSMBinarySensor):
    """Representation a Synology Upgrade binary sensor."""

    @property
    def is_on(self) -> bool:
        """Return the state."""
        return bool(getattr(self._api.upgrade, self.entity_description.key))

    @property
    def available(self) -> bool:
        """Return True if entity is available."""
        return bool(self._api.upgrade)

    @property
    def extra_state_attributes(self) -> Mapping[str, Any] | None:
        """Return firmware details."""
        return {
            "installed_version": self._api.information.version_string,
            "latest_available_version": self._api.upgrade.available_version,
        }
<|endoftext|>