<commit_before>#Copyright (C) 2016 Paolo Galeone <nessuno@nerdz.eu>
# Based on Tensorflow cifar10_train.py file
# https://github.com/tensorflow/tensorflow/blob/r0.11/tensorflow/models/image/cifar10/cifar10_train.py
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
""" Train model with a single GPU. Evaluate it on the second one"""

import sys
from datetime import datetime
import os.path
import time
import math

import numpy as np
import tensorflow as tf
from models import model2 as vgg
from inputs import cifar10 as dataset
import evaluate

BATCH_SIZE = 128
STEP_PER_EPOCH = math.ceil(dataset.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
                           BATCH_SIZE)
MAX_EPOCH = 300
MAX_STEPS = STEP_PER_EPOCH * MAX_EPOCH

CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
LOG_DIR = CURRENT_DIR + "/log/" + vgg.NAME + '/keep_prob_decay'

MAX_KEEP_PROB = 1.0

def keep_prob_decay(validation_accuracy_,
                    keep_prob,
                    min_keep_prob,
                    num_updates,
                    decay_amount,
                    name=None):
    """ Decay keep_prob until it reaches min_keep_pro. Computation
    based on validation_accuracy_ variations
    """

    with tf.name_scope(name, "KeepProbDecay", [
            validation_accuracy_, keep_prob, min_keep_prob, num_updates,
            decay_amount
    ]) as name:
        validation_accuracy_ = tf.convert_to_tensor(
            validation_accuracy_,
            name="validation_accuracy_",
            dtype=tf.float32)
        decay_amount = tf.convert_to_tensor(
            decay_amount, name="decay_amount", dtype=tf.float32)
        keep_prob = tf.convert_to_tensor(
            keep_prob, name="keep_prob", dtype=tf.float32)
        # initialize keep_prob with keep_prob + decay_amount
        # to handle the case of the first decay
        # that always happen because of the constant ratio
        # of va/rolling_avg = va/va
        # Maintains the state of the computation.
        internal_keep_prob = tf.Variable(
            keep_prob + decay_amount, dtype=tf.float32, trainable=False)
        min_keep_prob = tf.convert_to_tensor(
            min_keep_prob, name="min_keep_prob", dtype=tf.float32)

        # crate a tensor with num_updates value, to accumulte validation accuracies
        accumulator = tf.Variable(
            tf.zeros(
                [num_updates], dtype=tf.float32), trainable=False)
        position = tf.Variable(0, dtype=tf.int32, trainable=False)
        accumulated = tf.Variable(0, dtype=tf.float32, trainable=False)

        # convert num_updates to a tensor
        num_updates = tf.convert_to_tensor(
            num_updates, name="num_updates", dtype=tf.float32)

        validation_accuracy = tf.Variable(0.0)
        validation_accuracy = tf.assign(validation_accuracy,
                                        validation_accuracy_)

        with tf.control_dependencies([validation_accuracy]):
            # calculate right position in the accumulator vector
            # where we put the va value
            position = tf.assign(
                position, tf.cast(tf.mod(accumulated, num_updates), tf.int32))
            # update value
            accumulator = tf.scatter_update(accumulator, position,
                                            validation_accuracy)
            # update the amount of accumulated value of the whole train process
            accumulated = tf.assign_add(accumulated, 1)

            # get the denominator
            denominator = tf.cond(
                tf.greater_equal(accumulated, num_updates),
                lambda: num_updates, lambda: accumulated)

            # calculate cumulative rolling average
            rolling_avg = tf.reduce_sum(accumulator) / denominator
            # trigger value: 0 (nop) or 1 (trigger)
            trigger = tf.abs(
                (tf.ceil(validation_accuracy / rolling_avg) - 2.0))

            internal_keep_prob = tf.assign(
                internal_keep_prob,
                tf.maximum(min_keep_prob,
                           internal_keep_prob - decay_amount * trigger))
            return internal_keep_prob

def train():
    """Train model"""
    with tf.Graph().as_default(), tf.device('/gpu:0'):
        global_step = tf.Variable(0, trainable=False)

        # Get images and labels for CIFAR-10.
        images, labels = dataset.distorted_inputs(BATCH_SIZE)

        # Build a Graph that computes the logits predictions from the
        # inference model.
        keep_prob_, logits = vgg.get_model(images, train_phase=True)

        # Calculate loss.
        loss = vgg.loss(logits, labels)

        # Build a Graph that trains the model with one batch of examples and
        # updates the model parameters.
        train_op = vgg.train(loss, global_step)

        # Create a saver.
        saver = tf.train.Saver(tf.trainable_variables() + [global_step])

        # Train accuracy ops
        top_k_op = tf.nn.in_top_k(logits, labels, 1)
        train_accuracy = tf.reduce_mean(tf.cast(top_k_op, tf.float32))
        # General validation summary
        accuracy_value_ = tf.placeholder(tf.float32, shape=())
        accuracy_summary = tf.scalar_summary('accuracy', accuracy_value_)

        # Initialize decay_keep_prob op
        # va placeholder required for keep_prob_decay
        validation_accuracy_ = tf.placeholder(
            tf.float32, shape=(), name="validation_accuracy_")
        get_keep_prob = keep_prob_decay(
            validation_accuracy_,
            keep_prob=MAX_KEEP_PROB,
            min_keep_prob=0.4,
            num_updates=3,
            decay_amount=0.1)
        keep_prob_summary = tf.scalar_summary('keep_prob', get_keep_prob)

        # read collection after keep_prob_decay that adds
        # the keep_prob summary
        train_summaries = tf.merge_summary(
            tf.get_collection_ref('train_summaries'))

        # Build an initialization operation to run below.
        init = tf.initialize_all_variables()

        # Start running operations on the Graph.
        with tf.Session(config=tf.ConfigProto(
                allow_soft_placement=True)) as sess:
            sess.run(init)

            # Start the queue runners.
            tf.train.start_queue_runners(sess=sess)
            train_log = tf.train.SummaryWriter(LOG_DIR + "/train", sess.graph)
            validation_log = tf.train.SummaryWriter(LOG_DIR + "/validation",
                                                    sess.graph)

            # Extract previous global step value
            old_gs = sess.run(global_step)

            # set initial keep_prob
            keep_prob = MAX_KEEP_PROB

            # Restart from where we were
            for step in range(old_gs, MAX_STEPS):
                start_time = time.time()
                _, loss_value, summary_lines = sess.run(
                    [train_op, loss, train_summaries],
                    feed_dict={keep_prob_: keep_prob})
                duration = time.time() - start_time

                assert not np.isnan(
                    loss_value), 'Model diverged with loss = NaN'

                # update logs every 10 iterations
                if step % 10 == 0:
                    num_examples_per_step = BATCH_SIZE
                    examples_per_sec = num_examples_per_step / duration
                    sec_per_batch = float(duration)

                    format_str = ('{}: step {}, loss = {:.2f} '
                                  '({:.1f} examples/sec; {:.3f} sec/batch)')
                    print(
                        format_str.format(datetime.now(), step, loss_value,
                                          examples_per_sec, sec_per_batch))
                    # log train values
                    train_log.add_summary(summary_lines, global_step=step)

                # Save the model checkpoint at the end of every epoch
                # evaluate train and validation performance
                if (step > 0 and
                        step % STEP_PER_EPOCH == 0) or (step + 1) == MAX_STEPS:
                    checkpoint_path = os.path.join(LOG_DIR, 'model.ckpt')
                    saver.save(sess, checkpoint_path, global_step=step)

                    # validation accuracy
                    validation_accuracy_value = evaluate.get_validation_accuracy(
                        LOG_DIR)
                    summary_line = sess.run(accuracy_summary,
                                            feed_dict={
                                                accuracy_value_:
                                                validation_accuracy_value
                                            })
                    validation_log.add_summary(summary_line, global_step=step)

                    # update keep_prob using new validation accuracy
                    keep_prob, summary_line = sess.run(
                        [get_keep_prob, keep_prob_summary],
                        feed_dict={
                            validation_accuracy_: validation_accuracy_value
                        })

                    train_log.add_summary(summary_line, global_step=step)

                    # train accuracy
                    train_accuracy_value = sess.run(
                        train_accuracy, feed_dict={keep_prob_: 1.0})
                    summary_line = sess.run(
                        accuracy_summary,
                        feed_dict={accuracy_value_: train_accuracy_value})
                    train_log.add_summary(summary_line, global_step=step)

                    print(
                        '{}: train accuracy = {:.3f} validation accuracy = {:.3f}'.
                        format(datetime.now(), train_accuracy_value,
                               validation_accuracy_value))

def main():
    """main function"""
    dataset.maybe_download_and_extract()
    if tf.gfile.Exists(LOG_DIR):
        tf.gfile.DeleteRecursively(LOG_DIR)
    tf.gfile.MakeDirs(LOG_DIR)
    train()
    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>train_decaying_keep_prob.py: reset accumulator every time keep_prob changes<commit_after>#Copyright (C) 2016 Paolo Galeone <nessuno@nerdz.eu>
# Based on Tensorflow cifar10_train.py file
# https://github.com/tensorflow/tensorflow/blob/r0.11/tensorflow/models/image/cifar10/cifar10_train.py
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
""" Train model with a single GPU. Evaluate it on the second one"""

import sys
from datetime import datetime
import os.path
import time
import math

import numpy as np
import tensorflow as tf
from models import model2 as vgg
from inputs import cifar10 as dataset
import evaluate

BATCH_SIZE = 128
STEP_PER_EPOCH = math.ceil(dataset.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
                           BATCH_SIZE)
MAX_EPOCH = 300
MAX_STEPS = STEP_PER_EPOCH * MAX_EPOCH

CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
LOG_DIR = CURRENT_DIR + "/log/" + vgg.NAME + '/keep_prob_decay'

MAX_KEEP_PROB = 1.0

def keep_prob_decay(validation_accuracy_,
                    keep_prob,
                    min_keep_prob,
                    num_updates,
                    decay_amount,
                    name=None):
    """ Decay keep_prob until it reaches min_keep_pro. Computation
    based on validation_accuracy_ variations
    """

    with tf.name_scope(name, "KeepProbDecay", [
            validation_accuracy_, keep_prob, min_keep_prob, num_updates,
            decay_amount
    ]) as name:
        validation_accuracy_ = tf.convert_to_tensor(
            validation_accuracy_,
            name="validation_accuracy_",
            dtype=tf.float32)
        decay_amount = tf.convert_to_tensor(
            decay_amount, name="decay_amount", dtype=tf.float32)
        keep_prob = tf.convert_to_tensor(
            keep_prob, name="keep_prob", dtype=tf.float32)
        # initialize keep_prob with keep_prob + decay_amount
        # to handle the case of the first decay
        # that always happen because of the constant ratio
        # of va/rolling_avg = va/va
        # Maintains the state of the computation.
        internal_keep_prob = tf.Variable(
            keep_prob + decay_amount, dtype=tf.float32, trainable=False)
        min_keep_prob = tf.convert_to_tensor(
            min_keep_prob, name="min_keep_prob", dtype=tf.float32)

        # crate a tensor with num_updates value, to accumulte validation accuracies
        accumulator = tf.Variable(
            tf.zeros(
                [num_updates], dtype=tf.float32), trainable=False)
        position = tf.Variable(0, dtype=tf.int32, trainable=False)
        accumulated = tf.Variable(0, dtype=tf.int32, trainable=False)

        # create a separate variable for num_updates tensor
        # that's not a python variabile
        num_updates_tensor = tf.convert_to_tensor(
            num_updates, name="num_updates", dtype=tf.int32)

        validation_accuracy = tf.Variable(0.0)
        validation_accuracy = tf.assign(validation_accuracy,
                                        validation_accuracy_)

        with tf.control_dependencies([validation_accuracy]):
            # calculate right position in the accumulator vector
            # where we put the va value
            position_op = tf.cast(
                tf.mod(accumulated, num_updates_tensor), tf.int32)
            position = tf.assign(position, position_op)
            # update value
            accumulator_op = tf.scatter_update(accumulator, position,
                                               validation_accuracy)
            accumulator = accumulator_op
            # update the amount of accumulated value of the whole train process
            accumulated_op = tf.assign_add(accumulated, 1)
            accumulated = accumulated_op

            # get the denominator
            denominator = tf.cast(
                tf.cond(
                    tf.greater_equal(accumulated, num_updates_tensor),
                    lambda: num_updates_tensor, lambda: accumulated),
                tf.float32)

            # calculate cumulative rolling average
            rolling_avg = tf.reduce_sum(accumulator) / denominator
            # trigger value: 0 (nop) or 1 (trigger)
            trigger = tf.abs(
                (tf.ceil(validation_accuracy / rolling_avg) - 2.0))

            with tf.control_dependencies([trigger]):
                # if triggered, reset
                position = tf.cond(
                    tf.equal(trigger, 1), lambda: tf.assign(position, 0),
                    lambda: position_op)

                def reset_accumulator():
                    """set past validation accuracies to 0 and place actual validation accuracy in position 0
                    of the accumulator"""
                    return tf.scatter_update(
                        accumulator, [i for i in range(num_updates)],
                        [validation_accuracy] +
                        [0.0 for i in range(1, num_updates)])

                accumulator = tf.cond(
                    tf.equal(trigger, 1), reset_accumulator,
                    lambda: accumulator_op)

                accumulated = tf.cond(
                    tf.equal(trigger, 1), lambda: tf.assign(accumulated, 1),
                    lambda: accumulated_op)

            # status variable
            internal_keep_prob = tf.assign(
                internal_keep_prob,
                tf.maximum(min_keep_prob,
                           internal_keep_prob - decay_amount * trigger))
            return internal_keep_prob

def train():
    """Train model"""
    with tf.Graph().as_default(), tf.device('/gpu:0'):
        global_step = tf.Variable(0, trainable=False)

        # Get images and labels for CIFAR-10.
        images, labels = dataset.distorted_inputs(BATCH_SIZE)

        # Build a Graph that computes the logits predictions from the
        # inference model.
        keep_prob_, logits = vgg.get_model(images, train_phase=True)

        # Calculate loss.
        loss = vgg.loss(logits, labels)

        # Build a Graph that trains the model with one batch of examples and
        # updates the model parameters.
        train_op = vgg.train(loss, global_step)

        # Create a saver.
        saver = tf.train.Saver(tf.trainable_variables() + [global_step])

        # Train accuracy ops
        top_k_op = tf.nn.in_top_k(logits, labels, 1)
        train_accuracy = tf.reduce_mean(tf.cast(top_k_op, tf.float32))
        # General validation summary
        accuracy_value_ = tf.placeholder(tf.float32, shape=())
        accuracy_summary = tf.scalar_summary('accuracy', accuracy_value_)

        # Initialize decay_keep_prob op
        # va placeholder required for keep_prob_decay
        validation_accuracy_ = tf.placeholder(
            tf.float32, shape=(), name="validation_accuracy_")
        get_keep_prob = keep_prob_decay(
            validation_accuracy_,
            keep_prob=MAX_KEEP_PROB,
            min_keep_prob=0.4,
            num_updates=10,
            decay_amount=0.1)
        keep_prob_summary = tf.scalar_summary('keep_prob', get_keep_prob)

        # read collection after keep_prob_decay that adds
        # the keep_prob summary
        train_summaries = tf.merge_summary(
            tf.get_collection_ref('train_summaries'))

        # Build an initialization operation to run below.
        init = tf.initialize_all_variables()

        # Start running operations on the Graph.
        with tf.Session(config=tf.ConfigProto(
                allow_soft_placement=True)) as sess:
            sess.run(init)

            # Start the queue runners.
            tf.train.start_queue_runners(sess=sess)
            train_log = tf.train.SummaryWriter(LOG_DIR + "/train", sess.graph)
            validation_log = tf.train.SummaryWriter(LOG_DIR + "/validation",
                                                    sess.graph)

            # Extract previous global step value
            old_gs = sess.run(global_step)

            # set initial keep_prob
            keep_prob = MAX_KEEP_PROB

            # Restart from where we were
            for step in range(old_gs, MAX_STEPS):
                start_time = time.time()
                _, loss_value, summary_lines = sess.run(
                    [train_op, loss, train_summaries],
                    feed_dict={keep_prob_: keep_prob})
                duration = time.time() - start_time

                assert not np.isnan(
                    loss_value), 'Model diverged with loss = NaN'

                # update logs every 10 iterations
                if step % 10 == 0:
                    num_examples_per_step = BATCH_SIZE
                    examples_per_sec = num_examples_per_step / duration
                    sec_per_batch = float(duration)

                    format_str = ('{}: step {}, loss = {:.2f} '
                                  '({:.1f} examples/sec; {:.3f} sec/batch)')
                    print(
                        format_str.format(datetime.now(), step, loss_value,
                                          examples_per_sec, sec_per_batch))
                    # log train values
                    train_log.add_summary(summary_lines, global_step=step)

                # Save the model checkpoint at the end of every epoch
                # evaluate train and validation performance
                if (step > 0 and
                        step % STEP_PER_EPOCH == 0) or (step + 1) == MAX_STEPS:
                    checkpoint_path = os.path.join(LOG_DIR, 'model.ckpt')
                    saver.save(sess, checkpoint_path, global_step=step)

                    # validation accuracy
                    validation_accuracy_value = evaluate.get_validation_accuracy(
                        LOG_DIR)
                    summary_line = sess.run(accuracy_summary,
                                            feed_dict={
                                                accuracy_value_:
                                                validation_accuracy_value
                                            })
                    validation_log.add_summary(summary_line, global_step=step)

                    # update keep_prob using new validation accuracy
                    keep_prob, summary_line = sess.run(
                        [get_keep_prob, keep_prob_summary],
                        feed_dict={
                            validation_accuracy_: validation_accuracy_value
                        })
                    train_log.add_summary(summary_line, global_step=step)

                    # train accuracy
                    train_accuracy_value = sess.run(
                        train_accuracy, feed_dict={keep_prob_: 1.0})
                    summary_line = sess.run(
                        accuracy_summary,
                        feed_dict={accuracy_value_: train_accuracy_value})
                    train_log.add_summary(summary_line, global_step=step)

                    print(
                        '{}: train accuracy = {:.3f} validation accuracy = {:.3f}'.
                        format(datetime.now(), train_accuracy_value,
                               validation_accuracy_value))

def main():
    """main function"""
    dataset.maybe_download_and_extract()
    if tf.gfile.Exists(LOG_DIR):
        tf.gfile.DeleteRecursively(LOG_DIR)
    tf.gfile.MakeDirs(LOG_DIR)
    train()
    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>"""
.. See the NOTICE file distributed with this work for additional information
   regarding copyright ownership.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""
from __future__ import print_function

import sys

try:
    if hasattr(sys, '_run_from_cmdl') is True:
        raise ImportError
    from pycompss.api.parameter import FILE_IN, FILE_OUT, FILE_INOUT, IN
    from pycompss.api.task import task
    from pycompss.api.api import compss_wait_on
    # from pycompss.api.constraint import constraint
except ImportError:
    print("[Warning] Cannot import \"pycompss\" API packages.")
    print("          Using mock decorators.")

    from utils.dummy_pycompss import FILE_IN, FILE_OUT, FILE_INOUT, IN
    from utils.dummy_pycompss import task
    from utils.dummy_pycompss import compss_wait_on
    # from utils.dummy_pycompss import constraint

from basic_modules.tool import Tool

from pytadbit import Chromosome
from pytadbit import load_hic_data_from_reads  # pylint: disable=import-error

# ------------------------------------------------------------------------------

class tbGenerateTADsTool(Tool):
    """
    Tool for taking the adjacency lists and predicting TADs
    """

    def __init__(self):
        """
        Init function
        """
        print("TADbit - Generate TADs")
        Tool.__init__(self)

    @task(adj_list=FILE_IN, resolution=IN, normalized=IN, returns=list)
    def tb_hic_chr(self, adj_list, resolution):  # pylint: disable=no-self-use
        """
        Get the list of chromosomes in the adjacency list
        """
        print("TB LOADED HIC MATRIX")
        hic_data = load_hic_data_from_reads(adj_list, resolution=int(resolution))

        print("TB LOADED HIC MATRIX")

        return hic_data.chromosomes.keys()

    @task(expt_name=IN, adj_list=FILE_IN, chrom=IN, resolution=IN, normalized=IN, tad_file=FILE_OUT)
    # @constraint(ProcessorCoreCount=16)
    def tb_generate_tads(self, expt_name, adj_list, chrom, resolution, normalized, tad_file):
        """
        Function to the predict TAD sites for a given resolution from the Hi-C
        matrix

        Parameters
        ----------
        expt_name : str
                Location of the adjacency list
        matrix_file : str
            Location of the HDF5 output matrix file
        resolution : int
            Resolution to read the Hi-C adjacency list at
        tad_file : str
            Location of the output TAD file

        Returns
        -------
        tad_file : str
            Location of the output TAD file

        """
        # chr_hic_data = read_matrix(matrix_file, resolution=int(resolution))

        print("TB TAD GENERATOR:", expt_name, adj_list, chrom, resolution, normalized, tad_file)

        hic_data = load_hic_data_from_reads(adj_list, resolution=int(resolution))

        if normalized is False:
            hic_data.normalize_hic(iterations=9, max_dev=0.1)

        save_matrix_file = adj_list + "_" + str(chrom) + "_tmp.txt"
        hic_data.write_matrix(save_matrix_file, (chrom, chrom), normalized=True)

        chr_hic_data = hic_data.get_matrix((chrom, chrom))
        print("TB - chr_hic_data:", chr_hic_data)

        my_chrom = Chromosome(name=chrom, centromere_search=True)
        my_chrom.add_experiment(expt_name, hic_data=save_matrix_file, resolution=int(resolution))

        # Run core TADbit function to find TADs on each expt.
        my_chrom.find_tad(expt_name, n_cpus=15)

        exp = my_chrom.experiments[expt_name]
        exp.write_tad_borders(savedata=tad_file + ".tmp")

        with open(tad_file, "wb") as f_out:
            with open(tad_file + ".tmp", "rb") as f_in:
                f_out.write(f_in.read())

        return True

    @task(input_file=FILE_IN, chrom=IN, resolution=IN, output_file=FILE_INOUT)
    def tb_merge_tad_files(self, input_file, chrom, resolution, output_file):
        """
        Merge 2 TAD adjacnecny list files
        """
        with open(output_file, 'a') as f_out:
            with open(input_file, 'r') as f_in:
                for line in f_in:
                    line = line.split("\t")
                    line[-1] = line[-1].rstrip()
                    f_out.write(
                        str(chrom) + "\t" + line[1] + "\t" + line[2] + "\tTADs_" + str(resolution) + "\t" + line[3] + "\t.\n")  # pylint: disable=line-too-long

        return True

    def run(self, input_files, output_files, metadata=None):  # pylint: disable=arguments-differ
        """
        The main function to the predict TAD sites for a given resolution from
        the Hi-C matrix

        Parameters
        ----------
        input_files : list
            adj_list : str
                Location of the adjacency list
        metadata : dict
            resolutions : list
                Levels of resolution for the adjacency list to be daved at
            assembly : str
                Assembly of the aligned sequences

        Returns
        -------
        output_files : list
            List of locations for the output files.
        output_metadata : list
            List of matching metadata dict objects

        """

        adj_list = input_files[0]

        resolutions = [1000000]
        if 'resolutions' in metadata:
            resolutions = metadata['resolutions']

        normalized = False
        if 'normalized' in metadata:
            normalized = metadata['normalized']

        # input and output share most metadata
        output_metadata = {}

        root_name = adj_list.split("/")

        tad_files = {}

        results = []
        for resolution in resolutions:
            print("TB LOADING Hi-C:", adj_list, resolution, normalized)
            hic_data_chr = self.tb_hic_chr(adj_list, resolution)
            hic_data_chr = compss_wait_on(hic_data_chr)
            print("TB LOADED Hi-C!")

            print("TB CHROMOSOMES", hic_data_chr)

            tad_files[resolution] = {}

            for chrom in hic_data_chr:
                save_tad_file = "/".join(root_name[0:-1]) + '/' + metadata['expt_name'] + '_tad_' + chrom + '_' + str(resolution) + '.tsv'  # pylint: disable=line-too-long
                tad_files[resolution][chrom] = save_tad_file

                expt_name = metadata['expt_name'] + '_tad_' + chrom + '_' + str(resolution)

                print("TB Generate TADS:", resolution, normalized)
                results.append(
                    self.tb_generate_tads(
                        expt_name, adj_list, chrom, resolution, normalized,
                        save_tad_file
                    )
                )

        #results = compss_wait_on(results)

        # Step to merge all the TAD files into a single bed file
        tad_bed_file = "/".join(root_name[0:-1]) + '/' + metadata['expt_name'] + '_tads.tsv'

        print("TB tad_files:", tad_files)

        # Step to merge all the TAD files into a single bed file
        tad_bed_file = "/".join(root_name[0:-1]) + '/' + metadata['expt_name'] + '_tads.tsv'
        f_prepare = open(tad_bed_file, 'w')
        f_prepare.close()

        for resolution in tad_files:
            for chrom in tad_files[resolution]:
                results = self.tb_merge_tad_files(
                    tad_files[resolution][chrom],
                    chrom,
                    resolution,
                    tad_bed_file
                )
                # results = compss_wait_on(results) # This one avoids all parallelism

        return ([tad_bed_file], output_metadata)

# ------------------------------------------------------------------------------
<commit_msg>Ignores for the import of the TADbit code which has been disabled for the master branch<commit_after>"""
.. See the NOTICE file distributed with this work for additional information
   regarding copyright ownership.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""
from __future__ import print_function

import sys

try:
    if hasattr(sys, '_run_from_cmdl') is True:
        raise ImportError
    from pycompss.api.parameter import FILE_IN, FILE_OUT, FILE_INOUT, IN
    from pycompss.api.task import task
    from pycompss.api.api import compss_wait_on
    # from pycompss.api.constraint import constraint
except ImportError:
    print("[Warning] Cannot import \"pycompss\" API packages.")
    print("          Using mock decorators.")

    from utils.dummy_pycompss import FILE_IN, FILE_OUT, FILE_INOUT, IN
    from utils.dummy_pycompss import task
    from utils.dummy_pycompss import compss_wait_on
    # from utils.dummy_pycompss import constraint

from basic_modules.tool import Tool

from pytadbit import Chromosome  # pylint: disable=import-error
from pytadbit import load_hic_data_from_reads  # pylint: disable=import-error

# ------------------------------------------------------------------------------

class tbGenerateTADsTool(Tool):
    """
    Tool for taking the adjacency lists and predicting TADs
    """

    def __init__(self):
        """
        Init function
        """
        print("TADbit - Generate TADs")
        Tool.__init__(self)

    @task(adj_list=FILE_IN, resolution=IN, normalized=IN, returns=list)
    def tb_hic_chr(self, adj_list, resolution):  # pylint: disable=no-self-use
        """
        Get the list of chromosomes in the adjacency list
        """
        print("TB LOADED HIC MATRIX")
        hic_data = load_hic_data_from_reads(adj_list, resolution=int(resolution))

        print("TB LOADED HIC MATRIX")

        return hic_data.chromosomes.keys()

    @task(expt_name=IN, adj_list=FILE_IN, chrom=IN, resolution=IN, normalized=IN, tad_file=FILE_OUT)
    # @constraint(ProcessorCoreCount=16)
    def tb_generate_tads(self, expt_name, adj_list, chrom, resolution, normalized, tad_file):
        """
        Function to the predict TAD sites for a given resolution from the Hi-C
        matrix

        Parameters
        ----------
        expt_name : str
                Location of the adjacency list
        matrix_file : str
            Location of the HDF5 output matrix file
        resolution : int
            Resolution to read the Hi-C adjacency list at
        tad_file : str
            Location of the output TAD file

        Returns
        -------
        tad_file : str
            Location of the output TAD file

        """
        # chr_hic_data = read_matrix(matrix_file, resolution=int(resolution))

        print("TB TAD GENERATOR:", expt_name, adj_list, chrom, resolution, normalized, tad_file)

        hic_data = load_hic_data_from_reads(adj_list, resolution=int(resolution))

        if normalized is False:
            hic_data.normalize_hic(iterations=9, max_dev=0.1)

        save_matrix_file = adj_list + "_" + str(chrom) + "_tmp.txt"
        hic_data.write_matrix(save_matrix_file, (chrom, chrom), normalized=True)

        chr_hic_data = hic_data.get_matrix((chrom, chrom))
        print("TB - chr_hic_data:", chr_hic_data)

        my_chrom = Chromosome(name=chrom, centromere_search=True)
        my_chrom.add_experiment(expt_name, hic_data=save_matrix_file, resolution=int(resolution))

        # Run core TADbit function to find TADs on each expt.
        my_chrom.find_tad(expt_name, n_cpus=15)

        exp = my_chrom.experiments[expt_name]
        exp.write_tad_borders(savedata=tad_file + ".tmp")

        with open(tad_file, "wb") as f_out:
            with open(tad_file + ".tmp", "rb") as f_in:
                f_out.write(f_in.read())

        return True

    @task(input_file=FILE_IN, chrom=IN, resolution=IN, output_file=FILE_INOUT)
    def tb_merge_tad_files(self, input_file, chrom, resolution, output_file):
        """
        Merge 2 TAD adjacnecny list files
        """
        with open(output_file, 'a') as f_out:
            with open(input_file, 'r') as f_in:
                for line in f_in:
                    line = line.split("\t")
                    line[-1] = line[-1].rstrip()
                    f_out.write(
                        str(chrom) + "\t" + line[1] + "\t" + line[2] + "\tTADs_" + str(resolution) + "\t" + line[3] + "\t.\n")  # pylint: disable=line-too-long

        return True

    def run(self, input_files, output_files, metadata=None):  # pylint: disable=arguments-differ
        """
        The main function to the predict TAD sites for a given resolution from
        the Hi-C matrix

        Parameters
        ----------
        input_files : list
            adj_list : str
                Location of the adjacency list
        metadata : dict
            resolutions : list
                Levels of resolution for the adjacency list to be daved at
            assembly : str
                Assembly of the aligned sequences

        Returns
        -------
        output_files : list
            List of locations for the output files.
        output_metadata : list
            List of matching metadata dict objects

        """

        adj_list = input_files[0]

        resolutions = [1000000]
        if 'resolutions' in metadata:
            resolutions = metadata['resolutions']

        normalized = False
        if 'normalized' in metadata:
            normalized = metadata['normalized']

        # input and output share most metadata
        output_metadata = {}

        root_name = adj_list.split("/")

        tad_files = {}

        results = []
        for resolution in resolutions:
            print("TB LOADING Hi-C:", adj_list, resolution, normalized)
            hic_data_chr = self.tb_hic_chr(adj_list, resolution)
            hic_data_chr = compss_wait_on(hic_data_chr)
            print("TB LOADED Hi-C!")

            print("TB CHROMOSOMES", hic_data_chr)

            tad_files[resolution] = {}

            for chrom in hic_data_chr:
                save_tad_file = "/".join(root_name[0:-1]) + '/' + metadata['expt_name'] + '_tad_' + chrom + '_' + str(resolution) + '.tsv'  # pylint: disable=line-too-long
                tad_files[resolution][chrom] = save_tad_file

                expt_name = metadata['expt_name'] + '_tad_' + chrom + '_' + str(resolution)

                print("TB Generate TADS:", resolution, normalized)
                results.append(
                    self.tb_generate_tads(
                        expt_name, adj_list, chrom, resolution, normalized,
                        save_tad_file
                    )
                )

        #results = compss_wait_on(results)

        # Step to merge all the TAD files into a single bed file
        tad_bed_file = "/".join(root_name[0:-1]) + '/' + metadata['expt_name'] + '_tads.tsv'

        print("TB tad_files:", tad_files)

        # Step to merge all the TAD files into a single bed file
        tad_bed_file = "/".join(root_name[0:-1]) + '/' + metadata['expt_name'] + '_tads.tsv'
        f_prepare = open(tad_bed_file, 'w')
        f_prepare.close()

        for resolution in tad_files:
            for chrom in tad_files[resolution]:
                results = self.tb_merge_tad_files(
                    tad_files[resolution][chrom],
                    chrom,
                    resolution,
                    tad_bed_file
                )
                # results = compss_wait_on(results) # This one avoids all parallelism

        return ([tad_bed_file], output_metadata)

# ------------------------------------------------------------------------------
<|endoftext|>


<commit_before>from django.contrib.auth.models import User, Group
from rest_framework import serializers

from clothing.models import ClothingTime, Clothing, BodyPart

class ClothingTimeSerializer(serializers.ModelSerializer):
    class Meta:
        model = ClothingTime
        fields = ('id', 'name', 'clothes', 'slug', 'image', )

class ClothingSerializer(serializers.ModelSerializer):
    class Meta:
        model = Clothing
        fields = ('id', 'name', 'slug', 'image', )

class BodyPartSerializer(serializers.ModelSerializer):
    class Meta:
        model = BodyPart
        fields = ('id', 'name', )

class UserSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = User
        fields = ('url', 'username', 'email', 'groups')

class GroupSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Group
        fields = ('url', 'name',)<commit_msg>Added bodyparts to clothing serialization<commit_after>from django.contrib.auth.models import User, Group
from rest_framework import serializers

from clothing.models import ClothingTime, Clothing, BodyPart

class ClothingTimeSerializer(serializers.ModelSerializer):
    class Meta:
        model = ClothingTime
        fields = ('id', 'name', 'clothes', 'slug', 'image', )

class ClothingSerializer(serializers.ModelSerializer):
    class Meta:
        model = Clothing
        fields = ('id', 'name', 'body_parts', 'slug', 'image', )

class BodyPartSerializer(serializers.ModelSerializer):
    class Meta:
        model = BodyPart
        fields = ('id', 'name', )

class UserSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = User
        fields = ('url', 'username', 'email', 'groups')

class GroupSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Group
        fields = ('url', 'name',)<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import with_statement
import sys
import hashlib
from saxtree import HTML, HEAD, META, BODY, P, Text, TABLE, TR, TD, LINK, SCRIPT, DIV, A, IMG, EM, BR, NAV
import magic
import time
import saxtree
import StringIO
import urllib
import os
import stat
import urlparse

def pathjoin(a, b):
    return a + b
def copy(source, destination):
	while True:
		b = source.read(8192)
		if b == "":
			break
		destination.write(b)
def loadStrings(name):
    with open(name, "r") as f:
        for line in f.xreadlines():
            yield line.rstrip("\n")

class Template(object):
    def __init__(self, generator):
        self.generator = generator
    def __enter__(self):
        if self.generator is not None:
          self.generator.next()
    def __exit__(self, ty, exc, tb):
        try:
            if self.generator is not None:
              self.generator.next()
        except StopIteration:
            pass
def serveSimpleResponse(req, statusCode, statusText, headers):
    """ serveSimpleResponse("200", "OK", [("Content-Type", "text/html; charset=utf-8")]) """
    startResponse(req.out, statusCode, statusText, headers)
    req.out.flush()

def servePage(req, statusCode, statusText, outHeaders, onload=None):
    """ serves a HTML page without content. If you want to add content, be sure to call using 'with Template(servePage(...)):'"""
    headers = [(k, v) for k, v in outHeaders if k.lower() != "Content-Type"]
    headers.append(("Content-Type", "text/html; charset=utf-8"))
    startResponse(req.out, statusCode, statusText, headers)
    saxtree.init(req.out)
    with HTML(xmlns="http://www.w3.org/1999/xhtml"):
        with HEAD():
            with META(**{"http-equiv": "Content-Type", "content": "application/xhtml+xml; charset=UTF-8"}): # IE
                pass
            with META(name="MSSmartTagsPreventParsing", content="TRUE"):
                pass
            with LINK(rel="openid.server", href="http://www.livejournal.com/"):
                pass
            with LINK(rel="openid.delegate", href="http://dannymi.livejournal.com/"):
                pass
            with LINK(rel="stylesheet", href="/common/scratchpost.css", type="text/css"):
                pass
            with LINK(rel="shortcut icon", href="/common/icon/scratchpost.PNG", type="image/png"):
                pass
            with LINK(rel="icon", href="/common/icon/scratchpost.PNG", type="image/png"):
                pass
            with LINK(rel="up", href="../"):
                pass
            with SCRIPT(type="text/javascript", src="/common/scratchpost.js"):
                with Text(" "):  # Opera workaround
                    pass
        with BODY(**({"onload": onload} if onload is not None else {})):
            with Template(navigationB(req)) as realBody:
                yield realBody
def urlFromFilename(name):
    return urllib.quote(name)

HTTP_DATE_FMT = "%a, %d %b %Y %H:%M:%S GMT"
def formatLastModified(mtime):
    humanTime = time.gmtime(mtime)
    return(time.strftime(HTTP_DATE_FMT, humanTime))
def startResponse(out, statusCode, statusText, headers):
    statusCode = int(statusCode)
    headers = headers[:]
    headers.append(("Date", formatLastModified(time.time())))
    headers.append(("Status", "%s %s" % (statusCode, statusText)))
    for k, v in headers:
       out.write("%s: %s\r\n" % (k, v)) # TODO escape
    out.write("\r\n")
def renderBinaryFile(f, out, bWriteBody):
    """ renders the file f, if it is binary.
        returns: whether the file f was rendered or not. """
    stat1 = os.fstat(f.fileno())
    regularFileFlag = stat.S_ISREG(stat1.st_mode)
    if regularFileFlag:
        mimeType = magic.getRegularFileType(f)
        if mimeType != "application/x-httpd-php": # or True:
                f.seek(0, 2)
                l = f.tell()
                f.seek(0)
                startResponse(out, 200, "OK", [("Content-Type", mimeType), ("Content-Length", l), ("Last-Modified", formatLastModified(stat1.st_mtime))])
                if bWriteBody:
                    copy(f, out)
                return(True)
    return(False)
# see <http://developer.yahoo.com/performance/rules.html#etags>

def printPathBar(dirName):
    io = StringIO.StringIO()
    parts = dirName.split("/")
    with P(): # path
        for part in parts:
            if part == "":
                continue
            with Text("/"):
                pass
            io.write("/")
            io.write(part)
            with A(href=urlFromFilename(io.getvalue())):
                with Text(part):
                    pass
def getEntries(containerName):
    try:
        webhidden = set(loadStrings("%s/.webhidden" % (containerName, )))
    except:
        webhidden = set()
    webhidden.add("index") # TODO except when we aren't it?
    for name in os.listdir(containerName):
        if name.startswith(".") or name in webhidden:
            continue
        title = name
        bmtime = 0
        try:
            bname = os.path.join(containerName, name)
            st = os.stat(bname)
            bmtime = st.st_mtime
            if stat.S_ISDIR(st.st_mode):
                title = "%s/" % (name, )
        except OSError:
            pass
        yield (title, name, bmtime)
def getEntries2(containerName):
    #yield from getEntries(containerName)
    for entry in sorted(getEntries(containerName)):
        yield entry
    try:
        webseealso = loadStrings("%s/.webseealso" % (containerName, ))
        webseealso = map(lambda line: line.split(" ", 1), webseealso) # URL, description
    except:
        webseealso = []
    for URL, description in webseealso:
        yield ("See also: %s" % (description, ), URL, None)
def getNewestSubpage(base): # -> (title, path, mtime)
    #st = os.stat(base)
    #mtime = st.st_mtime
    title = None
    name = None
    mtime = 0
    for xtitle, yname, xmtime in getEntries(base):
        xname = os.path.join(base, yname)
        if xtitle.endswith("/") and not xtitle.startswith("."):
            btitle, bname, bmtime = getNewestSubpage(xname)
            if bmtime > xmtime:
                xname = bname
                xmtime = bmtime
                xtitle = btitle
        if xmtime > mtime:
            name = xname
            mtime = xmtime
            title = xtitle
    return title, name, mtime
def md5hexdigestText(text):
    m = hashlib.md5()
    m.update(text)
    return(m.hexdigest())
def gallery(folderName, req):
    DOCUMENT_ROOT = req.env.get("DOCUMENT_ROOT", "R")
    ifolderName = "%s/image" % (folderName, )
    idirName = pathjoin(DOCUMENT_ROOT, ifolderName)
    with P():
       with Text(ifolderName):
          pass
    for title, href, mtime in sorted(getEntries(idirName)):
        #with A(href=href):
        #with DIV(**{"id": "anchor_%s" % (md5hexdigestText(href), ), "class": "image_of_the_day_big_image"}):
        with IMG(**{"alt": title, "src": urlFromFilename("%s/%s" % (ifolderName, href)), "class": "page"}):
            pass
    with BR(clear="both") as br:
        yield br
def navigationB(req):
    from common import saxdirtree
    DOCUMENT_ROOT = req.env.get("DOCUMENT_ROOT", "R")
    #PATH_INFO = req.env.get("REQUEST_URI", ".").split("?")[0] # PATH_INFO"]
    #PATH_INFO = urllib.unquote(PATH_INFO)
    PATH_INFO = req.env.get("PATH_INFO", "/.").split("?")[0] # PATH_INFO"]
    folderName = PATH_INFO # FIXME
    dirName = pathjoin(DOCUMENT_ROOT, folderName)
    with TABLE(**{"border": "0", "class": "main_navigation_area"}):
        with TR():
            with TD():
                with NAV(**{"class": "main_navigation", "id": "main_navigation"}):
                    printPathBar(PATH_INFO)
                    try:
                        saxdirtree.printList({"class": "main_directory"}, getEntries2(dirName))
                    except OSError:
                        pass
            with TD(**{"class": "main"}):
                # TODO H1
                with DIV(**{"class": "main_border", "id": "main_border"}):
                    with DIV(**{"class": "main_content"}) as main_content:
                        if os.path.exists(dirName + "/image") and (not os.path.exists(dirName + "/image/equation") or os.stat(dirName + "/image").st_nlink > 3):
                            with Template(gallery(folderName, req)):
                                pass
                        # everything else.
                        yield main_content
def modificationB(req, name):
    DOCUMENT_ROOT = req.env.get("DOCUMENT_ROOT", "R")
    dname = os.path.dirname(name)
    title, name, mtime = getNewestSubpage(dname)
    if name and name.startswith(DOCUMENT_ROOT):
        name = name[len(DOCUMENT_ROOT):]
        with P() as result:
            with Text("Last Modification in "):
                pass
            with A(href=urlFromFilename(name)):
                with Text(name):
                    pass
            with Text(" on "):
                pass
            humanReadableTime = time.asctime(time.gmtime(mtime))
            with Text(humanReadableTime):
                pass
            with Text("."):
                pass
            yield(result)
def serveDefaultPage(name, req, outHeaders, onload=None):
    with Template(servePage(req, 200, "OK", outHeaders, onload=onload)) as t:
        yield t
        with Template(modificationB(req, name)) as t2:
            pass
        # TODO webauthor

"""
urlparse.urlunparse
>>> urllib.urlencode({"a": "b", "c": "d"})
'a=b&c=d'
>>> urlparse.urlunparse(("http", "a", "x", ("a", "b"), "q", "frag"))
"http://a/x;('a', 'b')?q#frag"
>>> urlparse.urlunparse(("http", "a", "x", None, "X", "frag"))
'http://a/x?X#frag'
"""

def makeURL(scheme, netloc, url, params = None, query = None, fragment = None):
	"""
	>>> makeURL("http", "a", "x", None, {"a": "b", "c": "d"}, "frag")
	'http://a/x?a=b&c=d#frag'
	"""
	squery = urllib.urlencode(query)
	return urlparse.urlunparse((scheme, netloc, url, params, squery, fragment))

if __name__ == "__main__":
	makeURL("https", "api.dreamhost.com", "/", None, {
		"key": "xxx",
		"cmd": "a",
		"format": "json",
		"unique_id": hash("DNS_UPDATE"),
	}, None)
<commit_msg>make ".webseealso" tab-separated.<commit_after>#!/usr/bin/env python

from __future__ import with_statement
import sys
import hashlib
from saxtree import HTML, HEAD, META, BODY, P, Text, TABLE, TR, TD, LINK, SCRIPT, DIV, A, IMG, EM, BR, NAV
import magic
import time
import saxtree
import StringIO
import urllib
import os
import stat
import urlparse

def pathjoin(a, b):
    return a + b
def copy(source, destination):
	while True:
		b = source.read(8192)
		if b == "":
			break
		destination.write(b)
def loadStrings(name):
    with open(name, "r") as f:
        for line in f.xreadlines():
            yield line.rstrip("\n")

class Template(object):
    def __init__(self, generator):
        self.generator = generator
    def __enter__(self):
        if self.generator is not None:
          self.generator.next()
    def __exit__(self, ty, exc, tb):
        try:
            if self.generator is not None:
              self.generator.next()
        except StopIteration:
            pass
def serveSimpleResponse(req, statusCode, statusText, headers):
    """ serveSimpleResponse("200", "OK", [("Content-Type", "text/html; charset=utf-8")]) """
    startResponse(req.out, statusCode, statusText, headers)
    req.out.flush()

def servePage(req, statusCode, statusText, outHeaders, onload=None):
    """ serves a HTML page without content. If you want to add content, be sure to call using 'with Template(servePage(...)):'"""
    headers = [(k, v) for k, v in outHeaders if k.lower() != "Content-Type"]
    headers.append(("Content-Type", "text/html; charset=utf-8"))
    startResponse(req.out, statusCode, statusText, headers)
    saxtree.init(req.out)
    with HTML(xmlns="http://www.w3.org/1999/xhtml"):
        with HEAD():
            with META(**{"http-equiv": "Content-Type", "content": "application/xhtml+xml; charset=UTF-8"}): # IE
                pass
            with META(name="MSSmartTagsPreventParsing", content="TRUE"):
                pass
            with LINK(rel="openid.server", href="http://www.livejournal.com/"):
                pass
            with LINK(rel="openid.delegate", href="http://dannymi.livejournal.com/"):
                pass
            with LINK(rel="stylesheet", href="/common/scratchpost.css", type="text/css"):
                pass
            with LINK(rel="shortcut icon", href="/common/icon/scratchpost.PNG", type="image/png"):
                pass
            with LINK(rel="icon", href="/common/icon/scratchpost.PNG", type="image/png"):
                pass
            with LINK(rel="up", href="../"):
                pass
            with SCRIPT(type="text/javascript", src="/common/scratchpost.js"):
                with Text(" "):  # Opera workaround
                    pass
        with BODY(**({"onload": onload} if onload is not None else {})):
            with Template(navigationB(req)) as realBody:
                yield realBody
def urlFromFilename(name):
    return urllib.quote(name)

HTTP_DATE_FMT = "%a, %d %b %Y %H:%M:%S GMT"
def formatLastModified(mtime):
    humanTime = time.gmtime(mtime)
    return(time.strftime(HTTP_DATE_FMT, humanTime))
def startResponse(out, statusCode, statusText, headers):
    statusCode = int(statusCode)
    headers = headers[:]
    headers.append(("Date", formatLastModified(time.time())))
    headers.append(("Status", "%s %s" % (statusCode, statusText)))
    for k, v in headers:
       out.write("%s: %s\r\n" % (k, v)) # TODO escape
    out.write("\r\n")
def renderBinaryFile(f, out, bWriteBody):
    """ renders the file f, if it is binary.
        returns: whether the file f was rendered or not. """
    stat1 = os.fstat(f.fileno())
    regularFileFlag = stat.S_ISREG(stat1.st_mode)
    if regularFileFlag:
        mimeType = magic.getRegularFileType(f)
        if mimeType != "application/x-httpd-php": # or True:
                f.seek(0, 2)
                l = f.tell()
                f.seek(0)
                startResponse(out, 200, "OK", [("Content-Type", mimeType), ("Content-Length", l), ("Last-Modified", formatLastModified(stat1.st_mtime))])
                if bWriteBody:
                    copy(f, out)
                return(True)
    return(False)
# see <http://developer.yahoo.com/performance/rules.html#etags>

def printPathBar(dirName):
    io = StringIO.StringIO()
    parts = dirName.split("/")
    with P(): # path
        for part in parts:
            if part == "":
                continue
            with Text("/"):
                pass
            io.write("/")
            io.write(part)
            with A(href=urlFromFilename(io.getvalue())):
                with Text(part):
                    pass
def getEntries(containerName):
    try:
        webhidden = set(loadStrings("%s/.webhidden" % (containerName, )))
    except:
        webhidden = set()
    webhidden.add("index") # TODO except when we aren't it?
    for name in os.listdir(containerName):
        if name.startswith(".") or name in webhidden:
            continue
        title = name
        bmtime = 0
        try:
            bname = os.path.join(containerName, name)
            st = os.stat(bname)
            bmtime = st.st_mtime
            if stat.S_ISDIR(st.st_mode):
                title = "%s/" % (name, )
        except OSError:
            pass
        yield (title, name, bmtime)
def getEntries2(containerName):
    #yield from getEntries(containerName)
    for entry in sorted(getEntries(containerName)):
        yield entry
    try:
        webseealso = loadStrings("%s/.webseealso" % (containerName, ))
        webseealso = map(lambda line: line.split("\t", 1), webseealso) # URL, description
    except:
        webseealso = []
    for URL, description in webseealso:
        yield ("See also: %s" % (description, ), URL, None)
def getNewestSubpage(base): # -> (title, path, mtime)
    #st = os.stat(base)
    #mtime = st.st_mtime
    title = None
    name = None
    mtime = 0
    for xtitle, yname, xmtime in getEntries(base):
        xname = os.path.join(base, yname)
        if xtitle.endswith("/") and not xtitle.startswith("."):
            btitle, bname, bmtime = getNewestSubpage(xname)
            if bmtime > xmtime:
                xname = bname
                xmtime = bmtime
                xtitle = btitle
        if xmtime > mtime:
            name = xname
            mtime = xmtime
            title = xtitle
    return title, name, mtime
def md5hexdigestText(text):
    m = hashlib.md5()
    m.update(text)
    return(m.hexdigest())
def gallery(folderName, req):
    DOCUMENT_ROOT = req.env.get("DOCUMENT_ROOT", "R")
    ifolderName = "%s/image" % (folderName, )
    idirName = pathjoin(DOCUMENT_ROOT, ifolderName)
    with P():
       with Text(ifolderName):
          pass
    for title, href, mtime in sorted(getEntries(idirName)):
        #with A(href=href):
        #with DIV(**{"id": "anchor_%s" % (md5hexdigestText(href), ), "class": "image_of_the_day_big_image"}):
        with IMG(**{"alt": title, "src": urlFromFilename("%s/%s" % (ifolderName, href)), "class": "page"}):
            pass
    with BR(clear="both") as br:
        yield br
def navigationB(req):
    from common import saxdirtree
    DOCUMENT_ROOT = req.env.get("DOCUMENT_ROOT", "R")
    #PATH_INFO = req.env.get("REQUEST_URI", ".").split("?")[0] # PATH_INFO"]
    #PATH_INFO = urllib.unquote(PATH_INFO)
    PATH_INFO = req.env.get("PATH_INFO", "/.").split("?")[0] # PATH_INFO"]
    folderName = PATH_INFO # FIXME
    dirName = pathjoin(DOCUMENT_ROOT, folderName)
    with TABLE(**{"border": "0", "class": "main_navigation_area"}):
        with TR():
            with TD():
                with NAV(**{"class": "main_navigation", "id": "main_navigation"}):
                    printPathBar(PATH_INFO)
                    try:
                        saxdirtree.printList({"class": "main_directory"}, getEntries2(dirName))
                    except OSError:
                        pass
            with TD(**{"class": "main"}):
                # TODO H1
                with DIV(**{"class": "main_border", "id": "main_border"}):
                    with DIV(**{"class": "main_content"}) as main_content:
                        if os.path.exists(dirName + "/image") and (not os.path.exists(dirName + "/image/equation") or os.stat(dirName + "/image").st_nlink > 3):
                            with Template(gallery(folderName, req)):
                                pass
                        # everything else.
                        yield main_content
def modificationB(req, name):
    DOCUMENT_ROOT = req.env.get("DOCUMENT_ROOT", "R")
    dname = os.path.dirname(name)
    title, name, mtime = getNewestSubpage(dname)
    if name and name.startswith(DOCUMENT_ROOT):
        name = name[len(DOCUMENT_ROOT):]
        with P() as result:
            with Text("Last Modification in "):
                pass
            with A(href=urlFromFilename(name)):
                with Text(name):
                    pass
            with Text(" on "):
                pass
            humanReadableTime = time.asctime(time.gmtime(mtime))
            with Text(humanReadableTime):
                pass
            with Text("."):
                pass
            yield(result)
def serveDefaultPage(name, req, outHeaders, onload=None):
    with Template(servePage(req, 200, "OK", outHeaders, onload=onload)) as t:
        yield t
        with Template(modificationB(req, name)) as t2:
            pass
        # TODO webauthor

"""
urlparse.urlunparse
>>> urllib.urlencode({"a": "b", "c": "d"})
'a=b&c=d'
>>> urlparse.urlunparse(("http", "a", "x", ("a", "b"), "q", "frag"))
"http://a/x;('a', 'b')?q#frag"
>>> urlparse.urlunparse(("http", "a", "x", None, "X", "frag"))
'http://a/x?X#frag'
"""

def makeURL(scheme, netloc, url, params = None, query = None, fragment = None):
	"""
	>>> makeURL("http", "a", "x", None, {"a": "b", "c": "d"}, "frag")
	'http://a/x?a=b&c=d#frag'
	"""
	squery = urllib.urlencode(query)
	return urlparse.urlunparse((scheme, netloc, url, params, squery, fragment))

if __name__ == "__main__":
	makeURL("https", "api.dreamhost.com", "/", None, {
		"key": "xxx",
		"cmd": "a",
		"format": "json",
		"unique_id": hash("DNS_UPDATE"),
	}, None)
<|endoftext|>


<commit_before># tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
# so we need to create a synthetic argv.
import sys
if not hasattr(sys, 'argv'):
    sys.argv  = ['p3b1']

import json
import os
import p3b1
import runner_utils
import socket
import time

node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
print("node,pid: " + node_pid)

logger = None

def get_logger():
    """ Set up logging """
    global logger
    if logger is not None:
        return logger
    import logging, sys
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    h = logging.StreamHandler(stream=sys.stdout)
    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
                             datefmt='%Y/%m/%d %H:%M:%S')
    h.setFormatter(fmtr)
    logger.addHandler(h)
    return logger

def run(hyper_parameter_map):

    logger = get_logger()
    framework = hyper_parameter_map['framework']
    logger.debug("IMPORT START " + str(time.time()))
    if framework == 'keras':
        import p3b1_baseline_keras2
        pkg = p3b1_baseline_keras2
    else:
        raise ValueError("Unsupported framework: {}".format(framework))
    logger.debug("IMPORT STOP")

    # params is python dictionary
    params = pkg.initialize_parameters()
    runner_utils.format_params(hyper_parameter_map)

    for k,v in hyper_parameter_map.items():
        #if not k in params:
        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
        params[k] = v

    logger.debug("WRITE_PARAMS START")
    runner_utils.write_params(params, hyper_parameter_map)
    logger.debug("WRITE_PARAMS STOP")
    logger.debug("DO_N_FOLD START " + str(time.time()))
    avg_loss = pkg.do_n_fold(params)
    logger.debug("DO_N_FOLD STOP")

    if framework == 'keras':
        # works around this error:
        # https://github.com/tensorflow/tensorflow/issues/3388
        try:
            from keras import backend as K
            K.clear_session()
        except AttributeError:      # theano does not have this function
            pass

    return avg_loss

if __name__ == '__main__':
    logger = get_logger()
    logger.debug("RUN START")

    param_string = sys.argv[1]
    instance_directory = sys.argv[2]
    framework = sys.argv[3]
    exp_id = sys.argv[4]
    run_id = sys.argv[5]
    benchmark_timeout = int(sys.argv[6])

    logger.debug("RUN INIT START")

    hyper_parameter_map = runner_utils.init(param_string, instance_directory,
                                            framework, 'save_path')
    logger.debug("RUN INIT STOP")
    hyper_parameter_map['experiment_id'] = exp_id
    hyper_parameter_map['run_id'] = run_id
    hyper_parameter_map['timeout'] = benchmark_timeout
    # clear sys.argv so that argparse doesn't object
    sys.argv = ['p3b1_runner']
    result = run(hyper_parameter_map)
    logger.debug("WRITE OUTPUT START")
    runner_utils.write_output(result, instance_directory)
    logger.debug("WRITE OUTPUT STOP")
    logger.debug("RUN STOP")
<commit_msg>Disable verbose with new feature<commit_after># tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
# so we need to create a synthetic argv.
import sys
if not hasattr(sys, 'argv'):
    sys.argv  = ['p3b1']

import json
import os
import p3b1
import runner_utils
import socket
import time

node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
print("node,pid: " + node_pid)

logger = None

def get_logger():
    """ Set up logging """
    global logger
    if logger is not None:
        return logger
    import logging, sys
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    h = logging.StreamHandler(stream=sys.stdout)
    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
                             datefmt='%Y/%m/%d %H:%M:%S')
    h.setFormatter(fmtr)
    logger.addHandler(h)
    return logger

def run(hyper_parameter_map):

    logger = get_logger()
    framework = hyper_parameter_map['framework']
    logger.debug("IMPORT START " + str(time.time()))
    if framework == 'keras':
        import p3b1_baseline_keras2
        pkg = p3b1_baseline_keras2
    else:
        raise ValueError("Unsupported framework: {}".format(framework))
    logger.debug("IMPORT STOP")

    # params is python dictionary
    params = pkg.initialize_parameters()
    runner_utils.format_params(hyper_parameter_map)

    for k,v in hyper_parameter_map.items():
        #if not k in params:
        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
        params[k] = v

    logger.debug("WRITE_PARAMS START")
    runner_utils.write_params(params, hyper_parameter_map)
    logger.debug("WRITE_PARAMS STOP")
    logger.debug("DO_N_FOLD START " + str(time.time()))
    avg_loss = pkg.do_n_fold(params, verbose=0)
    logger.debug("DO_N_FOLD STOP")

    if framework == 'keras':
        # works around this error:
        # https://github.com/tensorflow/tensorflow/issues/3388
        try:
            from keras import backend as K
            K.clear_session()
        except AttributeError:      # theano does not have this function
            pass

    return avg_loss

if __name__ == '__main__':
    logger = get_logger()
    logger.debug("RUN START")

    param_string = sys.argv[1]
    instance_directory = sys.argv[2]
    framework = sys.argv[3]
    exp_id = sys.argv[4]
    run_id = sys.argv[5]
    benchmark_timeout = int(sys.argv[6])

    logger.debug("RUN INIT START")

    hyper_parameter_map = runner_utils.init(param_string, instance_directory,
                                            framework, 'save_path')
    logger.debug("RUN INIT STOP")
    hyper_parameter_map['experiment_id'] = exp_id
    hyper_parameter_map['run_id'] = run_id
    hyper_parameter_map['timeout'] = benchmark_timeout
    # clear sys.argv so that argparse doesn't object
    sys.argv = ['p3b1_runner']
    result = run(hyper_parameter_map)
    logger.debug("WRITE OUTPUT START")
    runner_utils.write_output(result, instance_directory)
    logger.debug("WRITE OUTPUT STOP")
    logger.debug("RUN STOP")
<|endoftext|>


<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import sys
import gc
from tempfile import NamedTemporaryFile

from pyspark.cloudpickle import print_exec
from pyspark.broadcast import Broadcast
from pyspark.broadcast import _from_id
from bigdl.nn.layer import Model

def _from_id_and_type(bid, bigdl_type):
    result = _from_id(bid)
    return ModelBroadcast(path=result._path, bigdl_type=bigdl_type)

def broadcast_model(sc, layer):
    return ModelBroadcast(sc, layer, sc._pickled_broadcast_vars)

class ModelBroadcast(Broadcast):

    def __init__(self, sc=None, layer=None, pickle_registry=None, path=None, bigdl_type="float"):
        """
        Should not be called directly by users -- use L{SparkContext.broadcast()}
        instead.
        """
        if layer is not None:
            self.bigdl_type = layer.bigdl_type
        else:
            self.bigdl_type = bigdl_type
        super(ModelBroadcast, self).__init__(sc, layer, pickle_registry, path)

    def dump(self, value, f):
        try:
            value.saveModel(f.name, over_write=True)
        except Exception as e:
            msg = "Could not serialize broadcast: %s" % e.__class__.__name__
            print_exec(sys.stderr)
            raise ValueError(msg)
        f.close()
        return f.name

    def _load(self, path):
        return Model.loadModel(path, bigdl_type=self.bigdl_type)

    @property
    def value(self):
        """ Return the broadcasted value
        """
        if not hasattr(self, "_value") and self._path is not None:
            self._value = self._load(self._path)
        return self._value

    def __reduce__(self):
        if self._jbroadcast is None:
            raise Exception("Broadcast can only be serialized in driver")
        self._pickle_registry.add(self)
        return _from_id_and_type, (self._jbroadcast.id(), self.bigdl_type)
<commit_msg>[WIP] spark 3.0 (#3054)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import sys
import gc
from tempfile import NamedTemporaryFile

from pyspark.broadcast import Broadcast
from pyspark.broadcast import _from_id
from bigdl.nn.layer import Model

def _from_id_and_type(bid, bigdl_type):
    result = _from_id(bid)
    return ModelBroadcast(path=result._path, bigdl_type=bigdl_type)

def broadcast_model(sc, layer):
    return ModelBroadcast(sc, layer, sc._pickled_broadcast_vars)

class ModelBroadcast(Broadcast):

    def __init__(self, sc=None, layer=None, pickle_registry=None, path=None, bigdl_type="float"):
        """
        Should not be called directly by users -- use L{SparkContext.broadcast()}
        instead.
        """
        if layer is not None:
            self.bigdl_type = layer.bigdl_type
        else:
            self.bigdl_type = bigdl_type
        super(ModelBroadcast, self).__init__(sc, layer, pickle_registry, path)

    def dump(self, value, f):
        try:
            value.saveModel(f.name, over_write=True)
        except Exception as e:
            msg = "Could not serialize broadcast: %s" % e.__class__.__name__
            if not self.sc.version.startswith("2.1"):
                from pyspark.cloudpickle import print_exec
            else:
                from pyspark.util import print_exec
            print_exec(sys.stderr)
            raise ValueError(msg)
        f.close()
        return f.name

    def _load(self, path):
        return Model.loadModel(path, bigdl_type=self.bigdl_type)

    @property
    def value(self):
        """ Return the broadcasted value
        """
        if not hasattr(self, "_value") and self._path is not None:
            self._value = self._load(self._path)
        return self._value

    def __reduce__(self):
        if self._jbroadcast is None:
            raise Exception("Broadcast can only be serialized in driver")
        self._pickle_registry.add(self)
        return _from_id_and_type, (self._jbroadcast.id(), self.bigdl_type)
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test
from django.core.urlresolvers import reverse_lazy
from django.forms.models import modelformset_factory
from django.shortcuts import redirect, render
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, UpdateView, DeleteView

from . forms import CSVUploadForm, InterestGroupSelectionForm
from . models import Action, Category, Interest, InterestGroup
from . utils import get_max_order

def superuser_required(user):
    """Verifies that a user is authenticated and a super user."""
    return user.is_authenticated() and user.is_superuser

class SuperuserRequiredMixin(object):
    """A Mixin that requires the user to be a superuser in order to access
    the view.

    NOTE: Eventually we'll want to have more granular permissions, here. We
    probably need a group or object-level permissions and check for those
    instead (namely, once we have more than just our group editing content)
    """
    @classmethod
    def as_view(cls, **initkwargs):
        view = super(SuperuserRequiredMixin, cls).as_view(**initkwargs)
        dec = user_passes_test(superuser_required, login_url=settings.LOGIN_URL)
        return dec(view)

@user_passes_test(superuser_required, login_url='/')
def upload_csv(request):
    """Allow a user to upload a CSV file to populate our data backend."""
    if request.method == "POST":
        form = CSVUploadForm(request.POST, request.FILES)
        if form.is_valid():
            try:
                form.save()
                messages.success(request, 'CSV File uploaded, successfully.')
                return redirect("goals:index")
            except CSVUploadForm.InvalidFormat as e:
                messages.error(
                    request, "The uploaded file could not be "
                    "processed. Please check the format and try again: "
                    " {0}".format(e)
                )
        else:
            messages.warning(
                request, "This form didn't validate. Please try again."
            )
    else:
        form = CSVUploadForm()

    context = {'form': form}
    return render(request, 'goals/upload_csv.html', context)

class CategoryListView(SuperuserRequiredMixin, ListView):
    model = Category
    context_object_name = 'categories'
    template_name = "goals/index.html"

class CategoryDetailView(SuperuserRequiredMixin, DetailView):
    queryset = Category.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class CategoryCreateView(SuperuserRequiredMixin, CreateView):
    model = Category
    fields = ['order', 'name', 'description', 'icon', 'notes']

    def get_initial(self, *args, **kwargs):
        """Pre-populate the value for the initial order. This can't be done
        at the class level because we want to query the value each time."""
        initial = super(CategoryCreateView, self).get_initial(*args, **kwargs)
        if 'order' not in initial:
            initial['order'] = get_max_order(Category)
        return initial

    def get_context_data(self, **kwargs):
        context = super(CategoryCreateView, self).get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        if 'formset' not in kwargs:
            context['formset'] = self.get_interestgroup_formset()
        return context

    def get_interestgroup_formset(self, post_data=None):
        InterestGroupFormset = modelformset_factory(
            InterestGroup,
            fields=('name', ),
            extra=6
        )
        if post_data:
            formset = InterestGroupFormset(post_data, prefix="ig")
        else:
            formset = InterestGroupFormset(
                queryset=InterestGroup.objects.none(),
                prefix="ig"
            )
        return formset

    def post(self, request, *args, **kwargs):
        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        formset = self.get_interestgroup_formset(request.POST)
        if form.is_valid() and formset.is_valid():
            return self.form_valid(form, formset)
        else:
            return self.form_invalid(form, formset)

    def form_invalid(self, form, formset=None):
        context = self.get_context_data(form=form, formset=formset)
        return self.render_to_response(context)

    def form_valid(self, form, formset):
        self.object = form.save()
        for instance in formset.save(commit=False):
            instance.category = self.object
            instance.save()
        formset.save_m2m()
        return super(CategoryCreateView, self).form_valid(form)

class CategoryUpdateView(SuperuserRequiredMixin, UpdateView):
    model = Category
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = ['order', 'name', 'description', 'icon', 'notes']

    def get_context_data(self, **kwargs):
        context = super(CategoryUpdateView, self).get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        if 'formset' not in kwargs:
            context['formset'] = self.get_interestgroup_formset()
        return context

    def get_interestgroup_formset(self, post_data=None):
        InterestGroupFormset = modelformset_factory(
            InterestGroup,
            fields=('name', ),
            extra=3
        )
        if post_data:
            formset = InterestGroupFormset(post_data, prefix="ig")
        else:
            formset = InterestGroupFormset(queryset=self.object.groups, prefix="ig")
        return formset

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        formset = self.get_interestgroup_formset(request.POST)
        if form.is_valid() and formset.is_valid():
            return self.form_valid(form, formset)
        else:
            return self.form_invalid(form, formset)

    def form_invalid(self, form, formset=None):
        context = self.get_context_data(form=form, formset=formset)
        return self.render_to_response(context)

    def form_valid(self, form, formset):
        self.object = form.save()
        for instance in formset.save(commit=False):
            instance.category = self.object
            instance.save()
        formset.save_m2m()
        return super(CategoryUpdateView, self).form_valid(form)

class CategoryDeleteView(SuperuserRequiredMixin, DeleteView):
    model = Category
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')

class InterestListView(SuperuserRequiredMixin, ListView):
    model = Interest
    context_object_name = 'interests'

class InterestDetailView(SuperuserRequiredMixin, DetailView):
    queryset = Interest.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class InterestCreateView(SuperuserRequiredMixin, CreateView):
    model = Interest
    fields = [
        'order', 'name', 'description', 'notes', 'source_name', 'source_link'
    ]

    def get_initial(self, *args, **kwargs):
        """Pre-populate the value for the initial order. This can't be done
        at the class level because we want to query the value each time."""
        initial = super(InterestCreateView, self).get_initial(*args, **kwargs)
        if 'order' not in initial:
            initial['order'] = get_max_order(Interest)
        return initial

    def get_interestgroup_form(self, post_data=None):
        if post_data:
            form = InterestGroupSelectionForm(post_data, prefix="ig")
        else:
            form = InterestGroupSelectionForm(prefix="ig")
        return form

    def get_context_data(self, **kwargs):
        context = super(InterestCreateView, self).get_context_data(**kwargs)
        context['interests'] = Interest.objects.all()
        if 'ig_form' not in kwargs:
            context['ig_form'] = self.get_interestgroup_form()
        return context

    def post(self, request, *args, **kwargs):
        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        ig_form = self.get_interestgroup_form(request.POST)
        if form.is_valid() and ig_form.is_valid():
            return self.form_valid(form, ig_form)
        else:
            return self.form_invalid(form, ig_form)

    def form_invalid(self, form, ig_form=None):
        context = self.get_context_data(form=form, ig_form=ig_form)
        return self.render_to_response(context)

    def form_valid(self, form, ig_form):
        self.object = form.save()
        for ig in ig_form.cleaned_data['add_to_groups']:
            ig.interests.add(self.object)
        return super(InterestCreateView, self).form_valid(form)

class InterestUpdateView(SuperuserRequiredMixin, UpdateView):
    model = Interest
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = [
        'order', 'name', 'description', 'notes', 'source_name', 'source_link'
    ]

    def get_interestgroup_form(self, post_data=None):
        if post_data:
            form = InterestGroupSelectionForm(post_data, prefix="ig")
        else:
            form = InterestGroupSelectionForm(initial=self.object.groups, prefix="ig")
        return form

    def get_context_data(self, **kwargs):
        context = super(InterestUpdateView, self).get_context_data(**kwargs)
        context['interests'] = Interest.objects.all()
        if 'ig_form' not in kwargs:
            context['ig_form'] = self.get_interestgroup_form()
        return context

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        ig_form = self.get_interestgroup_form(request.POST)
        if form.is_valid() and ig_form.is_valid():
            return self.form_valid(form, ig_form)
        else:
            return self.form_invalid(form, ig_form)

    def form_invalid(self, form, ig_form=None):
        context = self.get_context_data(form=form, ig_form=ig_form)
        return self.render_to_response(context)

    def form_valid(self, form, ig_form):
        self.object = form.save()
        self.object.remove_from_interestgroups()  # Remove all existing groups.
        for ig in ig_form.cleaned_data['add_to_groups']:
            ig.interests.add(self.object)
        return super(InterestUpdateView, self).form_valid(form)

class InterestDeleteView(SuperuserRequiredMixin, DeleteView):
    model = Interest
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')

class InterestGroupListView(SuperuserRequiredMixin, ListView):
    model = InterestGroup
    context_object_name = 'interestgroups'

class InterestGroupDetailView(SuperuserRequiredMixin, DetailView):
    queryset = InterestGroup.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class InterestGroupCreateView(SuperuserRequiredMixin, CreateView):
    model = InterestGroup
    fields = ['category', 'interests', 'name']

    def get_context_data(self, **kwargs):
        context = super(InterestGroupCreateView, self).get_context_data(**kwargs)
        context['interestgroups'] = InterestGroup.objects.all()
        return context

class InterestGroupUpdateView(SuperuserRequiredMixin, UpdateView):
    model = InterestGroup
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = ['category', 'interests', 'name']

    def get_context_data(self, **kwargs):
        context = super(InterestGroupUpdateView, self).get_context_data(**kwargs)
        context['interestgroups'] = InterestGroup.objects.all()
        return context

class InterestGroupDeleteView(SuperuserRequiredMixin, DeleteView):
    model = InterestGroup
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')

class ActionListView(SuperuserRequiredMixin, ListView):
    model = Action
    context_object_name = 'actions'

class ActionDetailView(SuperuserRequiredMixin, DetailView):
    queryset = Action.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class ActionCreateView(SuperuserRequiredMixin, CreateView):
    model = Action
    fields = [
        'order', 'name', 'summary', 'description', 'interests',
        'default_reminder_time', 'default_reminder_frequency',
        'icon', 'notes', 'source_name', 'source_link',
    ]

    def get_initial(self, *args, **kwargs):
        """Pre-populate the value for the initial order. This can't be done
        at the class level because we want to query the value each time."""
        initial = super(ActionCreateView, self).get_initial(*args, **kwargs)
        if 'order' not in initial:
            initial['order'] = get_max_order(Action)
        return initial

    def get_context_data(self, **kwargs):
        context = super(ActionCreateView, self).get_context_data(**kwargs)
        context['actions'] = Action.objects.all()
        return context

class ActionUpdateView(SuperuserRequiredMixin, UpdateView):
    model = Action
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = [
        'order', 'name', 'summary', 'description', 'interests',
        'default_reminder_time', 'default_reminder_frequency',
        'icon', 'notes', 'source_name', 'source_link',
    ]

    def get_context_data(self, **kwargs):
        context = super(ActionUpdateView, self).get_context_data(**kwargs)
        context['actions'] = Action.objects.all()
        return context

class ActionDeleteView(SuperuserRequiredMixin, DeleteView):
    model = Action
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')
<commit_msg>we probably only need 3 of those<commit_after>from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test
from django.core.urlresolvers import reverse_lazy
from django.forms.models import modelformset_factory
from django.shortcuts import redirect, render
from django.views.generic import DetailView, ListView
from django.views.generic.edit import CreateView, UpdateView, DeleteView

from . forms import CSVUploadForm, InterestGroupSelectionForm
from . models import Action, Category, Interest, InterestGroup
from . utils import get_max_order

def superuser_required(user):
    """Verifies that a user is authenticated and a super user."""
    return user.is_authenticated() and user.is_superuser

class SuperuserRequiredMixin(object):
    """A Mixin that requires the user to be a superuser in order to access
    the view.

    NOTE: Eventually we'll want to have more granular permissions, here. We
    probably need a group or object-level permissions and check for those
    instead (namely, once we have more than just our group editing content)
    """
    @classmethod
    def as_view(cls, **initkwargs):
        view = super(SuperuserRequiredMixin, cls).as_view(**initkwargs)
        dec = user_passes_test(superuser_required, login_url=settings.LOGIN_URL)
        return dec(view)

@user_passes_test(superuser_required, login_url='/')
def upload_csv(request):
    """Allow a user to upload a CSV file to populate our data backend."""
    if request.method == "POST":
        form = CSVUploadForm(request.POST, request.FILES)
        if form.is_valid():
            try:
                form.save()
                messages.success(request, 'CSV File uploaded, successfully.')
                return redirect("goals:index")
            except CSVUploadForm.InvalidFormat as e:
                messages.error(
                    request, "The uploaded file could not be "
                    "processed. Please check the format and try again: "
                    " {0}".format(e)
                )
        else:
            messages.warning(
                request, "This form didn't validate. Please try again."
            )
    else:
        form = CSVUploadForm()

    context = {'form': form}
    return render(request, 'goals/upload_csv.html', context)

class CategoryListView(SuperuserRequiredMixin, ListView):
    model = Category
    context_object_name = 'categories'
    template_name = "goals/index.html"

class CategoryDetailView(SuperuserRequiredMixin, DetailView):
    queryset = Category.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class CategoryCreateView(SuperuserRequiredMixin, CreateView):
    model = Category
    fields = ['order', 'name', 'description', 'icon', 'notes']

    def get_initial(self, *args, **kwargs):
        """Pre-populate the value for the initial order. This can't be done
        at the class level because we want to query the value each time."""
        initial = super(CategoryCreateView, self).get_initial(*args, **kwargs)
        if 'order' not in initial:
            initial['order'] = get_max_order(Category)
        return initial

    def get_context_data(self, **kwargs):
        context = super(CategoryCreateView, self).get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        if 'formset' not in kwargs:
            context['formset'] = self.get_interestgroup_formset()
        return context

    def get_interestgroup_formset(self, post_data=None):
        InterestGroupFormset = modelformset_factory(
            InterestGroup,
            fields=('name', ),
            extra=3
        )
        if post_data:
            formset = InterestGroupFormset(post_data, prefix="ig")
        else:
            formset = InterestGroupFormset(
                queryset=InterestGroup.objects.none(),
                prefix="ig"
            )
        return formset

    def post(self, request, *args, **kwargs):
        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        formset = self.get_interestgroup_formset(request.POST)
        if form.is_valid() and formset.is_valid():
            return self.form_valid(form, formset)
        else:
            return self.form_invalid(form, formset)

    def form_invalid(self, form, formset=None):
        context = self.get_context_data(form=form, formset=formset)
        return self.render_to_response(context)

    def form_valid(self, form, formset):
        self.object = form.save()
        for instance in formset.save(commit=False):
            instance.category = self.object
            instance.save()
        formset.save_m2m()
        return super(CategoryCreateView, self).form_valid(form)

class CategoryUpdateView(SuperuserRequiredMixin, UpdateView):
    model = Category
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = ['order', 'name', 'description', 'icon', 'notes']

    def get_context_data(self, **kwargs):
        context = super(CategoryUpdateView, self).get_context_data(**kwargs)
        context['categories'] = Category.objects.all()
        if 'formset' not in kwargs:
            context['formset'] = self.get_interestgroup_formset()
        return context

    def get_interestgroup_formset(self, post_data=None):
        InterestGroupFormset = modelformset_factory(
            InterestGroup,
            fields=('name', ),
            extra=3
        )
        if post_data:
            formset = InterestGroupFormset(post_data, prefix="ig")
        else:
            formset = InterestGroupFormset(queryset=self.object.groups, prefix="ig")
        return formset

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        formset = self.get_interestgroup_formset(request.POST)
        if form.is_valid() and formset.is_valid():
            return self.form_valid(form, formset)
        else:
            return self.form_invalid(form, formset)

    def form_invalid(self, form, formset=None):
        context = self.get_context_data(form=form, formset=formset)
        return self.render_to_response(context)

    def form_valid(self, form, formset):
        self.object = form.save()
        for instance in formset.save(commit=False):
            instance.category = self.object
            instance.save()
        formset.save_m2m()
        return super(CategoryUpdateView, self).form_valid(form)

class CategoryDeleteView(SuperuserRequiredMixin, DeleteView):
    model = Category
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')

class InterestListView(SuperuserRequiredMixin, ListView):
    model = Interest
    context_object_name = 'interests'

class InterestDetailView(SuperuserRequiredMixin, DetailView):
    queryset = Interest.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class InterestCreateView(SuperuserRequiredMixin, CreateView):
    model = Interest
    fields = [
        'order', 'name', 'description', 'notes', 'source_name', 'source_link'
    ]

    def get_initial(self, *args, **kwargs):
        """Pre-populate the value for the initial order. This can't be done
        at the class level because we want to query the value each time."""
        initial = super(InterestCreateView, self).get_initial(*args, **kwargs)
        if 'order' not in initial:
            initial['order'] = get_max_order(Interest)
        return initial

    def get_interestgroup_form(self, post_data=None):
        if post_data:
            form = InterestGroupSelectionForm(post_data, prefix="ig")
        else:
            form = InterestGroupSelectionForm(prefix="ig")
        return form

    def get_context_data(self, **kwargs):
        context = super(InterestCreateView, self).get_context_data(**kwargs)
        context['interests'] = Interest.objects.all()
        if 'ig_form' not in kwargs:
            context['ig_form'] = self.get_interestgroup_form()
        return context

    def post(self, request, *args, **kwargs):
        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        ig_form = self.get_interestgroup_form(request.POST)
        if form.is_valid() and ig_form.is_valid():
            return self.form_valid(form, ig_form)
        else:
            return self.form_invalid(form, ig_form)

    def form_invalid(self, form, ig_form=None):
        context = self.get_context_data(form=form, ig_form=ig_form)
        return self.render_to_response(context)

    def form_valid(self, form, ig_form):
        self.object = form.save()
        for ig in ig_form.cleaned_data['add_to_groups']:
            ig.interests.add(self.object)
        return super(InterestCreateView, self).form_valid(form)

class InterestUpdateView(SuperuserRequiredMixin, UpdateView):
    model = Interest
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = [
        'order', 'name', 'description', 'notes', 'source_name', 'source_link'
    ]

    def get_interestgroup_form(self, post_data=None):
        if post_data:
            form = InterestGroupSelectionForm(post_data, prefix="ig")
        else:
            form = InterestGroupSelectionForm(initial=self.object.groups, prefix="ig")
        return form

    def get_context_data(self, **kwargs):
        context = super(InterestUpdateView, self).get_context_data(**kwargs)
        context['interests'] = Interest.objects.all()
        if 'ig_form' not in kwargs:
            context['ig_form'] = self.get_interestgroup_form()
        return context

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        form_class = self.get_form_class()
        form = self.get_form(form_class)
        ig_form = self.get_interestgroup_form(request.POST)
        if form.is_valid() and ig_form.is_valid():
            return self.form_valid(form, ig_form)
        else:
            return self.form_invalid(form, ig_form)

    def form_invalid(self, form, ig_form=None):
        context = self.get_context_data(form=form, ig_form=ig_form)
        return self.render_to_response(context)

    def form_valid(self, form, ig_form):
        self.object = form.save()
        self.object.remove_from_interestgroups()  # Remove all existing groups.
        for ig in ig_form.cleaned_data['add_to_groups']:
            ig.interests.add(self.object)
        return super(InterestUpdateView, self).form_valid(form)

class InterestDeleteView(SuperuserRequiredMixin, DeleteView):
    model = Interest
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')

class InterestGroupListView(SuperuserRequiredMixin, ListView):
    model = InterestGroup
    context_object_name = 'interestgroups'

class InterestGroupDetailView(SuperuserRequiredMixin, DetailView):
    queryset = InterestGroup.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class InterestGroupCreateView(SuperuserRequiredMixin, CreateView):
    model = InterestGroup
    fields = ['category', 'interests', 'name']

    def get_context_data(self, **kwargs):
        context = super(InterestGroupCreateView, self).get_context_data(**kwargs)
        context['interestgroups'] = InterestGroup.objects.all()
        return context

class InterestGroupUpdateView(SuperuserRequiredMixin, UpdateView):
    model = InterestGroup
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = ['category', 'interests', 'name']

    def get_context_data(self, **kwargs):
        context = super(InterestGroupUpdateView, self).get_context_data(**kwargs)
        context['interestgroups'] = InterestGroup.objects.all()
        return context

class InterestGroupDeleteView(SuperuserRequiredMixin, DeleteView):
    model = InterestGroup
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')

class ActionListView(SuperuserRequiredMixin, ListView):
    model = Action
    context_object_name = 'actions'

class ActionDetailView(SuperuserRequiredMixin, DetailView):
    queryset = Action.objects.all()
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"

class ActionCreateView(SuperuserRequiredMixin, CreateView):
    model = Action
    fields = [
        'order', 'name', 'summary', 'description', 'interests',
        'default_reminder_time', 'default_reminder_frequency',
        'icon', 'notes', 'source_name', 'source_link',
    ]

    def get_initial(self, *args, **kwargs):
        """Pre-populate the value for the initial order. This can't be done
        at the class level because we want to query the value each time."""
        initial = super(ActionCreateView, self).get_initial(*args, **kwargs)
        if 'order' not in initial:
            initial['order'] = get_max_order(Action)
        return initial

    def get_context_data(self, **kwargs):
        context = super(ActionCreateView, self).get_context_data(**kwargs)
        context['actions'] = Action.objects.all()
        return context

class ActionUpdateView(SuperuserRequiredMixin, UpdateView):
    model = Action
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    fields = [
        'order', 'name', 'summary', 'description', 'interests',
        'default_reminder_time', 'default_reminder_frequency',
        'icon', 'notes', 'source_name', 'source_link',
    ]

    def get_context_data(self, **kwargs):
        context = super(ActionUpdateView, self).get_context_data(**kwargs)
        context['actions'] = Action.objects.all()
        return context

class ActionDeleteView(SuperuserRequiredMixin, DeleteView):
    model = Action
    slug_field = "name_slug"
    slug_url_kwarg = "name_slug"
    success_url = reverse_lazy('goals:index')
<|endoftext|>