<commit_before>
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from datetime import datetime
import os.path
import re
import time

import numpy as np
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf
import cnnt_input
import cnn_model
from sklearn.metrics import precision_score, recall_score, f1_score

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('train_dir', './train/',
                           """Directory where to write event logs """
                           """and checkpoint.""")
tf.app.flags.DEFINE_integer('max_steps', 1000000,
                            """Number of batches to run.""")
tf.app.flags.DEFINE_integer('num_gpus', 4,
                            """How many GPUs to use.""")
tf.app.flags.DEFINE_boolean('log_device_placement', False,
                            """Whether to log device placement.""")
tf.app.flags.DEFINE_float('dropout_keep_prob', 0.8, """dropout_keep_prob""")

def tower_loss(namescope, target, batch_size=4):
    """Calculate the total loss on a single tower running the CIFAR model.

    Args:
    namescope: unique prefix string identifying the tweets tower, e.g. 'tower_0'

    Returns:
     Tensor of shape [] containing the total loss for a batch of data
    """
    # Get images and labels for tweets
    txts, labels = cnnt_input.get_inputs(target, batch_size=batch_size)

    # if target == 'trn':
    #     txts, labels = cnnt_input.get_inputs('trn', batch_size=batch_size)
    #
    # else:  # 'dev'
    #     txts, labels = cnnt_input.get_inputs('tst', batch_size=batch_size)

    # Build inference Graph.
    if target=='trn':
        logits = cnn_model.inference(txts, dropout_keep_prob=FLAGS.dropout_keep_prob)

    else:
        logits = cnn_model.inference(txts)

    y_true = tf.argmax(labels, 1, name="golds")
    y_pred = tf.argmax(logits, 1, name="predictions")

    # Build the portion of the Graph calculating the losses. Note that we will
    # assemble the total_loss using a custom function below.
    _, accuracy = cnn_model.loss(logits, labels)

    # Assemble all of the losses for the current tower only.
    losses = tf.get_collection('losses', namescope)

    # Calculate the total loss for the current tower.
    total_loss = tf.add_n(losses, name='total_loss')

    # Attach a scalar summary to all individual losses and the total loss; do the
    # same for the averaged version of the losses.
    for l in losses + [total_loss]:
        # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
        # session. This helps the clarity of presentation on tensorboard.
        loss_name = re.sub('%s_[0-9]*/' % cnn_model.TOWER_NAME, '', l.op.name)
        tf.summary.scalar(loss_name, l)

    return total_loss, accuracy, logits, y_true, y_pred

def average_gradients(tower_grads):
    """Calculate the average gradient for each shared variable across all towers.

    Note that this function provides a synchronization point across all towers.

    Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
        is over individual gradients. The inner list is over the gradient
        calculation for each tower.
    Returns:
        List of pairs of (gradient, variable) where the gradient has been averaged
        across all towers.
    """
    average_grads = []
    for grad_and_vars in zip(*tower_grads):
        # Note that each grad_and_vars looks like the following:
        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
        grads = []
        for g, _ in grad_and_vars:
            # Add 0 dimension to the gradients to represent the tower.
            expanded_g = tf.expand_dims(g, 0)

            # Append on a 'tower' dimension which we will average over below.
            grads.append(expanded_g)

        # Average over the 'tower' dimension.
        grad = tf.concat(axis=0, values=grads)
        grad = tf.reduce_mean(grad, 0)

        # Keep in mind that the Variables are redundant because they are shared
        # across towers. So .. we will just return the first tower's pointer to
        # the Variable.
        v = grad_and_vars[0][1]
        grad_and_var = (grad, v)
        average_grads.append(grad_and_var)

    return average_grads

def train():
    """Train cnnt for a number of steps."""
    with tf.Graph().as_default(), tf.device('/cpu:0'):
        # Create a variable to count the number of train() calls. This equals the
        # number of batches processed * FLAGS.num_gpus.
        global_step = tf.get_variable(
            'global_step', [],
            initializer=tf.constant_initializer(0), trainable=False)

        # Calculate the learning rate schedule.
        # num_batches_per_epoch = (cnnt_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
        #                          FLAGS.batch_size)
        # decay_steps = int(num_batches_per_epoch * cnnt_input.NUM_EPOCHS_PER_DECAY)

        # Decay the learning rate exponentially based on the number of steps.
        # lr = tf.train.exponential_decay(cnnt_input.INITIAL_LEARNING_RATE,
        #                                 global_step,
        #                                 decay_steps,
        #                                 cnnt_input.LEARNING_RATE_DECAY_FACTOR,
        #                                 staircase=True)

        # Create an optimizer that performs gradient descent.
        # opt = tf.train.GradientDescentOptimizer(lr)
        opt = tf.train.AdamOptimizer(1e-3)

        # Calculate the gradients for each model tower.
        tower_grads = []

        with tf.variable_scope(tf.get_variable_scope()):
            with tf.device('/gpu:%d' % 0):
                with tf.name_scope('%s_%d_dev' % (cnn_model.TOWER_NAME, 0)) as namescope:
                    loss_dev, accuracy_dev, logits_dev, y_true_dev, y_pred_dev = \
                        tower_loss(namescope, 'dev', batch_size=1588)
                    # Reuse variables for the next tower.
                    tf.get_variable_scope().reuse_variables()

            with tf.device('/gpu:%d' % 1):
                with tf.name_scope('%s_%d_tst' % (cnn_model.TOWER_NAME, 0)) as namescope:
                    loss_tst, accuracy_tst, logits_tst, y_true_tst, y_pred_tst = \
                        tower_loss(namescope, 'tst', batch_size=20632)
                    # Reuse variables for the next tower.
                    tf.get_variable_scope().reuse_variables()

            for i in xrange(FLAGS.num_gpus):
                with tf.device('/gpu:%d' % i):
                    with tf.name_scope('%s_%d' % (cnn_model.TOWER_NAME, i)) as namescope:
                        # Calculate the loss for one tower of the CIFAR model. This function
                        # constructs the entire CIFAR model but shares the variables across
                        # all towers.
                        loss, accuracy, _, _, _ = tower_loss(namescope, 'trn', batch_size=FLAGS.batch_size)

                        # Reuse variables for the next tower.
                        tf.get_variable_scope().reuse_variables()

                        # Retain the summaries from the final tower.
                        summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, namescope)

                        # Calculate the gradients for the batch of data on this CIFAR tower.
                        grads = opt.compute_gradients(loss)

                        # Keep track of the gradients across all towers.
                        tower_grads.append(grads)

        # We must calculate the mean of each gradient. Note that this is the
        # synchronization point across all towers.
        grads = average_gradients(tower_grads)

        # # Add a summary to track the learning rate.
        # summaries.append(tf.summary.scalar('learning_rate', lr))

        # Add histograms for gradients.
        for grad, var in grads:
            if grad is not None:
                summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad))

        # Apply the gradients to adjust the shared variables.
        apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)

        # Add histograms for trainable variables.
        for var in tf.trainable_variables():
            summaries.append(tf.summary.histogram(var.op.name, var))

        # Track the moving averages of all trainable variables.
        variable_averages = tf.train.ExponentialMovingAverage(
            cnnt_input.MOVING_AVERAGE_DECAY, global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        # Group all updates to into a single train op.
        train_op = tf.group(apply_gradient_op, variables_averages_op)

        # Create a saver.
        saver = tf.train.Saver(tf.global_variables())

        # Build the summary operation from the last tower summaries.
        summary_op = tf.summary.merge(summaries)

        # Build an initialization operation to run below.
        init = tf.global_variables_initializer()

        # Start running operations on the Graph. allow_soft_placement must be set to
        # True to build towers on GPU, as some of the ops do not have GPU
        # implementations.
        # gpu_options = tf.GPUOptions(visible_device_list=str('0,1,2,3'), allow_growth=True)
        # gpu_options = tf.GPUOptions(visible_device_list=str('1,2,3'), allow_growth=True) # dead
        # gpu_options = tf.GPUOptions(visible_device_list=str('0,2,3'), allow_growth=True)  # dead
        gpu_options = tf.GPUOptions(visible_device_list=str('2,3'), allow_growth=True) # ok
        # gpu_options = tf.GPUOptions(visible_device_list=str('1,2'), allow_growth=True)  # dead
        sess = tf.Session(config=tf.ConfigProto(
            gpu_options=gpu_options,
            allow_soft_placement=True,
            log_device_placement=FLAGS.log_device_placement))
        sess.run(init)

        # Start the queue runners.
        tf.train.start_queue_runners(sess=sess)

        summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)

        for step in xrange(FLAGS.max_steps):
            start_time = time.time()
            # _, loss_value = sess.run([train_op, loss])
            _, loss_value, accuracy_val = sess.run([train_op, loss, accuracy])

            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            if step > 1 and step % 10 == 0:
                num_examples_per_step = FLAGS.batch_size * FLAGS.num_gpus
                examples_per_sec = num_examples_per_step / duration
                sec_per_batch = duration / FLAGS.num_gpus

                format_str = ('%s: step %d, loss = %.4f, acc = %.4f (%.1f examples/sec; %.3f '
                              'sec/batch)')
                print (format_str % (datetime.now(), step, loss_value, accuracy_val,
                                     examples_per_sec, sec_per_batch))

            if step > 1 and step % 100 == 0:
                summary_str = sess.run(summary_op)
                summary_writer.add_summary(summary_str, step)

                loss_dev_value, accuracy_dev_value, logits_dev_value, y_true_dev_value, y_pred_dev_value = \
                    sess.run([loss_dev, accuracy_dev, logits_dev, y_true_dev, y_pred_dev])

                # p_neg_dev = precision_score(y_true_dev_value==0, y_pred_dev_value==0)
                # r_neg_dev = recall_score(y_true_dev_value==0, y_pred_dev_value==0)
                f1_neg_dev = f1_score(y_true_dev_value==0, y_pred_dev_value==0)
                # p_pos_dev = precision_score(y_true_dev_value == 2, y_pred_dev_value == 2)
                # r_pos_dev = recall_score(y_true_dev_value == 2, y_pred_dev_value == 2)
                f1_pos_dev = f1_score(y_true_dev_value == 2, y_pred_dev_value == 2)
                f1_avg_dev = (f1_neg_dev+f1_pos_dev)/2

                format_str = ('[Eval] %s: step %d, loss = %.4f, acc = %.4f, f1neg = %.4f, f1pos = %.4f, f1 = %.4f')
                print(format_str % (datetime.now(), step, loss_dev_value, accuracy_dev_value,
                                    f1_neg_dev, f1_pos_dev, f1_avg_dev))

                loss_tst_value, accuracy_tst_value, logits_tst_value, y_true_tst_value, y_pred_tst_value = \
                    sess.run([loss_tst, accuracy_tst, logits_tst, y_true_tst, y_pred_tst])
                f1_neg_tst = f1_score(y_true_tst_value == 0, y_pred_tst_value == 0)
                f1_pos_tst = f1_score(y_true_tst_value == 2, y_pred_tst_value == 2)
                f1_avg_tst = (f1_neg_tst + f1_pos_tst) / 2
                format_str = ('[Test] %s: step %d, loss = %.4f, acc = %.4f, f1neg = %.4f, f1pos = %.4f, f1 = %.4f')
                print(format_str % (datetime.now(), step, loss_tst_value, accuracy_tst_value,
                                    f1_neg_tst, f1_pos_tst, f1_avg_tst))

            # Save the model checkpoint periodically.
            if step > 1 and step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
                checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
                saver.save(sess, checkpoint_path, global_step=step)

def main(argv=None):  # pylint: disable=unused-argument
    # cifar10.maybe_download_and_extract()
    if tf.gfile.Exists(FLAGS.train_dir):
        tf.gfile.DeleteRecursively(FLAGS.train_dir)
    tf.gfile.MakeDirs(FLAGS.train_dir)
    train()

if __name__ == '__main__':
    tf.app.run()
<commit_msg>xavier<commit_after>
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from datetime import datetime
import os.path
import re
import time

import numpy as np
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf
import cnnt_input
import cnn_model
from sklearn.metrics import precision_score, recall_score, f1_score

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('train_dir', './train/',
                           """Directory where to write event logs """
                           """and checkpoint.""")
tf.app.flags.DEFINE_integer('max_steps', 1000000,
                            """Number of batches to run.""")
tf.app.flags.DEFINE_integer('num_gpus', 4,
                            """How many GPUs to use.""")
tf.app.flags.DEFINE_boolean('log_device_placement', False,
                            """Whether to log device placement.""")
tf.app.flags.DEFINE_float('dropout_keep_prob', 0.8, """dropout_keep_prob""")

def tower_loss(namescope, target, batch_size=4):
    """Calculate the total loss on a single tower running the CIFAR model.

    Args:
    namescope: unique prefix string identifying the tweets tower, e.g. 'tower_0'

    Returns:
     Tensor of shape [] containing the total loss for a batch of data
    """
    # Get images and labels for tweets
    txts, labels = cnnt_input.get_inputs(target, batch_size=batch_size)

    # if target == 'trn':
    #     txts, labels = cnnt_input.get_inputs('trn', batch_size=batch_size)
    #
    # else:  # 'dev'
    #     txts, labels = cnnt_input.get_inputs('tst', batch_size=batch_size)

    # Build inference Graph.
    if target=='trn':
        logits = cnn_model.inference(txts, dropout_keep_prob=FLAGS.dropout_keep_prob)

    else:
        logits = cnn_model.inference(txts)

    y_true = tf.argmax(labels, 1, name="golds")
    y_pred = tf.argmax(logits, 1, name="predictions")

    # Build the portion of the Graph calculating the losses. Note that we will
    # assemble the total_loss using a custom function below.
    _, accuracy = cnn_model.loss(logits, labels)

    # Assemble all of the losses for the current tower only.
    losses = tf.get_collection('losses', namescope)

    # Calculate the total loss for the current tower.
    total_loss = tf.add_n(losses, name='total_loss')

    # Attach a scalar summary to all individual losses and the total loss; do the
    # same for the averaged version of the losses.
    for l in losses + [total_loss]:
        # Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
        # session. This helps the clarity of presentation on tensorboard.
        loss_name = re.sub('%s_[0-9]*/' % cnn_model.TOWER_NAME, '', l.op.name)
        tf.summary.scalar(loss_name, l)

    return total_loss, accuracy, logits, y_true, y_pred

def average_gradients(tower_grads):
    """Calculate the average gradient for each shared variable across all towers.

    Note that this function provides a synchronization point across all towers.

    Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
        is over individual gradients. The inner list is over the gradient
        calculation for each tower.
    Returns:
        List of pairs of (gradient, variable) where the gradient has been averaged
        across all towers.
    """
    average_grads = []
    for grad_and_vars in zip(*tower_grads):
        # Note that each grad_and_vars looks like the following:
        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
        grads = []
        for g, _ in grad_and_vars:
            # Add 0 dimension to the gradients to represent the tower.
            expanded_g = tf.expand_dims(g, 0)

            # Append on a 'tower' dimension which we will average over below.
            grads.append(expanded_g)

        # Average over the 'tower' dimension.
        grad = tf.concat(axis=0, values=grads)
        grad = tf.reduce_mean(grad, 0)

        # Keep in mind that the Variables are redundant because they are shared
        # across towers. So .. we will just return the first tower's pointer to
        # the Variable.
        v = grad_and_vars[0][1]
        grad_and_var = (grad, v)
        average_grads.append(grad_and_var)

    return average_grads

def train():
    """Train cnnt for a number of steps."""
    with tf.Graph().as_default(), tf.device('/cpu:0'):
        # Create a variable to count the number of train() calls. This equals the
        # number of batches processed * FLAGS.num_gpus.
        global_step = tf.get_variable(
            'global_step', [],
            initializer=tf.constant_initializer(0), trainable=False)

        # Calculate the learning rate schedule.
        # num_batches_per_epoch = (cnnt_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
        #                          FLAGS.batch_size)
        # decay_steps = int(num_batches_per_epoch * cnnt_input.NUM_EPOCHS_PER_DECAY)

        # Decay the learning rate exponentially based on the number of steps.
        # lr = tf.train.exponential_decay(cnnt_input.INITIAL_LEARNING_RATE,
        #                                 global_step,
        #                                 decay_steps,
        #                                 cnnt_input.LEARNING_RATE_DECAY_FACTOR,
        #                                 staircase=True)

        # Create an optimizer that performs gradient descent.
        # opt = tf.train.GradientDescentOptimizer(lr)
        opt = tf.train.AdamOptimizer(1e-3)

        # Calculate the gradients for each model tower.
        tower_grads = []

        with tf.variable_scope(tf.get_variable_scope()):
            with tf.device('/gpu:%d' % 0):
                with tf.name_scope('%s_%d_dev' % (cnn_model.TOWER_NAME, 0)) as namescope:
                    loss_dev, accuracy_dev, logits_dev, y_true_dev, y_pred_dev = \
                        tower_loss(namescope, 'dev', batch_size=1588)
                    # Reuse variables for the next tower.
                    tf.get_variable_scope().reuse_variables()

            with tf.device('/gpu:%d' % 1):
                with tf.name_scope('%s_%d_tst' % (cnn_model.TOWER_NAME, 0)) as namescope:
                    loss_tst, accuracy_tst, logits_tst, y_true_tst, y_pred_tst = \
                        tower_loss(namescope, 'tst', batch_size=20632)
                    # Reuse variables for the next tower.
                    tf.get_variable_scope().reuse_variables()

            for i in xrange(FLAGS.num_gpus):
                with tf.device('/gpu:%d' % i):
                    with tf.name_scope('%s_%d' % (cnn_model.TOWER_NAME, i)) as namescope:
                        # Calculate the loss for one tower of the CIFAR model. This function
                        # constructs the entire CIFAR model but shares the variables across
                        # all towers.
                        loss, accuracy, _, _, _ = tower_loss(namescope, 'trn', batch_size=FLAGS.batch_size)

                        # Reuse variables for the next tower.
                        tf.get_variable_scope().reuse_variables()

                        # Retain the summaries from the final tower.
                        summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, namescope)

                        # Calculate the gradients for the batch of data on this CIFAR tower.
                        grads = opt.compute_gradients(loss)

                        # Keep track of the gradients across all towers.
                        tower_grads.append(grads)

        # We must calculate the mean of each gradient. Note that this is the
        # synchronization point across all towers.
        grads = average_gradients(tower_grads)

        # # Add a summary to track the learning rate.
        # summaries.append(tf.summary.scalar('learning_rate', lr))

        # Add histograms for gradients.
        for grad, var in grads:
            if grad is not None:
                summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad))

        # Apply the gradients to adjust the shared variables.
        apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)

        # Add histograms for trainable variables.
        for var in tf.trainable_variables():
            summaries.append(tf.summary.histogram(var.op.name, var))

        # Track the moving averages of all trainable variables.
        variable_averages = tf.train.ExponentialMovingAverage(
            cnnt_input.MOVING_AVERAGE_DECAY, global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        # Group all updates to into a single train op.
        train_op = tf.group(apply_gradient_op, variables_averages_op)

        # Create a saver.
        saver = tf.train.Saver(tf.global_variables())

        # Build the summary operation from the last tower summaries.
        summary_op = tf.summary.merge(summaries)

        # Build an initialization operation to run below.
        init = tf.global_variables_initializer()

        # Start running operations on the Graph. allow_soft_placement must be set to
        # True to build towers on GPU, as some of the ops do not have GPU
        # implementations.
        # gpu_options = tf.GPUOptions(visible_device_list=str('0,1,2,3'), allow_growth=True)
        # gpu_options = tf.GPUOptions(visible_device_list=str('1,2,3'), allow_growth=True) # dead
        # gpu_options = tf.GPUOptions(visible_device_list=str('0,2,3'), allow_growth=True)  # dead
        gpu_options = tf.GPUOptions(visible_device_list=str('2,3'), allow_growth=True) # ok
        # gpu_options = tf.GPUOptions(visible_device_list=str('1,2'), allow_growth=True)  # dead
        sess = tf.Session(config=tf.ConfigProto(
            gpu_options=gpu_options,
            allow_soft_placement=True,
            log_device_placement=FLAGS.log_device_placement))
        sess.run(init)

        # Start the queue runners.
        tf.train.start_queue_runners(sess=sess)

        summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)

        for step in xrange(FLAGS.max_steps):
            start_time = time.time()
            # _, loss_value = sess.run([train_op, loss])
            _, loss_value, accuracy_val = sess.run([train_op, loss, accuracy])

            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            if step % 10 == 0:
                num_examples_per_step = FLAGS.batch_size * FLAGS.num_gpus
                examples_per_sec = num_examples_per_step / duration
                sec_per_batch = duration / FLAGS.num_gpus

                format_str = ('%s: step %d, loss = %.4f, acc = %.4f (%.1f examples/sec; %.3f '
                              'sec/batch)')
                print (format_str % (datetime.now(), step, loss_value, accuracy_val,
                                     examples_per_sec, sec_per_batch))

            if step % 100 == 0:
                summary_str = sess.run(summary_op)
                summary_writer.add_summary(summary_str, step)

                loss_dev_value, accuracy_dev_value, logits_dev_value, y_true_dev_value, y_pred_dev_value = \
                    sess.run([loss_dev, accuracy_dev, logits_dev, y_true_dev, y_pred_dev])

                # p_neg_dev = precision_score(y_true_dev_value==0, y_pred_dev_value==0)
                # r_neg_dev = recall_score(y_true_dev_value==0, y_pred_dev_value==0)
                f1_neg_dev = f1_score(y_true_dev_value==0, y_pred_dev_value==0)
                # p_pos_dev = precision_score(y_true_dev_value == 2, y_pred_dev_value == 2)
                # r_pos_dev = recall_score(y_true_dev_value == 2, y_pred_dev_value == 2)
                f1_pos_dev = f1_score(y_true_dev_value == 2, y_pred_dev_value == 2)
                f1_avg_dev = (f1_neg_dev+f1_pos_dev)/2

                format_str = ('[Eval] %s: step %d, loss = %.4f, acc = %.4f, f1neg = %.4f, f1pos = %.4f, f1 = %.4f')
                print(format_str % (datetime.now(), step, loss_dev_value, accuracy_dev_value,
                                    f1_neg_dev, f1_pos_dev, f1_avg_dev))

                loss_tst_value, accuracy_tst_value, logits_tst_value, y_true_tst_value, y_pred_tst_value = \
                    sess.run([loss_tst, accuracy_tst, logits_tst, y_true_tst, y_pred_tst])
                f1_neg_tst = f1_score(y_true_tst_value == 0, y_pred_tst_value == 0)
                f1_pos_tst = f1_score(y_true_tst_value == 2, y_pred_tst_value == 2)
                f1_avg_tst = (f1_neg_tst + f1_pos_tst) / 2
                format_str = ('[Test] %s: step %d, loss = %.4f, acc = %.4f, f1neg = %.4f, f1pos = %.4f, f1 = %.4f')
                print(format_str % (datetime.now(), step, loss_tst_value, accuracy_tst_value,
                                    f1_neg_tst, f1_pos_tst, f1_avg_tst))

            # Save the model checkpoint periodically.
            if step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
                checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
                saver.save(sess, checkpoint_path, global_step=step)

def main(argv=None):  # pylint: disable=unused-argument
    # cifar10.maybe_download_and_extract()
    if tf.gfile.Exists(FLAGS.train_dir):
        tf.gfile.DeleteRecursively(FLAGS.train_dir)
    tf.gfile.MakeDirs(FLAGS.train_dir)
    train()

if __name__ == '__main__':
    tf.app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import os
import shutil
from collections import OrderedDict

EXTRACT_TAGS = ["description", "brief_description", "member", "constant", "theme_item", "link"]
HEADER = """\
# LANGUAGE translation of the Godot Engine class reference.
# Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.
# Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).
# This file is distributed under the same license as the Godot source code.
#
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: Godot Engine class reference\\n"
"Report-Msgid-Bugs-To: https://github.com/godotengine/godot\\n"
"MIME-Version: 1.0\\n"
"Content-Type: text/plain; charset=UTF-8\\n"
"Content-Transfer-Encoding: 8-bit\\n"

"""
# Some strings used by make_rst.py are normally part of the editor translations,
# so we need to include them manually here for the online docs.
BASE_STRINGS = [
    "Description",
    "Tutorials",
    "Properties",
    "Constructors",
    "Methods",
    "Operators",
    "Theme Properties",
    "Signals",
    "Enumerations",
    "Constants",
    "Property Descriptions",
    "Constructor Descriptions",
    "Method Descriptions",
    "Operator Descriptions",
    "Theme Property Descriptions",
    "Inherits:",
    "Inherited By:",
    "(overrides %s)",
    "Default",
    "Setter",
    "value",
    "Getter",
    "This method should typically be overridden by the user to have any effect.",
    "This method has no side effects. It doesn't modify any of the instance's member variables.",
    "This method accepts any number of arguments after the ones described here.",
    "This method is used to construct a type.",
    "This method doesn't need an instance to be called, so it can be called directly using the class name.",
    "This method describes a valid operator to use with this type as left-hand operand.",
]

## <xml-line-number-hack from="https://stackoverflow.com/a/36430270/10846399">
import sys

sys.modules["_elementtree"] = None
import xml.etree.ElementTree as ET

## override the parser to get the line number
class LineNumberingParser(ET.XMLParser):
    def _start(self, *args, **kwargs):
        ## Here we assume the default XML parser which is expat
        ## and copy its element position attributes into output Elements
        element = super(self.__class__, self)._start(*args, **kwargs)
        element._start_line_number = self.parser.CurrentLineNumber
        element._start_column_number = self.parser.CurrentColumnNumber
        element._start_byte_index = self.parser.CurrentByteIndex
        return element

    def _end(self, *args, **kwargs):
        element = super(self.__class__, self)._end(*args, **kwargs)
        element._end_line_number = self.parser.CurrentLineNumber
        element._end_column_number = self.parser.CurrentColumnNumber
        element._end_byte_index = self.parser.CurrentByteIndex
        return element

## </xml-line-number-hack>

class Desc:
    def __init__(self, line_no, msg, desc_list=None):
        ## line_no   : the line number where the desc is
        ## msg       : the description string
        ## desc_list : the DescList it belongs to
        self.line_no = line_no
        self.msg = msg
        self.desc_list = desc_list

class DescList:
    def __init__(self, doc, path):
        ## doc  : root xml element of the document
        ## path : file path of the xml document
        ## list : list of Desc objects for this document
        self.doc = doc
        self.path = path
        self.list = []

def print_error(error):
    print("ERROR: {}".format(error))

## build classes with xml elements recursively
def _collect_classes_dir(path, classes):
    if not os.path.isdir(path):
        print_error("Invalid directory path: {}".format(path))
        exit(1)
    for _dir in map(lambda dir: os.path.join(path, dir), os.listdir(path)):
        if os.path.isdir(_dir):
            _collect_classes_dir(_dir, classes)
        elif os.path.isfile(_dir):
            if not _dir.endswith(".xml"):
                # print("Got non-.xml file '{}', skipping.".format(path))
                continue
            _collect_classes_file(_dir, classes)

## opens a file and parse xml add to classes
def _collect_classes_file(path, classes):
    if not os.path.isfile(path) or not path.endswith(".xml"):
        print_error("Invalid xml file path: {}".format(path))
        exit(1)
    print("Collecting file: {}".format(os.path.basename(path)))

    try:
        tree = ET.parse(path, parser=LineNumberingParser())
    except ET.ParseError as e:
        print_error("Parse error reading file '{}': {}".format(path, e))
        exit(1)

    doc = tree.getroot()

    if "name" in doc.attrib:
        if "version" not in doc.attrib:
            print_error("Version missing from 'doc', file: {}".format(path))

        name = doc.attrib["name"]
        if name in classes:
            print_error("Duplicate class {} at path {}".format(name, path))
            exit(1)
        classes[name] = DescList(doc, path)
    else:
        print_error("Unknown XML file {}, skipping".format(path))

## regions are list of tuples with size 3 (start_index, end_index, indent)
## indication in string where the codeblock starts, ends, and it's indent
## if i inside the region returns the indent, else returns -1
def _get_xml_indent(i, regions):
    for region in regions:
        if region[0] < i < region[1]:
            return region[2]
    return -1

## find and build all regions of codeblock which we need later
def _make_codeblock_regions(desc, path=""):
    code_block_end = False
    code_block_index = 0
    code_block_regions = []
    while not code_block_end:
        code_block_index = desc.find("[codeblock]", code_block_index)
        if code_block_index < 0:
            break
        xml_indent = 0
        while True:
            ## [codeblock] always have a trailing new line and some tabs
            ## those tabs are belongs to xml indentations not code indent
            if desc[code_block_index + len("[codeblock]\n") + xml_indent] == "\t":
                xml_indent += 1
            else:
                break
        end_index = desc.find("[/codeblock]", code_block_index)
        if end_index < 0:
            print_error("Non terminating codeblock: {}".format(path))
            exit(1)
        code_block_regions.append((code_block_index, end_index, xml_indent))
        code_block_index += 1
    return code_block_regions

def _strip_and_split_desc(desc, code_block_regions):
    desc_strip = ""  ## a stripped desc msg
    total_indent = 0  ## code indent = total indent - xml indent
    for i in range(len(desc)):
        c = desc[i]
        if c == "\n":
            c = "\\n"
        if c == '"':
            c = '\\"'
        if c == "\\":
            c = "\\\\"  ## <element \> is invalid for msgmerge
        if c == "\t":
            xml_indent = _get_xml_indent(i, code_block_regions)
            if xml_indent >= 0:
                total_indent += 1
                if xml_indent < total_indent:
                    c = "\\t"
                else:
                    continue
            else:
                continue
        desc_strip += c
        if c == "\\n":
            total_indent = 0
    return desc_strip

## make catalog strings from xml elements
def _make_translation_catalog(classes):
    unique_msgs = OrderedDict()
    for class_name in classes:
        desc_list = classes[class_name]
        for elem in desc_list.doc.iter():
            if elem.tag in EXTRACT_TAGS:
                if not elem.text or len(elem.text) == 0:
                    continue
                line_no = elem._start_line_number if elem.text[0] != "\n" else elem._start_line_number + 1
                desc_str = elem.text.strip()
                code_block_regions = _make_codeblock_regions(desc_str, desc_list.path)
                desc_msg = _strip_and_split_desc(desc_str, code_block_regions)
                desc_obj = Desc(line_no, desc_msg, desc_list)
                desc_list.list.append(desc_obj)

                if desc_msg not in unique_msgs:
                    unique_msgs[desc_msg] = [desc_obj]
                else:
                    unique_msgs[desc_msg].append(desc_obj)
    return unique_msgs

## generate the catalog file
def _generate_translation_catalog_file(unique_msgs, output, location_line=False):
    with open(output, "w", encoding="utf8") as f:
        f.write(HEADER)
        for msg in BASE_STRINGS:
            f.write("#: doc/tools/make_rst.py\n")
            f.write('msgid "{}"\n'.format(msg))
            f.write('msgstr ""\n\n')
        for msg in unique_msgs:
            if len(msg) == 0 or msg in BASE_STRINGS:
                continue

            f.write("#:")
            desc_list = unique_msgs[msg]
            for desc in desc_list:
                path = desc.desc_list.path.replace("\\", "/")
                if path.startswith("./"):
                    path = path[2:]
                if location_line:  # Can be skipped as diffs on line numbers are spammy.
                    f.write(" {}:{}".format(path, desc.line_no))
                else:
                    f.write(" {}".format(path))
            f.write("\n")

            f.write('msgid "{}"\n'.format(msg))
            f.write('msgstr ""\n\n')

    ## TODO: what if 'nt'?
    if os.name == "posix":
        print("Wrapping template at 79 characters for compatibility with Weblate.")
        os.system("msgmerge -w79 {0} {0} > {0}.wrap".format(output))
        shutil.move("{}.wrap".format(output), output)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--path", "-p", nargs="+", default=".", help="The directory or directories containing XML files to collect."
    )
    parser.add_argument("--output", "-o", default="translation_catalog.pot", help="The path to the output file.")
    args = parser.parse_args()

    output = os.path.abspath(args.output)
    if not os.path.isdir(os.path.dirname(output)) or not output.endswith(".pot"):
        print_error("Invalid output path: {}".format(output))
        exit(1)

    classes = OrderedDict()
    for path in args.path:
        if not os.path.isdir(path):
            print_error("Invalid working directory path: {}".format(path))
            exit(1)

        print("\nCurrent working dir: {}".format(path))

        path_classes = OrderedDict()  ## dictionary of key=class_name, value=DescList objects
        _collect_classes_dir(path, path_classes)
        classes.update(path_classes)

    classes = OrderedDict(sorted(classes.items(), key=lambda kv: kv[0].lower()))
    unique_msgs = _make_translation_catalog(classes)
    _generate_translation_catalog_file(unique_msgs, output)

if __name__ == "__main__":
    main()
<commit_msg>i18n: Exclude `$DOCS_URL` links from classref translations<commit_after>#!/usr/bin/env python3

import argparse
import os
import shutil
from collections import OrderedDict

EXTRACT_TAGS = ["description", "brief_description", "member", "constant", "theme_item", "link"]
HEADER = """\
# LANGUAGE translation of the Godot Engine class reference.
# Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.
# Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).
# This file is distributed under the same license as the Godot source code.
#
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: Godot Engine class reference\\n"
"Report-Msgid-Bugs-To: https://github.com/godotengine/godot\\n"
"MIME-Version: 1.0\\n"
"Content-Type: text/plain; charset=UTF-8\\n"
"Content-Transfer-Encoding: 8-bit\\n"

"""
# Some strings used by make_rst.py are normally part of the editor translations,
# so we need to include them manually here for the online docs.
BASE_STRINGS = [
    "Description",
    "Tutorials",
    "Properties",
    "Constructors",
    "Methods",
    "Operators",
    "Theme Properties",
    "Signals",
    "Enumerations",
    "Constants",
    "Property Descriptions",
    "Constructor Descriptions",
    "Method Descriptions",
    "Operator Descriptions",
    "Theme Property Descriptions",
    "Inherits:",
    "Inherited By:",
    "(overrides %s)",
    "Default",
    "Setter",
    "value",
    "Getter",
    "This method should typically be overridden by the user to have any effect.",
    "This method has no side effects. It doesn't modify any of the instance's member variables.",
    "This method accepts any number of arguments after the ones described here.",
    "This method is used to construct a type.",
    "This method doesn't need an instance to be called, so it can be called directly using the class name.",
    "This method describes a valid operator to use with this type as left-hand operand.",
]

## <xml-line-number-hack from="https://stackoverflow.com/a/36430270/10846399">
import sys

sys.modules["_elementtree"] = None
import xml.etree.ElementTree as ET

## override the parser to get the line number
class LineNumberingParser(ET.XMLParser):
    def _start(self, *args, **kwargs):
        ## Here we assume the default XML parser which is expat
        ## and copy its element position attributes into output Elements
        element = super(self.__class__, self)._start(*args, **kwargs)
        element._start_line_number = self.parser.CurrentLineNumber
        element._start_column_number = self.parser.CurrentColumnNumber
        element._start_byte_index = self.parser.CurrentByteIndex
        return element

    def _end(self, *args, **kwargs):
        element = super(self.__class__, self)._end(*args, **kwargs)
        element._end_line_number = self.parser.CurrentLineNumber
        element._end_column_number = self.parser.CurrentColumnNumber
        element._end_byte_index = self.parser.CurrentByteIndex
        return element

## </xml-line-number-hack>

class Desc:
    def __init__(self, line_no, msg, desc_list=None):
        ## line_no   : the line number where the desc is
        ## msg       : the description string
        ## desc_list : the DescList it belongs to
        self.line_no = line_no
        self.msg = msg
        self.desc_list = desc_list

class DescList:
    def __init__(self, doc, path):
        ## doc  : root xml element of the document
        ## path : file path of the xml document
        ## list : list of Desc objects for this document
        self.doc = doc
        self.path = path
        self.list = []

def print_error(error):
    print("ERROR: {}".format(error))

## build classes with xml elements recursively
def _collect_classes_dir(path, classes):
    if not os.path.isdir(path):
        print_error("Invalid directory path: {}".format(path))
        exit(1)
    for _dir in map(lambda dir: os.path.join(path, dir), os.listdir(path)):
        if os.path.isdir(_dir):
            _collect_classes_dir(_dir, classes)
        elif os.path.isfile(_dir):
            if not _dir.endswith(".xml"):
                # print("Got non-.xml file '{}', skipping.".format(path))
                continue
            _collect_classes_file(_dir, classes)

## opens a file and parse xml add to classes
def _collect_classes_file(path, classes):
    if not os.path.isfile(path) or not path.endswith(".xml"):
        print_error("Invalid xml file path: {}".format(path))
        exit(1)
    print("Collecting file: {}".format(os.path.basename(path)))

    try:
        tree = ET.parse(path, parser=LineNumberingParser())
    except ET.ParseError as e:
        print_error("Parse error reading file '{}': {}".format(path, e))
        exit(1)

    doc = tree.getroot()

    if "name" in doc.attrib:
        if "version" not in doc.attrib:
            print_error("Version missing from 'doc', file: {}".format(path))

        name = doc.attrib["name"]
        if name in classes:
            print_error("Duplicate class {} at path {}".format(name, path))
            exit(1)
        classes[name] = DescList(doc, path)
    else:
        print_error("Unknown XML file {}, skipping".format(path))

## regions are list of tuples with size 3 (start_index, end_index, indent)
## indication in string where the codeblock starts, ends, and it's indent
## if i inside the region returns the indent, else returns -1
def _get_xml_indent(i, regions):
    for region in regions:
        if region[0] < i < region[1]:
            return region[2]
    return -1

## find and build all regions of codeblock which we need later
def _make_codeblock_regions(desc, path=""):
    code_block_end = False
    code_block_index = 0
    code_block_regions = []
    while not code_block_end:
        code_block_index = desc.find("[codeblock]", code_block_index)
        if code_block_index < 0:
            break
        xml_indent = 0
        while True:
            ## [codeblock] always have a trailing new line and some tabs
            ## those tabs are belongs to xml indentations not code indent
            if desc[code_block_index + len("[codeblock]\n") + xml_indent] == "\t":
                xml_indent += 1
            else:
                break
        end_index = desc.find("[/codeblock]", code_block_index)
        if end_index < 0:
            print_error("Non terminating codeblock: {}".format(path))
            exit(1)
        code_block_regions.append((code_block_index, end_index, xml_indent))
        code_block_index += 1
    return code_block_regions

def _strip_and_split_desc(desc, code_block_regions):
    desc_strip = ""  ## a stripped desc msg
    total_indent = 0  ## code indent = total indent - xml indent
    for i in range(len(desc)):
        c = desc[i]
        if c == "\n":
            c = "\\n"
        if c == '"':
            c = '\\"'
        if c == "\\":
            c = "\\\\"  ## <element \> is invalid for msgmerge
        if c == "\t":
            xml_indent = _get_xml_indent(i, code_block_regions)
            if xml_indent >= 0:
                total_indent += 1
                if xml_indent < total_indent:
                    c = "\\t"
                else:
                    continue
            else:
                continue
        desc_strip += c
        if c == "\\n":
            total_indent = 0
    return desc_strip

## make catalog strings from xml elements
def _make_translation_catalog(classes):
    unique_msgs = OrderedDict()
    for class_name in classes:
        desc_list = classes[class_name]
        for elem in desc_list.doc.iter():
            if elem.tag in EXTRACT_TAGS:
                if not elem.text or len(elem.text) == 0:
                    continue
                if elem.tag == "link" and "$DOCS_URL" in elem.text:  # No need to localize.
                    continue
                line_no = elem._start_line_number if elem.text[0] != "\n" else elem._start_line_number + 1
                desc_str = elem.text.strip()
                code_block_regions = _make_codeblock_regions(desc_str, desc_list.path)
                desc_msg = _strip_and_split_desc(desc_str, code_block_regions)
                desc_obj = Desc(line_no, desc_msg, desc_list)
                desc_list.list.append(desc_obj)

                if desc_msg not in unique_msgs:
                    unique_msgs[desc_msg] = [desc_obj]
                else:
                    unique_msgs[desc_msg].append(desc_obj)
    return unique_msgs

## generate the catalog file
def _generate_translation_catalog_file(unique_msgs, output, location_line=False):
    with open(output, "w", encoding="utf8") as f:
        f.write(HEADER)
        for msg in BASE_STRINGS:
            f.write("#: doc/tools/make_rst.py\n")
            f.write('msgid "{}"\n'.format(msg))
            f.write('msgstr ""\n\n')
        for msg in unique_msgs:
            if len(msg) == 0 or msg in BASE_STRINGS:
                continue

            f.write("#:")
            desc_list = unique_msgs[msg]
            for desc in desc_list:
                path = desc.desc_list.path.replace("\\", "/")
                if path.startswith("./"):
                    path = path[2:]
                if location_line:  # Can be skipped as diffs on line numbers are spammy.
                    f.write(" {}:{}".format(path, desc.line_no))
                else:
                    f.write(" {}".format(path))
            f.write("\n")

            f.write('msgid "{}"\n'.format(msg))
            f.write('msgstr ""\n\n')

    ## TODO: what if 'nt'?
    if os.name == "posix":
        print("Wrapping template at 79 characters for compatibility with Weblate.")
        os.system("msgmerge -w79 {0} {0} > {0}.wrap".format(output))
        shutil.move("{}.wrap".format(output), output)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--path", "-p", nargs="+", default=".", help="The directory or directories containing XML files to collect."
    )
    parser.add_argument("--output", "-o", default="translation_catalog.pot", help="The path to the output file.")
    args = parser.parse_args()

    output = os.path.abspath(args.output)
    if not os.path.isdir(os.path.dirname(output)) or not output.endswith(".pot"):
        print_error("Invalid output path: {}".format(output))
        exit(1)

    classes = OrderedDict()
    for path in args.path:
        if not os.path.isdir(path):
            print_error("Invalid working directory path: {}".format(path))
            exit(1)

        print("\nCurrent working dir: {}".format(path))

        path_classes = OrderedDict()  ## dictionary of key=class_name, value=DescList objects
        _collect_classes_dir(path, path_classes)
        classes.update(path_classes)

    classes = OrderedDict(sorted(classes.items(), key=lambda kv: kv[0].lower()))
    unique_msgs = _make_translation_catalog(classes)
    _generate_translation_catalog_file(unique_msgs, output)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (C) 2008-2012 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Netius System. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2012 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import math
import uuid
import time
import types
import struct
import hashlib

import netius.common
import netius.clients

PIECE_SIZE = 16384
""" The typical size of piece that is going to be retrieved
using the current torrent infra-structure, this value conditions
most of the torrent operations and should be defined carefully """

class TorrentTask(netius.Observable):
    """
    Describes a task (operation) that is going to be performed
    using the peer to peer mesh network of the torrent protocol.

    Each of the download operations should be able to be described
    by this task object (for latter reference).
    """

    def __init__(self, owner, target_path, torrent_path = None, info_hash = None):
        netius.Observable.__init__(self)

        self.owner = owner
        self.target_path = target_path
        self.start = time.time()
        self.uploaded = 0
        self.downloaded = 0
        self.left = 0
        self.peers = []

        if torrent_path: self.info = self.load_info(torrent_path)
        else: self.info = dict(info_hash = info_hash)

        self.pieces_tracker()
        self.peers_tracker()

        self.load_file()
        self.load_pieces()

    def load_info(self, torrent_path):
        file = open(torrent_path, "rb")
        try: data = file.read()
        finally: file.close()

        struct = netius.common.bdecode(data)
        struct["info_hash"] = netius.common.info_hash(struct)
        return struct

    def load_file(self):
        self.file = open(self.target_path, "wb")
        self.file.seek(780140544 - 1)
        self.file.write("\0")
        self.file.flush()

    def load_pieces(self):
        number_pieces = self.info["number_pieces"]
        number_parts = self.info["number_parts"]
        self.bitfield = [True for _index in xrange(number_pieces)]
        self.mask = [True for _index in xrange(number_pieces * number_parts)]

    def pieces_tracker(self):
        info = self.info.get("info", {})
        pieces = info.get("pieces", "")
        piece_length = info.get("piece length", 1)
        number_parts = math.ceil(float(piece_length) / float(PIECE_SIZE))
        number_parts = int(number_parts)
        self.info["pieces"] = [piece for piece in netius.common.chunks(pieces, 20)]
        self.info["number_pieces"] = len(self.info["pieces"])
        self.info["number_parts"] = number_parts

    def set_data(self, data, index, begin):
        piece_length = self.info["info"]["piece length"]
        self.file.seek(index * piece_length + begin)
        self.file.write(data)
        self.file.flush()
        self.downloaded += len(data)
        self.trigger("part", self, data, index, begin)

    def peers_tracker(self):
        """
        Tries to retrieve as much information as possible about the
        peers from the currently loaded tracker information.

        It's possible that no tracker information exits for the current
        task and for such situations no state change will occur.
        """

        announce = self.info.get("announce", None)
        announce_list = self.info.get("announce-list", [[announce]])

        for tracker in announce_list:
            tracker_url = tracker[0]
            result = netius.clients.HTTPClient.get_s(
                tracker_url,
                params = dict(
                    info_hash = self.info["info_hash"],
                    peer_id = self.owner.peer_id,
                    port = "1000",
                    uploaded = self.uploaded,
                    downloaded = self.downloaded,
                    left = self.left,
                    compact = "0"
                ),
                async = False
            )

            data = result["data"]
            if not data: continue

            response = netius.common.bdecode(data)
            peers = response["peers"]

            if type(peers) == types.DictType:
                self.peers = peers
                continue

            peers = [peer for peer in netius.common.chunks(peers, 6)]
            for peer in peers:
                address, port = struct.unpack("!LH", peer)
                ip = netius.common.addr_to_ip4(address)
                peer = dict(
                    ip = ip,
                    port = port
                )
                self.peers.append(peer)

    def connect_peers(self):
        for peer in self.peers: self.connect_peer(peer)

    def connect_peer(self, peer):
        self.owner.client.peer(self, peer["ip"], peer["port"])

    #@todo: must change this !!! to a different place
    def _and(self, first, second):
        result = []
        for _first, _second in zip(first, second):
            if _first and _second: value = True
            else: value = False
            result.append(value)
        return result

    def speed(self):
        """
        Retrieves a float number representing the global speed
        of the task in bytes per second, this value is computed
        using the original creation time of the task and so it
        may not represent the most accurate speedup.

        @rtype: float
        @return: The current speed of download, defined as bytes
        per second from the original task creation time.
        """

        current = time.time()
        delta = current - self.start
        bytes_second = self.downloaded / delta
        return bytes_second

    def pop_piece(self, bitfield):
        index = 0
        result = self._and(bitfield, self.bitfield)
        for bit in result:
            if bit == True: break
            index += 1

        if index == len(result): return None

        begin = self.pop_part(index)
        self.update_piece(index)

        return (index, begin)

    def pop_part(self, index):
        number_parts = self.info["number_parts"]
        base = index * number_parts

        for part_index in xrange(number_parts):
            state = self.mask[base + part_index]
            if state == True: break

        self.mask[base + part_index] = False
        return part_index * PIECE_SIZE

    def update_piece(self, index):
        number_parts = self.info["number_parts"]
        base = index * number_parts
        piece_state = False

        for part_index in xrange(number_parts):
            state = self.mask[base + part_index]
            if state == False: continue
            piece_state = True
            break

        self.bitfield[index] = piece_state

    def verify_piece(self, index):
        file = open(self.target_path, "rb")
        try: self._verify_piece(index, file)
        finally: file.close()

    def _verify_piece(self, index, file):
        piece = self.info["pieces"][index]
        piece_length = self.info["info"]["piece length"]
        file = open(self.target_path, "rb")
        file.seek(index * piece_length)
        pending = piece_length
        hash = hashlib.sha1()
        while True:
            if pending == 0: break
            count = PIECE_SIZE if pending > PIECE_SIZE else pending
            data = file.read(count)
            hash.update(data)
            pending -= count
        digest = hash.digest()
        if not digest ==  piece:
            raise RuntimeError("error verifying piece index '%d'" % index)

class TorrentServer(netius.StreamServer):

    def __init__(self, *args, **kwargs):
        netius.StreamServer.__init__(self, *args, **kwargs)
        self.peer_id = self._generate_id()
        self.client = netius.clients.TorrentClient()

    def download(self, target_path, torrent_path = None, info_hash = None):
        """
        Starts the "downloading" process of a torrent associated file
        using the defined peer to peer torrent strategy suing either
        the provided torrent path as reference or just the info hash
        of the file that is going to be downloaded.

        Note that if only the info hash is provided a DHT bases strategy
        is going to be used to retrieve the peers list.

        The returned value is the task entity representing the newly created
        task for the downloading of the requested file, this object may be
        used for the operations and listening of events.

        @type target_path: String
        @param target_path: The path to the file that will be used to store
        the binary information resulting from the download, this file may also
        be used to store some temporary information on state of download.
        @type torrent_path: String
        @param torrent_path: The path to the file that contains the torrent
        information that is going to be used for file processing.
        @type info_hash: String
        @param info_hash: The info hash value of the file that is going
        to be downloaded, may be used for magnet torrents (DHT).
        @rtype: TorrentTask
        @return: The torrent task object that represents the task that has been
        created for downloading of the requested file.
        """

        task = TorrentTask(
            self,
            target_path,
            torrent_path = torrent_path,
            info_hash = info_hash
        )
        task.connect_peers()
        return task

    def _generate_id(self):
        random = str(uuid.uuid4())
        hash = hashlib.sha1(random)
        digest = hash.hexdigest()
        id = "-NE1000-%s" % digest[:12]
        return id

if __name__ == "__main__":
    def handler(task, data, index, begin):
        speed = task.speed()
        speed_s = netius.common.size_round_unit(speed)
        print "%s/s" % speed_s

    torrent_server = TorrentServer()
    task = torrent_server.download("C:/tobias.download", "C:\ubuntu.torrent")
    task.bind("part", handler)
    torrent_server.serve(env = True)
<commit_msg>renamed part to block (as specification)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (C) 2008-2012 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Netius System. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2012 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import math
import uuid
import time
import types
import struct
import hashlib

import netius.common
import netius.clients

PIECE_SIZE = 16384
""" The typical size of piece that is going to be retrieved
using the current torrent infra-structure, this value conditions
most of the torrent operations and should be defined carefully """

class TorrentTask(netius.Observable):
    """
    Describes a task (operation) that is going to be performed
    using the peer to peer mesh network of the torrent protocol.

    Each of the download operations should be able to be described
    by this task object (for latter reference).
    """

    def __init__(self, owner, target_path, torrent_path = None, info_hash = None):
        netius.Observable.__init__(self)

        self.owner = owner
        self.target_path = target_path
        self.start = time.time()
        self.uploaded = 0
        self.downloaded = 0
        self.left = 0
        self.peers = []

        if torrent_path: self.info = self.load_info(torrent_path)
        else: self.info = dict(info_hash = info_hash)

        self.pieces_tracker()
        self.peers_tracker()

        self.load_file()
        self.load_pieces()

    def load_info(self, torrent_path):
        file = open(torrent_path, "rb")
        try: data = file.read()
        finally: file.close()

        struct = netius.common.bdecode(data)
        struct["info_hash"] = netius.common.info_hash(struct)
        return struct

    def load_file(self):
        self.file = open(self.target_path, "wb")
        self.file.seek(780140544 - 1)
        self.file.write("\0")
        self.file.flush()

    def load_pieces(self):
        number_pieces = self.info["number_pieces"]
        number_blocks = self.info["number_blocks"]
        self.bitfield = [True for _index in xrange(number_pieces)]
        self.mask = [True for _index in xrange(number_pieces * number_blocks)]

    def pieces_tracker(self):
        info = self.info.get("info", {})
        pieces = info.get("pieces", "")
        piece_length = info.get("piece length", 1)
        number_blocks = math.ceil(float(piece_length) / float(PIECE_SIZE))
        number_blocks = int(number_blocks)
        self.info["pieces"] = [piece for piece in netius.common.chunks(pieces, 20)]
        self.info["number_pieces"] = len(self.info["pieces"])
        self.info["number_blocks"] = number_blocks

    def set_data(self, data, index, begin):
        piece_length = self.info["info"]["piece length"]
        self.file.seek(index * piece_length + begin)
        self.file.write(data)
        self.file.flush()
        self.downloaded += len(data)
        self.trigger("block", self, data, index, begin)

    def peers_tracker(self):
        """
        Tries to retrieve as much information as possible about the
        peers from the currently loaded tracker information.

        It's possible that no tracker information exits for the current
        task and for such situations no state change will occur.
        """

        announce = self.info.get("announce", None)
        announce_list = self.info.get("announce-list", [[announce]])

        for tracker in announce_list:
            tracker_url = tracker[0]
            result = netius.clients.HTTPClient.get_s(
                tracker_url,
                params = dict(
                    info_hash = self.info["info_hash"],
                    peer_id = self.owner.peer_id,
                    port = "1000",
                    uploaded = self.uploaded,
                    downloaded = self.downloaded,
                    left = self.left,
                    compact = "0"
                ),
                async = False
            )

            data = result["data"]
            if not data: continue

            response = netius.common.bdecode(data)
            peers = response["peers"]

            if type(peers) == types.DictType:
                self.peers = peers
                continue

            peers = [peer for peer in netius.common.chunks(peers, 6)]
            for peer in peers:
                address, port = struct.unpack("!LH", peer)
                ip = netius.common.addr_to_ip4(address)
                peer = dict(
                    ip = ip,
                    port = port
                )
                self.peers.append(peer)

    def connect_peers(self):
        for peer in self.peers: self.connect_peer(peer)

    def connect_peer(self, peer):
        self.owner.client.peer(self, peer["ip"], peer["port"])

    #@todo: must change this !!! to a different place
    def _and(self, first, second):
        result = []
        for _first, _second in zip(first, second):
            if _first and _second: value = True
            else: value = False
            result.append(value)
        return result

    def speed(self):
        """
        Retrieves a float number representing the global speed
        of the task in bytes per second, this value is computed
        using the original creation time of the task and so it
        may not represent the most accurate speedup.

        @rtype: float
        @return: The current speed of download, defined as bytes
        per second from the original task creation time.
        """

        current = time.time()
        delta = current - self.start
        bytes_second = self.downloaded / delta
        return bytes_second

    def pop_piece(self, bitfield):
        index = 0
        result = self._and(bitfield, self.bitfield)
        for bit in result:
            if bit == True: break
            index += 1

        if index == len(result): return None

        begin = self.pop_block(index)
        self.update_piece(index)

        return (index, begin)

    def pop_block(self, index):
        number_blocks = self.info["number_blocks"]
        base = index * number_blocks

        for block_index in xrange(number_blocks):
            state = self.mask[base + block_index]
            if state == True: break

        self.mask[base + block_index] = False
        return block_index * PIECE_SIZE

    def update_piece(self, index):
        number_blocks = self.info["number_blocks"]
        base = index * number_blocks
        piece_state = False

        for block_index in xrange(number_blocks):
            state = self.mask[base + block_index]
            if state == False: continue
            piece_state = True
            break

        self.bitfield[index] = piece_state

    def verify_piece(self, index):
        file = open(self.target_path, "rb")
        try: self._verify_piece(index, file)
        finally: file.close()

    def _verify_piece(self, index, file):
        piece = self.info["pieces"][index]
        piece_length = self.info["info"]["piece length"]
        file = open(self.target_path, "rb")
        file.seek(index * piece_length)
        pending = piece_length
        hash = hashlib.sha1()
        while True:
            if pending == 0: break
            count = PIECE_SIZE if pending > PIECE_SIZE else pending
            data = file.read(count)
            hash.update(data)
            pending -= count
        digest = hash.digest()
        if not digest ==  piece:
            raise RuntimeError("error verifying piece index '%d'" % index)

class TorrentServer(netius.StreamServer):

    def __init__(self, *args, **kwargs):
        netius.StreamServer.__init__(self, *args, **kwargs)
        self.peer_id = self._generate_id()
        self.client = netius.clients.TorrentClient()

    def download(self, target_path, torrent_path = None, info_hash = None):
        """
        Starts the "downloading" process of a torrent associated file
        using the defined peer to peer torrent strategy suing either
        the provided torrent path as reference or just the info hash
        of the file that is going to be downloaded.

        Note that if only the info hash is provided a DHT bases strategy
        is going to be used to retrieve the peers list.

        The returned value is the task entity representing the newly created
        task for the downloading of the requested file, this object may be
        used for the operations and listening of events.

        @type target_path: String
        @param target_path: The path to the file that will be used to store
        the binary information resulting from the download, this file may also
        be used to store some temporary information on state of download.
        @type torrent_path: String
        @param torrent_path: The path to the file that contains the torrent
        information that is going to be used for file processing.
        @type info_hash: String
        @param info_hash: The info hash value of the file that is going
        to be downloaded, may be used for magnet torrents (DHT).
        @rtype: TorrentTask
        @return: The torrent task object that represents the task that has been
        created for downloading of the requested file.
        """

        task = TorrentTask(
            self,
            target_path,
            torrent_path = torrent_path,
            info_hash = info_hash
        )
        task.connect_peers()
        return task

    def _generate_id(self):
        random = str(uuid.uuid4())
        hash = hashlib.sha1(random)
        digest = hash.hexdigest()
        id = "-NE1000-%s" % digest[:12]
        return id

if __name__ == "__main__":
    def handler(task, data, index, begin):
        speed = task.speed()
        speed_s = netius.common.size_round_unit(speed)
        print "%s/s" % speed_s

    torrent_server = TorrentServer()
    task = torrent_server.download("C:/tobias.download", "C:\ubuntu.torrent")
    task.bind("block", handler)
    torrent_server.serve(env = True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''Base TextCase class for OSF unittests. Uses a temporary MongoDB database.'''
import os
import unittest

from pymongo import MongoClient

from framework import storage, set_up_storage
from framework.auth.model import User
from framework.sessions.model import Session
from framework.search.model import Keyword
from website.project.model import (ApiKey, Node, NodeLog, NodeFile, NodeWikiPage,
                                   Tag, WatchConfig)

from new_style import app

# All Models
MODELS = (User, ApiKey, Keyword, Node, NodeLog, NodeFile, NodeWikiPage,
          Tag, WatchConfig, Session)

class DbTestCase(unittest.TestCase):
    '''Base TestCase for tests that require a temporary MongoDB database.
    '''
    # DB settings
    db_name = os.environ.get('MONGO_DATABASE', 'osf_test')
    db_host = os.environ.get('MONGO_HOST', 'localhost')
    db_port = int(os.environ.get('MONGO_PORT', '20771'))

    @classmethod
    def setUpClass(klass):
        '''Before running this TestCase, set up a temporary MongoDB database'''
        klass._client = MongoClient(host=klass.db_host, port=klass.db_port)
        klass.db = klass._client[klass.db_name]
        # Set storage backend to MongoDb
        set_up_storage(MODELS, storage.MongoStorage, db=klass.db)

    @classmethod
    def tearDownClass(klass):
        '''Drop the database when all tests finish.'''
        klass._client.drop_database(klass.db)

class AppTestCase(unittest.TestCase):
    '''Base TestCase for OSF tests that require the WSGI app (but no database).
    '''

    def setUp(self):
        self.app = app
        self.ctx = app.app_context()
        self.ctx.push()

    def tearDown(self):
        self.ctx.pop()
<commit_msg>Use list of all models in website.models.MODELS<commit_after># -*- coding: utf-8 -*-
'''Base TextCase class for OSF unittests. Uses a temporary MongoDB database.'''
import os
import unittest

from pymongo import MongoClient

from framework import storage, set_up_storage
import website.models

from new_style import app

class DbTestCase(unittest.TestCase):
    '''Base TestCase for tests that require a temporary MongoDB database.
    '''
    # DB settings
    db_name = os.environ.get('MONGO_DATABASE', 'osf_test')
    db_host = os.environ.get('MONGO_HOST', 'localhost')
    db_port = int(os.environ.get('MONGO_PORT', '20771'))

    @classmethod
    def setUpClass(klass):
        '''Before running this TestCase, set up a temporary MongoDB database'''
        klass._client = MongoClient(host=klass.db_host, port=klass.db_port)
        klass.db = klass._client[klass.db_name]
        # Set storage backend to MongoDb
        set_up_storage(website.models.MODELS, storage.MongoStorage, db=klass.db)

    @classmethod
    def tearDownClass(klass):
        '''Drop the database when all tests finish.'''
        klass._client.drop_database(klass.db)

class AppTestCase(unittest.TestCase):
    '''Base TestCase for OSF tests that require the WSGI app (but no database).
    '''

    def setUp(self):
        self.app = app
        self.ctx = app.app_context()
        self.ctx.push()

    def tearDown(self):
        self.ctx.pop()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from nose.plugins.attrib import attr

from openquake.baselib.general import writetmp
from openquake.calculators.views import view
from openquake.calculators.tests import CalculatorTestCase, strip_calc_id
from openquake.commonlib.export import export
from openquake.calculators.tests import check_platform
from openquake.qa_tests_data.event_based_risk import (
    case_1, case_2, case_3, case_4, case_4a, case_master, case_miriam,
    occupants)

class EventBasedRiskTestCase(CalculatorTestCase):

    def assert_stats_ok(self, pkg, job_ini, individual_curves='false'):
        out = self.run_calc(pkg.__file__, job_ini, exports='csv',
                            individual_curves=individual_curves,
                            concurrent_tasks='4')
        # NB: it is important to use concurrent_tasks > 1 to test the
        # complications of concurrency (for instance the noncommutativity of
        # numpy.float32 addition when computing the average losses)
        all_csv = []
        for fnames in out.values():
            for fname in fnames:
                if 'rlz' in fname and individual_curves == 'false':
                    continue
                elif fname.endswith('.csv') and any(x in fname for x in (
                        'loss_curve', 'loss_map', 'agg_loss', 'avg_loss')):
                    all_csv.append(fname)
        assert all_csv, 'Could not find any CSV file??'
        for fname in all_csv:
            self.assertEqualFiles(
                'expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_1(self):
        check_platform('xenial')  # FIXME: remove this line?
        self.assert_stats_ok(case_1, 'job.ini', individual_curves='true')

        # make sure the XML and JSON exporters run
        ekeys = [
            ('loss_curves-stats', 'xml'),
            ('loss_curves-stats', 'geojson'),
            ('rcurves-rlzs', 'xml'),
            ('rcurves-rlzs', 'geojson'),

            ('loss_maps-stats', 'xml'),
            ('loss_maps-stats', 'geojson'),
            ('loss_maps-rlzs', 'xml'),
            ('loss_maps-rlzs', 'geojson'),

            ('agg_curve-stats', 'xml'),
        ]
        for ekey in ekeys:
            export(ekey, self.calc.datastore)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2(self):
        self.assert_stats_ok(case_2, 'job.ini', individual_curves='true')
        fname = writetmp(view('mean_avg_losses', self.calc.datastore))
        self.assertEqualFiles('expected/mean_avg_losses.txt', fname)

        # test the composite_risk_model keys (i.e. slash escaping)
        crm = sorted(self.calc.datastore.getitem('composite_risk_model'))
        self.assertEqual(crm, ['RC%2B', 'RM', 'W%2F1'])
        # export a specific eid
        [fname] = export(('ass_loss_ratios:0', 'csv'), self.calc.datastore)
        self.assertEqualFiles('expected/losses-eid=0.csv', fname)

        # test the case when all GMFs are filtered out
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job.ini', minimum_intensity='10.0')
        self.assertEqual(
            str(ctx.exception),
            'No GMFs were generated, perhaps they were all below the '
            'minimum_intensity threshold')

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2_correlation(self):
        out = self.run_calc(case_2.__file__, 'job_loss.ini', exports='csv',
                            asset_correlation=1.0)
        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses.csv', fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_missing_taxonomy(self):
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job_err.ini')
        self.assertIn('not in the risk model', str(ctx.exception))

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_3(self):
        # this is a test with statistics and without conditional_loss_poes
        self.run_calc(case_3.__file__, 'job.ini',
                      exports='xml', individual_curves='false',
                      concurrent_tasks='4')
        [fname] = export(('agg_curve-stats', 'xml'), self.calc.datastore)
        self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_4(self):
        # Turkey with SHARE logic tree
        out = self.run_calc(case_4.__file__, 'job.ini',
                            exports='csv', individual_curves='true')
        [fname] = export(('avg_losses-stats', 'csv'), self.calc.datastore)
        self.assertEqualFiles('expected/avg_losses-mean.csv', fname)

        fnames = out['agg_loss_table', 'csv']
        assert fnames, 'No agg_losses exported??'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_occupants(self):
        out = self.run_calc(occupants.__file__, 'job.ini',
                            exports='xml', individual_curves='true')
        fnames = out['loss_maps-rlzs', 'xml'] + out['agg_curve-rlzs', 'xml']
        self.assertEqual(len(fnames), 3)  # 2 loss_maps + 1 agg_curve
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_master(self):
        self.assert_stats_ok(case_master, 'job.ini', individual_curves='true')

        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles('expected/portfolio_loss.txt', fname, delta=1E-5)

        # check rup_data is stored correctly
        fname = writetmp(view('ruptures_events', self.calc.datastore))
        self.assertEqualFiles('expected/ruptures_events.txt', fname)

        # export a specific eid
        fnames = export(('ass_loss_ratios:0', 'csv'), self.calc.datastore)
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)
        self.assertEqualFiles('expected/losses-eid=0.csv', fname)

        # export a specific pair (sm_id, eid)
        fnames = export(('ass_loss_ratios:1:0', 'csv'),
                        self.calc.datastore)
        for fname in fnames:
            self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_miriam(self):
        # this is a case with a grid and asset-hazard association
        out = self.run_calc(case_miriam.__file__, 'job.ini', exports='csv')
        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv',
                              fname)
        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles(
            'expected/portfolio_loss.txt', fname, delta=1E-5)

    # now a couple of hazard tests

    @attr('qa', 'hazard', 'event_based')
    def test_case_4_hazard(self):
        # Turkey with SHARE logic tree; TODO: add site model
        out = self.run_calc(case_4.__file__, 'job.ini',
                            calculation_mode='event_based',
                            ground_motion_fields='false', exports='csv')
        [fname] = out['hcurves', 'csv']
        self.assertEqualFiles('expected/hazard_curve-mean.csv', fname)
        [fname] = out['hmaps', 'csv']
        self.assertEqualFiles('expected/hazard_map-mean.csv', fname)

        fnames = export(('hmaps', 'xml'), self.calc.datastore)
        self.assertEqual(len(fnames), 4)  # 2 IMT x 2 poes

    @attr('qa', 'hazard', 'event_based')
    def test_case_4a(self):
        # the case of a site_model.xml with 7 sites but only 1 asset
        out = self.run_calc(case_4a.__file__, 'job_hazard.ini',
                            exports='txt')
        [fname] = out['gmf_data', 'txt']
        self.assertEqualFiles(
            'expected/gmf-smltp_b1-gsimltp_b1.txt', fname)
<commit_msg>Removed a skipped test<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from nose.plugins.attrib import attr

from openquake.baselib.general import writetmp
from openquake.calculators.views import view
from openquake.calculators.tests import CalculatorTestCase, strip_calc_id
from openquake.commonlib.export import export
from openquake.calculators.tests import check_platform
from openquake.qa_tests_data.event_based_risk import (
    case_1, case_2, case_3, case_4, case_4a, case_master, case_miriam,
    occupants)

class EventBasedRiskTestCase(CalculatorTestCase):

    def assert_stats_ok(self, pkg, job_ini, individual_curves='false'):
        out = self.run_calc(pkg.__file__, job_ini, exports='csv',
                            individual_curves=individual_curves,
                            concurrent_tasks='4')
        # NB: it is important to use concurrent_tasks > 1 to test the
        # complications of concurrency (for instance the noncommutativity of
        # numpy.float32 addition when computing the average losses)
        all_csv = []
        for fnames in out.values():
            for fname in fnames:
                if 'rlz' in fname and individual_curves == 'false':
                    continue
                elif fname.endswith('.csv') and any(x in fname for x in (
                        'loss_curve', 'loss_map', 'agg_loss', 'avg_loss')):
                    all_csv.append(fname)
        assert all_csv, 'Could not find any CSV file??'
        for fname in all_csv:
            self.assertEqualFiles(
                'expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_1(self):
        self.assert_stats_ok(case_1, 'job.ini', individual_curves='true')

        # make sure the XML and JSON exporters run
        ekeys = [
            ('loss_curves-stats', 'xml'),
            ('loss_curves-stats', 'geojson'),
            ('rcurves-rlzs', 'xml'),
            ('rcurves-rlzs', 'geojson'),

            ('loss_maps-stats', 'xml'),
            ('loss_maps-stats', 'geojson'),
            ('loss_maps-rlzs', 'xml'),
            ('loss_maps-rlzs', 'geojson'),

            ('agg_curve-stats', 'xml'),
        ]
        for ekey in ekeys:
            export(ekey, self.calc.datastore)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2(self):
        self.assert_stats_ok(case_2, 'job.ini', individual_curves='true')
        fname = writetmp(view('mean_avg_losses', self.calc.datastore))
        self.assertEqualFiles('expected/mean_avg_losses.txt', fname)

        # test the composite_risk_model keys (i.e. slash escaping)
        crm = sorted(self.calc.datastore.getitem('composite_risk_model'))
        self.assertEqual(crm, ['RC%2B', 'RM', 'W%2F1'])
        # export a specific eid
        [fname] = export(('ass_loss_ratios:0', 'csv'), self.calc.datastore)
        self.assertEqualFiles('expected/losses-eid=0.csv', fname)

        # test the case when all GMFs are filtered out
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job.ini', minimum_intensity='10.0')
        self.assertEqual(
            str(ctx.exception),
            'No GMFs were generated, perhaps they were all below the '
            'minimum_intensity threshold')

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2_correlation(self):
        out = self.run_calc(case_2.__file__, 'job_loss.ini', exports='csv',
                            asset_correlation=1.0)
        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses.csv', fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_missing_taxonomy(self):
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job_err.ini')
        self.assertIn('not in the risk model', str(ctx.exception))

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_3(self):
        # this is a test with statistics and without conditional_loss_poes
        self.run_calc(case_3.__file__, 'job.ini',
                      exports='xml', individual_curves='false',
                      concurrent_tasks='4')
        [fname] = export(('agg_curve-stats', 'xml'), self.calc.datastore)
        self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_4(self):
        # Turkey with SHARE logic tree
        out = self.run_calc(case_4.__file__, 'job.ini',
                            exports='csv', individual_curves='true')
        [fname] = export(('avg_losses-stats', 'csv'), self.calc.datastore)
        self.assertEqualFiles('expected/avg_losses-mean.csv', fname)

        fnames = out['agg_loss_table', 'csv']
        assert fnames, 'No agg_losses exported??'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_occupants(self):
        out = self.run_calc(occupants.__file__, 'job.ini',
                            exports='xml', individual_curves='true')
        fnames = out['loss_maps-rlzs', 'xml'] + out['agg_curve-rlzs', 'xml']
        self.assertEqual(len(fnames), 3)  # 2 loss_maps + 1 agg_curve
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_master(self):
        self.assert_stats_ok(case_master, 'job.ini', individual_curves='true')

        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles('expected/portfolio_loss.txt', fname, delta=1E-5)

        # check rup_data is stored correctly
        fname = writetmp(view('ruptures_events', self.calc.datastore))
        self.assertEqualFiles('expected/ruptures_events.txt', fname)

        # export a specific eid
        fnames = export(('ass_loss_ratios:0', 'csv'), self.calc.datastore)
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)
        self.assertEqualFiles('expected/losses-eid=0.csv', fname)

        # export a specific pair (sm_id, eid)
        fnames = export(('ass_loss_ratios:1:0', 'csv'),
                        self.calc.datastore)
        for fname in fnames:
            self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_miriam(self):
        # this is a case with a grid and asset-hazard association
        out = self.run_calc(case_miriam.__file__, 'job.ini', exports='csv')
        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv',
                              fname)
        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles(
            'expected/portfolio_loss.txt', fname, delta=1E-5)

    # now a couple of hazard tests

    @attr('qa', 'hazard', 'event_based')
    def test_case_4_hazard(self):
        # Turkey with SHARE logic tree; TODO: add site model
        out = self.run_calc(case_4.__file__, 'job.ini',
                            calculation_mode='event_based',
                            ground_motion_fields='false', exports='csv')
        [fname] = out['hcurves', 'csv']
        self.assertEqualFiles('expected/hazard_curve-mean.csv', fname)
        [fname] = out['hmaps', 'csv']
        self.assertEqualFiles('expected/hazard_map-mean.csv', fname)

        fnames = export(('hmaps', 'xml'), self.calc.datastore)
        self.assertEqual(len(fnames), 4)  # 2 IMT x 2 poes

    @attr('qa', 'hazard', 'event_based')
    def test_case_4a(self):
        # the case of a site_model.xml with 7 sites but only 1 asset
        out = self.run_calc(case_4a.__file__, 'job_hazard.ini',
                            exports='txt')
        [fname] = out['gmf_data', 'txt']
        self.assertEqualFiles(
            'expected/gmf-smltp_b1-gsimltp_b1.txt', fname)
<|endoftext|>


<commit_before>
def singleNodeLauncher (cmd_string, taskBlocks, walltime=None):
    ''' Worker launcher that wraps the user's cmd_string with the framework to
    launch multiple cmd_string invocations in parallel. This wrapper sets the
    bash env variable CORES to the number of cores on the machine. By setting
    taskBlocks to an integer or to a bash expression the number of invocations
    of the cmd_string to be launched can be controlled.

    Args:
        - cmd_string (string): The command string to be launched
        - taskBlock (string) : bash evaluated string.

    KWargs:
        - walltime (int) : This is not used by this launcher.
    '''

    x = '''export CORES=$(grep -c ^processor /proc/cpuinfo)
echo "Found cores : $CORES"
WORKERCOUNT={1}

CMD ( ) {{
{0}
}}

for COUNT in $(seq 1 1 $WORKERCOUNT)
do
    echo "Launching worker: $COUNT"
    CMD &
done
wait
echo "All workers done"
'''.format(cmd_string, taskBlocks)
    return x

<commit_msg>Adding Srun launch codes<commit_after>
def singleNodeLauncher (cmd_string, taskBlocks, walltime=None):
    ''' Worker launcher that wraps the user's cmd_string with the framework to
    launch multiple cmd_string invocations in parallel. This wrapper sets the
    bash env variable CORES to the number of cores on the machine. By setting
    taskBlocks to an integer or to a bash expression the number of invocations
    of the cmd_string to be launched can be controlled.

    Args:
        - cmd_string (string): The command string to be launched
        - taskBlock (string) : bash evaluated string.

    KWargs:
        - walltime (int) : This is not used by this launcher.
    '''

    x = '''export CORES=$(grep -c ^processor /proc/cpuinfo)
echo "Found cores : $CORES"
WORKERCOUNT={1}

CMD ( ) {{
{0}
}}

for COUNT in $(seq 1 1 $WORKERCOUNT)
do
    echo "Launching worker: $COUNT"
    CMD &
done
wait
echo "All workers done"
'''.format(cmd_string, taskBlocks)
    return x

def srunLauncher (cmd_string, taskBlocks, walltime=None):
    ''' Worker launcher that wraps the user's cmd_string with the SRUN launch framework
    to launch multiple cmd invocations in parallel on a single job allocation.

    Args:
        - cmd_string (string): The command string to be launched
        - taskBlock (string) : bash evaluated string.

    KWargs:
        - walltime (int) : This is not used by this launcher.
    '''

    x = '''export CORES=$SLURM_CPUS_ON_NODE
export NODES=$SLURM_JOB_NUM_NODES

echo "Found cores : $CORES"
echo "Found nodes : $NODES"
WORKERCOUNT={1}

cat << SLURM_EOF > cmd_$SLURM_JOB_NAME.sh
{0}
SLURM_EOF
chmod a+x cmd_$SLURM_JOB_NAME.sh

srun -n{1} -l bash cmd_$SLURM_JOB_NAME.sh
echo "Done"
'''.format(cmd_string, taskBlocks)
    return x
<|endoftext|>