<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import time
import tensorflow as tf

from lib.models import nn

from vocab import Vocab
from lib.models.parsers.base_parser import BaseParser

#***************************************************************
class Parser(BaseParser):
  """"""

  #=============================================================
  def __call__(self, dataset, moving_params=None):
    """"""

    vocabs = dataset.vocabs
    inputs = dataset.inputs
    targets = dataset.targets

    reuse = (moving_params is not None)
    self.tokens_to_keep3D = tf.expand_dims(tf.to_float(tf.greater(inputs[:,:,0], vocabs[0].ROOT)), 2)
    self.sequence_lengths = tf.reshape(tf.reduce_sum(self.tokens_to_keep3D, [1, 2]), [-1,1])
    self.n_tokens = tf.reduce_sum(self.sequence_lengths)
    self.moving_params = moving_params

    word_inputs, pret_inputs = vocabs[0].embedding_lookup(inputs[:,:,0], inputs[:,:,1], moving_params=self.moving_params)
    tag_inputs = vocabs[1].embedding_lookup(inputs[:,:,2], moving_params=self.moving_params)
    if self.add_to_pretrained:
      word_inputs += pret_inputs
    if self.word_l2_reg > 0:
      unk_mask = tf.expand_dims(tf.to_float(tf.greater(inputs[:,:,1], vocabs[0].UNK)),2)
      word_loss = self.word_l2_reg*tf.nn.l2_loss((word_inputs - pret_inputs) * unk_mask)
    embed_inputs = self.embed_concat(word_inputs, tag_inputs)

    top_recur = embed_inputs

    kernel = 3
    hidden_size = self.num_heads * self.head_size
    print("n_recur: ", self.n_recur)
    print("num heads: ", self.num_heads)
    print("cnn dim: ", self.cnn_dim)
    print("relu hidden size: ", self.relu_hidden_size)
    print("head size: ", self.head_size)

    attn_dropout = 0.67
    prepost_dropout = 0.67
    relu_dropout = 0.67
    # if moving_params is not None:
    #   attn_dropout = 1.0
    #   prepost_dropout = 1.0
    #   relu_dropout = 1.0
    #   self.recur_keep_prob = 1.0

    for i in xrange(self.cnn_layers):
      with tf.variable_scope('CNN%d' % i, reuse=reuse):
        top_recur = self.CNN(top_recur, 1, kernel, self.cnn_dim, self.recur_keep_prob, self.info_func)

    if self.n_recur > 0:
      with tf.variable_scope('proj1', reuse=reuse):
        top_recur = self.MLP(top_recur, hidden_size, n_splits=1)

      top_recur = nn.add_timing_signal_1d(top_recur)

    # add a transformer layer here y not
    # Transformer:
    for i in range(self.n_recur):
      with tf.variable_scope('Transformer%d' % i, reuse=reuse):
        top_recur = self.transformer(top_recur, hidden_size, self.num_heads,
                                     attn_dropout, relu_dropout, prepost_dropout, self.relu_hidden_size,
                                     self.info_func, reuse)
    # if normalization is done in layer_preprocess, then it shuold also be done
    # on the output, since the output can grow very large, being the sum of
    # a whole stack of unnormalized layer outputs.
    if self.n_recur > 0:
      top_recur = nn.layer_norm(top_recur, reuse)

    with tf.variable_scope('proj2', reuse=reuse):
      top_recur_rows, top_recur_cols = self.MLP(top_recur, self.cnn_dim_2d//2, n_splits=2)

    top_recur_rows = nn.add_timing_signal_1d(top_recur_rows)
    top_recur_cols = nn.add_timing_signal_1d(top_recur_cols)

    with tf.variable_scope('2d', reuse=reuse):
      # set up input (split -> 2d)
      input_shape = tf.shape(embed_inputs)
      bucket_size = input_shape[1]
      top_recur_rows = tf.tile(tf.expand_dims(top_recur_rows, 1), [1, bucket_size, 1, 1])
      top_recur_cols = tf.tile(tf.expand_dims(top_recur_cols, 2), [1, 1, bucket_size, 1])
      top_recur_2d = tf.concat([top_recur_cols, top_recur_rows], axis=-1)

      # apply num_convs 2d conv layers
      for i in xrange(self.cnn2d_layers):  # todo pass this in
        with tf.variable_scope('CNN%d' % i, reuse=reuse):
          top_recur_2d = self.CNN(top_recur_2d, kernel, kernel, self.cnn_dim_2d,  # todo pass this in
                                  self.recur_keep_prob if i < self.n_recur - 1 else 1.0,
                                  self.info_func if i < self.n_recur - 1 else tf.identity)
          # top_recur_2d = nn.layer_norm(top_recur_2d, reuse)

    # should normalize before adding
    # top_recur = nn.layer_norm(top_recur, reuse)

    with tf.variable_scope('Arcs', reuse=reuse):
      arc_logits = self.MLP(top_recur_2d, 1, n_splits=1)
      arc_logits = tf.squeeze(arc_logits, axis=-1)
      arc_output = self.output_svd(arc_logits, targets[:, :, 1])
      if moving_params is None:
        predictions = targets[:, :, 1]
      else:
        predictions = arc_output['predictions']

    # Project each predicted (or gold) edge into head and dep rel representations
    with tf.variable_scope('MLP', reuse=reuse):
      # flat_labels = tf.reshape(predictions, [-1])
      original_shape = tf.shape(arc_logits)
      batch_size = original_shape[0]
      bucket_size = original_shape[1]
      # num_classes = len(vocabs[2])
      i1, i2 = tf.meshgrid(tf.range(batch_size), tf.range(bucket_size), indexing="ij")
      targ = i1 * bucket_size * bucket_size + i2 * bucket_size + predictions
      idx = tf.reshape(targ, [-1])
      conditioned = tf.gather(tf.reshape(top_recur_2d, [-1, self.cnn_dim_2d]), idx)
      conditioned = tf.reshape(conditioned, [batch_size, bucket_size, self.cnn_dim_2d])
      dep_rel_mlp, head_rel_mlp = self.MLP(conditioned, self.class_mlp_size + self.attn_mlp_size, n_splits=2)

    with tf.variable_scope('Rels', reuse=reuse):
      rel_logits, rel_logits_cond = self.conditional_bilinear_classifier(dep_rel_mlp, head_rel_mlp, len(vocabs[2]), predictions)
      rel_output = self.output(rel_logits, targets[:, :, 2])
      rel_output['probabilities'] = self.conditional_probabilities(rel_logits_cond)

    output = {}
    output['probabilities'] = tf.tuple([arc_output['probabilities'],
                                        rel_output['probabilities']])
    output['predictions'] = tf.stack([arc_output['predictions'],
                                      rel_output['predictions']])
    output['correct'] = arc_output['correct'] * rel_output['correct']
    output['tokens'] = arc_output['tokens']
    output['n_correct'] = tf.reduce_sum(output['correct'])
    output['n_tokens'] = self.n_tokens
    output['accuracy'] = output['n_correct'] / output['n_tokens']
    output['loss'] = arc_output['loss'] + rel_output['loss']
    if self.word_l2_reg > 0:
      output['loss'] += word_loss

    output['embed'] = embed_inputs
    output['recur'] = top_recur
    # output['dep_arc'] = dep_arc_mlp
    # output['head_dep'] = head_arc_mlp
    output['dep_rel'] = dep_rel_mlp
    output['head_rel'] = head_rel_mlp
    output['arc_logits'] = arc_logits
    output['rel_logits'] = rel_logits

    output['rel_loss'] = rel_output['loss']
    output['log_loss'] = arc_output['log_loss']
    output['2cycle_loss'] = arc_output['2cycle_loss']
    output['roots_loss'] = arc_output['roots_loss']
    output['svd_loss'] = arc_output['svd_loss']

    #### OLD: TRANSFORMER ####
    # top_recur = nn.add_timing_signal_1d(top_recur)
    #
    # for i in xrange(self.n_recur):
    #   # RNN:
    #   # with tf.variable_scope('RNN%d' % i, reuse=reuse):
    #   #   top_recur, _ = self.RNN(top_recur)
    #
    #   # Transformer:
    #   with tf.variable_scope('Transformer%d' % i, reuse=reuse):
    #     top_recur = self.transformer(top_recur, hidden_size, self.num_heads,
    #                                  attn_dropout, relu_dropout, prepost_dropout, self.relu_hidden_size,
    #                                  self.info_func, reuse)
    # # if normalization is done in layer_preprocess, then it shuold also be done
    # # on the output, since the output can grow very large, being the sum of
    # # a whole stack of unnormalized layer outputs.
    # top_recur = nn.layer_norm(top_recur, reuse)
    #
    # with tf.variable_scope('MLP', reuse=reuse):
    #   dep_mlp, head_mlp = self.MLP(top_recur, self.class_mlp_size+self.attn_mlp_size, n_splits=2)
    #   dep_arc_mlp, dep_rel_mlp = dep_mlp[:,:,:self.attn_mlp_size], dep_mlp[:,:,self.attn_mlp_size:]
    #   head_arc_mlp, head_rel_mlp = head_mlp[:,:,:self.attn_mlp_size], head_mlp[:,:,self.attn_mlp_size:]
    #
    # with tf.variable_scope('Arcs', reuse=reuse):
    #   arc_logits = self.bilinear_classifier(dep_arc_mlp, head_arc_mlp)
    #   # arc_output = self.output(arc_logits, targets[:,:,1])
    #   arc_output = self.output_svd(arc_logits, targets[:,:,1])
    #   if moving_params is None:
    #     predictions = targets[:,:,1]
    #   else:
    #     predictions = arc_output['predictions']
    # with tf.variable_scope('Rels', reuse=reuse):
    #   rel_logits, rel_logits_cond = self.conditional_bilinear_classifier(dep_rel_mlp, head_rel_mlp, len(vocabs[2]), predictions)
    #   rel_output = self.output(rel_logits, targets[:,:,2])
    #   rel_output['probabilities'] = self.conditional_probabilities(rel_logits_cond)
    #
    # output = {}
    # output['probabilities'] = tf.tuple([arc_output['probabilities'],
    #                                     rel_output['probabilities']])
    # output['predictions'] = tf.stack([arc_output['predictions'],
    #                                  rel_output['predictions']])
    # output['correct'] = arc_output['correct'] * rel_output['correct']
    # output['tokens'] = arc_output['tokens']
    # output['n_correct'] = tf.reduce_sum(output['correct'])
    # output['n_tokens'] = self.n_tokens
    # output['accuracy'] = output['n_correct'] / output['n_tokens']
    # output['loss'] = arc_output['loss'] + rel_output['loss']
    # if self.word_l2_reg > 0:
    #   output['loss'] += word_loss
    #
    # output['embed'] = embed_inputs
    # output['recur'] = top_recur
    # output['dep_arc'] = dep_arc_mlp
    # output['head_dep'] = head_arc_mlp
    # output['dep_rel'] = dep_rel_mlp
    # output['head_rel'] = head_rel_mlp
    # output['arc_logits'] = arc_logits
    # output['rel_logits'] = rel_logits
    #
    # output['rel_loss'] = rel_output['loss']
    # output['log_loss'] = arc_output['log_loss']
    # output['2cycle_loss'] = arc_output['2cycle_loss']
    # output['roots_loss'] = arc_output['roots_loss']
    # output['svd_loss'] = arc_output['svd_loss']
    return output

  #=============================================================
  def prob_argmax(self, parse_probs, rel_probs, tokens_to_keep):
    """"""
    start_time = time.time()
    parse_preds, roots_lt, roots_gt, cycles_2, cycles_n = self.parse_argmax(parse_probs, tokens_to_keep)
    rel_probs = rel_probs[np.arange(len(parse_preds)), parse_preds]
    rel_preds = self.rel_argmax(rel_probs, tokens_to_keep)
    total_time = time.time() - start_time
    return parse_preds, rel_preds, total_time, roots_lt, roots_gt, cycles_2, cycles_n
<commit_msg>layer norm more things<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import time
import tensorflow as tf

from lib.models import nn

from vocab import Vocab
from lib.models.parsers.base_parser import BaseParser

#***************************************************************
class Parser(BaseParser):
  """"""

  #=============================================================
  def __call__(self, dataset, moving_params=None):
    """"""

    vocabs = dataset.vocabs
    inputs = dataset.inputs
    targets = dataset.targets

    reuse = (moving_params is not None)
    self.tokens_to_keep3D = tf.expand_dims(tf.to_float(tf.greater(inputs[:,:,0], vocabs[0].ROOT)), 2)
    self.sequence_lengths = tf.reshape(tf.reduce_sum(self.tokens_to_keep3D, [1, 2]), [-1,1])
    self.n_tokens = tf.reduce_sum(self.sequence_lengths)
    self.moving_params = moving_params

    word_inputs, pret_inputs = vocabs[0].embedding_lookup(inputs[:,:,0], inputs[:,:,1], moving_params=self.moving_params)
    tag_inputs = vocabs[1].embedding_lookup(inputs[:,:,2], moving_params=self.moving_params)
    if self.add_to_pretrained:
      word_inputs += pret_inputs
    if self.word_l2_reg > 0:
      unk_mask = tf.expand_dims(tf.to_float(tf.greater(inputs[:,:,1], vocabs[0].UNK)),2)
      word_loss = self.word_l2_reg*tf.nn.l2_loss((word_inputs - pret_inputs) * unk_mask)
    embed_inputs = self.embed_concat(word_inputs, tag_inputs)

    top_recur = embed_inputs

    kernel = 3
    hidden_size = self.num_heads * self.head_size
    print("n_recur: ", self.n_recur)
    print("num heads: ", self.num_heads)
    print("cnn dim: ", self.cnn_dim)
    print("relu hidden size: ", self.relu_hidden_size)
    print("head size: ", self.head_size)

    print("cnn2d_layers: ", self.cnn2d_layers)
    print("cnn_dim_2d: ", self.cnn_dim_2d)

    attn_dropout = 0.67
    prepost_dropout = 0.67
    relu_dropout = 0.67
    # if moving_params is not None:
    #   attn_dropout = 1.0
    #   prepost_dropout = 1.0
    #   relu_dropout = 1.0
    #   self.recur_keep_prob = 1.0

    for i in xrange(self.cnn_layers):
      with tf.variable_scope('CNN%d' % i, reuse=reuse):
        top_recur = self.CNN(top_recur, 1, kernel, self.cnn_dim, self.recur_keep_prob, self.info_func)

    if self.n_recur > 0:
      with tf.variable_scope('proj1', reuse=reuse):
        top_recur = self.MLP(top_recur, hidden_size, n_splits=1)

      top_recur = nn.add_timing_signal_1d(top_recur)

    # add a transformer layer here y not
    # Transformer:
    for i in range(self.n_recur):
      with tf.variable_scope('Transformer%d' % i, reuse=reuse):
        top_recur = self.transformer(top_recur, hidden_size, self.num_heads,
                                     attn_dropout, relu_dropout, prepost_dropout, self.relu_hidden_size,
                                     self.info_func, reuse)
    # if normalization is done in layer_preprocess, then it shuold also be done
    # on the output, since the output can grow very large, being the sum of
    # a whole stack of unnormalized layer outputs.
    if self.n_recur > 0:
      top_recur = nn.layer_norm(top_recur, reuse)

    with tf.variable_scope('proj2', reuse=reuse):
      top_recur_rows, top_recur_cols = self.MLP(top_recur, self.cnn_dim_2d//2, n_splits=2)

    top_recur_rows = nn.add_timing_signal_1d(top_recur_rows)
    top_recur_cols = nn.add_timing_signal_1d(top_recur_cols)

    with tf.variable_scope('2d', reuse=reuse):
      # set up input (split -> 2d)
      input_shape = tf.shape(embed_inputs)
      bucket_size = input_shape[1]
      top_recur_rows = tf.tile(tf.expand_dims(top_recur_rows, 1), [1, bucket_size, 1, 1])
      top_recur_cols = tf.tile(tf.expand_dims(top_recur_cols, 2), [1, 1, bucket_size, 1])
      top_recur_2d = tf.concat([top_recur_cols, top_recur_rows], axis=-1)

      # apply num_convs 2d conv layers
      for i in xrange(self.cnn2d_layers):  # todo pass this in
        with tf.variable_scope('CNN%d' % i, reuse=reuse):
          top_recur_2d = self.CNN(top_recur_2d, kernel, kernel, self.cnn_dim_2d,  # todo pass this in
                                  self.recur_keep_prob if i < self.n_recur - 1 else 1.0,
                                  self.info_func if i < self.n_recur - 1 else tf.identity)
          # top_recur_2d = nn.layer_norm(top_recur_2d, reuse)

    # should normalize before adding
    # top_recur = nn.layer_norm(top_recur, reuse)

    with tf.variable_scope('Arcs', reuse=reuse):
      arc_logits = self.MLP(top_recur_2d, 1, n_splits=1)
      arc_logits = tf.squeeze(arc_logits, axis=-1)
      arc_output = self.output_svd(arc_logits, targets[:, :, 1])
      if moving_params is None:
        predictions = targets[:, :, 1]
      else:
        predictions = arc_output['predictions']

    # Project each predicted (or gold) edge into head and dep rel representations
    with tf.variable_scope('MLP', reuse=reuse):
      # flat_labels = tf.reshape(predictions, [-1])
      original_shape = tf.shape(arc_logits)
      batch_size = original_shape[0]
      bucket_size = original_shape[1]
      # num_classes = len(vocabs[2])
      i1, i2 = tf.meshgrid(tf.range(batch_size), tf.range(bucket_size), indexing="ij")
      targ = i1 * bucket_size * bucket_size + i2 * bucket_size + predictions
      idx = tf.reshape(targ, [-1])
      conditioned = tf.gather(tf.reshape(top_recur_2d, [-1, self.cnn_dim_2d]), idx)
      conditioned = tf.reshape(conditioned, [batch_size, bucket_size, self.cnn_dim_2d])
      dep_rel_mlp, head_rel_mlp = self.MLP(conditioned, self.class_mlp_size + self.attn_mlp_size, n_splits=2)

    with tf.variable_scope('Rels', reuse=reuse):
      rel_logits, rel_logits_cond = self.conditional_bilinear_classifier(dep_rel_mlp, head_rel_mlp, len(vocabs[2]), predictions)
      rel_output = self.output(rel_logits, targets[:, :, 2])
      rel_output['probabilities'] = self.conditional_probabilities(rel_logits_cond)

    output = {}
    output['probabilities'] = tf.tuple([arc_output['probabilities'],
                                        rel_output['probabilities']])
    output['predictions'] = tf.stack([arc_output['predictions'],
                                      rel_output['predictions']])
    output['correct'] = arc_output['correct'] * rel_output['correct']
    output['tokens'] = arc_output['tokens']
    output['n_correct'] = tf.reduce_sum(output['correct'])
    output['n_tokens'] = self.n_tokens
    output['accuracy'] = output['n_correct'] / output['n_tokens']
    output['loss'] = arc_output['loss'] + rel_output['loss']
    if self.word_l2_reg > 0:
      output['loss'] += word_loss

    output['embed'] = embed_inputs
    output['recur'] = top_recur
    # output['dep_arc'] = dep_arc_mlp
    # output['head_dep'] = head_arc_mlp
    output['dep_rel'] = dep_rel_mlp
    output['head_rel'] = head_rel_mlp
    output['arc_logits'] = arc_logits
    output['rel_logits'] = rel_logits

    output['rel_loss'] = rel_output['loss']
    output['log_loss'] = arc_output['log_loss']
    output['2cycle_loss'] = arc_output['2cycle_loss']
    output['roots_loss'] = arc_output['roots_loss']
    output['svd_loss'] = arc_output['svd_loss']

    #### OLD: TRANSFORMER ####
    # top_recur = nn.add_timing_signal_1d(top_recur)
    #
    # for i in xrange(self.n_recur):
    #   # RNN:
    #   # with tf.variable_scope('RNN%d' % i, reuse=reuse):
    #   #   top_recur, _ = self.RNN(top_recur)
    #
    #   # Transformer:
    #   with tf.variable_scope('Transformer%d' % i, reuse=reuse):
    #     top_recur = self.transformer(top_recur, hidden_size, self.num_heads,
    #                                  attn_dropout, relu_dropout, prepost_dropout, self.relu_hidden_size,
    #                                  self.info_func, reuse)
    # # if normalization is done in layer_preprocess, then it shuold also be done
    # # on the output, since the output can grow very large, being the sum of
    # # a whole stack of unnormalized layer outputs.
    # top_recur = nn.layer_norm(top_recur, reuse)
    #
    # with tf.variable_scope('MLP', reuse=reuse):
    #   dep_mlp, head_mlp = self.MLP(top_recur, self.class_mlp_size+self.attn_mlp_size, n_splits=2)
    #   dep_arc_mlp, dep_rel_mlp = dep_mlp[:,:,:self.attn_mlp_size], dep_mlp[:,:,self.attn_mlp_size:]
    #   head_arc_mlp, head_rel_mlp = head_mlp[:,:,:self.attn_mlp_size], head_mlp[:,:,self.attn_mlp_size:]
    #
    # with tf.variable_scope('Arcs', reuse=reuse):
    #   arc_logits = self.bilinear_classifier(dep_arc_mlp, head_arc_mlp)
    #   # arc_output = self.output(arc_logits, targets[:,:,1])
    #   arc_output = self.output_svd(arc_logits, targets[:,:,1])
    #   if moving_params is None:
    #     predictions = targets[:,:,1]
    #   else:
    #     predictions = arc_output['predictions']
    # with tf.variable_scope('Rels', reuse=reuse):
    #   rel_logits, rel_logits_cond = self.conditional_bilinear_classifier(dep_rel_mlp, head_rel_mlp, len(vocabs[2]), predictions)
    #   rel_output = self.output(rel_logits, targets[:,:,2])
    #   rel_output['probabilities'] = self.conditional_probabilities(rel_logits_cond)
    #
    # output = {}
    # output['probabilities'] = tf.tuple([arc_output['probabilities'],
    #                                     rel_output['probabilities']])
    # output['predictions'] = tf.stack([arc_output['predictions'],
    #                                  rel_output['predictions']])
    # output['correct'] = arc_output['correct'] * rel_output['correct']
    # output['tokens'] = arc_output['tokens']
    # output['n_correct'] = tf.reduce_sum(output['correct'])
    # output['n_tokens'] = self.n_tokens
    # output['accuracy'] = output['n_correct'] / output['n_tokens']
    # output['loss'] = arc_output['loss'] + rel_output['loss']
    # if self.word_l2_reg > 0:
    #   output['loss'] += word_loss
    #
    # output['embed'] = embed_inputs
    # output['recur'] = top_recur
    # output['dep_arc'] = dep_arc_mlp
    # output['head_dep'] = head_arc_mlp
    # output['dep_rel'] = dep_rel_mlp
    # output['head_rel'] = head_rel_mlp
    # output['arc_logits'] = arc_logits
    # output['rel_logits'] = rel_logits
    #
    # output['rel_loss'] = rel_output['loss']
    # output['log_loss'] = arc_output['log_loss']
    # output['2cycle_loss'] = arc_output['2cycle_loss']
    # output['roots_loss'] = arc_output['roots_loss']
    # output['svd_loss'] = arc_output['svd_loss']
    return output

  #=============================================================
  def prob_argmax(self, parse_probs, rel_probs, tokens_to_keep):
    """"""
    start_time = time.time()
    parse_preds, roots_lt, roots_gt, cycles_2, cycles_n = self.parse_argmax(parse_probs, tokens_to_keep)
    rel_probs = rel_probs[np.arange(len(parse_preds)), parse_preds]
    rel_preds = self.rel_argmax(rel_probs, tokens_to_keep)
    total_time = time.time() - start_time
    return parse_preds, rel_preds, total_time, roots_lt, roots_gt, cycles_2, cycles_n
<|endoftext|>


<commit_before>import sys
import argparse
import logging
from closeio_api import Client as CloseIO_API, APIError
import csv
reload(sys)
sys.setdefaultencoding('utf-8')

parser = argparse.ArgumentParser(description='Download a CSV of calls from/to a specific Close.io number over a specified time range')

parser.add_argument('--api-key', '-k', required=True, help='API Key')
parser.add_argument('--direction', '-d', default=None, choices=['inbound', 'outbound'],
                    help='Use this field to only export inbound calls or outbound calls. Leave this field blank to export both.')
parser.add_argument('--missed-or-voicemail', '-m', action='store_true', help='Use this field to only export missed calls, voicemails, or calls of a duration 0')
parser.add_argument('--end-date', '-e',
                    help='The end of the date range you want to export call data for in yyyy-mm-dd format.')
parser.add_argument('--start-date', '-s',
                    help='The start of the date range you want to export call data for in yyyy-mm-dd format.')
parser.add_argument('--phone-number', '-p',
                    help='The phone number you\'d like to export the calls for in E164 international format. Example: +18552567346')
parser.add_argument('--unattached-only', '-o', action='store_true', help='Use this field if you only want to find calls not attached to a lead')
parser.add_argument('--user-id', '-u', help='Use this field if you only want to find calls for a specific user')

args = parser.parse_args()

api = CloseIO_API(args.api_key)

org_id = api.get('api_key/' + args.api_key)['organization_id']
org_name = api.get('organization/' + org_id)['name'].replace('/', "")

params = {}
has_more = True
offset = 0
calls = []
display_names = {}
query = '(has:calls'

if args.start_date:
    params['date_created__gte'] = args.start_date
    query = query + ' date >= "%s"' % args.start_date

if args.end_date:
    params['date_created__lte'] = args.end_date
    query = query + ' date <= "%s"' % args.end_date
query = query + ")"

if args.user_id:
    params['user_id'] = args.user_id

while has_more:

    resp = api.get('lead', params={ '_skip': offset, 'query': query, '_fields': 'id,display_name' })
    for lead in resp['data']:
        display_names[lead['id']] = lead['display_name']

    offset+=len(resp['data'])
    has_more = resp['has_more']

has_more = True
offset = 0
params['_fields'] = 'id,user_id,duration,direction,date_created,remote_phone,local_phone,voicemail_url,recording_url,source,lead_id,updated_by_name'
print "Getting Calls:"
while has_more:
    params['_skip'] = offset
    resp = api.get('activity/call', params=params)
    if args.missed_or_voicemail:
        resp['data'] = [i for i in resp['data'] if i['duration'] == 0]
    if args.direction:
        resp['data'] = [i for i in resp['data'] if i['direction'] == args.direction]
    if args.phone_number:
        resp['data'] = [i for i in resp['data'] if i['local_phone'] == args.phone_number]
    if args.unattached_only:
        resp['data'] = [i for i in resp['data'] if i['lead_id'] == None]
    for call in resp['data']:
        if call.get('lead_id') and display_names.get(call['lead_id']):
            call['lead_name'] = display_names[call['lead_id']]
        else:
            call['lead_name'] = ""
        calls.append(call)
    offset+=len(resp['data'])
    print offset
    has_more = resp['has_more']

f = open('%s Calls.csv' % org_name, 'wt')
try:
    keys = ['date_created', 'updated_by_name'] + [i for i in calls[0].keys() if i not in ['date_created', 'updated_by_name']]
    writer = csv.DictWriter(f, keys)
    writer.writeheader()
    writer.writerows(calls)
finally:
    f.close()<commit_msg>Fix export_call_data script pagination issue and update the csv creation to handle the case where there are no calls<commit_after>import sys
import argparse
import logging
from closeio_api import Client as CloseIO_API, APIError
import csv
reload(sys)
sys.setdefaultencoding('utf-8')

parser = argparse.ArgumentParser(description='Download a CSV of calls from/to a specific Close.io number over a specified time range')

parser.add_argument('--api-key', '-k', required=True, help='API Key')
parser.add_argument('--direction', '-d', default=None, choices=['inbound', 'outbound'],
                    help='Use this field to only export inbound calls or outbound calls. Leave this field blank to export both.')
parser.add_argument('--missed-or-voicemail', '-m', action='store_true', help='Use this field to only export missed calls, voicemails, or calls of a duration 0')
parser.add_argument('--end-date', '-e',
                    help='The end of the date range you want to export call data for in yyyy-mm-dd format.')
parser.add_argument('--start-date', '-s',
                    help='The start of the date range you want to export call data for in yyyy-mm-dd format.')
parser.add_argument('--phone-number', '-p',
                    help='The phone number you\'d like to export the calls for in E164 international format. Example: +18552567346')
parser.add_argument('--unattached-only', '-o', action='store_true', help='Use this field if you only want to find calls not attached to a lead')
parser.add_argument('--user-id', '-u', help='Use this field if you only want to find calls for a specific user')
parser.add_argument('--include-cost', '-c', action='store_true', help='Use this field if you want to export the cost of each call')

args = parser.parse_args()

api = CloseIO_API(args.api_key)

org_id = api.get('api_key/' + args.api_key)['organization_id']
org_name = api.get('organization/' + org_id)['name'].replace('/', "")

params = {}
has_more = True
offset = 0
calls = []
display_names = {}
query = '(has:calls'

if args.start_date:
    params['date_created__gte'] = args.start_date
    query = query + ' date >= "%s"' % args.start_date

if args.end_date:
    params['date_created__lte'] = args.end_date
    query = query + ' date <= "%s"' % args.end_date
query = query + ")"

if args.user_id:
    params['user_id'] = args.user_id

if not args.unattached_only:
    print "Getting Lead Display Names..."
    while has_more:
        resp = api.get('lead', params={ '_skip': offset, 'query': query, '_fields': 'id,display_name' })
        for lead in resp['data']:
            display_names[lead['id']] = lead['display_name']
        print offset
        offset+=len(resp['data'])
        has_more = resp['has_more']

has_more = True
offset = 0
params['_fields'] = 'id,user_id,duration,direction,date_created,remote_phone,local_phone,voicemail_url,recording_url,source,lead_id,updated_by_name'
print "Getting Calls:"
while has_more:
    params['_skip'] = offset
    resp = api.get('activity/call', params=params)
    for call in resp['data']:
        if call.get('lead_id') and display_names.get(call['lead_id']):
            call['lead_name'] = display_names[call['lead_id']]
        else:
            call['lead_name'] = ""
        calls.append(call)
    offset+=len(resp['data'])
    print offset
    has_more = resp['has_more']

if args.missed_or_voicemail:
    calls = [i for i in calls if i['duration'] == 0]
if args.direction:
    calls = [i for i in calls if i['direction'] == args.direction]
if args.phone_number:
    calls = [i for i in calls if i['local_phone'] == args.phone_number]
if args.unattached_only:
    calls = [i for i in calls if i['lead_id'] == None]

f = open('%s Calls.csv' % org_name, 'wt')
try:
    keys = ['date_created', 'updated_by_name'] + [i for i in params['_fields'].split(',') if i not in ['date_created', 'updated_by_name', 'lead_id']] + ['lead_id', 'lead_name']
    writer = csv.DictWriter(f, keys)
    writer.writeheader()
    writer.writerows(calls)
finally:
    f.close()<|endoftext|>


<commit_before>from __future__ import unicode_literals
import os, sys
import sqlite3
from pkg_resources import resource_string

class DBError(Exception):
    pass

def get_schema(version):
    schema_bytes = resource_string("wormhole.server",
                                   "db-schemas/v%d.sql" % version)
    return schema_bytes.decode("utf-8")

def get_upgrader(new_version):
    schema_bytes = resource_string("wormhole.server",
                                   "db-schemas/upgrade-to-v%d.sql" % new_version)
    return schema_bytes.decode("utf-8")

TARGET_VERSION = 3

def dict_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d

def get_db(dbfile, target_version=TARGET_VERSION, stderr=sys.stderr):
    """Open or create the given db file. The parent directory must exist.
    Returns the db connection object, or raises DBError.
    """

    must_create = (dbfile == ":memory:") or not os.path.exists(dbfile)
    try:
        db = sqlite3.connect(dbfile)
    except (EnvironmentError, sqlite3.OperationalError) as e:
        raise DBError("Unable to create/open db file %s: %s" % (dbfile, e))
    db.row_factory = dict_factory

    if must_create:
        schema = get_schema(target_version)
        db.executescript(schema)
        db.execute("INSERT INTO version (version) VALUES (?)",
                   (target_version,))
        db.commit()

    try:
        version = db.execute("SELECT version FROM version").fetchone()["version"]
    except sqlite3.DatabaseError as e:
        # this indicates that the file is not a compatible database format.
        # Perhaps it was created with an old version, or it might be junk.
        raise DBError("db file is unusable: %s" % e)

    while version < target_version:
        try:
            upgrader = get_upgrader(version+1)
        except ValueError: # ResourceError??
            raise DBError("Unable to upgrade %s to version %s, left at %s"
                          % (dbfile, version+1, version))
        db.executescript(upgrader)
        db.commit()
        version = version+1

    if version != target_version:
        raise DBError("Unable to handle db version %s" % version)

    return db

def dump_db(db):
    # to let _iterdump work, we need to restore the original row factory
    orig = db.row_factory
    try:
        db.row_factory = sqlite3.Row
        return "".join(db.iterdump())
    finally:
        db.row_factory = orig
<commit_msg>db: turn on foreign keys<commit_after>from __future__ import unicode_literals
import os, sys
import sqlite3
from pkg_resources import resource_string

class DBError(Exception):
    pass

def get_schema(version):
    schema_bytes = resource_string("wormhole.server",
                                   "db-schemas/v%d.sql" % version)
    return schema_bytes.decode("utf-8")

def get_upgrader(new_version):
    schema_bytes = resource_string("wormhole.server",
                                   "db-schemas/upgrade-to-v%d.sql" % new_version)
    return schema_bytes.decode("utf-8")

TARGET_VERSION = 3

def dict_factory(cursor, row):
    d = {}
    for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
    return d

def get_db(dbfile, target_version=TARGET_VERSION, stderr=sys.stderr):
    """Open or create the given db file. The parent directory must exist.
    Returns the db connection object, or raises DBError.
    """

    must_create = (dbfile == ":memory:") or not os.path.exists(dbfile)
    try:
        db = sqlite3.connect(dbfile)
    except (EnvironmentError, sqlite3.OperationalError) as e:
        raise DBError("Unable to create/open db file %s: %s" % (dbfile, e))
    db.row_factory = dict_factory
    db.execute("PRAGMA foreign_keys = ON")
    problems = db.execute("PRAGMA foreign_key_check").fetchall()
    if problems:
        raise DBError("failed foreign key check: %s" % (problems,))

    if must_create:
        schema = get_schema(target_version)
        db.executescript(schema)
        db.execute("INSERT INTO version (version) VALUES (?)",
                   (target_version,))
        db.commit()

    try:
        version = db.execute("SELECT version FROM version").fetchone()["version"]
    except sqlite3.DatabaseError as e:
        # this indicates that the file is not a compatible database format.
        # Perhaps it was created with an old version, or it might be junk.
        raise DBError("db file is unusable: %s" % e)

    while version < target_version:
        try:
            upgrader = get_upgrader(version+1)
        except ValueError: # ResourceError??
            raise DBError("Unable to upgrade %s to version %s, left at %s"
                          % (dbfile, version+1, version))
        db.executescript(upgrader)
        db.commit()
        version = version+1

    if version != target_version:
        raise DBError("Unable to handle db version %s" % version)

    return db

def dump_db(db):
    # to let _iterdump work, we need to restore the original row factory
    orig = db.row_factory
    try:
        db.row_factory = sqlite3.Row
        return "".join(db.iterdump())
    finally:
        db.row_factory = orig
<|endoftext|>


<commit_before>import settings
from django.conf.urls.defaults import *
from django.contrib import admin

admin.autodiscover()

subdir = 'andre/'
urlpatterns = patterns('',
    (r'^%sadmin/(.*)' % subdir, admin.site.root),
    (r'^%spublication/(.*)' % subdir, include('stuff.publications.urls')),
    (r'^%sfile/(.*)' % subdir, include('stuff.files.urls')),
    (r'^%sphoto/(.*)' % subdir, include('stuff.picasaweb.urls')),
    (r'^%s$' % subdir, 'stuff.views.index'),

    # Media serving
    (r'^%smedia/(?P<path>.*)$' % subdir,
     'django.views.static.serve',
     {'document_root': settings.MEDIA_ROOT,
     'show_indexes': True}
     ),
    )
<commit_msg>Back to previous configuration for this project's URLs<commit_after>import settings
from django.conf.urls.defaults import *
from django.contrib import admin

admin.autodiscover()

subdir = 'andre/'
urlpatterns = patterns('',
    (r'^%sadmin/(.*)' % subdir, admin.site.root),
    (r'^%spublication/' % subdir, include('stuff.publications.urls')),
    (r'^%sfile/' % subdir, include('stuff.files.urls')),
    (r'^%sphoto/' % subdir, include('stuff.picasaweb.urls')),
    (r'^%s$' % subdir, 'stuff.views.index'),

    # Media serving
    (r'^%smedia/(?P<path>.*)$' % subdir,
     'django.views.static.serve',
     {'document_root': settings.MEDIA_ROOT,
     'show_indexes': True}
     ),
    )
<|endoftext|>


<commit_before>import unittest
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.alert import Alert
from selenium.webdriver.support.ui import Select, WebDriverWait
from test.testmodeltestcase import TestModelTestCase as Super

EXPECTED_TYPES = ['Bank', 'Broke', 'Employment Period', 'Has Address',
    'Has Secretarys', 'Important Person', 'Random Interface', 'Range', 'Secretary', 'Thing', 'Types']

class QueryBuilderTestCase(Super):

    def setUp(self):
        Super.setUp(self)
        self.browser.get(self.base_url + '/begin.do')
        self.browser.get(self.base_url + '/customQuery.do')

    def test_on_right_page(self):
        self.assertIn('Custom query', self.browser.title)

    def test_browse_data_model(self):
        link = self.findLink("Browse data model")
        self.assertIsNotNone(link)
        link.click()
        help_text = self.elem('.body > p').text
        self.assertIn("browse the tree", help_text)
        for type_name in EXPECTED_TYPES:
            self.assertIsNotNone(self.findLink(type_name))

        self.findLink('Bank').click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals('Bank', self.elem('.typeSelected').text)

    def start_query_from_select(self):
        cls = 'Employee'
        Select(self.elem("#queryClassSelector")).select_by_visible_text(cls)
        self.elem("#submitClassSelect").click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals(cls, self.elem('.typeSelected').text)
        self.elem('a[title="Show Employee in results"] > img.arrow').click()

    def test_query_tree(self):
        Select(self.elem("#queryClassSelector")).select_by_visible_text("Bank")
        self.elem("#submitClassSelect").click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals('Bank', self.elem('.typeSelected').text)
        self.assertEquals('Name', self.elem('.attributeField').text)
        cc = self.browser.find_element_by_id('drag_Bank.corporateCustomers')
        self.assertEquals('Corporate Customers', cc.text)
        ds = self.browser.find_element_by_id('drag_Bank.debtors')
        self.assertEquals('Debtors', ds.text)
        # Add a view element.
        self.browser.find_element_by_id('img_Bank.debtors').click()
        self.elem('a[title="Show debt in results"] > img.arrow').click()
        self.assertEquals(1, len(self.browser.find_elements_by_class_name('viewpath')))
        self.elem('a[title="Show name in results"] > img.arrow').click()
        self.assertEquals(2, len(self.browser.find_elements_by_class_name('viewpath')))
        # Add a constraint
        self.elem('a[title="Add a constraint to debt"]').click()
        Select(self.elem("#attribute5")).select_by_visible_text(">")
        self.elem("#attribute8").clear()
        self.elem("#attribute8").send_keys("1000")
        self.elem("#attributeSubmit").click()
        constraints = self.elems('span.constraint')
        self.assertEquals(1, len(constraints))
        self.assertEquals('> 1000', constraints[0].text)
        self.elem('a[title="Export this query as XML"]').click()
        expected_query = """
<query name="" model="testmodel" view="Bank.debtors.debt Bank.name" longDescription="" sortOrder="Bank.debtors.debt asc">
  <constraint path="Bank.debtors.debt" op=">" value="1000"/>
</query>
        """
        self.assertEquals(expected_query.strip(), self.elem('body').text)

    def test_run_query(self):
        query = """
        <query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt ASC" >
          <constraint path="Bank.debtors.debt" op=">" value="25000000" />
        </query>
        """
        self.findLink("Import query from XML").click()
        self.elem('#xml').send_keys(query)
        self.elem('#importQueriesForm input[type="submit"]').click()
        self.elem('#showResult').click()
        summary = self.elem(".im-table-summary")
        self.assertRowCountIs(22)

    def assertRowCountIs(self, n):
        self.assertEquals(n, len(self.elems('.im-table-container tbody tr')))

    def test_add_constraint_set_and(self):
        query = """
        <query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt ASC" >
        </query>
        """
        expected_query = "\n".join([
            ' '.join([
                '<query',
                'name=""',
                'model="testmodel"',
                'view="Bank.name Bank.debtors.debt"',
                'longDescription=""',
                'sortOrder="Bank.debtors.debt asc"',
                'constraintLogic="A and B">'
            ]),
            '  <constraint path="Bank.debtors.debt" code="A" op=">" value="35,000,000"/>',
            '  <constraint path="Bank.name" code="B" op="=" value="Gringotts"/>',
            '</query>'
        ])
        self.findLink("Import query from XML").click()
        self.elem('#xml').send_keys(query)
        self.elem('#importQueriesForm input[type="submit"]').click()
        # Add constraint: Bank.debtors.debt > 35e6
        self.browser.find_element_by_id('img_Bank.debtors').click()
        self.elem('a[title="Add a constraint to debt"]').click()
        Select(self.elem("#attribute5")).select_by_visible_text(">")
        self.elem('#attribute8').send_keys('35,000,000')
        self.elem('#attributeSubmit').click()
        # Add constraint: Bank.name = Gringotts
        self.elem('a[title="Add a constraint to name"]').click()
        Select(self.elem("#attribute7")).select_by_visible_text("Gringotts")
        self.elem('#attributeSubmit').click()
        # Check that the query is as expected.
        self.elem('a[title="Export this query as XML"]').click()
        self.assertEquals(expected_query.strip(), self.elem('body').text)
        self.browser.back()
        # Check that the results are as expected.
        self.elem('#showResult').click()
        self.assertRowCountIs(2)

    def test_add_constraint_set_or(self):
        query = """
        <query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt ASC" >
        </query>
        """
        expected_query = "\n".join([
            ' '.join([
                '<query',
                'name=""',
                'model="testmodel"',
                'view="Bank.name Bank.debtors.debt"',
                'longDescription=""',
                'sortOrder="Bank.debtors.debt asc"',
                'constraintLogic="A or B">'
            ]),
            '  <constraint path="Bank.debtors.debt" code="A" op=">" value="35,000,000"/>',
            '  <constraint path="Bank.name" code="B" op="=" value="Gringotts"/>',
            '</query>'
        ])

        # Perform actions.
        self.findLink("Import query from XML").click()
        self.elem('#xml').send_keys(query)
        self.elem('#importQueriesForm input[type="submit"]').click()
        # Add constraint: Bank.debtors.debt > 35e6
        self.browser.find_element_by_id('img_Bank.debtors').click()
        self.elem('a[title="Add a constraint to debt"]').click()
        Select(self.elem("#attribute5")).select_by_visible_text(">")
        self.elem('#attribute8').send_keys('35,000,000')
        self.elem('#attributeSubmit').click()
        # Add constraint: Bank.name = Gringotts
        self.elem('a[title="Add a constraint to name"]').click()
        Select(self.elem("#attribute7")).select_by_visible_text("Gringotts")
        self.elem('#attributeSubmit').click()
        # Switch the constraint logic to A or B
        self.elem('#constraintLogic').click()
        logic = self.elem('#expr')
        logic.clear()
        logic.send_keys('A or B')
        self.browser.find_element_by_id('editconstraintlogic').click()

        # Check that the query is as expected.
        self.elem('a[title="Export this query as XML"]').click()
        self.assertEquals(expected_query.strip(), self.elem('body').text)
        self.browser.back()
        # Check that the results are as expected.
        self.elem('#showResult').click()
        self.assertRowCountIs(24)

    def load_queries_into_history(self):
        query_1 = ''.join([
            '<query model="testmodel" view="Bank.debtors.debt" sortOrder="Bank.debtors.debt asc">',
            '</query>'
            ])
        query_2 = ''.join([
            '<query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt asc">',
            '<constraint path="Bank.debtors.debt" op=">" value="35,000,000"/>',
            '</query>'
            ])
        # Load queries into session history.
        for q in [query_1, query_2]:
            self.browser.get(self.base_url + '/customQuery.do')
            self.findLink("Import query from XML").click()
            self.elem('#xml').send_keys(q)
            self.elem('#importQueriesForm input[type="submit"]').click()
            self.elem('#showResult').click()
        self.browser.get(self.base_url + '/customQuery.do')

    def test_edit_template(self):
        self.browser.get(self.base_url + '/template.do?name=ManagerLookup&scope=all')
        self.elem('input.editQueryBuilder').click()
        self.assertIn('Query builder', self.browser.title)
        # Edit the constraint.
        self.elem('img[title="Edit this constraint"]').click()
        con_value = self.elem('#attribute8')
        con_value.clear()
        con_value.send_keys('Anne')
        self.elem('#attributeSubmit').click()
        # Check export.
        self.elem('a[title="Export this query as XML"]').click()
        expected_query = '\n'.join([
            '<query name="" model="testmodel" view="Manager.name Manager.title" longDescription="">',
            '  <constraint path="Manager" op="LOOKUP" value="Anne" extraValue=""/>',
            '</query>'])
        self.assertEquals(expected_query, self.elem('body').text)
        self.browser.back()
        # Check results.
        self.elem('#showResult').click()
        self.assertRowCountIs(1)

    def test_query_history(self):
        self.load_queries_into_history()
        self.assertIn('Custom query', self.browser.title)
        self.assertEquals(2, len(self.elems('#modifyQueryForm tbody tr')))
        self.assertEquals('query_2', self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(2)').text)
        root = self.elem('#modifyQueryForm tbody tr:nth-child(2) .historySummaryRoot').text
        self.assertEquals('Bank', root)
        showing = self.elems('#modifyQueryForm tbody tr:nth-child(2) .historySummaryShowing')
        self.assertEquals(2, len(showing))
        self.assertEquals(['Name', 'Debt'], [s.text for s in showing])

    def test_delete_query_from_history(self):
        self.load_queries_into_history()
        self.assertEquals(2, len(self.elems('#modifyQueryForm tbody tr')))
        self.elem('#selected_history_1').click()
        self.elem('#delete_button').click()
        Alert(self.browser).accept()
        self.assertEquals(1, len(self.elems('#modifyQueryForm tbody tr')))

    def test_run_query_in_query_history(self):
        self.load_queries_into_history()

        self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(7) span.fakelink:nth-child(1)').click()
        self.assertRowCountIs(16)

    def test_edit_query_in_query_history(self):
        self.load_queries_into_history()

        self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(7) span.fakelink:nth-child(2)').click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals('Bank', self.elem('.typeSelected').text)
        # Edit a constraint.
        self.elem('img[title="Edit this constraint"]').click()
        con_value = self.elem('#attribute8')
        con_value.clear()
        con_value.send_keys('40,000,000')
        self.elem('#attributeSubmit').click()
        # Check results.
        self.elem('#showResult').click()
        self.assertRowCountIs(15)

    def test_export_query_in_query_history(self):
        self.load_queries_into_history()
        expected_query = '\n'.join([
            ' '.join([
                '<query',
                'name="query_2"',
                'model="testmodel"',
                'view="Bank.name Bank.debtors.debt"',
                'longDescription=""',
                'sortOrder="Bank.debtors.debt asc">'
                ]),
            '  <constraint path="Bank.debtors.debt" op=">" value="35,000,000"/>',
            '</query>'])

        self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(7) span.fakelink:nth-child(3)').click()
        self.assertEquals(expected_query, self.elem('body').text)

    def test_import_query(self):
        link = self.findLink("Import query from XML")
        self.assertIsNotNone(link)
        link.click()
        self.assertIn('Import Query', self.browser.title)
        input_box = self.elem('#xml')
        self.assertIsNotNone(input_box)
        query = ''.join([
            '<query model="testmodel" view="Bank.debtors.debt" sortOrder="Bank.debtors.debt asc">',
            '<constraint path="Bank.debtors.debt" op=">" value="1000"/>',
            '</query>'
            ])

        input_box.send_keys(query)
        self.assertEquals('true', self.elem('#file').get_attribute('disabled'))
        self.elem('#importQueriesForm input[type="submit"]').click()
        wait = WebDriverWait(self.browser, 10)
        wait.until(EC.title_contains('Query builder'))

        self.assertEquals('Bank', self.elem('.typeSelected').text)
        constraints = self.elems('span.constraint')
        self.assertEquals(1, len(constraints))
        self.assertEquals('> 1000', constraints[0].text)
        self.assertEquals(1, len(self.browser.find_elements_by_class_name('viewpath')))

    def test_login_to_view_saved(self):
        link = self.findLink("Login to view saved queries")
        self.assertIsNotNone(link)
<commit_msg>Added test for unicode constraint value.<commit_after># -*- coding: utf-8 -*-

import unittest
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.alert import Alert
from selenium.webdriver.support.ui import Select, WebDriverWait
from test.testmodeltestcase import TestModelTestCase as Super

EXPECTED_TYPES = ['Bank', 'Broke', 'Employment Period', 'Has Address',
    'Has Secretarys', 'Important Person', 'Random Interface', 'Range', 'Secretary', 'Thing', 'Types']

class QueryBuilderTestCase(Super):

    def setUp(self):
        Super.setUp(self)
        self.browser.get(self.base_url + '/begin.do')
        self.browser.get(self.base_url + '/customQuery.do')

    def test_on_right_page(self):
        self.assertIn('Custom query', self.browser.title)

    def test_browse_data_model(self):
        link = self.findLink("Browse data model")
        self.assertIsNotNone(link)
        link.click()
        help_text = self.elem('.body > p').text
        self.assertIn("browse the tree", help_text)
        for type_name in EXPECTED_TYPES:
            self.assertIsNotNone(self.findLink(type_name))

        self.findLink('Bank').click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals('Bank', self.elem('.typeSelected').text)

    def start_query_from_select(self):
        cls = 'Employee'
        Select(self.elem("#queryClassSelector")).select_by_visible_text(cls)
        self.elem("#submitClassSelect").click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals(cls, self.elem('.typeSelected').text)
        self.elem('a[title="Show Employee in results"] > img.arrow').click()

    def test_query_tree(self):
        Select(self.elem("#queryClassSelector")).select_by_visible_text("Bank")
        self.elem("#submitClassSelect").click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals('Bank', self.elem('.typeSelected').text)
        self.assertEquals('Name', self.elem('.attributeField').text)
        cc = self.browser.find_element_by_id('drag_Bank.corporateCustomers')
        self.assertEquals('Corporate Customers', cc.text)
        ds = self.browser.find_element_by_id('drag_Bank.debtors')
        self.assertEquals('Debtors', ds.text)
        # Add a view element.
        self.browser.find_element_by_id('img_Bank.debtors').click()
        self.elem('a[title="Show debt in results"] > img.arrow').click()
        self.assertEquals(1, len(self.browser.find_elements_by_class_name('viewpath')))
        self.elem('a[title="Show name in results"] > img.arrow').click()
        self.assertEquals(2, len(self.browser.find_elements_by_class_name('viewpath')))
        # Add a constraint
        self.elem('a[title="Add a constraint to debt"]').click()
        Select(self.elem("#attribute5")).select_by_visible_text(">")
        self.elem("#attribute8").clear()
        self.elem("#attribute8").send_keys("1000")
        self.elem("#attributeSubmit").click()
        constraints = self.elems('span.constraint')
        self.assertEquals(1, len(constraints))
        self.assertEquals('> 1000', constraints[0].text)
        self.elem('a[title="Export this query as XML"]').click()
        expected_query = """
<query name="" model="testmodel" view="Bank.debtors.debt Bank.name" longDescription="" sortOrder="Bank.debtors.debt asc">
  <constraint path="Bank.debtors.debt" op=">" value="1000"/>
</query>
        """
        self.assertEquals(expected_query.strip(), self.elem('body').text)

    def test_build_query(self):
        Select(self.elem("#queryClassSelector")).select_by_visible_text("Employee")
        self.elem("#submitClassSelect").click()
        self.elem('a[title="Show name in results"]').click()
        self.elem('a[title="Add a constraint to name"]').click()
        self.elem("#attribute8").clear()
        self.elem("#attribute8").send_keys(u"*ö*")
        self.elem("#attributeSubmit").click()
        self.run_and_expect(4)

    def test_run_query(self):
        query = """
        <query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt ASC" >
          <constraint path="Bank.debtors.debt" op=">" value="25000000" />
        </query>
        """
        self.findLink("Import query from XML").click()
        self.elem('#xml').send_keys(query)
        self.elem('#importQueriesForm input[type="submit"]').click()
        self.run_and_expect(22)

    def run_and_expect(self, n):
        self.elem('#showResult').click()
        summary = self.elem(".im-table-summary")
        self.assertRowCountIs(n)

    def assertRowCountIs(self, n):
        self.assertEquals(n, len(self.elems('.im-table-container tbody tr')))

    def test_add_constraint_set_and(self):
        query = """
        <query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt ASC" >
        </query>
        """
        expected_query = "\n".join([
            ' '.join([
                '<query',
                'name=""',
                'model="testmodel"',
                'view="Bank.name Bank.debtors.debt"',
                'longDescription=""',
                'sortOrder="Bank.debtors.debt asc"',
                'constraintLogic="A and B">'
            ]),
            '  <constraint path="Bank.debtors.debt" code="A" op=">" value="35,000,000"/>',
            '  <constraint path="Bank.name" code="B" op="=" value="Gringotts"/>',
            '</query>'
        ])
        self.findLink("Import query from XML").click()
        self.elem('#xml').send_keys(query)
        self.elem('#importQueriesForm input[type="submit"]').click()
        # Add constraint: Bank.debtors.debt > 35e6
        self.browser.find_element_by_id('img_Bank.debtors').click()
        self.elem('a[title="Add a constraint to debt"]').click()
        Select(self.elem("#attribute5")).select_by_visible_text(">")
        self.elem('#attribute8').send_keys('35,000,000')
        self.elem('#attributeSubmit').click()
        # Add constraint: Bank.name = Gringotts
        self.elem('a[title="Add a constraint to name"]').click()
        Select(self.elem("#attribute7")).select_by_visible_text("Gringotts")
        self.elem('#attributeSubmit').click()
        # Check that the query is as expected.
        self.elem('a[title="Export this query as XML"]').click()
        self.assertEquals(expected_query.strip(), self.elem('body').text)
        self.browser.back()
        # Check that the results are as expected.
        self.elem('#showResult').click()
        self.assertRowCountIs(2)

    def test_add_constraint_set_or(self):
        query = """
        <query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt ASC" >
        </query>
        """
        expected_query = "\n".join([
            ' '.join([
                '<query',
                'name=""',
                'model="testmodel"',
                'view="Bank.name Bank.debtors.debt"',
                'longDescription=""',
                'sortOrder="Bank.debtors.debt asc"',
                'constraintLogic="A or B">'
            ]),
            '  <constraint path="Bank.debtors.debt" code="A" op=">" value="35,000,000"/>',
            '  <constraint path="Bank.name" code="B" op="=" value="Gringotts"/>',
            '</query>'
        ])

        # Perform actions.
        self.findLink("Import query from XML").click()
        self.elem('#xml').send_keys(query)
        self.elem('#importQueriesForm input[type="submit"]').click()
        # Add constraint: Bank.debtors.debt > 35e6
        self.browser.find_element_by_id('img_Bank.debtors').click()
        self.elem('a[title="Add a constraint to debt"]').click()
        Select(self.elem("#attribute5")).select_by_visible_text(">")
        self.elem('#attribute8').send_keys('35,000,000')
        self.elem('#attributeSubmit').click()
        # Add constraint: Bank.name = Gringotts
        self.elem('a[title="Add a constraint to name"]').click()
        Select(self.elem("#attribute7")).select_by_visible_text("Gringotts")
        self.elem('#attributeSubmit').click()
        # Switch the constraint logic to A or B
        self.elem('#constraintLogic').click()
        logic = self.elem('#expr')
        logic.clear()
        logic.send_keys('A or B')
        self.browser.find_element_by_id('editconstraintlogic').click()

        # Check that the query is as expected.
        self.elem('a[title="Export this query as XML"]').click()
        self.assertEquals(expected_query.strip(), self.elem('body').text)
        self.browser.back()
        # Check that the results are as expected.
        self.elem('#showResult').click()
        self.assertRowCountIs(24)

    def load_queries_into_history(self):
        query_1 = ''.join([
            '<query model="testmodel" view="Bank.debtors.debt" sortOrder="Bank.debtors.debt asc">',
            '</query>'
            ])
        query_2 = ''.join([
            '<query model="testmodel" view="Bank.name Bank.debtors.debt" sortOrder="Bank.debtors.debt asc">',
            '<constraint path="Bank.debtors.debt" op=">" value="35,000,000"/>',
            '</query>'
            ])
        # Load queries into session history.
        for q in [query_1, query_2]:
            self.browser.get(self.base_url + '/customQuery.do')
            self.findLink("Import query from XML").click()
            self.elem('#xml').send_keys(q)
            self.elem('#importQueriesForm input[type="submit"]').click()
            self.elem('#showResult').click()
        self.browser.get(self.base_url + '/customQuery.do')

    def test_edit_template(self):
        self.browser.get(self.base_url + '/template.do?name=ManagerLookup&scope=all')
        self.elem('input.editQueryBuilder').click()
        self.assertIn('Query builder', self.browser.title)
        # Edit the constraint.
        self.elem('img[title="Edit this constraint"]').click()
        con_value = self.elem('#attribute8')
        con_value.clear()
        con_value.send_keys('Anne')
        self.elem('#attributeSubmit').click()
        # Check export.
        self.elem('a[title="Export this query as XML"]').click()
        expected_query = '\n'.join([
            '<query name="" model="testmodel" view="Manager.name Manager.title" longDescription="">',
            '  <constraint path="Manager" op="LOOKUP" value="Anne" extraValue=""/>',
            '</query>'])
        self.assertEquals(expected_query, self.elem('body').text)
        self.browser.back()
        # Check results.
        self.elem('#showResult').click()
        self.assertRowCountIs(1)

    def test_query_history(self):
        self.load_queries_into_history()
        self.assertIn('Custom query', self.browser.title)
        self.assertEquals(2, len(self.elems('#modifyQueryForm tbody tr')))
        self.assertEquals('query_2', self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(2)').text)
        root = self.elem('#modifyQueryForm tbody tr:nth-child(2) .historySummaryRoot').text
        self.assertEquals('Bank', root)
        showing = self.elems('#modifyQueryForm tbody tr:nth-child(2) .historySummaryShowing')
        self.assertEquals(2, len(showing))
        self.assertEquals(['Name', 'Debt'], [s.text for s in showing])

    def test_delete_query_from_history(self):
        self.load_queries_into_history()
        self.assertEquals(2, len(self.elems('#modifyQueryForm tbody tr')))
        self.elem('#selected_history_1').click()
        self.elem('#delete_button').click()
        Alert(self.browser).accept()
        self.assertEquals(1, len(self.elems('#modifyQueryForm tbody tr')))

    def test_run_query_in_query_history(self):
        self.load_queries_into_history()

        self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(7) span.fakelink:nth-child(1)').click()
        self.assertRowCountIs(16)

    def test_edit_query_in_query_history(self):
        self.load_queries_into_history()

        self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(7) span.fakelink:nth-child(2)').click()
        self.assertIn('Query builder', self.browser.title)
        self.assertEquals('Bank', self.elem('.typeSelected').text)
        # Edit a constraint.
        self.elem('img[title="Edit this constraint"]').click()
        con_value = self.elem('#attribute8')
        con_value.clear()
        con_value.send_keys('40,000,000')
        self.elem('#attributeSubmit').click()
        # Check results.
        self.elem('#showResult').click()
        self.assertRowCountIs(15)

    def test_export_query_in_query_history(self):
        self.load_queries_into_history()
        expected_query = '\n'.join([
            ' '.join([
                '<query',
                'name="query_2"',
                'model="testmodel"',
                'view="Bank.name Bank.debtors.debt"',
                'longDescription=""',
                'sortOrder="Bank.debtors.debt asc">'
                ]),
            '  <constraint path="Bank.debtors.debt" op=">" value="35,000,000"/>',
            '</query>'])

        self.elem('#modifyQueryForm tbody tr:nth-child(2) td:nth-child(7) span.fakelink:nth-child(3)').click()
        self.assertEquals(expected_query, self.elem('body').text)

    def test_import_query(self):
        link = self.findLink("Import query from XML")
        self.assertIsNotNone(link)
        link.click()
        self.assertIn('Import Query', self.browser.title)
        input_box = self.elem('#xml')
        self.assertIsNotNone(input_box)
        query = ''.join([
            '<query model="testmodel" view="Bank.debtors.debt" sortOrder="Bank.debtors.debt asc">',
            '<constraint path="Bank.debtors.debt" op=">" value="1000"/>',
            '</query>'
            ])

        input_box.send_keys(query)
        self.assertEquals('true', self.elem('#file').get_attribute('disabled'))
        self.elem('#importQueriesForm input[type="submit"]').click()
        wait = WebDriverWait(self.browser, 10)
        wait.until(EC.title_contains('Query builder'))

        self.assertEquals('Bank', self.elem('.typeSelected').text)
        constraints = self.elems('span.constraint')
        self.assertEquals(1, len(constraints))
        self.assertEquals('> 1000', constraints[0].text)
        self.assertEquals(1, len(self.browser.find_elements_by_class_name('viewpath')))

    def test_login_to_view_saved(self):
        link = self.findLink("Login to view saved queries")
        self.assertIsNotNone(link)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
This script is for the accounting purposes of Jetstream
The goal:
    Print a CSV of:
"""
CSV_HEADER="Instance ID, Instance Alias, Username, Provider, Instance Start Date, Image Name, Version Name, Featured Image, Active, Deploy Error, Error, Aborted"

import django; django.setup()
from core.models import Provider, Instance, InstanceStatusHistory, ObjectDoesNotExist

print CSV_HEADER
inst_list = Instance.objects.all()
#inst_list = Instance.objects.filter(source__providermachine__application_version__application__tags__name__icontains='featured')
for inst in inst_list.order_by('id'):
    first_history = inst.get_first_history()
    try:
        instance_id = inst.provider_alias
        username = inst.created_by.username
        machine = inst.source.providermachine
        application_version = machine.application_version
        application = application_version.application
        provider = machine.provider.location
        image_name = application.name
        version_name = application_version.name
        featured_image = application.tags.filter(name__icontains='featured').count() > 0
    except ObjectDoesNotExist:
        image_name = "Deleted Image"
        version_name = "N/A"
        featured_image = False
    last_history = inst.get_last_history()
    history = first_history
    hit_aborted = hit_active = hit_deploy_error = hit_error = False
    while True:
        try:
            #print history.instance.id, history.status, history.start_date
            hit_active = hit_active or (history.status.name.lower() == 'active')
            hit_error = hit_error or (history.status.name.lower() == 'error')
            hit_deploy_error = hit_deploy_error or (history.status.name.lower() == 'deploy_error')
            history = history.next()
        except LookupError:
            break
        except ValueError:
            #NOTE: These value-errors are important debugging tools. but they can be ignored if you aren't hunting down problems.
            #raise
            break
    if not hit_active and not hit_error and not hit_deploy_error:
        hit_aborted = True
    featured_image = 1 if featured_image else 0
    #Magic goes here.
    if hit_active:
        hit_error = 0
        hit_deploy_error = 0
    if hit_error and hit_deploy_error:
        hit_error = 0
    #
    hit_active = 1 if hit_active else 0
    hit_aborted = 1 if hit_aborted else 0
    hit_error = 1 if hit_error else 0
    hit_deploy_error = 1 if hit_deploy_error else 0
    arg_list = [inst.id, instance_id, username, provider, inst.start_date.strftime("%x %X"), image_name, version_name, featured_image, hit_active, hit_deploy_error, hit_error, hit_aborted]
    print ",".join(map(str,arg_list))

<commit_msg>Include 'size' in the instance metrics<commit_after>#!/usr/bin/env python
"""
This script is for the accounting purposes of Jetstream
The goal:
    Print a CSV of:
"""
CSV_HEADER="Instance ID, Instance Alias, Username, Provider, Instance Start Date, Image Name, Version Name, Size Name, Size Alias, Size cpu, Size mem, Size disk, Featured Image, Active, Deploy Error, Error, Aborted"

import django; django.setup()
from core.models import Provider, Instance, InstanceStatusHistory, ObjectDoesNotExist

print CSV_HEADER
inst_list = Instance.objects.all()
#inst_list = Instance.objects.filter(source__providermachine__application_version__application__tags__name__icontains='featured')
for inst in inst_list.order_by('id'):
    first_history = inst.get_first_history()
    try:
        instance_id = inst.provider_alias
        username = inst.created_by.username
        size = inst.get_size()
        machine = inst.source.providermachine
        application_version = machine.application_version
        application = application_version.application
        provider = machine.provider.location
        image_name = application.name
        version_name = application_version.name
        featured_image = application.tags.filter(name__icontains='featured').count() > 0
    except ObjectDoesNotExist:
        image_name = "Deleted Image"
        version_name = "N/A"
        featured_image = False
    last_history = inst.get_last_history()
    history = first_history
    hit_aborted = hit_active = hit_deploy_error = hit_error = False
    while True:
        try:
            #print history.instance.id, history.status, history.start_date
            hit_active = hit_active or (history.status.name.lower() == 'active')
            hit_error = hit_error or (history.status.name.lower() == 'error')
            hit_deploy_error = hit_deploy_error or (history.status.name.lower() == 'deploy_error')
            history = history.next()
        except LookupError:
            break
        except ValueError:
            #NOTE: These value-errors are important debugging tools. but they can be ignored if you aren't hunting down problems.
            #raise
            break
    if not hit_active and not hit_error and not hit_deploy_error:
        hit_aborted = True
    featured_image = 1 if featured_image else 0
    #Magic goes here.
    if hit_active:
        hit_error = 0
        hit_deploy_error = 0
    if hit_error and hit_deploy_error:
        hit_error = 0
    #
    hit_active = 1 if hit_active else 0
    hit_aborted = 1 if hit_aborted else 0
    hit_error = 1 if hit_error else 0
    hit_deploy_error = 1 if hit_deploy_error else 0
    arg_list = [inst.id, instance_id, username, provider, inst.start_date.strftime("%x %X"), image_name, version_name, size.name, size.alias, size.cpu, size.mem, size.disk, featured_image, hit_active, hit_deploy_error, hit_error, hit_aborted]
    print ",".join(map(str,arg_list))

<|endoftext|>


<commit_before># encoding: utf-8

from __future__ import unicode_literals

import re

from marrow.schema.compat import unicode
from marrow.schema.validation.base import truthy, falsy, Instance
from marrow.schema.validation.exc import Concern
from marrow.schema.validation.testing import ValidationTest
from marrow.schema.validation.container import *

INVALID = (None, 1, True)
EMPTY = (tuple(), list(), dict(), "")
STRINGS = (('a', 'b'), ['a', 'b'], {'a': "one", 'b': "two"}, set(['a', 'b']), "foo")
INTEGERS = ((0, 1, 2), [0, 1, 2], {0: 0, 1: 1, 2: 2}, set([0, 1, 2]))
TRUTHY = ((1, True, 'foo'), [1, True, 'foo'], {'a': 1, 'b': True, 1: 'foo'}, set([1, True, 'foo']))
FALSY = ((0, False, ''), [0, False, ''], {None: 0, '': False, 0: ''}, set([0, False, '']))

class TestIterable(ValidationTest):
	validator = Iterable().validate
	valid = EMPTY + STRINGS + INTEGERS + TRUTHY + FALSY
	invalid = INVALID

class TestTruthyIterable(ValidationTest):
	validator = Iterable([truthy]).validate
	valid = EMPTY + TRUTHY + STRINGS
	invalid = INVALID + FALSY + INTEGERS

class TestFalsyIterable(ValidationTest):
	validator = Iterable([falsy]).validate
	valid = EMPTY + FALSY
	invalid = INVALID + TRUTHY

class TestStringyIterable(ValidationTest):
	class Validator(Iterable):
		stringy = Instance(unicode)

	validator = Validator().validate
	valid = EMPTY + STRINGS
	invalid = INVALID + INTEGERS + TRUTHY + FALSY

class TestIterableConcerns(object):
	def test_singular_failure(self):
		try:
			Iterable([truthy]).validate([True, False])
		except Concern as e:
			assert "Element 1" in e.message, "Should identify element failing validation."
			assert not e.concerns, "Should not contain child concerns."

	def test_multiple_failure(self):
		try:
			Iterable([truthy]).validate([0, False])
		except Concern as e:
			assert "multiple" in e.message.lower(), "Should indicate multiple failures."
			assert "Element 1" in e.concerns[1].message, "Should identify element failing validation."
<commit_msg>Added Mapping container tests.<commit_after># encoding: utf-8

from __future__ import unicode_literals

import re

from marrow.schema.compat import unicode
from marrow.schema.validation.base import truthy, falsy, Instance
from marrow.schema.validation.exc import Concern
from marrow.schema.validation.testing import ValidationTest
from marrow.schema.validation.container import *

INVALID = (None, 1, True)
EMPTY = (tuple(), list(), dict(), "")
STRINGS = (('a', 'b'), ['a', 'b'], {'a': "one", 'b': "two"}, set(['a', 'b']), "foo")
INTEGERS = ((0, 1, 2), [0, 1, 2], {0: 0, 1: 1, 2: 2}, set([0, 1, 2]))
TRUTHY = ((1, True, 'foo'), [1, True, 'foo'], {'a': 1, 'b': True, 1: 'foo'}, set([1, True, 'foo']))
FALSY = ((0, False, ''), [0, False, ''], {None: 0, '': False, 0: ''}, set([0, False, '']))
NONMAP = (tuple(), list(), "")

class TestIterable(ValidationTest):
	validator = Iterable().validate
	valid = EMPTY + STRINGS + INTEGERS + TRUTHY + FALSY
	invalid = INVALID

class TestTruthyIterable(ValidationTest):
	validator = Iterable([truthy]).validate
	valid = EMPTY + TRUTHY + STRINGS
	invalid = INVALID + FALSY + INTEGERS

class TestFalsyIterable(ValidationTest):
	validator = Iterable([falsy]).validate
	valid = EMPTY + FALSY
	invalid = INVALID + TRUTHY

class TestStringyIterable(ValidationTest):
	class Validator(Iterable):
		stringy = Instance(unicode)

	validator = Validator().validate
	valid = EMPTY + STRINGS
	invalid = INVALID + INTEGERS + TRUTHY + FALSY

class TestIterableConcerns(object):
	def test_singular_failure(self):
		try:
			Iterable([truthy]).validate([True, False])
		except Concern as e:
			assert "Element 1" in e.message, "Should identify element failing validation."
			assert not e.concerns, "Should not contain child concerns."

	def test_multiple_failure(self):
		try:
			Iterable([truthy]).validate([0, False])
		except Concern as e:
			assert "multiple" in e.message.lower(), "Should indicate multiple failures."
			assert "Element 1" in e.concerns[1].message, "Should identify element failing validation."

class TestMapping(ValidationTest):
	validator = Mapping([Instance(unicode)]).validate
	valid = ({}, {1: 'foo', 2: 'bar'})
	invalid = INVALID + NONMAP + ({1: 2, 2: 'baz'}, )

class TestMappingConcerns(object):
	def test_singular_failure(self):
		try:
			Mapping([truthy]).validate({'bob': True, 'dole': False})
		except Concern as e:
			assert "dole" in e.message, "Should identify element failing validation."
			assert not e.concerns, "Should not contain child concerns."

	def test_multiple_failure(self):
		try:
			Mapping([truthy]).validate({'bob': False, 'dole': False})
		except Concern as e:
			assert "multiple" in e.message.lower(), "Should indicate multiple failures."
			assert "dole" in e.concerns[0].message or "dole" in e.concerns[1].message, "Should identify element failing validation."
<|endoftext|>


<commit_before>from mocket.mocket import Mocket, MocketEntry, Mocketizer, mocketize

__all__ = ("mocketize", "Mocket", "MocketEntry", "Mocketizer")

__version__ = "3.9.43"
<commit_msg>Update __init__.py<commit_after>from mocket.mocket import Mocket, MocketEntry, Mocketizer, mocketize

__all__ = ("mocketize", "Mocket", "MocketEntry", "Mocketizer")

__version__ = "3.9.44"
<|endoftext|>


<commit_before>from __future__ import print_function
import io, os, sys, json, binascii, six, tempfile, zipfile
from twisted.internet import reactor, defer
from twisted.internet.defer import inlineCallbacks, returnValue
from txwormhole.transcribe import Wormhole, WrongPasswordError
from txwormhole.transit import TransitReceiver
from wormhole.errors import TransferError
from .progress import ProgressPrinter

APPID = u"lothar.com/wormhole/text-or-file-xfer"

class RespondError(Exception):
    def __init__(self, response):
        self.response = response

def receive_twisted(args, reactor=reactor):
    return TwistedReceiver(args, reactor).go()

class TwistedReceiver:
    def __init__(self, args, reactor=reactor):
        assert isinstance(args.relay_url, type(u""))
        self.args = args
        self._reactor = reactor

    def msg(self, *args, **kwargs):
        print(*args, file=self.args.stdout, **kwargs)

    # TODO: @handle_server_error
    def go(self):
        d = defer.succeed(None)
        tor_manager = None
        if self.args.tor:
            _start = self.args.timing.add_event("import TorManager")
            from txwormhole.tor_manager import TorManager
            self.args.timing.finish_event(_start)
            tor_manager = TorManager(self._reactor, timing=self.args.timing)
            # For now, block everything until Tor has started. Soon: launch
            # tor in parallel with everything else, make sure the TorManager
            # can lazy-provide an endpoint, and overlap the startup process
            # with the user handing off the wormhole code
            d.addCallback(lambda _: tor_manager.start())
        def _make_wormhole(_):
            self._w = Wormhole(APPID, self.args.relay_url, tor_manager,
                               timing=self.args.timing,
                               reactor=self._reactor)
        d.addCallback(_make_wormhole)
        d.addCallback(lambda _: self._go(self._w, tor_manager))
        def _always_close(res):
            d2 = self._w.close()
            d2.addBoth(lambda _: res)
            return d2
        d.addBoth(_always_close)
        # I wanted to do this instead:
        #
        #    try:
        #        yield self._go(w, tor_manager)
        #    finally:
        #        yield w.close()
        #
        # but when _go had a UsageError, the stacktrace was always displayed
        # as coming from the "yield self._go" line, which wasn't very useful
        # for tracking it down.
        return d

    @inlineCallbacks
    def _go(self, w, tor_manager):
        yield self.handle_code(w)
        verifier = yield w.get_verifier()
        self.show_verifier(verifier)
        them_d = yield self.get_data(w)
        try:
            if "message" in them_d:
                yield self.handle_text(them_d, w)
                returnValue(0)
            if "file" in them_d:
                f = self.handle_file(them_d)
                rp = yield self.establish_transit(w, them_d, tor_manager)
                yield self.transfer_data(rp, f)
                self.write_file(f)
                yield self.close_transit(rp)
            elif "directory" in them_d:
                f = self.handle_directory(them_d)
                rp = yield self.establish_transit(w, them_d, tor_manager)
                yield self.transfer_data(rp, f)
                self.write_directory(f)
                yield self.close_transit(rp)
            else:
                self.msg(u"I don't know what they're offering\n")
                self.msg(u"Offer details:", them_d)
                raise RespondError({"error": "unknown offer type"})
        except RespondError as r:
            data = json.dumps(r.response).encode("utf-8")
            yield w.send_data(data)
            raise SystemExit(1)
        returnValue(0)

    @inlineCallbacks
    def handle_code(self, w):
        code = self.args.code
        if self.args.zeromode:
            assert not code
            code = u"0-"
        if not code:
            code = yield w.input_code("Enter receive wormhole code: ",
                                      self.args.code_length)
        yield w.set_code(code)

    def show_verifier(self, verifier):
        verifier_hex = binascii.hexlify(verifier).decode("ascii")
        if self.args.verify:
            self.msg(u"Verifier %s." % verifier_hex)

    @inlineCallbacks
    def get_data(self, w):
        try:
            them_bytes = yield w.get_data()
        except WrongPasswordError as e:
            raise TransferError(u"ERROR: " + e.explain())
        them_d = json.loads(them_bytes.decode("utf-8"))
        if "error" in them_d:
            raise TransferError(u"ERROR: " + them_d["error"])
        returnValue(them_d)

    @inlineCallbacks
    def handle_text(self, them_d, w):
        # we're receiving a text message
        self.msg(them_d["message"])
        data = json.dumps({"message_ack": "ok"}).encode("utf-8")
        yield w.send_data(data)

    def handle_file(self, them_d):
        file_data = them_d["file"]
        self.abs_destname = self.decide_destname("file",
                                                 file_data["filename"])
        self.xfersize = file_data["filesize"]

        self.msg(u"Receiving file (%d bytes) into: %s" %
                 (self.xfersize, os.path.basename(self.abs_destname)))
        self.ask_permission()
        tmp_destname = self.abs_destname + ".tmp"
        return open(tmp_destname, "wb")

    def handle_directory(self, them_d):
        file_data = them_d["directory"]
        zipmode = file_data["mode"]
        if zipmode != "zipfile/deflated":
            self.msg(u"Error: unknown directory-transfer mode '%s'" % (zipmode,))
            raise RespondError({"error": "unknown mode"})
        self.abs_destname = self.decide_destname("directory",
                                                 file_data["dirname"])
        self.xfersize = file_data["zipsize"]

        self.msg(u"Receiving directory (%d bytes) into: %s/" %
                 (self.xfersize, os.path.basename(self.abs_destname)))
        self.msg(u"%d files, %d bytes (uncompressed)" %
                 (file_data["numfiles"], file_data["numbytes"]))
        self.ask_permission()
        return tempfile.SpooledTemporaryFile()

    def decide_destname(self, mode, destname):
        # the basename() is intended to protect us against
        # "~/.ssh/authorized_keys" and other attacks
        destname = os.path.basename(destname)
        if self.args.output_file:
            destname = self.args.output_file # override
        abs_destname = os.path.join(self.args.cwd, destname)

        # get confirmation from the user before writing to the local directory
        if os.path.exists(abs_destname):
            self.msg(u"Error: refusing to overwrite existing %s %s" %
                     (mode, destname))
            raise RespondError({"error": "%s already exists" % mode})
        return abs_destname

    def ask_permission(self):
        _start = self.args.timing.add_event("permission", waiting="user")
        while True and not self.args.accept_file:
            ok = six.moves.input("ok? (y/n): ")
            if ok.lower().startswith("y"):
                break
            print(u"transfer rejected", file=sys.stderr)
            self.args.timing.finish_event(_start, answer="no")
            raise RespondError({"error": "transfer rejected"})
        self.args.timing.finish_event(_start, answer="yes")

    @inlineCallbacks
    def establish_transit(self, w, them_d, tor_manager):
        transit_key = w.derive_key(APPID+u"/transit-key")
        transit_receiver = TransitReceiver(self.args.transit_helper,
                                           no_listen=self.args.no_listen,
                                           tor_manager=tor_manager,
                                           reactor=self._reactor,
                                           timing=self.args.timing)
        transit_receiver.set_transit_key(transit_key)
        direct_hints = yield transit_receiver.get_direct_hints()
        relay_hints = yield transit_receiver.get_relay_hints()
        data = json.dumps({
            "file_ack": "ok",
            "transit": {
                "direct_connection_hints": direct_hints,
                "relay_connection_hints": relay_hints,
                },
            }).encode("utf-8")
        yield w.send_data(data)

        # now receive the rest of the owl
        tdata = them_d["transit"]
        transit_receiver.add_their_direct_hints(tdata["direct_connection_hints"])
        transit_receiver.add_their_relay_hints(tdata["relay_connection_hints"])
        record_pipe = yield transit_receiver.connect()
        returnValue(record_pipe)

    @inlineCallbacks
    def transfer_data(self, record_pipe, f):
        self.msg(u"Receiving (%s).." % record_pipe.describe())

        _start = self.args.timing.add_event("rx file")
        progress_stdout = self.args.stdout
        if self.args.hide_progress:
            progress_stdout = io.StringIO()
        progress = ProgressPrinter(self.xfersize, progress_stdout)

        progress.start()
        received = yield record_pipe.writeToFile(f, self.xfersize,
                                                 progress.update)
        progress.finish()
        self.args.timing.finish_event(_start)

        # except TransitError
        if received < self.xfersize:
            self.msg()
            self.msg(u"Connection dropped before full file received")
            self.msg(u"got %d bytes, wanted %d" % (received, self.xfersize))
            returnValue(1) # TODO: exit properly
        assert received == self.xfersize

    def write_file(self, f):
        tmp_name = f.name
        f.close()
        os.rename(tmp_name, self.abs_destname)
        self.msg(u"Received file written to %s" %
                 os.path.basename(self.abs_destname))

    def write_directory(self, f):
        self.msg(u"Unpacking zipfile..")
        _start = self.args.timing.add_event("unpack zip")
        with zipfile.ZipFile(f, "r", zipfile.ZIP_DEFLATED) as zf:
            zf.extractall(path=self.abs_destname)
            # extractall() appears to offer some protection against
            # malicious pathnames. For example, "/tmp/oops" and
            # "../tmp/oops" both do the same thing as the (safe)
            # "tmp/oops".
        self.msg(u"Received files written to %s/" %
                 os.path.basename(self.abs_destname))
        f.close()
        self.args.timing.finish_event(_start)

    @inlineCallbacks
    def close_transit(self, record_pipe):
        _start = self.args.timing.add_event("ack")
        yield record_pipe.send_record(b"ok\n")
        yield record_pipe.close()
        self.args.timing.finish_event(_start)
<commit_msg>cmd_receive: use wait=True for the last ack<commit_after>from __future__ import print_function
import io, os, sys, json, binascii, six, tempfile, zipfile
from twisted.internet import reactor, defer
from twisted.internet.defer import inlineCallbacks, returnValue
from txwormhole.transcribe import Wormhole, WrongPasswordError
from txwormhole.transit import TransitReceiver
from wormhole.errors import TransferError
from .progress import ProgressPrinter

APPID = u"lothar.com/wormhole/text-or-file-xfer"

class RespondError(Exception):
    def __init__(self, response):
        self.response = response

def receive_twisted(args, reactor=reactor):
    return TwistedReceiver(args, reactor).go()

class TwistedReceiver:
    def __init__(self, args, reactor=reactor):
        assert isinstance(args.relay_url, type(u""))
        self.args = args
        self._reactor = reactor

    def msg(self, *args, **kwargs):
        print(*args, file=self.args.stdout, **kwargs)

    # TODO: @handle_server_error
    def go(self):
        d = defer.succeed(None)
        tor_manager = None
        if self.args.tor:
            _start = self.args.timing.add_event("import TorManager")
            from txwormhole.tor_manager import TorManager
            self.args.timing.finish_event(_start)
            tor_manager = TorManager(self._reactor, timing=self.args.timing)
            # For now, block everything until Tor has started. Soon: launch
            # tor in parallel with everything else, make sure the TorManager
            # can lazy-provide an endpoint, and overlap the startup process
            # with the user handing off the wormhole code
            d.addCallback(lambda _: tor_manager.start())
        def _make_wormhole(_):
            self._w = Wormhole(APPID, self.args.relay_url, tor_manager,
                               timing=self.args.timing,
                               reactor=self._reactor)
        d.addCallback(_make_wormhole)
        d.addCallback(lambda _: self._go(self._w, tor_manager))
        def _always_close(res):
            d2 = self._w.close()
            d2.addBoth(lambda _: res)
            return d2
        d.addBoth(_always_close)
        # I wanted to do this instead:
        #
        #    try:
        #        yield self._go(w, tor_manager)
        #    finally:
        #        yield w.close()
        #
        # but when _go had a UsageError, the stacktrace was always displayed
        # as coming from the "yield self._go" line, which wasn't very useful
        # for tracking it down.
        return d

    @inlineCallbacks
    def _go(self, w, tor_manager):
        yield self.handle_code(w)
        verifier = yield w.get_verifier()
        self.show_verifier(verifier)
        them_d = yield self.get_data(w)
        try:
            if "message" in them_d:
                yield self.handle_text(them_d, w)
                returnValue(0)
            if "file" in them_d:
                f = self.handle_file(them_d)
                rp = yield self.establish_transit(w, them_d, tor_manager)
                yield self.transfer_data(rp, f)
                self.write_file(f)
                yield self.close_transit(rp)
            elif "directory" in them_d:
                f = self.handle_directory(them_d)
                rp = yield self.establish_transit(w, them_d, tor_manager)
                yield self.transfer_data(rp, f)
                self.write_directory(f)
                yield self.close_transit(rp)
            else:
                self.msg(u"I don't know what they're offering\n")
                self.msg(u"Offer details:", them_d)
                raise RespondError({"error": "unknown offer type"})
        except RespondError as r:
            data = json.dumps(r.response).encode("utf-8")
            yield w.send_data(data)
            raise SystemExit(1)
        returnValue(0)

    @inlineCallbacks
    def handle_code(self, w):
        code = self.args.code
        if self.args.zeromode:
            assert not code
            code = u"0-"
        if not code:
            code = yield w.input_code("Enter receive wormhole code: ",
                                      self.args.code_length)
        yield w.set_code(code)

    def show_verifier(self, verifier):
        verifier_hex = binascii.hexlify(verifier).decode("ascii")
        if self.args.verify:
            self.msg(u"Verifier %s." % verifier_hex)

    @inlineCallbacks
    def get_data(self, w):
        try:
            them_bytes = yield w.get_data()
        except WrongPasswordError as e:
            raise TransferError(u"ERROR: " + e.explain())
        them_d = json.loads(them_bytes.decode("utf-8"))
        if "error" in them_d:
            raise TransferError(u"ERROR: " + them_d["error"])
        returnValue(them_d)

    @inlineCallbacks
    def handle_text(self, them_d, w):
        # we're receiving a text message
        self.msg(them_d["message"])
        data = json.dumps({"message_ack": "ok"}).encode("utf-8")
        yield w.send_data(data, wait=True)

    def handle_file(self, them_d):
        file_data = them_d["file"]
        self.abs_destname = self.decide_destname("file",
                                                 file_data["filename"])
        self.xfersize = file_data["filesize"]

        self.msg(u"Receiving file (%d bytes) into: %s" %
                 (self.xfersize, os.path.basename(self.abs_destname)))
        self.ask_permission()
        tmp_destname = self.abs_destname + ".tmp"
        return open(tmp_destname, "wb")

    def handle_directory(self, them_d):
        file_data = them_d["directory"]
        zipmode = file_data["mode"]
        if zipmode != "zipfile/deflated":
            self.msg(u"Error: unknown directory-transfer mode '%s'" % (zipmode,))
            raise RespondError({"error": "unknown mode"})
        self.abs_destname = self.decide_destname("directory",
                                                 file_data["dirname"])
        self.xfersize = file_data["zipsize"]

        self.msg(u"Receiving directory (%d bytes) into: %s/" %
                 (self.xfersize, os.path.basename(self.abs_destname)))
        self.msg(u"%d files, %d bytes (uncompressed)" %
                 (file_data["numfiles"], file_data["numbytes"]))
        self.ask_permission()
        return tempfile.SpooledTemporaryFile()

    def decide_destname(self, mode, destname):
        # the basename() is intended to protect us against
        # "~/.ssh/authorized_keys" and other attacks
        destname = os.path.basename(destname)
        if self.args.output_file:
            destname = self.args.output_file # override
        abs_destname = os.path.join(self.args.cwd, destname)

        # get confirmation from the user before writing to the local directory
        if os.path.exists(abs_destname):
            self.msg(u"Error: refusing to overwrite existing %s %s" %
                     (mode, destname))
            raise RespondError({"error": "%s already exists" % mode})
        return abs_destname

    def ask_permission(self):
        _start = self.args.timing.add_event("permission", waiting="user")
        while True and not self.args.accept_file:
            ok = six.moves.input("ok? (y/n): ")
            if ok.lower().startswith("y"):
                break
            print(u"transfer rejected", file=sys.stderr)
            self.args.timing.finish_event(_start, answer="no")
            raise RespondError({"error": "transfer rejected"})
        self.args.timing.finish_event(_start, answer="yes")

    @inlineCallbacks
    def establish_transit(self, w, them_d, tor_manager):
        transit_key = w.derive_key(APPID+u"/transit-key")
        transit_receiver = TransitReceiver(self.args.transit_helper,
                                           no_listen=self.args.no_listen,
                                           tor_manager=tor_manager,
                                           reactor=self._reactor,
                                           timing=self.args.timing)
        transit_receiver.set_transit_key(transit_key)
        direct_hints = yield transit_receiver.get_direct_hints()
        relay_hints = yield transit_receiver.get_relay_hints()
        data = json.dumps({
            "file_ack": "ok",
            "transit": {
                "direct_connection_hints": direct_hints,
                "relay_connection_hints": relay_hints,
                },
            }).encode("utf-8")
        yield w.send_data(data)

        # now receive the rest of the owl
        tdata = them_d["transit"]
        transit_receiver.add_their_direct_hints(tdata["direct_connection_hints"])
        transit_receiver.add_their_relay_hints(tdata["relay_connection_hints"])
        record_pipe = yield transit_receiver.connect()
        returnValue(record_pipe)

    @inlineCallbacks
    def transfer_data(self, record_pipe, f):
        self.msg(u"Receiving (%s).." % record_pipe.describe())

        _start = self.args.timing.add_event("rx file")
        progress_stdout = self.args.stdout
        if self.args.hide_progress:
            progress_stdout = io.StringIO()
        progress = ProgressPrinter(self.xfersize, progress_stdout)

        progress.start()
        received = yield record_pipe.writeToFile(f, self.xfersize,
                                                 progress.update)
        progress.finish()
        self.args.timing.finish_event(_start)

        # except TransitError
        if received < self.xfersize:
            self.msg()
            self.msg(u"Connection dropped before full file received")
            self.msg(u"got %d bytes, wanted %d" % (received, self.xfersize))
            returnValue(1) # TODO: exit properly
        assert received == self.xfersize

    def write_file(self, f):
        tmp_name = f.name
        f.close()
        os.rename(tmp_name, self.abs_destname)
        self.msg(u"Received file written to %s" %
                 os.path.basename(self.abs_destname))

    def write_directory(self, f):
        self.msg(u"Unpacking zipfile..")
        _start = self.args.timing.add_event("unpack zip")
        with zipfile.ZipFile(f, "r", zipfile.ZIP_DEFLATED) as zf:
            zf.extractall(path=self.abs_destname)
            # extractall() appears to offer some protection against
            # malicious pathnames. For example, "/tmp/oops" and
            # "../tmp/oops" both do the same thing as the (safe)
            # "tmp/oops".
        self.msg(u"Received files written to %s/" %
                 os.path.basename(self.abs_destname))
        f.close()
        self.args.timing.finish_event(_start)

    @inlineCallbacks
    def close_transit(self, record_pipe):
        _start = self.args.timing.add_event("ack")
        yield record_pipe.send_record(b"ok\n")
        yield record_pipe.close()
        self.args.timing.finish_event(_start)
<|endoftext|>