<commit_before>import numpy as np
import tensorflow as tf
import h5py
from sklearn.preprocessing import OneHotEncoder
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import time
import visualizationHelper

startTime = time.time()

print('==> Experiment 1i')

def loadData(filepath):

	print('==> Loading data from {}'.format(filepath))
	f = h5py.File(filepath)
	X_train = np.array(f.get('trainingFeatures'))
	y_train = np.array(f.get('trainingLabels'))
	X_test = np.array(f.get('validationFeatures'))
	y_test = np.array(f.get('validationLabels'))
	del f
	print('==> Data sizes:',X_train.shape, y_train.shape, X_test.shape, y_test.shape)

	# Transform labels into on-hot encoding form
	enc = OneHotEncoder()
	y_train = enc.fit_transform(y_train.copy()).astype(int).toarray()
	y_test = enc.fit_transform(y_test.copy()).astype(int).toarray()

	return [X_train, y_train, X_test, y_test]

# Neural-network model set-up
# Functions for initializing neural nets parameters
def init_weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float64)
  return tf.Variable(initial)

def init_bias_variable(shape):
  initial = tf.constant(0.1, shape=shape, dtype=tf.float64)
  return tf.Variable(initial)

def runNeuralNet(num_features, hidden_layer_size, X_train, y_train, X_test, y_test, batchSize, numEpochs):

	'''
		NN config parameters
	'''

	num_classes = y_test.shape[1]

	print('==> Creating Neural net with %d features, %d hidden units, and %d classes'%(num_features, hidden_layer_size, num_classes))

	# Set-up NN layers
	x = tf.placeholder(tf.float64, [None, num_features])
	W1 = init_weight_variable([num_features, hidden_layer_size])
	b1 = init_bias_variable([hidden_layer_size])

	# Hidden layer activation function: ReLU
	h1 = tf.nn.relu(tf.matmul(x, W1) + b1)

	W2 = init_weight_variable([hidden_layer_size, num_classes])
	b2 = init_bias_variable([num_classes])

	# Softmax layer (Output), dtype = float64
	y = tf.matmul(h1, W2) + b2

	# NN desired value (labels)
	y_ = tf.placeholder(tf.float64, [None, num_classes])

	# Loss function
	cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
	train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

	sess = tf.InteractiveSession()

	correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
	accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))
	sess.run(tf.global_variables_initializer())

	'''
		Training config
	'''
	numTrainingVec = len(X_train)
	print_freq = 5

	train_accuracies = []
	test_accuracies = []
	train_costs = []
	test_costs = []
	# track which epochs you grab data on
	epoch_numbers = []

	print('Training with %d samples, a batch size of %d, for %d epochs'%(numTrainingVec, batchSize, numEpochs))

	for epoch in range(numEpochs):

		epochStart = time.time()
		for i in range(0,numTrainingVec,batchSize):

			# Batch Data
			batchEndPoint = min(i+batchSize, numTrainingVec)
			trainBatchData = X_train[i:batchEndPoint]
			trainBatchLabel = y_train[i:batchEndPoint]

			train_step.run(feed_dict={x: trainBatchData, y_: trainBatchLabel})

		epochEnd = time.time()

		# Print accuracy

		if (epoch + 1) % print_freq == 0:
			# calculate the accuracies and costs at this epoch
			train_accuracy = accuracy.eval(feed_dict={x:X_train, y_: y_train})
			test_accuracy = accuracy.eval(feed_dict={x: X_test, y_: y_test})
			train_cost = cross_entropy.eval(feed_dict={x:X_train, y_: y_train})
			test_cost = cross_entropy.eval(feed_dict={x: X_test, y_: y_test})
			# update the lists
			train_accuracies += [train_accuracy]
			test_accuracies += [test_accuracy]
			train_costs += [train_cost]
			test_costs += [test_cost]
			epoch_numbers += [epoch]
			print("epoch: %d, time: %g, t acc, v acc, t cost, v cost: %g, %g, %g, %g"%(epoch+1, epochEnd - epochStart, train_accuracy, test_accuracy, train_cost, test_cost))

	# Validation
	train_accuracy = accuracy.eval(feed_dict={x:X_train, y_: y_train})
	test_accuracy = accuracy.eval(feed_dict={x: X_test, y_: y_test})
	train_cost = cross_entropy.eval(feed_dict={x:X_train, y_: y_train})
	test_cost = cross_entropy.eval(feed_dict={x:X_test, y_: y_test})
	print("test accuracy %g"%(test_accuracy))
	return [train_accuracies, test_accuracies, train_costs, test_costs, epoch_numbers, W1.eval(), W2.eval()]

'''
our main  - compare regularization methods
'''

'''
 set params
'''
startExperiment = time.time()

numEpochs = 5
batchSize = 1000
numHiddenNeurons = 100

[X_train, y_train, X_test, y_test] = loadData('pylon2/ci560sp/cstrong/taylorswift_smallDataset_71_7.mat')

# cubic root
X_train_cr = np.cbrt(X_train)
X_test_cr = np.cbrt(X_test)

[trainingAccuracies, testAccuracies, trainingCosts, testCosts, epochNumbers, W1, W2] = runNeuralNet(121, numHiddenNeurons, X_train_cr, y_train, X_test_cr, y_test, batchSize, numEpochs)

#W1cols = [W1[:,i] for i in range(W1.shape[1])]
# make them intro matrices by nesting them again
#W1colsAsMatrices = [[curCol.tolist()] for curCol in W1cols]
#visualizeWeights(W1colsAsMatrices, 10, 'testvisualize.png')
visualizationHelper.visualizeColVecs(W1, 5, 'testvisualizecols.png')

visualizationHelper.visualizeColVecsGroupedByOctave(W1, 5, 'testvisualizecols_grouped.png')

visualizationHelper.visualizeWeights([W1, W2], 1, 'testvisualizeweights.png')

endExperiment = time.time()

'''
Printing results
'''
print("--------------------------")
print("Summary Of Results - exp1o, Visualization")
print("--------------------------")
print("Epoch Numbers: %s"%str(epochNumbers))
print("Training Accuracy: %s"%str(trainingAccuracies))
print("Test Accuracy: %s"%str(testAccuracies))
print("Training Cost: %s"%str(trainingCosts))
print("Test Cost: %s"%str(testCosts))
print("Time: %g"%(endExperiment - startExperiment))

'''
Plotting results
'''

<commit_msg>exp1o update<commit_after>import numpy as np
import tensorflow as tf
import h5py
from sklearn.preprocessing import OneHotEncoder
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import time
import visualizationHelper

startTime = time.time()

print('==> Experiment 1i')

def loadData(filepath):

	print('==> Loading data from {}'.format(filepath))
	f = h5py.File(filepath)
	X_train = np.array(f.get('trainingFeatures'))
	y_train = np.array(f.get('trainingLabels'))
	X_test = np.array(f.get('validationFeatures'))
	y_test = np.array(f.get('validationLabels'))
	del f
	print('==> Data sizes:',X_train.shape, y_train.shape, X_test.shape, y_test.shape)

	# Transform labels into on-hot encoding form
	enc = OneHotEncoder()
	y_train = enc.fit_transform(y_train.copy()).astype(int).toarray()
	y_test = enc.fit_transform(y_test.copy()).astype(int).toarray()

	return [X_train, y_train, X_test, y_test]

# Neural-network model set-up
# Functions for initializing neural nets parameters
def init_weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float64)
  return tf.Variable(initial)

def init_bias_variable(shape):
  initial = tf.constant(0.1, shape=shape, dtype=tf.float64)
  return tf.Variable(initial)

def runNeuralNet(num_features, hidden_layer_size, X_train, y_train, X_test, y_test, batchSize, numEpochs):

	'''
		NN config parameters
	'''

	num_classes = y_test.shape[1]

	print('==> Creating Neural net with %d features, %d hidden units, and %d classes'%(num_features, hidden_layer_size, num_classes))

	# Set-up NN layers
	x = tf.placeholder(tf.float64, [None, num_features])
	W1 = init_weight_variable([num_features, hidden_layer_size])
	b1 = init_bias_variable([hidden_layer_size])

	# Hidden layer activation function: ReLU
	h1 = tf.nn.relu(tf.matmul(x, W1) + b1)

	W2 = init_weight_variable([hidden_layer_size, num_classes])
	b2 = init_bias_variable([num_classes])

	# Softmax layer (Output), dtype = float64
	y = tf.matmul(h1, W2) + b2

	# NN desired value (labels)
	y_ = tf.placeholder(tf.float64, [None, num_classes])

	# Loss function
	cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
	train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

	sess = tf.InteractiveSession()

	correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
	accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))
	sess.run(tf.global_variables_initializer())

	'''
		Training config
	'''
	numTrainingVec = len(X_train)
	print_freq = 5

	train_accuracies = []
	test_accuracies = []
	train_costs = []
	test_costs = []
	# track which epochs you grab data on
	epoch_numbers = []

	print('Training with %d samples, a batch size of %d, for %d epochs'%(numTrainingVec, batchSize, numEpochs))

	for epoch in range(numEpochs):

		epochStart = time.time()
		for i in range(0,numTrainingVec,batchSize):

			# Batch Data
			batchEndPoint = min(i+batchSize, numTrainingVec)
			trainBatchData = X_train[i:batchEndPoint]
			trainBatchLabel = y_train[i:batchEndPoint]

			train_step.run(feed_dict={x: trainBatchData, y_: trainBatchLabel})

		epochEnd = time.time()

		# Print accuracy

		if (epoch + 1) % print_freq == 0:
			# calculate the accuracies and costs at this epoch
			train_accuracy = accuracy.eval(feed_dict={x:X_train, y_: y_train})
			test_accuracy = accuracy.eval(feed_dict={x: X_test, y_: y_test})
			train_cost = cross_entropy.eval(feed_dict={x:X_train, y_: y_train})
			test_cost = cross_entropy.eval(feed_dict={x: X_test, y_: y_test})
			# update the lists
			train_accuracies += [train_accuracy]
			test_accuracies += [test_accuracy]
			train_costs += [train_cost]
			test_costs += [test_cost]
			epoch_numbers += [epoch]
			print("epoch: %d, time: %g, t acc, v acc, t cost, v cost: %g, %g, %g, %g"%(epoch+1, epochEnd - epochStart, train_accuracy, test_accuracy, train_cost, test_cost))

	# Validation
	train_accuracy = accuracy.eval(feed_dict={x:X_train, y_: y_train})
	test_accuracy = accuracy.eval(feed_dict={x: X_test, y_: y_test})
	train_cost = cross_entropy.eval(feed_dict={x:X_train, y_: y_train})
	test_cost = cross_entropy.eval(feed_dict={x:X_test, y_: y_test})
	print("test accuracy %g"%(test_accuracy))
	return [train_accuracies, test_accuracies, train_costs, test_costs, epoch_numbers, W1.eval(), W2.eval()]

'''
our main  - compare regularization methods
'''

'''
 set params
'''
startExperiment = time.time()

numEpochs = 5
batchSize = 1000
numHiddenNeurons = 100

[X_train, y_train, X_test, y_test] = loadData('pylon2/ci560sp/cstrong/exp1/taylorswift_smallDataset_71_7.mat')

# cubic root
X_train_cr = np.cbrt(X_train)
X_test_cr = np.cbrt(X_test)

[trainingAccuracies, testAccuracies, trainingCosts, testCosts, epochNumbers, W1, W2] = runNeuralNet(121, numHiddenNeurons, X_train_cr, y_train, X_test_cr, y_test, batchSize, numEpochs)

#W1cols = [W1[:,i] for i in range(W1.shape[1])]
# make them intro matrices by nesting them again
#W1colsAsMatrices = [[curCol.tolist()] for curCol in W1cols]
#visualizeWeights(W1colsAsMatrices, 10, 'testvisualize.png')
visualizationHelper.visualizeColVecs(W1, 5, 'testvisualizecols.png')

visualizationHelper.visualizeColVecsGroupedByOctave(W1, 5, 24, 'testvisualizecols_grouped.png')

visualizationHelper.visualizeWeights([W1, W2], 1, 'testvisualizeweights.png')

endExperiment = time.time()

'''
Printing results
'''
print("--------------------------")
print("Summary Of Results - exp1o, Visualization")
print("--------------------------")
print("Epoch Numbers: %s"%str(epochNumbers))
print("Training Accuracy: %s"%str(trainingAccuracies))
print("Test Accuracy: %s"%str(testAccuracies))
print("Training Cost: %s"%str(trainingCosts))
print("Test Cost: %s"%str(testCosts))
print("Time: %g"%(endExperiment - startExperiment))

'''
Plotting results
'''

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
hbasta._api

API Wrapper for HBase Thrift Client
"""

import sys
import logging
from itertools import imap

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol

from hbase import Hbase
from hbase.ttypes import *

class HBasta(object):
    """HBase API entry point"""

    LOG = logging.getLogger("HBasta")

    def __init__(self, host, port):
        """Initialize client.

        Params:
            hostnport - Tuple of (host, port) to connect to
        """
        self._hostnport = (host, int(port))
        self._client = None

    @property
    def client(self):
        """Lazy load of the underlying client"""
        if not self._client:
            self.LOG.debug("* Connecting to HBase at: %s", self._hostnport)
            host, port = self._hostnport
            transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port))
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            self._client  = Hbase.Client(protocol)
            transport.open()
        return self._client

    def create_table(self, table, col_families):
        """Create a new HBase table

        Params
            table - Table name
            col_families - list of column family names
        """
        self.client.createTable(table, [ColumnDescriptor({'name': c+':'}) \
                                    for c in col_families])

    def enable_table(self, table):
        """Enable an HBase table"""
        self.client.enableTable(table)

    def disable_table(self, table):
        """Disable an HBase table"""
        self.client.disableTable(table)

    def is_table_enabled(self, table):
        """Check if table is enabled"""
        return self.client.isTableEnabled(table)

    def get_table_names(self):
        """Get list of all available table names"""
        return self.client.getTableNames()

    def get_row(self, table, row, colspec=None):
        """Get single row of data, possibly filtered
        using the colspec construct

        Params:
            table - Table name
            key - Row key
            colspec - Specifier of which columns to return, in the form of list of column names
        """
        if not colspec:
            ret = self.client.getRow(table, row)
        else:
            ret = self.client.getRowWithColumns(table, row, colspec)

        if ret:
            return dict(imap(lambda i: (i[0], i[1].value), \
                            ret[0].columns.iteritems()))
        else:
            return None

    def scanner_open(self, table, start_row, colspec):
        """Open a scanner for table at given start_row,
        fetching columns as specified in colspec"""
        return self.client.scannerOpen(table, start_row, colspec)

    def scanner_close(self, scanner_id):
        """Close a scanner"""
        self.client.scannerClose(scanner_id)

    def scanner_get(self, scanner_id):
        """Return current row scanner is pointing to."""

        rows = self.client.scannerGet(scanner_id)
        if rows:
            return dict(imap(lambda i: (i[0], i[1].value), \
                            rows[0].columns.iteritems()))
        else:
            return None

<commit_msg>more api stuff<commit_after>#!/usr/bin/env python
"""
hbasta._api

API Wrapper for HBase Thrift Client
"""

import sys
import logging
from itertools import imap

from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol

from hbase import Hbase
from hbase.ttypes import *

def _row_to_dict(row):
    """Convert an HBase Row as returned by the Thrift API
    to a native python dictionary, mapping column names to values"""
    return dict(imap(lambda i: (i[0], i[1].value), \
                     row.columns.iteritems()))

class HBasta(object):
    """HBase API entry point"""

    LOG = logging.getLogger("HBasta")

    def __init__(self, host, port):
        """Initialize client.

        Params:
            hostnport - Tuple of (host, port) to connect to
        """
        self._hostnport = (host, int(port))
        self._client = None

    @property
    def client(self):
        """Lazy load of the underlying client"""
        if not self._client:
            self.LOG.debug("* Connecting to HBase at: %s", self._hostnport)
            host, port = self._hostnport
            transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port))
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            self._client  = Hbase.Client(protocol)
            transport.open()
        return self._client

    def create_table(self, table, col_families):
        """Create a new HBase table

        Params
            table - Table name
            col_families - list of column family names
        """
        self.client.createTable(table, [ColumnDescriptor({'name': c+':'}) \
                                    for c in col_families])

    def enable_table(self, table):
        """Enable an HBase table"""
        self.client.enableTable(table)

    def disable_table(self, table):
        """Disable an HBase table"""
        self.client.disableTable(table)

    def is_table_enabled(self, table):
        """Check if table is enabled"""
        return self.client.isTableEnabled(table)

    def get_table_names(self):
        """Get list of all available table names"""
        return self.client.getTableNames()

    def get_row(self, table, row, colspec=None):
        """Get single row of data, possibly filtered
        using the colspec construct

        Params:
            table - Table name
            key - Row key
            colspec - Specifier of which columns to return, in the form of list of column names
        """
        if not colspec:
            rows = self.client.getRow(table, row)
        else:
            rows = self.client.getRowWithColumns(table, row, colspec)

        if rows:
            return _row_to_dict(rows[0])
        else:
            return None

    def delete_row(self, table, row):
        """Completely delete all data associated with row"""
        self.client.deleteAllRow(table, row)

    def atomic_increment(self, table, row, column, val=1):
        """Atomic increment of value for given column by the
        value specified"""
        return self.client.atomicIncrement(table, row, column, val)

    def scanner_open(self, table, start_row, colspec):
        """Open a scanner for table at given start_row,
        fetching columns as specified in colspec"""
        return self.client.scannerOpen(table, start_row, colspec)

    def scanner_open_with_stop(self, start_row, stop_row, colspec):
        """Open a scanner for table at given start_row, scanning up to
        specified stop_row"""
        return self.client.scannerOpenWithStop(table, start_row, stop_row, colspec)

    def scanner_open_with_prefix(self, start_prefix, colspec):
        """Open a scanner for a given prefix on row name"""
        return self.client.scannerOpenWithPrefix(start_prefix, colspec)

    def scanner_close(self, scanner_id):
        """Close a scanner"""
        self.client.scannerClose(scanner_id)

    def scanner_get(self, scanner_id):
        """Return current row scanner is pointing to."""

        rows = self.client.scannerGet(scanner_id)
        if rows:
            return _row_to_dict(rows[0])
        else:
            return None

    def scanner_get_list(self, scanner_id, num_rows):
        """Returns up to num_rows rows starting at current
        scanner location. Returns as a generator expression."""
        rows = self.client.scannerGetList(scanner_id, num_rows)
        if rows:
            for row in rows:
                yield _row_to_dict(row)
        else:
            return None
<|endoftext|>


<commit_before>import h5py
import time
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

SMALL_FLAG = 0
print('==> Experiment 6a: LSTM memory sizes')
filepath = '/pylon2/ci560sp/haunter/exp3_taylorswift_d15_1s_C1C8.mat'
if SMALL_FLAG:
    filepath = '/pylon2/ci560sp/haunter/exp3_small.mat'
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
#               reading data
# ==============================================
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
        seconds'.format(t = t_end - t_start))
del f

print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
#               RNN configs
# ==============================================
# Network Parameters
num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)

max_iter = 300
print_freq = 10
if SMALL_FLAG:
    max_iter = 10
    print_freq = 1

batch_size = 1000
learning_rate = 0.001
n_input = num_freq # number of sequences (rows)
n_steps = num_frames # size of each sequence (number of columns), timesteps
n_hidden = 128 # hidden layer num of features
n_classes = int(max(y_train.max(), y_val.max()) + 1)

try:
    n_hidden = int(sys.argv[1])
except Exception, e:
    print('-- {}'.format(e))
print('-- LSTM size: '.format(n_hidden))

# ==============================================
#               RNN architecture
# ==============================================
# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), n_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), n_classes)

# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])

# Define weights
weights = {
    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
}
biases = {
    'out': tf.Variable(tf.random_normal([n_classes]))
}

def RNN(x, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)

    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.unstack(x, n_steps, 1)

    # Define a lstm cell with tensorflow
    lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)

    # Get lstm cell output
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights['out']) + biases['out']

pred = RNN(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.global_variables_initializer()

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# ==============================================
#               RNN training
# ==============================================
# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    y_train = sess.run(y_train_OHEnc)[:, 0, :]
    y_val = sess.run(y_val_OHEnc)[:, 0, :]

    print('==> Training the full network...')
    t_start = time.time()
    # Keep training until reach max iterations
    for epoch in range(max_iter):
        for i in range(0, num_training_vec, batch_size):
            end_ind = min(i + batch_size, num_training_vec)
            batch_x = X_train[i : end_ind]
            batch_y = y_train[i : end_ind]
            # Reshape data to get 28 seq of 28 elements
            batch_x = batch_x.reshape((-1, n_steps, n_input))
            # Run optimization op (backprop)
            sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
        if (epoch + 1) % print_freq == 0:
            train_acc = accuracy.eval(feed_dict={x: X_train.reshape((-1, n_steps, n_input)),\
             y: y_train})
            train_acc_list.append(train_acc)
            val_acc = accuracy.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
             y: y_val})
            val_acc_list.append(val_acc)
            train_err = cost.eval(feed_dict={x: X_train.reshape((-1, n_steps, n_input)),\
             y: y_train})
            train_err_list.append(train_err)
            val_err = cost.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
             y: y_val})
            val_err_list.append(val_err)
            print("-- epoch: %d, training error %g"%(epoch + 1, train_err))

    t_end = time.time()
    print('--Time elapsed for training: {t:.2f} \
        seconds'.format(t = t_end - t_start))

# ==============================================
#               RNN Evaluation
# ==============================================
# Reports
print('-- Training accuracy: {:.4f}'.format(train_acc_list[-1]))
print('-- Validation accuracy: {:.4f}'.format(val_acc_list[-1]))
print('-- Training error: {:.4E}'.format(train_err_list[-1]))
print('-- Validation error: {:.4E}'.format(val_err_list[-1]))

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs with {} Hidden Units'.format(n_hidden))
plt.legend(loc='best')
plt.savefig('exp6a_{}.png'.format(n_hidden), format='png')
plt.close()

print('==> Finished!')
<commit_msg>switch flag exp6a<commit_after>import h5py
import time
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

FAST_FLAG = 1
print('==> Experiment 6a: LSTM memory sizes')
filepath = '/pylon2/ci560sp/haunter/exp3_taylorswift_d15_1s_C1C8.mat'
if FAST_FLAG:
    filepath = '/pylon2/ci560sp/haunter/exp3_small.mat'
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
#               reading data
# ==============================================
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
        seconds'.format(t = t_end - t_start))
del f

print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
#               RNN configs
# ==============================================
# Network Parameters
num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)

max_iter = 300
print_freq = 10
if SMALL_FLAG:
    max_iter = 10
    print_freq = 1

batch_size = 1000
learning_rate = 0.001
n_input = num_freq # number of sequences (rows)
n_steps = num_frames # size of each sequence (number of columns), timesteps
n_hidden = 128 # hidden layer num of features
n_classes = int(max(y_train.max(), y_val.max()) + 1)

try:
    n_hidden = int(sys.argv[1])
except Exception, e:
    print('-- {}'.format(e))
print('-- LSTM size: '.format(n_hidden))

# ==============================================
#               RNN architecture
# ==============================================
# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), n_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), n_classes)

# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])

# Define weights
weights = {
    'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
}
biases = {
    'out': tf.Variable(tf.random_normal([n_classes]))
}

def RNN(x, weights, biases):

    # Prepare data shape to match `rnn` function requirements
    # Current data input shape: (batch_size, n_steps, n_input)
    # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)

    # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input)
    x = tf.unstack(x, n_steps, 1)

    # Define a lstm cell with tensorflow
    lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)

    # Get lstm cell output
    outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights['out']) + biases['out']

pred = RNN(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.global_variables_initializer()

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

# ==============================================
#               RNN training
# ==============================================
# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    y_train = sess.run(y_train_OHEnc)[:, 0, :]
    y_val = sess.run(y_val_OHEnc)[:, 0, :]

    print('==> Training the full network...')
    t_start = time.time()
    # Keep training until reach max iterations
    for epoch in range(max_iter):
        for i in range(0, num_training_vec, batch_size):
            end_ind = min(i + batch_size, num_training_vec)
            batch_x = X_train[i : end_ind]
            batch_y = y_train[i : end_ind]
            # Reshape data to get 28 seq of 28 elements
            batch_x = batch_x.reshape((-1, n_steps, n_input))
            # Run optimization op (backprop)
            sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
        if (epoch + 1) % print_freq == 0:
            train_acc = accuracy.eval(feed_dict={x: X_train.reshape((-1, n_steps, n_input)),\
             y: y_train})
            train_acc_list.append(train_acc)
            val_acc = accuracy.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
             y: y_val})
            val_acc_list.append(val_acc)
            train_err = cost.eval(feed_dict={x: X_train.reshape((-1, n_steps, n_input)),\
             y: y_train})
            train_err_list.append(train_err)
            val_err = cost.eval(feed_dict={x: X_val.reshape((-1, n_steps, n_input)),\
             y: y_val})
            val_err_list.append(val_err)
            print("-- epoch: %d, training error %g"%(epoch + 1, train_err))

    t_end = time.time()
    print('--Time elapsed for training: {t:.2f} \
        seconds'.format(t = t_end - t_start))

# ==============================================
#               RNN Evaluation
# ==============================================
# Reports
print('-- Training accuracy: {:.4f}'.format(train_acc_list[-1]))
print('-- Validation accuracy: {:.4f}'.format(val_acc_list[-1]))
print('-- Training error: {:.4E}'.format(train_err_list[-1]))
print('-- Validation error: {:.4E}'.format(val_err_list[-1]))

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs with {} Hidden Units'.format(n_hidden))
plt.legend(loc='best')
plt.savefig('exp6a_{}.png'.format(n_hidden), format='png')
plt.close()

print('==> Finished!')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import itertools
from collections import defaultdict
from ..errors import *
from ..dev import is_experimental
from ..operation import Operation, OperationList, extract_signatures
from ..common import get_logger
from ..threadlocal import LocalProxy

__all__ = (
            "OperationContext",
            "default_context",
            "LoggingContextObserver",
            "CollectingContextObserver",
        )

"""List of modules with operations to be loaded for the default contex."""

# FIXME: make this more intelligent and lazy
_default_op_modules = (
            "bubbles.backends.sql.ops",
            "bubbles.backends.mongo.ops",
            "bubbles.ops.rows",
            "bubbles.ops.generic",
        )

class _OperationReference(object):
    def __init__(self, context, name):
        """Creates a reference to an operation within a context"""
        self.context = context
        self.name = name

    def __call__(self, *args, **kwargs):
        return self.context.call(self.name, *args, **kwargs)

class OperationContext(object):
    # TODO: add parent_context
    def __init__(self, retry_count=10):
        """Creates an operation context. Default `retry_count` is 10.

        Use:

        .. code-block:: python

            c = Context()
            duplicate_records = c.op.duplicates(table)

        Context uses multiple dispatch based on operation arguments. The
        `k.duplicates()` operation might be different, based on available
        representations of `table` object.
        """

        super().__init__()
        self.operations = defaultdict(OperationList)
        self.operation_retry_count = retry_count
        self.op = _OperationGetter(self)
        # TODO: depreciate
        self.o = self.op

        self.logger = get_logger()
        self.observer = LoggingContextObserver(self.logger)
        self.retry_count = retry_count

        self.retry_allow = []
        self.retry_deny = []

    def operation_list(self, name):
        """Get list of operation variants for an operation with `name`."""

        if name not in self.operations:
            self.operation_not_found(name)

        try:
            ops = self.operations[name]
        except KeyError:
            raise OperationError("Unable to find operation %s" % name)

        return ops

    def operation_prototype(self, name):
        """Returns a prototype signature for operation `name`"""
        oplist = self.operation_list(name)
        return oplist.prototype

    def add_operations_from(self, obj):
        """Import operations from `obj`. All attributes of `obj` are
        inspected. If they are instances of Operation, then the operation is
        imported."""

        for name in dir(obj):
            op = getattr(obj, name)
            if isinstance(op, Operation):
                self.add_operation(op)

    def add_operation(self, op):
        """Registers a decorated operation.  operation is considered to be a
        context's method and would receive the context as first argument."""

        if op.name not in self.operations:
            self.operations[op.name] = OperationList()

        if any(c.signature == op.signature for c in self.operations[op.name]):
            raise ArgumentError("Operation %s with signature %s already "
                                "registered" % (op.name, op.signature))

        self.operations[op.name].append(op)

    def remove_operation(self, name, signature=None):
        """Removes all operations with `name` and `signature`. If no
        `signature` is specified, then all operations with given name are
        removed, regardles of the signature."""

        operations = self.operations.get(name)
        if not operations:
            return
        elif not signature:
            del self.operations[name]
            return

        newops = [op for op in operations if op.signature != signature]
        self.operations[name] = newops

    def operation_list(self, name):
        """Return a list of operations with `name`. If no list is found, then
        `operation_not_found(name)` is called. See `operation_not_found` for
        more information"""
        # Get all signatures registered for the operation
        operations = self.operations[name]
        if not operations:
            self.operation_not_found(name)
            operations = self.operations[name]
            if not operations:
                raise OperationError("Unable to find operation %s" % name)

        return operations

    def operation_not_found(self, name):
        """Subclasses might override this method to load necessary modules and
        register operations using `register_operation()`. Default
        implementation raises an exception."""
        raise OperationError("Operation '%s' not found" % name)

    def get_operation(self, name, signature):
        """Returns an operation with given name and signature. Requires exact
        signature match."""
        # Get all signatures registered for the operation
        operations = self.operation_list(name)

        for op in operations:
            if op.signature == signature:
                return op
        raise OperationError("No operation '%s' with signature '%s'" %
                                                        (name, signature))

    def operation(self, name):
        """Returns a bound operation with `name`"""

        return _OperationReference(self, name)

    def debug_print_catalogue(self):
        """Pretty prints the operation catalogue – all operations, their
        signatures and function references. Used for debugging purposes."""

        print("== OPERATIONS ==")
        keys = list(self.operations.keys())
        keys.sort()
        for name in keys:
            ops = self.operations[name]
            print("* %s:" % name)

            for op in ops:
                print("    - %s:%s" % \
                            (op.signature.signature, op.function) )

    def can_retry(self, opname):
        """Returns `True` if an operation can be retried. By default, all
        operations can be retried. `Context.retry_allow` contains list of
        allowed operations, `Context.retry_deny` contains list of denied
        operations. When the allow list is set, then only operations from the
        list are allowed. When the deny list is set, the operations in the
        deny list are not allowed."""

        if self.retry_deny and opname in self.retry_deny:
            return False

        if self.retry_allow and opname not in self.retry_allow:
            return False

        return True

    def call(self, op_name, *args, **kwargs):
        """Call operation with `name`. Arguments are passed to the
        operation"""

        oplist = self.operation_list(op_name)
        argc = oplist.prototype.operand_count
        match_objects = args[0:argc]

        # Find best signature match for the operation based on object
        # arguments
        op = self.lookup_operation(op_name, *match_objects)

        if self.observer:
            self.observer.will_call_operation(self, op)

        result = None
        retries = 0

        # We try to perform requested operation. If the operation raises
        # RetryOperation exception, then we use signature from the exception
        # for another operation. Operation can be retried retry_count number
        # of times.
        # Observer is notified about each retry.

        # TODO: allow retry without a signature
        # This will require to have a prepared list of matching operations and
        # number of retries will be number of operations in the list.

        try:
            if is_experimental(op.function):
                self.logger.warn("operation %s is experimental" % \
                                    op_name)
            result = op.function(self, *args, **kwargs)
        except RetryOperation as e:
            if not self.can_retry(op_name):
                raise RetryError("Retry of operation '%s' is not allowed")

            signature = e.signature
            reason = e.reason
            success = False

            for i in range(0, self.retry_count):
                op = self.get_operation(op_name, signature)
                retries += 1

                if self.observer:
                    self.observer.will_retry_operation(self, op, reason)

                try:
                    result = op.function(self, *args, **kwargs)
                except RetryOperation as e:
                    signature = e.signature
                    reason = e.reason
                else:
                    success = True
                    break

            if not success:
                raise RetryError("Operation retry limit reached "
                                     "(allowed: %d)" % self.retry_count)

        # Let the observer know which operation was called at last and
        # completed sucessfully

        if self.observer:
            self.observer.did_call_operation(self, op, retries)

        return result

    def lookup_operation(self, name, *objlist):
        """Returns a matching operation for given data objects as arguments.
        This is the main lookup method of the operation context.

        Lookup is performed for all representations of objects, the first
        matching signature is accepted. Order of representations returned by
        object's `representations()` matters here.

        Returned object is a OpFunction tuple with attributes: `name`, `func`,
        `signature`.

        Note: If the match does not fit your expectations, it is recommended
        to pefrom explicit object conversion to an object with desired
        representation.
        """

        operations = self.operation_list(name)
        if not operations:
            # TODO: check parent context
            raise OperationError("No known signatures for operation '%s'" %
                                    name)

        # TODO: add some fall-back mechanism when no operation signature is
        # found
        # Extract signatures from arguments

        arg_signatures = extract_signatures(*objlist)
        match = None
        for arguments in itertools.product(*arg_signatures):
            for op in operations:
                if op.signature.matches(*arguments):
                    match = op
                    break
            if match:
                break

        if match:
            return match

        raise OperationError("No matching signature found for operation '%s' "
                             " (args: %s)" %
                                (name, arg_signatures))

class LoggingContextObserver(object):
    def __init__(self, logger=None):
        self.logger = logger

    def will_call_operation(self, ctx, op):
        logger = self.logger or ctx.logger
        logger.debug("calling %s(%s)" % (op, op.signature))

    def did_call_operation(self, ctx, op, retries):
        logger = self.logger or ctx.logger

        if not retries:
            logger.info("called %s(%s)" % (op, op.signature))
        else:
            logger.info("called %s(%s) wth %s retries" % \
                                (op, op.signature, retries))

    def will_retry_operation(self, ctx, op, reason):
        logger = self.logger or ctx.logger
        logger.debug("retry %s(%s), reason: %s" %
                                        (op, op.signature, reason))

class CollectingContextObserver(object):
    def __init__(self):
        self.history = []
        self.current = None
        self.tried = []

    def will_call_operation(self, ctx, op):
        self.current = op
        self.tried = []

    def will_retry_operation(self, ctx, op, reason):
        self.tried.append( (self.current, reason) )
        self.current = op

    def did_call_operation(self, ctx, op, retries):
        line = (op, retries, self.tried)
        self.history.append(line)

class _OperationGetter(object):
    def __init__(self, context):
        self.context = context

    def __getattr__(self, name):
        return self.context.operation(name)

    def __getitem__(self, name):
        return self.context.operation(name)

def create_default_context():
    """Creates a ExecutionContext with default operations."""
    context = OperationContext()

    for modname in _default_op_modules:
        mod = _load_module(modname)
        context.add_operations_from(mod)

    return context

def _load_module(modulepath):
    mod = __import__(modulepath)
    path = []
    for token in modulepath.split(".")[1:]:
       path.append(token)
       try:
           mod = getattr(mod, token)
       except AttributeError:
           raise BubblesError("Unable to get %s" % (path, ))
    return mod

default_context = LocalProxy("default_context",
                             factory=create_default_context)

# FIXME: k is for backward prototype compatibility reasons and means 'kernel'
k = default_context
<commit_msg>Display call intention and retry in INFO level, actual call in DEBUG<commit_after># -*- coding: utf-8 -*-
import itertools
from collections import defaultdict
from ..errors import *
from ..dev import is_experimental
from ..operation import Operation, OperationList, extract_signatures
from ..common import get_logger
from ..threadlocal import LocalProxy

__all__ = (
            "OperationContext",
            "default_context",
            "LoggingContextObserver",
            "CollectingContextObserver",
        )

"""List of modules with operations to be loaded for the default contex."""

# FIXME: make this more intelligent and lazy
_default_op_modules = (
            "bubbles.backends.sql.ops",
            "bubbles.backends.mongo.ops",
            "bubbles.ops.rows",
            "bubbles.ops.generic",
        )

class _OperationReference(object):
    def __init__(self, context, name):
        """Creates a reference to an operation within a context"""
        self.context = context
        self.name = name

    def __call__(self, *args, **kwargs):
        return self.context.call(self.name, *args, **kwargs)

class OperationContext(object):
    # TODO: add parent_context
    def __init__(self, retry_count=10):
        """Creates an operation context. Default `retry_count` is 10.

        Use:

        .. code-block:: python

            c = Context()
            duplicate_records = c.op.duplicates(table)

        Context uses multiple dispatch based on operation arguments. The
        `k.duplicates()` operation might be different, based on available
        representations of `table` object.
        """

        super().__init__()
        self.operations = defaultdict(OperationList)
        self.operation_retry_count = retry_count
        self.op = _OperationGetter(self)
        # TODO: depreciate
        self.o = self.op

        self.logger = get_logger()
        self.observer = LoggingContextObserver(self.logger)
        self.retry_count = retry_count

        self.retry_allow = []
        self.retry_deny = []

    def operation_list(self, name):
        """Get list of operation variants for an operation with `name`."""

        if name not in self.operations:
            self.operation_not_found(name)

        try:
            ops = self.operations[name]
        except KeyError:
            raise OperationError("Unable to find operation %s" % name)

        return ops

    def operation_prototype(self, name):
        """Returns a prototype signature for operation `name`"""
        oplist = self.operation_list(name)
        return oplist.prototype

    def add_operations_from(self, obj):
        """Import operations from `obj`. All attributes of `obj` are
        inspected. If they are instances of Operation, then the operation is
        imported."""

        for name in dir(obj):
            op = getattr(obj, name)
            if isinstance(op, Operation):
                self.add_operation(op)

    def add_operation(self, op):
        """Registers a decorated operation.  operation is considered to be a
        context's method and would receive the context as first argument."""

        if op.name not in self.operations:
            self.operations[op.name] = OperationList()

        if any(c.signature == op.signature for c in self.operations[op.name]):
            raise ArgumentError("Operation %s with signature %s already "
                                "registered" % (op.name, op.signature))

        self.operations[op.name].append(op)

    def remove_operation(self, name, signature=None):
        """Removes all operations with `name` and `signature`. If no
        `signature` is specified, then all operations with given name are
        removed, regardles of the signature."""

        operations = self.operations.get(name)
        if not operations:
            return
        elif not signature:
            del self.operations[name]
            return

        newops = [op for op in operations if op.signature != signature]
        self.operations[name] = newops

    def operation_list(self, name):
        """Return a list of operations with `name`. If no list is found, then
        `operation_not_found(name)` is called. See `operation_not_found` for
        more information"""
        # Get all signatures registered for the operation
        operations = self.operations[name]
        if not operations:
            self.operation_not_found(name)
            operations = self.operations[name]
            if not operations:
                raise OperationError("Unable to find operation %s" % name)

        return operations

    def operation_not_found(self, name):
        """Subclasses might override this method to load necessary modules and
        register operations using `register_operation()`. Default
        implementation raises an exception."""
        raise OperationError("Operation '%s' not found" % name)

    def get_operation(self, name, signature):
        """Returns an operation with given name and signature. Requires exact
        signature match."""
        # Get all signatures registered for the operation
        operations = self.operation_list(name)

        for op in operations:
            if op.signature == signature:
                return op
        raise OperationError("No operation '%s' with signature '%s'" %
                                                        (name, signature))

    def operation(self, name):
        """Returns a bound operation with `name`"""

        return _OperationReference(self, name)

    def debug_print_catalogue(self):
        """Pretty prints the operation catalogue – all operations, their
        signatures and function references. Used for debugging purposes."""

        print("== OPERATIONS ==")
        keys = list(self.operations.keys())
        keys.sort()
        for name in keys:
            ops = self.operations[name]
            print("* %s:" % name)

            for op in ops:
                print("    - %s:%s" % \
                            (op.signature.signature, op.function) )

    def can_retry(self, opname):
        """Returns `True` if an operation can be retried. By default, all
        operations can be retried. `Context.retry_allow` contains list of
        allowed operations, `Context.retry_deny` contains list of denied
        operations. When the allow list is set, then only operations from the
        list are allowed. When the deny list is set, the operations in the
        deny list are not allowed."""

        if self.retry_deny and opname in self.retry_deny:
            return False

        if self.retry_allow and opname not in self.retry_allow:
            return False

        return True

    def call(self, op_name, *args, **kwargs):
        """Call operation with `name`. Arguments are passed to the
        operation"""

        oplist = self.operation_list(op_name)
        argc = oplist.prototype.operand_count
        match_objects = args[0:argc]

        # Find best signature match for the operation based on object
        # arguments
        op = self.lookup_operation(op_name, *match_objects)

        if self.observer:
            self.observer.will_call_operation(self, op)

        result = None
        retries = 0

        # We try to perform requested operation. If the operation raises
        # RetryOperation exception, then we use signature from the exception
        # for another operation. Operation can be retried retry_count number
        # of times.
        # Observer is notified about each retry.

        # TODO: allow retry without a signature
        # This will require to have a prepared list of matching operations and
        # number of retries will be number of operations in the list.

        try:
            if is_experimental(op.function):
                self.logger.warn("operation %s is experimental" % \
                                    op_name)
            result = op.function(self, *args, **kwargs)
        except RetryOperation as e:
            if not self.can_retry(op_name):
                raise RetryError("Retry of operation '%s' is not allowed")

            signature = e.signature
            reason = e.reason
            success = False

            for i in range(0, self.retry_count):
                op = self.get_operation(op_name, signature)
                retries += 1

                if self.observer:
                    self.observer.will_retry_operation(self, op, reason)

                try:
                    result = op.function(self, *args, **kwargs)
                except RetryOperation as e:
                    signature = e.signature
                    reason = e.reason
                else:
                    success = True
                    break

            if not success:
                raise RetryError("Operation retry limit reached "
                                     "(allowed: %d)" % self.retry_count)

        # Let the observer know which operation was called at last and
        # completed sucessfully

        if self.observer:
            self.observer.did_call_operation(self, op, retries)

        return result

    def lookup_operation(self, name, *objlist):
        """Returns a matching operation for given data objects as arguments.
        This is the main lookup method of the operation context.

        Lookup is performed for all representations of objects, the first
        matching signature is accepted. Order of representations returned by
        object's `representations()` matters here.

        Returned object is a OpFunction tuple with attributes: `name`, `func`,
        `signature`.

        Note: If the match does not fit your expectations, it is recommended
        to pefrom explicit object conversion to an object with desired
        representation.
        """

        operations = self.operation_list(name)
        if not operations:
            # TODO: check parent context
            raise OperationError("No known signatures for operation '%s'" %
                                    name)

        # TODO: add some fall-back mechanism when no operation signature is
        # found
        # Extract signatures from arguments

        arg_signatures = extract_signatures(*objlist)
        match = None
        for arguments in itertools.product(*arg_signatures):
            for op in operations:
                if op.signature.matches(*arguments):
                    match = op
                    break
            if match:
                break

        if match:
            return match

        raise OperationError("No matching signature found for operation '%s' "
                             " (args: %s)" %
                                (name, arg_signatures))

class LoggingContextObserver(object):
    def __init__(self, logger=None):
        self.logger = logger

    def will_call_operation(self, ctx, op):
        logger = self.logger or ctx.logger
        logger.info("calling %s(%s)" % (op, op.signature))

    def did_call_operation(self, ctx, op, retries):
        logger = self.logger or ctx.logger

        if not retries:
            logger.deubug("called %s(%s)" % (op, op.signature))
        else:
            logger.deubug("called %s(%s) wth %s retries" % \
                                (op, op.signature, retries))

    def will_retry_operation(self, ctx, op, reason):
        logger = self.logger or ctx.logger
        logger.info("retry %s(%s), reason: %s" %
                                        (op, op.signature, reason))

class CollectingContextObserver(object):
    def __init__(self):
        self.history = []
        self.current = None
        self.tried = []

    def will_call_operation(self, ctx, op):
        self.current = op
        self.tried = []

    def will_retry_operation(self, ctx, op, reason):
        self.tried.append( (self.current, reason) )
        self.current = op

    def did_call_operation(self, ctx, op, retries):
        line = (op, retries, self.tried)
        self.history.append(line)

class _OperationGetter(object):
    def __init__(self, context):
        self.context = context

    def __getattr__(self, name):
        return self.context.operation(name)

    def __getitem__(self, name):
        return self.context.operation(name)

def create_default_context():
    """Creates a ExecutionContext with default operations."""
    context = OperationContext()

    for modname in _default_op_modules:
        mod = _load_module(modname)
        context.add_operations_from(mod)

    return context

def _load_module(modulepath):
    mod = __import__(modulepath)
    path = []
    for token in modulepath.split(".")[1:]:
       path.append(token)
       try:
           mod = getattr(mod, token)
       except AttributeError:
           raise BubblesError("Unable to get %s" % (path, ))
    return mod

default_context = LocalProxy("default_context",
                             factory=create_default_context)

# FIXME: k is for backward prototype compatibility reasons and means 'kernel'
k = default_context
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import operator
import re

class ApacheCombined(object):
    def __init__(self, remote_host, remote_user, date_time, request_line, status_code, bytes_sent, referrer, useragent):
        self.remote_host = remote_host
        self.remote_user = remote_user
        self.date_time = date_time
        self.request_line = request_line
        self.status_code = status_code
        self.bytes_sent = bytes_sent
        self.referrer = referrer
        self.useragent = useragent
        pass

class ApacheCombinedLineParser(object):
    def __init__(self):
        """
        Combined format is:

        %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"

        where the items are:

        %h - Remote host name (or IP address if enableLookups for the connector is false)
        %l - Remote logical username from identd (always returns '-')
        %u - Remote user that was authenticated
        %t - Date and time, in Common Log Format format
        %r - First line of the request
        %s - HTTP status code of the response
        %b - Bytes sent, excluding HTTP headers, or '-' if no bytes were sent
        """
        self.__lineexpression = re.compile('^(\d+\.\d+\.\d+\.\d+)\s-\s(\S+)\s\[([^\]]+)\]\s"(.+)HTTP/1.[01]"\s(\d+)\s(\d+|-)\s"([^"]*)"\s"(.*)"$')

    def parse(self, line):
        """
        Parse a log line returning a ApacheCombined object or None if the line is not valid
        """
        match = self.__lineexpression.match(line)
        if match:
            remote_host = match.group(1)
            remote_user = match.group(2)
            date_time = match.group(3)
            request_line = match.group(4)
            status_code = match.group(5)
            bytes_sent = match.group(6)
            referrer = match.group(7)
            useragent = match.group(8)
            return ApacheCombined(remote_host, remote_user, date_time, request_line, status_code, bytes_sent, referrer, useragent)
        else:
            return None

class BotStatistics(object):
    def __init__(self):
        self.__agents = {}
        self.__bot_agents = {}
        self.__agents_seen = 0

        self.__bot_expressions = []
        self.__bot_expressions.append(re.compile('(ads|google|bing|msn|yandex|baidu|ro|career|seznam|)bot', re.IGNORECASE))
        self.__bot_expressions.append(re.compile('(baidu|jike|symantec)spider', re.IGNORECASE))
        self.__bot_expressions.append(re.compile('scanner', re.IGNORECASE))
        self.__bot_expressions.append(re.compile('(web)crawler', re.IGNORECASE))
        pass

    def _isBot(self, useragent):
        """
        Classify the useragent as bot or not a bot.
        """
        for expression in self.__bot_expressions:
            match = expression.match(useragent)
            if match:
                return True
        return False

    def _addAgent(self, agent):
        if agent in self.__agents:
            self.__agents[agent] = self.__agents[agent] + 1
        else:
            self.__agents[agent] = 1

    def _addBotAgent(self, agent):
        if agent in self.__bot_agents:
            self.__bot_agents[agent] = self.__bot_agents[agent] + 1
        else:
            self.__bot_agents[agent] = 1

    def consume(self, log_entry):
        agent = log_entry.useragent
        self.__agents_seen = self.__agents_seen + 1
        if self._isBot(agent):
            self._addBotAgent(agent)
        else:
            self._addAgent(agent)

    def print_statistics(self):
        ordered_most_seen = sorted(self.__agents.items(), key=operator.itemgetter(1))
        for t in ordered_most_seen:
            print(t)
        print("Unique user agent strings {:d} out of {:d} entries".format(len(self.__agents), self.__agents_seen))
        ordered_bots = sorted(self.__bot_agents.items(), key=operator.itemgetter(1))
        for t in ordered_bots:
            print(t)
        print("Unique bot user agent strings {:d} out of {:d} entries".format(len(self.__bot_agents), self.__agents_seen))

class UserAgents(object):
    """
    User agents object for summeriazing the user agent data.
    """
    def __init__(self, files, bot):
        """
        Construct the UserAgents object with a list of file to process.
        """
        self.__files = files;
        self.__bot_feature = bot
        pass

    def process(self):
        """
        Process the list of files.
        """
        parser = ApacheCombinedLineParser()
        bots = BotStatistics()
        for logfile in self.__files:
            lines = 0
            matchedlines = 0
            for line in logfile:
                lines = lines + 1
                match = parser.parse(line)
                if(match):
                    matchedlines = matchedlines + 1
                    if(self.__bot_feature):
                        bots.consume(match)
                else:
                    print(line)
            print('File {:s} has lines {:d} where {:d} matches'.format(logfile.name, lines, matchedlines))

        if(self.__bot_feature):
            bots.print_statistics()

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='User agents from Tomcat combined access log.')
    parser.add_argument('--bot', dest='bot', action='store_true')
    parser.set_defaults(bot=False)
    parser.add_argument('files', metavar='FILE', type=argparse.FileType(mode='r', encoding='latin-1'), nargs='+', help='File to parse')
    args = parser.parse_args()

    userAgents = UserAgents(args.files, args.bot)
    userAgents.process()

<commit_msg>search instead of match<commit_after>#!/usr/bin/env python

import argparse
import operator
import re

class ApacheCombined(object):
    def __init__(self, remote_host, remote_user, date_time, request_line, status_code, bytes_sent, referrer, useragent):
        self.remote_host = remote_host
        self.remote_user = remote_user
        self.date_time = date_time
        self.request_line = request_line
        self.status_code = status_code
        self.bytes_sent = bytes_sent
        self.referrer = referrer
        self.useragent = useragent
        pass

class ApacheCombinedLineParser(object):
    def __init__(self):
        """
        Combined format is:

        %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"

        where the items are:

        %h - Remote host name (or IP address if enableLookups for the connector is false)
        %l - Remote logical username from identd (always returns '-')
        %u - Remote user that was authenticated
        %t - Date and time, in Common Log Format format
        %r - First line of the request
        %s - HTTP status code of the response
        %b - Bytes sent, excluding HTTP headers, or '-' if no bytes were sent
        """
        self.__lineexpression = re.compile('^(\d+\.\d+\.\d+\.\d+)\s-\s(\S+)\s\[([^\]]+)\]\s"(.+)HTTP/1.[01]"\s(\d+)\s(\d+|-)\s"([^"]*)"\s"(.*)"$')

    def parse(self, line):
        """
        Parse a log line returning a ApacheCombined object or None if the line is not valid
        """
        match = self.__lineexpression.match(line)
        if match:
            remote_host = match.group(1)
            remote_user = match.group(2)
            date_time = match.group(3)
            request_line = match.group(4)
            status_code = match.group(5)
            bytes_sent = match.group(6)
            referrer = match.group(7)
            useragent = match.group(8)
            return ApacheCombined(remote_host, remote_user, date_time, request_line, status_code, bytes_sent, referrer, useragent)
        else:
            return None

class BotStatistics(object):
    def __init__(self):
        self.__agents = {}
        self.__bot_agents = {}
        self.__agents_seen = 0

        self.__bot_expressions = []
        self.__bot_expressions.append(re.compile('(ads|google|bing|msn|yandex|baidu|ro|career|seznam|)bot', re.IGNORECASE))
        self.__bot_expressions.append(re.compile('(baidu|jike|symantec)spider', re.IGNORECASE))
        self.__bot_expressions.append(re.compile('scanner', re.IGNORECASE))
        self.__bot_expressions.append(re.compile('(web)crawler', re.IGNORECASE))
        pass

    def _isBot(self, useragent):
        """
        Classify the useragent as bot or not a bot.
        """
        for expression in self.__bot_expressions:
            match = expression.search(useragent)
            if match:
                return True
        return False

    def _addAgent(self, agent):
        if agent in self.__agents:
            self.__agents[agent] = self.__agents[agent] + 1
        else:
            self.__agents[agent] = 1

    def _addBotAgent(self, agent):
        if agent in self.__bot_agents:
            self.__bot_agents[agent] = self.__bot_agents[agent] + 1
        else:
            self.__bot_agents[agent] = 1

    def consume(self, log_entry):
        agent = log_entry.useragent
        self.__agents_seen = self.__agents_seen + 1
        if self._isBot(agent):
            self._addBotAgent(agent)
        else:
            self._addAgent(agent)

    def print_statistics(self):
        ordered_most_seen = sorted(self.__agents.items(), key=operator.itemgetter(1))
        for t in ordered_most_seen:
            print(t)
        print("Unique user agent strings {:d} out of {:d} entries".format(len(self.__agents), self.__agents_seen))
        ordered_bots = sorted(self.__bot_agents.items(), key=operator.itemgetter(1))
        for t in ordered_bots:
            print(t)
        print("Unique bot user agent strings {:d} out of {:d} entries".format(len(self.__bot_agents), self.__agents_seen))

class UserAgents(object):
    """
    User agents object for summeriazing the user agent data.
    """
    def __init__(self, files, bot):
        """
        Construct the UserAgents object with a list of file to process.
        """
        self.__files = files;
        self.__bot_feature = bot
        pass

    def process(self):
        """
        Process the list of files.
        """
        parser = ApacheCombinedLineParser()
        bots = BotStatistics()
        for logfile in self.__files:
            lines = 0
            matchedlines = 0
            for line in logfile:
                lines = lines + 1
                match = parser.parse(line)
                if(match):
                    matchedlines = matchedlines + 1
                    if(self.__bot_feature):
                        bots.consume(match)
                else:
                    print(line)
            print('File {:s} has lines {:d} where {:d} matches'.format(logfile.name, lines, matchedlines))

        if(self.__bot_feature):
            bots.print_statistics()

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='User agents from Tomcat combined access log.')
    parser.add_argument('--bot', dest='bot', action='store_true')
    parser.set_defaults(bot=False)
    parser.add_argument('files', metavar='FILE', type=argparse.FileType(mode='r', encoding='latin-1'), nargs='+', help='File to parse')
    args = parser.parse_args()

    userAgents = UserAgents(args.files, args.bot)
    userAgents.process()

<|endoftext|>


<commit_before>import client
import csv
import queries
import itertools
import numpy as np
from operator import itemgetter

hostname = "dbserver02.cs.washington.edu"
port = "10032"

client.init_connection(hostname, port)

def experiment(filename, exp_queries):
    with open(filename, "wb") as f:
        writer = csv.writer(f)
        writer.writerow(
            ["name", "qid", "time", "algebra", "profilingMode", "success"])
        for query in exp_queries:
            # submit queries
            result, status = client.execute_query(query)
            _, algebra, profie, _, name = query
            # log experiment result
            if result == 'success':
                print "success"
                time = float(status["elapsedNanos"]) / client.NANO_IN_ONE_SEC
                writer.writerow(
                    [name, status["queryId"], time, algebra, profie, result])
            else:
                print "error"
                writer.writerow(
                    [name, "N.A.", "N.A.", algebra, profie, result])

exp_raw_queries = [
    (queries.triangle, 'triangle'),
    (queries.fb_q1, 'fb_q1'),
    (queries.rectangle, 'rectangle'),
    (queries.fb_q2, 'fb_q2'),
    (queries.two_rings, 'two_rings'),
    (queries.fb_q3, 'fb_q3'),
    (queries.clique, 'clique'),
    (queries.fb_q4, 'fb_q4')
]

phys_algebras = [
    ('RS_HJ',),
    ('HC_HJ',),
    ('BR_HJ',),
    ('HC_LFJ',),
    ('BR_LFJ',)
]

languages = [('myrial',)]

# experiment 1:  resouce usage
def resource_exp():
    profilingModes = [('RESOURCE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("resource_exp.csv", exp_queries)

# experiment 2: profile query execution only
def profile_exp():
    profilingModes = [('QUERY',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("profile_exp.csv", exp_queries)

# experiment 3: cold cache experiment
def cold_cache_exp(filename):
    profilingModes = [('NONE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 4: regular shuffle with leapfrog join
def cold_cache_rslfj_exp(filename):
    profilingModes = [('NONE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

def collect_network_data(query_id):
    """collect skew and """
    send = list(client.connection.get_sent_logs(query_id))
    # sanitiy checking
    assert len(send[0]) == 4
    # remove header
    send = send[1:]
    # convert to long
    send = [map(long, r) for r in send]
    # compute total number
    total_num_tuples = sum([r[3] for r in send])
    # group by fragmentid
    key_func = lambda x: x[1]
    send = sorted(send, key=lambda x: x[1])
    sends = itertools.groupby(send, key_func)
    groups = []
    for fid, shuffles in sends:
        groups.append([r for r in shuffles])

    # compute skew per group
    def compute_skew(group):
        data = [r[3] for r in group]
        skew = np.max(data)/np.mean(data)
        return skew
    skews = map(compute_skew, groups)
    # return number of tuples being shuffled, and max skew
    return total_num_tuples, max(skews)

def add_network_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["shuffled tuples", "max skew"])
        for i, row in enumerate(data):
            if i != 0:
                shuffled_tupels, max_skew = collect_network_data(row[1])
                data[i].extend([shuffled_tupels, max_skew])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

def collect_resource_data(query_id):
    resource = list(client.connection.get_resource_log(query_id))
    # remove header
    resource = resource[1:]
    # convert numeric fields to long
    # row = [timestamp,opId,measurement,value,queryId,subqueryId,workerId]
    resource = [
        [long(row[0]),
         long(row[1]),
         row[2],
         long(row[3]),
         long(row[4]),
         long(row[5]),
         long(row[6])] for row in resource]
    # filter out row with zero value
    resource = filter(lambda row: row[3] != 0, resource)
    # group by opId, measurement and worker
    resource = sorted(resource, key=itemgetter(1, 2, 6))
    resource = itertools.groupby(resource, key=itemgetter(1, 2, 6))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    resource = [max(grp, key=itemgetter(0)) for grp in groups]
    # store processed raw data as csv
    with open("query_{}_resource.csv".format(query_id), "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerow(["opId", "measure", "value", "queryId", "workerId"])
        for row in resource:
            csvwriter.writerow([row[1], row[2], row[3], row[4], row[6]])
    # group by measurement
    resource = sorted(resource, key=itemgetter(2))
    resource = itertools.groupby(resource, key=itemgetter(2))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    result = []
    for grp in groups:
        values = [row[3] for row in grp]
        result.append([grp[0][2], sum(values)])
    cpu, hash_table_size = 0, 0
    for measure, value in result:
        if measure == 'cpuTotal':
            cpu = value
        elif measure == 'hashTableSize':
            hash_table_size = value
    return cpu, hash_table_size

def add_resource_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["cpu time", "hash table size"])
        for i, row in enumerate(data):
            if i != 0:
                cpu, ht_size = collect_resource_data(row[1])
                data[i].extend([cpu, ht_size])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

if __name__ == '__main__':
    #resource_exp()
    #profile_exp()
    #cold_cache_exp("cold_cache_1.csv")
    #cold_cache_exp("cold_cache_2.csv")
    #cold_cache_exp("cold_cache_3.csv")
    #cold_cache_exp("cold_cache_4.csv")
    #cold_cache_exp("cold_cache_5.csv")
    cold_cache_rslfj_exp("cold_cache_rslfj.csv")
    #add_resource_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/resource_exp.csv",
    #    "resource_extend.csv")
    #q = ('myrial', "RS_HJ", "NONE", queries.triangle, 'whatever')
    #execute_query(q)
    # collect_network_data(1031)
    # add_network_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/profile_exp.csv",
    #    "profile_extend.csv")
<commit_msg>query execution and resource usage experiments.<commit_after>import client
import csv
import queries
import itertools
import numpy as np
from operator import itemgetter

hostname = "dbserver02.cs.washington.edu"
port = "10032"

client.init_connection(hostname, port)

def experiment(filename, exp_queries):
    with open(filename, "wb") as f:
        writer = csv.writer(f)
        writer.writerow(
            ["name", "qid", "time", "algebra", "profilingMode", "success"])
        for query in exp_queries:
            # submit queries
            result, status = client.execute_query(query)
            _, algebra, profie, _, name = query
            # log experiment result
            if result == 'success':
                print "success"
                time = float(status["elapsedNanos"]) / client.NANO_IN_ONE_SEC
                writer.writerow(
                    [name, status["queryId"], time, algebra, profie, result])
            else:
                print "error"
                writer.writerow(
                    [name, "N.A.", "N.A.", algebra, profie, result])

exp_raw_queries = [
    (queries.triangle, 'triangle'),
    (queries.fb_q1, 'fb_q1'),
    (queries.rectangle, 'rectangle'),
    (queries.fb_q2, 'fb_q2'),
    (queries.two_rings, 'two_rings'),
    (queries.fb_q3, 'fb_q3'),
    (queries.clique, 'clique'),
    (queries.fb_q4, 'fb_q4')
]

phys_algebras = [
    ('RS_HJ',),
    ('HC_HJ',),
    ('BR_HJ',),
    ('HC_LFJ',),
    ('BR_LFJ',)
]

languages = [('myrial',)]

# experiment 1:  resouce usage
def resource_exp():
    profilingModes = [('RESOURCE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("resource_exp.csv", exp_queries)

# experiment 2: profile query execution only
def profile_exp():
    profilingModes = [('QUERY',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment("profile_exp.csv", exp_queries)

# experiment 3: cold cache experiment
def cold_cache_exp(filename):
    profilingModes = [('NONE',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 4: regular shuffle with leapfrog join
def cold_cache_rslfj_exp(filename):
    profilingModes = [('NONE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 5: rs lfj profile
def profile_rslfj(filename):
    profilingModes = [('QUERY',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

# experiment 6: rs lfj resource
def resource_rslfj(filename):
    profilingModes = [('RESOURCE',)]
    phys_algebras = [('RS_LFJ',)]
    exp_queries = itertools.product(
        languages, phys_algebras, profilingModes, exp_raw_queries)
    exp_queries = [
        reduce(lambda t1, t2: t1 + t2, query) for query in exp_queries]
    experiment(filename, exp_queries)

def collect_network_data(query_id):
    """collect skew and """
    send = list(client.connection.get_sent_logs(query_id))
    # sanitiy checking
    assert len(send[0]) == 4
    # remove header
    send = send[1:]
    # convert to long
    send = [map(long, r) for r in send]
    # compute total number
    total_num_tuples = sum([r[3] for r in send])
    # group by fragmentid
    key_func = lambda x: x[1]
    send = sorted(send, key=lambda x: x[1])
    sends = itertools.groupby(send, key_func)
    groups = []
    for fid, shuffles in sends:
        groups.append([r for r in shuffles])

    # compute skew per group
    def compute_skew(group):
        data = [r[3] for r in group]
        skew = np.max(data)/np.mean(data)
        return skew
    skews = map(compute_skew, groups)
    # return number of tuples being shuffled, and max skew
    return total_num_tuples, max(skews)

def add_network_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["shuffled tuples", "max skew"])
        for i, row in enumerate(data):
            if i != 0:
                shuffled_tupels, max_skew = collect_network_data(row[1])
                data[i].extend([shuffled_tupels, max_skew])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

def collect_resource_data(query_id):
    resource = list(client.connection.get_resource_log(query_id))
    # remove header
    resource = resource[1:]
    # convert numeric fields to long
    # row = [timestamp,opId,measurement,value,queryId,subqueryId,workerId]
    resource = [
        [long(row[0]),
         long(row[1]),
         row[2],
         long(row[3]),
         long(row[4]),
         long(row[5]),
         long(row[6])] for row in resource]
    # filter out row with zero value
    resource = filter(lambda row: row[3] != 0, resource)
    # group by opId, measurement and worker
    resource = sorted(resource, key=itemgetter(1, 2, 6))
    resource = itertools.groupby(resource, key=itemgetter(1, 2, 6))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    resource = [max(grp, key=itemgetter(0)) for grp in groups]
    # store processed raw data as csv
    with open("query_{}_resource.csv".format(query_id), "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerow(["opId", "measure", "value", "queryId", "workerId"])
        for row in resource:
            csvwriter.writerow([row[1], row[2], row[3], row[4], row[6]])
    # group by measurement
    resource = sorted(resource, key=itemgetter(2))
    resource = itertools.groupby(resource, key=itemgetter(2))
    groups = []
    for key, grp in resource:
        groups.append([row for row in grp])
    result = []
    for grp in groups:
        values = [row[3] for row in grp]
        result.append([grp[0][2], sum(values)])
    cpu, hash_table_size = 0, 0
    for measure, value in result:
        if measure == 'cpuTotal':
            cpu = value
        elif measure == 'hashTableSize':
            hash_table_size = value
    return cpu, hash_table_size

def add_resource_data(query_file, output_file):
    with open(query_file, "rU") as f:
        csvreader = csv.reader(f)
        data = [r for r in csvreader]
        data[0].extend(["cpu time", "hash table size"])
        for i, row in enumerate(data):
            if i != 0:
                cpu, ht_size = collect_resource_data(row[1])
                data[i].extend([cpu, ht_size])

    with open(output_file, "wb") as f:
        csvwriter = csv.writer(f)
        csvwriter.writerows(data)

if __name__ == '__main__':
    #resource_exp()
    #profile_exp()
    #cold_cache_exp("cold_cache_1.csv")
    #cold_cache_exp("cold_cache_2.csv")
    #cold_cache_exp("cold_cache_3.csv")
    #cold_cache_exp("cold_cache_4.csv")
    #cold_cache_exp("cold_cache_5.csv")
    #cold_cache_rslfj_exp("cold_cache_rslfj.csv")
    profile_rslfj("profile_rslfj.csv")
    resource_rslfj("resource_rslfj.csv")
    #add_resource_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/resource_exp.csv",
    #    "resource_extend.csv")
    #q = ('myrial', "RS_HJ", "NONE", queries.triangle, 'whatever')
    #execute_query(q)
    # collect_network_data(1031)
    # add_network_data(
    #    "/Users/chushumo/Project/papers/2014-multiwayjoin/profile_exp.csv",
    #    "profile_extend.csv")
<|endoftext|>


<commit_before>"""
A wrapper for different Python bindings for Qt.
"""
import os
import importlib
from collections import namedtuple

# the order in the tuple represents the search order when importing the package
packages = ('PySide6', 'PyQt6', 'PySide2', 'PyQt5')

# one can create a QT_API environment variable to select which Qt API to use
qt_api = os.getenv('QT_API')
if qt_api:
    if qt_api not in packages:
        raise ValueError('Invalid QT_API environment variable {!r}'.format(qt_api))
else:
    for package in packages:
        try:
            importlib.import_module(package)
        except ImportError:
            continue
        else:
            qt_api = package
            break

if not qt_api:
    raise ImportError('One of the following Qt packages must be installed: ' + ', '.join(packages))

Binding = namedtuple('Binding', ['name', 'version', 'qt_version', 'version_info', 'qt_version_info'])

# do not use, for example, QtGui = importlib.import_module(qt_api + '.QtGui')
# for the following because this does not allow for IDE's to use autocompletion
if qt_api == 'PyQt5':
    import PyQt5 as api
    from PyQt5 import QtGui
    from PyQt5 import QtCore
    from PyQt5 import QtWidgets
    from PyQt5 import QtSvg
elif qt_api == 'PySide2':
    import PySide2 as api
    from PySide2 import QtGui
    from PySide2 import QtCore
    from PySide2 import QtWidgets
    from PySide2 import QtSvg
elif qt_api == 'PyQt6':
    import PyQt6 as api
    from PyQt6 import QtGui
    from PyQt6 import QtCore
    from PyQt6 import QtWidgets
    from PyQt6 import QtSvg
else:
    import PySide6 as api
    from PySide6 import QtGui
    from PySide6 import QtCore
    from PySide6 import QtWidgets
    from PySide6 import QtSvg

Qt = QtCore.Qt

if qt_api.startswith('PySide'):
    Signal = QtCore.Signal
    Slot = QtCore.Slot
    binding = Binding(
        name=api.__name__,
        version=api.__version__,
        qt_version=QtCore.__version__,
        version_info=api.__version_info__,
        qt_version_info=QtCore.__version_info__,
    )
else:
    def _to_version_info(version):
        major = (version >> 16) & 0xff
        minor = (version >> 8) & 0xff
        patch = version & 0xff
        return major, minor, patch

    Signal = QtCore.pyqtSignal
    Slot = QtCore.pyqtSlot
    binding = Binding(
        name=api.__name__,
        version=QtCore.PYQT_VERSION_STR,
        qt_version=QtCore.QT_VERSION_STR,
        version_info=_to_version_info(QtCore.PYQT_VERSION),
        qt_version_info=_to_version_info(QtCore.QT_VERSION),
    )

if not hasattr(QtWidgets.QApplication, 'exec'):
    QtWidgets.QApplication.exec = QtWidgets.QApplication.exec_

if not hasattr(QtWidgets.QDialog, 'exec'):
    QtWidgets.QDialog.exec = QtWidgets.QDialog.exec_

# PyQt6 uses scoped enums. Want to support unscoped enums to be
# consistent with PySide6, PySide2 and PyQt5
if qt_api == 'PyQt6':
    from enum import EnumMeta
    from inspect import getmembers, isclass

    def isenum(obj):
        return isinstance(obj, EnumMeta)

    for qt in (QtCore, QtWidgets, QtGui, QtSvg):
        for _, cls in getmembers(qt, isclass):
            for _, enum in getmembers(cls, isenum):
                for item in enum:
                    setattr(cls, item.name, item)

# make Qt5 to be compatible with Qt6
if qt_api in ('PySide2', 'PyQt5'):

    # QAction and QActionGroup were moved from QtWidgets (Qt5) to QtGui (Qt6)
    QtGui.QAction = QtWidgets.QAction
    QtGui.QActionGroup = QtWidgets.QActionGroup

__all__ = (
    'Qt',
    'QtGui',
    'QtWidgets',
    'QtCore',
    'QtSvg',
    'Signal',
    'Slot',
    'binding',
)
<commit_msg>check if qt_api startswith PyQt<commit_after>"""
A wrapper for different Python bindings for Qt.
"""
import os
import importlib
from collections import namedtuple

# the order in the tuple represents the search order when importing the package
packages = ('PySide6', 'PyQt6', 'PySide2', 'PyQt5')

# one can create a QT_API environment variable to select which Qt API to use
qt_api = os.getenv('QT_API')
if qt_api:
    if qt_api not in packages:
        raise ValueError('Invalid QT_API environment variable {!r}'.format(qt_api))
else:
    for package in packages:
        try:
            importlib.import_module(package)
        except ImportError:
            continue
        else:
            qt_api = package
            break

if not qt_api:
    raise ImportError('One of the following Qt packages must be installed: ' + ', '.join(packages))

Binding = namedtuple('Binding', ['name', 'version', 'qt_version', 'version_info', 'qt_version_info'])

# do not use, for example, QtGui = importlib.import_module(qt_api + '.QtGui')
# for the following because this does not allow for IDE's to use autocompletion
if qt_api == 'PyQt5':
    import PyQt5 as api
    from PyQt5 import QtGui
    from PyQt5 import QtCore
    from PyQt5 import QtWidgets
    from PyQt5 import QtSvg
elif qt_api == 'PySide2':
    import PySide2 as api
    from PySide2 import QtGui
    from PySide2 import QtCore
    from PySide2 import QtWidgets
    from PySide2 import QtSvg
elif qt_api == 'PyQt6':
    import PyQt6 as api
    from PyQt6 import QtGui
    from PyQt6 import QtCore
    from PyQt6 import QtWidgets
    from PyQt6 import QtSvg
else:
    import PySide6 as api
    from PySide6 import QtGui
    from PySide6 import QtCore
    from PySide6 import QtWidgets
    from PySide6 import QtSvg

Qt = QtCore.Qt

if qt_api.startswith('PyQt'):
    def to_version_info(version):
        major = (version >> 16) & 0xff
        minor = (version >> 8) & 0xff
        patch = version & 0xff
        return major, minor, patch

    Signal = QtCore.pyqtSignal
    Slot = QtCore.pyqtSlot
    binding = Binding(
        name=api.__name__,
        version=QtCore.PYQT_VERSION_STR,
        qt_version=QtCore.QT_VERSION_STR,
        version_info=to_version_info(QtCore.PYQT_VERSION),
        qt_version_info=to_version_info(QtCore.QT_VERSION),
    )
else:
    Signal = QtCore.Signal
    Slot = QtCore.Slot
    binding = Binding(
        name=api.__name__,
        version=api.__version__,
        qt_version=QtCore.__version__,
        version_info=api.__version_info__,
        qt_version_info=QtCore.__version_info__,
    )

if not hasattr(QtWidgets.QApplication, 'exec'):
    QtWidgets.QApplication.exec = QtWidgets.QApplication.exec_

if not hasattr(QtWidgets.QDialog, 'exec'):
    QtWidgets.QDialog.exec = QtWidgets.QDialog.exec_

# PyQt6 uses scoped enums. Want to support unscoped enums to be
# consistent with PySide6, PySide2 and PyQt5
if qt_api == 'PyQt6':
    from enum import EnumMeta
    from inspect import getmembers, isclass

    def isenum(obj):
        return isinstance(obj, EnumMeta)

    for qt in (QtCore, QtWidgets, QtGui, QtSvg):
        for _, cls in getmembers(qt, isclass):
            for _, enum in getmembers(cls, isenum):
                for item in enum:
                    setattr(cls, item.name, item)

# make Qt5 to be compatible with Qt6
if qt_api in ('PySide2', 'PyQt5'):

    # QAction and QActionGroup were moved from QtWidgets (Qt5) to QtGui (Qt6)
    QtGui.QAction = QtWidgets.QAction
    QtGui.QActionGroup = QtWidgets.QActionGroup

__all__ = (
    'Qt',
    'QtGui',
    'QtWidgets',
    'QtCore',
    'QtSvg',
    'Signal',
    'Slot',
    'binding',
)
<|endoftext|>