<commit_before>#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""This example builds deep residual network for mnist data.

Reference Paper: http://arxiv.org/pdf/1512.03385.pdf

Note that this is still a work-in-progress. Feel free to submit a PR
to make this better.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import namedtuple
from math import sqrt
import os

from sklearn import metrics
import tensorflow as tf
from tensorflow.contrib import learn
from tensorflow.examples.tutorials.mnist import input_data

def res_net(x, y, activation=tf.nn.relu):
  """Builds a residual network.

  Note that if the input tensor is 2D, it must be square in order to be
  converted to a 4D tensor.

  Borrowed structure from:
  github.com/pkmital/tensorflow_tutorials/blob/master/10_residual_network.py

  Args:
    x: Input of the network
    y: Output of the network
    activation: Activation function to apply after each convolution

  Returns:
    Predictions and loss tensors.
  """

  # Configurations for each bottleneck block.
  BottleneckBlock = namedtuple(
      'BottleneckBlock', ['num_layers', 'num_filters', 'bottleneck_size'])
  blocks = [BottleneckBlock(3, 128, 32),
            BottleneckBlock(3, 256, 64),
            BottleneckBlock(3, 512, 128),
            BottleneckBlock(3, 1024, 256)]

  input_shape = x.get_shape().as_list()

  # Reshape the input into the right shape if it's 2D tensor
  if len(input_shape) == 2:
    ndim = int(sqrt(input_shape[1]))
    x = tf.reshape(x, [-1, ndim, ndim, 1])

  # First convolution expands to 64 channels
  with tf.variable_scope('conv_layer1'):
    net = learn.ops.conv2d(x, 64, [7, 7], batch_norm=True,
                           activation=activation, bias=False)

  # Max pool
  net = tf.nn.max_pool(
      net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')

  # First chain of resnets
  with tf.variable_scope('conv_layer2'):
    net = learn.ops.conv2d(net, blocks[0].num_filters,
                           [1, 1], [1, 1, 1, 1],
                           padding='VALID', bias=True)

  # Create each bottleneck building block for each layer
  for block_i, block in enumerate(blocks):
    for layer_i in range(block.num_layers):

      name = 'block_%d/layer_%d' % (block_i, layer_i)

      # 1x1 convolution responsible for reducing dimension
      with tf.variable_scope(name + '/conv_in'):
        conv = learn.ops.conv2d(net, block.bottleneck_size,
                                [1, 1], [1, 1, 1, 1],
                                padding='VALID',
                                activation=activation,
                                batch_norm=True,
                                bias=False)

      with tf.variable_scope(name + '/conv_bottleneck'):
        conv = learn.ops.conv2d(conv, block.bottleneck_size,
                                [3, 3], [1, 1, 1, 1],
                                padding='SAME',
                                activation=activation,
                                batch_norm=True,
                                bias=False)

      # 1x1 convolution responsible for restoring dimension
      with tf.variable_scope(name + '/conv_out'):
        conv = learn.ops.conv2d(conv, block.num_filters,
                                [1, 1], [1, 1, 1, 1],
                                padding='VALID',
                                activation=activation,
                                batch_norm=True,
                                bias=False)

      # shortcut connections that turn the network into its counterpart
      # residual function (identity shortcut)
      net = conv + net

      try:
        # upscale to the next block size
        next_block = blocks[block_i + 1]
        with tf.variable_scope('block_%d/conv_upscale' % block_i):
          net = learn.ops.conv2d(net, next_block.num_filters,
                                 [1, 1], [1, 1, 1, 1],
                                 bias=False,
                                 padding='SAME')
      except IndexError:
        pass

  net_shape = net.get_shape().as_list()
  net = tf.nn.avg_pool(net,
                       ksize=[1, net_shape[1], net_shape[2], 1],
                       strides=[1, 1, 1, 1], padding='VALID')

  net_shape = net.get_shape().as_list()
  net = tf.reshape(net, [-1, net_shape[1] * net_shape[2] * net_shape[3]])

  return learn.models.logistic_regression(net, y)

# Download and load MNIST data.
mnist = input_data.read_data_sets('MNIST_data')

# Restore model if graph is saved into a folder.
if os.path.exists('models/resnet/graph.pbtxt'):
  classifier = learn.TensorFlowEstimator.restore('models/resnet/')
else:
  # Create a new resnet classifier.
  classifier = learn.TensorFlowEstimator(
      model_fn=res_net, n_classes=10, batch_size=100, steps=100,
      learning_rate=0.001, continue_training=True)

while True:
  # Train model and save summaries into logdir.
  classifier.fit(
      mnist.train.images, mnist.train.labels, logdir='models/resnet/')

  # Calculate accuracy.
  score = metrics.accuracy_score(
      mnist.test.labels, classifier.predict(mnist.test.images, batch_size=64))
  print('Accuracy: {0:f}'.format(score))

  # Save model graph and checkpoints.
  classifier.save('models/resnet/')
<commit_msg>fix dim mismatch bug in resnet example. (#2863)<commit_after>#  Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""This example builds deep residual network for mnist data.

Reference Paper: http://arxiv.org/pdf/1512.03385.pdf

Note that this is still a work-in-progress. Feel free to submit a PR
to make this better.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import namedtuple
from math import sqrt
import os

from sklearn import metrics
import tensorflow as tf
from tensorflow.contrib import learn
from tensorflow.examples.tutorials.mnist import input_data

def res_net(x, y, activation=tf.nn.relu):
  """Builds a residual network.

  Note that if the input tensor is 2D, it must be square in order to be
  converted to a 4D tensor.

  Borrowed structure from:
  github.com/pkmital/tensorflow_tutorials/blob/master/10_residual_network.py

  Args:
    x: Input of the network
    y: Output of the network
    activation: Activation function to apply after each convolution

  Returns:
    Predictions and loss tensors.
  """

  # Configurations for each bottleneck group.
  BottleneckGroup = namedtuple(
      'BottleneckGroup', ['num_blocks', 'num_filters', 'bottleneck_size'])
  groups = [BottleneckGroup(3, 128, 32),
            BottleneckGroup(3, 256, 64),
            BottleneckGroup(3, 512, 128),
            BottleneckGroup(3, 1024, 256)]

  input_shape = x.get_shape().as_list()

  # Reshape the input into the right shape if it's 2D tensor
  if len(input_shape) == 2:
    ndim = int(sqrt(input_shape[1]))
    x = tf.reshape(x, [-1, ndim, ndim, 1])

  # First convolution expands to 64 channels
  with tf.variable_scope('conv_layer1'):
    net = learn.ops.conv2d(x, 64, [7, 7], batch_norm=True,
                           activation=activation, bias=False)

  # Max pool
  net = tf.nn.max_pool(
      net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')

  # First chain of resnets
  with tf.variable_scope('conv_layer2'):
    net = learn.ops.conv2d(net, groups[0].num_filters,
                           [1, 1], [1, 1, 1, 1],
                           padding='VALID', bias=True)

  # Create the bottleneck groups, each of which contains `num_blocks`
  # bottleneck groups.
  for group_i, group in enumerate(groups):
    for block_i in range(group.num_blocks):
      name = 'group_%d/block_%d' % (group_i, block_i)

      # 1x1 convolution responsible for reducing dimension
      with tf.variable_scope(name + '/conv_in'):
        conv = learn.ops.conv2d(net, group.bottleneck_size,
                                [1, 1], [1, 1, 1, 1],
                                padding='VALID',
                                activation=activation,
                                batch_norm=True,
                                bias=False)

      with tf.variable_scope(name + '/conv_bottleneck'):
        conv = learn.ops.conv2d(conv, group.bottleneck_size,
                                [3, 3], [1, 1, 1, 1],
                                padding='SAME',
                                activation=activation,
                                batch_norm=True,
                                bias=False)

      # 1x1 convolution responsible for restoring dimension
      with tf.variable_scope(name + '/conv_out'):
        input_dim = net.get_shape()[-1].value
        conv = learn.ops.conv2d(conv, input_dim,
                                [1, 1], [1, 1, 1, 1],
                                padding='VALID',
                                activation=activation,
                                batch_norm=True,
                                bias=False)

      # shortcut connections that turn the network into its counterpart
      # residual function (identity shortcut)
      net = conv + net

    try:
      # upscale to the next group size
      next_group = groups[group_i + 1]
      with tf.variable_scope('block_%d/conv_upscale' % group_i):
        net = learn.ops.conv2d(net, next_group.num_filters,
                               [1, 1], [1, 1, 1, 1],
                               bias=False,
                               padding='SAME')
    except IndexError:
      pass

  net_shape = net.get_shape().as_list()
  net = tf.nn.avg_pool(net,
                       ksize=[1, net_shape[1], net_shape[2], 1],
                       strides=[1, 1, 1, 1], padding='VALID')

  net_shape = net.get_shape().as_list()
  net = tf.reshape(net, [-1, net_shape[1] * net_shape[2] * net_shape[3]])

  return learn.models.logistic_regression(net, y)

# Download and load MNIST data.
mnist = input_data.read_data_sets('MNIST_data')

# Restore model if graph is saved into a folder.
if os.path.exists('models/resnet/graph.pbtxt'):
  classifier = learn.TensorFlowEstimator.restore('models/resnet/')

while True:
  # Train model and save summaries into logdir.
  classifier.fit(
      mnist.train.images, mnist.train.labels, logdir='models/resnet/')

  # Calculate accuracy.
  score = metrics.accuracy_score(
      mnist.test.labels, classifier.predict(mnist.test.images, batch_size=64))
  print('Accuracy: {0:f}'.format(score))
<|endoftext|>


<commit_before>#!/usr/bin/python3

import contextlib
import time
import re

import reap

def match(re, string):
    return reap.execute_threaded(reap.generate(reap.parse_2rp(re)), string)

@contextlib.contextmanager
def timer():
    t0 = time.clock()
    yield
    print('duration', time.clock() - t0, 'seconds')

def main():
    for i in range(20):
        tre = 'a?'*i + 'a'*i
        s = 'a'*i
        print()
        print(i, repr(tre), 'against', repr(s))
        print('re   ', end='')
        with timer():
            re.compile(tre).match(s)

        print('reap ', end='')
        with timer():
            match(tre, s)

if __name__ == '__main__':
    main()
<commit_msg>don't time the compilation<commit_after>#!/usr/bin/python3

import contextlib
import time
import re

import reap

def match(re, string):
    return reap.execute_threaded(reap.generate(reap.parse_2rp(re)), string)

@contextlib.contextmanager
def timer():
    t0 = time.clock()
    yield
    print('duration', time.clock() - t0, 'seconds')

def main():
    for i in range(20):
        tre = 'a?'*i + 'a'*i
        s = 'a'*i
        print()
        print(i, repr(tre), 'against', repr(s))
        print('re   ', end='')
        c = re.compile(tre)
        with timer():
            c.match(s)

        print('reap ', end='')
        c = reap.generate(reap.parse_2rp(tre))
        with timer():
            reap.execute_threaded(c, s)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import random
from Queue import Queue
import timeit
import subprocess
import sys
import os

class Leaf(object):
    '''Create leaf.'''
    def __init__(self, key, left=None, right=None, parent=None):
        self.left = left
        self.key = key
        self.right = right
        self.parent = parent

    def _get_dot(self):
        """recursively prepare a dot graph entry for this node."""
        if self.parent is not None:
            yield "\t%s -> %s;" % (self.key, self.parent.key)
        if self.left is not None:
            yield "\t%s -> %s;" % (self.key, self.left.key)
            for i in self.left._get_dot():
                yield i
        elif self.right is not None:
            r = random.randint(0, 1e9)
            yield "\tnull%s [shape=point];" % r
            yield "\t%s -> null%s;" % (self.key, r)
        if self.right is not None:
            yield "\t%s -> %s;" % (self.key, self.right.key)
            for i in self.right._get_dot():
                yield i
        elif self.left is not None:
            r = random.randint(0, 1e9)
            yield "\tnull%s [shape=point];" % r
            yield "\t%s -> null%s;" % (self.key, r)

class Tree(object):
    '''Create tree'''

    def __init__(self, root=None):
        self.root = root

    def _contains(self, key, leaf):
        if leaf is None or leaf.key == key:
            return leaf
        elif key < leaf.key:
            return self._contains(key, leaf.left)
        else:
            return self._contains(key, leaf.right)

    def contains(self, key):
        return bool(self._contains(key, self.root))

    def insert(self, key):
        if not self.root:
            self.root = Leaf(key)
        else:
            self._insert(key, self.root)

    def _insert(self, key, leaf):
        if key == leaf.key:
            return leaf
        elif key < leaf.key:
            if leaf.left:
                self._insert(key, leaf.left)
            else:
                leaf.left = Leaf(key, parent=leaf)
        else:
            if leaf.right:
                self._insert(key, leaf.right)
            else:
                leaf.right = Leaf(key, parent=leaf)

    def _find_replacement(self, leaf):
        current = leaf
        while current.right:
            current = current.right
        return current

    def _remove_node(self, leaf, replacement_leaf=None):
        if leaf.parent:
            if leaf.parent.left == leaf:
                leaf.parent.left = leaf.left
            else:
                leaf.parent.right = leaf.right
        if replacement_leaf:
            replacement_leaf.parent = leaf.parent

    def delete(self, target):
        self._delete(target, self.root)

    def _delete(self, target, start):
        leaf = self._contains(target, start)
        if leaf:
            if leaf.left and leaf.right:
                successor = self._find_replacement(leaf.left)
                leaf.key = successor.key
                self._delete(successor.key, successor)
            elif leaf.left:
                self._remove_node(leaf, leaf.left)
            elif leaf.right:
                self._remove_node(leaf, leaf.right)
            else:
                self._remove_node(leaf)

    def size(self):
        if not self.root:
            return 0
        else:
            return self._size(self.root, 1)

    def _size(self, leaf, count):
        if leaf.left:
            count += 1
            count = self._size(leaf.left, count)
        if leaf.right:
            count += 1
            count = self._size(leaf.right, count)
        return count

    def depth(self):
        if not self.root:
            return 0
        else:
            return max(self._depth(self.root))+1

    def _depth(self, leaf):
        if leaf is None:
            return 0
        else:
            depthL = self._depth(leaf.left)
            depthR = self._depth(leaf.right)
        if leaf == self.root:
            return depthL, depthR
        return max([depthL, depthR]) + 1

    def balance(self):
        return self._balance(self.root)

    def _balance(self, leaf):
        depthL, depthR = self._depth(leaf)
        return depthL - depthR

    def in_order(self):
        return self._in_order(self.root)

    def _in_order(self, leaf):
        if leaf is None:
            return
        for val in self._in_order(leaf.left):
            yield val
        yield leaf.key
        for val in self._in_order(leaf.right):
            yield val

    def pre_order(self):
        return self._pre_order(self.root)

    def _pre_order(self, leaf):
        if leaf is None:
            return
        yield leaf.key
        for val in self._pre_order(leaf.left):
            yield val
        for val in self._pre_order(leaf.right):
            yield val

    def post_order(self):
        return self._post_order(self.root)

    def _post_order(self, leaf):
        if leaf is None:
            return
        for val in self._post_order(leaf.left):
            yield val
        for val in self._post_order(leaf.right):
            yield val
        yield leaf.key

    def breadth_first_traversal(self):
        q = Queue()
        q.put(self.root)
        while not q.empty():
            leaf = q.get()
            yield leaf.key
            if leaf.left:
                q.put(leaf.left)
            if leaf.right:
                q.put(leaf.right)

    def get_dot(self):
        """return the tree with root 'self' as a dot graph for visualization"""
        return "digraph G{\n%s}" % ("" if self.root.key is None else (
            "\t%s;\n%s\n" % (
                self.root.key,
                "\n".join(self.root._get_dot())
            )
        ))

    def graph(self, file_name):
        t = subprocess.Popen(["dot", "-Tpng"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        png_data = t.communicate(self.get_dot())
        with open('{}.png'.format(file_name), 'w') as png_file:
            png_file.write(png_data[0])
        os.system('open {}.png'.format(file_name))

    def nuclear_option(self):
        generator = self.in_order()
        num = []
        while True:
            try:
                num.append(generator.next())
            except StopIteration:
                self.root = self._nuclear_option(num, 0, len(num)-1, None)
                return

    def _nuclear_option(self, num, begin, end, parent):
        if begin > end:
            return None
        midPoint = (begin+end)//2
        root = Leaf(num[midPoint], parent=parent)
        root.left = self._nuclear_option(num, begin, midPoint-1, root)
        root.right = self._nuclear_option(num, midPoint+1, end, root)
        return root

    def avl_insert(self, key):
        if not self.root:
            self.root = Leaf(key)
        else:
            leaf = self._insert(key, self.root)
        if not -1 >= self.balance(leaf.parent.parent) >= 1:
            self._avl_insert(leaf)

    def _avl_insert(self, leaf):
        parent = leaf.parent.parent
        current = leaf.parent
        while parent:
            if current == parent.left:
                if self._balance(parent) == 2:
                    if self._balance(current) == -1:
                        self._rotate_left(current)
                    self._rotate_right(parent)
                    return
                if self._balance(parent) == -1:
                    return
            else:
                if self._balance(parent) == -2:
                    if self._balance(current) == 1:
                        self._rotate_right(current)
                    self._rotate_left(parent)
                    return
                if self._balance(parent) == 1:
                    return
            current, parent = parent, parent.parent

    def _rotate_left(self, leaf):
        leaf.right.parent = leaf.parent
        leaf.parent.left = leaf.right
        leaf.right.left = leaf
        leaf.parent = leaf.right

    def _rotate_right(self, leaf):
        leaf.left.parent = leaf.parent
        leaf.parent.right = leaf.left
        leaf.left.right = leaf
        leaf.parent = leaf.left

if __name__ == '__main__':

    def makeBalancedTree(num):
        return makeBalance(num, 0, len(num)-1)

    def makeBalance(num, begin, end):
        if begin > end:
            return None
        midPoint = (begin+end)//2
        root = Leaf(num[midPoint])
        root.left = makeBalance(num, begin, midPoint-1)
        root.right = makeBalance(num, midPoint+1, end)
        return root

    if len(sys.argv) > 1:
        if sys.argv[1] == 't':
            nums = []
            nums.append(range(0, 5))
            nums.append(range(0, 50))
            nums.append(range(0, 500))

            for num in nums:
                T_easy = Tree(makeBalancedTree(num))
                T_hard = Tree()
                for i in num:
                    T_hard.insert(i)

                def test_contains_easy():
                    T_easy.contains(num[-1])

                def test_contains_hard():
                    T_hard.contains(num[-1])

                best_case = '{:10.10f}s :Best Case,  Balanced,   Leaves {},  Depth {}'
                worst_case = '{:10.10f}s :Worst Case, Unbalanced, Leaves {},  Depth {}'
                time_easy = timeit.Timer(
                    "test_contains_easy()",
                    setup="from __main__ import test_contains_easy").timeit(
                    number=10000)
                time_hard = timeit.Timer(
                    "test_contains_hard()",
                    setup="from __main__ import test_contains_hard").timeit(
                    number=10000)

                print(best_case.format(time_easy, len(num), T_easy.depth()))
                print(worst_case.format(time_hard, len(num), T_hard.depth()))
                print('{} times slower'.format(float(time_hard)/float(time_easy)))
                print('\n')
        elif sys.argv[1] == 'g':
            sample = range(0, 37)
            T = Tree(makeBalancedTree(sample))
            T.graph('bst')
<commit_msg>work on avl insert<commit_after>import random
from Queue import Queue
import timeit
import subprocess
import sys
import os

class Leaf(object):
    '''Create leaf.'''
    def __init__(self, key, left=None, right=None, parent=None):
        self.left = left
        self.key = key
        self.right = right
        self.parent = parent

    def _get_dot(self):
        """recursively prepare a dot graph entry for this node."""
        if self.parent is not None:
            yield "\t%s -> %s;" % (self.key, self.parent.key)
        if self.left is not None:
            yield "\t%s -> %s;" % (self.key, self.left.key)
            for i in self.left._get_dot():
                yield i
        elif self.right is not None:
            r = random.randint(0, 1e9)
            yield "\tnull%s [shape=point];" % r
            yield "\t%s -> null%s;" % (self.key, r)
        if self.right is not None:
            yield "\t%s -> %s;" % (self.key, self.right.key)
            for i in self.right._get_dot():
                yield i
        elif self.left is not None:
            r = random.randint(0, 1e9)
            yield "\tnull%s [shape=point];" % r
            yield "\t%s -> null%s;" % (self.key, r)

class Tree(object):
    '''Create tree'''

    def __init__(self, root=None):
        self.root = root

    def _contains(self, key, leaf):
        if leaf is None or leaf.key == key:
            return leaf
        elif key < leaf.key:
            return self._contains(key, leaf.left)
        else:
            return self._contains(key, leaf.right)

    def contains(self, key):
        return bool(self._contains(key, self.root))

    def insert(self, key):
        if not self.root:
            self.root = Leaf(key)
        else:
            self._insert(key, self.root)

    def _insert(self, key, leaf):
        if key == leaf.key:
            return leaf
        elif key < leaf.key:
            if leaf.left:
                return self._insert(key, leaf.left)
            else:
                leaf.left = Leaf(key, parent=leaf)
                return leaf.left
        else:
            if leaf.right:
                return self._insert(key, leaf.right)
            else:
                leaf.right = Leaf(key, parent=leaf)
                return leaf.right

    def _find_replacement(self, leaf):
        current = leaf
        while current.right:
            current = current.right
        return current

    def _remove_node(self, leaf, replacement_leaf=None):
        if leaf.parent:
            if leaf.parent.left == leaf:
                leaf.parent.left = leaf.left
            else:
                leaf.parent.right = leaf.right
        if replacement_leaf:
            replacement_leaf.parent = leaf.parent

    def delete(self, target):
        self._delete(target, self.root)

    def _delete(self, target, start):
        leaf = self._contains(target, start)
        if leaf:
            if leaf.left and leaf.right:
                successor = self._find_replacement(leaf.left)
                leaf.key = successor.key
                self._delete(successor.key, successor)
            elif leaf.left:
                self._remove_node(leaf, leaf.left)
            elif leaf.right:
                self._remove_node(leaf, leaf.right)
            else:
                self._remove_node(leaf)

    def size(self):
        if not self.root:
            return 0
        else:
            return self._size(self.root, 1)

    def _size(self, leaf, count):
        if leaf.left:
            count += 1
            count = self._size(leaf.left, count)
        if leaf.right:
            count += 1
            count = self._size(leaf.right, count)
        return count

    def depth(self):
        if not self.root:
            return 0
        else:
            return max(self._depth(self.root))+1

    def _depth(self, leaf):
        if leaf is None:
            return 0
        else:
            depthL = self._depth(leaf.left)
            depthR = self._depth(leaf.right)
        if leaf == self.root:
            return depthL, depthR
        return max([depthL, depthR]) + 1

    def balance(self, start=None):
        if not start:
            return self._balance(self.root)
        else:
            return self._balance(start)

    def _balance(self, leaf):
        depthL, depthR = self._depth(leaf)
        return depthL - depthR

    def in_order(self):
        return self._in_order(self.root)

    def _in_order(self, leaf):
        if leaf is None:
            return
        for val in self._in_order(leaf.left):
            yield val
        yield leaf.key
        for val in self._in_order(leaf.right):
            yield val

    def pre_order(self):
        return self._pre_order(self.root)

    def _pre_order(self, leaf):
        if leaf is None:
            return
        yield leaf.key
        for val in self._pre_order(leaf.left):
            yield val
        for val in self._pre_order(leaf.right):
            yield val

    def post_order(self):
        return self._post_order(self.root)

    def _post_order(self, leaf):
        if leaf is None:
            return
        for val in self._post_order(leaf.left):
            yield val
        for val in self._post_order(leaf.right):
            yield val
        yield leaf.key

    def breadth_first_traversal(self):
        q = Queue()
        q.put(self.root)
        while not q.empty():
            leaf = q.get()
            yield leaf.key
            if leaf.left:
                q.put(leaf.left)
            if leaf.right:
                q.put(leaf.right)

    def get_dot(self):
        """return the tree with root 'self' as a dot graph for visualization"""
        return "digraph G{\n%s}" % ("" if self.root.key is None else (
            "\t%s;\n%s\n" % (
                self.root.key,
                "\n".join(self.root._get_dot())
            )
        ))

    def graph(self, file_name):
        t = subprocess.Popen(["dot", "-Tpng"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        png_data = t.communicate(self.get_dot())
        with open('{}.png'.format(file_name), 'w') as png_file:
            png_file.write(png_data[0])
        os.system('open {}.png'.format(file_name))

    def nuclear_option(self):
        generator = self.in_order()
        num = []
        while True:
            try:
                num.append(generator.next())
            except StopIteration:
                self.root = self._nuclear_option(num, 0, len(num)-1, None)
                return

    def _nuclear_option(self, num, begin, end, parent):
        if begin > end:
            return None
        midPoint = (begin+end)//2
        root = Leaf(num[midPoint], parent=parent)
        root.left = self._nuclear_option(num, begin, midPoint-1, root)
        root.right = self._nuclear_option(num, midPoint+1, end, root)
        return root

    def avl_insert(self, key):
        if not self.root:
            self.root = Leaf(key)
        else:
            leaf = self._insert(key, self.root)
            if not -1 >= self.balance(leaf.parent.parent) >= 1:
                self._avl_insert(leaf)

    def _avl_insert(self, leaf):
        parent = leaf.parent.parent
        current = leaf.parent
        while parent:
            if current == parent.left:
                if self._balance(parent) == 2:
                    import pdb; pdb.set_trace()
                    if self._balance(current) == -1:
                        self._rotate_left(current)
                    self._rotate_right(parent)
                    return
                if self._balance(parent) == -1:
                    return
            else:
                if self._balance(parent) == -2:
                    if self._balance(current) == 1:
                        self._rotate_right(current)
                    self._rotate_left(parent)
                    return
                if self._balance(parent) == 1:
                    return
            current, parent = parent, parent.parent

    def _rotate_left(self, leaf):
        leaf.right.parent = leaf.parent
        leaf.parent.left = leaf.right
        leaf.right.left = leaf
        leaf.parent = leaf.right

    def _rotate_right(self, leaf):
        leaf.left.parent = leaf.parent
        leaf.parent.right = leaf.left
        leaf.left.right = leaf
        leaf.parent = leaf.left

if __name__ == '__main__':

    def makeBalancedTree(num):
        return makeBalance(num, 0, len(num)-1)

    def makeBalance(num, begin, end):
        if begin > end:
            return None
        midPoint = (begin+end)//2
        root = Leaf(num[midPoint])
        root.left = makeBalance(num, begin, midPoint-1)
        root.right = makeBalance(num, midPoint+1, end)
        return root

    if len(sys.argv) > 1:
        if sys.argv[1] == 't':
            nums = []
            nums.append(range(0, 5))
            nums.append(range(0, 50))
            nums.append(range(0, 500))

            for num in nums:
                T_easy = Tree(makeBalancedTree(num))
                T_hard = Tree()
                for i in num:
                    T_hard.insert(i)

                def test_contains_easy():
                    T_easy.contains(num[-1])

                def test_contains_hard():
                    T_hard.contains(num[-1])

                best_case = '{:10.10f}s :Best Case,  Balanced,   Leaves {},  Depth {}'
                worst_case = '{:10.10f}s :Worst Case, Unbalanced, Leaves {},  Depth {}'
                time_easy = timeit.Timer(
                    "test_contains_easy()",
                    setup="from __main__ import test_contains_easy").timeit(
                    number=10000)
                time_hard = timeit.Timer(
                    "test_contains_hard()",
                    setup="from __main__ import test_contains_hard").timeit(
                    number=10000)

                print(best_case.format(time_easy, len(num), T_easy.depth()))
                print(worst_case.format(time_hard, len(num), T_hard.depth()))
                print('{} times slower'.format(float(time_hard)/float(time_easy)))
                print('\n')
        elif sys.argv[1] == 'g':
            sample = range(0, 37)
            T = Tree(makeBalancedTree(sample))
            T.graph('bst')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class BinarySearchTree(object):
  """Implements a basic binary search tree in Python.

  This BST does not do any kind of balancing, and thus can degenerate into a
  linked list in the worst case.

  This BST also does NOT permit duplicates.
  """

  class _Node(object):
    """Represents a node in the binary search tree."""

    left = None
    right = None

    def __init__(self, value):
      self.value = value

  _root = None
  _size = 0

  def __init__(self, data=None):
    """Creates a new binary search tree.

    Args:
      data: An optional tuple or list containing data to be inserted IN ORDER
      into the BST.  If not supplied, the BST is created empty.
    """
    for datum in (data or []):
      self.insert(datum)

  @property
  def inorder(self):
    """Returns values of all nodes in order, as a list.

    For an empty BST, this returns an empty list.
    """
    return self.__inorder(self._root)

  def insert(self, value):
    """Inserts a new value into the BST (without rebalancing the tree).

    Returns True if the value was successfully inserted, False otherwise (e.g.,
    probably because the value duplicated an existing value in the BST).
    """
    self._root, result = self.__insert(value, self._root)
    return result

  @property
  def root(self):
    """Returns the value of the root node.

    If BST is empty, raises a ValueError (i.e., there's no root node to query).
    """
    try:
      return self._root.value
    except AttributeError:
      raise ValueError('Tree is empty; no root node to query.')

  @property
  def size(self):
    """Returns the number of elements in the BST as an int."""
    return self._size

  def __inorder(self, root):
    """Returns values of all nodes in a subtree in order, as a list.

    For an empty subtree (defined by a root of None), this returns an empty
    list.
    """
    if not root:
      return []
    return self.__inorder(root.left) + [root.value] + self.__inorder(root.right)

  def __insert(self, value, root):
    """Attempts to insert the given value into the subtree with the given root.

    Returns True if the value was successfully inserted, False otherwise (e.g.,
    probably because the value duplicated an existing value in the BST).
    """
    if not root:
      return self._Node(value), True
    if value < root.value:
      return self.__insert(value, root.left)
    if value > root.value:
      return self.__insert(value, root.right)
    return root, False
<commit_msg>missing update of size<commit_after># -*- coding: utf-8 -*-

class BinarySearchTree(object):
  """Implements a basic binary search tree in Python.

  This BST does not do any kind of balancing, and thus can degenerate into a
  linked list in the worst case.

  This BST also does NOT permit duplicates.
  """

  class _Node(object):
    """Represents a node in the binary search tree."""

    left = None
    right = None

    def __init__(self, value):
      self.value = value

  _root = None
  _size = 0

  def __init__(self, data=None):
    """Creates a new binary search tree.

    Args:
      data: An optional tuple or list containing data to be inserted IN ORDER
      into the BST.  If not supplied, the BST is created empty.
    """
    for datum in (data or []):
      self.insert(datum)

  @property
  def inorder(self):
    """Returns values of all nodes in order, as a list.

    For an empty BST, this returns an empty list.
    """
    return self.__inorder(self._root)

  def insert(self, value):
    """Inserts a new value into the BST (without rebalancing the tree).

    Returns True if the value was successfully inserted, False otherwise (e.g.,
    probably because the value duplicated an existing value in the BST).
    """
    self._root, result = self.__insert(value, self._root)
    self._size += int(result)
    return result

  @property
  def root(self):
    """Returns the value of the root node.

    If BST is empty, raises a ValueError (i.e., there's no root node to query).
    """
    try:
      return self._root.value
    except AttributeError:
      raise ValueError('Tree is empty; no root node to query.')

  @property
  def size(self):
    """Returns the number of elements in the BST as an int."""
    return self._size

  def __inorder(self, root):
    """Returns values of all nodes in a subtree in order, as a list.

    For an empty subtree (defined by a root of None), this returns an empty
    list.
    """
    if not root:
      return []
    return self.__inorder(root.left) + [root.value] + self.__inorder(root.right)

  def __insert(self, value, root):
    """Attempts to insert the given value into the subtree with the given root.

    Returns True if the value was successfully inserted, False otherwise (e.g.,
    probably because the value duplicated an existing value in the BST).
    """
    if not root:
      return self._Node(value), True
    if value < root.value:
      return self.__insert(value, root.left)
    if value > root.value:
      return self.__insert(value, root.right)
    return root, False
<|endoftext|>


<commit_before>from tables import HuffmanTree

huffmanRoot = HuffmanTree.create()

# 6.1 Integer Representation (encode)
def packIntRepresentation(I, N):
    if I < (1 << N) - 1:
        return [I]
    else:
        buf = [(1 << N) - 1]
        I -= (1 << N) - 1
        while I >= 0x80:
            buf.append(I & 0x7f | 0x80)
            I = (I >> 7)
        buf.append(I)
        return buf

#TODO here should use shift operator
def packContent(content, huffman):
    wire = ""
    if huffman:
        if not content:
            # when value is ''
            return '80'
        enc, actualLen = HuffmanTree.encode(content)
        intRep = packIntRepresentation(actualLen, 7)
        intRep[0] = intRep[0] | 0x80
        wire += "".join([hex(b)[2:].zfill(2) for b in intRep]) + enc
    else:
        intRep = packIntRepresentation(len(content), 7)
        wire += "".join([hex(b)[2:].zfill(2) for b in intRep])
        wire += "".join([hex(ord(c))[2:].zfill(2) for c in content])
    return wire

def encode(headers, fromStaticTable, fromHeaderTable, huffman, table):
    wire = ""
    for header in headers:
        match = table.find(header[0], header[1])
        # 7.1 Indexed Header Field Representation
        if fromStaticTable and match[0]:
            intRep = packIntRepresentation(match[1], 4)
            intRep[0] = intRep[0] | 0x00 if not fromHeaderTable else intRep[0] | 0x80
            wire += "".join([hex(b)[2:].zfill(2) for b in intRep])
            if not fromHeaderTable:
                wire += packContent(header[1], huffman)

        # 7.2.1 Literal Header Field with Incremental Indexing
        elif fromStaticTable and not match[0] and match[1]:
            if fromHeaderTable:
                intRep = packIntRepresentation(match[1], 6)
                intRep[0] |= 0x40
                table.add(header)
            else:
                intRep = packIntRepresentation(match[1], 4)
                intRep[0] |= 0x00
            wire += "".join([hex(b)[2:].zfill(2) for b in intRep])
            wire += packContent(header[1], huffman)
        else:
            content = packContent(header[0], huffman) + packContent(header[1], huffman)
            prefix = "40" if fromHeaderTable else "00"
            wire += prefix + content
            if fromHeaderTable:
                table.add(header)

    return wire

# 6.1 Integer Representation (decode)
def parseIntRepresentation(buf, N):
    I = (buf[0] & ((1 << N) - 1))
    cursor = 1
    if I < (1 << N) - 1:
        return I, cursor
    else:
        M = 0
        while buf[cursor] & 0x80:
            I += (buf[cursor] & 0x7f) * (1 << M)
            M += 7
            cursor += 1
        I += (buf[cursor] & 0x7f) * (1 << M)
        return I, cursor + 1

def extractContent(subBuf, length, isHuffman):
    if isHuffman:
        return huffmanRoot.decode(subBuf, length)
    else:
        return "".join([chr(subBuf[i]) for i in range(length)])

def parseHeader(index, table, subBuf, isIndexed):
    def parseFromByte(buf):
        isHuffman = buf[0] & 0x80
        length, cursor = parseIntRepresentation(buf, 7)
        content = extractContent(buf[cursor:], length, isHuffman)
        cursor += length
        return content, cursor

    cursor = 0
    name = value = ""
    if not isIndexed:
        if not index:
            name, c = parseFromByte(subBuf[cursor:])
            cursor += c
        value, c = parseFromByte(subBuf[cursor:])
        cursor += c

    if index:
        header = table.get(index)
        name = header[0]
        value = value or header[1]

    return name, value, cursor

def decode(data, table):
    buf = [int(data[i:i+2], 16) for i in range(0, len(data), 2)]
    cursor = 0
    index = 0
    headers = []
    while cursor < len(buf):
        isIndexed = False
        isIncremental = False
        if buf[cursor] & 0xe0 == 0x20:
            # 7.3 Header Table Size Update
           table.setMaxHeaderTableSize(buf[cursor] & 0x1f)
           cursor += 1
        elif buf[cursor] & 0x80:
            # 7.1 Indexd Header Field
            if not buf[cursor] & 0x7f:
                print("error")
            index, c = parseIntRepresentation(buf[cursor:], 7)
            cursor += c
            isIndexed = True
        else :
            if buf[cursor] & 0xc0 == 0x40:
                # 7.2.1 Literal Header Field with Incremental Indexing
                index, c = parseIntRepresentation(buf[cursor:], 6)
                isIncremental = True
            elif buf[cursor] & 0xf0 == 0xf0:
                # 7.2.3 Literal Header Field never Indexed
                index, c = parseIntRepresentation(buf[cursor:], 4)
            else:
                # 7.2.2 Literal Header Field without Indexing
                index, c = parseIntRepresentation(buf[cursor:], 4)
            cursor += c

        name, value, c = parseHeader(index, table, buf[cursor:], isIndexed)
        cursor += c

        if isIncremental:
            table.add([name, value])
        headers.append({name:value})

    return headers

if __name__ == "__main__":
    data = "1FA18DB701" #3000000
    data = "00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"
    for i in packIntRepresentation(3000000, 5):
        print(hex(i))
    #print parseIntRepresentation("".join([str(hex(b))[2:] for b in buf]), 5)
<commit_msg>added toHex function which remove '0x' prefix<commit_after>from tables import HuffmanTree

huffmanRoot = HuffmanTree.create()

# 6.1 Integer Representation (encode)
def packIntRepresentation(I, N):
    if I < (1 << N) - 1:
        return [I]
    else:
        buf = [(1 << N) - 1]
        I -= (1 << N) - 1
        while I >= 0x80:
            buf.append(I & 0x7f | 0x80)
            I = (I >> 7)
        buf.append(I)
        return buf

def toHex(num):
    return hex(num)[2:].zfill(2)

def packContent(content, huffman):
    wire = ""
    if huffman:
        if not content:
            # when value is ''
            return '80'
        enc, actualLen = HuffmanTree.encode(content)
        intRep = packIntRepresentation(actualLen, 7)
        intRep[0] = intRep[0] | 0x80
        wire += "".join([toHex(b) for b in intRep]) + enc
    else:
        intRep = packIntRepresentation(len(content), 7)
        wire += "".join([toHex(b) for b in intRep])
        wire += "".join([toHex(ord(c)) for c in content])
    return wire

def encode(headers, fromStaticTable, fromHeaderTable, huffman, table):
    wire = ""
    for header in headers:
        match = table.find(header[0], header[1])
        # 7.1 Indexed Header Field Representation
        if fromStaticTable and match[0]:
            intRep = packIntRepresentation(match[1], 4)
            intRep[0] = intRep[0] | 0x00 if not fromHeaderTable else intRep[0] | 0x80
            wire += "".join([toHex(b) for b in intRep])
            if not fromHeaderTable:
                wire += packContent(header[1], huffman)

        # 7.2.1 Literal Header Field with Incremental Indexing
        elif fromStaticTable and not match[0] and match[1]:
            if fromHeaderTable:
                intRep = packIntRepresentation(match[1], 6)
                intRep[0] |= 0x40
                table.add(header)
            else:
                intRep = packIntRepresentation(match[1], 4)
                intRep[0] |= 0x00
            wire += "".join([toHex(b) for b in intRep])
            wire += packContent(header[1], huffman)
        else:
            content = packContent(header[0], huffman) + packContent(header[1], huffman)
            prefix = "40" if fromHeaderTable else "00"
            wire += prefix + content
            if fromHeaderTable:
                table.add(header)

    return wire

# 6.1 Integer Representation (decode)
def parseIntRepresentation(buf, N):
    I = (buf[0] & ((1 << N) - 1))
    cursor = 1
    if I < (1 << N) - 1:
        return I, cursor
    else:
        M = 0
        while buf[cursor] & 0x80:
            I += (buf[cursor] & 0x7f) * (1 << M)
            M += 7
            cursor += 1
        I += (buf[cursor] & 0x7f) * (1 << M)
        return I, cursor + 1

def extractContent(subBuf, length, isHuffman):
    if isHuffman:
        return huffmanRoot.decode(subBuf, length)
    else:
        return "".join([chr(subBuf[i]) for i in range(length)])

def parseHeader(index, table, subBuf, isIndexed):
    def parseFromByte(buf):
        isHuffman = buf[0] & 0x80
        length, cursor = parseIntRepresentation(buf, 7)
        content = extractContent(buf[cursor:], length, isHuffman)
        cursor += length
        return content, cursor

    cursor = 0
    name = value = ""
    if not isIndexed:
        if not index:
            name, c = parseFromByte(subBuf[cursor:])
            cursor += c
        value, c = parseFromByte(subBuf[cursor:])
        cursor += c

    if index:
        header = table.get(index)
        name = header[0]
        value = value or header[1]

    return name, value, cursor

def decode(data, table):
    buf = [int(data[i:i+2], 16) for i in range(0, len(data), 2)]
    cursor = 0
    index = 0
    headers = []
    while cursor < len(buf):
        isIndexed = False
        isIncremental = False
        if buf[cursor] & 0xe0 == 0x20:
            # 7.3 Header Table Size Update
           table.setMaxHeaderTableSize(buf[cursor] & 0x1f)
           cursor += 1
        elif buf[cursor] & 0x80:
            # 7.1 Indexd Header Field
            if not buf[cursor] & 0x7f:
                print("error")
            index, c = parseIntRepresentation(buf[cursor:], 7)
            cursor += c
            isIndexed = True
        else :
            if buf[cursor] & 0xc0 == 0x40:
                # 7.2.1 Literal Header Field with Incremental Indexing
                index, c = parseIntRepresentation(buf[cursor:], 6)
                isIncremental = True
            elif buf[cursor] & 0xf0 == 0xf0:
                # 7.2.3 Literal Header Field never Indexed
                index, c = parseIntRepresentation(buf[cursor:], 4)
            else:
                # 7.2.2 Literal Header Field without Indexing
                index, c = parseIntRepresentation(buf[cursor:], 4)
            cursor += c

        name, value, c = parseHeader(index, table, buf[cursor:], isIndexed)
        cursor += c

        if isIncremental:
            table.add([name, value])
        headers.append({name:value})

    return headers

if __name__ == "__main__":
    data = "1FA18DB701" #3000000
    data = "00073a6d6574686f640347455400073a736368656d650468747470000a3a617574686f726974790f7777772e7961686f6f2e636f2e6a7000053a70617468012f"
    for i in packIntRepresentation(3000000, 5):
        print(hex(i))
    #print parseIntRepresentation("".join([str(hex(b))[2:] for b in buf]), 5)
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import operator
import os
import re
import glob
import trace
"""
data_dir = '../data2/shubhankar/subset_data/'
#data_dir = '/data2/gmrtarch/cycle20/'
VALID_LIST = '../parser/filter_healthy/healthy2_file.txt'
valid_observations = open(VALID_LIST, 'r').read().split('\n')[0:-1]
all_observations = os.listdir(data_dir)
ltahdr_error_dir = open('ltahdr_error.txt', 'wa')
lta_success_file = open('ltahdr_success.txt', 'wa')
"""
def INVALID_OBS():
    for DIR_NAME in all_observations:
        current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')
        """
        if current_obslog == []:
            print DIR_NAME
            break
        """
        #Extract substring that contains obslog relative path
        relative_path = re.findall(r'[/][\d]+[.]obslog', current_obslog[0])[0][1:]
        #Invalid file (not fitting given constraints i.e. < 900 MHz and IF BW != 6,16,32)
        if relative_path not in valid_observations:
            print data_dir+DIR_NAME
        #Valid obslog file with no LTA file in the DIR

        if relative_path in valid_observations:
            if glob.glob(data_dir+DIR_NAME+'/'+'*.lta') == []:
                print data_dir+DIR_NAME

def NUM_LTA():
    valid_obs = []
    all_lta_files = []
    for DIR_NAME in all_observations:
        current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')

        #Extract substring that contains obslog relative path
        relative_path = re.findall(r'[/][\d]+[.]obslog', current_obslog[0])[0][1:]

        if relative_path in valid_observations:
            current_dir_lta_files = glob.glob(data_dir+DIR_NAME+'/'+'*.lta*')
            #print current_dir_lta_files
            if(current_dir_lta_files != []):
                valid_obs.append(data_dir+DIR_NAME)
                all_lta_files.append(current_dir_lta_files)
    return len([i for j in all_lta_files for i in j])

def VALID_OBS():
    valid_obs = []
    for DIR_NAME in all_observations:
        current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')

            #Extract substring that contains obslog relative path
        relative_path = re.findall(r'[/][\d]+[.]obslog', current_obslog[0])[0][1:]

        if relative_path in valid_observations:
            if glob.glob(data_dir+DIR_NAME+'/'+'*.lta') != []:
                valid_obs.append(data_dir+DIR_NAME)
    return valid_obs

def extract( file_name ):
    with open(file_name) as f:
        for i,line in enumerate(f,1):
            if "SCN" in line:
                return i

def main(lta_name):
    calibrator_list = ['3C48', '3C147', '3C286']
    os.system('ltahdr -i'+ lta_name + '> lta_file.txt')
    dictionary = {}
    try:
        skipped_rows = extract('lta_file.txt')-1

        header = pd.read_csv('lta_file.txt',skiprows=skipped_rows,delimiter=r"\s+")
        flux = list(set(header["OBJECT"]))
        #print flux

        header['Nrecs'] = header['Nrecs'].astype(float)

        for i in flux :
            temp = header.loc[header.OBJECT==i,'Nrecs'].values
            temp = np.mean(temp)
            dictionary[i]=temp

        #Sort the list of targets according to the number of recordings
        list_of_targets = sorted(dictionary.iteritems(),key=operator.itemgetter(1), reverse=True)
        for i in list_of_targets:
            if i[0] in calibrator_list:
                continue
            else:
                os.system('rm lta_file.txt')
                return i[0]
        #Return this value if entire list of targets is populated with calibrators
        return -1
    except:
        pass

def test_validity():
    #valid_observations = VALID_OBS()
    #lta_file_count = 0
    #for DIR in valid_observations:

    all_lta_files = glob.glob('*.lta*')
    output_list = []
    for LTA_FILE in all_lta_files:
        try:
            output = main(LTA_FILE)
            #Check if valid target name is found
            if output != -1:
                lta_success_file.write(LTA_FILE + '\n' + output + '\n')
                lta_file_count += 1
                output_list.append([output, LTA_FILE])
                """
                print(LTA_FILE)
                print output
                print lta_file_count
                """
            else:
                continue
                ltahdr_error_dir.write(DIR+ '/' + LTA_FILE + '\n')
                lta_file_count += 1
        except:
            #Log the error directory to a file
            ltahdr_error_dir.write(DIR+ '/' + LTA_FILE + '\n')
            lta_file_count += 1
    return output_list
<commit_msg>Changed and added test_validity file<commit_after>import pandas as pd
import numpy as np
import operator
import os
import re
import glob
import trace
def INVALID_OBS():
    invalid_obs = []
    for DIR_NAME in all_observations:
        current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')
        if current_obslog == []:
            continue

        #Extract substring that contains obslog relative path
        relative_path = re.findall(r'[/][\d]+[.]obslog', current_obslog[0])[0][1:]
        #Invalid file (not fitting given constraints i.e. < 900 MHz and IF BW != 6,16,32)
        if relative_path not in valid_observations:
 	    invalid_obs.append(data_dir+DIR_NAME)

	#Valid obslog file with no LTA file in the DIR
        if relative_path in valid_observations:
            if glob.glob(data_dir+DIR_NAME+'/'+'*.lta') == []:
                #print data_dir+DIR_NAME
		invalid_obs.append(data_dir+DIR_NAME)
    return invalid_obs

def VALID_OBS():
    valid_obs = []
    for DIR_NAME in all_observations:
            current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')
	    if current_obslog == []:
            	continue
            #Extract substring that contains obslog relative path
            relative_path = re.findall(r'[/][\d]+[.]obslog', current_obslog[0])[0][1:]

            if relative_path in valid_observations:
                if glob.glob(data_dir+DIR_NAME+'/'+'*.lta') != []:
                    valid_obs.append(data_dir+DIR_NAME)
    return valid_obs

def NUM_LTA():
    valid_obs = []
    all_lta_files = []
    for DIR_NAME in all_observations:
        current_obslog = glob.glob(data_dir+DIR_NAME+'/'+'*.obslog')

        #Extract substring that contains obslog relative path
        relative_path = re.findall(r'[/][\d]+[.]obslog', current_obslog[0])[0][1:]

        if relative_path in valid_observations:
            current_dir_lta_files = glob.glob(data_dir+DIR_NAME+'/'+'*.lta*')
            #print current_dir_lta_files
            if(current_dir_lta_files != []):
                valid_obs.append(data_dir+DIR_NAME)
                all_lta_files.append(current_dir_lta_files)
    return len([i for j in all_lta_files for i in j])

def extract( file_name ):
    with open(file_name) as f:
        for i,line in enumerate(f,1):
            if "SCN" in line:
                return i

def main(lta_name):
    calibrator_list = ['3C48', '3C147', '3C286']
    os.system('ltahdr -i'+ lta_name + '> lta_file.txt')
    dictionary = {}
    try:
        skipped_rows = extract('lta_file.txt')-1

        header = pd.read_csv('lta_file.txt',skiprows=skipped_rows,delimiter=r"\s+")
        flux = list(set(header["OBJECT"]))
        #print flux

        header['Nrecs'] = header['Nrecs'].astype(float)

        for i in flux :
            temp = header.loc[header.OBJECT==i,'Nrecs'].values
            temp = np.mean(temp)
            dictionary[i]=temp

        #Sort the list of targets according to the number of recordings
        list_of_targets = sorted(dictionary.iteritems(),key=operator.itemgetter(1), reverse=True)
        for i in list_of_targets:
            if i[0] in calibrator_list:
                continue
            else:
                os.system('rm lta_file.txt')
                return i[0]
        #Return this value if entire list of targets is populated with calibrators
        return -1
    except:
        pass

def test_validity():
    #valid_observations = VALID_OBS()
    #lta_file_count = 0
    #for DIR in valid_observations:

    lta_uvfits_files = glob.glob('*.lta*')
    uvfits_files = glob.glob('*.lta*.UVFITS')
    all_lta_files = [i for i in lta_uvfits_files if i not in uvfits_files]
    output_list = []
    for LTA_FILE in all_lta_files:
        try:
            output = main(LTA_FILE)
            #Check if valid target name is found
            #if output != -1:
                #lta_success_file.write(LTA_FILE + '\n' + output + '\n')
                #lta_file_count += 1
             #   output_list.append([output, LTA_FILE])

            if output == -1:
                continue
                #ltahdr_error_dir.write(DIR+ '/' + LTA_FILE + '\n')
                #lta_file_count += 1
        except:
	    continue
            #Log the error directory to a file
            #ltahdr_error_dir.write(DIR+ '/' + LTA_FILE + '\n')
            #lta_file_count += 1
    return output
<|endoftext|>


<commit_before>FEATURES_CUSTOM_CALLBACK = {
  'global_namespace' : str,
  'global_ttl' : int,
  'policiesByServerPoolFeature' : bool,
  'messagesBySecFeature' : bool,
  'messagesBySecStoreDays' : int,
  'feederFeaturesEnabled' : bool,
  'featuresByServerPool' : bool,
  'learnFeature' : bool,
  'learnPredictSafeValue' : float,
  'learnOnlyOnce' : bool,
  'learnEscalationValue' : float,
  'learnTimeFrameValue' : list,
  'learnBlockMinutes' : float,
  'spfStrictRejectFeature' : bool,
  'rateReputationFeature' : bool,
  'rateReputationBlockHitValue' : int,
  'rateReputationDecreaseValue' : float,
  'rateReputationIncreaseMinutes' : float,
  'ipReputationFeature' : bool,
  'ipReputationHitValue' : int,
  'ipReputationDecreaseValue' : float,
  'ipReputationIncreaseMinutes' : float,
  'subjectReputationFeature' : bool,
  'subjectReputationHitValue' : int,
  'subjectReputationDecreaseValue' : float,
  'subjectReputationIncreaseMinutes' : float,
  'global_conditions' : list,
  'global_custom_block' : int
}

METADATA_CUSTOM_CALLBACK = {
  'learnPredictSafeValue' : float,
  'learnEscalationValue' : float,
  'subject_repeated_count' : int,
  'ip_reputation_lastupdate' : float,
  'subject_lastupdate' : float,
  'sentmessages_lastupdate' : float,
  'last_update' : float,
  'last_subject' : str,
  'blue_creation_date' : float,
  'block_count' : int,
  'learningBlueMode' : bool,
  'learningRedMode' : bool,
  'manual_block' : bool,
  'predictBy' : str,
  'bypass' : bool
}

POLICY_CUSTOM_CALLBACK = {
  'enable' : bool,
  'stophere' : bool,
  'requestsmon' : bool,
  'countrcpt' : bool,
  'ipprobation' : float,
  'subjectprobation' : float,
  'countsentprobation' : float,
  'blockprobation' : float,
  'priority' : int,
  'jailspec' : list,
  'onlyheaders' : bool,
  'spf' : bool,
  'actionheaders' : dict
}

DEFAULT_FEATURES_VALUES = {
  'global_namespace' : 'AI',
  'global_ttl' : 86400,
  'policiesByServerPoolFeature' : True,
  'featuresByServerPool' : False,
  'messagesBySecFeature' : True,
  'messagesBySecStoreDays' : 1,
  'feederFeaturesEnabled' : False,
  'learnFeature' : False,
  'learnPredictSafeValue' : 30.0,
  'learnOnlyOnce' : True,
  'learnEscalationValue' : 0.2,
  'learnTimeFrameValue' : [['1', '1']],
  'learnBlockMinutes' : 2,
  'spfStrictRejectFeature' : False,
  'rateReputationFeature' : False,
  'rateReputationBlockHitValue' : 5,
  'rateReputationDecreaseValue' : 0.20,
  'rateReputationIncreaseMinutes' : 0,
  'ipReputationFeature' : False,
  'ipReputationHitValue' : 4,
  'ipReputationDecreaseValue' : 0.20,
  'ipReputationIncreaseMinutes' : 0,
  'subjectReputationFeature' : False,
  'subjectReputationHitValue' : 10,
  'subjectReputationDecreaseValue' : 0.10,
  'subjectReputationIncreaseMinutes' : 0,
  'global_conditions' : [ [720, 500] ],
  'global_custom_block' : 0
}

DEFAULT_POLICY_PARAMS = {
  'Enable' : True,
  'Type' : 'regular',
  'Priority' : 5,
  'JailBy' : 'Sender+',
  'JailSpec' : '0:0',
  'JailAction' : 'monitor',
  'ReplyData' : 'Limit reached. Blocking for %s second(s)',
  'CountSentProbation' : 1,
  'CountRCPT' : False,
  'StopHere' : False,
  'RequestsMon' : False,
  'SubjectProbation' : 0.5,
  'IpProbation' : 0.5,
  'BlockProbation' : 0.5,
  'CountSentProbation' : 1,
  'ActionHeaders' : {},
  'OnlyHeaders' : False,
  'SPF' : False,
}

RESERVERD_KEYWORDS = ['global', 'any', 'list', 'all', 'metadata', 'pool', 'policy', 'policies', 'group']<commit_msg>new key fallbackToNonPoolPolicies<commit_after>FEATURES_CUSTOM_CALLBACK = {
  'global_namespace' : str,
  'global_ttl' : int,
  'policiesByServerPoolFeature' : bool,
  'messagesBySecFeature' : bool,
  'messagesBySecStoreDays' : int,
  'feederFeaturesEnabled' : bool,
  'fallbackToNonPoolPolicies' : bool,
  'featuresByServerPool' : bool,
  'learnFeature' : bool,
  'learnPredictSafeValue' : float,
  'learnOnlyOnce' : bool,
  'learnEscalationValue' : float,
  'learnTimeFrameValue' : list,
  'learnBlockMinutes' : float,
  'spfStrictRejectFeature' : bool,
  'rateReputationFeature' : bool,
  'rateReputationBlockHitValue' : int,
  'rateReputationDecreaseValue' : float,
  'rateReputationIncreaseMinutes' : float,
  'ipReputationFeature' : bool,
  'ipReputationHitValue' : int,
  'ipReputationDecreaseValue' : float,
  'ipReputationIncreaseMinutes' : float,
  'subjectReputationFeature' : bool,
  'subjectReputationHitValue' : int,
  'subjectReputationDecreaseValue' : float,
  'subjectReputationIncreaseMinutes' : float,
  'global_conditions' : list,
  'global_custom_block' : int
}

METADATA_CUSTOM_CALLBACK = {
  'learnPredictSafeValue' : float,
  'learnEscalationValue' : float,
  'subject_repeated_count' : int,
  'ip_reputation_lastupdate' : float,
  'subject_lastupdate' : float,
  'sentmessages_lastupdate' : float,
  'last_update' : float,
  'last_subject' : str,
  'blue_creation_date' : float,
  'block_count' : int,
  'learningBlueMode' : bool,
  'learningRedMode' : bool,
  'manual_block' : bool,
  'predictBy' : str,
  'bypass' : bool
}

POLICY_CUSTOM_CALLBACK = {
  'enable' : bool,
  'stophere' : bool,
  'requestsmon' : bool,
  'countrcpt' : bool,
  'ipprobation' : float,
  'subjectprobation' : float,
  'countsentprobation' : float,
  'blockprobation' : float,
  'priority' : int,
  'jailspec' : list,
  'onlyheaders' : bool,
  'spf' : bool,
  'actionheaders' : dict
}

DEFAULT_FEATURES_VALUES = {
  'global_namespace' : 'AI',
  'global_ttl' : 86400,
  'policiesByServerPoolFeature' : True,
  'featuresByServerPool' : False,
  'messagesBySecFeature' : True,
  'messagesBySecStoreDays' : 1,
  'feederFeaturesEnabled' : False,
  'fallbackToNonPoolPolicies' : False,
  'learnFeature' : False,
  'learnPredictSafeValue' : 30.0,
  'learnOnlyOnce' : True,
  'learnEscalationValue' : 0.2,
  'learnTimeFrameValue' : [['1', '1']],
  'learnBlockMinutes' : 2,
  'spfStrictRejectFeature' : False,
  'rateReputationFeature' : False,
  'rateReputationBlockHitValue' : 5,
  'rateReputationDecreaseValue' : 0.20,
  'rateReputationIncreaseMinutes' : 0,
  'ipReputationFeature' : False,
  'ipReputationHitValue' : 4,
  'ipReputationDecreaseValue' : 0.20,
  'ipReputationIncreaseMinutes' : 0,
  'subjectReputationFeature' : False,
  'subjectReputationHitValue' : 10,
  'subjectReputationDecreaseValue' : 0.10,
  'subjectReputationIncreaseMinutes' : 0,
  'global_conditions' : [ [720, 500] ],
  'global_custom_block' : 0
}

DEFAULT_POLICY_PARAMS = {
  'Enable' : True,
  'Type' : 'regular',
  'Priority' : 5,
  'JailBy' : 'Sender+',
  'JailSpec' : '0:0',
  'JailAction' : 'monitor',
  'ReplyData' : 'Limit reached. Blocking for %s second(s)',
  'CountSentProbation' : 1,
  'CountRCPT' : False,
  'StopHere' : False,
  'RequestsMon' : False,
  'SubjectProbation' : 0.5,
  'IpProbation' : 0.5,
  'BlockProbation' : 0.5,
  'CountSentProbation' : 1,
  'ActionHeaders' : {},
  'OnlyHeaders' : False,
  'SPF' : False
}

RESERVERD_KEYWORDS = ['global', 'any', 'list', 'all', 'metadata', 'pool', 'policy', 'policies', 'group']<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import errno
import os.path
from os.path import dirname, realpath, join
from subprocess import Popen, PIPE
import re
import shutil
import tempfile

class TestFailure(Exception):
    pass

# http://stackoverflow.com/a/600612/119527
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

def do_bgrep(pattern, paths, options=[], retcode=0, quiet=False):
    args = [bgrep_path]
    args += list(options)
    args.append(pattern)
    args += list(paths)

    if not quiet:
        print '$ ' + ' '.join(args)

    p = Popen(args, stdout=PIPE)
    stdout, stderr = p.communicate()

    if p.returncode != retcode:
        raise TestFailure('Return code: {0}, expected: {1}'.format(p.returncode, retcode))

    result = {}
    if p.returncode == 0:
        pat = re.compile('^(.*):(0x[0-9A-Fa-f]+).*')

        for line in stdout.splitlines():
            if not quiet:
                print line

            m = pat.match(line)
            if not m: continue

            filename = m.group(1)
            offset = int(m.group(2), 16)

            if not filename in result:
                result[filename] = set()

            result[filename].add(offset)

    return result

def assert_equal(expected, actual):
    if not expected == actual:
        raise TestFailure('Expected: {0}, Actual {1}'.format(expected, actual))

def single_test(data, pattern, offsets, retcode=0, options=[]):
    filename = 'test.bin'
    with open(filename, 'wb') as f:
        f.write(data)

    result = do_bgrep(pattern, [filename], retcode=retcode, options=options)

    for retfilename, retoffsets in result.iteritems():
        assert_equal(filename, retfilename)
        assert_equal(set(offsets), retoffsets)

    # Don't put this in a try/finally, so it hangs around on failure
    # for post-mortem
    os.remove(filename)

def gen_padded_data(offsets, insert, padchar='\0'):
    data = ''
    for o in sorted(offsets):
        padamt = o - len(data)
        assert padamt >= 0
        data += padchar*padamt + insert
    return data

################################################################################
# Tests

def basic_test():
    offsets = [100]
    pattern  = '\x12\x34\x56\x78'
    data = gen_padded_data(offsets, pattern)
    single_test(data, pattern.encode('hex'), offsets)
    single_test(data, pattern, offsets, options=['-s'])

def no_find():
    data = 'notherenotgonnafindit'
    single_test(data, 'HOLYGRAIL'.encode('hex'), [], retcode=1)
    single_test(data, 'HOLYGRAIL', [], retcode=1, options=['-s'])

def multiple_offsets():
    offsets = [4, 27, 369, 630, 750]
    pattern = '\x12\x34\x56\x78'
    data = gen_padded_data(offsets, pattern)
    single_test(data, pattern.encode('hex'), offsets)
    single_test(data, pattern, offsets, options=['-s'])

def no_overlap():
    offsets = [123, 456]
    data = gen_padded_data(offsets, 'cacac')
    single_test(data, 'cac'.encode('hex'), offsets)
    single_test(data, 'cac', offsets, options=['-s'])

def pattern_wild_full():
    for p in xrange(0, 10):
        pad = '00'*p
        data = (pad + '12345678' + pad).decode('hex')
        pattern = '..345678'
        offsets = [p]
        single_test(data, pattern, offsets)

def pattern_wild_high():
    for p in xrange(0, 10):
        pad = '00'*p
        data = (pad + '12345678' + pad).decode('hex')
        pattern = '.2345678'
        offsets = [p]
        single_test(data, pattern, offsets)

def pattern_wild_low():
    for p in xrange(0, 10):
        pad = '00'*p
        data = (pad + '12345678' + pad).decode('hex')
        pattern = '1.345678'
        offsets = [p]
        single_test(data, pattern, offsets)

def recursive():
    basedir = tempfile.mkdtemp(prefix='bgreptest')

    files = {
        join(basedir,'some.bin'):       {0x60, 0x20},
        join(basedir,'other.bin'):      {0x68, 0x78, 0x92},
        join(basedir,'a/data.bin'):     {0x70},
        join(basedir,'a/b/data.bin'):   {0x80, 0x74},
        join(basedir,'a/b/c/foo.bin'):  {0x120},
        join(basedir,'a/b/c/bar.bin'):  {0x230},
        join(basedir,'x/data.bin'):     {0x90},
        join(basedir,'x/y/foo.bin'):    {0x100},
        join(basedir,'x/y/bar.bin'):    {0x110},
    }

    pattern = '\x12\x34\x56\x78'

    for path,offsets in files.iteritems():
        dname, fname = os.path.split(path)
        data = gen_padded_data(offsets, pattern)
        mkdir_p(dname)
        with open(path, 'wb') as f:
            f.write(data)

    result = do_bgrep(pattern.encode('hex'), [basedir], options=['-r'])
    assert_equal(files, result)

    shutil.rmtree(basedir)

def invalid_pattern_byte():
    do_bgrep('abcdeZ', ['n/a'], retcode=1)

def invalid_pattern_highnib():
    do_bgrep('abcdZ.', ['n/a'], retcode=1)

def invalid_pattern_lownib():
    do_bgrep('abcd.Z', ['n/a'], retcode=1)

def invalid_pattern_oddlen():
    do_bgrep('abcde', ['n/a'], retcode=1)

all_tests = [
    basic_test,
    no_find,
    multiple_offsets,
    no_overlap,

    pattern_wild_full,
    pattern_wild_high,
    pattern_wild_low,

    recursive,

    invalid_pattern_byte,
    invalid_pattern_highnib,
    invalid_pattern_lownib,
    invalid_pattern_oddlen,
]

################################################################################

def setup():
    global bgrep_path
    my_path = realpath(join(dirname(__file__), '..'))
    bgrep_path = join(my_path, 'bgrep')
    print 'bgrep path:', bgrep_path

def termcolor(n):
    return '\033[{0}m'.format(n)
RED = 31
GREEN = 32

def main():
    setup()

    failures = 0
    passes = 0

    def message(m, color=0):
        print '[{0:03}] {1}: '.format(n, name) + \
              termcolor(color) + m + termcolor(0)

    for n,test in enumerate(all_tests):
        name = test.__name__
        print '-'*80
        message('Starting')
        try:
            test()
        except TestFailure as tf:
            message('Failure: {0}'.format(tf), RED)
            failures += 1
        else:
            message('Success', GREEN)
            passes += 1

    print '-'*80
    print '{0}/{1} tests passed.'.format(passes, len(all_tests))

    if failures:
        print termcolor(RED) + 'FAILURE' + termcolor(0)
        sys.exit(1)
    else:
        print termcolor(GREEN) + 'Success!' + termcolor(0)
        sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>test: invoke bgrep -v and check for sane output<commit_after>#!/usr/bin/env python
import sys
import os
import errno
import os.path
from os.path import dirname, realpath, join
from subprocess import Popen, PIPE
import re
import shutil
import tempfile

class TestFailure(Exception):
    pass

# http://stackoverflow.com/a/600612/119527
def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

def do_bgrep(pattern, paths, options=[], retcode=0, quiet=False, raw_output=False):
    args = [bgrep_path]
    args += list(options)
    if pattern:
        args.append(pattern)
    args += list(paths)

    if not quiet:
        print '$ ' + ' '.join(args)

    p = Popen(args, stdout=PIPE)
    stdout, stderr = p.communicate()

    if p.returncode != retcode:
        raise TestFailure('Return code: {0}, expected: {1}'.format(p.returncode, retcode))

    if raw_output:
        if not quiet:
            print stdout,
        return stdout

    result = {}
    if p.returncode == 0:
        pat = re.compile('^(.*):(0x[0-9A-Fa-f]+).*')

        for line in stdout.splitlines():
            if not quiet:
                print line

            m = pat.match(line)
            if not m: continue

            filename = m.group(1)
            offset = int(m.group(2), 16)

            if not filename in result:
                result[filename] = set()

            result[filename].add(offset)

    return result

def assert_equal(expected, actual):
    if not expected == actual:
        raise TestFailure('Expected: {0}, Actual {1}'.format(expected, actual))

def single_test(data, pattern, offsets, retcode=0, options=[]):
    filename = 'test.bin'
    with open(filename, 'wb') as f:
        f.write(data)

    result = do_bgrep(pattern, [filename], retcode=retcode, options=options)

    for retfilename, retoffsets in result.iteritems():
        assert_equal(filename, retfilename)
        assert_equal(set(offsets), retoffsets)

    # Don't put this in a try/finally, so it hangs around on failure
    # for post-mortem
    os.remove(filename)

def gen_padded_data(offsets, insert, padchar='\0'):
    data = ''
    for o in sorted(offsets):
        padamt = o - len(data)
        assert padamt >= 0
        data += padchar*padamt + insert
    return data

################################################################################
# Tests

def basic_test():
    offsets = [100]
    pattern  = '\x12\x34\x56\x78'
    data = gen_padded_data(offsets, pattern)
    single_test(data, pattern.encode('hex'), offsets)
    single_test(data, pattern, offsets, options=['-s'])

def no_find():
    data = 'notherenotgonnafindit'
    single_test(data, 'HOLYGRAIL'.encode('hex'), [], retcode=1)
    single_test(data, 'HOLYGRAIL', [], retcode=1, options=['-s'])

def multiple_offsets():
    offsets = [4, 27, 369, 630, 750]
    pattern = '\x12\x34\x56\x78'
    data = gen_padded_data(offsets, pattern)
    single_test(data, pattern.encode('hex'), offsets)
    single_test(data, pattern, offsets, options=['-s'])

def no_overlap():
    offsets = [123, 456]
    data = gen_padded_data(offsets, 'cacac')
    single_test(data, 'cac'.encode('hex'), offsets)
    single_test(data, 'cac', offsets, options=['-s'])

def pattern_wild_full():
    for p in xrange(0, 10):
        pad = '00'*p
        data = (pad + '12345678' + pad).decode('hex')
        pattern = '..345678'
        offsets = [p]
        single_test(data, pattern, offsets)

def pattern_wild_high():
    for p in xrange(0, 10):
        pad = '00'*p
        data = (pad + '12345678' + pad).decode('hex')
        pattern = '.2345678'
        offsets = [p]
        single_test(data, pattern, offsets)

def pattern_wild_low():
    for p in xrange(0, 10):
        pad = '00'*p
        data = (pad + '12345678' + pad).decode('hex')
        pattern = '1.345678'
        offsets = [p]
        single_test(data, pattern, offsets)

def recursive():
    basedir = tempfile.mkdtemp(prefix='bgreptest')

    files = {
        join(basedir,'some.bin'):       {0x60, 0x20},
        join(basedir,'other.bin'):      {0x68, 0x78, 0x92},
        join(basedir,'a/data.bin'):     {0x70},
        join(basedir,'a/b/data.bin'):   {0x80, 0x74},
        join(basedir,'a/b/c/foo.bin'):  {0x120},
        join(basedir,'a/b/c/bar.bin'):  {0x230},
        join(basedir,'x/data.bin'):     {0x90},
        join(basedir,'x/y/foo.bin'):    {0x100},
        join(basedir,'x/y/bar.bin'):    {0x110},
    }

    pattern = '\x12\x34\x56\x78'

    for path,offsets in files.iteritems():
        dname, fname = os.path.split(path)
        data = gen_padded_data(offsets, pattern)
        mkdir_p(dname)
        with open(path, 'wb') as f:
            f.write(data)

    result = do_bgrep(pattern.encode('hex'), [basedir], options=['-r'])
    assert_equal(files, result)

    shutil.rmtree(basedir)

def invalid_pattern_byte():
    do_bgrep('abcdeZ', ['n/a'], retcode=1)

def invalid_pattern_highnib():
    do_bgrep('abcdZ.', ['n/a'], retcode=1)

def invalid_pattern_lownib():
    do_bgrep('abcd.Z', ['n/a'], retcode=1)

def invalid_pattern_oddlen():
    do_bgrep('abcde', ['n/a'], retcode=1)

def test_version():
    result = do_bgrep(None, [], options=['-v'], raw_output=True)
    if not 'bgrep version' in result:
        raise TestFailure('Unexpcted -v output: ' + result)

all_tests = [
    basic_test,
    no_find,
    multiple_offsets,
    no_overlap,

    pattern_wild_full,
    pattern_wild_high,
    pattern_wild_low,

    recursive,

    invalid_pattern_byte,
    invalid_pattern_highnib,
    invalid_pattern_lownib,
    invalid_pattern_oddlen,

    test_version,
]

################################################################################

def setup():
    global bgrep_path
    my_path = realpath(join(dirname(__file__), '..'))
    bgrep_path = join(my_path, 'bgrep')
    print 'bgrep path:', bgrep_path

def termcolor(n):
    return '\033[{0}m'.format(n)
RED = 31
GREEN = 32

def main():
    setup()

    failures = 0
    passes = 0

    def message(m, color=0):
        print '[{0:03}] {1}: '.format(n, name) + \
              termcolor(color) + m + termcolor(0)

    for n,test in enumerate(all_tests):
        name = test.__name__
        print '-'*80
        message('Starting')
        try:
            test()
        except TestFailure as tf:
            message('Failure: {0}'.format(tf), RED)
            failures += 1
        else:
            message('Success', GREEN)
            passes += 1

    print '-'*80
    print '{0}/{1} tests passed.'.format(passes, len(all_tests))

    if failures:
        print termcolor(RED) + 'FAILURE' + termcolor(0)
        sys.exit(1)
    else:
        print termcolor(GREEN) + 'Success!' + termcolor(0)
        sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys
import os
import h5py
import cv2
import numpy as np
import logging
import json
import yaml
import ruamel.yaml
import traceback
from abc import ABCMeta
from abc import abstractmethod
from gi.repository import Vips

logger = logging.getLogger(__name__)

'''
Writer Classes.

All writers make use of the
`with statement context manager <https://docs.python.org/2/reference/datamodel.html#context-managers>`_.
and follow a similar syntax::

    with Writer('/path/to/folder') as writer:
        writer.write('name_of_file')

    with Writer() as writer:
        writer.write('/path/to/file')
'''

class Writer(object):
    '''
    Abstract writer base class.
    '''

    __metaclass__ = ABCMeta

    def __init__(self, directory=None):
        self.directory = directory

    def __enter__(self):
        return self

    def __exit__(self, except_type, except_value, except_trace):
        if except_value:
            sys.stdout.write('The following error occurred:\n%s'
                             % str(except_value))
            for tb in traceback.format_tb(except_trace):
                sys.stdout.write(tb)

class TextWriter(Writer):

    '''
    Abstract base class for writing text to files on disk.
    '''

    __metaclass__ = ABCMeta

    @abstractmethod
    def write(self, filename, data, **kwargs):
        pass

class XmlWriter(TextWriter):

    def __init__(self, directory=None):
        '''
        Initialize an object of class XmlWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(XmlWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Parameters
        ----------
        filename: str
            name of the XML file
        data: list or dict
            the XML string that should be written to the file
        **kwargs: dict
            additional arguments as key-value pairs (none implemented)
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write data to file: %s' % filename)
        with open(filename, 'w') as f:
            f.write(data)

class JsonWriter(TextWriter):

    def __init__(self, directory=None):
        '''
        Initialize an object of class JsonWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(JsonWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Write data to JSON file.

        Parameters
        ----------
        filename: str
            name of the JSON file
        data: list or dict
            the JSON string that should be written to the file
        **kwargs: dict
            additional arguments as key-value pairs
            ("naicify": *bool*, whether `data` should be pretty-printed)

        Note
        ----
        `filename` will be truncated in case it already exists.
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write data to file: %s' % filename)
        if 'naicify' in kwargs:
            naicify = kwargs['naicify']
        else:
            naicify = False
        with open(filename, 'w') as f:
            if naicify:
                json.dump(data, f, sort_keys=True,
                          indent=4, separators=(',', ': '))
            else:
                json.dump(data, f, sort_keys=True)

class YamlWriter(TextWriter):

    def __init__(self, directory=None):
        '''
        Initialize an object of class YamlWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(YamlWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Write data to YAML file.

        Parameters
        ----------
        filename: str
            name of the YAML file
        data: list or dict
            the YAML string that should be written to the file
        **kwargs: dict
            additional arguments as key-value pairs
            ("use_ruamel": *bool*, when `ruamel.yaml` library should be used)

        Note
        ----
        `filename` will be truncated in case it already exists.
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write data to file: %s' % filename)
        if 'use_ruamel' in kwargs:
            use_ruamel = kwargs['use_ruamel']
        else:
            use_ruamel = False
        with open(filename, 'w') as f:
            if use_ruamel:
                f.write(ruamel.yaml.dump(data,
                        Dumper=ruamel.yaml.RoundTripDumper,
                        explicit_start=True))
            else:
                f.write(yaml.dump(data,
                        default_flow_style=False,
                        explicit_start=True))

class ImageWriter(Writer):

    def __init__(self, directory=None):
        '''
        Initialize an object of class ImageWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(ImageWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Write an image to file.

        Parameters
        ----------
        data: numpy.ndarray or Vips.Image
            image that should be saved
        filename: str
            path to the image file
        **kwargs: dict
            additional arguments as key-value pairs (none implemented)

        Raises
        ------
        TypeError
            when `data` is not of type "numpy.ndarray" or "Vips.Image"
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write image to file: %s' % filename)
        if isinstance(data, np.ndarray):
            cv2.imwrite(filename, data)
        elif isinstance(data, Vips.Image):
            data.write_to_file(filename)
        else:
            raise TypeError(
                    'Image must have type "numpy.ndarray" or "Vips.Image"')

class DatasetWriter(object):

    '''
    Base class for writing datasets and attributes to HDF5 files on disk.
    '''

    def __init__(self, filename, truncate=False):
        '''
        Initialize an instance of class DatasetWriter.

        Parameters
        ----------
        filename: str
            absolute path to HDF5 file
        truncate: bool, optional
            whether an existing file should be truncated, i.e. a new file
            created
        '''
        self.filename = filename
        self.truncate = truncate
        logger.debug('write data to file: %s', filename)

    def __enter__(self):
        if self.truncate:
            self._stream = h5py.File(self.filename, 'w')
        else:
            self._stream = h5py.File(self.filename, 'a')
        return self

    def exists(self, path):
        '''
        Check whether a `path` exists within the file.

        Parameters
        ----------
        path: str
            absolute path to a group or dataset in the file

        Returns
        -------
        bool
            ``True`` if `path` exists and ``False`` otherwise
        '''
        if path in self._stream:
            return True
        else:
            return False

    def write(self, path, data):
        '''
        Create a dataset.

        Parameters
        ----------
        path: str
            absolute path to the dataset within the file
        data:
            dataset; will be put through ``numpy.array(data)``

        Note
        ----
        If `data` is a nested list or array of arrays,
        a *variable_length* dataset with dimensions ``(len(data),)`` is
        created. For more information on *variable-length* types, see
        `h5py docs <http://docs.h5py.org/en/latest/special.html>`_.
        '''
        if isinstance(data, basestring):
            data = np.string_(data)
        if ((isinstance(data, np.ndarray) or isinstance(data, list)) and
                all([isinstance(d, basestring) for d in data])):
            data = [np.string_(d) for d in data]
        if isinstance(data, list):
            data = np.array(data)
        if isinstance(data, np.ndarray) and data.dtype == 'O':
            logger.debug('write dataset "%s" as variable length', path)
            self._write_vlen(path, data)
        else:
            logger.debug('write dataset "%s"', path)
            self._stream.create_dataset(path, data=data)

    def _write_vlen(self, path, data):
        data_type = np.unique([d.dtype for d in data])
        if len(data_type) == 0:
            dt = h5py.special_dtype(vlen=np.int64)
        else:
            dt = h5py.special_dtype(vlen=data_type[0])
        dataset = self._stream.create_dataset(path, (len(data),), dtype=dt)
        # dataset = self._stream.create_dataset(path, data.shape, dtype=dt)
        for i, d in enumerate(data):
            dataset[i] = d.tolist()  # doesn't work with numpy.ndarray!!!

    def set_attribute(self, path, name, data):
        '''
        Attach an attribute to a dataset.

        Parameters
        ----------
        path: str
            absolute path to the dataset within the file
        name: str
            name of the attribute
        data:
            value of the attribute; will be put through ``numpy.array(data)``
        '''
        if isinstance(data, basestring):
            data = np.string_(data)
        elif isinstance(data, list):
            data = [
                np.string(d) if isinstance(d, basestring) else d
                for d in data
            ]
        self._stream[path].create(name, data)

    def __exit__(self, except_type, except_value, except_trace):
        self._stream.close()
<commit_msg>Fix bug in writing attributes to HDF5 files<commit_after>import sys
import os
import h5py
import cv2
import numpy as np
import logging
import json
import yaml
import ruamel.yaml
import traceback
from abc import ABCMeta
from abc import abstractmethod
from gi.repository import Vips

logger = logging.getLogger(__name__)

'''
Writer Classes.

All writers make use of the
`with statement context manager <https://docs.python.org/2/reference/datamodel.html#context-managers>`_.
and follow a similar syntax::

    with Writer('/path/to/folder') as writer:
        writer.write('name_of_file')

    with Writer() as writer:
        writer.write('/path/to/file')
'''

class Writer(object):
    '''
    Abstract writer base class.
    '''

    __metaclass__ = ABCMeta

    def __init__(self, directory=None):
        self.directory = directory

    def __enter__(self):
        return self

    def __exit__(self, except_type, except_value, except_trace):
        if except_value:
            sys.stdout.write('The following error occurred:\n%s'
                             % str(except_value))
            for tb in traceback.format_tb(except_trace):
                sys.stdout.write(tb)

class TextWriter(Writer):

    '''
    Abstract base class for writing text to files on disk.
    '''

    __metaclass__ = ABCMeta

    @abstractmethod
    def write(self, filename, data, **kwargs):
        pass

class XmlWriter(TextWriter):

    def __init__(self, directory=None):
        '''
        Initialize an object of class XmlWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(XmlWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Parameters
        ----------
        filename: str
            name of the XML file
        data: list or dict
            the XML string that should be written to the file
        **kwargs: dict
            additional arguments as key-value pairs (none implemented)
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write data to file: %s' % filename)
        with open(filename, 'w') as f:
            f.write(data)

class JsonWriter(TextWriter):

    def __init__(self, directory=None):
        '''
        Initialize an object of class JsonWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(JsonWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Write data to JSON file.

        Parameters
        ----------
        filename: str
            name of the JSON file
        data: list or dict
            the JSON string that should be written to the file
        **kwargs: dict
            additional arguments as key-value pairs
            ("naicify": *bool*, whether `data` should be pretty-printed)

        Note
        ----
        `filename` will be truncated in case it already exists.
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write data to file: %s' % filename)
        if 'naicify' in kwargs:
            naicify = kwargs['naicify']
        else:
            naicify = False
        with open(filename, 'w') as f:
            if naicify:
                json.dump(data, f, sort_keys=True,
                          indent=4, separators=(',', ': '))
            else:
                json.dump(data, f, sort_keys=True)

class YamlWriter(TextWriter):

    def __init__(self, directory=None):
        '''
        Initialize an object of class YamlWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(YamlWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Write data to YAML file.

        Parameters
        ----------
        filename: str
            name of the YAML file
        data: list or dict
            the YAML string that should be written to the file
        **kwargs: dict
            additional arguments as key-value pairs
            ("use_ruamel": *bool*, when `ruamel.yaml` library should be used)

        Note
        ----
        `filename` will be truncated in case it already exists.
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write data to file: %s' % filename)
        if 'use_ruamel' in kwargs:
            use_ruamel = kwargs['use_ruamel']
        else:
            use_ruamel = False
        with open(filename, 'w') as f:
            if use_ruamel:
                f.write(ruamel.yaml.dump(data,
                        Dumper=ruamel.yaml.RoundTripDumper,
                        explicit_start=True))
            else:
                f.write(yaml.dump(data,
                        default_flow_style=False,
                        explicit_start=True))

class ImageWriter(Writer):

    def __init__(self, directory=None):
        '''
        Initialize an object of class ImageWriter.

        Parameters
        ----------
        directory: str, optional
            absolute path to a directory where files are located
        '''
        super(ImageWriter, self).__init__(directory)
        self.directory = directory

    def write(self, filename, data, **kwargs):
        '''
        Write an image to file.

        Parameters
        ----------
        data: numpy.ndarray or Vips.Image
            image that should be saved
        filename: str
            path to the image file
        **kwargs: dict
            additional arguments as key-value pairs (none implemented)

        Raises
        ------
        TypeError
            when `data` is not of type "numpy.ndarray" or "Vips.Image"
        '''
        if self.directory:
            filename = os.path.join(self.directory, filename)
        logger.debug('write image to file: %s' % filename)
        if isinstance(data, np.ndarray):
            cv2.imwrite(filename, data)
        elif isinstance(data, Vips.Image):
            data.write_to_file(filename)
        else:
            raise TypeError(
                    'Image must have type "numpy.ndarray" or "Vips.Image"')

class DatasetWriter(object):

    '''
    Base class for writing datasets and attributes to HDF5 files on disk.
    '''

    def __init__(self, filename, truncate=False):
        '''
        Initialize an instance of class DatasetWriter.

        Parameters
        ----------
        filename: str
            absolute path to HDF5 file
        truncate: bool, optional
            whether an existing file should be truncated, i.e. a new file
            created
        '''
        self.filename = filename
        self.truncate = truncate
        logger.debug('write data to file: %s', filename)

    def __enter__(self):
        if self.truncate:
            self._stream = h5py.File(self.filename, 'w')
        else:
            self._stream = h5py.File(self.filename, 'a')
        return self

    def exists(self, path):
        '''
        Check whether a `path` exists within the file.

        Parameters
        ----------
        path: str
            absolute path to a group or dataset in the file

        Returns
        -------
        bool
            ``True`` if `path` exists and ``False`` otherwise
        '''
        if path in self._stream:
            return True
        else:
            return False

    def write(self, path, data):
        '''
        Create a dataset.

        Parameters
        ----------
        path: str
            absolute path to the dataset within the file
        data:
            dataset; will be put through ``numpy.array(data)``

        Note
        ----
        If `data` is a nested list or array of arrays,
        a *variable_length* dataset with dimensions ``(len(data),)`` is
        created. For more information on *variable-length* types, see
        `h5py docs <http://docs.h5py.org/en/latest/special.html>`_.
        '''
        if isinstance(data, basestring):
            data = np.string_(data)
        if ((isinstance(data, np.ndarray) or isinstance(data, list)) and
                all([isinstance(d, basestring) for d in data])):
            data = [np.string_(d) for d in data]
        if isinstance(data, list):
            data = np.array(data)
        if isinstance(data, np.ndarray) and data.dtype == 'O':
            logger.debug('write dataset "%s" as variable length', path)
            self._write_vlen(path, data)
        else:
            logger.debug('write dataset "%s"', path)
            self._stream.create_dataset(path, data=data)

    def _write_vlen(self, path, data):
        data_type = np.unique([d.dtype for d in data])
        if len(data_type) == 0:
            dt = h5py.special_dtype(vlen=np.int64)
        else:
            dt = h5py.special_dtype(vlen=data_type[0])
        dataset = self._stream.create_dataset(path, (len(data),), dtype=dt)
        # dataset = self._stream.create_dataset(path, data.shape, dtype=dt)
        for i, d in enumerate(data):
            dataset[i] = d.tolist()  # doesn't work with numpy.ndarray!!!

    def set_attribute(self, path, name, data):
        '''
        Attach an attribute to a dataset.

        Parameters
        ----------
        path: str
            absolute path to the dataset within the file
        name: str
            name of the attribute
        data:
            value of the attribute; will be put through ``numpy.array(data)``
        '''
        if isinstance(data, basestring):
            data = np.string_(data)
        elif isinstance(data, list):
            data = [
                np.string_(d) if isinstance(d, basestring) else d
                for d in data
            ]
        self._stream[path].attrs.create(name, data)

    def __exit__(self, except_type, except_value, except_trace):
        self._stream.close()
<|endoftext|>