<commit_before>"""Utils for training neural networks.
"""
import os
import Image
from time import time
from datetime import datetime
from copy import deepcopy
import cPickle

import numpy

import theano
import theano.tensor as T
from theano.sandbox.cuda.basic_ops import gpu_contiguous
from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
from pylearn2.space import Conv2DSpace, VectorSpace, CompositeSpace
from pylearn2.datasets import cifar10

def load_checkpoint(model, checkpoint_path):
    # Added supervised finetuning parameters to this line
    all_parameters = model.all_parameters_symbol+model.all_parameters_classify_symbol
    f = open(checkpoint_path, 'rb')
    checkpoint = cPickle.load(f)
    f.close()

    [model_param.set_value(checkpoint_param) for model_param, checkpoint_param
     in zip(all_parameters, checkpoint)]

def save_checkpoint(model):
    all_parameters_unsupervised = model.all_parameters_symbol
    # Added these two lines to incorporate supervised finetuning parameters
    all_parameters_supervised_finetune = model.all_parameters_classify_symbol
    all_parameters = all_parameters_unsupervised+all_parameters_supervised_finetune

    checkpoint = [param.get_value() for param in all_parameters]
    tt = datetime.now()
    time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
    checkpoint_name = '%s-%s.pkl' % (model.name, time_string)
    print(model.path)
    checkpoint_path = os.path.join(model.path, 'checkpoints', checkpoint_name)

    print 'Saving model checkpoint to: %s' % checkpoint_path
    f = open(checkpoint_path, 'wb')
    cPickle.dump(checkpoint, f)
    f.close()

# def load_checkpoint(model, checkpoint_path):
#     all_parameters = model.all_parameters_symbol
#     f = open(checkpoint_path, 'rb')
#     checkpoint = cPickle.load(f)
#     f.close()

#     [model_param.set_value(checkpoint_param) for model_param, checkpoint_param
#      in zip(all_parameters, checkpoint)]

# def save_checkpoint(model):
#     all_parameters = model.all_parameters_symbol
#     checkpoint = [param.get_value() for param in all_parameters]
#     tt = datetime.now()
#     time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
#     checkpoint_name = '%s-%s.pkl' % (model.name, time_string)
#     print(model.path)
#     checkpoint_path = os.path.join(model.path, 'checkpoints', checkpoint_name)

#     print 'Saving model checkpoint to: %s' % checkpoint_path
#     f = open(checkpoint_path, 'wb')
#     cPickle.dump(checkpoint, f)
#     f.close()

def rescale(data):
    data = data/2.0*255.0
    data[data > 255.0] = 255.0
    return data

class ReconVisualizer(object):
    def __init__(self, model, batch, steps=2000):
        self.model = model
        self.batch = deepcopy(batch)
        self.count = 0
        self.steps = steps
        self.save_path = os.path.join(self.model.path, 'recon')

        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)

    def run(self):
        if self.count % self.steps == 0:
            self._save()
        self.count += 1

    def _save(self):
        tt = datetime.now()
        time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
        sub_path = os.path.join(self.save_path, time_string)

        if not os.path.exists(sub_path):
            os.makedirs(sub_path)

        prediction = self.model.prediction(self.batch)
        for i in range(128):
            image = self.batch[:, :, :, i]
            image = image.transpose(1, 2, 0)

            recon = numpy.array(prediction[:, :, :, i])
            recon = recon.transpose(1, 2, 0)

            image_array = numpy.uint8(rescale(numpy.hstack((image, recon))))

            to_save = Image.fromarray(image_array)
            filename = 'recon-%02d.jpeg' % i
            filepath = os.path.join(sub_path, filename)
            to_save.save(filepath)

class NormReconVisualizer(object):
    def __init__(self, model, batch, steps=2000):
        self.model = model
        self.batch = deepcopy(batch)
        self.count = 0
        self.steps = steps
        self.save_path = os.path.join(self.model.path, 'recon')

        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)

    def run(self):
        if self.count % self.steps == 0:
            self._save()
        self.count += 1

    def _save(self):
        tt = datetime.now()
        time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
        sub_path = os.path.join(self.save_path, time_string)

        if not os.path.exists(sub_path):
            os.makedirs(sub_path)

        prediction = self.model.prediction(self.batch)
        for i in range(128):
            image = self.batch[:, :, :, i]
            image = image.transpose(1, 2, 0)
            image -= image.min()
            image /= image.max()
            image *= 255

            recon = numpy.array(prediction[:, :, :, i])
            recon = recon.transpose(1, 2, 0)
            recon2 = deepcopy(recon)*1.0

            recon -= image.min()
            recon /= image.max()
            recon *= 255

            recon2 -= recon2.min()
            recon2 /= recon2.max()
            recon2 *= 255

            image_array = numpy.uint8(numpy.hstack((image, recon, recon2)))

            to_save = Image.fromarray(image_array)
            filename = 'recon-%02d.jpeg' % i
            filepath = os.path.join(sub_path, filename)
            to_save.save(filepath)

class FilterVisualizer(object):
    def __init__(self, model, steps=2000):
        self.model = model
        self.count = 0
        self.steps = steps
        self.save_path = os.path.join(self.model.path, 'filters')

        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)

    def run(self):
        if self.count % self.steps == 0:
            self._save()
        self.count += 1

    def _save(self):
        tt = datetime.now()
        time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')

        W = self.model.conv1.W.get_value()
        W = W.transpose(1,2,0,3)

        row_list = []
        img_list = []
        k = 0

        rows = W.shape[3]/16
        for i in range(rows):
            for j in range(16):
                W0 = W[:,:,:,k]
                W0[:,:,0] -= W0[:,:,0].min()
                W0[:,:,0] /= W0[:,:,0].max()
                W0[:,:,1] -= W0[:,:,1].min()
                W0[:,:,1] /= W0[:,:,1].max()
                W0[:,:,2] -= W0[:,:,2].min()
                W0[:,:,2] /= W0[:,:,2].max()
                row_list.append(W0)
                k += 1
            row_image = numpy.hstack(row_list)
            row_list = []
            img_list.append(row_image)
        img_image = numpy.vstack(img_list)

        to_save = Image.fromarray(numpy.uint8(255*img_image))
        filename = 'filters_'+time_string+'.png'
        filepath = os.path.join(self.save_path, filename)
        to_save.save(filepath)

class Monitor(object):
    errors = []
    times = []
    big_errors = []
    big_times = []

    def __init__(self, model, step_number=0, best=1, short_steps=10, long_steps=50, save_steps=2000, test_steps=50):
        self.step_number = step_number
        self.best = best
        self.short_steps = short_steps
        self.long_steps = long_steps
        self.save_steps = save_steps
        self.model = model
        self.test = False
        self.test_steps = test_steps

        # Check if model.path exists, if not create it (with a checkpoint folder)
        if model.path and not os.path.exists(os.path.join(model.path, 'checkpoints')):
            os.makedirs(os.path.join(model.path, 'checkpoints'))

    def start(self):
        self.tic = time()

    def stop_test(self, error):
        if self.test:
            self.toc = time()
            _time = self.toc - self.tic
            print '&%d, test error: %.5f, time: %.2f' % (self.step_number, error, _time)

    def stop(self, error):
        self.toc = time()
        _time = self.toc-self.tic
        self.errors.append(error)
        self.times.append(_time)
        self.big_errors.append(error)
        self.big_times.append(_time)
        if self.step_number % self.test_steps == 0:
            self.test = True
        else:
            self.test = False
        if self.step_number % self.save_steps == 0:
            save_checkpoint(self.model)
        if self.step_number % self.long_steps == 0:
            mean_error = numpy.mean(self.big_errors)
            mean_time = numpy.mean(self.big_times)
            print '*%d, train error: %.5f, time: %.2f' % (self.step_number, mean_error, mean_time)
            self.big_errors = []
            self.big_times = []
        if self.step_number % self.short_steps == 0:
            mean_error = numpy.mean(self.errors)
            mean_time = numpy.mean(self.times)
            print '%d, train error: %.5f, time: %.2f' % (self.step_number, mean_error, mean_time)
            self.errors = []
            self.times = []
        self.step_number += 1

class Evaluator(object):
    def __init__(self, model, dataset, steps=100):
        self.step_number = 1
        self.steps = steps
        self.model = model
        self.dataset = dataset

    def run(self):
        input_convspace = Conv2DSpace(shape=(32, 32), num_channels=3,
                                      axes=('c', 0, 1, 'b'))
        data_specs = (input_convspace,'features')
        iterator = self.dataset.iterator(mode='sequential', batch_size=128, data_specs=data_specs)
        errors = []
        if self.step_number % self.steps == 0:
            tic = time()
            for batch in iterator:
                error = self.model.eval(batch)
                errors.append(error)
            toc = time()
            _time = toc-tic
            mean_error = numpy.mean(errors)
            print '%d, test error: %.3f, time: %.2f' % (self.step_number, mean_error, _time)
        self.step_number += 1

def get_cifar_iterator(which_set, mode='sequential', batch_size=128, num_batches=None, center=False, rescale=True, axes=['c', 0, 1, 'b'], targets=False):
    dataset = cifar10.CIFAR10(which_set=which_set,
                              center=center,
                              rescale=rescale,
                              axes=axes
                              )
    input_convspace = Conv2DSpace(shape=(32, 32), num_channels=3,
                              axes=axes)
    #target_space = VectorSpace(dim=10)
    #data_specs = (input_convspace,'features')

    target_space = VectorSpace(dim=10)
    data_specs = (CompositeSpace((input_convspace, target_space)), ("features", "targets"))

    if num_batches:
        iterator = dataset.iterator(mode=mode, batch_size=batch_size, num_batches=num_batches, data_specs=data_specs)
    else:
        iterator = dataset.iterator(mode=mode, batch_size=batch_size, data_specs=data_specs)
    return iterator

class Normer(object):
    def __init__(self, filter_size=7):

        # magic numbers that make things work for stl10
        self.filter_size = 7
        self.pad = self.filter_size/2 -1
        self.num_channels = 3
        self.num_filters = 16
        input = T.ftensor4(name='input')
        filter = T.ftensor4(name='filter')
        gpu_input = gpu_contiguous(input)
        gpu_filter = gpu_contiguous(filter)

        self.conv_func = theano.function([input, filter], FilterActs(pad=3)(gpu_input, gpu_filter))
        n = self.num_channels * self.filter_size * self.filter_size
        self.w = numpy.float32(numpy.ones((self.num_channels, self.filter_size, self.filter_size, self.num_filters)))/n

    def run(self, x_batch):
        mean_batch = self.conv_func(x_batch, self.w)
        mean_batch = numpy.tile(numpy.array(mean_batch[0,:,:,:])[None, :, :], (3, 1, 1, 1))
        diff_batch = x_batch - mean_batch
        std_batch = self.conv_func(diff_batch**2, self.w)
        std_batch = numpy.tile(numpy.array(std_batch[0,:,:,:])[None, :, :], (3, 1, 1, 1))
        norm_batch = diff_batch/(numpy.array(std_batch)**(1/2))
        return norm_batch

class PatchGrabber(object):
    def __init__(self, num_patches, patch_size):
        self.num_patches = num_patches
        self.patch_size = patch_size

    def run(self, x_batch):
        image_size = x_batch.shape[1]
        batch_size = x_batch.shape[-1]

        patches = numpy.zeros((3, self.patch_size, self.patch_size, self.num_patches),dtype=numpy.float32)

        for i_patch in range(self.num_patches):
            x_start = numpy.random.randint(image_size-self.patch_size)
            y_start = numpy.random.randint(image_size-self.patch_size)
            image_id = numpy.random.randint(batch_size)
            x_slice = slice(x_start, x_start+self.patch_size)
            y_slice = slice(y_start, y_start+self.patch_size)
            patch = x_batch[:, x_slice, y_slice, image_id]
            patches[:, :, :, i_patch] = patch

        return patches<commit_msg>Fixed image scaling in NormReconVisualizer.<commit_after>"""Utils for training neural networks.
"""
import os
import Image
from time import time
from datetime import datetime
from copy import deepcopy
import cPickle

import numpy

import theano
import theano.tensor as T
from theano.sandbox.cuda.basic_ops import gpu_contiguous
from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
from pylearn2.space import Conv2DSpace, VectorSpace, CompositeSpace
from pylearn2.datasets import cifar10

def load_checkpoint(model, checkpoint_path):
    # Added supervised finetuning parameters to this line
    all_parameters = model.all_parameters_symbol+model.all_parameters_classify_symbol
    f = open(checkpoint_path, 'rb')
    checkpoint = cPickle.load(f)
    f.close()

    [model_param.set_value(checkpoint_param) for model_param, checkpoint_param
     in zip(all_parameters, checkpoint)]

def save_checkpoint(model):
    all_parameters_unsupervised = model.all_parameters_symbol
    # Added these two lines to incorporate supervised finetuning parameters
    all_parameters_supervised_finetune = model.all_parameters_classify_symbol
    all_parameters = all_parameters_unsupervised+all_parameters_supervised_finetune

    checkpoint = [param.get_value() for param in all_parameters]
    tt = datetime.now()
    time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
    checkpoint_name = '%s-%s.pkl' % (model.name, time_string)
    print(model.path)
    checkpoint_path = os.path.join(model.path, 'checkpoints', checkpoint_name)

    print 'Saving model checkpoint to: %s' % checkpoint_path
    f = open(checkpoint_path, 'wb')
    cPickle.dump(checkpoint, f)
    f.close()

# def load_checkpoint(model, checkpoint_path):
#     all_parameters = model.all_parameters_symbol
#     f = open(checkpoint_path, 'rb')
#     checkpoint = cPickle.load(f)
#     f.close()

#     [model_param.set_value(checkpoint_param) for model_param, checkpoint_param
#      in zip(all_parameters, checkpoint)]

# def save_checkpoint(model):
#     all_parameters = model.all_parameters_symbol
#     checkpoint = [param.get_value() for param in all_parameters]
#     tt = datetime.now()
#     time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
#     checkpoint_name = '%s-%s.pkl' % (model.name, time_string)
#     print(model.path)
#     checkpoint_path = os.path.join(model.path, 'checkpoints', checkpoint_name)

#     print 'Saving model checkpoint to: %s' % checkpoint_path
#     f = open(checkpoint_path, 'wb')
#     cPickle.dump(checkpoint, f)
#     f.close()

def rescale(data):
    data = data/2.0*255.0
    data[data > 255.0] = 255.0
    return data

class ReconVisualizer(object):
    def __init__(self, model, batch, steps=2000):
        self.model = model
        self.batch = deepcopy(batch)
        self.count = 0
        self.steps = steps
        self.save_path = os.path.join(self.model.path, 'recon')

        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)

    def run(self):
        if self.count % self.steps == 0:
            self._save()
        self.count += 1

    def _save(self):
        tt = datetime.now()
        time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
        sub_path = os.path.join(self.save_path, time_string)

        if not os.path.exists(sub_path):
            os.makedirs(sub_path)

        prediction = self.model.prediction(self.batch)
        for i in range(128):
            image = self.batch[:, :, :, i]
            image = image.transpose(1, 2, 0)

            recon = numpy.array(prediction[:, :, :, i])
            recon = recon.transpose(1, 2, 0)

            image_array = numpy.uint8(rescale(numpy.hstack((image, recon))))

            to_save = Image.fromarray(image_array)
            filename = 'recon-%02d.jpeg' % i
            filepath = os.path.join(sub_path, filename)
            to_save.save(filepath)

class NormReconVisualizer(object):
    def __init__(self, model, batch, steps=2000):
        self.model = model
        self.batch = deepcopy(batch)
        self.count = 0
        self.steps = steps
        self.save_path = os.path.join(self.model.path, 'recon')

        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)

    def run(self):
        if self.count % self.steps == 0:
            self._save()
        self.count += 1

    def _save(self):
        tt = datetime.now()
        time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')
        sub_path = os.path.join(self.save_path, time_string)

        if not os.path.exists(sub_path):
            os.makedirs(sub_path)

        prediction = self.model.prediction(self.batch)
        for i in range(128):
            image = deepcopy(self.batch[:, :, :, i])
            image = image.transpose(1, 2, 0)
            image_min = image.min()
            image -= image_min
            image_max = image.max()
            image /= image_max
            image *= 255

            recon = numpy.array(prediction[:, :, :, i])
            recon = recon.transpose(1, 2, 0)
            recon2 = deepcopy(recon)*1.0

            recon -= image_min
            recon /= image_max
            recon *= 255

            recon2 -= recon2.min()
            recon2 /= recon2.max()
            recon2 *= 255

            image_array = numpy.uint8(numpy.hstack((image, recon, recon2)))

            to_save = Image.fromarray(image_array)
            filename = 'recon-%02d.jpeg' % i
            filepath = os.path.join(sub_path, filename)
            to_save.save(filepath)

class FilterVisualizer(object):
    def __init__(self, model, steps=2000):
        self.model = model
        self.count = 0
        self.steps = steps
        self.save_path = os.path.join(self.model.path, 'filters')

        if not os.path.exists(self.save_path):
            os.makedirs(self.save_path)

    def run(self):
        if self.count % self.steps == 0:
            self._save()
        self.count += 1

    def _save(self):
        tt = datetime.now()
        time_string = tt.strftime('%mm-%dd-%Hh-%Mm-%Ss')

        W = self.model.conv1.W.get_value()
        W = W.transpose(1,2,0,3)

        row_list = []
        img_list = []
        k = 0

        rows = W.shape[3]/16
        for i in range(rows):
            for j in range(16):
                W0 = W[:,:,:,k]
                W0[:,:,0] -= W0[:,:,0].min()
                W0[:,:,0] /= W0[:,:,0].max()
                W0[:,:,1] -= W0[:,:,1].min()
                W0[:,:,1] /= W0[:,:,1].max()
                W0[:,:,2] -= W0[:,:,2].min()
                W0[:,:,2] /= W0[:,:,2].max()
                row_list.append(W0)
                k += 1
            row_image = numpy.hstack(row_list)
            row_list = []
            img_list.append(row_image)
        img_image = numpy.vstack(img_list)

        to_save = Image.fromarray(numpy.uint8(255*img_image))
        filename = 'filters_'+time_string+'.png'
        filepath = os.path.join(self.save_path, filename)
        to_save.save(filepath)

class Monitor(object):
    errors = []
    times = []
    big_errors = []
    big_times = []

    def __init__(self, model, step_number=0, best=1, short_steps=10, long_steps=50, save_steps=2000, test_steps=50):
        self.step_number = step_number
        self.best = best
        self.short_steps = short_steps
        self.long_steps = long_steps
        self.save_steps = save_steps
        self.model = model
        self.test = False
        self.test_steps = test_steps

        # Check if model.path exists, if not create it (with a checkpoint folder)
        if model.path and not os.path.exists(os.path.join(model.path, 'checkpoints')):
            os.makedirs(os.path.join(model.path, 'checkpoints'))

    def start(self):
        self.tic = time()

    def stop_test(self, error):
        if self.test:
            self.toc = time()
            _time = self.toc - self.tic
            print '&%d, test error: %.5f, time: %.2f' % (self.step_number, error, _time)

    def stop(self, error):
        self.toc = time()
        _time = self.toc-self.tic
        self.errors.append(error)
        self.times.append(_time)
        self.big_errors.append(error)
        self.big_times.append(_time)
        if self.step_number % self.test_steps == 0:
            self.test = True
        else:
            self.test = False
        if self.step_number % self.save_steps == 0:
            save_checkpoint(self.model)
        if self.step_number % self.long_steps == 0:
            mean_error = numpy.mean(self.big_errors)
            mean_time = numpy.mean(self.big_times)
            print '*%d, train error: %.5f, time: %.2f' % (self.step_number, mean_error, mean_time)
            self.big_errors = []
            self.big_times = []
        if self.step_number % self.short_steps == 0:
            mean_error = numpy.mean(self.errors)
            mean_time = numpy.mean(self.times)
            print '%d, train error: %.5f, time: %.2f' % (self.step_number, mean_error, mean_time)
            self.errors = []
            self.times = []
        self.step_number += 1

class Evaluator(object):
    def __init__(self, model, dataset, steps=100):
        self.step_number = 1
        self.steps = steps
        self.model = model
        self.dataset = dataset

    def run(self):
        input_convspace = Conv2DSpace(shape=(32, 32), num_channels=3,
                                      axes=('c', 0, 1, 'b'))
        data_specs = (input_convspace,'features')
        iterator = self.dataset.iterator(mode='sequential', batch_size=128, data_specs=data_specs)
        errors = []
        if self.step_number % self.steps == 0:
            tic = time()
            for batch in iterator:
                error = self.model.eval(batch)
                errors.append(error)
            toc = time()
            _time = toc-tic
            mean_error = numpy.mean(errors)
            print '%d, test error: %.3f, time: %.2f' % (self.step_number, mean_error, _time)
        self.step_number += 1

def get_cifar_iterator(which_set, mode='sequential', batch_size=128, num_batches=None, center=False, rescale=True, axes=['c', 0, 1, 'b'], targets=False):
    dataset = cifar10.CIFAR10(which_set=which_set,
                              center=center,
                              rescale=rescale,
                              axes=axes
                              )
    input_convspace = Conv2DSpace(shape=(32, 32), num_channels=3,
                              axes=axes)
    #target_space = VectorSpace(dim=10)
    #data_specs = (input_convspace,'features')

    target_space = VectorSpace(dim=10)
    data_specs = (CompositeSpace((input_convspace, target_space)), ("features", "targets"))

    if num_batches:
        iterator = dataset.iterator(mode=mode, batch_size=batch_size, num_batches=num_batches, data_specs=data_specs)
    else:
        iterator = dataset.iterator(mode=mode, batch_size=batch_size, data_specs=data_specs)
    return iterator

class Normer(object):
    def __init__(self, filter_size=7):

        # magic numbers that make things work for stl10
        self.filter_size = 7
        self.pad = self.filter_size/2 -1
        self.num_channels = 3
        self.num_filters = 16
        input = T.ftensor4(name='input')
        filter = T.ftensor4(name='filter')
        gpu_input = gpu_contiguous(input)
        gpu_filter = gpu_contiguous(filter)

        self.conv_func = theano.function([input, filter], FilterActs(pad=3)(gpu_input, gpu_filter))
        n = self.num_channels * self.filter_size * self.filter_size
        self.w = numpy.float32(numpy.ones((self.num_channels, self.filter_size, self.filter_size, self.num_filters)))/n

    def run(self, x_batch):
        mean_batch = self.conv_func(x_batch, self.w)
        mean_batch = numpy.tile(numpy.array(mean_batch[0,:,:,:])[None, :, :], (3, 1, 1, 1))
        diff_batch = x_batch - mean_batch
        std_batch = self.conv_func(diff_batch**2, self.w)
        std_batch = numpy.tile(numpy.array(std_batch[0,:,:,:])[None, :, :], (3, 1, 1, 1))
        norm_batch = diff_batch/(numpy.array(std_batch)**(1/2))
        return norm_batch

class PatchGrabber(object):
    def __init__(self, num_patches, patch_size):
        self.num_patches = num_patches
        self.patch_size = patch_size

    def run(self, x_batch):
        image_size = x_batch.shape[1]
        batch_size = x_batch.shape[-1]

        patches = numpy.zeros((3, self.patch_size, self.patch_size, self.num_patches),dtype=numpy.float32)

        for i_patch in range(self.num_patches):
            x_start = numpy.random.randint(image_size-self.patch_size)
            y_start = numpy.random.randint(image_size-self.patch_size)
            image_id = numpy.random.randint(batch_size)
            x_slice = slice(x_start, x_start+self.patch_size)
            y_slice = slice(y_start, y_start+self.patch_size)
            patch = x_batch[:, x_slice, y_slice, image_id]
            patches[:, :, :, i_patch] = patch

        return patches<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Classes used by the tournament program to run computer continuous Pairs

class Dealer:
    """This asks Strategy classes to play the game and tracks the game state.

    >>> d = Dealer()

    >>> d.gameState.deck
    [1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]

    """

    def __init__(self, noPlayers = 5):

        self.gameState = Information()
        self.gameState.noPlayers = noPlayers
        self.verbose = False
        for n in range(self.gameState.noPlayers):
            self.gameState.players.append(Player())

        for i in range(1, 11):
            self.gameState.deck += [i] * i

    def deal(self):
        for player in self.gameState.players:
            newCard = self.gameState.draw()
            player.stack = [newCard]
            player.strategy.player = player

        cardList = [sum(player.stack) for player in self.gameState.players]
        minPlayers = [player for player in self.gameState.players
                       if sum(player.stack) == min(cardList)]

        while len(minPlayers) != 1:
            for player in list(minPlayers):
                if sum(player.stack) != min(cardList):
                    minPlayers.remove(player)
                else:
                    player.hit(self.gameState.draw())
                    while player.whichPair() != False:
                        self.gameState.discards.append(player.stack.pop(-1))
                        player.hit(self.gameState.draw())
            cardList = [sum(player.stack) for player in minPlayers]
            minPlayers = [player for player in minPlayers
                           if sum(player.stack) == min(cardList)]

        startPlayer = minPlayers[0]
        self.gameState.startIndex = self.gameState.players.index(startPlayer)

        i = 0
        for player in self.gameState.players:
            i += 1
            self.vPrint('Player ' + str(i) + '\'s stack: ' + str(player.stack))

    def play(self):
        from copy import deepcopy
        self.deal() # should be called by Tournament?
        highestScore = max(int(60 / self.gameState.noPlayers) + 1, 11)

        allScores = [player.getScore() for player in self.gameState.players]
        currentIndex = self.gameState.startIndex

        while max(allScores) < highestScore:
            currentPlayer = self.gameState.players[currentIndex]
            info = deepcopy(self.gameState)
            inStacks = self.gameState.inStacks()
            if inStacks == [] or \
             len(currentPlayer.stack) == 0 or \
             (min(inStacks) + currentPlayer.getScore()) >= highestScore:
                reply = 'hit.'
                self.vPrint('Player '+str(currentIndex+1)+' was forced to hit.')
            else:
                reply = currentPlayer.strategy.play(info)
                self.vPrint('Player '+str(currentIndex+1)+' decided to '+str(reply))

            if reply == 'fold':
                foldFrom = self.gameState.bestFold()
                self.gameState.players[foldFrom[0]].steal(foldFrom[1])
                currentPlayer.catch(foldFrom[1])
                self.vPrint('You just folded for ' + str(foldFrom[1]) +
                    ' Your stack: ' + str(currentPlayer.stack) +
                    ' Your score: ' + str(currentPlayer.getScore()))
            else:
                try:
                    self.gameState.players[reply[0]].steal(reply[1])
                    currentPlayer.catch(reply[1])
                    self.vPrint('You just folded for ' + str(reply[1]) +
                           ' Your stack: ' +
                           str(currentPlayer.stack) +
                            ' Your score: ' +
                            str(currentPlayer.getScore()))
                except TypeError:
                    hitCard = self.gameState.draw()
                    currentPlayer.hit(hitCard)
                    whichPair = currentPlayer.whichPair()
                    if whichPair:
                        currentPlayer.catch(hitCard)
                        self.vPrint('You just paired for ' + str(hitCard) +
                               ' Your stack: ' +
                               str(currentPlayer.stack) +
                                ' Your score: ' +
                                str(currentPlayer.getScore()))
                    else:
                        self.vPrint('You just hit for ' + str(hitCard) +
                               ' Your stack: ' +
                               str(currentPlayer.stack) +
                                ' Your score: ' +
                                str(currentPlayer.getScore()))

            allScores = [player.getScore() for player in self.gameState.players]
            currentIndex = (currentIndex + 1) % self.gameState.noPlayers

        return ((currentIndex - 1) % self.gameState.noPlayers)+1

    def vPrint(self, *args):
        if self.verbose:
            print(*args)

class Information:
    """This the the game state information provided to Strategy classes to make
    decisions with.
    """
    def __init__(self):
        self.deck = []
        self.discards = []
        self.players = []
        self.burn = 5
        self.startIndex = 0 # Dealer.deal will set this properly

    def bestFold(self):
        smallest = min(self.inStacks())
        best = []
        for i in range(len(self.players)):
            if smallest in self.Players[i].inStack():
                best += (i, smallest)
        return best

    def draw(self):
        from random import choice
        if len(self.deck) <= self.burn: # time to shuffle
            self.deck = []
            for i in range(1, 11):
                self.deck += [i] * i
            for i in self.inPoints() + self.inStacks():
                self.deck.remove(i)
            self.discards = []
        card = choice(self.deck)
        self.deck.remove(card)
        return card

    def inPoints(self):
        self.allPoints = []
        for player in self.players:
            self.allPoints += player.points
        return self.allPoints

    def inStacks(self):
        self.allStacks= []
        for player in self.players:
            self.allStacks += player.stack
        return self.allStacks

class Player:
    """This holds information about the cards that a player has.
    """
    def __init__(self):
        self.stack = []
        self.points = []
        self.strategy = SimpletonStrategy()

    def catch(self, card):
        self.stack = []
        self.points.append(card)

    def hit(self, card):
        self.stack.append(card)

    def index(self, newIndex=None):
        if newIndex == None:
            return self._index
        self._index = newIndex

    def getScore(self):
        return sum(self.points)

    def getSmallest(self):
        return min(self.stack)

    def steal(self, card):
        self.stack.remove(card)

    def whichPair(self):
        if self.stack == []:
            return False
        from collections import Counter
        mostCommon = Counter(self.stack).most_common(1)[0]
        if mostCommon[1] > 1:
            return mostCommon[0]
        else:
            return False

class SimpletonStrategy:
    """This is an example strategy"""
    def __init__(self):
        self.shouldIHit = True
    def play(self, info):
        if self.shouldIHit:
            return 'Hit me; I can\'t lose!'
        else:
            return 'fold'

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>Fix bug with bestFold and rename to bestFolds<commit_after># -*- coding: utf-8 -*-
# Classes used by the tournament program to run computer continuous Pairs

class Dealer:
    """This asks Strategy classes to play the game and tracks the game state.

    >>> d = Dealer()

    >>> d.gameState.deck
    [1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]

    """

    def __init__(self, noPlayers = 5):

        self.gameState = Information()
        self.gameState.noPlayers = noPlayers
        self.verbose = False
        for n in range(self.gameState.noPlayers):
            self.gameState.players.append(Player())

        for i in range(1, 11):
            self.gameState.deck += [i] * i

    def deal(self):
        for player in self.gameState.players:
            newCard = self.gameState.draw()
            player.stack = [newCard]
            player.strategy.player = player

        cardList = [sum(player.stack) for player in self.gameState.players]
        minPlayers = [player for player in self.gameState.players
                       if sum(player.stack) == min(cardList)]

        while len(minPlayers) != 1:
            for player in list(minPlayers):
                if sum(player.stack) != min(cardList):
                    minPlayers.remove(player)
                else:
                    player.hit(self.gameState.draw())
                    while player.whichPair() != False:
                        self.gameState.discards.append(player.stack.pop(-1))
                        player.hit(self.gameState.draw())
            cardList = [sum(player.stack) for player in minPlayers]
            minPlayers = [player for player in minPlayers
                           if sum(player.stack) == min(cardList)]

        startPlayer = minPlayers[0]
        self.gameState.startIndex = self.gameState.players.index(startPlayer)

        i = 0
        for player in self.gameState.players:
            i += 1
            self.vPrint('Player ' + str(i) + '\'s stack: ' + str(player.stack))

    def play(self):
        from copy import deepcopy
        self.deal() # should be called by Tournament?
        highestScore = max(int(60 / self.gameState.noPlayers) + 1, 11)

        allScores = [player.getScore() for player in self.gameState.players]
        currentIndex = self.gameState.startIndex

        while max(allScores) < highestScore:
            currentPlayer = self.gameState.players[currentIndex]
            info = deepcopy(self.gameState)
            inStacks = self.gameState.inStacks()
            if inStacks == [] or \
             len(currentPlayer.stack) == 0 or \
             (min(inStacks) + currentPlayer.getScore()) >= highestScore:
                reply = 'hit.'
                self.vPrint('Player '+str(currentIndex+1)+' was forced to hit.')
            else:
                reply = currentPlayer.strategy.play(info)
                self.vPrint('Player '+str(currentIndex+1)+' decided to '+str(reply))

            if reply == 'fold':
                foldFrom = self.gameState.bestFold()
                self.gameState.players[foldFrom[0]].steal(foldFrom[1])
                currentPlayer.catch(foldFrom[1])
                self.vPrint('You just folded for ' + str(foldFrom[1]) +
                    ' Your stack: ' + str(currentPlayer.stack) +
                    ' Your score: ' + str(currentPlayer.getScore()))
            else:
                try:
                    self.gameState.players[reply[0]].steal(reply[1])
                    currentPlayer.catch(reply[1])
                    self.vPrint('You just folded for ' + str(reply[1]) +
                           ' Your stack: ' +
                           str(currentPlayer.stack) +
                            ' Your score: ' +
                            str(currentPlayer.getScore()))
                except TypeError:
                    hitCard = self.gameState.draw()
                    currentPlayer.hit(hitCard)
                    whichPair = currentPlayer.whichPair()
                    if whichPair:
                        currentPlayer.catch(hitCard)
                        self.vPrint('You just paired for ' + str(hitCard) +
                               ' Your stack: ' +
                               str(currentPlayer.stack) +
                                ' Your score: ' +
                                str(currentPlayer.getScore()))
                    else:
                        self.vPrint('You just hit for ' + str(hitCard) +
                               ' Your stack: ' +
                               str(currentPlayer.stack) +
                                ' Your score: ' +
                                str(currentPlayer.getScore()))

            allScores = [player.getScore() for player in self.gameState.players]
            currentIndex = (currentIndex + 1) % self.gameState.noPlayers

        return ((currentIndex - 1) % self.gameState.noPlayers)+1

    def vPrint(self, *args):
        if self.verbose:
            print(*args)

class Information:
    """This the the game state information provided to Strategy classes to make
    decisions with.
    """
    def __init__(self):
        self.deck = []
        self.discards = []
        self.players = []
        self.burn = 5
        self.startIndex = 0 # Dealer.deal will set this properly

    def bestFolds(self):
        smallest = min(self.inStacks())
        best = []
        for i in range(len(self.players)):
            if smallest in self.players[i].stack:
                best += [(i, smallest)]
        return best

    def bestFold(self):
        return self.bestFolds()[0]

    def draw(self):
        from random import choice
        if len(self.deck) <= self.burn: # time to shuffle
            self.deck = []
            for i in range(1, 11):
                self.deck += [i] * i
            for i in self.inPoints() + self.inStacks():
                self.deck.remove(i)
            self.discards = []
        card = choice(self.deck)
        self.deck.remove(card)
        return card

    def inPoints(self):
        self.allPoints = []
        for player in self.players:
            self.allPoints += player.points
        return self.allPoints

    def inStacks(self):
        self.allStacks= []
        for player in self.players:
            self.allStacks += player.stack
        return self.allStacks

class Player:
    """This holds information about the cards that a player has.
    """
    def __init__(self):
        self.stack = []
        self.points = []
        self.strategy = SimpletonStrategy()

    def catch(self, card):
        self.stack = []
        self.points.append(card)

    def hit(self, card):
        self.stack.append(card)

    def index(self, newIndex=None):
        if newIndex == None:
            return self._index
        self._index = newIndex

    def getScore(self):
        return sum(self.points)

    def getSmallest(self):
        return min(self.stack)

    def steal(self, card):
        self.stack.remove(card)

    def whichPair(self):
        if self.stack == []:
            return False
        from collections import Counter
        mostCommon = Counter(self.stack).most_common(1)[0]
        if mostCommon[1] > 1:
            return mostCommon[0]
        else:
            return False

class SimpletonStrategy:
    """This is an example strategy"""
    def __init__(self):
        self.shouldIHit = True
    def play(self, info):
        if self.shouldIHit:
            return 'Hit me; I can\'t lose!'
        else:
            return 'fold'

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>"""Code for the B{0desktop} command."""

# Copyright (C) 2009, Thomas Leonard
# See the README file for details, or visit http://0install.net.

from __future__ import print_function

from zeroinstall import _, logger
import sys
from optparse import OptionParser
import logging

def main(command_args):
	"""Implements the logic of the 0desktop command.
	@param command_args: the command-line arguments"""
	parser = OptionParser(usage=_("usage: %prog [options] [URI]"))
	parser.add_option("-m", "--manage", help=_("manage added applications"), action='store_true')
	parser.add_option("-v", "--verbose", help=_("more verbose output"), action='count')
	parser.add_option("-V", "--version", help=_("display version information"), action='store_true')

	(options, args) = parser.parse_args(command_args)

	if options.verbose:
		if options.verbose == 1:
			logger.setLevel(logging.INFO)
		else:
			logger.setLevel(logging.DEBUG)
		hdlr = logging.StreamHandler()
		fmt = logging.Formatter("%(levelname)s:%(message)s")
		hdlr.setFormatter(fmt)
		logger.addHandler(hdlr)

	if options.version:
		import zeroinstall
		print("0desktop (zero-install) " + zeroinstall.version)
		print("Copyright (C) 2013 Thomas Leonard")
		print(_("This program comes with ABSOLUTELY NO WARRANTY,"
				"\nto the extent permitted by law."
				"\nYou may redistribute copies of this program"
				"\nunder the terms of the GNU Lesser General Public License."
				"\nFor more information about these matters, see the file named COPYING."))
		sys.exit(0)

	if not args:
		interface_uri = None
	elif len(args) == 1:
		interface_uri = args[0]
	else:
		parser.print_help()
		sys.exit(1)

	if sys.version_info[0] < 3:
		import pygtk
		pygtk.require('2.0')
	else:
		from zeroinstall.gtkui import pygtkcompat
		pygtkcompat.enable()
		pygtkcompat.enable_gtk(version = '3.0')
	import gtk

	if options.manage:
		from zeroinstall.gtkui.applistbox import AppListBox, AppList
		from zeroinstall.injector.iface_cache import iface_cache
		box = AppListBox(iface_cache, AppList())
	else:
		from zeroinstall.gtkui.addbox import AddBox
		box = AddBox(interface_uri)

	box.window.connect('destroy', gtk.main_quit)
	box.window.show()
	gtk.main()
<commit_msg>Removed unused code<commit_after><|endoftext|>


<commit_before>"""
Utils package.
"""

from _compat import *

import logging as _logging
import functools as _functools
from inspect import ismethod

from util.collections import is_container
from util.collections import is_mapping
from util.operator import invoker
import pprint

def identity(x):
    return x

_logging.DUMP = _logging.DEBUG // 2
_logging.addLevelName(_logging.DUMP, 'DUMP')

logging_defaults = dict(
    level=_logging.DEBUG,
    filemode='w',
    format='%(levelname)-8s%(name)s:\t%(message)s',
)

def init_logging(filename_or_stream, **kwargs):
    init_dict = dict(logging_defaults, **kwargs)

    is_string = isinstance(filename_or_stream, str)
    init_dict['filename' if is_string else 'stream'] = filename_or_stream

    _logging.basicConfig(**init_dict)

def get_extended_logger(name):
	return extend_logger(_logging.getLogger(name))

def extend_logger(logger):
	logger.dump = _functools.partial(logger_dump, logger)
	logger.wrap = _functools.partial(logger_wrap, logger)
	return logger

def logger_wrap(logger, func, width=60):
    msg = ' {func.__name__} '.format(**locals())
    header = '---{msg:-<{width}}'.format(**locals())
    footer = '{msg:->{width}}---'.format(**locals())

    @_functools.wraps(func)
    def decorated(*args, **kwargs):
        logger.debug(header)
        try:
            return func(*args, **kwargs)
        finally:
            logger.debug(footer)

    return decorated

def logger_dump(logger, target, attrs=None):
    if not logger.isEnabledFor(_logging.DEBUG):
    	return

    if isinstance(attrs, str):
        attrs = attrs.split()
    if attrs is None:
        try:
            attrs = target._dump_attrs
        except AttributeError:
            attrs = filternot(invoker.startswith('_'), dir(target))

    logger.log(_logging.DUMP, '%r', target)
    for attr in attrs:
        try:
            obj = getattr(target, attr)
        except AttributeError as e:
            obj = e
        else:
            if ismethod(obj):
                continue
            obj = _log_dump_normalize(obj)

        try:
            obj_len = len(obj)
        except TypeError:
            msg = '.{0}:'.format(attr)
        else:
            msg = '.{0}: (len={1})'.format(attr, obj_len)

        logger.log(_logging.DUMP, '\t||%s', msg)
        for line in pprint.pformat(obj).splitlines():
            logger.log(_logging.DUMP, '\t||\t\t%s', line)

def _log_dump_normalize(obj):
    if is_mapping(obj):
        obj = dict((k, _log_dump_normalize(v)) for k, v in iteritems(obj))
    elif is_container(obj):
        obj = sorted(obj, key=repr)
    return obj

<commit_msg>logging: Fix filemode argument necessary for python 3.2<commit_after>"""
Utils package.
"""

from _compat import *

import logging as _logging
import functools as _functools
from inspect import ismethod

from util.collections import is_container
from util.collections import is_mapping
from util.operator import invoker
import pprint

def identity(x):
    return x

_logging.DUMP = _logging.DEBUG // 2
_logging.addLevelName(_logging.DUMP, 'DUMP')

logging_defaults = dict(
    level=_logging.DEBUG,
    format='%(levelname)-8s%(name)s:\t%(message)s',
)

def init_logging(filename_or_stream, **kwargs):
    init_dict = dict(logging_defaults, **kwargs)

    if isinstance(filename_or_stream, str):
        init_dict['filename'] = filename_or_stream
        init_dict['filemode'] = 'w'
    else:
        init_dict['stream'] = filename_or_stream

    _logging.basicConfig(**init_dict)

def get_extended_logger(name):
	return extend_logger(_logging.getLogger(name))

def extend_logger(logger):
	logger.dump = _functools.partial(logger_dump, logger)
	logger.wrap = _functools.partial(logger_wrap, logger)
	return logger

def logger_wrap(logger, func, width=60):
    msg = ' {func.__name__} '.format(**locals())
    header = '---{msg:-<{width}}'.format(**locals())
    footer = '{msg:->{width}}---'.format(**locals())

    @_functools.wraps(func)
    def decorated(*args, **kwargs):
        logger.debug(header)
        try:
            return func(*args, **kwargs)
        finally:
            logger.debug(footer)

    return decorated

def logger_dump(logger, target, attrs=None):
    if not logger.isEnabledFor(_logging.DEBUG):
    	return

    if isinstance(attrs, str):
        attrs = attrs.split()
    if attrs is None:
        try:
            attrs = target._dump_attrs
        except AttributeError:
            attrs = filternot(invoker.startswith('_'), dir(target))

    logger.log(_logging.DUMP, '%r', target)
    for attr in attrs:
        try:
            obj = getattr(target, attr)
        except AttributeError as e:
            obj = e
        else:
            if ismethod(obj):
                continue
            obj = _log_dump_normalize(obj)

        try:
            obj_len = len(obj)
        except TypeError:
            msg = '.{0}:'.format(attr)
        else:
            msg = '.{0}: (len={1})'.format(attr, obj_len)

        logger.log(_logging.DUMP, '\t||%s', msg)
        for line in pprint.pformat(obj).splitlines():
            logger.log(_logging.DUMP, '\t||\t\t%s', line)

def _log_dump_normalize(obj):
    if is_mapping(obj):
        obj = dict((k, _log_dump_normalize(v)) for k, v in iteritems(obj))
    elif is_container(obj):
        obj = sorted(obj, key=repr)
    return obj

<|endoftext|>


<commit_before>"""Build command for etest Docker images."""
import os
import sys
from pathlib import Path
from typing import List

import click

@click.command(name="etest-build")
@click.option("--hardened/--no-hardened", default=False, help="Use a hardened profile.")
@click.option("--multilib/--no-multilib", default=True, help="Use a multilib profile.")
@click.option("--systemd/--no-systemd", default=False, help="Use a systemd profile.")
# The following could and should be excised to an architecture module that
# handles the nuances of architecture differences in Gentoo and Docker.
@click.option(
    "--architecture",
    "--arch",
    type=click.Choice(["amd64", "x86", "arm64", "armv5", "armv7", "ppc64"], case_sensitive=False),
    default="amd64",
    help="Architecture for the built image.",
)
@click.option("--environment", "--env", multiple=True, help="Pass an environment variable through to Docker.")
@click.option("--name", type=Path, default=os.path.basename(sys.argv[0]), help="Name of the directory?")
def main(hardened: bool, multilib: bool, systemd: bool, architecture: str, environment: List[str], name: Path) -> None:
    """Build the etest image."""
    pass
<commit_msg>Ensure vulture is happy with etest-build.<commit_after>"""Build command for etest Docker images."""
import os
import sys
from pathlib import Path
from typing import List

import click

@click.command(name="etest-build")
@click.option("--hardened/--no-hardened", default=False, help="Use a hardened profile.")
@click.option("--multilib/--no-multilib", default=True, help="Use a multilib profile.")
@click.option("--systemd/--no-systemd", default=False, help="Use a systemd profile.")
# The following could and should be excised to an architecture module that
# handles the nuances of architecture differences in Gentoo and Docker.
@click.option(
    "--architecture",
    "--arch",
    type=click.Choice(["amd64", "x86", "arm64", "armv5", "armv7", "ppc64"], case_sensitive=False),
    default="amd64",
    help="Architecture for the built image.",
)
@click.option("--environment", "--env", multiple=True, help="Pass an environment variable through to Docker.")
@click.option("--name", type=Path, default=os.path.basename(sys.argv[0]), help="Name of the directory?")
def main(hardened: bool, multilib: bool, systemd: bool, architecture: str, environment: List[str], name: Path) -> None:
    """Build the etest image."""
    click.echo(f"hardened: {hardened}")
    click.echo(f"multilib: {multilib}")
    click.echo(f"systemd: {systemd}")
    click.echo(f"architecture: {architecture}")
    click.echo(f"environment: {environment}")
    click.echo(f"name: {name}")
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import attr
import fnmatch
import json
import pathlib
import requests
import yaml

from http import HTTPStatus
from os import environ
from rich import print as rprint
from rich.console import Console
from rich.pretty import pprint

console = Console()

@attr.s
class Droplet:
    droplet_id = attr.ib()
    minion_id = attr.ib()
    memory = attr.ib()
    disk = attr.ib()
    price_monthly = attr.ib()
    tags = attr.ib()
    domains = attr.ib()

def display_domains(domains):
    count = 0
    for domain_name, config in domains.items():
        count = count + 1
        ssl = False
        print()
        if "ssl" in config:
            ssl = True
        console.print(
            "{}. {}{}".format(
                count, "https://" if ssl else "http://", domain_name
            )
        )
        console.print("- {}".format(config["minion"]))
        if "testing" in config:
            console.print("- Test Site")
        if "profile" in config:
            profile = config["profile"]
            if profile == "php":
                profile = "{} (PHP)".format(config["php_profile"].capitalize())
            else:
                profile = "{}".format(profile.capitalize())
            console.print("- {}".format(profile))
        if "backup" in config:
            console.print("- backup")
        is_promtail = is_raygun = False
        if "env" in config:
            env = config["env"]
            if "norecaptcha_site_key" in env:
                console.print("- captcha (Google)")
            if "raygun4py_api_key" in env:
                is_raygun = True
            if "sparkpost_api_key" in env:
                console.print("- email (SparkPost)")
        if "promtail" in config:
            is_promtail = True
        if is_promtail or is_raygun:
            x = "{}".format("Promtail" if is_promtail else "RayGun")
            console.print("- monitor ({})".format(x))
        if ssl:
            certificate = ""
            if "letsencrypt" in config:
                certificate = "LetsEncrypt"
            console.print(
                "- ssl{}".format(
                    " ({})".format(certificate) if certificate else ""
                )
            )

def get_digital_ocean():
    """

    How To Use Web APIs in Python 3
    https://www.digitalocean.com/community/tutorials/how-to-use-web-apis-in-python-#!/usr/bin/env python3

    """
    result = []
    api_token = environ["DIGITAL_OCEAN_TOKEN"]
    api_url_base = "https://api.digitalocean.com/v2/"
    headers = {
        "Content-Type": "application/json",
        "Authorization": "Bearer {0}".format(api_token),
    }
    api_url = "{}droplets".format(api_url_base)
    response = requests.get(api_url, headers=headers)
    if response.status_code == HTTPStatus.OK:
        data = json.loads(response.content.decode("utf-8"))
        droplets = data["droplets"]
        for droplet in droplets:
            minion_id = droplet["name"]
            size = droplet["size"]
            tags = droplet["tags"]
            result.append(
                Droplet(
                    droplet_id=droplet["id"],
                    minion_id=minion_id,
                    memory=droplet["memory"],
                    disk=droplet["disk"],
                    price_monthly=size["price_monthly"],
                    tags=[x for x in tags],
                    domains=[],
                )
            )
    else:
        pprint(response, expand_all=True)
        raise Exception(
            "Error from the Digital Ocean API: {}".format(response.status_code)
        )
    return result
    # return dict(sorted(result.items()))

def get_linode():
    """

    Create a Linode Using the Linode API
    https://www.linode.com/docs/guides/getting-started-with-the-linode-api/

    """
    result = []
    api_token = environ["LINODE_TOKEN"]
    api_url_base = "https://api.linode.com/v4/linode/"
    headers = {
        "Content-Type": "application/json",
        "Authorization": "Bearer {0}".format(api_token),
    }
    api_url = "{}instances".format(api_url_base)
    response = requests.get(api_url, headers=headers)
    if response.status_code == HTTPStatus.OK:
        data = json.loads(response.content.decode("utf-8"))
        # pprint(data, expand_all=True)
        droplets = data["data"]
        for droplet in droplets:
            minion_id = droplet["label"]
            # size = droplet["size"]
            # tags = droplet["tags"]
            specs = droplet["specs"]
            result.append(
                Droplet(
                    droplet_id=droplet["id"],
                    minion_id=minion_id,
                    memory=specs["memory"],
                    disk=specs["disk"],
                    price_monthly=None,  # size["price_monthly"],
                    tags=[],
                    domains=[],
                )
            )
    else:
        pprint(response, expand_all=True)
        raise Exception(
            "Error from the Linode API: {}".format(response.status_code)
        )
    return result

def get_domains():
    """Get the config for each site (domain name) from the Salt pillar."""
    result = {}
    pillar_folder = pathlib.Path.home().joinpath("Private", "deploy")
    for folder in pillar_folder.iterdir():
        if folder.is_dir() and folder.name.startswith("pillar-"):
            wildcard = get_wildcard(folder)
            result.update(get_domain_names(folder, wildcard))
    return result

def get_wildcard(pillar_folder):
    """Get the config for the wildcard include files.

    The ``top.sls`` file for our pillar, includes wildcards e.g::

      '*':
        - global.users
      'kb-* and not kb-vpn':
        - config.django
        - config.monitor
        - config.nginx

    This method retrieves the *wildcard* config, so it can be merged into the
    site later on (using ``match_minion``).

    """
    result = {}
    with open(pathlib.Path(pillar_folder, "top.sls")) as f:
        # load the 'top.sls' file
        data = yaml.safe_load(f)
        base = data["base"]
        for host_name, config in base.items():
            if "*" in host_name:
                result[host_name] = {}
                for include in config:
                    if isinstance(include, str):
                        # not sure if / why we need to do this...
                        if include.startswith("sites"):
                            pass
                        else:
                            # convert include to file and path
                            # e.g. 'config.monitor' to 'config/monitor.sls'
                            path_file = include.split(".")
                            path_file[-1] = path_file[-1] + ".sls"
                            with open(
                                pathlib.Path(pillar_folder, *path_file)
                            ) as f:
                                include_config = yaml.safe_load(f)
                                result[host_name].update(include_config)
    return result

def get_domain_names(pillar_folder, wildcard):
    """Find the server configuration for each site / domain name.

    .. warning:: We add the configuration for the site / domain name after
                 merging the configuration for the server
                 (including *wildcard* data).
                 Salt probably resolves this in a different order!

    """
    result = {}
    with open(pathlib.Path(pillar_folder, "top.sls")) as f:
        # load the 'top.sls' file
        data = yaml.safe_load(f)
        base = data["base"]
        for host_name, base_config in base.items():
            if "*" in host_name:
                # exclude wildcard e.g. 'pc-*' (see 'get_wildcard')
                pass
            else:
                server_config = {}
                for include in base_config:
                    if isinstance(include, str):
                        # convert include to file and path
                        # e.g. 'sites.cw-3' to 'sites/cw-3.sls'
                        path_file = include.split(".")
                        path_file[-1] = path_file[-1] + ".sls"
                        with open(pathlib.Path(pillar_folder, *path_file)) as f:
                            server_config.update(yaml.safe_load(f))
                if "sites" in server_config:
                    sites = server_config.pop("sites")
                    for domain_name, domain_config in sites.items():
                        # config for domain name (site)
                        result[domain_name] = {}
                        # merge in the *wildcard* config
                        result[domain_name].update(
                            merge_wildcard(host_name, wildcard)
                        )
                        # merge in the config for this server
                        result[domain_name].update(server_config)
                        # add the server (host) name
                        result[domain_name].update({"minion": host_name})
                        # finally - merge the domain config
                        result[domain_name].update(domain_config)
    return result

def json_dump_domains(data):
    file_name = "domains.json"
    with open(file_name, "w") as f:
        json.dump(data, f, indent=4)
    rprint("[yellow]'json_dump_domains' to '{}'...".format(file_name))

def json_dump_droplets(droplets):
    data = []
    file_name = "droplets.json"
    for droplet in droplets:
        data.append(attr.asdict(droplet))
    with open(file_name, "w") as f:
        json.dump(data, f, indent=4)
    rprint("[yellow]'json_dump_droplets' to '{}'...".format(file_name))

def match_minion(minion_id, salt_top):
    """Copied from ``_match`` (see ``lib/pillarinfo.py`` in ``fabric``."""
    result = False
    if minion_id is None:
        result = True
    else:
        # 20/04/2021, Try and handle ``and not``
        # (when I don't really understand parsing)
        # e.g. 'drop-* and not drop-vpn'
        tokens = salt_top.split(" ")
        index_and = index_not = minion_not = None
        try:
            index_and = tokens.index("and")
            index_not = tokens.index("not")
        except ValueError:
            pass
        # if minion_id == "kb-a" and index_and and index_not:
        # import pdb; pdb.set_trace()
        if (
            index_and
            and index_not
            and index_and > 0
            and (index_and + 1 == index_not)
            and (index_not + 2 == len(tokens))
        ):
            minion_not = tokens[index_not + 1].strip()
            pos = salt_top.find(" and ")
            salt_top = salt_top[:pos].strip()
        # exclude the ``and not`` value e.g. ``drop-vpn``
        if minion_not and minion_not == minion_id:
            pass
        else:
            result = fnmatch.fnmatch(minion_id, salt_top)
            if not result:
                for item in salt_top.split(","):
                    result = fnmatch.fnmatch(minion_id, item)
                    if result:
                        break
    return result

def merge_wildcard(host_name, wildcard):
    """Merge configuration from files matching the wildcard.

    e.g. ``nc-*`` matches ``nc-a``, so ``nc-a`` will include the
    configuration data from ``nc-*``.

    """
    result = {}
    for wildcard_host, config in wildcard.items():
        if match_minion(host_name, wildcard_host):
            result.update(config)
    return result

def main():
    domains = dict(sorted(get_domains().items()))
    json_dump_domains(domains)
    # display_domains(domains)

    # find the sites (domain names) for each minion (server)
    minions = {}
    for domain_name, config in domains.items():
        minion_id = config["minion"]
        if not minion_id in minions:
            minions[minion_id] = []
        minions[minion_id].append(domain_name)
    # pprint(minions, expand_all=True)

    droplets = []
    droplets = droplets + get_digital_ocean()
    droplets = droplets + get_linode()

    # add the sites (domain names) to the droplet
    for droplet in droplets:
        minion_id = droplet.minion_id
        if minion_id in minions:
            droplet.domains = minions.pop(minion_id)
    json_dump_droplets(droplets)

    # use the tag to link droplets to a contact
    contacts = {}
    for droplet in droplets:
        for tag in droplet.tags:
            if not tag in contacts:
                contacts[tag] = []
            contacts[tag].append(droplet)
    pprint(contacts, expand_all=True)

    # display any minions which are not allocated to a droplet
    # note: this is most likely because the customer is paying for the hosting!
    pprint(minions, expand_all=True)

if __name__ == "__main__":
    main()
<commit_msg>wip - calculate hosting costs #5774<commit_after># -*- encoding: utf-8 -*-
import attr
import fnmatch
import json
import pathlib
import requests
import yaml

from http import HTTPStatus
from os import environ
from rich import print as rprint
from rich.console import Console
from rich.pretty import pprint

console = Console()

@attr.s
class Droplet:
    droplet_id = attr.ib()
    minion_id = attr.ib()
    memory = attr.ib()
    disk = attr.ib()
    price_monthly = attr.ib()
    tags = attr.ib()
    domains = attr.ib()

def display_domains(domains):
    count = 0
    for domain_name, config in domains.items():
        count = count + 1
        ssl = False
        print()
        if "ssl" in config:
            ssl = True
        console.print(
            "{}. {}{}".format(
                count, "https://" if ssl else "http://", domain_name
            )
        )
        console.print("- {}".format(config["minion"]))
        if "testing" in config:
            console.print("- Test Site")
        if "profile" in config:
            profile = config["profile"]
            if profile == "php":
                profile = "{} (PHP)".format(config["php_profile"].capitalize())
            else:
                profile = "{}".format(profile.capitalize())
            console.print("- {}".format(profile))
        if "backup" in config:
            console.print("- backup")
        is_promtail = is_raygun = False
        if "env" in config:
            env = config["env"]
            if "norecaptcha_site_key" in env:
                console.print("- captcha (Google)")
            if "raygun4py_api_key" in env:
                is_raygun = True
            if "sparkpost_api_key" in env:
                console.print("- email (SparkPost)")
        if "promtail" in config:
            is_promtail = True
        if is_promtail or is_raygun:
            x = "{}".format("Promtail" if is_promtail else "RayGun")
            console.print("- monitor ({})".format(x))
        if ssl:
            certificate = ""
            if "letsencrypt" in config:
                certificate = "LetsEncrypt"
            console.print(
                "- ssl{}".format(
                    " ({})".format(certificate) if certificate else ""
                )
            )

def get_digital_ocean():
    """

    How To Use Web APIs in Python 3
    https://www.digitalocean.com/community/tutorials/how-to-use-web-apis-in-python-#!/usr/bin/env python3

    """
    result = []
    api_token = environ["DIGITAL_OCEAN_TOKEN"]
    api_url_base = "https://api.digitalocean.com/v2/"
    headers = {
        "Content-Type": "application/json",
        "Authorization": "Bearer {0}".format(api_token),
    }
    api_url = "{}droplets".format(api_url_base)
    response = requests.get(api_url, headers=headers)
    if response.status_code == HTTPStatus.OK:
        data = json.loads(response.content.decode("utf-8"))
        droplets = data["droplets"]
        for droplet in droplets:
            minion_id = droplet["name"]
            size = droplet["size"]
            # 06/05/2022, The project does not appear in the Droplet data, so
            # use the tags for now...
            tags = droplet["tags"]
            result.append(
                Droplet(
                    droplet_id=droplet["id"],
                    minion_id=minion_id,
                    memory=droplet["memory"],
                    disk=droplet["disk"],
                    price_monthly=size["price_monthly"],
                    tags=[x for x in tags],
                    domains=[],
                )
            )
    else:
        pprint(response, expand_all=True)
        raise Exception(
            "Error from the Digital Ocean API: {}".format(response.status_code)
        )
    return result
    # return dict(sorted(result.items()))

class Linode:
    def __init__(self):
        self.api_token = environ["LINODE_TOKEN"]
        self.api_url_base = "https://api.linode.com/v4/linode/"
        self.headers = {
            "Content-Type": "application/json",
            "Authorization": "Bearer {0}".format(self.api_token),
        }
        self.linode_types = {}

    def get_linodes(self):
        """

        From, Create a Linode Using the Linode API
        https://www.linode.com/docs/guides/getting-started-with-the-linode-api/

        """
        result = []
        api_url = "{}instances".format(self.api_url_base)
        response = requests.get(api_url, headers=self.headers)
        if response.status_code == HTTPStatus.OK:
            data = json.loads(response.content.decode("utf-8"))
            # pprint(data, expand_all=True)
            droplets = data["data"]
            for droplet in droplets:
                minion_id = droplet["label"]
                linode_type = self.get_linode_type(droplet["type"])
                # size = droplet["size"]
                # tags = droplet["tags"]
                specs = droplet["specs"]
                result.append(
                    Droplet(
                        droplet_id=droplet["id"],
                        minion_id=minion_id,
                        memory=specs["memory"],
                        disk=specs["disk"],
                        price_monthly=linode_type["price"]["monthly"],
                        tags=[],
                        domains=[],
                    )
                )
        else:
            pprint(response, expand_all=True)
            raise Exception(
                "Error from the Linode API: {}".format(response.status_code)
            )
        return result

    def get_linode_type(self, linode_type):
        """

        https://www.linode.com/docs/api/linode-types/#type-view

        """
        if linode_type in self.linode_types:
            pass
        else:
            api_url = "{}types/{}".format(self.api_url_base, linode_type)
            response = requests.get(api_url, headers=self.headers)
            if response.status_code == HTTPStatus.OK:
                data = json.loads(response.content.decode("utf-8"))
                self.linode_types["linode_type"] = data
            else:
                pprint(response, expand_all=True)
                raise Exception(
                    "Error from the Linode types API: {}".format(
                        response.status_code
                    )
                )
        return self.linode_types["linode_type"]

def get_domains():
    """Get the config for each site (domain name) from the Salt pillar."""
    result = {}
    pillar_folder = pathlib.Path.home().joinpath("Private", "deploy")
    for folder in pillar_folder.iterdir():
        if folder.is_dir() and folder.name.startswith("pillar-"):
            wildcard = get_wildcard(folder)
            result.update(get_domain_names(folder, wildcard))
    return result

def get_wildcard(pillar_folder):
    """Get the config for the wildcard include files.

    The ``top.sls`` file for our pillar, includes wildcards e.g::

      '*':
        - global.users
      'kb-* and not kb-vpn':
        - config.django
        - config.monitor
        - config.nginx

    This method retrieves the *wildcard* config, so it can be merged into the
    site later on (using ``match_minion``).

    """
    result = {}
    with open(pathlib.Path(pillar_folder, "top.sls")) as f:
        # load the 'top.sls' file
        data = yaml.safe_load(f)
        base = data["base"]
        for host_name, config in base.items():
            if "*" in host_name:
                result[host_name] = {}
                for include in config:
                    if isinstance(include, str):
                        # not sure if / why we need to do this...
                        if include.startswith("sites"):
                            pass
                        else:
                            # convert include to file and path
                            # e.g. 'config.monitor' to 'config/monitor.sls'
                            path_file = include.split(".")
                            path_file[-1] = path_file[-1] + ".sls"
                            with open(
                                pathlib.Path(pillar_folder, *path_file)
                            ) as f:
                                include_config = yaml.safe_load(f)
                                result[host_name].update(include_config)
    return result

def get_domain_names(pillar_folder, wildcard):
    """Find the server configuration for each site / domain name.

    .. warning:: We add the configuration for the site / domain name after
                 merging the configuration for the server
                 (including *wildcard* data).
                 Salt probably resolves this in a different order!

    """
    result = {}
    with open(pathlib.Path(pillar_folder, "top.sls")) as f:
        # load the 'top.sls' file
        data = yaml.safe_load(f)
        base = data["base"]
        for host_name, base_config in base.items():
            if "*" in host_name:
                # exclude wildcard e.g. 'pc-*' (see 'get_wildcard')
                pass
            else:
                server_config = {}
                for include in base_config:
                    if isinstance(include, str):
                        # convert include to file and path
                        # e.g. 'sites.cw-3' to 'sites/cw-3.sls'
                        path_file = include.split(".")
                        path_file[-1] = path_file[-1] + ".sls"
                        with open(pathlib.Path(pillar_folder, *path_file)) as f:
                            server_config.update(yaml.safe_load(f))
                if "sites" in server_config:
                    sites = server_config.pop("sites")
                    for domain_name, domain_config in sites.items():
                        # config for domain name (site)
                        result[domain_name] = {}
                        # merge in the *wildcard* config
                        result[domain_name].update(
                            merge_wildcard(host_name, wildcard)
                        )
                        # merge in the config for this server
                        result[domain_name].update(server_config)
                        # add the server (host) name
                        result[domain_name].update({"minion": host_name})
                        # finally - merge the domain config
                        result[domain_name].update(domain_config)
    return result

def json_dump_domains(domains):
    # file_name = "temp.json"
    # with open(file_name, "w") as f:
    #     json.dump(domains, f, indent=4)
    # write only required data to ``domains.json``
    # if you want to see all the data, then uncomment just above...
    data = {}
    for domain_name, config in domains.items():
        data.update({domain_name: {"minion_id": config["minion"]}})
    file_name = "domains.json"
    with open(file_name, "w") as f:
        json.dump(data, f, indent=4)
    rprint("[yellow]1. 'json_dump_domains' to '{}'...".format(file_name))

def json_dump_droplets(droplets):
    data = []
    file_name = "droplets.json"
    for droplet in droplets:
        data.append(attr.asdict(droplet))
    with open(file_name, "w") as f:
        json.dump(data, f, indent=4)
    rprint("[yellow]2. 'json_dump_droplets' to '{}'...".format(file_name))

def match_minion(minion_id, salt_top):
    """Copied from ``_match`` (see ``lib/pillarinfo.py`` in ``fabric``."""
    result = False
    if minion_id is None:
        result = True
    else:
        # 20/04/2021, Try and handle ``and not``
        # (when I don't really understand parsing)
        # e.g. 'drop-* and not drop-vpn'
        tokens = salt_top.split(" ")
        index_and = index_not = minion_not = None
        try:
            index_and = tokens.index("and")
            index_not = tokens.index("not")
        except ValueError:
            pass
        # if minion_id == "kb-a" and index_and and index_not:
        # import pdb; pdb.set_trace()
        if (
            index_and
            and index_not
            and index_and > 0
            and (index_and + 1 == index_not)
            and (index_not + 2 == len(tokens))
        ):
            minion_not = tokens[index_not + 1].strip()
            pos = salt_top.find(" and ")
            salt_top = salt_top[:pos].strip()
        # exclude the ``and not`` value e.g. ``drop-vpn``
        if minion_not and minion_not == minion_id:
            pass
        else:
            result = fnmatch.fnmatch(minion_id, salt_top)
            if not result:
                for item in salt_top.split(","):
                    result = fnmatch.fnmatch(minion_id, item)
                    if result:
                        break
    return result

def merge_wildcard(host_name, wildcard):
    """Merge configuration from files matching the wildcard.

    e.g. ``nc-*`` matches ``nc-a``, so ``nc-a`` will include the
    configuration data from ``nc-*``.

    """
    result = {}
    for wildcard_host, config in wildcard.items():
        if match_minion(host_name, wildcard_host):
            result.update(config)
    return result

def main():
    # find all the domains in the salt pillar
    domains = dict(sorted(get_domains().items()))
    json_dump_domains(domains)
    # display_domains(domains)

    # find the domain names for each minion (server)
    minions = {}
    for domain_name, config in domains.items():
        minion_id = config["minion"]
        if not minion_id in minions:
            minions[minion_id] = []
        minions[minion_id].append(domain_name)
    # pprint(minions, expand_all=True)

    # get a list of cloud servers (droplets)
    droplets = []
    droplets = droplets + get_digital_ocean()
    droplets = droplets + Linode().get_linodes()

    # link the domain names to the droplets
    for droplet in droplets:
        minion_id = droplet.minion_id
        if minion_id in minions:
            droplet.domains = minions.pop(minion_id)
    json_dump_droplets(droplets)

    # use the tag to link droplets to a contact
    # contacts = {}
    # for droplet in droplets:
    #     for tag in droplet.tags:
    #         if not tag in contacts:
    #             contacts[tag] = []
    #         contacts[tag].append(droplet)
    # pprint(contacts, expand_all=True)

    # display any minions which are not allocated to a droplet
    print()
    rprint("[cyan]List of Salt minions NOT allocated to a Cloud Server...")
    rprint(
        "[cyan]Note: this is most likely because "
        "the customer is paying for the hosting!"
    )
    pprint(minions, expand_all=True)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import tensorflow as tf

def resize_image(image, bboxes=None, min_size=None, max_size=None):
    """
    We need to resize image and (optionally) bounding boxes when the biggest
    side dimension is bigger than `max_size` or when the smaller side is
    smaller than `min_size`. If no max_size defined it won't scale down and if
    no min_size defined it won't scale up.

    Then, using the ratio we used, we need to properly scale the bounding
    boxes.

    Args:
        image: Tensor with image of shape (H, W, 3).
        bboxes: Optional Tensor with bounding boxes with shape (num_bboxes, 5).
            where we have (x_min, y_min, x_max, y_max, label) for each one.
        min_size: Min size of width or height.
        max_size: Max size of width or height.

    Returns:
        Dictionary containing:
            image: Tensor with scaled image.
            bboxes: Tensor with scaled (using the same factor as the image)
                bounding boxes with shape (num_bboxes, 5).
            scale_factor: Scale factor used to modify the image (1.0 means no
                change).
    """
    image_shape = tf.to_float(tf.shape(image))
    height = image_shape[0]
    width = image_shape[1]

    if min_size is not None:
        # We calculate the upscale factor, the rate we need to use to end up
        # with an image with it's lowest dimension at least `image_min_size`.
        # In case of being big enough the scale factor is 1. (no change)
        min_size = tf.to_float(min_size)
        min_dimension = tf.minimum(height, width)
        upscale_factor = tf.maximum(min_size / min_dimension, 1.)
    else:
        upscale_factor = tf.constant(1.)

    if max_size:
        # We do the same calculating the downscale factor, to end up with an
        # image where the biggest dimension is less than `image_max_size`.
        # When the image is small enough the scale factor is 1. (no change)
        max_size = tf.to_float(max_size)
        max_dimension = tf.maximum(height, width)
        downscale_factor = tf.minimum(max_size / max_dimension, 1.)
    else:
        downscale_factor = tf.constant(1.)

    scale_factor = upscale_factor * downscale_factor

    # New size is calculate using the scale factor and rounding to int.
    new_height = height * scale_factor
    new_width = width * scale_factor

    # Resize image using TensorFlow's own `resize_image` utility.
    image = tf.image.resize_images(
        image, tf.stack(tf.to_int32([new_height, new_width])),
        method=tf.image.ResizeMethod.BILINEAR
    )

    if bboxes is not None:
        # We normalize bounding boxes points before modifying the image.
        bboxes_float = tf.to_float(bboxes)
        x_min, y_min, x_max, y_max, label = tf.unstack(bboxes_float, axis=1)

        x_min = x_min / width
        y_min = y_min / height
        x_max = x_max / width
        y_max = y_max / height

        # Use new size to scale back the bboxes points to absolute values.
        x_min = tf.to_int32(x_min * new_width)
        y_min = tf.to_int32(y_min * new_height)
        x_max = tf.to_int32(x_max * new_width)
        y_max = tf.to_int32(y_max * new_height)
        label = tf.to_int32(label)  # Cast back to int.

        # Concat points and label to return a [num_bboxes, 5] tensor.
        bboxes = tf.stack([x_min, y_min, x_max, y_max, label], axis=1)
        return {
            'image': image,
            'bboxes': bboxes,
            'scale_factor': scale_factor,
        }

    return {
        'image': image,
        'scale_factor': scale_factor,
    }

def flip_image(image, bboxes=None, left_right=True, up_down=False):
    """Flips image on its axis for data augmentation.

    Args:
        image: Tensor with image of shape (H, W, 3).
        bboxes: Optional Tensor with bounding boxes with shape
            (total_bboxes, 5).
        left_right: Boolean flag to flip the image horizontally
            (left to right).
        up_down: Boolean flag to flip the image vertically (upside down)
    Returns:
        image: Flipped image with the same shape.
        bboxes: Tensor with the same shape.
    """

    image_shape = tf.shape(image)
    height = image_shape[0]
    width = image_shape[1]

    if bboxes is not None:
        # bboxes usually come from dataset as ints, but just in case we are
        # using flip for preprocessing, where bboxes usually are represented as
        # floats, we cast them.
        bboxes = tf.to_int32(bboxes)

    if left_right:
        image = tf.image.flip_left_right(image)
        if bboxes is not None:
            x_min, y_min, x_max, y_max, label = tf.unstack(bboxes, axis=1)
            new_x_min = width - x_max - 1
            new_y_min = y_min
            new_x_max = new_x_min + (x_max - x_min)
            new_y_max = y_max
            bboxes = tf.stack(
                [new_x_min, new_y_min, new_x_max, new_y_max, label], axis=1
            )

    if up_down:
        image = tf.image.flip_up_down(image)
        if bboxes is not None:
            x_min, y_min, x_max, y_max, label = tf.unstack(bboxes, axis=1)
            new_x_min = x_min
            new_y_min = height - y_max - 1
            new_x_max = x_max
            new_y_max = new_y_min + (y_max - y_min)
            bboxes = tf.stack(
                [new_x_min, new_y_min, new_x_max, new_y_max, label], axis=1
            )

    return_dict = {'image': image}
    if bboxes is not None:
        return_dict['bboxes'] = bboxes

    return return_dict
<commit_msg>Add random_patch for data augmentation<commit_after>import tensorflow as tf

from utils.bbox_transform_tf import clip_boxes

def resize_image(image, bboxes=None, min_size=None, max_size=None):
    """
    We need to resize image and (optionally) bounding boxes when the biggest
    side dimension is bigger than `max_size` or when the smaller side is
    smaller than `min_size`. If no max_size defined it won't scale down and if
    no min_size defined it won't scale up.

    Then, using the ratio we used, we need to properly scale the bounding
    boxes.

    Args:
        image: Tensor with image of shape (H, W, 3).
        bboxes: Optional Tensor with bounding boxes with shape (num_bboxes, 5).
            where we have (x_min, y_min, x_max, y_max, label) for each one.
        min_size: Min size of width or height.
        max_size: Max size of width or height.

    Returns:
        Dictionary containing:
            image: Tensor with scaled image.
            bboxes: Tensor with scaled (using the same factor as the image)
                bounding boxes with shape (num_bboxes, 5).
            scale_factor: Scale factor used to modify the image (1.0 means no
                change).
    """
    image_shape = tf.to_float(tf.shape(image))
    height = image_shape[0]
    width = image_shape[1]

    if min_size is not None:
        # We calculate the upscale factor, the rate we need to use to end up
        # with an image with it's lowest dimension at least `image_min_size`.
        # In case of being big enough the scale factor is 1. (no change)
        min_size = tf.to_float(min_size)
        min_dimension = tf.minimum(height, width)
        upscale_factor = tf.maximum(min_size / min_dimension, 1.)
    else:
        upscale_factor = tf.constant(1.)

    if max_size:
        # We do the same calculating the downscale factor, to end up with an
        # image where the biggest dimension is less than `image_max_size`.
        # When the image is small enough the scale factor is 1. (no change)
        max_size = tf.to_float(max_size)
        max_dimension = tf.maximum(height, width)
        downscale_factor = tf.minimum(max_size / max_dimension, 1.)
    else:
        downscale_factor = tf.constant(1.)

    scale_factor = upscale_factor * downscale_factor

    # New size is calculate using the scale factor and rounding to int.
    new_height = height * scale_factor
    new_width = width * scale_factor

    # Resize image using TensorFlow's own `resize_image` utility.
    image = tf.image.resize_images(
        image, tf.stack(tf.to_int32([new_height, new_width])),
        method=tf.image.ResizeMethod.BILINEAR
    )

    if bboxes is not None:
        # We normalize bounding boxes points before modifying the image.
        bboxes_float = tf.to_float(bboxes)
        x_min, y_min, x_max, y_max, label = tf.unstack(bboxes_float, axis=1)

        x_min = x_min / width
        y_min = y_min / height
        x_max = x_max / width
        y_max = y_max / height

        # Use new size to scale back the bboxes points to absolute values.
        x_min = tf.to_int32(x_min * new_width)
        y_min = tf.to_int32(y_min * new_height)
        x_max = tf.to_int32(x_max * new_width)
        y_max = tf.to_int32(y_max * new_height)
        label = tf.to_int32(label)  # Cast back to int.

        # Concat points and label to return a [num_bboxes, 5] tensor.
        bboxes = tf.stack([x_min, y_min, x_max, y_max, label], axis=1)
        return {
            'image': image,
            'bboxes': bboxes,
            'scale_factor': scale_factor,
        }

    return {
        'image': image,
        'scale_factor': scale_factor,
    }

def flip_image(image, bboxes=None, left_right=True, up_down=False):
    """Flips image on its axis for data augmentation.

    Args:
        image: Tensor with image of shape (H, W, 3).
        bboxes: Optional Tensor with bounding boxes with shape
            (total_bboxes, 5).
        left_right: Boolean flag to flip the image horizontally
            (left to right).
        up_down: Boolean flag to flip the image vertically (upside down)
    Returns:
        image: Flipped image with the same shape.
        bboxes: Tensor with the same shape.
    """

    image_shape = tf.shape(image)
    height = image_shape[0]
    width = image_shape[1]

    if bboxes is not None:
        # bboxes usually come from dataset as ints, but just in case we are
        # using flip for preprocessing, where bboxes usually are represented as
        # floats, we cast them.
        bboxes = tf.to_int32(bboxes)

    if left_right:
        image = tf.image.flip_left_right(image)
        if bboxes is not None:
            x_min, y_min, x_max, y_max, label = tf.unstack(bboxes, axis=1)
            new_x_min = width - x_max - 1
            new_y_min = y_min
            new_x_max = new_x_min + (x_max - x_min)
            new_y_max = y_max
            bboxes = tf.stack(
                [new_x_min, new_y_min, new_x_max, new_y_max, label], axis=1
            )

    if up_down:
        image = tf.image.flip_up_down(image)
        if bboxes is not None:
            x_min, y_min, x_max, y_max, label = tf.unstack(bboxes, axis=1)
            new_x_min = x_min
            new_y_min = height - y_max - 1
            new_x_max = x_max
            new_y_max = new_y_min + (y_max - y_min)
            bboxes = tf.stack(
                [new_x_min, new_y_min, new_x_max, new_y_max, label], axis=1
            )

    return_dict = {'image': image}
    if bboxes is not None:
        return_dict['bboxes'] = bboxes

    return return_dict

def random_patch(image, bboxes=None, debug=False):
    """Gets a random patch from an image.

    Args:
        image: Tensor with shape (H, W, 3).
        bboxes: Tensor with the ground-truth boxes. Shaped (total_boxes, 5).
            The last element in each box is the category label.

    Returns:
        image: Tensor with shape (H', W', 3), with H' <= H and W' <= W. A
            random patch of the input image.
        bboxes: Tensor with shape (new_total_boxes, 5), where we keep
            bboxes that have their center inside the patch, cropping
            them to the patch boundaries.
    """
    if debug:
        seed = 0
    else:
        seed = None
    # See the documentation on tf.crop_to_bounding_box for the meaning of
    # these variables.
    offset_width = tf.random_uniform(
        shape=[],
        minval=0,
        maxval=tf.subtract(
            tf.shape(image)[1],
            1
        ),
        dtype=tf.int32,
        seed=seed
    )
    offset_height = tf.random_uniform(
        shape=[],
        minval=0,
        maxval=tf.subtract(
            tf.shape(image)[0],
            1
        ),
        dtype=tf.int32,
        seed=seed
    )
    target_width = tf.random_uniform(
        shape=[],
        minval=1,
        maxval=tf.subtract(
            tf.shape(image)[1],
            offset_width
        ),
        dtype=tf.int32,
        seed=seed
    )
    target_height = tf.random_uniform(
        shape=[],
        minval=1,
        maxval=tf.subtract(
            tf.shape(image)[0],
            offset_height
        ),
        dtype=tf.int32,
        seed=seed
    )
    new_image = tf.image.crop_to_bounding_box(
        image,
        offset_height, offset_width,
        target_height, target_width
    )

    return_dict = {'image': new_image}

    # Return if we didn't have bboxes.
    if bboxes is None:
        return_dict['bboxes'] = tf.constant(-1.)
        return return_dict

    # Now we will remove all bboxes whose centers are not inside the cropped
    # image.

    # First get the x  and y coordinates of the center of each of the
    # bboxes.
    bboxes_center_x = tf.reduce_mean(
        tf.concat(
            [
                # bboxes[:, 0] gets a Tensor with shape (20,).
                # We do this to get a Tensor with shape (20, 1).
                bboxes[:, 0:1],
                bboxes[:, 2:3]
            ],
            axis=1
        )
    )
    bboxes_center_y = tf.reduce_mean(
        tf.concat(
            [
                bboxes[:, 1:2],
                bboxes[:, 3:4]
            ],
            axis=1
        ),
        axis=1
    )

    # Now we get a boolean tensor holding for each of the bboxes' centers
    # wheter they are inside the patch.
    center_x_is_inside = tf.logical_and(
        tf.greater(
            bboxes_center_x,
            offset_width
        ),
        tf.less(
            bboxes_center_x,
            tf.add(target_width, offset_width)
        )
    )
    center_y_is_inside = tf.logical_and(
        tf.greater(
            bboxes_center_y,
            offset_height
        ),
        tf.less(
            bboxes_center_y,
            tf.add(target_height, offset_height)
        )
    )
    center_is_inside = tf.logical_and(
        center_x_is_inside,
        center_y_is_inside
    )

    # Now we mask the bboxes, removing all those whose centers are outside
    # the patch.
    masked_bboxes = tf.boolean_mask(bboxes, center_is_inside)
    # We move the bboxes to the right place, clipping them if
    # necessary.
    new_bboxes_unclipped = tf.concat(
        [
            tf.subtract(masked_bboxes[:, 0:1], offset_width),
            tf.subtract(masked_bboxes[:, 1:2], offset_height),
            tf.subtract(masked_bboxes[:, 2:3], offset_width),
            tf.subtract(masked_bboxes[:, 3:4], offset_height),
        ],
        axis=1,
    )
    # Finally, we clip the boxes and add back the labels.
    new_bboxes = tf.concat(
        [
            clip_boxes(
                new_bboxes_unclipped[:, :4],
                imshape=tf.shape(new_image)[:2]
            ),
            tf.cast(masked_bboxes[:, 4:], tf.float32)
        ],
        axis=1
    )
    return_dict['bboxes'] = new_bboxes
    return return_dict
<|endoftext|>


<commit_before>from .helpers import codetest

class TestRepeat(object):
    """
    """
    def test_constant_return_0_args_1_ret(self):
        ret = codetest("""
                function foo()
                    return 2;
                end
                x = foo()
                return x
                """)
        assert ret.getval() == 2

    def test_call_1_arg_1_ret(self):
        ret = codetest("""
                function foo(x)
                    return x+1;
                end
                x = foo(10)
                return x
                """)
        assert ret.getval() == 11

    def test_call_2_args_1_ret(self):
        ret = codetest("""
                function foo(x, y)
                    return x+y;
                end
                x = foo(30, 10)
                return x
                """)
        assert ret.getval() == 40

    def test_call_2_args_ret_without_var(self):
        ret = codetest("""
                function foo(x, y)
                    return x+y;
                end
                return foo(30, 10)
                """)
        assert ret.getval() == 40

    def test_call_recursive(self):
        ret = codetest("""
                function fac(n)
                    if n == 1 then
                        return 1
                    end
                    x = n * fac(n-1);
                    print(n)
                    return x
                end
                x = fac(10)
                return x
                """)
        assert ret.getval() == 3628800
<commit_msg>Add some tests for CALLM and CALLMT<commit_after>from .helpers import codetest

class TestRepeat(object):
    """
    """
    def test_constant_return_0_args_1_ret(self):
        ret = codetest("""
                function foo()
                    return 2;
                end
                x = foo()
                return x
                """)
        assert ret.getval() == 2

    def test_call_1_arg_1_ret(self):
        ret = codetest("""
                function foo(x)
                    return x+1;
                end
                x = foo(10)
                return x
                """)
        assert ret.getval() == 11

    def test_call_2_args_1_ret(self):
        ret = codetest("""
                function foo(x, y)
                    return x+y;
                end
                x = foo(30, 10)
                return x
                """)
        assert ret.getval() == 40

    def test_call_2_args_ret_without_var(self):
        ret = codetest("""
                function foo(x, y)
                    return x+y;
                end
                return foo(30, 10)
                """)
        assert ret.getval() == 40

    def test_call_recursive(self):
        ret = codetest("""
                function fac(n)
                    if n == 1 then
                        return 1
                    end
                    x = n * fac(n-1);
                    print(n)
                    return x
                end
                x = fac(10)
                return x
                """)
        assert ret.getval() == 3628800

    def test_nested_function_tailcall(self):
        """
        Tests CALLMT with user defined function
        """
        ret = codetest("""
                function f(x)
                    y = x+1
                    return y
                end
                return f(f(5))
                """)
        assert ret.getval() == 7

    def test_nested_function_tailcall_builtin(self):
        """
        Tests CALLMT with builtin function
        """
        ret = codetest("""
                return math.sin(math.sin(1))
                """)
        assert ret.getval() == 0.7456241416655579

    def test_nested_function_call(self):
        """
        Tests CALLM with user function
        """
        ret = codetest("""
                function f(x)
                    y = x+1
                    return y
                end
                x = f(f(5))
                return x
                """)
        assert ret.getval() == 7

    def test_nested_function_call_builtin(self):
        """
        Tests CALLM with builtin function
        """
        ret = codetest("""
                x = math.sin(math.sin(1))
                return x
                """)
        assert ret.getval() == 0.7456241416655579
<|endoftext|>