<commit_before>import tensorflow as tf

import vgg19

tf.set_random_seed(777)  # reproducibility

def sub_pixel_conv2d(x, f, s=2):
    """
    # ref : https://github.com/tensorlayer/SRGAN/blob/master/tensorlayer/layers.py
    """
    if f is None:
        f = int(int(x.get_shape()[-1]) / (s ** 2))

    bsize, a, b, c = x.get_shape().as_list()
    bsize = tf.shape(x)[0]

    x_s = tf.split(x, s, 3)
    x_r = tf.concat(x_s, 2)

    return tf.reshape(x_r, (bsize, s * a, s * b, f))

def conv2d(x, f=64, k=3, s=1, act=None, pad='SAME', name='conv2d'):
    """
    :param x: input
    :param f: filters, default 64
    :param k: kernel size, default 3
    :param s: strides, default 1
    :param act: activation function, default None
    :param pad: padding (valid or same), default same
    :param name: scope name, default conv2d
    :return: covn2d net
    """
    return tf.layers.conv2d(x,
                            filters=f, kernel_size=k, strides=s,
                            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),
                            kernel_regularizer=tf.contrib.layers.l2_regularizer(5e-4),
                            bias_initializer=tf.zeros_initializer(),
                            activation=act,
                            padding=pad,
                            name=name)

def batch_norm(x, momentum=0.9, eps=1e-5, is_train=True):
    return tf.layers.batch_normalization(inputs=x,
                                         momentum=momentum,
                                         epsilon=eps,
                                         scale=True,
                                         trainable=is_train)

class SRGAN:

    def __init__(self, s, batch_size=16, input_height=384, input_width=384, input_channel=3,
                 sample_num=1 * 1, sample_size=1, output_height=384, output_width=384,
                 df_dim=64, gf_dim=64, g_lr=1e-4, d_lr=1e-4):

        """ Super-Resolution GAN Class
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 16
        :param input_height: input image height, default 384
        :param input_width: input image width, default 384
        :param input_channel: input image channel, default 3 (RGB)
        - in case of DIV2K-HR, image size is 384x384x3(HWC).

        # Output Settings
        :param sample_num: the number of output images, default 1
        :param sample_size: sample image size, default 1
        :param output_height: output images height, default 384
        :param output_width: output images width, default 384

        # For CNN model
        :param df_dim: discriminator filter, default 64
        :param gf_dim: generator filter, default 64

        # Training Option
        :param g_lr: generator learning rate, default 1e-4
        :param d_lr: discriminator learning rate, default 1e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.input_height = input_height
        self.input_width = input_width
        self.input_channel = input_channel

        self.lr_image_shape = [None, self.input_height // 4, self.input_width // 4, self.input_channel]
        self.hr_image_shape = [None, self.input_height, self.input_width, self.input_channel]

        self.vgg_image_shape = [224, 224, 3]

        self.sample_num = sample_num
        self.sample_size = sample_size
        self.output_height = output_height
        self.output_width = output_width

        self.df_dim = df_dim
        self.gf_dim = gf_dim

        self.beta1 = 0.9
        self.beta2 = 0.999

        self.d_lr = d_lr
        self.g_lr = g_lr
        self.lr_decay_rate = 1e-1
        self.lr_decay_epoch = 100

        # pre-defined
        self.d_real = 0.
        self.d_fake = 0.
        self.d_loss = 0.
        self.g_adv_loss = 0.
        self.g_cnt_loss = 0.
        self.g_mse_loss = 0.
        self.g_loss = 0.

        self.vgg19 = None

        self.g = None
        self.g_test = None
        self.adv_scaling = 1e-3
        self.vgg_scaling = 2e-6

        self.d_op = None
        self.g_op = None
        self.g_init_op = None
        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x_hr = tf.placeholder(tf.float32, shape=self.hr_image_shape, name="x-image-hr")  # (-1, 384, 384, 3)
        self.x_lr = tf.placeholder(tf.float32, shape=self.lr_image_shape, name="x-image-lr")  # (-1, 96, 96, 3)

        self.build_srgan()  # build SRGAN model

    def discriminator(self, x, reuse=None):
        """
        # Following a network architecture referred in the paper
        :param x: Input images (-1, 384, 384, 3)
        :param reuse: re-usability
        :return: HR (High Resolution) or SR (Super Resolution) images
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            x = conv2d(x, self.df_dim, act=tf.nn.leaky_relu, name='n64s1-1')

            strides = [2, 1]
            filters = [1, 2, 2, 4, 4, 8, 8]

            for i, f in enumerate(filters):
                x = conv2d(x, f=f, s=strides[i % 2], name='n%ds%d-%d' % (f, strides[i % 2], i + 1))
                x = batch_norm(x)
                x = tf.nn.leaky_relu(x)

            x = tf.layers.flatten(x)  # (-1, 96 * 96 * 64)

            x = tf.layers.dense(x, 1024, activation=tf.nn.leaky_relu, name='d-fc-0')
            logits = tf.layers.dense(x, 1, name='d-fc-1')
            prob = tf.nn.sigmoid(logits)

            return logits, prob

    def generator(self, x, reuse=None, is_train=True):
        """
        :param x: LR (Low Resolution) images, (-1, 96, 96, 3)
        :param reuse: scope re-usability
        :param is_train: is trainable, default True
        :return: SR (Super Resolution) images, (-1, 384, 384, 3)
        """

        with tf.variable_scope("generator", reuse=reuse):
            def residual_block(x, name="", _is_train=True):
                with tf.variable_scope(name):
                    x = conv2d(x, self.gf_dim, name="n64s1-2")
                    x = batch_norm(x, is_train=_is_train)
                    x = tf.nn.relu(x)

                    return x

            x = conv2d(x, self.gf_dim, act=tf.nn.relu, name='n64s1-1')
            x_ = x  # for later, used at layer concat

            # B residual blocks
            for i in range(1, 17):  # (1, 9)
                xx = residual_block(x, name='b-residual_block_%d' % (i * 2 - 1), _is_train=is_train)
                xx = residual_block(xx, name='b-residual_block_%d' % (i * 2), _is_train=is_train)
                xx = tf.add(x, xx)
                x = xx

            x = conv2d(x, self.gf_dim, name='n64s1-3')
            x = batch_norm(x, is_train=is_train)

            x = tf.add(x_, x)

            # subpixel conv blocks
            for i in range(1, 3):
                x = conv2d(x, self.gf_dim * 4, name='n256s1-%d' % (i + 2))
                x = sub_pixel_conv2d(x, f=None, s=2)
                x = tf.nn.relu(x)

            x = conv2d(x, self.input_channel, act=tf.nn.tanh, k=1, name='n3s1')  # (-1, 384, 384, 3)

            return x

    def build_vgg19(self, x, reuse=None):
        with tf.variable_scope("vgg19", reuse=reuse):
            # image re-scaling
            x = tf.cast((x + 1) / 2, dtype=tf.float32)  # [-1, 1] to [0, 1]
            x = tf.cast(x * 255., dtype=tf.float32)     # [0, 1]  to [0, 255]

            r, g, b = tf.split(x, 3, 3)
            bgr = tf.concat([b - self.vgg_mean[0],
                             g - self.vgg_mean[1],
                             r - self.vgg_mean[2]], axis=3)

            self.vgg19 = vgg19.VGG19(bgr)

            net = self.vgg19.vgg19_net['conv5_4']

            return net  # last layer

    def build_srgan(self):
        def mse_loss(pred, data):
            return tf.reduce_mean(tf.reduce_sum(tf.square(pred - data), axis=[3]))

        def sigmoid_loss(logits, label):
            return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=label))

        # Generator
        self.g = self.generator(self.x_lr)
        self.g_test = self.generator(self.x_lr, reuse=True, is_train=False)

        # Discriminator
        d_real, d_real_prob = self.discriminator(self.x_hr)
        d_fake, d_fake_prob = self.discriminator(self.g, reuse=True)

        # VGG19
        x_vgg_real = tf.image.resize_images(self.x_hr, size=self.vgg_image_shape[:2])
        x_vgg_fake = tf.image.resize_images(self.g, size=self.vgg_image_shape[:2])

        # vgg_bottle_real = self.vgg_model(x_vgg_real, weights=self.vgg_weights)
        # vgg_bottle_fake = self.vgg_model(x_vgg_fake, weights=self.vgg_weights, reuse=True)
        vgg_bottle_real = self.build_vgg19(x_vgg_real)
        vgg_bottle_fake = self.build_vgg19(x_vgg_fake, reuse=True)

        # Losses
        d_real_loss = sigmoid_loss(d_real, tf.ones_like(d_real))
        d_fake_loss = sigmoid_loss(d_fake, tf.zeros_like(d_fake))
        self.d_loss = d_real_loss + d_fake_loss

        self.g_adv_loss = self.adv_scaling * sigmoid_loss(d_fake, tf.ones_like(d_fake))
        self.g_mse_loss = mse_loss(self.g, self.x_hr)
        # tf.losses.mean_squared_error(self.g, self.x_hr, reduction=tf.losses.Reduction.MEAN)
        self.g_cnt_loss = self.vgg_scaling * mse_loss(vgg_bottle_real, vgg_bottle_fake)
        # tf.losses.mean_squared_error(vgg_bottle_fake, vgg_bottle_real, reduction=tf.losses.Reduction.MEAN)
        self.g_loss = self.g_adv_loss + self.g_mse_loss + self.g_cnt_loss

        # Summary
        tf.summary.scalar("loss/d_real_loss", d_real_loss)
        tf.summary.scalar("loss/d_fake_loss", d_fake_loss)
        tf.summary.scalar("loss/d_loss", self.d_loss)
        tf.summary.scalar("loss/g_cnt_loss", self.g_cnt_loss)
        tf.summary.scalar("loss/g_mse_loss", self.g_mse_loss)
        tf.summary.scalar("loss/g_adv_loss", self.g_adv_loss)
        tf.summary.scalar("loss/g_loss", self.g_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]

        self.d_op = tf.train.AdamOptimizer(learning_rate=self.d_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(loss=self.d_loss,
                                                                                var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(loss=self.g_loss,
                                                                                var_list=g_params)

        # pre-train
        self.g_init_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                                beta1=self.beta1, beta2=self.beta2).minimize(loss=self.g_mse_loss,
                                                                                var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<commit_msg>recover a part of previous commit<commit_after>import tensorflow as tf

import vgg19

tf.set_random_seed(777)  # reproducibility

def sub_pixel_conv2d(x, f, s=2):
    """
    # ref : https://github.com/tensorlayer/SRGAN/blob/master/tensorlayer/layers.py
    """
    if f is None:
        f = int(int(x.get_shape()[-1]) / (s ** 2))

    bsize, a, b, c = x.get_shape().as_list()
    bsize = tf.shape(x)[0]

    x_s = tf.split(x, s, 3)
    x_r = tf.concat(x_s, 2)

    return tf.reshape(x_r, (bsize, s * a, s * b, f))

def conv2d(x, f=64, k=3, s=1, act=None, pad='SAME', name='conv2d'):
    """
    :param x: input
    :param f: filters, default 64
    :param k: kernel size, default 3
    :param s: strides, default 1
    :param act: activation function, default None
    :param pad: padding (valid or same), default same
    :param name: scope name, default conv2d
    :return: covn2d net
    """
    return tf.layers.conv2d(x,
                            filters=f, kernel_size=k, strides=s,
                            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),
                            kernel_regularizer=tf.contrib.layers.l2_regularizer(5e-4),
                            bias_initializer=tf.zeros_initializer(),
                            activation=act,
                            padding=pad,
                            name=name)

def batch_norm(x, momentum=0.9, eps=1e-5, is_train=True):
    return tf.layers.batch_normalization(inputs=x,
                                         momentum=momentum,
                                         epsilon=eps,
                                         scale=True,
                                         trainable=is_train)

class SRGAN:

    def __init__(self, s, batch_size=16, input_height=384, input_width=384, input_channel=3,
                 sample_num=1 * 1, sample_size=1, output_height=384, output_width=384,
                 df_dim=64, gf_dim=64, g_lr=1e-4, d_lr=1e-4):

        """ Super-Resolution GAN Class
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 16
        :param input_height: input image height, default 384
        :param input_width: input image width, default 384
        :param input_channel: input image channel, default 3 (RGB)
        - in case of DIV2K-HR, image size is 384x384x3(HWC).

        # Output Settings
        :param sample_num: the number of output images, default 1
        :param sample_size: sample image size, default 1
        :param output_height: output images height, default 384
        :param output_width: output images width, default 384

        # For CNN model
        :param df_dim: discriminator filter, default 64
        :param gf_dim: generator filter, default 64

        # Training Option
        :param g_lr: generator learning rate, default 1e-4
        :param d_lr: discriminator learning rate, default 1e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.input_height = input_height
        self.input_width = input_width
        self.input_channel = input_channel

        self.lr_image_shape = [None, self.input_height // 4, self.input_width // 4, self.input_channel]
        self.hr_image_shape = [None, self.input_height, self.input_width, self.input_channel]

        self.vgg_image_shape = [224, 224, 3]

        self.sample_num = sample_num
        self.sample_size = sample_size
        self.output_height = output_height
        self.output_width = output_width

        self.df_dim = df_dim
        self.gf_dim = gf_dim

        self.beta1 = 0.9
        self.beta2 = 0.999

        self.d_lr = d_lr
        self.g_lr = g_lr
        self.lr_decay_rate = 1e-1
        self.lr_decay_epoch = 100

        self.vgg_mean = [103.939, 116.779, 123.68]

        # pre-defined
        self.d_real = 0.
        self.d_fake = 0.
        self.d_loss = 0.
        self.g_adv_loss = 0.
        self.g_cnt_loss = 0.
        self.g_mse_loss = 0.
        self.g_loss = 0.

        self.vgg19 = None

        self.g = None
        self.g_test = None
        self.adv_scaling = 1e-3
        self.vgg_scaling = 2e-6

        self.d_op = None
        self.g_op = None
        self.g_init_op = None
        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x_hr = tf.placeholder(tf.float32, shape=self.hr_image_shape, name="x-image-hr")  # (-1, 384, 384, 3)
        self.x_lr = tf.placeholder(tf.float32, shape=self.lr_image_shape, name="x-image-lr")  # (-1, 96, 96, 3)

        self.build_srgan()  # build SRGAN model

    def discriminator(self, x, reuse=None):
        """
        # Following a network architecture referred in the paper
        :param x: Input images (-1, 384, 384, 3)
        :param reuse: re-usability
        :return: HR (High Resolution) or SR (Super Resolution) images
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            x = conv2d(x, self.df_dim, act=tf.nn.leaky_relu, name='n64s1-1')

            strides = [2, 1]
            filters = [1, 2, 2, 4, 4, 8, 8]

            for i, f in enumerate(filters):
                x = conv2d(x, f=f, s=strides[i % 2], name='n%ds%d-%d' % (f, strides[i % 2], i + 1))
                x = batch_norm(x)
                x = tf.nn.leaky_relu(x)

            x = tf.layers.flatten(x)  # (-1, 96 * 96 * 64)

            x = tf.layers.dense(x, 1024, activation=tf.nn.leaky_relu, name='d-fc-0')
            logits = tf.layers.dense(x, 1, name='d-fc-1')
            prob = tf.nn.sigmoid(logits)

            return logits, prob

    def generator(self, x, reuse=None, is_train=True):
        """
        :param x: LR (Low Resolution) images, (-1, 96, 96, 3)
        :param reuse: scope re-usability
        :param is_train: is trainable, default True
        :return: SR (Super Resolution) images, (-1, 384, 384, 3)
        """

        with tf.variable_scope("generator", reuse=reuse):
            def residual_block(x, name="", _is_train=True):
                with tf.variable_scope(name):
                    x = conv2d(x, self.gf_dim, name="n64s1-2")
                    x = batch_norm(x, is_train=_is_train)
                    x = tf.nn.relu(x)

                    return x

            x = conv2d(x, self.gf_dim, act=tf.nn.relu, name='n64s1-1')
            x_ = x  # for later, used at layer concat

            # B residual blocks
            for i in range(1, 17):  # (1, 9)
                xx = residual_block(x, name='b-residual_block_%d' % (i * 2 - 1), _is_train=is_train)
                xx = residual_block(xx, name='b-residual_block_%d' % (i * 2), _is_train=is_train)
                xx = tf.add(x, xx)
                x = xx

            x = conv2d(x, self.gf_dim, name='n64s1-3')
            x = batch_norm(x, is_train=is_train)

            x = tf.add(x_, x)

            # subpixel conv blocks
            for i in range(1, 3):
                x = conv2d(x, self.gf_dim * 4, name='n256s1-%d' % (i + 2))
                x = sub_pixel_conv2d(x, f=None, s=2)
                x = tf.nn.relu(x)

            x = conv2d(x, self.input_channel, act=tf.nn.tanh, k=1, name='n3s1')  # (-1, 384, 384, 3)

            return x

    def build_vgg19(self, x, reuse=None):
        with tf.variable_scope("vgg19", reuse=reuse):
            # image re-scaling
            x = tf.cast((x + 1) / 2, dtype=tf.float32)  # [-1, 1] to [0, 1]
            x = tf.cast(x * 255., dtype=tf.float32)     # [0, 1]  to [0, 255]

            r, g, b = tf.split(x, 3, 3)
            bgr = tf.concat([b - self.vgg_mean[0],
                             g - self.vgg_mean[1],
                             r - self.vgg_mean[2]], axis=3)

            self.vgg19 = vgg19.VGG19(bgr)

            net = self.vgg19.vgg19_net['conv5_4']

            return net  # last layer

    def build_srgan(self):
        def mse_loss(pred, data):
            return tf.reduce_mean(tf.reduce_sum(tf.square(pred - data), axis=[3]))

        def sigmoid_loss(logits, label):
            return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=label))

        # Generator
        self.g = self.generator(self.x_lr)
        self.g_test = self.generator(self.x_lr, reuse=True, is_train=False)

        # Discriminator
        d_real, d_real_prob = self.discriminator(self.x_hr)
        d_fake, d_fake_prob = self.discriminator(self.g, reuse=True)

        # VGG19
        x_vgg_real = tf.image.resize_images(self.x_hr, size=self.vgg_image_shape[:2])
        x_vgg_fake = tf.image.resize_images(self.g, size=self.vgg_image_shape[:2])

        # vgg_bottle_real = self.vgg_model(x_vgg_real, weights=self.vgg_weights)
        # vgg_bottle_fake = self.vgg_model(x_vgg_fake, weights=self.vgg_weights, reuse=True)
        vgg_bottle_real = self.build_vgg19(x_vgg_real)
        vgg_bottle_fake = self.build_vgg19(x_vgg_fake, reuse=True)

        # Losses
        d_real_loss = sigmoid_loss(d_real, tf.ones_like(d_real))
        d_fake_loss = sigmoid_loss(d_fake, tf.zeros_like(d_fake))
        self.d_loss = d_real_loss + d_fake_loss

        self.g_adv_loss = self.adv_scaling * sigmoid_loss(d_fake, tf.ones_like(d_fake))
        self.g_mse_loss = mse_loss(self.g, self.x_hr)
        # tf.losses.mean_squared_error(self.g, self.x_hr, reduction=tf.losses.Reduction.MEAN)
        self.g_cnt_loss = self.vgg_scaling * mse_loss(vgg_bottle_real, vgg_bottle_fake)
        # tf.losses.mean_squared_error(vgg_bottle_fake, vgg_bottle_real, reduction=tf.losses.Reduction.MEAN)
        self.g_loss = self.g_adv_loss + self.g_mse_loss + self.g_cnt_loss

        # Summary
        tf.summary.scalar("loss/d_real_loss", d_real_loss)
        tf.summary.scalar("loss/d_fake_loss", d_fake_loss)
        tf.summary.scalar("loss/d_loss", self.d_loss)
        tf.summary.scalar("loss/g_cnt_loss", self.g_cnt_loss)
        tf.summary.scalar("loss/g_mse_loss", self.g_mse_loss)
        tf.summary.scalar("loss/g_adv_loss", self.g_adv_loss)
        tf.summary.scalar("loss/g_loss", self.g_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]

        self.d_op = tf.train.AdamOptimizer(learning_rate=self.d_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(loss=self.d_loss,
                                                                                var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(loss=self.g_loss,
                                                                                var_list=g_params)

        # pre-train
        self.g_init_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                                beta1=self.beta1, beta2=self.beta2).minimize(loss=self.g_mse_loss,
                                                                                var_list=g_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# (c) 2016, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os

from ansible.module_utils.six.moves import configparser
from ansible.module_utils.six import integer_types, string_types
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.urls import fetch_url

EXO_DNS_BASEURL = "https://api.exoscale.ch/dns/v1"

def exo_dns_argument_spec():
    return dict(
        api_key=dict(default=None, no_log=True),
        api_secret=dict(default=None, no_log=True),
        api_timeout=dict(type='int', default=10),
        api_region=dict(default='cloudstack'),
        validate_certs=dict(default='yes', type='bool'),
    )

def exo_dns_required_together():
    return [['api_key', 'api_secret']]

class ExoDns(object):

    def __init__(self, module):
        self.module = module

        self.api_key = self.module.params. get('api_key')
        self.api_secret = self.module.params.get('api_secret')
        if not (self.api_key and self.api_secret):
            try:
                region = self.module.params.get('api_region')
                config = self.read_config(ini_group=region)
                self.api_key = config['key']
                self.api_secret = config['secret']
            except Exception as e:
                self.module.fail_json(msg="Error while processing config: %s" % to_native(e))

        self.headers = {
            'X-DNS-Token': "%s:%s" % (self.api_key, self.api_secret),
            'Content-Type': 'application/json',
            'Accept': 'application/json',
        }
        self.result = {
            'changed': False,
            'diff': {
                'before': {},
                'after': {},
            }
        }

    def read_config(self, ini_group=None):
        if not ini_group:
            ini_group = os.environ.get('CLOUDSTACK_REGION', 'cloudstack')

        keys = ['key', 'secret']
        env_conf = {}
        for key in keys:
            if 'CLOUDSTACK_%s' % key.upper() not in os.environ:
                break
            else:
                env_conf[key] = os.environ['CLOUDSTACK_%s' % key.upper()]
        else:
            return env_conf

        # Config file: $PWD/cloudstack.ini or $HOME/.cloudstack.ini
        # Last read wins in configparser
        paths = (
            os.path.join(os.path.expanduser('~'), '.cloudstack.ini'),
            os.path.join(os.getcwd(), 'cloudstack.ini'),
        )
        # Look at CLOUDSTACK_CONFIG first if present
        if 'CLOUDSTACK_CONFIG' in os.environ:
            paths += (os.path.expanduser(os.environ['CLOUDSTACK_CONFIG']),)
        if not any([os.path.exists(c) for c in paths]):
            self.module.fail_json(msg="Config file not found. Tried : %s" % ", ".join(paths))

        conf = configparser.ConfigParser()
        conf.read(paths)
        return dict(conf.items(ini_group))

    def api_query(self, resource="/domains", method="GET", data=None):
        url = EXO_DNS_BASEURL + resource
        if data:
            data = self.module.jsonify(data)

        response, info = fetch_url(
            module=self.module,
            url=url,
            data=data,
            method=method,
            headers=self.headers,
            timeout=self.module.params.get('api_timeout'),
        )

        if info['status'] not in (200, 201, 204):
            self.module.fail_json(msg="%s returned %s, with body: %s" % (url, info['status'], info['msg']))

        try:
            return self.module.from_json(to_text(response.read()))

        except Exception as e:
            self.module.fail_json(msg="Could not process response into json: %s" % to_native(e))

    def has_changed(self, want_dict, current_dict, only_keys=None):
        changed = False
        for key, value in want_dict.items():
            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue
            # Skip None values
            if value is None:
                continue
            if key in current_dict:
                if isinstance(current_dict[key], integer_types):
                    if value != current_dict[key]:
                        self.result['diff']['before'][key] = current_dict[key]
                        self.result['diff']['after'][key] = value
                        changed = True
                elif isinstance(current_dict[key], string_types):
                    if value.lower() != current_dict[key].lower():
                        self.result['diff']['before'][key] = current_dict[key]
                        self.result['diff']['after'][key] = value
                        changed = True
                else:
                    self.module.fail_json(msg="Unable to determine comparison for key %s" % key)
            else:
                self.result['diff']['after'][key] = value
                changed = True
        return changed
<commit_msg>exoscale: fix api_key param auth typo<commit_after># -*- coding: utf-8 -*-
#
# (c) 2016, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os

from ansible.module_utils.six.moves import configparser
from ansible.module_utils.six import integer_types, string_types
from ansible.module_utils._text import to_native, to_text
from ansible.module_utils.urls import fetch_url

EXO_DNS_BASEURL = "https://api.exoscale.ch/dns/v1"

def exo_dns_argument_spec():
    return dict(
        api_key=dict(default=None, no_log=True),
        api_secret=dict(default=None, no_log=True),
        api_timeout=dict(type='int', default=10),
        api_region=dict(default='cloudstack'),
        validate_certs=dict(default='yes', type='bool'),
    )

def exo_dns_required_together():
    return [['api_key', 'api_secret']]

class ExoDns(object):

    def __init__(self, module):
        self.module = module

        self.api_key = self.module.params.get('api_key')
        self.api_secret = self.module.params.get('api_secret')
        if not (self.api_key and self.api_secret):
            try:
                region = self.module.params.get('api_region')
                config = self.read_config(ini_group=region)
                self.api_key = config['key']
                self.api_secret = config['secret']
            except Exception as e:
                self.module.fail_json(msg="Error while processing config: %s" % to_native(e))

        self.headers = {
            'X-DNS-Token': "%s:%s" % (self.api_key, self.api_secret),
            'Content-Type': 'application/json',
            'Accept': 'application/json',
        }
        self.result = {
            'changed': False,
            'diff': {
                'before': {},
                'after': {},
            }
        }

    def read_config(self, ini_group=None):
        if not ini_group:
            ini_group = os.environ.get('CLOUDSTACK_REGION', 'cloudstack')

        keys = ['key', 'secret']
        env_conf = {}
        for key in keys:
            if 'CLOUDSTACK_%s' % key.upper() not in os.environ:
                break
            else:
                env_conf[key] = os.environ['CLOUDSTACK_%s' % key.upper()]
        else:
            return env_conf

        # Config file: $PWD/cloudstack.ini or $HOME/.cloudstack.ini
        # Last read wins in configparser
        paths = (
            os.path.join(os.path.expanduser('~'), '.cloudstack.ini'),
            os.path.join(os.getcwd(), 'cloudstack.ini'),
        )
        # Look at CLOUDSTACK_CONFIG first if present
        if 'CLOUDSTACK_CONFIG' in os.environ:
            paths += (os.path.expanduser(os.environ['CLOUDSTACK_CONFIG']),)
        if not any([os.path.exists(c) for c in paths]):
            self.module.fail_json(msg="Config file not found. Tried : %s" % ", ".join(paths))

        conf = configparser.ConfigParser()
        conf.read(paths)
        return dict(conf.items(ini_group))

    def api_query(self, resource="/domains", method="GET", data=None):
        url = EXO_DNS_BASEURL + resource
        if data:
            data = self.module.jsonify(data)

        response, info = fetch_url(
            module=self.module,
            url=url,
            data=data,
            method=method,
            headers=self.headers,
            timeout=self.module.params.get('api_timeout'),
        )

        if info['status'] not in (200, 201, 204):
            self.module.fail_json(msg="%s returned %s, with body: %s" % (url, info['status'], info['msg']))

        try:
            return self.module.from_json(to_text(response.read()))

        except Exception as e:
            self.module.fail_json(msg="Could not process response into json: %s" % to_native(e))

    def has_changed(self, want_dict, current_dict, only_keys=None):
        changed = False
        for key, value in want_dict.items():
            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue
            # Skip None values
            if value is None:
                continue
            if key in current_dict:
                if isinstance(current_dict[key], integer_types):
                    if value != current_dict[key]:
                        self.result['diff']['before'][key] = current_dict[key]
                        self.result['diff']['after'][key] = value
                        changed = True
                elif isinstance(current_dict[key], string_types):
                    if value.lower() != current_dict[key].lower():
                        self.result['diff']['before'][key] = current_dict[key]
                        self.result['diff']['after'][key] = value
                        changed = True
                else:
                    self.module.fail_json(msg="Unable to determine comparison for key %s" % key)
            else:
                self.result['diff']['after'][key] = value
                changed = True
        return changed
<|endoftext|>


<commit_before>from __future__ import with_statement

from fabric.api import env
from fabric.api import hide
from fabric.api import run
from fabric.api import local
from fabric.api import sudo
from fabric.api import task
from fabric.api import abort
from fabric.context_managers import cd

import os
import sys

# Constant
CUSTOM_KVM = os.path.abspath("./src/cloudlet/lib/bin/x86_64/cloudlet_qemu-system-x86_64")

def check_support():
    if run("egrep '^flags.*(vmx|svm)' /proc/cpuinfo > /dev/null").failed:
        abort("Need hardware VM support (vmx)")

def disable_EPT():
    if run("egrep '^flags.*(ept)' /proc/cpuinfo > /dev/null").failed:
        return
    else:
        # disable EPT
        sudo('modprobe -r kvm_intel')
        sudo('modprobe kvm_intel "ept=0"')

def install_kvm():
    global CUSTOM_KVM

    dest_path = "/usr/bin/qemu-system-x86_64"
    if CUSTOM_KVM != dest_path:
        sudo("cp %s %s.old" % (dest_path, dest_path))
        sudo("cp %s %s" % (CUSTOM_KVM, dest_path))

@task
def localhost():
    env.run = local
    env.warn_only = True
    env.hosts = ['localhost']

@task
def install():
    global CUSTOM_KVM
    check_support()

    # install dependent package
    with hide('stdout'):
        sudo("apt-get update")
    if sudo("apt-get install -y qemu-kvm libvirt-bin gvncviewer " +
            "python-libvirt python-xdelta3 python-dev openjdk-6-jre  " +
            "liblzma-dev apparmor-utils libc6-i386 python-pip libxml2-dev " +
            "libxslt1-dev").failed:
        abort("Failed to install libraries")
    if sudo("pip install bson pyliblzma psutil sqlalchemy python-dateutil requests").failed:
        abort("Failed to install python libraries")

    # disable libvirtd from appArmor to enable custom KVM
    if sudo("aa-complain /usr/sbin/libvirtd").failed:
        abort("Failed to disable AppArmor for custom KVM")

    # add current user to groups (optional)
    username = env.get('user')
    if sudo("adduser %s kvm" % username).failed:
        abort("Cannot add user to kvm group")
    if sudo("adduser %s libvirtd" % username).failed:
        abort("Cannot add user to libvirtd group")
    if sudo("adduser %s fuse" % username).failed:
        abort("Cannot add user to fuse group")

    # Make sure to have fuse support
    # qemu-kvm changes the permission of /dev/fuse, so we revert back the
    # permission. This bug is fixed from udev-175-0ubuntu26
    # Please see https://bugs.launchpad.net/ubuntu/+source/udev/+bug/1152718
    if sudo("chmod 1666 /dev/fuse").failed:
        abort("Failed to enable fuse for the user")
    if sudo("chmod 644 /etc/fuse.conf").failed:
        abort("Failed to change permission of fuse configuration")
    if sudo("sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf"):
        abort("Failed to allow other user to access FUSE file")

    # install custom KVM
    install_kvm()

    # (Optional) disable EPT support
    # When you use EPT support with FUSE+mmap, it randomly causes kernel panic.
    # We're investigating it whether it's Linux kernel bug or not.
    disable_EPT()

    # install cloudlet package
    current_dir = os.path.abspath(os.curdir)
    with cd(current_dir):
        if sudo("python setup.py install").failed:
            abort("cannot install cloudlet library")

    sys.stdout.write("[SUCCESS] VM synthesis code is installed\n")

<commit_msg>library dependency update<commit_after>from __future__ import with_statement

from fabric.api import env
from fabric.api import hide
from fabric.api import run
from fabric.api import local
from fabric.api import sudo
from fabric.api import task
from fabric.api import abort
from fabric.context_managers import cd

import os
import sys

# Constant
CUSTOM_KVM = os.path.abspath("./src/cloudlet/lib/bin/x86_64/cloudlet_qemu-system-x86_64")

def check_support():
    if run("egrep '^flags.*(vmx|svm)' /proc/cpuinfo > /dev/null").failed:
        abort("Need hardware VM support (vmx)")

def disable_EPT():
    if run("egrep '^flags.*(ept)' /proc/cpuinfo > /dev/null").failed:
        return
    else:
        # disable EPT
        sudo('modprobe -r kvm_intel')
        sudo('modprobe kvm_intel "ept=0"')

def install_kvm():
    global CUSTOM_KVM

    dest_path = "/usr/bin/qemu-system-x86_64"
    if CUSTOM_KVM != dest_path:
        sudo("cp %s %s.old" % (dest_path, dest_path))
        sudo("cp %s %s" % (CUSTOM_KVM, dest_path))

@task
def localhost():
    env.run = local
    env.warn_only = True
    env.hosts = ['localhost']

@task
def install():
    global CUSTOM_KVM
    check_support()

    # install dependent package
    with hide('stdout'):
        sudo("apt-get update")
    if sudo("apt-get install -y qemu-kvm libvirt-bin gvncviewer " +
            "python-libvirt python-xdelta3 python-dev openjdk-6-jre  " +
            "liblzma-dev apparmor-utils libc6-i386 python-pip libxml2-dev " +
            "libxslt1-dev").failed:
        abort("Failed to install libraries")
    if sudo("pip install bson pyliblzma psutil sqlalchemy python-dateutil requests lxml").failed:
        abort("Failed to install python libraries")

    # disable libvirtd from appArmor to enable custom KVM
    if sudo("aa-complain /usr/sbin/libvirtd").failed:
        abort("Failed to disable AppArmor for custom KVM")

    # add current user to groups (optional)
    username = env.get('user')
    if sudo("adduser %s kvm" % username).failed:
        abort("Cannot add user to kvm group")
    if sudo("adduser %s libvirtd" % username).failed:
        abort("Cannot add user to libvirtd group")
    if sudo("adduser %s fuse" % username).failed:
        abort("Cannot add user to fuse group")

    # Make sure to have fuse support
    # qemu-kvm changes the permission of /dev/fuse, so we revert back the
    # permission. This bug is fixed from udev-175-0ubuntu26
    # Please see https://bugs.launchpad.net/ubuntu/+source/udev/+bug/1152718
    if sudo("chmod 1666 /dev/fuse").failed:
        abort("Failed to enable fuse for the user")
    if sudo("chmod 644 /etc/fuse.conf").failed:
        abort("Failed to change permission of fuse configuration")
    if sudo("sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf"):
        abort("Failed to allow other user to access FUSE file")

    # install custom KVM
    install_kvm()

    # (Optional) disable EPT support
    # When you use EPT support with FUSE+mmap, it randomly causes kernel panic.
    # We're investigating it whether it's Linux kernel bug or not.
    disable_EPT()

    # install cloudlet package
    current_dir = os.path.abspath(os.curdir)
    with cd(current_dir):
        if sudo("python setup.py install").failed:
            abort("cannot install cloudlet library")

    sys.stdout.write("[SUCCESS] VM synthesis code is installed\n")

<|endoftext|>


<commit_before>import keras as keras
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Dropout, TimeDistributed, Concatenate, Input, UpSampling2D
from keras.layers import GRU, LSTM, Conv2D, Conv1D, Reshape, Flatten, Permute, AveragePooling2D, MaxPooling2D, RepeatVector, Conv2DTranspose, LocallyConnected2D
import keras.optimizers as optimizers
import numpy as np

from custom_objects import CustomObjects

class ModelDef(object):

  layers=[]

  model = None
  utils = None
  started = False

  stateful = False

  encoder_trainable = True
  decoder_trainable = True
  generator_trainable = False

  def __init__(self, utils, config):
    self.utils = utils
    self.config = config
    self.models = {}
    self.layers = []

    self.utils.log("encoder_trainable:", self.encoder_trainable)
    self.utils.log("decoder_trainable:", self.decoder_trainable)
    self.utils.log("generator_trainable:", self.generator_trainable)

  def define_model(self, frame_seq_len, framelen, num_frame_seqs):
    self.utils.log("Defining model")
    config = self.config
    overlap_sequence = config.overlap_sequence
    short_input_len = frame_seq_len - overlap_sequence*2

    self.conv_count = 16
    enc_params = 20

    generator_trainable = self.generator_trainable

    print("short_input_len", short_input_len)

    main_input = Input(shape=(frame_seq_len, framelen), dtype='float32', name="main_input")
    # short_input = Input(shape=(short_input_len, framelen), dtype='float32', name="short_input")

    encoder_output = self.encoder_model(enc_params, shape=(frame_seq_len, framelen))(main_input)
    conf = self.encoder_model(framelen)
    print("encoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    encoder_output_len = conf.get_output_shape_at(0)[1]

    # Run the decoder portion of autoencoder
    mid_output = self.decoder_model(framelen, (encoder_output_len, enc_params))(encoder_output)

    conf = self.decoder_model(framelen)
    print("decoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    # Generator

    conf = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_0'
        , trainable=generator_trainable
    )
    res = conf(encoder_output)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_1'
        , trainable=generator_trainable
    )(res)

    res = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_postconcat'
        , trainable=generator_trainable
    )(res)

    conf = TimeDistributed(
        Dense(
          enc_params
          , activation="relu"
          , trainable=generator_trainable
        )
        , name='generator_TD_Dense_0'
    )
    res = generator_output = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # res = self.decoder_model(framelen, (-1, enc_params))(generator_output)

    conf = self.decoder_model(framelen)
    res = conf(res)
    print("decoder_model shapes for input / output 1")
    print(conf.get_input_shape_at(1))
    print(conf.get_output_shape_at(1))

    main_output = res

    model = Model(
        #inputs=[main_input, short_input],
        inputs=[main_input],
        outputs=[main_output, mid_output, generator_output, encoder_output]
    )

    self.model = model
    return model

  def encoder_model(self, enc_params, shape=(-1,0)):

    if self.models.get('encoder_model'):
      print("Reusing encoder model")
      return self.models.get('encoder_model')
    framelen = shape[1]
    in_scale = 2
    in_count = framelen * in_scale
    encoder_trainable = self.encoder_trainable

    conv_count = self.conv_count

    res = encoder_input = Input(shape=shape, dtype='float32', name="encoder_input")
    print("Encoder model input shape:", shape)

    res = keras.layers.concatenate([res, res])

    conf = TimeDistributed(keras.layers.Reshape((in_count, 1), trainable=encoder_trainable))
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,14), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,13), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = keras.layers.Reshape((-1, conv_count), trainable=encoder_trainable)(res)

    # Second chain - pooling input data parameter by parameter
    conf = TimeDistributed(keras.layers.Reshape((framelen, 1), trainable=encoder_trainable))
    shaped_input = conf(encoder_input)

    conf = AveragePooling2D(pool_size=(3,1), strides=(2,1), padding="valid", trainable=encoder_trainable)
    res_mean = conf(shaped_input)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)
    conf = AveragePooling2D(pool_size=(3,1), strides=(2,1), padding="valid", trainable=encoder_trainable)
    res_mean = conf(res_mean)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)
    conf = keras.layers.Reshape((-1, framelen), trainable=encoder_trainable)
    res_mean = conf(res_mean)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)
    # conf = TimeDistributed(
    #     Dense(
    #         enc_params
    #         , activation="relu"
    #         , trainable=encoder_trainable
    #         , name="encoder_post_average_dense"
    #     )
    # )
    # res_mean = conf(res_mean)
    # print(conf.get_config())
    # print(conf.input_shape)
    # print(conf.output_shape)

    res = keras.layers.concatenate([res, res_mean])

    conf = TimeDistributed(
        Dense(
            enc_params * 5
            , activation="relu"
            , trainable=encoder_trainable
            , name="encoder_concat_chains_dense"
        )
    )
    res = conf(res)

    conf = TimeDistributed(
        Dense(
            enc_params
            , activation="relu"
            , trainable=encoder_trainable
            , name="encoder_final_dense"
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    encoder_output = res

    self.models['encoder_model'] = Model(encoder_input, encoder_output)
    return self.models['encoder_model']

  # Shared Decoder model, taking Conv2D encoded input, or an attempt at predicted
  # sequences of the encoded data.
  #
  # Returns a single instance of a decoder model. May be called any number of times.
  #
  # Arguments:
  #  framelen: standard frame length for a timestep
  #  shape: two element tensor, typically (-1, enc_params), where enc_params is the
  #         width of the input timestep. -1 saves us having to enter the
  #         frame sequence (sub-batch) length
  #
  # Learning can be disabled by setting self.decoder_trainable = False
  #
  # Example:
  #    main_output = self.decoder_model(framelen, (-1, enc_params))(generator_output)
  def decoder_model(self, framelen, shape=(-1, 0)):

    if self.models.get('decoder_model'):
      return self.models.get('decoder_model')

    decoder_trainable = self.decoder_trainable
    enc_params = shape[1]
    conv_count = self.conv_count

    res = decoder_input = Input(shape=shape, dtype='float32', name="decoder_input")

    conf = TimeDistributed(
            keras.layers.Reshape(
              (1, enc_params)
              , trainable=decoder_trainable
            )
          )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,13),
              padding='valid',
              strides=(2,1),
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,14),
              strides=(2,1),
              padding='valid',
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )

    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Return a single filter pulling together the results of all conv_count filters
    conf = Conv2D(
              1,
              kernel_size=(2,14),
              #  strides=(2,1),
              padding='valid',
              activation='sigmoid',
              name='decoder_conv_squash',
              trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf  = keras.layers.Reshape((-1, framelen), trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Pass through the input

    conf = TimeDistributed(
        Dense(
            enc_params
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_pass_thru_dense_in'
        )
    )
    res_pt = conf(decoder_input)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = UpSampling2D(
        size=[4,1]
        , data_format='channels_last'
        , trainable=decoder_trainable
    )
    res_pt = conf(res_pt)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(
        Dense(
            framelen * 5
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_pass_thru_dense_wide'
        )
    )
    res_pt = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(
        Dense(
            framelen
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_pass_thru_dense'
        )
    )
    res_pt = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = keras.layers.concatenate([res, res_pt])

    conf = TimeDistributed(
        Dense(
            enc_params * 5
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_post_conv_dense'
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(
        Dense(
            framelen
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_final_dense'
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    decoder_output = res

    self.models['decoder_model'] = Model(decoder_input, decoder_output)
    return self.models['decoder_model']

  def compile_model(self):
    self.utils.log("Compiling model")

    loss = CustomObjects.codec2_param_error_td
    # other loss options: CustomObjects.codec2_param_mean_square_error; 'mean_absolute_error'; 'cosine_proximity'

    encoder_loss_prop = 0

    main_loss_prop = 0.2
    mid_loss_prop = 0.4
    generator_loss_prop = 0.4

    if self.decoder_trainable and not self.generator_trainable:
      main_loss_prop = 0
      mid_loss_prop = 1
      generator_loss_prop = 0
    elif not self.decoder_trainable and self.generator_trainable:
      mid_loss_prop = 0
      main_loss_prop = 0.0
      generator_loss_prop = 1.0

    self.utils.log("Loss weightings:", main_loss_prop, mid_loss_prop, generator_loss_prop, encoder_loss_prop)

    self.model.compile(
        loss=[loss, loss, 'mean_absolute_error', 'mean_absolute_error'],
        loss_weights=[main_loss_prop, mid_loss_prop, generator_loss_prop, encoder_loss_prop],
        optimizer=self.get_optimizer_from_config())
    self.utils.log_model_summary()

  def fit(self, input_seq, output_seq, batch_size=None, epochs=1, shuffle=False, callbacks=None):
      inputs = input_seq[0]
      outputs = output_seq

      # s = output_seq[0].shape
      # dummy_encoded_output = np.zeros((s[0], 24, 64), dtype=np.float32)
      # outputs.append( dummy_encoded_output)

      print("X shape:", inputs.shape)
      print("y shape:", outputs[0].shape, outputs[1].shape, outputs[2].shape, outputs[3].shape)

      self.model.fit(inputs, outputs, batch_size=batch_size, epochs=epochs, shuffle=shuffle,
       callbacks=callbacks
      )

  # we can wrap the model.add method, since in the future we may wish to
  # provide additional processing at this level
  def add_layer(self, layer):
    self.model.add(layer)
    return layer

  # start training GRU 1, then 1&2, then 3
  def before_iteration(self, iteration):
#    if iteration == 541:
#      self.utils.log("Adding frame rotation to reduce memory usage")
#      self.config.limit_frames = self.config.num_frames / 100
#      self.model_updates_lstm_1234_trainable()
#      self.config.log_attrs()
#
#    elif iteration == 481:
#      self.model_updates_lstm3_trainable()
#

    if not self.started:
      self.model_updates_onstart()

    self.started = True

  def model_updates_onstart(self):
#    self.utils.log("Make all lstms trainable")
#    self.model.layers[0].trainable=True
#    self.model.layers[1].trainable=True
#    self.model.layers[2].trainable=True
#    self.model.layers[3].trainable=True
#    self.model.layers[4].trainable=True

    self.compile_model()
    self.utils.save_json_model(0)

  def load_weights(self, fn, by_name=False):
    self.utils.log("Loading weights")
    self.model.load_weights(fn, by_name=by_name)

  def get_optimizer_from_config(self):
      optimizer_name = self.config.optimizer["name"]
      args = []
      self.optimizer = getattr(optimizers, optimizer_name)(*args, **self.config.optimizer["params"])
      return self.optimizer
<commit_msg>Learning func-27-1-2 ./test/critiquepracticalreason_00_kant_64kb.c2cb-3200 --load-weights=out/func-27-1-2/weights-440.h5<commit_after>import keras as keras
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Dropout, TimeDistributed, Concatenate, Input, UpSampling2D
from keras.layers import GRU, LSTM, Conv2D, Conv1D, Reshape, Flatten, Permute, AveragePooling2D, MaxPooling2D, RepeatVector, Conv2DTranspose, LocallyConnected2D
import keras.optimizers as optimizers
import numpy as np

from custom_objects import CustomObjects

class ModelDef(object):

  layers=[]

  model = None
  utils = None
  started = False

  stateful = False

  encoder_trainable = True
  decoder_trainable = True
  generator_trainable = False

  def __init__(self, utils, config):
    self.utils = utils
    self.config = config
    self.models = {}
    self.layers = []

    self.utils.log("encoder_trainable:", self.encoder_trainable)
    self.utils.log("decoder_trainable:", self.decoder_trainable)
    self.utils.log("generator_trainable:", self.generator_trainable)

  def define_model(self, frame_seq_len, framelen, num_frame_seqs):
    self.utils.log("Defining model")
    config = self.config
    overlap_sequence = config.overlap_sequence
    short_input_len = frame_seq_len - overlap_sequence*2

    self.conv_count = 16
    enc_params = 20

    generator_trainable = self.generator_trainable

    print("short_input_len", short_input_len)

    main_input = Input(shape=(frame_seq_len, framelen), dtype='float32', name="main_input")
    # short_input = Input(shape=(short_input_len, framelen), dtype='float32', name="short_input")

    encoder_output = self.encoder_model(enc_params, shape=(frame_seq_len, framelen))(main_input)
    conf = self.encoder_model(framelen)
    print("encoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    encoder_output_len = conf.get_output_shape_at(0)[1]

    # Run the decoder portion of autoencoder
    mid_output = self.decoder_model(framelen, (encoder_output_len, enc_params))(encoder_output)

    conf = self.decoder_model(framelen)
    print("decoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    # Generator

    conf = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_0'
        , trainable=generator_trainable
    )
    res = conf(encoder_output)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_1'
        , trainable=generator_trainable
    )(res)

    res = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_postconcat'
        , trainable=generator_trainable
    )(res)

    conf = TimeDistributed(
        Dense(
          enc_params
          , activation="relu"
          , trainable=generator_trainable
        )
        , name='generator_TD_Dense_0'
    )
    res = generator_output = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # res = self.decoder_model(framelen, (-1, enc_params))(generator_output)

    conf = self.decoder_model(framelen)
    res = conf(res)
    print("decoder_model shapes for input / output 1")
    print(conf.get_input_shape_at(1))
    print(conf.get_output_shape_at(1))

    main_output = res

    model = Model(
        #inputs=[main_input, short_input],
        inputs=[main_input],
        outputs=[main_output, mid_output, generator_output, encoder_output]
    )

    self.model = model
    return model

  def encoder_model(self, enc_params, shape=(-1,0)):

    if self.models.get('encoder_model'):
      print("Reusing encoder model")
      return self.models.get('encoder_model')
    framelen = shape[1]
    in_scale = 2
    in_count = framelen * in_scale
    encoder_trainable = self.encoder_trainable

    conv_count = self.conv_count

    res = encoder_input = Input(shape=shape, dtype='float32', name="encoder_input")
    print("Encoder model input shape:", shape)

    res = keras.layers.concatenate([res, res])

    conf = TimeDistributed(keras.layers.Reshape((in_count, 1), trainable=encoder_trainable))
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,14), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,13), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = keras.layers.Reshape((-1, conv_count), trainable=encoder_trainable)(res)

    # Second chain - pooling input data parameter by parameter
    conf = TimeDistributed(keras.layers.Reshape((framelen, 1), trainable=encoder_trainable))
    shaped_input = conf(encoder_input)

    conf = AveragePooling2D(pool_size=(3,1), strides=(2,1), padding="valid", trainable=encoder_trainable)
    res_mean = conf(shaped_input)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)
    conf = AveragePooling2D(pool_size=(3,1), strides=(2,1), padding="valid", trainable=encoder_trainable)
    res_mean = conf(res_mean)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)
    conf = keras.layers.Reshape((-1, framelen), trainable=encoder_trainable)
    res_mean = conf(res_mean)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)
    # conf = TimeDistributed(
    #     Dense(
    #         enc_params
    #         , activation="relu"
    #         , trainable=encoder_trainable
    #         , name="encoder_post_average_dense"
    #     )
    # )
    # res_mean = conf(res_mean)
    # print(conf.get_config())
    # print(conf.input_shape)
    # print(conf.output_shape)

    res = keras.layers.concatenate([res, res_mean])

    conf = TimeDistributed(
        Dense(
            enc_params * 5
            , activation="relu"
            , trainable=encoder_trainable
            , name="encoder_concat_chains_dense"
        )
    )
    res = conf(res)

    conf = TimeDistributed(
        Dense(
            enc_params
            , activation="relu"
            , trainable=encoder_trainable
            , name="encoder_final_dense"
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    encoder_output = res

    self.models['encoder_model'] = Model(encoder_input, encoder_output)
    return self.models['encoder_model']

  # Shared Decoder model, taking Conv2D encoded input, or an attempt at predicted
  # sequences of the encoded data.
  #
  # Returns a single instance of a decoder model. May be called any number of times.
  #
  # Arguments:
  #  framelen: standard frame length for a timestep
  #  shape: two element tensor, typically (-1, enc_params), where enc_params is the
  #         width of the input timestep. -1 saves us having to enter the
  #         frame sequence (sub-batch) length
  #
  # Learning can be disabled by setting self.decoder_trainable = False
  #
  # Example:
  #    main_output = self.decoder_model(framelen, (-1, enc_params))(generator_output)
  def decoder_model(self, framelen, shape=(-1, 0)):

    if self.models.get('decoder_model'):
      return self.models.get('decoder_model')

    decoder_trainable = self.decoder_trainable
    enc_params = shape[1]
    conv_count = self.conv_count

    res = decoder_input = Input(shape=shape, dtype='float32', name="decoder_input")

    conf = TimeDistributed(
            keras.layers.Reshape(
              (1, enc_params)
              , trainable=decoder_trainable
            )
          )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,13),
              padding='valid',
              strides=(2,1),
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,14),
              strides=(2,1),
              padding='valid',
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )

    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Return a single filter pulling together the results of all conv_count filters
    conf = Conv2D(
              1,
              kernel_size=(2,14),
              #  strides=(2,1),
              padding='valid',
              activation='sigmoid',
              name='decoder_conv_squash',
              trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf  = keras.layers.Reshape((-1, framelen), trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Pass through the input

    conf = TimeDistributed(
        Dense(
            enc_params
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_pass_thru_dense_in'
        )
    )
    res_pt = conf(decoder_input)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(keras.layers.Reshape((enc_params, 1), trainable=decoder_trainable))
    res = conf(res_pt)

    conf = UpSampling2D(
        size=[4,1]
        , data_format='channels_last'
        , trainable=decoder_trainable
    )
    res_pt = conf(res_pt)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(keras.layers.Reshape((-1, enc_params), trainable=decoder_trainable))
    res = conf(res_pt)

    conf = TimeDistributed(
        Dense(
            framelen * 5
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_pass_thru_dense_wide'
        )
    )
    res_pt = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(
        Dense(
            framelen
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_pass_thru_dense'
        )
    )
    res_pt = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = keras.layers.concatenate([res, res_pt])

    conf = TimeDistributed(
        Dense(
            enc_params * 5
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_post_conv_dense'
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = TimeDistributed(
        Dense(
            framelen
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_final_dense'
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    decoder_output = res

    self.models['decoder_model'] = Model(decoder_input, decoder_output)
    return self.models['decoder_model']

  def compile_model(self):
    self.utils.log("Compiling model")

    loss = CustomObjects.codec2_param_error_td
    # other loss options: CustomObjects.codec2_param_mean_square_error; 'mean_absolute_error'; 'cosine_proximity'

    encoder_loss_prop = 0

    main_loss_prop = 0.2
    mid_loss_prop = 0.4
    generator_loss_prop = 0.4

    if self.decoder_trainable and not self.generator_trainable:
      main_loss_prop = 0
      mid_loss_prop = 1
      generator_loss_prop = 0
    elif not self.decoder_trainable and self.generator_trainable:
      mid_loss_prop = 0
      main_loss_prop = 0.0
      generator_loss_prop = 1.0

    self.utils.log("Loss weightings:", main_loss_prop, mid_loss_prop, generator_loss_prop, encoder_loss_prop)

    self.model.compile(
        loss=[loss, loss, 'mean_absolute_error', 'mean_absolute_error'],
        loss_weights=[main_loss_prop, mid_loss_prop, generator_loss_prop, encoder_loss_prop],
        optimizer=self.get_optimizer_from_config())
    self.utils.log_model_summary()

  def fit(self, input_seq, output_seq, batch_size=None, epochs=1, shuffle=False, callbacks=None):
      inputs = input_seq[0]
      outputs = output_seq

      # s = output_seq[0].shape
      # dummy_encoded_output = np.zeros((s[0], 24, 64), dtype=np.float32)
      # outputs.append( dummy_encoded_output)

      print("X shape:", inputs.shape)
      print("y shape:", outputs[0].shape, outputs[1].shape, outputs[2].shape, outputs[3].shape)

      self.model.fit(inputs, outputs, batch_size=batch_size, epochs=epochs, shuffle=shuffle,
       callbacks=callbacks
      )

  # we can wrap the model.add method, since in the future we may wish to
  # provide additional processing at this level
  def add_layer(self, layer):
    self.model.add(layer)
    return layer

  # start training GRU 1, then 1&2, then 3
  def before_iteration(self, iteration):
#    if iteration == 541:
#      self.utils.log("Adding frame rotation to reduce memory usage")
#      self.config.limit_frames = self.config.num_frames / 100
#      self.model_updates_lstm_1234_trainable()
#      self.config.log_attrs()
#
#    elif iteration == 481:
#      self.model_updates_lstm3_trainable()
#

    if not self.started:
      self.model_updates_onstart()

    self.started = True

  def model_updates_onstart(self):
#    self.utils.log("Make all lstms trainable")
#    self.model.layers[0].trainable=True
#    self.model.layers[1].trainable=True
#    self.model.layers[2].trainable=True
#    self.model.layers[3].trainable=True
#    self.model.layers[4].trainable=True

    self.compile_model()
    self.utils.save_json_model(0)

  def load_weights(self, fn, by_name=False):
    self.utils.log("Loading weights")
    self.model.load_weights(fn, by_name=by_name)

  def get_optimizer_from_config(self):
      optimizer_name = self.config.optimizer["name"]
      args = []
      self.optimizer = getattr(optimizers, optimizer_name)(*args, **self.config.optimizer["params"])
      return self.optimizer
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012, David "DaviXX" CHANIAL <david.chanial@gmail.com>
# (c) 2014, James Tanner <tanner.jc@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import absolute_import, division, print_function
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['stableinterface'],
                    'supported_by': 'core'}

DOCUMENTATION = '''
---
module: sysctl
short_description: Manage entries in sysctl.conf.
description:
    - This module manipulates sysctl entries and optionally performs a C(/sbin/sysctl -p) after changing them.
version_added: "1.0"
options:
    name:
        description:
            - The dot-separated path (aka I(key)) specifying the sysctl variable.
        required: true
        aliases: [ 'key' ]
    value:
        description:
            - Desired value of the sysctl key.
        aliases: [ 'val' ]
    state:
        description:
            - Whether the entry should be present or absent in the sysctl file.
        choices: [ "present", "absent" ]
        default: present
    ignoreerrors:
        description:
            - Use this option to ignore errors about unknown keys.
        type: bool
        default: 'no'
    reload:
        description:
            - If C(yes), performs a I(/sbin/sysctl -p) if the C(sysctl_file) is
              updated. If C(no), does not reload I(sysctl) even if the
              C(sysctl_file) is updated.
        type: bool
        default: 'yes'
    sysctl_file:
        description:
            - Specifies the absolute path to C(sysctl.conf), if not C(/etc/sysctl.conf).
        default: /etc/sysctl.conf
    sysctl_set:
        description:
            - Verify token value with the sysctl command and set with -w if necessary
        type: bool
        default: 'no'
        version_added: 1.5
author: "David CHANIAL (@davixx) <david.chanial@gmail.com>"
'''

EXAMPLES = '''
# Set vm.swappiness to 5 in /etc/sysctl.conf
- sysctl:
    name: vm.swappiness
    value: 5
    state: present

# Remove kernel.panic entry from /etc/sysctl.conf
- sysctl:
    name: kernel.panic
    state: absent
    sysctl_file: /etc/sysctl.conf

# Set kernel.panic to 3 in /tmp/test_sysctl.conf
- sysctl:
    name: kernel.panic
    value: 3
    sysctl_file: /tmp/test_sysctl.conf
    reload: no

# Set ip forwarding on in /proc and do not reload the sysctl file
- sysctl:
    name: net.ipv4.ip_forward
    value: 1
    sysctl_set: yes

# Set ip forwarding on in /proc and in the sysctl file and reload if necessary
- sysctl:
    name: net.ipv4.ip_forward
    value: 1
    sysctl_set: yes
    state: present
    reload: yes
'''

# ==============================================================

import os
import tempfile

from ansible.module_utils.basic import get_platform, AnsibleModule
from ansible.module_utils.six import string_types
from ansible.module_utils.parsing.convert_bool import BOOLEANS_FALSE, BOOLEANS_TRUE
from ansible.module_utils._text import to_native

class SysctlModule(object):

    def __init__(self, module):
        self.module = module
        self.args = self.module.params

        self.sysctl_cmd = self.module.get_bin_path('sysctl', required=True)
        self.sysctl_file = self.args['sysctl_file']

        self.proc_value = None  # current token value in proc fs
        self.file_value = None  # current token value in file
        self.file_lines = []    # all lines in the file
        self.file_values = {}   # dict of token values

        self.changed = False    # will change occur
        self.set_proc = False   # does sysctl need to set value
        self.write_file = False  # does the sysctl file need to be reloaded

        self.process()

    # ==============================================================
    #   LOGIC
    # ==============================================================

    def process(self):

        self.platform = get_platform().lower()

        # Whitespace is bad
        self.args['name'] = self.args['name'].strip()
        self.args['value'] = self._parse_value(self.args['value'])

        thisname = self.args['name']

        # get the current proc fs value
        self.proc_value = self.get_token_curr_value(thisname)

        # get the currect sysctl file value
        self.read_sysctl_file()
        if thisname not in self.file_values:
            self.file_values[thisname] = None

        # update file contents with desired token/value
        self.fix_lines()

        # what do we need to do now?
        if self.file_values[thisname] is None and self.args['state'] == "present":
            self.changed = True
            self.write_file = True
        elif self.file_values[thisname] is None and self.args['state'] == "absent":
            self.changed = False
        elif self.file_values[thisname] and self.args['state'] == "absent":
            self.changed = True
            self.write_file = True
        elif self.file_values[thisname] != self.args['value']:
            self.changed = True
            self.write_file = True

        # use the sysctl command or not?
        if self.args['sysctl_set']:
            if self.proc_value is None:
                self.changed = True
            elif not self._values_is_equal(self.proc_value, self.args['value']):
                self.changed = True
                self.set_proc = True

        # Do the work
        if not self.module.check_mode:
            if self.write_file:
                self.write_sysctl()
            if self.write_file and self.args['reload']:
                self.reload_sysctl()
            if self.set_proc:
                self.set_token_value(self.args['name'], self.args['value'])

    def _values_is_equal(self, a, b):
        """Expects two string values. It will split the string by whitespace
        and compare each value. It will return True if both lists are the same,
        contain the same elements and the same order."""
        if a is None or b is None:
            return False

        a = a.split()
        b = b.split()

        if len(a) != len(b):
            return False

        return len([i for i, j in zip(a, b) if i == j]) == len(a)

    def _parse_value(self, value):
        if value is None:
            return ''
        elif isinstance(value, bool):
            if value:
                return '1'
            else:
                return '0'
        elif isinstance(value, string_types):
            if value.lower() in BOOLEANS_TRUE:
                return '1'
            elif value.lower() in BOOLEANS_FALSE:
                return '0'
            else:
                return value.strip()
        else:
            return value

    # ==============================================================
    #   SYSCTL COMMAND MANAGEMENT
    # ==============================================================

    # Use the sysctl command to find the current value
    def get_token_curr_value(self, token):
        if self.platform == 'openbsd':
            # openbsd doesn't support -e, just drop it
            thiscmd = "%s -n %s" % (self.sysctl_cmd, token)
        else:
            thiscmd = "%s -e -n %s" % (self.sysctl_cmd, token)
        rc, out, err = self.module.run_command(thiscmd)
        if rc != 0:
            return None
        else:
            return out

    # Use the sysctl command to set the current value
    def set_token_value(self, token, value):
        if len(value.split()) > 0:
            value = '"' + value + '"'
        if self.platform == 'openbsd':
            # openbsd doesn't accept -w, but since it's not needed, just drop it
            thiscmd = "%s %s=%s" % (self.sysctl_cmd, token, value)
        elif self.platform == 'freebsd':
            ignore_missing = ''
            if self.args['ignoreerrors']:
                ignore_missing = '-i'
            # freebsd doesn't accept -w, but since it's not needed, just drop it
            thiscmd = "%s %s %s=%s" % (self.sysctl_cmd, ignore_missing, token, value)
        else:
            ignore_missing = ''
            if self.args['ignoreerrors']:
                ignore_missing = '-e'
            thiscmd = "%s %s -w %s=%s" % (self.sysctl_cmd, ignore_missing, token, value)
        rc, out, err = self.module.run_command(thiscmd)
        if rc != 0:
            self.module.fail_json(msg='setting %s failed: %s' % (token, out + err))
        else:
            return rc

    # Run sysctl -p
    def reload_sysctl(self):
        # do it
        if self.platform == 'freebsd':
            # freebsd doesn't support -p, so reload the sysctl service
            rc, out, err = self.module.run_command('/etc/rc.d/sysctl reload')
        elif self.platform == 'openbsd':
            # openbsd doesn't support -p and doesn't have a sysctl service,
            # so we have to set every value with its own sysctl call
            for k, v in self.file_values.items():
                rc = 0
                if k != self.args['name']:
                    rc = self.set_token_value(k, v)
                    if rc != 0:
                        break
            if rc == 0 and self.args['state'] == "present":
                rc = self.set_token_value(self.args['name'], self.args['value'])
        else:
            # system supports reloading via the -p flag to sysctl, so we'll use that
            sysctl_args = [self.sysctl_cmd, '-p', self.sysctl_file]
            if self.args['ignoreerrors']:
                sysctl_args.insert(1, '-e')

            rc, out, err = self.module.run_command(sysctl_args)

        if rc != 0:
            self.module.fail_json(msg="Failed to reload sysctl: %s" % to_native(out) + to_native(err))

    # ==============================================================
    #   SYSCTL FILE MANAGEMENT
    # ==============================================================

    # Get the token value from the sysctl file
    def read_sysctl_file(self):

        lines = []
        if os.path.isfile(self.sysctl_file):
            try:
                with open(self.sysctl_file, "r") as read_file:
                    lines = read_file.readlines()
            except IOError as e:
                self.module.fail_json(msg="Failed to open %s: %s" % (to_native(self.sysctl_file), to_native(e)))

        for line in lines:
            line = line.strip()
            self.file_lines.append(line)

            # don't split empty lines or comments or line without equal sign
            if not line or line.startswith(("#", ";")) or "=" not in line:
                continue

            k, v = line.split('=', 1)
            k = k.strip()
            v = v.strip()
            self.file_values[k] = v.strip()

    # Fix the value in the sysctl file content
    def fix_lines(self):
        checked = []
        self.fixed_lines = []
        for line in self.file_lines:
            if not line.strip() or line.strip().startswith(("#", ";")) or "=" not in line:
                self.fixed_lines.append(line)
                continue
            tmpline = line.strip()
            k, v = tmpline.split('=', 1)
            k = k.strip()
            v = v.strip()
            if k not in checked:
                checked.append(k)
                if k == self.args['name']:
                    if self.args['state'] == "present":
                        new_line = "%s=%s\n" % (k, self.args['value'])
                        self.fixed_lines.append(new_line)
                else:
                    new_line = "%s=%s\n" % (k, v)
                    self.fixed_lines.append(new_line)

        if self.args['name'] not in checked and self.args['state'] == "present":
            new_line = "%s=%s\n" % (self.args['name'], self.args['value'])
            self.fixed_lines.append(new_line)

    # Completely rewrite the sysctl file
    def write_sysctl(self):
        # open a tmp file
        fd, tmp_path = tempfile.mkstemp('.conf', '.ansible_m_sysctl_', os.path.dirname(self.sysctl_file))
        f = open(tmp_path, "w")
        try:
            for l in self.fixed_lines:
                f.write(l.strip() + "\n")
        except IOError as e:
            self.module.fail_json(msg="Failed to write to file %s: %s" % (tmp_path, to_native(e)))
        f.flush()
        f.close()

        # replace the real one
        self.module.atomic_move(tmp_path, self.sysctl_file)

# ==============================================================
# main

def main():

    # defining module
    module = AnsibleModule(
        argument_spec=dict(
            name=dict(aliases=['key'], required=True),
            value=dict(aliases=['val'], required=False, type='str'),
            state=dict(default='present', choices=['present', 'absent']),
            reload=dict(default=True, type='bool'),
            sysctl_set=dict(default=False, type='bool'),
            ignoreerrors=dict(default=False, type='bool'),
            sysctl_file=dict(default='/etc/sysctl.conf', type='path')
        ),
        supports_check_mode=True,
        required_if=[('state', 'present', ['value'])],
    )

    if module.params['name'] is None:
        module.fail_json(msg="name cannot be None")
    if module.params['state'] == 'present' and module.params['value'] is None:
        module.fail_json(msg="value cannot be None")

    # In case of in-line params
    if module.params['name'] == '':
        module.fail_json(msg="name cannot be blank")
    if module.params['state'] == 'present' and module.params['value'] == '':
        module.fail_json(msg="value cannot be blank")

    result = SysctlModule(module)

    module.exit_json(changed=result.changed)

if __name__ == '__main__':
    main()
<commit_msg>Fixes the comment in the example of the sysctl module(#51547)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012, David "DaviXX" CHANIAL <david.chanial@gmail.com>
# (c) 2014, James Tanner <tanner.jc@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import absolute_import, division, print_function
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['stableinterface'],
                    'supported_by': 'core'}

DOCUMENTATION = '''
---
module: sysctl
short_description: Manage entries in sysctl.conf.
description:
    - This module manipulates sysctl entries and optionally performs a C(/sbin/sysctl -p) after changing them.
version_added: "1.0"
options:
    name:
        description:
            - The dot-separated path (aka I(key)) specifying the sysctl variable.
        required: true
        aliases: [ 'key' ]
    value:
        description:
            - Desired value of the sysctl key.
        aliases: [ 'val' ]
    state:
        description:
            - Whether the entry should be present or absent in the sysctl file.
        choices: [ "present", "absent" ]
        default: present
    ignoreerrors:
        description:
            - Use this option to ignore errors about unknown keys.
        type: bool
        default: 'no'
    reload:
        description:
            - If C(yes), performs a I(/sbin/sysctl -p) if the C(sysctl_file) is
              updated. If C(no), does not reload I(sysctl) even if the
              C(sysctl_file) is updated.
        type: bool
        default: 'yes'
    sysctl_file:
        description:
            - Specifies the absolute path to C(sysctl.conf), if not C(/etc/sysctl.conf).
        default: /etc/sysctl.conf
    sysctl_set:
        description:
            - Verify token value with the sysctl command and set with -w if necessary
        type: bool
        default: 'no'
        version_added: 1.5
author: "David CHANIAL (@davixx) <david.chanial@gmail.com>"
'''

EXAMPLES = '''
# Set vm.swappiness to 5 in /etc/sysctl.conf
- sysctl:
    name: vm.swappiness
    value: 5
    state: present

# Remove kernel.panic entry from /etc/sysctl.conf
- sysctl:
    name: kernel.panic
    state: absent
    sysctl_file: /etc/sysctl.conf

# Set kernel.panic to 3 in /tmp/test_sysctl.conf
- sysctl:
    name: kernel.panic
    value: 3
    sysctl_file: /tmp/test_sysctl.conf
    reload: no

# Set ip forwarding on in /proc and verify token value with the sysctl command
- sysctl:
    name: net.ipv4.ip_forward
    value: 1
    sysctl_set: yes

# Set ip forwarding on in /proc and in the sysctl file and reload if necessary
- sysctl:
    name: net.ipv4.ip_forward
    value: 1
    sysctl_set: yes
    state: present
    reload: yes
'''

# ==============================================================

import os
import tempfile

from ansible.module_utils.basic import get_platform, AnsibleModule
from ansible.module_utils.six import string_types
from ansible.module_utils.parsing.convert_bool import BOOLEANS_FALSE, BOOLEANS_TRUE
from ansible.module_utils._text import to_native

class SysctlModule(object):

    def __init__(self, module):
        self.module = module
        self.args = self.module.params

        self.sysctl_cmd = self.module.get_bin_path('sysctl', required=True)
        self.sysctl_file = self.args['sysctl_file']

        self.proc_value = None  # current token value in proc fs
        self.file_value = None  # current token value in file
        self.file_lines = []    # all lines in the file
        self.file_values = {}   # dict of token values

        self.changed = False    # will change occur
        self.set_proc = False   # does sysctl need to set value
        self.write_file = False  # does the sysctl file need to be reloaded

        self.process()

    # ==============================================================
    #   LOGIC
    # ==============================================================

    def process(self):

        self.platform = get_platform().lower()

        # Whitespace is bad
        self.args['name'] = self.args['name'].strip()
        self.args['value'] = self._parse_value(self.args['value'])

        thisname = self.args['name']

        # get the current proc fs value
        self.proc_value = self.get_token_curr_value(thisname)

        # get the currect sysctl file value
        self.read_sysctl_file()
        if thisname not in self.file_values:
            self.file_values[thisname] = None

        # update file contents with desired token/value
        self.fix_lines()

        # what do we need to do now?
        if self.file_values[thisname] is None and self.args['state'] == "present":
            self.changed = True
            self.write_file = True
        elif self.file_values[thisname] is None and self.args['state'] == "absent":
            self.changed = False
        elif self.file_values[thisname] and self.args['state'] == "absent":
            self.changed = True
            self.write_file = True
        elif self.file_values[thisname] != self.args['value']:
            self.changed = True
            self.write_file = True

        # use the sysctl command or not?
        if self.args['sysctl_set']:
            if self.proc_value is None:
                self.changed = True
            elif not self._values_is_equal(self.proc_value, self.args['value']):
                self.changed = True
                self.set_proc = True

        # Do the work
        if not self.module.check_mode:
            if self.write_file:
                self.write_sysctl()
            if self.write_file and self.args['reload']:
                self.reload_sysctl()
            if self.set_proc:
                self.set_token_value(self.args['name'], self.args['value'])

    def _values_is_equal(self, a, b):
        """Expects two string values. It will split the string by whitespace
        and compare each value. It will return True if both lists are the same,
        contain the same elements and the same order."""
        if a is None or b is None:
            return False

        a = a.split()
        b = b.split()

        if len(a) != len(b):
            return False

        return len([i for i, j in zip(a, b) if i == j]) == len(a)

    def _parse_value(self, value):
        if value is None:
            return ''
        elif isinstance(value, bool):
            if value:
                return '1'
            else:
                return '0'
        elif isinstance(value, string_types):
            if value.lower() in BOOLEANS_TRUE:
                return '1'
            elif value.lower() in BOOLEANS_FALSE:
                return '0'
            else:
                return value.strip()
        else:
            return value

    # ==============================================================
    #   SYSCTL COMMAND MANAGEMENT
    # ==============================================================

    # Use the sysctl command to find the current value
    def get_token_curr_value(self, token):
        if self.platform == 'openbsd':
            # openbsd doesn't support -e, just drop it
            thiscmd = "%s -n %s" % (self.sysctl_cmd, token)
        else:
            thiscmd = "%s -e -n %s" % (self.sysctl_cmd, token)
        rc, out, err = self.module.run_command(thiscmd)
        if rc != 0:
            return None
        else:
            return out

    # Use the sysctl command to set the current value
    def set_token_value(self, token, value):
        if len(value.split()) > 0:
            value = '"' + value + '"'
        if self.platform == 'openbsd':
            # openbsd doesn't accept -w, but since it's not needed, just drop it
            thiscmd = "%s %s=%s" % (self.sysctl_cmd, token, value)
        elif self.platform == 'freebsd':
            ignore_missing = ''
            if self.args['ignoreerrors']:
                ignore_missing = '-i'
            # freebsd doesn't accept -w, but since it's not needed, just drop it
            thiscmd = "%s %s %s=%s" % (self.sysctl_cmd, ignore_missing, token, value)
        else:
            ignore_missing = ''
            if self.args['ignoreerrors']:
                ignore_missing = '-e'
            thiscmd = "%s %s -w %s=%s" % (self.sysctl_cmd, ignore_missing, token, value)
        rc, out, err = self.module.run_command(thiscmd)
        if rc != 0:
            self.module.fail_json(msg='setting %s failed: %s' % (token, out + err))
        else:
            return rc

    # Run sysctl -p
    def reload_sysctl(self):
        # do it
        if self.platform == 'freebsd':
            # freebsd doesn't support -p, so reload the sysctl service
            rc, out, err = self.module.run_command('/etc/rc.d/sysctl reload')
        elif self.platform == 'openbsd':
            # openbsd doesn't support -p and doesn't have a sysctl service,
            # so we have to set every value with its own sysctl call
            for k, v in self.file_values.items():
                rc = 0
                if k != self.args['name']:
                    rc = self.set_token_value(k, v)
                    if rc != 0:
                        break
            if rc == 0 and self.args['state'] == "present":
                rc = self.set_token_value(self.args['name'], self.args['value'])
        else:
            # system supports reloading via the -p flag to sysctl, so we'll use that
            sysctl_args = [self.sysctl_cmd, '-p', self.sysctl_file]
            if self.args['ignoreerrors']:
                sysctl_args.insert(1, '-e')

            rc, out, err = self.module.run_command(sysctl_args)

        if rc != 0:
            self.module.fail_json(msg="Failed to reload sysctl: %s" % to_native(out) + to_native(err))

    # ==============================================================
    #   SYSCTL FILE MANAGEMENT
    # ==============================================================

    # Get the token value from the sysctl file
    def read_sysctl_file(self):

        lines = []
        if os.path.isfile(self.sysctl_file):
            try:
                with open(self.sysctl_file, "r") as read_file:
                    lines = read_file.readlines()
            except IOError as e:
                self.module.fail_json(msg="Failed to open %s: %s" % (to_native(self.sysctl_file), to_native(e)))

        for line in lines:
            line = line.strip()
            self.file_lines.append(line)

            # don't split empty lines or comments or line without equal sign
            if not line or line.startswith(("#", ";")) or "=" not in line:
                continue

            k, v = line.split('=', 1)
            k = k.strip()
            v = v.strip()
            self.file_values[k] = v.strip()

    # Fix the value in the sysctl file content
    def fix_lines(self):
        checked = []
        self.fixed_lines = []
        for line in self.file_lines:
            if not line.strip() or line.strip().startswith(("#", ";")) or "=" not in line:
                self.fixed_lines.append(line)
                continue
            tmpline = line.strip()
            k, v = tmpline.split('=', 1)
            k = k.strip()
            v = v.strip()
            if k not in checked:
                checked.append(k)
                if k == self.args['name']:
                    if self.args['state'] == "present":
                        new_line = "%s=%s\n" % (k, self.args['value'])
                        self.fixed_lines.append(new_line)
                else:
                    new_line = "%s=%s\n" % (k, v)
                    self.fixed_lines.append(new_line)

        if self.args['name'] not in checked and self.args['state'] == "present":
            new_line = "%s=%s\n" % (self.args['name'], self.args['value'])
            self.fixed_lines.append(new_line)

    # Completely rewrite the sysctl file
    def write_sysctl(self):
        # open a tmp file
        fd, tmp_path = tempfile.mkstemp('.conf', '.ansible_m_sysctl_', os.path.dirname(self.sysctl_file))
        f = open(tmp_path, "w")
        try:
            for l in self.fixed_lines:
                f.write(l.strip() + "\n")
        except IOError as e:
            self.module.fail_json(msg="Failed to write to file %s: %s" % (tmp_path, to_native(e)))
        f.flush()
        f.close()

        # replace the real one
        self.module.atomic_move(tmp_path, self.sysctl_file)

# ==============================================================
# main

def main():

    # defining module
    module = AnsibleModule(
        argument_spec=dict(
            name=dict(aliases=['key'], required=True),
            value=dict(aliases=['val'], required=False, type='str'),
            state=dict(default='present', choices=['present', 'absent']),
            reload=dict(default=True, type='bool'),
            sysctl_set=dict(default=False, type='bool'),
            ignoreerrors=dict(default=False, type='bool'),
            sysctl_file=dict(default='/etc/sysctl.conf', type='path')
        ),
        supports_check_mode=True,
        required_if=[('state', 'present', ['value'])],
    )

    if module.params['name'] is None:
        module.fail_json(msg="name cannot be None")
    if module.params['state'] == 'present' and module.params['value'] is None:
        module.fail_json(msg="value cannot be None")

    # In case of in-line params
    if module.params['name'] == '':
        module.fail_json(msg="name cannot be blank")
    if module.params['state'] == 'present' and module.params['value'] == '':
        module.fail_json(msg="value cannot be blank")

    result = SysctlModule(module)

    module.exit_json(changed=result.changed)

if __name__ == '__main__':
    main()
<|endoftext|>