<commit_before>import keras as keras
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Dropout, TimeDistributed, Concatenate, Input
from keras.layers import GRU, LSTM, Conv2D, Conv1D, Reshape, Flatten, Permute, AveragePooling2D, MaxPooling2D, RepeatVector, Conv2DTranspose
import keras.optimizers as optimizers
import numpy as np

from custom_objects import CustomObjects

class ModelDef(object):

  layers=[]

  model = None
  utils = None
  started = False

  stateful = False

  encoder_trainable = True
  decoder_trainable = True
  generator_trainable = False

  def __init__(self, utils, config):
    self.utils = utils
    self.config = config
    self.models = {}
    self.layers = []

    self.utils.log("encoder_trainable:", self.encoder_trainable)
    self.utils.log("decoder_trainable:", self.decoder_trainable)
    self.utils.log("generator_trainable:", self.generator_trainable)

  def define_model(self, frame_seq_len, framelen, num_frame_seqs):
    self.utils.log("Defining model")
    config = self.config
    overlap_sequence = config.overlap_sequence
    short_input_len = frame_seq_len - overlap_sequence*2

    self.conv_count = 64
    enc_params = 64

    generator_trainable = self.generator_trainable

    print("short_input_len", short_input_len)

    main_input = Input(shape=(frame_seq_len, framelen), dtype='float32', name="main_input")
    # short_input = Input(shape=(short_input_len, framelen), dtype='float32', name="short_input")

    encoder_output = self.encoder_model(enc_params, shape=(frame_seq_len, framelen))(main_input)
    conf = self.encoder_model(framelen)
    print("encoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    encoder_output_len = conf.get_output_shape_at(0)[1]

    # Run the decoder portion of autoencoder
    mid_output = self.decoder_model(framelen, (encoder_output_len, enc_params))(encoder_output)

    conf = self.decoder_model(framelen)
    print("decoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    # Generator

    conf = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_0'
        , trainable=generator_trainable
    )
    res = conf(encoder_output)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_1'
        , trainable=generator_trainable
    )(res)

    conf = TimeDistributed(
        Dense(
          enc_params
          , activation="relu"
          , trainable=generator_trainable
        )
        , name='generator_TD_Dense_0'
    )
    res = generator_output = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # res = self.decoder_model(framelen, (-1, enc_params))(generator_output)

    conf = self.decoder_model(framelen)
    res = conf(res)
    print("decoder_model shapes for input / output 1")
    print(conf.get_input_shape_at(1))
    print(conf.get_output_shape_at(1))

    main_output = res

    model = Model(
        #inputs=[main_input, short_input],
        inputs=[main_input],
        outputs=[main_output, mid_output, generator_output]
    )

    self.model = model
    return model

  def encoder_model(self, enc_params, shape=(-1,0)):

    if self.models.get('encoder_model'):
      print("Reusing encoder model")
      return self.models.get('encoder_model')
    framelen = shape[1]
    in_scale = 2
    in_count = framelen * in_scale
    encoder_trainable = self.encoder_trainable

    conv_count = self.conv_count

    res = encoder_input = Input(shape=shape, dtype='float32', name="encoder_input")
    print("Encoder model input shape:", shape)

    res = keras.layers.concatenate([res, res])

    conf = TimeDistributed(keras.layers.Reshape((in_count, 1), trainable=encoder_trainable))
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,14), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,13), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = keras.layers.Reshape((-1, conv_count), trainable=encoder_trainable)(res)

    conf = TimeDistributed(
        Dense(
            enc_params
            , activation="relu"
            , trainable=encoder_trainable
            , name="encoder_final_dense"
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    encoder_output = res

    self.models['encoder_model'] = Model(encoder_input, encoder_output)
    return self.models['encoder_model']

  # Shared Decoder model, taking Conv2D encoded input, or an attempt at predicted
  # sequences of the encoded data.
  #
  # Returns a single instance of a decoder model. May be called any number of times.
  #
  # Arguments:
  #  framelen: standard frame length for a timestep
  #  shape: two element tensor, typically (-1, enc_params), where enc_params is the
  #         width of the input timestep. -1 saves us having to enter the
  #         frame sequence (sub-batch) length
  #
  # Learning can be disabled by setting self.decoder_trainable = False
  #
  # Example:
  #    main_output = self.decoder_model(framelen, (-1, enc_params))(generator_output)
  def decoder_model(self, framelen, shape=(-1, 0)):

    if self.models.get('decoder_model'):
      return self.models.get('decoder_model')

    decoder_trainable = self.decoder_trainable
    enc_params = shape[1]
    conv_count = self.conv_count

    res = decoder_input = Input(shape=shape, dtype='float32', name="decoder_input")

    conf = TimeDistributed(
            keras.layers.Reshape(
              (1, enc_params)
              , trainable=decoder_trainable
            )
          )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,13),
              padding='valid',
              strides=(2,1),
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,14),
              strides=(2,1),
              padding='valid',
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )

    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Return a single filter pulling together the results of all conv_count filters
    conf = Conv2D(
              1,
              kernel_size=(2,14),
              #  strides=(2,1),
              padding='valid',
              activation='sigmoid',
              name='decoder_conv_squash',
              trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf  = keras.layers.Reshape((-1, framelen), trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Trying to reduce autoencoder loss below 0.78
    res = LSTM(
        framelen * 3
        , return_sequences=True
        , trainable=decoder_trainable
    )(res)

    conf = TimeDistributed(
        Dense(
            framelen
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_final_dense'
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    decoder_output = res

    self.models['decoder_model'] = Model(decoder_input, decoder_output)
    return self.models['decoder_model']

  def compile_model(self):
    self.utils.log("Compiling model")

    loss = CustomObjects.codec2_param_error_td
    # other loss options: CustomObjects.codec2_param_mean_square_error; 'mean_absolute_error'; 'cosine_proximity'

    main_loss_prop = 0.2
    mid_loss_prop = 0.4
    generator_loss_prop = 0.4

    if self.decoder_trainable and not self.generator_trainable:
      main_loss_prop = 0
      mid_loss_prop = 1
      generator_loss_prop = 0
    elif not self.decoder_trainable and self.generator_trainable:
      mid_loss_prop = 0
      generator_loss_prop = 0.5
      main_loss_prop = 0.5

    self.utils.log("Loss weightings:", main_loss_prop, mid_loss_prop, generator_loss_prop)

    self.model.compile(
        loss=[loss, loss, 'mean_squared_error'],
        loss_weights=[main_loss_prop, mid_loss_prop, generator_loss_prop],
        optimizer=self.get_optimizer_from_config())
    self.utils.log_model_summary()

  def fit(self, input_seq, output_seq, batch_size=None, epochs=1, shuffle=False, callbacks=None):
      inputs = input_seq[0]
      outputs = output_seq

      # s = output_seq[0].shape
      # dummy_encoded_output = np.zeros((s[0], 24, 64), dtype=np.float32)
      # outputs.append( dummy_encoded_output)

      print("X shape:", inputs.shape)
      print("y shape:", outputs[0].shape, outputs[1].shape, outputs[2].shape)

      self.model.fit(inputs, outputs, batch_size=batch_size, epochs=epochs, shuffle=shuffle,
       callbacks=callbacks
      )

  # we can wrap the model.add method, since in the future we may wish to
  # provide additional processing at this level
  def add_layer(self, layer):
    self.model.add(layer)
    return layer

  # start training GRU 1, then 1&2, then 3
  def before_iteration(self, iteration):
#    if iteration == 541:
#      self.utils.log("Adding frame rotation to reduce memory usage")
#      self.config.limit_frames = self.config.num_frames / 100
#      self.model_updates_lstm_1234_trainable()
#      self.config.log_attrs()
#
#    elif iteration == 481:
#      self.model_updates_lstm3_trainable()
#

    if not self.started:
      self.model_updates_onstart()

    self.started = True

  def model_updates_onstart(self):
#    self.utils.log("Make all lstms trainable")
#    self.model.layers[0].trainable=True
#    self.model.layers[1].trainable=True
#    self.model.layers[2].trainable=True
#    self.model.layers[3].trainable=True
#    self.model.layers[4].trainable=True

    self.compile_model()
    self.utils.save_json_model(0)

  def load_weights(self, fn, by_name=False):
    self.utils.log("Loading weights")
    self.model.load_weights(fn, by_name=by_name)

  def get_optimizer_from_config(self):
      optimizer_name = self.config.optimizer["name"]
      args = []
      self.optimizer = getattr(optimizers, optimizer_name)(*args, **self.config.optimizer["params"])
      return self.optimizer
<commit_msg>Learning func-26-1-8 ./test/critiquepracticalreason_00_kant_64kb.c2cb-3200 --load-weights=out/func-26-1-8/weights-1482.h5<commit_after>import keras as keras
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Dropout, TimeDistributed, Concatenate, Input
from keras.layers import GRU, LSTM, Conv2D, Conv1D, Reshape, Flatten, Permute, AveragePooling2D, MaxPooling2D, RepeatVector, Conv2DTranspose
import keras.optimizers as optimizers
import numpy as np

from custom_objects import CustomObjects

class ModelDef(object):

  layers=[]

  model = None
  utils = None
  started = False

  stateful = False

  encoder_trainable = True
  decoder_trainable = True
  generator_trainable = False

  def __init__(self, utils, config):
    self.utils = utils
    self.config = config
    self.models = {}
    self.layers = []

    self.utils.log("encoder_trainable:", self.encoder_trainable)
    self.utils.log("decoder_trainable:", self.decoder_trainable)
    self.utils.log("generator_trainable:", self.generator_trainable)

  def define_model(self, frame_seq_len, framelen, num_frame_seqs):
    self.utils.log("Defining model")
    config = self.config
    overlap_sequence = config.overlap_sequence
    short_input_len = frame_seq_len - overlap_sequence*2

    self.conv_count = 64
    enc_params = 64

    generator_trainable = self.generator_trainable

    print("short_input_len", short_input_len)

    main_input = Input(shape=(frame_seq_len, framelen), dtype='float32', name="main_input")
    # short_input = Input(shape=(short_input_len, framelen), dtype='float32', name="short_input")

    encoder_output = self.encoder_model(enc_params, shape=(frame_seq_len, framelen))(main_input)
    conf = self.encoder_model(framelen)
    print("encoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    encoder_output_len = conf.get_output_shape_at(0)[1]

    # Run the decoder portion of autoencoder
    mid_output = self.decoder_model(framelen, (encoder_output_len, enc_params))(encoder_output)

    conf = self.decoder_model(framelen)
    print("decoder_model shapes for input / output 0")
    print(conf.get_input_shape_at(0))
    print(conf.get_output_shape_at(0))

    # Generator

    conf = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_0'
        , trainable=generator_trainable
    )
    res = conf(encoder_output)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = LSTM(
        128
        , return_sequences=True
        , name='generator_LSTM_1'
        , trainable=generator_trainable
    )(res)

    conf = TimeDistributed(
        Dense(
          enc_params
          , activation="relu"
          , trainable=generator_trainable
        )
        , name='generator_TD_Dense_0'
    )
    res = generator_output = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # res = self.decoder_model(framelen, (-1, enc_params))(generator_output)

    conf = self.decoder_model(framelen)
    res = conf(res)
    print("decoder_model shapes for input / output 1")
    print(conf.get_input_shape_at(1))
    print(conf.get_output_shape_at(1))

    main_output = res

    model = Model(
        #inputs=[main_input, short_input],
        inputs=[main_input],
        outputs=[main_output, mid_output, generator_output]
    )

    self.model = model
    return model

  def encoder_model(self, enc_params, shape=(-1,0)):

    if self.models.get('encoder_model'):
      print("Reusing encoder model")
      return self.models.get('encoder_model')
    framelen = shape[1]
    in_scale = 2
    in_count = framelen * in_scale
    encoder_trainable = self.encoder_trainable

    conv_count = self.conv_count

    res = encoder_input = Input(shape=shape, dtype='float32', name="encoder_input")
    print("Encoder model input shape:", shape)

    res = keras.layers.concatenate([res, res])

    conf = TimeDistributed(keras.layers.Reshape((in_count, 1), trainable=encoder_trainable))
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,14), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2D(conv_count, (3,13), strides=(2,1), padding='valid', data_format='channels_last', activation='relu', trainable=encoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    res = keras.layers.Reshape((-1, conv_count), trainable=encoder_trainable)(res)

    conf = TimeDistributed(
        Dense(
            enc_params
            , activation="relu"
            , trainable=encoder_trainable
            , name="encoder_final_dense"
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    encoder_output = res

    self.models['encoder_model'] = Model(encoder_input, encoder_output)
    return self.models['encoder_model']

  # Shared Decoder model, taking Conv2D encoded input, or an attempt at predicted
  # sequences of the encoded data.
  #
  # Returns a single instance of a decoder model. May be called any number of times.
  #
  # Arguments:
  #  framelen: standard frame length for a timestep
  #  shape: two element tensor, typically (-1, enc_params), where enc_params is the
  #         width of the input timestep. -1 saves us having to enter the
  #         frame sequence (sub-batch) length
  #
  # Learning can be disabled by setting self.decoder_trainable = False
  #
  # Example:
  #    main_output = self.decoder_model(framelen, (-1, enc_params))(generator_output)
  def decoder_model(self, framelen, shape=(-1, 0)):

    if self.models.get('decoder_model'):
      return self.models.get('decoder_model')

    decoder_trainable = self.decoder_trainable
    enc_params = shape[1]
    conv_count = self.conv_count

    res = decoder_input = Input(shape=shape, dtype='float32', name="decoder_input")

    conf = TimeDistributed(
            keras.layers.Reshape(
              (1, enc_params)
              , trainable=decoder_trainable
            )
          )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,13),
              padding='valid',
              strides=(2,1),
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf = Conv2DTranspose(
              conv_count,
              kernel_size=(3,14),
              strides=(2,1),
              padding='valid',
              activation='relu',
              data_format="channels_last",
              trainable=decoder_trainable
    )

    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Return a single filter pulling together the results of all conv_count filters
    conf = Conv2D(
              1,
              kernel_size=(2,14),
              #  strides=(2,1),
              padding='valid',
              activation='sigmoid',
              name='decoder_conv_squash',
              trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    conf  = keras.layers.Reshape((-1, framelen), trainable=decoder_trainable)
    res = conf(res)

    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    # Trying to reduce autoencoder loss below 0.78
    res = LSTM(
        framelen * 3
        , return_sequences=True
        , name='decoder_lstm'
        , trainable=decoder_trainable
    )(res)

    conf = TimeDistributed(
        Dense(
            framelen
            , activation="relu"
            , trainable=decoder_trainable
            , name='decoder_final_dense'
        )
    )
    res = conf(res)
    print(conf.get_config())
    print(conf.input_shape)
    print(conf.output_shape)

    decoder_output = res

    self.models['decoder_model'] = Model(decoder_input, decoder_output)
    return self.models['decoder_model']

  def compile_model(self):
    self.utils.log("Compiling model")

    loss = CustomObjects.codec2_param_error_td
    # other loss options: CustomObjects.codec2_param_mean_square_error; 'mean_absolute_error'; 'cosine_proximity'

    main_loss_prop = 0.2
    mid_loss_prop = 0.4
    generator_loss_prop = 0.4

    if self.decoder_trainable and not self.generator_trainable:
      main_loss_prop = 0
      mid_loss_prop = 1
      generator_loss_prop = 0
    elif not self.decoder_trainable and self.generator_trainable:
      mid_loss_prop = 0
      generator_loss_prop = 0.5
      main_loss_prop = 0.5

    self.utils.log("Loss weightings:", main_loss_prop, mid_loss_prop, generator_loss_prop)

    self.model.compile(
        loss=[loss, loss, 'mean_squared_error'],
        loss_weights=[main_loss_prop, mid_loss_prop, generator_loss_prop],
        optimizer=self.get_optimizer_from_config())
    self.utils.log_model_summary()

  def fit(self, input_seq, output_seq, batch_size=None, epochs=1, shuffle=False, callbacks=None):
      inputs = input_seq[0]
      outputs = output_seq

      # s = output_seq[0].shape
      # dummy_encoded_output = np.zeros((s[0], 24, 64), dtype=np.float32)
      # outputs.append( dummy_encoded_output)

      print("X shape:", inputs.shape)
      print("y shape:", outputs[0].shape, outputs[1].shape, outputs[2].shape)

      self.model.fit(inputs, outputs, batch_size=batch_size, epochs=epochs, shuffle=shuffle,
       callbacks=callbacks
      )

  # we can wrap the model.add method, since in the future we may wish to
  # provide additional processing at this level
  def add_layer(self, layer):
    self.model.add(layer)
    return layer

  # start training GRU 1, then 1&2, then 3
  def before_iteration(self, iteration):
#    if iteration == 541:
#      self.utils.log("Adding frame rotation to reduce memory usage")
#      self.config.limit_frames = self.config.num_frames / 100
#      self.model_updates_lstm_1234_trainable()
#      self.config.log_attrs()
#
#    elif iteration == 481:
#      self.model_updates_lstm3_trainable()
#

    if not self.started:
      self.model_updates_onstart()

    self.started = True

  def model_updates_onstart(self):
#    self.utils.log("Make all lstms trainable")
#    self.model.layers[0].trainable=True
#    self.model.layers[1].trainable=True
#    self.model.layers[2].trainable=True
#    self.model.layers[3].trainable=True
#    self.model.layers[4].trainable=True

    self.compile_model()
    self.utils.save_json_model(0)

  def load_weights(self, fn, by_name=False):
    self.utils.log("Loading weights")
    self.model.load_weights(fn, by_name=by_name)

  def get_optimizer_from_config(self):
      optimizer_name = self.config.optimizer["name"]
      args = []
      self.optimizer = getattr(optimizers, optimizer_name)(*args, **self.config.optimizer["params"])
      return self.optimizer
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

db = DAL('sqlite://storage.sqlite',pool_size=1,check_reserved=['all'])

response.generic_patterns = ['*'] if request.is_local else []

from gluon.tools import Auth, Crud, Service, PluginManager, prettydate
auth = Auth(db)
crud, service, plugins = Crud(db), Service(), PluginManager()

## create all tables needed by auth if not custom tables
auth.define_tables(username=False, signature=False)

## configure email
mail = auth.settings.mailer
mail.settings.server = 'logging' or 'smtp.gmail.com:587'
mail.settings.sender = 'you@gmail.com'
mail.settings.login = 'username:password'

## configure auth policy
auth.settings.registration_requires_verification = False
auth.settings.registration_requires_approval = False
auth.settings.reset_password_requires_verification = True

# Definicion de tablas

Contenedor = db.define_table('contenedor',
<<<<<<< HEAD
			Field('nombre'),
			Field('descripcion', 'text'),
			Field('contenedor_superior', 'reference contenedor'),
			Field('es_contenedor', 'boolean'),
			auth.signature,
			format='%(nombre)s'
=======
				Field('nombre'),
				Field('descripcion', 'text'),
				Field('contenedor_superior', 'reference contenedor'),
				Field('es_contenedor', 'boolean'),
				auth.signature,
				format='%(nombre)s'
>>>>>>> 677807e06e49bb0fa081ab49d01a159e55132f4e
			)

Libro = db.define_table('libro',
			Field('titulo'),
			Field('descripcion'),
			Field('isbn'),
			Field('autor'),
			Field('editorial'),
			Field('fecha_publicacion', 'date'),
			Field('ubicacion', 'reference contenedor'),
			Field('cantidad_total', 'integer'),
			Field('cantidad_prestados', 'integer'),
			Field('cantidad_disponible'),
			auth.signature,
			format='%(titulo)s'
			)

tipos = ['Alumno', 'Docente', 'No Docente']
Persona = db.define_table('persona',
			Field('nombre'),
			Field('apellido'),
			Field('dni', 'integer'),
			Field('domicilio'),
			Field('email'),
			Field('tipo', 'list:string', requires=IS_IN_SET(tipos)),
			Field('telefono', 'integer'),
			Field('curso'),
			auth.signature,
			format='%(nombre)s'
			)

estados = ['Estado1', 'Estado2', 'Estado3']
Movimientos = db.define_table('movimientos',
			Field('libro_id', 'reference libro'),
			Field('persona_id', 'reference persona'),
			Field('cantidad', 'integer'),
			Field('estado', 'list:string', requires=IS_IN_SET(estados)),
			auth.signature,
			)
<commit_msg>cambio model.py<commit_after># -*- coding: utf-8 -*-

db = DAL('sqlite://storage.sqlite',pool_size=1,check_reserved=['all'])

response.generic_patterns = ['*'] if request.is_local else []

from gluon.tools import Auth, Crud, Service, PluginManager, prettydate
auth = Auth(db)
crud, service, plugins = Crud(db), Service(), PluginManager()

## create all tables needed by auth if not custom tables
auth.define_tables(username=False, signature=False)

## configure email
mail = auth.settings.mailer
mail.settings.server = 'logging' or 'smtp.gmail.com:587'
mail.settings.sender = 'you@gmail.com'
mail.settings.login = 'username:password'

## configure auth policy
auth.settings.registration_requires_verification = False
auth.settings.registration_requires_approval = False
auth.settings.reset_password_requires_verification = True

# Definicion de tablas

Contenedor = db.define_table('contenedor',
				Field('nombre'),
				Field('descripcion', 'text'),
				Field('contenedor_superior', 'reference contenedor'),
				Field('es_contenedor', 'boolean'),
				auth.signature,
				format='%(nombre)s'
			)

Libro = db.define_table('libro',
			Field('titulo'),
			Field('descripcion'),
			Field('isbn'),
			Field('autor'),
			Field('editorial'),
			Field('fecha_publicacion', 'date'),
			Field('ubicacion', 'reference contenedor'),
			Field('cantidad_total', 'integer'),
			Field('cantidad_prestados', 'integer'),
			Field('cantidad_disponible'),
			auth.signature,
			format='%(titulo)s'
			)

tipos = ['Alumno', 'Docente', 'No Docente']
Persona = db.define_table('persona',
			Field('nombre'),
			Field('apellido'),
			Field('dni', 'integer'),
			Field('domicilio'),
			Field('email'),
			Field('tipo', 'list:string', requires=IS_IN_SET(tipos)),
			Field('telefono', 'integer'),
			Field('curso'),
			auth.signature,
			format='%(nombre)s'
			)

estados = ['Estado1', 'Estado2', 'Estado3']
Movimientos = db.define_table('movimientos',
			Field('libro_id', 'reference libro'),
			Field('persona_id', 'reference persona'),
			Field('cantidad', 'integer'),
			Field('estado', 'list:string', requires=IS_IN_SET(estados)),
			auth.signature,
			)
<|endoftext|>


<commit_before>"""
======================================================
Classification of text documents using sparse features
======================================================

This is an example showing how the scikit-learn can be used to classify
documents by topics using a bag-of-words approach. This example uses
a scipy.sparse matrix to store the features instead of standard numpy arrays
and demos various classifiers that can efficiently handle sparse matrices.

The dataset used in this example is the 20 newsgroups dataset which will be
automatically downloaded and then cached.

You can adjust the number of categories by giving there name to the dataset
loader or setting them to None to get the 20 of them.

To run this example use::

  % python examples/document_classification_20newsgroups.py [options]

Options are:

  --report
      Print a detailed classification report.
  --confusion-matrix
      Print the confusion matrix.

"""
print __doc__

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
# License: Simplified BSD

from time import time
import logging
import os
import sys

from scikits.learn.datasets import fetch_20newsgroups
from scikits.learn.feature_extraction.text import Vectorizer
from scikits.learn.linear_model import RidgeClassifier
from scikits.learn.svm.sparse import LinearSVC
from scikits.learn.linear_model.sparse import SGDClassifier
from scikits.learn.naive_bayes import BernoulliNB, MultinomialNB
from scikits.learn.neighbors import NeighborsClassifier
from scikits.learn import metrics

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
argv = sys.argv[1:]
if "--report" in argv:
    print_report = True
else:
    print_report = False
if "--confusion-matrix" in argv:
    print_cm = True
else:
    print_cm = False

################################################################################
# Load some categories from the training set
categories = [
    'alt.atheism',
    'talk.religion.misc',
    'comp.graphics',
    'sci.space',
]
# Uncomment the following to do the analysis on all the categories
categories = None

print "Loading 20 newsgroups dataset for categories:"
print categories if categories else "all"

data_train = fetch_20newsgroups(subset='train', categories=categories,
                               shuffle=True, random_state=42)

data_test = fetch_20newsgroups(subset='test', categories=categories,
                              shuffle=True, random_state=42)

print "%d documents (training set)" % len(data_train.filenames)
print "%d documents (testing set)" % len(data_test.filenames)
print "%d categories" % len(data_train.target_names)
print

# split a training set and a test set
filenames_train, filenames_test = data_train.filenames, data_test.filenames
y_train, y_test = data_train.target, data_test.target

print "Extracting features from the training dataset using a sparse vectorizer"
t0 = time()
vectorizer = Vectorizer()
X_train = vectorizer.fit_transform((open(f).read() for f in filenames_train))
print "done in %fs" % (time() - t0)
print "n_samples: %d, n_features: %d" % X_train.shape
print

print "Extracting features from the test dataset using the same vectorizer"
t0 = time()
X_test = vectorizer.transform((open(f).read() for f in filenames_test))
print "done in %fs" % (time() - t0)
print "n_samples: %d, n_features: %d" % X_test.shape
print

################################################################################
# Benchmark classifiers
def benchmark(clf):
    print 80 * '_'
    print "Training: "
    print clf
    t0 = time()
    clf.fit(X_train, y_train)
    train_time = time() - t0
    print "train time: %0.3fs" % train_time

    t0 = time()
    pred = clf.predict(X_test)
    test_time = time() - t0
    print "test time:  %0.3fs" % test_time

    score = metrics.f1_score(y_test, pred)
    print "f1-score:   %0.3f" % score

    if hasattr(clf, 'coef_'):
        nnz = clf.coef_.nonzero()[0].shape[0]
        print "non-zero coef: %d" % nnz
        print

    if print_report:
        print "classification report:"
        print metrics.classification_report(y_test, pred,
                                            target_names=categories)

    if print_cm:
        print "confusion matrix:"
        print metrics.confusion_matrix(y_test, pred)

    print
    return score, train_time, test_time

for clf, name in ((RidgeClassifier(tol=1e-1), "Ridge Classifier"),
                  (NeighborsClassifier(n_neighbors=10), "kNN")):
    print 80*'='
    print name
    results = benchmark(clf)

for penalty in ["l2", "l1"]:
    print 80 * '='
    print "%s penalty" % penalty.upper()
    # Train Liblinear model
    liblinear_results = benchmark(LinearSVC(loss='l2', penalty=penalty, C=1000,
                                            dual=False, tol=1e-3))

    # Train SGD model
    sgd_results = benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                          penalty=penalty))

# Train SGD with Elastic Net penalty
print 80 * '='
print "Elastic-Net penalty"
sgd_results = benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                      penalty="elasticnet"))

# Train sparse Naive Bayes classifiers
print 80 * '='
print "Naive Bayes"
mnnb_results = benchmark(MultinomialNB(alpha=.01))
bnb_result = benchmark(BernoulliNB(alpha=.01))
<commit_msg>Use four categories instead of all in doc. class. example<commit_after>"""
======================================================
Classification of text documents using sparse features
======================================================

This is an example showing how the scikit-learn can be used to classify
documents by topics using a bag-of-words approach. This example uses
a scipy.sparse matrix to store the features instead of standard numpy arrays
and demos various classifiers that can efficiently handle sparse matrices.

The dataset used in this example is the 20 newsgroups dataset which will be
automatically downloaded and then cached.

You can adjust the number of categories by giving there name to the dataset
loader or setting them to None to get the 20 of them.

To run this example use::

  % python examples/document_classification_20newsgroups.py [options]

Options are:

  --report
      Print a detailed classification report.
  --confusion-matrix
      Print the confusion matrix.

"""
print __doc__

# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
#         Olivier Grisel <olivier.grisel@ensta.org>
#         Mathieu Blondel <mathieu@mblondel.org>
# License: Simplified BSD

from time import time
import logging
import os
import sys

from scikits.learn.datasets import fetch_20newsgroups
from scikits.learn.feature_extraction.text import Vectorizer
from scikits.learn.linear_model import RidgeClassifier
from scikits.learn.svm.sparse import LinearSVC
from scikits.learn.linear_model.sparse import SGDClassifier
from scikits.learn.naive_bayes import BernoulliNB, MultinomialNB
from scikits.learn.neighbors import NeighborsClassifier
from scikits.learn import metrics

# Display progress logs on stdout
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

# parse commandline arguments
argv = sys.argv[1:]
if "--report" in argv:
    print_report = True
else:
    print_report = False
if "--confusion-matrix" in argv:
    print_cm = True
else:
    print_cm = False

################################################################################
# Load some categories from the training set
categories = [
    'alt.atheism',
    'talk.religion.misc',
    'comp.graphics',
    'sci.space',
]
# Uncomment the following to do the analysis on all the categories
#categories = None

print "Loading 20 newsgroups dataset for categories:"
print categories if categories else "all"

data_train = fetch_20newsgroups(subset='train', categories=categories,
                               shuffle=True, random_state=42)

data_test = fetch_20newsgroups(subset='test', categories=categories,
                              shuffle=True, random_state=42)

print "%d documents (training set)" % len(data_train.filenames)
print "%d documents (testing set)" % len(data_test.filenames)
print "%d categories" % len(data_train.target_names)
print

# split a training set and a test set
filenames_train, filenames_test = data_train.filenames, data_test.filenames
y_train, y_test = data_train.target, data_test.target

print "Extracting features from the training dataset using a sparse vectorizer"
t0 = time()
vectorizer = Vectorizer()
X_train = vectorizer.fit_transform((open(f).read() for f in filenames_train))
print "done in %fs" % (time() - t0)
print "n_samples: %d, n_features: %d" % X_train.shape
print

print "Extracting features from the test dataset using the same vectorizer"
t0 = time()
X_test = vectorizer.transform((open(f).read() for f in filenames_test))
print "done in %fs" % (time() - t0)
print "n_samples: %d, n_features: %d" % X_test.shape
print

################################################################################
# Benchmark classifiers
def benchmark(clf):
    print 80 * '_'
    print "Training: "
    print clf
    t0 = time()
    clf.fit(X_train, y_train)
    train_time = time() - t0
    print "train time: %0.3fs" % train_time

    t0 = time()
    pred = clf.predict(X_test)
    test_time = time() - t0
    print "test time:  %0.3fs" % test_time

    score = metrics.f1_score(y_test, pred)
    print "f1-score:   %0.3f" % score

    if hasattr(clf, 'coef_'):
        nnz = clf.coef_.nonzero()[0].shape[0]
        print "non-zero coef: %d" % nnz
        print

    if print_report:
        print "classification report:"
        print metrics.classification_report(y_test, pred,
                                            target_names=categories)

    if print_cm:
        print "confusion matrix:"
        print metrics.confusion_matrix(y_test, pred)

    print
    return score, train_time, test_time

for clf, name in ((RidgeClassifier(tol=1e-1), "Ridge Classifier"),
                  (NeighborsClassifier(n_neighbors=10), "kNN")):
    print 80*'='
    print name
    results = benchmark(clf)

for penalty in ["l2", "l1"]:
    print 80 * '='
    print "%s penalty" % penalty.upper()
    # Train Liblinear model
    liblinear_results = benchmark(LinearSVC(loss='l2', penalty=penalty, C=1000,
                                            dual=False, tol=1e-3))

    # Train SGD model
    sgd_results = benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                          penalty=penalty))

# Train SGD with Elastic Net penalty
print 80 * '='
print "Elastic-Net penalty"
sgd_results = benchmark(SGDClassifier(alpha=.0001, n_iter=50,
                                      penalty="elasticnet"))

# Train sparse Naive Bayes classifiers
print 80 * '='
print "Naive Bayes"
mnnb_results = benchmark(MultinomialNB(alpha=.01))
bnb_result = benchmark(BernoulliNB(alpha=.01))
<|endoftext|>


<commit_before>"""holds locking functionality that works across processes"""
from __future__ import absolute_import, unicode_literals

import logging
import os
from abc import ABCMeta, abstractmethod
from contextlib import contextmanager
from threading import Lock, RLock

from filelock import FileLock, Timeout
from six import add_metaclass

from virtualenv.util.path import Path

class _CountedFileLock(FileLock):
    def __init__(self, lock_file):
        parent = os.path.dirname(lock_file)
        if not os.path.isdir(parent):
            try:
                os.makedirs(parent)
            except OSError:
                pass
        super(_CountedFileLock, self).__init__(lock_file)
        self.count = 0
        self.thread_safe = RLock()

    def acquire(self, timeout=None, poll_intervall=0.05):
        with self.thread_safe:
            if self.count == 0:
                super(_CountedFileLock, self).acquire(timeout=timeout, poll_intervall=poll_intervall)
            self.count += 1

    def release(self, force=False):
        with self.thread_safe:
            if self.count == 1:
                super(_CountedFileLock, self).release(force=force)
            self.count = max(self.count - 1, 0)

_lock_store = {}
_store_lock = Lock()

@add_metaclass(ABCMeta)
class PathLockBase(object):
    def __init__(self, folder):
        path = Path(folder)
        self.path = path.resolve() if path.exists() else path

    def __repr__(self):
        return "{}({})".format(self.__class__.__name__, self.path)

    def __div__(self, other):
        return type(self)(self.path / other)

    def __truediv__(self, other):
        return self.__div__(other)

    @abstractmethod
    def __enter__(self):
        raise NotImplementedError

    @abstractmethod
    def __exit__(self, exc_type, exc_val, exc_tb):
        raise NotImplementedError

    @abstractmethod
    @contextmanager
    def lock_for_key(self, name, no_block=False):
        raise NotImplementedError

    @abstractmethod
    @contextmanager
    def non_reentrant_lock_for_key(name):
        raise NotImplementedError

class ReentrantFileLock(PathLockBase):
    def __init__(self, folder):
        super(ReentrantFileLock, self).__init__(folder)
        self._lock = None

    def _create_lock(self, name=""):
        lock_file = str(self.path / "{}.lock".format(name))
        with _store_lock:
            if lock_file not in _lock_store:
                _lock_store[lock_file] = _CountedFileLock(lock_file)
            return _lock_store[lock_file]

    @staticmethod
    def _del_lock(lock):
        with _store_lock:
            if lock is not None:
                with lock.thread_safe:
                    if lock.count == 0:
                        _lock_store.pop(lock.lock_file, None)

    def __del__(self):
        self._del_lock(self._lock)

    def __enter__(self):
        self._lock = self._create_lock()
        self._lock_file(self._lock)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._release(self._lock)

    def _lock_file(self, lock, no_block=False):
        # multiple processes might be trying to get a first lock... so we cannot check if this directory exist without
        # a lock, but that lock might then become expensive, and it's not clear where that lock should live.
        # Instead here we just ignore if we fail to create the directory.
        try:
            os.makedirs(str(self.path))
        except OSError:
            pass
        try:
            lock.acquire(0.0001)
        except Timeout:
            if no_block:
                raise
            logging.debug("lock file %s present, will block until released", lock.lock_file)
            lock.release()  # release the acquire try from above
            lock.acquire()

    @staticmethod
    def _release(lock):
        lock.release()

    @contextmanager
    def lock_for_key(self, name, no_block=False):
        lock = self._create_lock(name)
        try:
            try:
                self._lock_file(lock, no_block)
                yield
            finally:
                self._release(lock)
        finally:
            self._del_lock(lock)

    @contextmanager
    def non_reentrant_lock_for_key(self, name):
        with _CountedFileLock(str(self.path / "{}.lock".format(name))):
            yield

class NoOpFileLock(PathLockBase):
    def __enter__(self):
        raise NotImplementedError

    def __exit__(self, exc_type, exc_val, exc_tb):
        raise NotImplementedError

    @contextmanager
    def lock_for_key(self, name, no_block=False):
        yield

    @contextmanager
    def non_reentrant_lock_for_key(self, name):
        yield

__all__ = (
    "NoOpFileLock",
    "ReentrantFileLock",
    "Timeout",
)
<commit_msg>refactor to not do too much in ReentrantFileLock.__del__ (#2213)<commit_after>"""holds locking functionality that works across processes"""
from __future__ import absolute_import, unicode_literals

import logging
import os
from abc import ABCMeta, abstractmethod
from contextlib import contextmanager
from threading import Lock, RLock

from filelock import FileLock, Timeout
from six import add_metaclass

from virtualenv.util.path import Path

class _CountedFileLock(FileLock):
    def __init__(self, lock_file):
        parent = os.path.dirname(lock_file)
        if not os.path.isdir(parent):
            try:
                os.makedirs(parent)
            except OSError:
                pass
        super(_CountedFileLock, self).__init__(lock_file)
        self.count = 0
        self.thread_safe = RLock()

    def acquire(self, timeout=None, poll_intervall=0.05):
        with self.thread_safe:
            if self.count == 0:
                super(_CountedFileLock, self).acquire(timeout=timeout, poll_intervall=poll_intervall)
            self.count += 1

    def release(self, force=False):
        with self.thread_safe:
            if self.count == 1:
                super(_CountedFileLock, self).release(force=force)
            self.count = max(self.count - 1, 0)

_lock_store = {}
_store_lock = Lock()

@add_metaclass(ABCMeta)
class PathLockBase(object):
    def __init__(self, folder):
        path = Path(folder)
        self.path = path.resolve() if path.exists() else path

    def __repr__(self):
        return "{}({})".format(self.__class__.__name__, self.path)

    def __div__(self, other):
        return type(self)(self.path / other)

    def __truediv__(self, other):
        return self.__div__(other)

    @abstractmethod
    def __enter__(self):
        raise NotImplementedError

    @abstractmethod
    def __exit__(self, exc_type, exc_val, exc_tb):
        raise NotImplementedError

    @abstractmethod
    @contextmanager
    def lock_for_key(self, name, no_block=False):
        raise NotImplementedError

    @abstractmethod
    @contextmanager
    def non_reentrant_lock_for_key(name):
        raise NotImplementedError

class ReentrantFileLock(PathLockBase):
    def __init__(self, folder):
        super(ReentrantFileLock, self).__init__(folder)
        self._lock = None

    def _create_lock(self, name=""):
        lock_file = str(self.path / "{}.lock".format(name))
        with _store_lock:
            if lock_file not in _lock_store:
                _lock_store[lock_file] = _CountedFileLock(lock_file)
            return _lock_store[lock_file]

    @staticmethod
    def _del_lock(lock):
        if lock is not None:
            with _store_lock:
                with lock.thread_safe:
                    if lock.count == 0:
                        _lock_store.pop(lock.lock_file, None)

    def __del__(self):
        self._del_lock(self._lock)

    def __enter__(self):
        self._lock = self._create_lock()
        self._lock_file(self._lock)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._release(self._lock)
        self._del_lock(self._lock)
        self._lock = None

    def _lock_file(self, lock, no_block=False):
        # multiple processes might be trying to get a first lock... so we cannot check if this directory exist without
        # a lock, but that lock might then become expensive, and it's not clear where that lock should live.
        # Instead here we just ignore if we fail to create the directory.
        try:
            os.makedirs(str(self.path))
        except OSError:
            pass
        try:
            lock.acquire(0.0001)
        except Timeout:
            if no_block:
                raise
            logging.debug("lock file %s present, will block until released", lock.lock_file)
            lock.release()  # release the acquire try from above
            lock.acquire()

    @staticmethod
    def _release(lock):
        lock.release()

    @contextmanager
    def lock_for_key(self, name, no_block=False):
        lock = self._create_lock(name)
        try:
            try:
                self._lock_file(lock, no_block)
                yield
            finally:
                self._release(lock)
        finally:
            self._del_lock(lock)
            lock = None

    @contextmanager
    def non_reentrant_lock_for_key(self, name):
        with _CountedFileLock(str(self.path / "{}.lock".format(name))):
            yield

class NoOpFileLock(PathLockBase):
    def __enter__(self):
        raise NotImplementedError

    def __exit__(self, exc_type, exc_val, exc_tb):
        raise NotImplementedError

    @contextmanager
    def lock_for_key(self, name, no_block=False):
        yield

    @contextmanager
    def non_reentrant_lock_for_key(self, name):
        yield

__all__ = (
    "NoOpFileLock",
    "ReentrantFileLock",
    "Timeout",
)
<|endoftext|>


<commit_before># -*-coding:utf-8-*-

import socket
import select
from karlooper.logger.logger import init_logger
from karlooper.web.__async_core_server import EchoServer, asyncore
from karlooper.web.http_connection import HttpConnection
from karlooper.web.http_io_buffer import HttpIOBuffer
from karlooper.http_parser.http_parser import HttpParser
from karlooper.config import get_cli_data, set_cli_data
from karlooper.config.config import SOCKET_RECEIVE_SIZE, DEFAULT_PORT, CLIENT_CONNECT_TO_SERVER_NUM

__author__ = 'karlvorndoenitz@gmail.com'

class Application(object):
    def __init__(self, handlers, settings=None, **kwargs):
        """

        :param handlers: handlers mapping, dict type
        :param settings: settings mapping, dict type
        :param kwargs: options

        """
        self.settings = settings
        self.handlers = handlers
        set_cli_data(self.settings)
        set_cli_data(kwargs)
        cli_data = get_cli_data()
        self.port = int(cli_data.get("port", DEFAULT_PORT))
        self.logger = init_logger()
        self.EOL1 = b'\n\n'
        self.EOL2 = b'\n\r\n'
        self.response = ""

    def listen(self, port):
        """listen port

        :param port: port that application listened
        :return: None

        """
        self.port = int(port)

    def __run_epoll(self):
        """
        run the application use epoll
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        epoll = select.epoll()
        epoll.register(server_socket.fileno(), select.EPOLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            while True:
                events = epoll.poll(1)
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            epoll.register(connection.fileno(), select.EPOLLIN)  # register socket read event to epoll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.EPOLLIN:  # when data in os's read buffer area
                            http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                            http_io_buffer.add_request(
                                fileno,
                                http_io_buffer.get_request(fileno) + http_request_buffer
                            )
                            if self.EOL1 in http_io_buffer.get_request(fileno) \
                                    or self.EOL2 in http_io_buffer.get_request(fileno):
                                request_data = http_io_buffer.get_request(fileno)[:-2] \
                                    if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                    else http_io_buffer.get_request(fileno)
                                data = HttpParser(request_data, self.handlers, settings=self.settings).parse()
                                http_io_buffer.add_response(
                                    fileno,
                                    http_io_buffer.get_response(fileno) + data
                                )
                                epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                        elif event & select.EPOLLOUT:  # if out mode
                            byteswritten = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[byteswritten:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                epoll.modify(fileno, 0)  # change file number to hup mode
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                epoll.modify(fileno, select.EPOLLHUP)
                        elif event & select.EPOLLHUP:  # if message sent and file number in epoll is hup
                            epoll.unregister(fileno)  # remove file number from epoll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_epoll", e)
                        continue
        finally:
            epoll.unregister(server_socket.fileno())
            epoll.close()
            server_socket.close()

    def __run_async_io(self):
        """
        run server use asyncore
        """
        EchoServer('0.0.0.0', self.port, self.handlers, self.settings)
        asyncore.loop()

    def __run_kqueue(self):
        """
        run server use kqueue
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)
        kq = select.kqueue()
        http_connection = HttpConnection()
        index = 1
        events = [select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)]
        while True:
            try:
                event_list = kq.control(events, 1)
            except select.error as e:
                self.logger.error("error in __run_kqueue", e)
                break
            if event_list:
                for each in event_list:
                    if each.ident == server_socket.fileno():
                        conn, addr = server_socket.accept()
                        http_connection.add_connection(index, conn)
                        events.append(
                            select.kevent(
                                http_connection.get_connection(index).fileno(),
                                select.KQ_FILTER_READ,
                                select.KQ_EV_ADD,
                                udata=index
                            )
                        )
                        index += 1
                    else:
                        try:
                            if each.udata >= 1 and each.flags == select.KQ_EV_ADD \
                                    and each.filter == select.KQ_FILTER_READ:
                                conn = http_connection.get_connection(each.udata)
                                request_data = conn.recv(SOCKET_RECEIVE_SIZE)
                                if request_data:
                                    request_data = request_data[:-2] if request_data.endswith("\r\n") else request_data
                                    data = HttpParser(
                                        request_data,
                                        handlers=self.handlers,
                                        settings=self.settings
                                    ).parse()
                                    conn.send(data)
                                else:
                                    conn.close()
                        except Exception, e:
                            self.logger.error("error in __run_kqueue event list", e)
                        finally:
                            conn.close()
        server_socket.close()

    def __run_poll(self):
        """
        run server use poll, I will modify __run_poll and __run_epoll in the future
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        poll = select.poll()
        poll.register(server_socket.fileno(), select.POLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            while True:
                events = poll.poll(1)
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            poll.register(connection.fileno(), select.POLLIN)  # register socket read event to poll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.POLLIN:  # when data in os's read buffer area
                            http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                            http_io_buffer.add_request(
                                fileno,
                                http_io_buffer.get_request(fileno) + http_request_buffer
                            )
                            if self.EOL1 in http_io_buffer.get_request(fileno) \
                                    or self.EOL2 in http_io_buffer.get_request(fileno):
                                request_data = http_io_buffer.get_request(fileno)[:-2] \
                                    if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                    else http_io_buffer.get_request(fileno)
                                data = HttpParser(request_data, self.handlers, settings=self.settings).parse()
                                http_io_buffer.add_response(
                                    fileno,
                                    http_io_buffer.get_response(fileno) + data
                                )
                                poll.modify(fileno, select.POLLOUT)  # change file number to poll out mode
                        elif event & select.POLLOUT:  # if out mode
                            byteswritten = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[byteswritten:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                poll.modify(fileno, 0)  # change file number to hup mode
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                poll.modify(fileno, select.POLLHUP)
                        elif event & select.POLLHUP:  # if message sent and file number in poll is hup
                            poll.unregister(fileno)  # remove file number from poll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_poll", e)
                        continue
        finally:
            poll.unregister(server_socket.fileno())
            poll.close()
            server_socket.close()

    def run(self):
        """
        run the web server
        """
        print "server run on port: %d" % self.port
        self.logger.info("server run on port: %d" % self.port)
        if hasattr(select, "epoll"):
            print "run with epoll"
            self.__run_epoll()
        if hasattr(select, "poll"):
            print "run with poll"
            self.__run_poll()
        if hasattr(select, "kqueue"):
            print "run with kqueue"
            self.__run_kqueue()
<commit_msg>add poll mod<commit_after># -*-coding:utf-8-*-

import socket
import select
from karlooper.logger.logger import init_logger
from karlooper.web.__async_core_server import EchoServer, asyncore
from karlooper.web.http_connection import HttpConnection
from karlooper.web.http_io_buffer import HttpIOBuffer
from karlooper.http_parser.http_parser import HttpParser
from karlooper.config import get_cli_data, set_cli_data
from karlooper.config.config import SOCKET_RECEIVE_SIZE, DEFAULT_PORT, CLIENT_CONNECT_TO_SERVER_NUM

__author__ = 'karlvorndoenitz@gmail.com'

class Application(object):
    def __init__(self, handlers, settings=None, **kwargs):
        """

        :param handlers: handlers mapping, dict type
        :param settings: settings mapping, dict type
        :param kwargs: options

        """
        self.settings = settings
        self.handlers = handlers
        set_cli_data(self.settings)
        set_cli_data(kwargs)
        cli_data = get_cli_data()
        self.port = int(cli_data.get("port", DEFAULT_PORT))
        self.logger = init_logger()
        self.EOL1 = b'\n\n'
        self.EOL2 = b'\n\r\n'
        self.response = ""

    def listen(self, port):
        """listen port

        :param port: port that application listened
        :return: None

        """
        self.port = int(port)

    def __run_epoll(self):
        """
        run the application use epoll
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        epoll = select.epoll()
        epoll.register(server_socket.fileno(), select.EPOLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            while True:
                events = epoll.poll(1)
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            epoll.register(connection.fileno(), select.EPOLLIN)  # register socket read event to epoll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.EPOLLIN:  # when data in os's read buffer area
                            http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                            http_io_buffer.add_request(
                                fileno,
                                http_io_buffer.get_request(fileno) + http_request_buffer
                            )
                            if self.EOL1 in http_io_buffer.get_request(fileno) \
                                    or self.EOL2 in http_io_buffer.get_request(fileno):
                                request_data = http_io_buffer.get_request(fileno)[:-2] \
                                    if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                    else http_io_buffer.get_request(fileno)
                                data = HttpParser(request_data, self.handlers, settings=self.settings).parse()
                                http_io_buffer.add_response(
                                    fileno,
                                    http_io_buffer.get_response(fileno) + data
                                )
                                epoll.modify(fileno, select.EPOLLOUT)  # change file number to epoll out mode
                        elif event & select.EPOLLOUT:  # if out mode
                            byteswritten = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[byteswritten:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                epoll.modify(fileno, 0)  # change file number to hup mode
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                epoll.modify(fileno, select.EPOLLHUP)
                        elif event & select.EPOLLHUP:  # if message sent and file number in epoll is hup
                            epoll.unregister(fileno)  # remove file number from epoll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_epoll", e)
                        continue
        finally:
            epoll.unregister(server_socket.fileno())
            epoll.close()
            server_socket.close()

    def __run_async_io(self):
        """
        run server use asyncore
        """
        EchoServer('0.0.0.0', self.port, self.handlers, self.settings)
        asyncore.loop()

    def __run_kqueue(self):
        """
        run server use kqueue
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)
        kq = select.kqueue()
        http_connection = HttpConnection()
        index = 1
        events = [select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)]
        while True:
            try:
                event_list = kq.control(events, 1)
            except select.error as e:
                self.logger.error("error in __run_kqueue", e)
                break
            if event_list:
                for each in event_list:
                    if each.ident == server_socket.fileno():
                        conn, addr = server_socket.accept()
                        http_connection.add_connection(index, conn)
                        events.append(
                            select.kevent(
                                http_connection.get_connection(index).fileno(),
                                select.KQ_FILTER_READ,
                                select.KQ_EV_ADD,
                                udata=index
                            )
                        )
                        index += 1
                    else:
                        try:
                            if each.udata >= 1 and each.flags == select.KQ_EV_ADD \
                                    and each.filter == select.KQ_FILTER_READ:
                                conn = http_connection.get_connection(each.udata)
                                request_data = conn.recv(SOCKET_RECEIVE_SIZE)
                                if request_data:
                                    request_data = request_data[:-2] if request_data.endswith("\r\n") else request_data
                                    data = HttpParser(
                                        request_data,
                                        handlers=self.handlers,
                                        settings=self.settings
                                    ).parse()
                                    conn.send(data)
                                else:
                                    conn.close()
                        except Exception, e:
                            self.logger.error("error in __run_kqueue event list", e)
                        finally:
                            conn.close()
        server_socket.close()

    def __run_poll(self):
        """
        run server use poll, I will modify __run_poll and __run_epoll in the future
        """
        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_socket.bind(('0.0.0.0', self.port))
        server_socket.listen(CLIENT_CONNECT_TO_SERVER_NUM)  # the number of client that connect to server
        server_socket.setblocking(0)  # set 0 not block other block
        server_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        poll = select.poll()
        poll.register(server_socket.fileno(), select.POLLIN)
        try:
            http_connection = HttpConnection()
            http_io_buffer = HttpIOBuffer()
            while True:
                events = poll.poll(1)
                for fileno, event in events:
                    try:
                        if fileno == server_socket.fileno():  # if request come
                            connection, address = server_socket.accept()  # waiting income connection
                            connection.setblocking(0)  # none block
                            poll.register(connection.fileno(), select.POLLIN)  # register socket read event to poll
                            http_connection.add_connection(connection.fileno(), connection)
                            http_io_buffer.add_request(connection.fileno(), b'')
                            http_io_buffer.add_response(connection.fileno(), self.response)
                        elif event & select.POLLIN:  # when data in os's read buffer area
                            http_request_buffer = http_connection.get_connection(fileno).recv(SOCKET_RECEIVE_SIZE)
                            http_io_buffer.add_request(
                                fileno,
                                http_io_buffer.get_request(fileno) + http_request_buffer
                            )
                            if self.EOL1 in http_io_buffer.get_request(fileno) \
                                    or self.EOL2 in http_io_buffer.get_request(fileno):
                                request_data = http_io_buffer.get_request(fileno)[:-2] \
                                    if http_io_buffer.get_request(fileno).endswith("\r\n") \
                                    else http_io_buffer.get_request(fileno)
                                data = HttpParser(request_data, self.handlers, settings=self.settings).parse()
                                http_io_buffer.add_response(
                                    fileno,
                                    http_io_buffer.get_response(fileno) + data
                                )
                                poll.modify(fileno, select.POLLOUT)  # change file number to poll out mode
                        elif event & select.POLLOUT:  # if out mode
                            byteswritten = http_connection.get_connection(fileno).send(
                                http_io_buffer.get_response(fileno)
                            )
                            http_io_buffer.add_response(fileno, http_io_buffer.get_response(fileno)[byteswritten:])
                            if len(http_io_buffer.get_response(fileno)) == 0:  # if file sent
                                poll.modify(fileno, 0)  # change file number to hup mode
                                http_connection.get_connection(fileno).shutdown(socket.SHUT_RDWR)
                                poll.modify(fileno, select.POLLHUP)
                        elif event & select.POLLHUP:  # if message sent and file number in poll is hup
                            poll.unregister(fileno)  # remove file number from poll
                            http_connection.get_connection(fileno).close()  # close connection
                            http_connection.remove_connection(fileno)  # delete connection from connections dict
                    except Exception, e:
                        self.logger.error("error in __run_poll", e)
                        continue
        finally:
            poll.unregister(server_socket.fileno())
            poll.close()
            server_socket.close()

    def run(self):
        """
        run the web server
        """
        print "server run on port: %d" % self.port
        self.logger.info("server run on port: %d" % self.port)
        if hasattr(select, "epoll"):
            print "run with epoll"
            self.__run_epoll()
        if hasattr(select, "kqueue"):
            print "run with kqueue"
            self.__run_kqueue()
        if hasattr(select, "poll"):
            print "run with poll"
            self.__run_poll()
<|endoftext|>


<commit_before>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

import sys

from twisted.internet import reactor
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File

from autobahn.twisted.websocket import WebSocketServerFactory, \
    WebSocketServerProtocol, \
    listenWS

from autobahn.websocket.http import HttpException

class BaseService:

    """
    Simple base for our services.
    """

    def __init__(self, proto):
        self.proto = proto

    def onOpen(self):
        pass

    def onClose(self, wasClean, code, reason):
        pass

    def onMessage(self, payload, isBinary):
        pass

class Echo1Service(BaseService):

    """
    Awesome Echo Service 1.
    """

    def onMessage(self, payload, isBinary):
        if not isBinary:
            msg = "Echo 1 - {}".format(payload.decode('utf8'))
            print(msg)
            self.proto.sendMessage(msg.encode('utf8'))

class Echo2Service(BaseService):

    """
    Awesome Echo Service 2.
    """

    def onMessage(self, payload, isBinary):
        if not isBinary:
            msg = "Echo 2 - {}".format(payload.decode('utf8'))
            print(msg)
            self.proto.sendMessage(msg.encode('utf8'))

class ServiceServerProtocol(WebSocketServerProtocol):

    SERVICEMAP = {'/echo1': Echo1Service,
                  '/echo2': Echo2Service}

    def __init__(self):
        self.service = None

    def onConnect(self, request):
        # request has all the information from the initial
        # WebSocket opening handshake ..
        print(request.peer)
        print(request.headers)
        print(request.host)
        print(request.path)
        print(request.params)
        print(request.version)
        print(request.origin)
        print(request.protocols)
        print(request.extensions)

        # We map to services based on path component of the URL the
        # WebSocket client requested. This is just an example. We could
        # use other information from request, such has HTTP headers,
        # WebSocket subprotocol, WebSocket origin etc etc
        ##
        if request.path in self.SERVICEMAP:
            cls = self.SERVICEMAP[request.path]
            self.service = cls(self)
        else:
            err = "No service under %s" % request.path
            print(err)
            raise HttpException(404, err)

    def onOpen(self):
        if self.service:
            self.service.onOpen()

    def onMessage(self, payload, isBinary):
        if self.service:
            self.service.onMessage(payload, isBinary)

    def onClose(self, wasClean, code, reason):
        if self.service:
            self.service.onClose(wasClean, code, reason)

if __name__ == '__main__':

    factory = WebSocketServerFactory(u"ws://127.0.0.1:9000")
    factory.protocol = ServiceServerProtocol
    listenWS(factory)

    reactor.run()
<commit_msg>Replaced deprecated HttpException for types.ConnectionDeny on onConnect<commit_after>###############################################################################
#
# The MIT License (MIT)
#
# Copyright (c) Tavendo GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

import txaio

from twisted.internet import reactor

from autobahn.twisted.websocket import WebSocketServerFactory, \
    WebSocketServerProtocol, \
    listenWS

from autobahn.websocket.types import ConnectionDeny

class BaseService:

    """
    Simple base for our services.
    """

    def __init__(self, proto):
        self.proto = proto
        self.is_closed = False

    def onOpen(self):
        pass

    def onClose(self, wasClean, code, reason):
        pass

    def onMessage(self, payload, isBinary):
        pass

class Echo1Service(BaseService):

    """
    Awesome Echo Service 1.
    """

    def onMessage(self, payload, isBinary):
        if not isBinary:
            msg = "Echo 1 - {}".format(payload.decode('utf8'))
            print(msg)
            self.proto.sendMessage(msg.encode('utf8'))

class Echo2Service(BaseService):

    """
    Awesome Echo Service 2.
    """

    def onMessage(self, payload, isBinary):
        if not isBinary:
            msg = "Echo 2 - {}".format(payload.decode('utf8'))
            print(msg)
            self.proto.sendMessage(msg.encode('utf8'))

class ServiceServerProtocol(WebSocketServerProtocol):

    SERVICEMAP = {'/echo1': Echo1Service,
                  '/echo2': Echo2Service}

    def __init__(self):
        self.service = None
        self.is_closed = txaio.create_future()

    def onConnect(self, request):
        # request has all the information from the initial
        # WebSocket opening handshake ..
        print(request.peer)
        print(request.headers)
        print(request.host)
        print(request.path)
        print(request.params)
        print(request.version)
        print(request.origin)
        print(request.protocols)
        print(request.extensions)

        # We map to services based on path component of the URL the
        # WebSocket client requested. This is just an example. We could
        # use other information from request, such has HTTP headers,
        # WebSocket subprotocol, WebSocket origin etc etc
        ##
        if request.path in self.SERVICEMAP:
            cls = self.SERVICEMAP[request.path]
            self.service = cls(self)
        else:
            err = "No service under %s" % request.path
            print(err)
            raise ConnectionDeny(404, unicode(err))

    def onOpen(self):
        if self.service:
            self.service.onOpen()

    def onMessage(self, payload, isBinary):
        if self.service:
            self.service.onMessage(payload, isBinary)

    def onClose(self, wasClean, code, reason):
        if self.service:
            self.service.onClose(wasClean, code, reason)

if __name__ == '__main__':

    factory = WebSocketServerFactory(u"ws://127.0.0.1:9000")
    factory.protocol = ServiceServerProtocol
    listenWS(factory)

    reactor.run()
<|endoftext|>


<commit_before>modulecfg = {
			#[MODULES]
			#Modules & System modules, you better not touch system modules
			"sysmodules":			"syscmd,logger_daemon,title,seendb", #Do not touch this line!!
			"modules":			"automodes,cmd,clock,op,version,fmi,currency,youtube,stats,wiki,seen,ylilauta,google,gt,geo,git,ubermod"
			}
<commit_msg>remove<commit_after>modulecfg = {
			#[MODULES]
			#Modules & System modules, you better not touch system modules
			"sysmodules":			"syscmd,logger_daemon,title,seendb", #Do not touch this line!!
			"modules":			"automodes,cmd,clock,op,version,fmi,currency,youtube,stats,wiki,seen,ylilauta,google,gt,geo,git"
			}
<|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
import yaml

from waldur_core.quotas import exceptions as quotas_exceptions
from waldur_openstack.openstack_tenant import models as openstack_tenant_models

from . import models

def get_unique_node_name(name, instance_spl, cluster_spl):
    names_instances = openstack_tenant_models.Instance.objects.filter(service_project_link=instance_spl)\
        .values_list('name', flat=True)
    names_nodes = models.Node.objects.filter(cluster__service_project_link=cluster_spl).values_list('name', flat=True)
    names = list(names_instances) + list(names_nodes)

    if name not in names:
        return name

    i = 1
    new_name = name

    while new_name in names:
        i += 1
        new_name = '%s_%s' % (name, i)

    return new_name

def expand_added_nodes(nodes, rancher_spl, cluster_name):
    for node in nodes:
        memory = node.pop('memory', None)
        cpu = node.pop('cpu', None)
        subnet = node.pop('subnet')
        flavor = node.pop('flavor', None)
        roles = node.pop('roles')
        system_volume_size = node.pop('system_volume_size', None)
        system_volume_type = node.pop('system_volume_type', None)
        data_volumes = node.pop('volumes', [])

        tenant_settings = subnet.settings
        project = rancher_spl.project

        for volume in data_volumes:
            volume_type = volume.get('volume_type')
            if volume_type and volume_type.settings != tenant_settings:
                raise serializers.ValidationError(
                    'Volume type %s should belong to the service settings %s.' % (
                        volume_type.name, tenant_settings.name,
                    ))

        if flavor:
            if flavor.settings != tenant_settings:
                raise serializers.ValidationError(
                    'Flavor %s should belong to the service se ttings %s.' % (
                        flavor.name, tenant_settings.name,
                    ))

        if not flavor:
            flavors = openstack_tenant_models.Flavor.objects.filter(
                cores__gte=cpu,
                ram__gte=memory,
                settings=tenant_settings). \
                order_by('cores', 'ram')
            flavor = flavors[0]

            if not flavors:
                raise serializers.ValidationError('No matching flavor found.')

        # validate flavor
        requirements = list(filter(lambda x: x[0] in list(roles),
                                   settings.WALDUR_RANCHER['ROLE_REQUIREMENT'].items()))
        cpu_requirements = max([t[1]['CPU'] for t in requirements])
        ram_requirements = max([t[1]['RAM'] for t in requirements])

        if flavor.cores < cpu_requirements:
            raise serializers.ValidationError('Flavor %s does not meet requirements. CPU requirement is %s'
                                              % (flavor, cpu_requirements))

        if flavor.ram < ram_requirements:
            raise serializers.ValidationError('Flavor %s does not meet requirements. RAM requirement is %s'
                                              % (flavor, ram_requirements))

        try:
            tenant_spl = openstack_tenant_models.OpenStackTenantServiceProjectLink.objects.get(
                project=project,
                service__settings=tenant_settings)
        except ObjectDoesNotExist:
            raise serializers.ValidationError(
                'Service project link for service %s and project %s is not found.' % (
                    tenant_settings.name, project.name
                ))

        try:
            base_image_name = rancher_spl.service.settings.get_option('base_image_name')
            image = openstack_tenant_models.Image.objects.get(
                name=base_image_name,
                settings=tenant_settings)
        except ObjectDoesNotExist:
            raise serializers.ValidationError('No matching image found.')

        try:
            group = openstack_tenant_models.SecurityGroup.objects.get(
                name='default',
                settings=tenant_settings)
        except ObjectDoesNotExist:
            raise serializers.ValidationError('No matching group found.')

        node['initial_data'] = {
            'flavor': flavor.uuid.hex,
            'vcpu': flavor.cores,
            'ram': flavor.ram,
            'image': image.uuid.hex,
            'subnet': subnet.uuid.hex,
            'tenant_service_project_link': tenant_spl.id,
            'group': group.uuid.hex,
            'system_volume_size': system_volume_size,
            'system_volume_type': system_volume_type,
            'data_volumes': [{
                'size': volume['size'],
                'volume_type': volume.get('volume_type') and volume.get('volume_type').uuid.hex,
            } for volume in data_volumes]
        }

        if 'controlplane' in list(roles):
            node['controlplane_role'] = True
        if 'etcd' in list(roles):
            node['etcd_role'] = True
        if 'worker' in list(roles):
            node['worker_role'] = True

        name = cluster_name + '_rancher_node'
        unique_name = get_unique_node_name(name, tenant_spl.id, rancher_spl)
        node['name'] = unique_name

        # check quotas
        quota_sources = [
            tenant_spl,
            tenant_spl.project,
            tenant_spl.customer,
            tenant_spl.service,
            tenant_spl.service.settings
        ]

        for quota_name in ['storage', 'vcpu', 'ram']:
            requested = sum([node['initial_data'][quota_name] for node in nodes])

            for source in quota_sources:
                try:
                    quota = source.quotas.get(name=quota_name)
                    if quota.limit != -1 and (quota.usage + requested > quota.limit):
                        raise quotas_exceptions.QuotaValidationError(
                            _('"%(name)s" quota is over limit. Required: %(usage)s, limit: %(limit)s.') % dict(
                                name=quota_name, usage=quota.usage + requested, limit=quota.limit))
                except ObjectDoesNotExist:
                    pass

def format_disk_id(index):
    return '/dev/vd' + (chr(ord('a') + index))

def format_node_command(node):
    roles_command = []

    if node.controlplane_role:
        roles_command.append('--controlplane')

    if node.etcd_role:
        roles_command.append('--etcd')

    if node.worker_role:
        roles_command.append('--worker')

    return node.cluster.node_command + ' ' + ' '.join(roles_command)

def format_node_cloud_config(node):
    node_command = format_node_command(node)
    user_data = settings.WALDUR_RANCHER['RANCHER_NODE_CLOUD_INIT_TEMPLATE'].format(command=node_command)
    data_volumes = sorted(node.initial_data.get('data_volumes'))

    if data_volumes:
        conf = yaml.parse(user_data)

        # First volume is reserved for system volume, other volumes are data volumes

        conf['mounts'] = [
            [format_disk_id(index + 1), volume['mount_point']]
            for index, volume in enumerate(data_volumes)
        ]

        conf['fs_setup'] = [
            {'device': format_disk_id(index + 1), 'filesystem': 'ext4'}
            for index, volume in enumerate(data_volumes)
        ]
        user_data = yaml.dump(conf)

    return user_data
<commit_msg>Fix typo.<commit_after>from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
import yaml

from waldur_core.quotas import exceptions as quotas_exceptions
from waldur_openstack.openstack_tenant import models as openstack_tenant_models

from . import models

def get_unique_node_name(name, instance_spl, cluster_spl):
    names_instances = openstack_tenant_models.Instance.objects.filter(service_project_link=instance_spl)\
        .values_list('name', flat=True)
    names_nodes = models.Node.objects.filter(cluster__service_project_link=cluster_spl).values_list('name', flat=True)
    names = list(names_instances) + list(names_nodes)

    if name not in names:
        return name

    i = 1
    new_name = name

    while new_name in names:
        i += 1
        new_name = '%s_%s' % (name, i)

    return new_name

def expand_added_nodes(nodes, rancher_spl, cluster_name):
    for node in nodes:
        memory = node.pop('memory', None)
        cpu = node.pop('cpu', None)
        subnet = node.pop('subnet')
        flavor = node.pop('flavor', None)
        roles = node.pop('roles')
        system_volume_size = node.pop('system_volume_size', None)
        system_volume_type = node.pop('system_volume_type', None)
        data_volumes = node.pop('data_volumes', [])

        tenant_settings = subnet.settings
        project = rancher_spl.project

        for volume in data_volumes:
            volume_type = volume.get('volume_type')
            if volume_type and volume_type.settings != tenant_settings:
                raise serializers.ValidationError(
                    'Volume type %s should belong to the service settings %s.' % (
                        volume_type.name, tenant_settings.name,
                    ))

        if flavor:
            if flavor.settings != tenant_settings:
                raise serializers.ValidationError(
                    'Flavor %s should belong to the service se ttings %s.' % (
                        flavor.name, tenant_settings.name,
                    ))

        if not flavor:
            flavors = openstack_tenant_models.Flavor.objects.filter(
                cores__gte=cpu,
                ram__gte=memory,
                settings=tenant_settings). \
                order_by('cores', 'ram')
            flavor = flavors[0]

            if not flavors:
                raise serializers.ValidationError('No matching flavor found.')

        # validate flavor
        requirements = list(filter(lambda x: x[0] in list(roles),
                                   settings.WALDUR_RANCHER['ROLE_REQUIREMENT'].items()))
        cpu_requirements = max([t[1]['CPU'] for t in requirements])
        ram_requirements = max([t[1]['RAM'] for t in requirements])

        if flavor.cores < cpu_requirements:
            raise serializers.ValidationError('Flavor %s does not meet requirements. CPU requirement is %s'
                                              % (flavor, cpu_requirements))

        if flavor.ram < ram_requirements:
            raise serializers.ValidationError('Flavor %s does not meet requirements. RAM requirement is %s'
                                              % (flavor, ram_requirements))

        try:
            tenant_spl = openstack_tenant_models.OpenStackTenantServiceProjectLink.objects.get(
                project=project,
                service__settings=tenant_settings)
        except ObjectDoesNotExist:
            raise serializers.ValidationError(
                'Service project link for service %s and project %s is not found.' % (
                    tenant_settings.name, project.name
                ))

        try:
            base_image_name = rancher_spl.service.settings.get_option('base_image_name')
            image = openstack_tenant_models.Image.objects.get(
                name=base_image_name,
                settings=tenant_settings)
        except ObjectDoesNotExist:
            raise serializers.ValidationError('No matching image found.')

        try:
            group = openstack_tenant_models.SecurityGroup.objects.get(
                name='default',
                settings=tenant_settings)
        except ObjectDoesNotExist:
            raise serializers.ValidationError('No matching group found.')

        node['initial_data'] = {
            'flavor': flavor.uuid.hex,
            'vcpu': flavor.cores,
            'ram': flavor.ram,
            'image': image.uuid.hex,
            'subnet': subnet.uuid.hex,
            'tenant_service_project_link': tenant_spl.id,
            'group': group.uuid.hex,
            'system_volume_size': system_volume_size,
            'system_volume_type': system_volume_type,
            'data_volumes': [{
                'size': volume['size'],
                'volume_type': volume.get('volume_type') and volume.get('volume_type').uuid.hex,
            } for volume in data_volumes]
        }

        if 'controlplane' in list(roles):
            node['controlplane_role'] = True
        if 'etcd' in list(roles):
            node['etcd_role'] = True
        if 'worker' in list(roles):
            node['worker_role'] = True

        name = cluster_name + '_rancher_node'
        unique_name = get_unique_node_name(name, tenant_spl.id, rancher_spl)
        node['name'] = unique_name

        # check quotas
        quota_sources = [
            tenant_spl,
            tenant_spl.project,
            tenant_spl.customer,
            tenant_spl.service,
            tenant_spl.service.settings
        ]

        for quota_name in ['storage', 'vcpu', 'ram']:
            requested = sum([node['initial_data'][quota_name] for node in nodes])

            for source in quota_sources:
                try:
                    quota = source.quotas.get(name=quota_name)
                    if quota.limit != -1 and (quota.usage + requested > quota.limit):
                        raise quotas_exceptions.QuotaValidationError(
                            _('"%(name)s" quota is over limit. Required: %(usage)s, limit: %(limit)s.') % dict(
                                name=quota_name, usage=quota.usage + requested, limit=quota.limit))
                except ObjectDoesNotExist:
                    pass

def format_disk_id(index):
    return '/dev/vd' + (chr(ord('a') + index))

def format_node_command(node):
    roles_command = []

    if node.controlplane_role:
        roles_command.append('--controlplane')

    if node.etcd_role:
        roles_command.append('--etcd')

    if node.worker_role:
        roles_command.append('--worker')

    return node.cluster.node_command + ' ' + ' '.join(roles_command)

def format_node_cloud_config(node):
    node_command = format_node_command(node)
    user_data = settings.WALDUR_RANCHER['RANCHER_NODE_CLOUD_INIT_TEMPLATE'].format(command=node_command)
    data_volumes = sorted(node.initial_data.get('data_volumes'))

    if data_volumes:
        conf = yaml.parse(user_data)

        # First volume is reserved for system volume, other volumes are data volumes

        conf['mounts'] = [
            [format_disk_id(index + 1), volume['mount_point']]
            for index, volume in enumerate(data_volumes)
        ]

        conf['fs_setup'] = [
            {'device': format_disk_id(index + 1), 'filesystem': 'ext4'}
            for index, volume in enumerate(data_volumes)
        ]
        user_data = yaml.dump(conf)

    return user_data
<|endoftext|>