<commit_before># coding=utf-8
# Copyright 2017-2019 The THUMT Authors

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import copy

import tensorflow as tf
import thumt.interface as interface
import thumt.layers as layers
import thumt.utils.getter as getter

def _copy_through(time, length, output, new_output):
    copy_cond = (time >= length)
    return tf.where(copy_cond, output, new_output)

def _gru_encoder(cell, inputs, sequence_length, initial_state, dtype=None):
    # Assume that the underlying cell is GRUCell-like
    output_size = cell.output_size
    dtype = dtype or inputs.dtype

    batch = tf.shape(inputs)[0]
    time_steps = tf.shape(inputs)[1]

    zero_output = tf.zeros([batch, output_size], dtype)

    if initial_state is None:
        initial_state = cell.zero_state(batch, dtype)

    input_ta = tf.TensorArray(dtype, time_steps,
                              tensor_array_name="input_array")
    output_ta = tf.TensorArray(dtype, time_steps,
                               tensor_array_name="output_array")
    input_ta = input_ta.unstack(tf.transpose(inputs, [1, 0, 2]))

    def loop_func(t, out_ta, state):
        inp_t = input_ta.read(t)
        cell_output, new_state = cell(inp_t, state)
        cell_output = _copy_through(t, sequence_length, zero_output,
                                    cell_output)
        new_state = _copy_through(t, sequence_length, state, new_state)
        out_ta = out_ta.write(t, cell_output)
        return t + 1, out_ta, new_state

    time = tf.constant(0, dtype=tf.int32, name="time")
    loop_vars = (time, output_ta, initial_state)

    outputs = tf.while_loop(lambda t, *_: t < time_steps, loop_func,
                            loop_vars, parallel_iterations=32,
                            swap_memory=True)

    output_final_ta = outputs[1]
    final_state = outputs[2]

    all_output = output_final_ta.stack()
    all_output.set_shape([None, None, output_size])
    all_output = tf.transpose(all_output, [1, 0, 2])

    return all_output, final_state

def _encoder(cell_fw, cell_bw, inputs, sequence_length, dtype=None,
             scope=None):
    with tf.variable_scope(scope or "encoder",
                           values=[inputs, sequence_length]):
        inputs_fw = inputs
        inputs_bw = tf.reverse_sequence(inputs, sequence_length,
                                        batch_axis=0, seq_axis=1)

        with tf.variable_scope("forward"):
            output_fw, state_fw = _gru_encoder(cell_fw, inputs_fw,
                                               sequence_length, None,
                                               dtype=dtype)

        with tf.variable_scope("backward"):
            output_bw, state_bw = _gru_encoder(cell_bw, inputs_bw,
                                               sequence_length, None,
                                               dtype=dtype)
            output_bw = tf.reverse_sequence(output_bw, sequence_length,
                                            batch_axis=0, seq_axis=1)

        results = {
            "annotation": tf.concat([output_fw, output_bw], axis=2),
            "outputs": {
                "forward": output_fw,
                "backward": output_bw
            },
            "final_states": {
                "forward": state_fw,
                "backward": state_bw
            }
        }

        return results

def _decoder(cell, inputs, memory, sequence_length, initial_state, dtype=None,
             scope=None):
    # Assume that the underlying cell is GRUCell-like
    batch = tf.shape(inputs)[0]
    time_steps = tf.shape(inputs)[1]
    dtype = dtype or inputs.dtype
    output_size = cell.output_size
    zero_output = tf.zeros([batch, output_size], dtype)
    zero_value = tf.zeros([batch, memory.shape[-1].value], dtype)

    with tf.variable_scope(scope or "decoder", dtype=dtype):
        inputs = tf.transpose(inputs, [1, 0, 2])
        mem_mask = tf.sequence_mask(sequence_length["source"],
                                    maxlen=tf.shape(memory)[1],
                                    dtype=dtype)
        bias = layers.attention.attention_bias(mem_mask, "masking",
                                               dtype=dtype)
        bias = tf.squeeze(bias, axis=[1, 2])
        cache = layers.attention.attention(None, memory, None, output_size)

        input_ta = tf.TensorArray(dtype, time_steps,
                                  tensor_array_name="input_array")
        output_ta = tf.TensorArray(dtype, time_steps,
                                   tensor_array_name="output_array")
        value_ta = tf.TensorArray(dtype, time_steps,
                                  tensor_array_name="value_array")
        alpha_ta = tf.TensorArray(dtype, time_steps,
                                  tensor_array_name="alpha_array")
        input_ta = input_ta.unstack(inputs)
        initial_state = layers.nn.linear(initial_state, output_size, True,
                                         False, scope="s_transform")
        initial_state = tf.tanh(initial_state)

        def loop_func(t, out_ta, att_ta, val_ta, state, cache_key):
            inp_t = input_ta.read(t)
            results = layers.attention.attention(state, memory, bias,
                                                 output_size,
                                                 cache={"key": cache_key})
            alpha = results["weight"]
            context = results["value"]
            cell_input = [inp_t, context]
            cell_output, new_state = cell(cell_input, state)
            cell_output = _copy_through(t, sequence_length["target"],
                                        zero_output, cell_output)
            new_state = _copy_through(t, sequence_length["target"], state,
                                      new_state)
            new_value = _copy_through(t, sequence_length["target"], zero_value,
                                      context)

            out_ta = out_ta.write(t, cell_output)
            att_ta = att_ta.write(t, alpha)
            val_ta = val_ta.write(t, new_value)
            cache_key = tf.identity(cache_key)
            return t + 1, out_ta, att_ta, val_ta, new_state, cache_key

        time = tf.constant(0, dtype=tf.int32, name="time")
        loop_vars = (time, output_ta, alpha_ta, value_ta, initial_state,
                     cache["key"])

        outputs = tf.while_loop(lambda t, *_: t < time_steps,
                                loop_func, loop_vars,
                                parallel_iterations=32,
                                swap_memory=True)

        output_final_ta = outputs[1]
        value_final_ta = outputs[3]

        final_output = output_final_ta.stack()
        final_output.set_shape([None, None, output_size])
        final_output = tf.transpose(final_output, [1, 0, 2])

        final_value = value_final_ta.stack()
        final_value.set_shape([None, None, memory.shape[-1].value])
        final_value = tf.transpose(final_value, [1, 0, 2])

        result = {
            "outputs": final_output,
            "values": final_value,
            "initial_state": initial_state
        }

    return result

def model_graph(features, mode, params):
    src_vocab_size = len(params.vocabulary["source"])
    tgt_vocab_size = len(params.vocabulary["target"])
    dtype = tf.get_variable_scope().dtype

    with tf.variable_scope("source_embedding"):
        src_emb = tf.get_variable("embedding",
                                  [src_vocab_size, params.embedding_size])
        src_bias = tf.get_variable("bias", [params.embedding_size])
        src_inputs = tf.nn.embedding_lookup(src_emb, features["source"])

    with tf.variable_scope("target_embedding"):
        tgt_emb = tf.get_variable("embedding",
                                  [tgt_vocab_size, params.embedding_size])
        tgt_bias = tf.get_variable("bias", [params.embedding_size])
        tgt_inputs = tf.nn.embedding_lookup(tgt_emb, features["target"])

    src_inputs = tf.nn.bias_add(src_inputs, src_bias)
    tgt_inputs = tf.nn.bias_add(tgt_inputs, tgt_bias)

    if params.dropout and not params.use_variational_dropout:
        src_inputs = tf.nn.dropout(src_inputs, 1.0 - params.dropout)
        tgt_inputs = tf.nn.dropout(tgt_inputs, 1.0 - params.dropout)

    # encoder
    cell_fw = layers.rnn_cell.LegacyGRUCell(params.hidden_size)
    cell_bw = layers.rnn_cell.LegacyGRUCell(params.hidden_size)

    if params.use_variational_dropout:
        cell_fw = tf.nn.rnn_cell.DropoutWrapper(
            cell_fw,
            input_keep_prob=1.0 - params.dropout,
            output_keep_prob=1.0 - params.dropout,
            state_keep_prob=1.0 - params.dropout,
            variational_recurrent=True,
            input_size=params.embedding_size,
            dtype=dtype
        )
        cell_bw = tf.nn.rnn_cell.DropoutWrapper(
            cell_bw,
            input_keep_prob=1.0 - params.dropout,
            output_keep_prob=1.0 - params.dropout,
            state_keep_prob=1.0 - params.dropout,
            variational_recurrent=True,
            input_size=params.embedding_size,
            dtype=dtype
        )

    encoder_output = _encoder(cell_fw, cell_bw, src_inputs,
                              features["source_length"], dtype=dtype)

    # decoder
    cell = layers.rnn_cell.LegacyGRUCell(params.hidden_size)

    if params.use_variational_dropout:
        cell = tf.nn.rnn_cell.DropoutWrapper(
            cell,
            input_keep_prob=1.0 - params.dropout,
            output_keep_prob=1.0 - params.dropout,
            state_keep_prob=1.0 - params.dropout,
            variational_recurrent=True,
            # input + context
            input_size=params.embedding_size + 2 * params.hidden_size,
            dtype=dtype
        )

    length = {
        "source": features["source_length"],
        "target": features["target_length"]
    }
    initial_state = encoder_output["final_states"]["backward"]
    decoder_output = _decoder(cell, tgt_inputs, encoder_output["annotation"],
                              length, initial_state, dtype=dtype)

    # Shift left
    shifted_tgt_inputs = tf.pad(tgt_inputs, [[0, 0], [1, 0], [0, 0]])
    shifted_tgt_inputs = shifted_tgt_inputs[:, :-1, :]

    all_outputs = tf.concat(
        [
            tf.expand_dims(decoder_output["initial_state"], axis=1),
            decoder_output["outputs"],
        ],
        axis=1
    )
    shifted_outputs = all_outputs[:, :-1, :]

    maxout_features = [
        shifted_tgt_inputs,
        shifted_outputs,
        decoder_output["values"]
    ]
    maxout_size = params.hidden_size // params.maxnum

    if mode is "infer":
        # Special case for non-incremental decoding
        maxout_features = [
            shifted_tgt_inputs[:, -1, :],
            shifted_outputs[:, -1, :],
            decoder_output["values"][:, -1, :]
        ]
        maxhid = layers.nn.maxout(maxout_features, maxout_size, params.maxnum,
                                  concat=False)
        readout = layers.nn.linear(maxhid, params.embedding_size, False,
                                   False, scope="deepout")

        # Prediction
        logits = layers.nn.linear(readout, tgt_vocab_size, True, False,
                                  scope="softmax")

        return tf.nn.log_softmax(logits)

    maxhid = layers.nn.maxout(maxout_features, maxout_size, params.maxnum,
                              concat=False)
    readout = layers.nn.linear(maxhid, params.embedding_size, False, False,
                               scope="deepout")

    if params.dropout and not params.use_variational_dropout:
        readout = tf.nn.dropout(readout, 1.0 - params.dropout)

    # Prediction
    logits = layers.nn.linear(readout, tgt_vocab_size, True, False,
                              scope="softmax")
    logits = tf.reshape(logits, [-1, tgt_vocab_size])
    labels = features["target"]

    ce = layers.nn.smoothed_softmax_cross_entropy_with_logits(
        logits=logits,
        labels=labels,
        smoothing=params.label_smoothing,
        normalize=True
    )

    ce = tf.reshape(ce, tf.shape(labels))
    tgt_mask = tf.to_float(
        tf.sequence_mask(
            features["target_length"],
            maxlen=tf.shape(features["target"])[1]
        )
    )

    if mode == "eval":
        return -tf.reduce_sum(ce * tgt_mask, axis=1)

    loss = tf.reduce_sum(ce * tgt_mask) / tf.reduce_sum(tgt_mask)

    return loss

class RNNsearch(interface.NMTModel):

    def __init__(self, params, scope="rnnsearch"):
        super(RNNsearch, self).__init__(params=params, scope=scope)

    def get_training_func(self, initializer, regularizer=None, dtype=None):
        def training_fn(features, params=None, reuse=None):
            if params is None:
                params = self.parameters

            if dtype != tf.float32:
                custom_getter = getter.fp32_variable_getter
            else:
                custom_getter = None

            with tf.variable_scope(self._scope, initializer=initializer,
                                   regularizer=regularizer, reuse=reuse,
                                   custom_getter=custom_getter, dtype=dtype):
                loss = model_graph(features, "train", params)
                return loss

        return training_fn

    def get_evaluation_func(self):
        def evaluation_fn(features, params=None):
            if params is None:
                params = copy.copy(self.parameters)
            else:
                params = copy.copy(params)

            params.dropout = 0.0
            params.use_variational_dropout = False
            params.label_smoothing = 0.0

            with tf.variable_scope(self._scope):
                score = model_graph(features, "eval", params)

            return score

        return evaluation_fn

    def get_inference_func(self):
        def inference_fn(features, params=None):
            if params is None:
                params = copy.copy(self.parameters)
            else:
                params = copy.copy(params)

            params.dropout = 0.0
            params.use_variational_dropout = False
            params.label_smoothing = 0.0

            with tf.variable_scope(self._scope):
                log_prob = model_graph(features, "infer", params)

            return log_prob

        return inference_fn

    @staticmethod
    def get_name():
        return "rnnsearch"

    @staticmethod
    def get_parameters():
        params = tf.contrib.training.HParams(
            # vocabulary
            pad="<pad>",
            unk="<unk>",
            eos="<eos>",
            bos="<eos>",
            append_eos=False,
            # model
            rnn_cell="LegacyGRUCell",
            embedding_size=620,
            hidden_size=1000,
            maxnum=2,
            # regularization
            dropout=0.2,
            use_variational_dropout=False,
            label_smoothing=0.1,
            constant_batch_size=True,
            batch_size=128,
            max_length=60,
            clip_grad_norm=5.0
        )

        return params
<commit_msg>Use == to compare strings<commit_after># coding=utf-8
# Copyright 2017-2019 The THUMT Authors

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import copy

import tensorflow as tf
import thumt.interface as interface
import thumt.layers as layers
import thumt.utils.getter as getter

def _copy_through(time, length, output, new_output):
    copy_cond = (time >= length)
    return tf.where(copy_cond, output, new_output)

def _gru_encoder(cell, inputs, sequence_length, initial_state, dtype=None):
    # Assume that the underlying cell is GRUCell-like
    output_size = cell.output_size
    dtype = dtype or inputs.dtype

    batch = tf.shape(inputs)[0]
    time_steps = tf.shape(inputs)[1]

    zero_output = tf.zeros([batch, output_size], dtype)

    if initial_state is None:
        initial_state = cell.zero_state(batch, dtype)

    input_ta = tf.TensorArray(dtype, time_steps,
                              tensor_array_name="input_array")
    output_ta = tf.TensorArray(dtype, time_steps,
                               tensor_array_name="output_array")
    input_ta = input_ta.unstack(tf.transpose(inputs, [1, 0, 2]))

    def loop_func(t, out_ta, state):
        inp_t = input_ta.read(t)
        cell_output, new_state = cell(inp_t, state)
        cell_output = _copy_through(t, sequence_length, zero_output,
                                    cell_output)
        new_state = _copy_through(t, sequence_length, state, new_state)
        out_ta = out_ta.write(t, cell_output)
        return t + 1, out_ta, new_state

    time = tf.constant(0, dtype=tf.int32, name="time")
    loop_vars = (time, output_ta, initial_state)

    outputs = tf.while_loop(lambda t, *_: t < time_steps, loop_func,
                            loop_vars, parallel_iterations=32,
                            swap_memory=True)

    output_final_ta = outputs[1]
    final_state = outputs[2]

    all_output = output_final_ta.stack()
    all_output.set_shape([None, None, output_size])
    all_output = tf.transpose(all_output, [1, 0, 2])

    return all_output, final_state

def _encoder(cell_fw, cell_bw, inputs, sequence_length, dtype=None,
             scope=None):
    with tf.variable_scope(scope or "encoder",
                           values=[inputs, sequence_length]):
        inputs_fw = inputs
        inputs_bw = tf.reverse_sequence(inputs, sequence_length,
                                        batch_axis=0, seq_axis=1)

        with tf.variable_scope("forward"):
            output_fw, state_fw = _gru_encoder(cell_fw, inputs_fw,
                                               sequence_length, None,
                                               dtype=dtype)

        with tf.variable_scope("backward"):
            output_bw, state_bw = _gru_encoder(cell_bw, inputs_bw,
                                               sequence_length, None,
                                               dtype=dtype)
            output_bw = tf.reverse_sequence(output_bw, sequence_length,
                                            batch_axis=0, seq_axis=1)

        results = {
            "annotation": tf.concat([output_fw, output_bw], axis=2),
            "outputs": {
                "forward": output_fw,
                "backward": output_bw
            },
            "final_states": {
                "forward": state_fw,
                "backward": state_bw
            }
        }

        return results

def _decoder(cell, inputs, memory, sequence_length, initial_state, dtype=None,
             scope=None):
    # Assume that the underlying cell is GRUCell-like
    batch = tf.shape(inputs)[0]
    time_steps = tf.shape(inputs)[1]
    dtype = dtype or inputs.dtype
    output_size = cell.output_size
    zero_output = tf.zeros([batch, output_size], dtype)
    zero_value = tf.zeros([batch, memory.shape[-1].value], dtype)

    with tf.variable_scope(scope or "decoder", dtype=dtype):
        inputs = tf.transpose(inputs, [1, 0, 2])
        mem_mask = tf.sequence_mask(sequence_length["source"],
                                    maxlen=tf.shape(memory)[1],
                                    dtype=dtype)
        bias = layers.attention.attention_bias(mem_mask, "masking",
                                               dtype=dtype)
        bias = tf.squeeze(bias, axis=[1, 2])
        cache = layers.attention.attention(None, memory, None, output_size)

        input_ta = tf.TensorArray(dtype, time_steps,
                                  tensor_array_name="input_array")
        output_ta = tf.TensorArray(dtype, time_steps,
                                   tensor_array_name="output_array")
        value_ta = tf.TensorArray(dtype, time_steps,
                                  tensor_array_name="value_array")
        alpha_ta = tf.TensorArray(dtype, time_steps,
                                  tensor_array_name="alpha_array")
        input_ta = input_ta.unstack(inputs)
        initial_state = layers.nn.linear(initial_state, output_size, True,
                                         False, scope="s_transform")
        initial_state = tf.tanh(initial_state)

        def loop_func(t, out_ta, att_ta, val_ta, state, cache_key):
            inp_t = input_ta.read(t)
            results = layers.attention.attention(state, memory, bias,
                                                 output_size,
                                                 cache={"key": cache_key})
            alpha = results["weight"]
            context = results["value"]
            cell_input = [inp_t, context]
            cell_output, new_state = cell(cell_input, state)
            cell_output = _copy_through(t, sequence_length["target"],
                                        zero_output, cell_output)
            new_state = _copy_through(t, sequence_length["target"], state,
                                      new_state)
            new_value = _copy_through(t, sequence_length["target"], zero_value,
                                      context)

            out_ta = out_ta.write(t, cell_output)
            att_ta = att_ta.write(t, alpha)
            val_ta = val_ta.write(t, new_value)
            cache_key = tf.identity(cache_key)
            return t + 1, out_ta, att_ta, val_ta, new_state, cache_key

        time = tf.constant(0, dtype=tf.int32, name="time")
        loop_vars = (time, output_ta, alpha_ta, value_ta, initial_state,
                     cache["key"])

        outputs = tf.while_loop(lambda t, *_: t < time_steps,
                                loop_func, loop_vars,
                                parallel_iterations=32,
                                swap_memory=True)

        output_final_ta = outputs[1]
        value_final_ta = outputs[3]

        final_output = output_final_ta.stack()
        final_output.set_shape([None, None, output_size])
        final_output = tf.transpose(final_output, [1, 0, 2])

        final_value = value_final_ta.stack()
        final_value.set_shape([None, None, memory.shape[-1].value])
        final_value = tf.transpose(final_value, [1, 0, 2])

        result = {
            "outputs": final_output,
            "values": final_value,
            "initial_state": initial_state
        }

    return result

def model_graph(features, mode, params):
    src_vocab_size = len(params.vocabulary["source"])
    tgt_vocab_size = len(params.vocabulary["target"])
    dtype = tf.get_variable_scope().dtype

    with tf.variable_scope("source_embedding"):
        src_emb = tf.get_variable("embedding",
                                  [src_vocab_size, params.embedding_size])
        src_bias = tf.get_variable("bias", [params.embedding_size])
        src_inputs = tf.nn.embedding_lookup(src_emb, features["source"])

    with tf.variable_scope("target_embedding"):
        tgt_emb = tf.get_variable("embedding",
                                  [tgt_vocab_size, params.embedding_size])
        tgt_bias = tf.get_variable("bias", [params.embedding_size])
        tgt_inputs = tf.nn.embedding_lookup(tgt_emb, features["target"])

    src_inputs = tf.nn.bias_add(src_inputs, src_bias)
    tgt_inputs = tf.nn.bias_add(tgt_inputs, tgt_bias)

    if params.dropout and not params.use_variational_dropout:
        src_inputs = tf.nn.dropout(src_inputs, 1.0 - params.dropout)
        tgt_inputs = tf.nn.dropout(tgt_inputs, 1.0 - params.dropout)

    # encoder
    cell_fw = layers.rnn_cell.LegacyGRUCell(params.hidden_size)
    cell_bw = layers.rnn_cell.LegacyGRUCell(params.hidden_size)

    if params.use_variational_dropout:
        cell_fw = tf.nn.rnn_cell.DropoutWrapper(
            cell_fw,
            input_keep_prob=1.0 - params.dropout,
            output_keep_prob=1.0 - params.dropout,
            state_keep_prob=1.0 - params.dropout,
            variational_recurrent=True,
            input_size=params.embedding_size,
            dtype=dtype
        )
        cell_bw = tf.nn.rnn_cell.DropoutWrapper(
            cell_bw,
            input_keep_prob=1.0 - params.dropout,
            output_keep_prob=1.0 - params.dropout,
            state_keep_prob=1.0 - params.dropout,
            variational_recurrent=True,
            input_size=params.embedding_size,
            dtype=dtype
        )

    encoder_output = _encoder(cell_fw, cell_bw, src_inputs,
                              features["source_length"], dtype=dtype)

    # decoder
    cell = layers.rnn_cell.LegacyGRUCell(params.hidden_size)

    if params.use_variational_dropout:
        cell = tf.nn.rnn_cell.DropoutWrapper(
            cell,
            input_keep_prob=1.0 - params.dropout,
            output_keep_prob=1.0 - params.dropout,
            state_keep_prob=1.0 - params.dropout,
            variational_recurrent=True,
            # input + context
            input_size=params.embedding_size + 2 * params.hidden_size,
            dtype=dtype
        )

    length = {
        "source": features["source_length"],
        "target": features["target_length"]
    }
    initial_state = encoder_output["final_states"]["backward"]
    decoder_output = _decoder(cell, tgt_inputs, encoder_output["annotation"],
                              length, initial_state, dtype=dtype)

    # Shift left
    shifted_tgt_inputs = tf.pad(tgt_inputs, [[0, 0], [1, 0], [0, 0]])
    shifted_tgt_inputs = shifted_tgt_inputs[:, :-1, :]

    all_outputs = tf.concat(
        [
            tf.expand_dims(decoder_output["initial_state"], axis=1),
            decoder_output["outputs"],
        ],
        axis=1
    )
    shifted_outputs = all_outputs[:, :-1, :]

    maxout_features = [
        shifted_tgt_inputs,
        shifted_outputs,
        decoder_output["values"]
    ]
    maxout_size = params.hidden_size // params.maxnum

    if mode == "infer":
        # Special case for non-incremental decoding
        maxout_features = [
            shifted_tgt_inputs[:, -1, :],
            shifted_outputs[:, -1, :],
            decoder_output["values"][:, -1, :]
        ]
        maxhid = layers.nn.maxout(maxout_features, maxout_size, params.maxnum,
                                  concat=False)
        readout = layers.nn.linear(maxhid, params.embedding_size, False,
                                   False, scope="deepout")

        # Prediction
        logits = layers.nn.linear(readout, tgt_vocab_size, True, False,
                                  scope="softmax")

        return tf.nn.log_softmax(logits)

    maxhid = layers.nn.maxout(maxout_features, maxout_size, params.maxnum,
                              concat=False)
    readout = layers.nn.linear(maxhid, params.embedding_size, False, False,
                               scope="deepout")

    if params.dropout and not params.use_variational_dropout:
        readout = tf.nn.dropout(readout, 1.0 - params.dropout)

    # Prediction
    logits = layers.nn.linear(readout, tgt_vocab_size, True, False,
                              scope="softmax")
    logits = tf.reshape(logits, [-1, tgt_vocab_size])
    labels = features["target"]

    ce = layers.nn.smoothed_softmax_cross_entropy_with_logits(
        logits=logits,
        labels=labels,
        smoothing=params.label_smoothing,
        normalize=True
    )

    ce = tf.reshape(ce, tf.shape(labels))
    tgt_mask = tf.to_float(
        tf.sequence_mask(
            features["target_length"],
            maxlen=tf.shape(features["target"])[1]
        )
    )

    if mode == "eval":
        return -tf.reduce_sum(ce * tgt_mask, axis=1)

    loss = tf.reduce_sum(ce * tgt_mask) / tf.reduce_sum(tgt_mask)

    return loss

class RNNsearch(interface.NMTModel):

    def __init__(self, params, scope="rnnsearch"):
        super(RNNsearch, self).__init__(params=params, scope=scope)

    def get_training_func(self, initializer, regularizer=None, dtype=None):
        def training_fn(features, params=None, reuse=None):
            if params is None:
                params = self.parameters

            if dtype != tf.float32:
                custom_getter = getter.fp32_variable_getter
            else:
                custom_getter = None

            with tf.variable_scope(self._scope, initializer=initializer,
                                   regularizer=regularizer, reuse=reuse,
                                   custom_getter=custom_getter, dtype=dtype):
                loss = model_graph(features, "train", params)
                return loss

        return training_fn

    def get_evaluation_func(self):
        def evaluation_fn(features, params=None):
            if params is None:
                params = copy.copy(self.parameters)
            else:
                params = copy.copy(params)

            params.dropout = 0.0
            params.use_variational_dropout = False
            params.label_smoothing = 0.0

            with tf.variable_scope(self._scope):
                score = model_graph(features, "eval", params)

            return score

        return evaluation_fn

    def get_inference_func(self):
        def inference_fn(features, params=None):
            if params is None:
                params = copy.copy(self.parameters)
            else:
                params = copy.copy(params)

            params.dropout = 0.0
            params.use_variational_dropout = False
            params.label_smoothing = 0.0

            with tf.variable_scope(self._scope):
                log_prob = model_graph(features, "infer", params)

            return log_prob

        return inference_fn

    @staticmethod
    def get_name():
        return "rnnsearch"

    @staticmethod
    def get_parameters():
        params = tf.contrib.training.HParams(
            # vocabulary
            pad="<pad>",
            unk="<unk>",
            eos="<eos>",
            bos="<eos>",
            append_eos=False,
            # model
            rnn_cell="LegacyGRUCell",
            embedding_size=620,
            hidden_size=1000,
            maxnum=2,
            # regularization
            dropout=0.2,
            use_variational_dropout=False,
            label_smoothing=0.1,
            constant_batch_size=True,
            batch_size=128,
            max_length=60,
            clip_grad_norm=5.0
        )

        return params
<|endoftext|>


<commit_before>"""
A subclass of tiddlywebplugins.sqlalchemy with mysql specific
tune ups, including a fulltext index and 'indexer' support
for accelerating filters.

http://github.com/cdent/tiddlywebplugins.mysql
http://tiddlyweb.com/

ALPHA!
"""
from __future__ import absolute_import

from sqlalchemy.engine import create_engine
from sqlalchemy.exc import ProgrammingError
from sqlalchemy.sql.expression import and_, or_, text as text_, alias
from sqlalchemy.sql import func

from sqlalchemy.dialects.mysql.base import VARCHAR, LONGTEXT

from tiddlyweb.model.tiddler import Tiddler
from tiddlyweb.serializer import Serializer
from tiddlyweb.store import StoreError

from tiddlywebplugins.sqlalchemy import (Store as SQLStore,
        sTiddler, sRevision, sTag, metadata, Session,
        field_table, tiddler_table, revision_table, bag_table,
        policy_table, recipe_table, role_table, user_table, tag_table)

from tiddlyweb.filters import FilterIndexRefused

from pyparsing import (printables, alphanums, OneOrMore, Group,
        Combine, Suppress, Optional, FollowedBy, Literal, CharsNotIn,
        Word, Keyword, Empty, White, Forward, QuotedString, StringEnd,
        ParseException)

#import logging
#logging.basicConfig()
#logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
#logging.getLogger('sqlalchemy.orm.unitofwork').setLevel(logging.DEBUG)
#logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG)

__version__ = '0.9.9'

ENGINE = None
MAPPED = False
TABLES = [field_table, revision_table, tiddler_table, bag_table,
        policy_table, recipe_table, role_table, user_table, tag_table]

class Store(SQLStore):

    def _init_store(self):
        """
        Establish the database engine and session,
        creating tables if needed.
        """
        global ENGINE, MAPPED
        if not ENGINE:
            ENGINE = create_engine(self._db_config(),
                    pool_recycle=3600,
                    pool_size=20,  # XXX these three ought to come from config
                    max_overflow=-1,
                    pool_timeout=2)
            metadata.bind = ENGINE
            Session.configure(bind=ENGINE)
        self.session = Session()
        self.serializer = Serializer('text')
        self.parser = DEFAULT_PARSER
        self.producer = Producer()

        if not MAPPED:
            for table in TABLES:
                table.kwargs['mysql_charset'] = 'utf8'
                if table.name == 'revision' or table.name == 'tiddler':
                    for column in table.columns:
                        if (column.name == 'tiddler_title'
                                or column.name == 'title'):
                            column.type = VARCHAR(length=128,
                                    convert_unicode=True, collation='utf8_bin')
                        if column.name == 'tags':
                            column.type = VARCHAR(length=1024,
                                    convert_unicode=True, collation='utf8_bin')
                        if column.name == 'text':
                            column.type = LONGTEXT(convert_unicode=True,
                                    collation='utf8_bin')

            metadata.create_all(ENGINE)
            MAPPED = True

    def search(self, search_query=''):
        query = self.session.query(sTiddler.bag_name, sTiddler.title)
        query = query.join(sTiddler.current)
        try:
            try:
                ast = self.parser(search_query)[0]
                query = self.producer.produce(ast, query)
            except ParseException, exc:
                raise StoreError('failed to parse search query: %s' % exc)

            try:
                for stiddler in query.all():
                    yield Tiddler(unicode(stiddler.title),
                            unicode(stiddler.bag_name))
                self.session.close()
            except ProgrammingError, exc:
                raise StoreError('generated search SQL incorrect: %s' % exc)
        except:
            self.session.rollback()
            raise

def index_query(environ, **kwargs):
    store = environ['tiddlyweb.store']

    queries = []
    for key, value in kwargs.items():
        if '"' in value:
            # XXX The current parser is currently unable to deal with
            # nested quotes. Rather than running the risk of tweaking
            # the parser with unclear results, we instead just refuse
            # for now. Later this can be fixed for real.
            raise FilterIndexRefused('unable to process values with quotes')
        queries.append('%s:"%s"' % (key, value))
    query = ' '.join(queries)

    storage = store.storage

    try:
        tiddlers = storage.search(search_query=query)
        return (store.get(tiddler) for tiddler in tiddlers)
    except StoreError, exc:
        raise FilterIndexRefused('error in the store: %s' % exc)

# XXX borrowed from Whoosh
def _make_default_parser():
    escapechar = "\\"

    wordtext = CharsNotIn('\\():"{}[] ')
    escape = Suppress(escapechar) + (Word(printables, exact=1) | White(exact=1))
    wordtoken = Combine(OneOrMore(wordtext | escape))
# A plain old word.
    plainWord = Group(wordtoken).setResultsName("Word")

# A wildcard word containing * or ?.
    wildchars = Word("?*")
# Start with word chars and then have wild chars mixed in
    wildmixed = wordtoken + OneOrMore(wildchars + Optional(wordtoken))
# Or, start with wildchars, and then either a mixture of word and wild chars, or the next token
    wildstart = wildchars + (OneOrMore(wordtoken + Optional(wildchars)) | FollowedBy(White() | StringEnd()))
    wildcard = Group(Combine(wildmixed | wildstart)).setResultsName("Wildcard")

# A range of terms
    startfence = Literal("[") | Literal("{")
    endfence = Literal("]") | Literal("}")
    rangeitem = QuotedString('"') | wordtoken
    openstartrange = Group(Empty()) + Suppress(Keyword("TO") + White()) + Group(rangeitem)
    openendrange = Group(rangeitem) + Suppress(White() + Keyword("TO")) + Group(Empty())
    normalrange = Group(rangeitem) + Suppress(White() + Keyword("TO") + White()) + Group(rangeitem)
    range = Group(startfence + (normalrange | openstartrange | openendrange) + endfence).setResultsName("Range")

# A word-like thing
    generalWord = range | wildcard | plainWord

# A quoted phrase
    quotedPhrase = Group(QuotedString('"')).setResultsName("Quotes")

    expression = Forward()

# Parentheses can enclose (group) any expression
    parenthetical = Group((Suppress("(") + expression + Suppress(")"))).setResultsName("Group")

    boostableUnit = generalWord | quotedPhrase
    boostedUnit = Group(boostableUnit + Suppress("^") + Word("0123456789", ".0123456789")).setResultsName("Boost")

# The user can flag that a parenthetical group, quoted phrase, or word
# should be searched in a particular field by prepending 'fn:', where fn is
# the name of the field.
    fieldableUnit = parenthetical | boostedUnit | boostableUnit
    fieldedUnit = Group(Word(alphanums + "_" + "-") + Suppress(':') + fieldableUnit).setResultsName("Field")

# Units of content
    unit = fieldedUnit | fieldableUnit

# A unit may be "not"-ed.
    operatorNot = Group(Suppress(Keyword("not", caseless=True)) + Suppress(White()) + unit).setResultsName("Not")
    generalUnit = operatorNot | unit

    andToken = Keyword("AND", caseless=False)
    orToken = Keyword("OR", caseless=False)
    andNotToken = Keyword("ANDNOT", caseless=False)

    operatorAnd = Group(generalUnit + OneOrMore(Suppress(White()) + Suppress(andToken) + Suppress(White()) + generalUnit)).setResultsName("And")
    operatorOr = Group(generalUnit + OneOrMore(Suppress(White()) + Suppress(orToken) + Suppress(White()) + generalUnit)).setResultsName("Or")
    operatorAndNot = Group(unit + OneOrMore(Suppress(White()) + Suppress(andNotToken) + Suppress(White()) + unit)).setResultsName("AndNot")

    expression << (OneOrMore(operatorAnd | operatorOr | operatorAndNot | generalUnit | Suppress(White())) | Empty())

    toplevel = Group(expression).setResultsName("Toplevel") + StringEnd()

    return toplevel.parseString

DEFAULT_PARSER = _make_default_parser()

class Producer(object):

    def produce(self, ast, query):
        expressions = self._eval(ast, None)
        return query.filter(expressions)

    def _eval(self, node, fieldname):
        name = node.getName()
        return getattr(self, "_" + name)(node, fieldname)

    def _Toplevel(self, node, fieldname):
        expressions = []
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        return and_(*expressions)

    def _Word(self, node, fieldname):
        value = node[0]
        if fieldname:
            like = False
            if value.endswith('*'):
                value = value.replace('*', '%')
                like = True

            if fieldname == 'ftitle' or fieldname == 'title':
                fieldname = 'tiddler_title'
            if fieldname == 'fbag' or fieldname == 'bag':
                fieldname = 'bag_name'

            if fieldname == 'id':
                bag, title = value.split(':', 1)
                expression = and_(sRevision.bag_name == bag,
                        sRevision.tiddler_title == title)
            elif fieldname == 'tag':
                stag_alias = alias(tag_table)
                if like:
                    expression = and_(
                            stag_alias.c.revision_number == sRevision.number,
                            stag_alias.c.tag.like(value))
                else:
                    expression = and_(
                            stag_alias.c.revision_number == sRevision.number,
                            stag_alias.c.tag == value)
            elif hasattr(sRevision, fieldname):
                if like:
                    expression = (getattr(sRevision, fieldname).like(value))
                else:
                    expression = (getattr(sRevision, fieldname) == value)
            else:
                sfield_alias = alias(field_table)
                expression = and_(
                        sfield_alias.c.revision_number == sRevision.number,
                        sfield_alias.c.name == fieldname)
                if like:
                    expression = and_(expression,
                            sfield_alias.c.value.like(value))
                else:
                    expression = and_(expression,
                            sfield_alias.c.value == value)
        else:
            expression = (text_(
                'MATCH(revision.tiddler_title, revision.text) '
                + 'AGAINST(:query in boolean mode)')
                .params(query=value))
        return expression

    def _Field(self, node, fieldname):
        return self._Word(node[1], node[0])

    def _Group(self, node, fieldname):
        expressions = []
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        return and_(*expressions)

    def _Or(self, node, fieldname):
        expressions = []
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        return or_(*expressions)

    def _And(self, node, fieldname):
        expressions = []
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        return and_(*expressions)

    def _Quotes(self, node, fieldname):
        node[0] = '"%s"' % node[0]
        return self._Word(node, fieldname)
<commit_msg>Add in_or and in_and state management so we can do correct joins.<commit_after>"""
A subclass of tiddlywebplugins.sqlalchemy with mysql specific
tune ups, including a fulltext index and 'indexer' support
for accelerating filters.

http://github.com/cdent/tiddlywebplugins.mysql
http://tiddlyweb.com/

ALPHA!
"""
from __future__ import absolute_import

from sqlalchemy.engine import create_engine
from sqlalchemy.exc import ProgrammingError
from sqlalchemy.sql.expression import (and_, or_, text as text_, alias,
        join as join_)
from sqlalchemy.sql import func

from sqlalchemy.dialects.mysql.base import VARCHAR, LONGTEXT

from tiddlyweb.model.tiddler import Tiddler
from tiddlyweb.serializer import Serializer
from tiddlyweb.store import StoreError

from tiddlywebplugins.sqlalchemy import (Store as SQLStore,
        sTiddler, sRevision, sTag, metadata, Session,
        field_table, tiddler_table, revision_table, bag_table,
        policy_table, recipe_table, role_table, user_table, tag_table)

from tiddlyweb.filters import FilterIndexRefused

from pyparsing import (printables, alphanums, OneOrMore, Group,
        Combine, Suppress, Optional, FollowedBy, Literal, CharsNotIn,
        Word, Keyword, Empty, White, Forward, QuotedString, StringEnd,
        ParseException)

#import logging
#logging.basicConfig()
#logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
#logging.getLogger('sqlalchemy.orm.unitofwork').setLevel(logging.DEBUG)
#logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG)

__version__ = '0.9.9'

ENGINE = None
MAPPED = False
TABLES = [field_table, revision_table, tiddler_table, bag_table,
        policy_table, recipe_table, role_table, user_table, tag_table]

class Store(SQLStore):

    def _init_store(self):
        """
        Establish the database engine and session,
        creating tables if needed.
        """
        global ENGINE, MAPPED
        if not ENGINE:
            ENGINE = create_engine(self._db_config(),
                    pool_recycle=3600,
                    pool_size=20,  # XXX these three ought to come from config
                    max_overflow=-1,
                    pool_timeout=2)
            metadata.bind = ENGINE
            Session.configure(bind=ENGINE)
        self.session = Session()
        self.serializer = Serializer('text')
        self.parser = DEFAULT_PARSER
        self.producer = Producer()

        if not MAPPED:
            for table in TABLES:
                table.kwargs['mysql_charset'] = 'utf8'
                if table.name == 'revision' or table.name == 'tiddler':
                    for column in table.columns:
                        if (column.name == 'tiddler_title'
                                or column.name == 'title'):
                            column.type = VARCHAR(length=128,
                                    convert_unicode=True, collation='utf8_bin')
                        if column.name == 'text':
                            column.type = LONGTEXT(convert_unicode=True,
                                    collation='utf8_bin')
                if table.name == 'tag':
                    for column in table.columns:
                        if column.name == 'tag':
                            column.type = VARCHAR(length=256,
                                    convert_unicode=True, collation='utf8_bin')

            metadata.create_all(ENGINE)
            MAPPED = True

    def search(self, search_query=''):
        query = self.session.query(sRevision.bag_name, sRevision.tiddler_title)
        query = query.join(sTiddler.current)
        try:
            try:
                ast = self.parser(search_query)[0]
                query = self.producer.produce(ast, query)
            except ParseException, exc:
                raise StoreError('failed to parse search query: %s' % exc)

            try:
                for stiddler in query.all():
                    yield Tiddler(unicode(stiddler.tiddler_title),
                            unicode(stiddler.bag_name))
                self.session.close()
            except ProgrammingError, exc:
                raise StoreError('generated search SQL incorrect: %s' % exc)
        except:
            self.session.rollback()
            raise

def index_query(environ, **kwargs):
    store = environ['tiddlyweb.store']

    queries = []
    for key, value in kwargs.items():
        if '"' in value:
            # XXX The current parser is currently unable to deal with
            # nested quotes. Rather than running the risk of tweaking
            # the parser with unclear results, we instead just refuse
            # for now. Later this can be fixed for real.
            raise FilterIndexRefused('unable to process values with quotes')
        queries.append('%s:"%s"' % (key, value))
    query = ' '.join(queries)

    storage = store.storage

    try:
        tiddlers = storage.search(search_query=query)
        return (store.get(tiddler) for tiddler in tiddlers)
    except StoreError, exc:
        raise FilterIndexRefused('error in the store: %s' % exc)

# XXX borrowed from Whoosh
def _make_default_parser():
    escapechar = "\\"

    wordtext = CharsNotIn('\\():"{}[] ')
    escape = Suppress(escapechar) + (Word(printables, exact=1) | White(exact=1))
    wordtoken = Combine(OneOrMore(wordtext | escape))
# A plain old word.
    plainWord = Group(wordtoken).setResultsName("Word")

# A wildcard word containing * or ?.
    wildchars = Word("?*")
# Start with word chars and then have wild chars mixed in
    wildmixed = wordtoken + OneOrMore(wildchars + Optional(wordtoken))
# Or, start with wildchars, and then either a mixture of word and wild chars, or the next token
    wildstart = wildchars + (OneOrMore(wordtoken + Optional(wildchars)) | FollowedBy(White() | StringEnd()))
    wildcard = Group(Combine(wildmixed | wildstart)).setResultsName("Wildcard")

# A range of terms
    startfence = Literal("[") | Literal("{")
    endfence = Literal("]") | Literal("}")
    rangeitem = QuotedString('"') | wordtoken
    openstartrange = Group(Empty()) + Suppress(Keyword("TO") + White()) + Group(rangeitem)
    openendrange = Group(rangeitem) + Suppress(White() + Keyword("TO")) + Group(Empty())
    normalrange = Group(rangeitem) + Suppress(White() + Keyword("TO") + White()) + Group(rangeitem)
    range = Group(startfence + (normalrange | openstartrange | openendrange) + endfence).setResultsName("Range")

# A word-like thing
    generalWord = range | wildcard | plainWord

# A quoted phrase
    quotedPhrase = Group(QuotedString('"')).setResultsName("Quotes")

    expression = Forward()

# Parentheses can enclose (group) any expression
    parenthetical = Group((Suppress("(") + expression + Suppress(")"))).setResultsName("Group")

    boostableUnit = generalWord | quotedPhrase
    boostedUnit = Group(boostableUnit + Suppress("^") + Word("0123456789", ".0123456789")).setResultsName("Boost")

# The user can flag that a parenthetical group, quoted phrase, or word
# should be searched in a particular field by prepending 'fn:', where fn is
# the name of the field.
    fieldableUnit = parenthetical | boostedUnit | boostableUnit
    fieldedUnit = Group(Word(alphanums + "_" + "-") + Suppress(':') + fieldableUnit).setResultsName("Field")

# Units of content
    unit = fieldedUnit | fieldableUnit

# A unit may be "not"-ed.
    operatorNot = Group(Suppress(Keyword("not", caseless=True)) + Suppress(White()) + unit).setResultsName("Not")
    generalUnit = operatorNot | unit

    andToken = Keyword("AND", caseless=False)
    orToken = Keyword("OR", caseless=False)
    andNotToken = Keyword("ANDNOT", caseless=False)

    operatorAnd = Group(generalUnit + OneOrMore(Suppress(White()) + Suppress(andToken) + Suppress(White()) + generalUnit)).setResultsName("And")
    operatorOr = Group(generalUnit + OneOrMore(Suppress(White()) + Suppress(orToken) + Suppress(White()) + generalUnit)).setResultsName("Or")
    operatorAndNot = Group(unit + OneOrMore(Suppress(White()) + Suppress(andNotToken) + Suppress(White()) + unit)).setResultsName("AndNot")

    expression << (OneOrMore(operatorAnd | operatorOr | operatorAndNot | generalUnit | Suppress(White())) | Empty())

    toplevel = Group(expression).setResultsName("Toplevel") + StringEnd()

    return toplevel.parseString

DEFAULT_PARSER = _make_default_parser()

class Producer(object):

    def produce(self, ast, query):
        self.joined_tags = False
        self.joined_fields = False
        self.in_and = False
        self.in_or = False
        self.query = query
        expressions = self._eval(ast, None)
        return self.query.filter(expressions)

    def _eval(self, node, fieldname):
        name = node.getName()
        return getattr(self, "_" + name)(node, fieldname)

    def _Toplevel(self, node, fieldname):
        expressions = []
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        return and_(*expressions)

    def _Word(self, node, fieldname):
        value = node[0]
        if fieldname:
            like = False
            if value.endswith('*'):
                value = value.replace('*', '%')
                like = True

            if fieldname == 'ftitle' or fieldname == 'title':
                fieldname = 'tiddler_title'
            if fieldname == 'fbag' or fieldname == 'bag':
                fieldname = 'bag_name'

            if fieldname == 'id':
                bag, title = value.split(':', 1)
                expression = and_(sRevision.bag_name == bag,
                        sRevision.tiddler_title == title)
            elif fieldname == 'tag':
                if self.in_and:
                    tag_alias = alias(tag_table)
                    self.query = self.query.join((tag_alias,
                            (tag_alias.c.revision_number
                                == revision_table.c.number)))
                    if like:
                        expression = (tag_alias.c.tag.like(value))
                    else:
                        expression = (tag_alias.c.tag == value)
                else:
                    if not self.joined_tags:
                        self.query = self.query.join((tag_table,
                                (tag_table.c.revision_number
                                    == revision_table.c.number)))
                        if like:
                            expression = (tag_table.c.tag.like(value))
                        else:
                            expression = (tag_table.c.tag == value)
                        self.joined_tags = True
                    else:
                        if like:
                            expression = (tag_table.c.tag.like(value))
                        else:
                            expression = (tag_table.c.tag == value)
            elif hasattr(sRevision, fieldname):
                if like:
                    expression = (getattr(sRevision, fieldname).like(value))
                else:
                    expression = (getattr(sRevision, fieldname) == value)
            else:
                print 'fieldname', fieldname, value, self.joined_fields, self.in_and, self.in_or
                if self.in_and:
                    field_alias = alias(field_table)
                    self.query = self.query.join((field_alias,
                            (field_alias.c.revision_number
                                == revision_table.c.number)))
                    expression = (field_alias.c.name == fieldname)
                    if like:
                        expression = and_(expression,
                                field_alias.c.value.like(value))
                    else:
                        expression = and_(expression,
                                field_alias.c.value == value)
                else:
                    if not self.joined_fields:
                        self.query = self.query.join((field_table,
                                (field_table.c.revision_number == revision_table.c.number)))
                        expression = (field_table.c.name == fieldname)
                        if like:
                            expression = and_(expression,
                                    field_table.c.value.like(value))
                        else:
                            expression = and_(expression,
                                    field_table.c.value == value)
                        self.joined_fields = True
                    else:
                        expression = (field_table.c.name == fieldname)
                        if like:
                            expression = and_(expression,
                                    field_table.c.value.like(value))
                        else:
                            expression = and_(expression,
                                    field_table.c.value == value)
        else:
            expression = (text_(
                'MATCH(revision.tiddler_title, revision.text) '
                + 'AGAINST(:query in boolean mode)')
                .params(query=value))
        return expression

    def _Field(self, node, fieldname):
        return self._Word(node[1], node[0])

    def _Group(self, node, fieldname):
        expressions = []
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        return and_(*expressions)

    def _Or(self, node, fieldname):
        expressions = []
        self.in_or = True
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        self.in_or = False
        return or_(*expressions)

    def _And(self, node, fieldname):
        expressions = []
        self.in_and = True
        for subnode in node:
            expressions.append(self._eval(subnode, fieldname))
        self.in_and = False
        return and_(*expressions)

    def _Quotes(self, node, fieldname):
        node[0] = '"%s"' % node[0]
        return self._Word(node, fieldname)
<|endoftext|>


<commit_before>""" Palette Loader """

# Palette
orange = "#f60"
light_orange = "#f80"
dark_magenta = "#608"
light_magenta = "#f0f"
light_red = "#f00"
light_green = "#0d0"
white = "white"
black = "black"
light_gray = "g82"
cool_gray = "g50"
warm_gray = "g15"
blue = "#08f"
dark_blue = "#00f"

# Styles
STYLES = [
    ("frame_header", "", "", "", white, orange),
    ("frame_subheader", "", "", "", white, warm_gray),
    ("frame_excerpt", "", "", "", cool_gray, ""),
    ("frame_footer", "", "", "", white, warm_gray),
    ("body", "", "", "", white, ""),
    ("button_primary", "", "", "", white, ""),
    ("button_primary focus", "", "", "", white, light_green),
    ("button_secondary", "", "", "", white, ""),
    ("button_secondary focus", "", "", "", white, cool_gray),
    ("menu_button", "", "", "", white, cool_gray),
    ("menu_button focus", "", "", "", white, cool_gray),
    ("info_minor", "", "", "", warm_gray, ""),
    ("info_major", "", "", "", light_green, ""),
    ("info_context", "", "", "", cool_gray, ""),
    ("error_major", "", "", "", light_red, ""),
    ("status_info", "", "", "", light_green, warm_gray),
    ("status_error", "", "", "", light_red, warm_gray),
    ("string_input", "", "", "", white, cool_gray),
    ("string_input focus", "", "", "", white, orange),
    ("dialog", "", "", "", white, warm_gray),
    ("radio_input", "", "", "", white, warm_gray),
    ("radio_input focus", "", "", "", orange, warm_gray),
    ("header_title", "", "", "", orange, ""),
    ("pending_icon_on", "", "", "", blue, ""),
    ("pending_icon", "", "", "", blue, ""),
    ("error_icon", "", "", "", light_red, ""),
    ("success_icon", "", "", "", light_green, ""),
    ("column_header", "", "", "", white, blue),
    ('deploy_highlight_start', "", "", "", warm_gray, light_green),
    ('deploy_highlight_end', "", "", "", warm_gray, cool_gray),
    ('disabled_button', "", "", "", light_gray, warm_gray),
    ('disabled_button_focus', "", "", "", light_gray, warm_gray),
    ('divider_line', "", "", "", cool_gray, ""),
    ('filter', "", "", "", black, light_gray),
    ('filter_focus', "", "", "", white, warm_gray),
    ('focus', "", "", "", white, warm_gray),
    ('radio focus', "", "", "", white, dark_magenta),
    ('status_extra', "", "", "", light_gray, warm_gray),
    ('error', "", "", "", white, light_red),
    ('info', "", "", "", light_green, ''),
    ('label', "", "", "", light_gray, ''),
]
<commit_msg>update palette for white bgs<commit_after>""" Palette Loader """

# Palette
orange = "#f60"
light_orange = "#f80"
dark_magenta = "#608"
light_magenta = "#f0f"
light_red = "#f00"
light_green = "#0d0"
white = "white"
black = "black"
light_gray = "g82"
cool_gray = "g50"
warm_gray = "g15"
blue = "#08f"
dark_blue = "#00f"

# Styles
STYLES = [
    ("frame_header", "", "", "", white, orange),
    ("frame_subheader", "", "", "", white, warm_gray),
    ("frame_excerpt", "", "", "", cool_gray, ""),
    ("frame_footer", "", "", "", white, warm_gray),
    ("body", "", "", "", "", ""),
    ("button_primary", "", "", "", "", ""),
    ("button_primary focus", "", "", "", white, light_green),
    ("button_secondary", "", "", "", "", ""),
    ("button_secondary focus", "", "", "", white, cool_gray),
    ("menu_button", "", "", "", white, cool_gray),
    ("menu_button focus", "", "", "", white, cool_gray),
    ("info_minor", "", "", "", warm_gray, ""),
    ("info_major", "", "", "", light_green, ""),
    ("info_context", "", "", "", cool_gray, ""),
    ("error_major", "", "", "", light_red, ""),
    ("status_info", "", "", "", light_green, warm_gray),
    ("status_error", "", "", "", light_red, warm_gray),
    ("string_input", "", "", "", white, cool_gray),
    ("string_input focus", "", "", "", white, orange),
    ("dialog", "", "", "", white, warm_gray),
    ("radio_input", "", "", "", white, warm_gray),
    ("radio_input focus", "", "", "", orange, warm_gray),
    ("header_title", "", "", "", orange, ""),
    ("pending_icon_on", "", "", "", blue, ""),
    ("pending_icon", "", "", "", blue, ""),
    ("error_icon", "", "", "", light_red, ""),
    ("success_icon", "", "", "", light_green, ""),
    ("column_header", "", "", "", white, blue),
    ('deploy_highlight_start', "", "", "", warm_gray, light_green),
    ('deploy_highlight_end', "", "", "", warm_gray, cool_gray),
    ('disabled_button', "", "", "", light_gray, warm_gray),
    ('disabled_button_focus', "", "", "", light_gray, warm_gray),
    ('divider_line', "", "", "", cool_gray, ""),
    ('filter', "", "", "", black, light_gray),
    ('filter_focus', "", "", "", white, warm_gray),
    ('focus', "", "", "", white, warm_gray),
    ('radio focus', "", "", "", white, dark_magenta),
    ('status_extra', "", "", "", light_gray, warm_gray),
    ('error', "", "", "", white, light_red),
    ('info', "", "", "", light_green, ''),
    ('label', "", "", "", light_gray, ''),
]
<|endoftext|>


<commit_before>import time
import requests

# TODO: html parser for status requests
# TODO: upload to pypi

BASE_URL = "http://evaluacion.ucuenca.edu.ec/ucuenca-rest-ws/api/v1/"
MAX_RETRIES = 10
DELAY = 0.5

class UcuencaException(Exception):
    def __init__(self, status_code, msg):
        self.status_code = status_code
        self.msg = msg

    def __str__(self):
        return "status code: {status_code} - {msg}".format(
            status_code=self.status_code,
            msg=self.msg,
        )

class Ucuenca:
    def __init__(self, token=None, lowercase_keys=False):
        self.token = token
        self.lowercase_keys = lowercase_keys

    def _get(self, service_name, params=None):
        url = self._get_url(service_name)
        response = self._request(
            url,
            params
        )
        return self._parse_response(response)

    def _get_url(self, field):
        return "{}{}".format(BASE_URL, field)

    def _request(self, url, params=None):
        retries = MAX_RETRIES
        delay = DELAY

        while retries:
            try:
                return self._get_response(url, params)
            except UcuencaException as e:
                if retries == 1 or e.status_code == 404:
                    raise e
            except Exception:
                if retries == 1:
                    raise UcuencaException(1, "Unknow error.")
            retries -= 1
            time.sleep(delay)
            delay += DELAY

    def _get_response(self, url, params):
        response = requests.get(url, params)
        status_code = response.status_code
        if status_code != 200:
            raise UcuencaException(status_code, "Error")
        elif status_code == 404:
            raise UcuencaException(404, "Resource not found.")
        return response

    def _parse_response(self, response):
        headers = response.headers
        if 'json' in headers['content-type']:
            result = response.json()
            result = None if not result else result[0]
            if self.lowercase_keys:
                result = self._keys_to_lower_case(result)
        else:
            raise UcuencaException(2, "Unknow response.")
        return result

    @staticmethod
    def _keys_to_lower_case(dictionary):
        return {
            k.lower(): v
            for k, v in dictionary.items()
        }

    def careers(self, student_id):
        """Returns the careers that a student has taken given an id."""
        return self._get(
            service_name='registroacademico',
            params={'idEstudiante': student_id}
        )

    def notes(self, student_id, career_id, period_id):
        """Returns the notes of a student given an id, career, and perdiod."""
        return self._get(
            service_name='registroacademico/notas',
            params={
                'idEstudiante': student_id,
                'idCarrera': career_id,
                'idPerlec': period_id
            }
        )

    def schedule(self, student_id):
        """Returns the current schedule of a student given an id."""
        return self._get(
            service_name='horarios',
            params={'idEstudiante': student_id}
        )
<commit_msg>Json like list<commit_after>import time
import requests

# TODO: html parser for status requests
# TODO: upload to pypi

BASE_URL = "http://evaluacion.ucuenca.edu.ec/ucuenca-rest-ws/api/v1/"
MAX_RETRIES = 10
DELAY = 0.5

class UcuencaException(Exception):
    def __init__(self, status_code, msg):
        self.status_code = status_code
        self.msg = msg

    def __str__(self):
        return "status code: {status_code} - {msg}".format(
            status_code=self.status_code,
            msg=self.msg,
        )

class Ucuenca:
    def __init__(self, token=None, lowercase_keys=False):
        self.token = token
        self.lowercase_keys = lowercase_keys

    def _get(self, service_name, params=None):
        url = self._get_url(service_name)
        response = self._request(
            url,
            params
        )
        return self._parse_response(response)

    def _get_url(self, field):
        return "{}{}".format(BASE_URL, field)

    def _request(self, url, params=None):
        retries = MAX_RETRIES
        delay = DELAY

        while retries:
            try:
                return self._get_response(url, params)
            except UcuencaException as e:
                if retries == 1 or e.status_code == 404:
                    raise e
            except Exception:
                if retries == 1:
                    raise UcuencaException(1, "Unknow error.")
            retries -= 1
            time.sleep(delay)
            delay += DELAY

    def _get_response(self, url, params):
        response = requests.get(url, params)
        status_code = response.status_code
        if status_code != 200:
            raise UcuencaException(status_code, "Error")
        elif status_code == 404:
            raise UcuencaException(404, "Resource not found.")
        return response

    def _parse_response(self, response):
        headers = response.headers
        if 'json' in headers['content-type']:
            result = response.json()
            if self.lowercase_keys:
                result = [
                    self._keys_to_lower_case(r)
                    for r in result
                ]
        else:
            raise UcuencaException(2, "Unknow response.")
        return result

    @staticmethod
    def _keys_to_lower_case(dictionary):
        return {
            k.lower(): v
            for k, v in dictionary.items()
        }

    def careers(self, student_id):
        """Returns the careers that a student has taken given an id."""
        response = self._get(
            service_name='registroacademico',
            params={'idEstudiante': student_id}
        )
        return None if not response else response[0]

    def notes(self, student_id, career_id, period_id):
        """Returns the notes of a student given an id, career, and perdiod."""
        response = self._get(
            service_name='registroacademico/notas',
            params={
                'idEstudiante': student_id,
                'idCarrera': career_id,
                'idPerlec': period_id
            }
        )
        return None if not response else response[0]

    def schedule(self, student_id):
        """Returns the current schedule of a student given an id."""
        response = self._get(
            service_name='horarios',
            params={'idEstudiante': student_id}
        )
        return None if not response else response[0]
<|endoftext|>


<commit_before>#!/usr/bin/python
'''
choose only the non-consecutive chars in a string
and make them a list.
'''

def unique_in_order(iterable):
    ls = [i for i in iterable]
    if len(ls) <= 1:
        return ls
    else:
        return [ls[index] for index in range(len(ls) - 1)
                if ls[index] != ls[index + 1]] + [ls[-1]]
'''
Sometimes a simple for loop is better than a list comprehension.
'''

def unique_in_order(iterable):
    result = []
    prev = None
    for char in iterable:
        if char != prev:
            result.append(char)
            prev = char
    return result
<commit_msg>2 more solutions<commit_after>#!/usr/bin/python

def unique_in_order(iterable):
    '''
    choose only the non-consecutive chars in a string
    and make them a list.
    '''

    ls = [i for i in iterable]
    if len(ls) <= 1:
        return ls
    else:
        return [ls[index] for index in range(len(ls) - 1)
                if ls[index] != ls[index + 1]] + [ls[-1]]

def unique_in_order(iterable):
    '''
    Sometimes a simple for loop is better than a list comprehension.
    '''

    result = []
    prev = None
    for char in iterable:
        if char != prev:
            result.append(char)
            prev = char
    return result

def unique_in_order(iterable):
    '''
    Itertools built-in
    '''

    from itertools import groupby
    return [k for (k, _) in groupby(iterable)]

def unique_in_order(iterable):
    '''
    more elegant solution than my own
    '''
    ls = [i for i in iterable]
    return [val for index, val in enumerate(ls)
            if i == 0 or ls[index - 1] != val]
<|endoftext|>


<commit_before># pedidos = Conexion con Interfaz Jugador
# Dados = Conexion con Dados
# listado_inicial conexion con ListadoCartas
@@ echo echo
class Jugador(object):
    """Representa a un jugador manejado por un usuario.
    Todo el manejo para pedirle y mostrarle cosas al usuario se hace utilizando su atributo "pedidos"
    que se encarga de dichas tareas.
    En este modulo no puede haber ninguna funcion raw_input ni print, ni ninguna semejante."""

    def __init__(self, nombre, posicion_inicial, listado_inicial, dados, pedidos):
        """Recibe su nombre, una posicion inicial, un listado ya inicializado, los dados a usar
        y alguien que le permita hacerle pedidos al usuario, de la manera que corresponda."""
		#Notes : Nombre = String ; posicion_inicial integer, listado_inicial = lista, dados = lista de objetos
		self.nombre = nombre
		self.pos_inicial = posicion_inicial[:]
		self.listado_inicial = listado_inicial[:]
		self.dados_user = dados[:]
		self.pedidos = pedidos[:] # Referencia a interfaz_jugador de cada jugador!

    def get_nombre(self):
        """Devuelve el nombre del jugador"""
        return str(self.nombre)

    def __eq___(self, otro):
        """Verifica si un jugador es igual a otro jugador.
        Dos jugadores son iguales cuando tienen el mismo nombre"""
        return (self.nombre == otro.nombre)

    def asignar_carta(self, carta):
        """Se le asigna una carta a la mano del jugador. Este la marca como vista en su listado
        de cartas."""
		self.listado_inicial[carta] = True

    def get_posicion(self):
        """Obtiene la posicion del jugador."""
		return int(self.pos_inicial)

    def alguna_carta(self, jugada):
        """Se fija si el jugador tiene alguna de las cartas indicadas en la jugada.
        Parametros:
            - jugada: iterable con cartas.
        Salida: si tiene al menos una de las cartas, debe preguntarle al usuario cual
        prefiere mostrarle. Si no tiene ninguna, devuelve None."""
		cartas_a_mostrar = list( set(self.listado_inicial) & set(jugada) )

		if not cartas_a_mostrar
			return None

		self.pedidos.pedir_carta_a_mostrar(self.nombre, cartas_a_mostrar)

    def arriesgar(self):
        """Devuelve arriesgo del usuario (personaje, arma, jugador), o None si no desea arriesgarse."""
        return self.pedidos.preguntar_arriesgo()

    def mover(self, tablero):
        """Lanza los dados y se mueve en algun sentido por el tablero. Le muestra al usuario el resultado de
        haber lanzado los dados, y le pide el sentido en el que debe moverse."""
		movimiento = self.dados.lanzar()
		sentido = self.pedidos.pedir_sentido()

		tablero.siguiente(int(self.pos_inicial), movimiento, sentido):

		self.pos_inicial += posicion

    def sugerir(self, tablero, otros_jugadores):
        """Si esta en algun lugar para hacer sugerencias, le pregunta al usuario si desea hacer una.
        En caso afirmativo, le muestra la mano al jugador, le muestra el listado de cartas que aun no vio,
        le pide la jugada, y le consulta al resto de los jugadores si tiene alguna
        de las cartas.
        Parametros:
            - tablero: tablero del juego.
            - otros_jugadores: un iterable con los demas jugadores, en el orden en el que se les debe consultar."""
        if tablero[self.pos_inicial] == None:
			return

		respuesta = self.pedidos.quiere_consultar(tablero[self.pos_inicial])

		if not respuesta:
			return

		self.listado_inicial.__str__()
		self.listado_inicial.obtener_no_vistas()
		self.pedidos.mostrar_mano(self.listado_inicial)

		for otros_pjs in otros_jugadores:
			self.pedidos.pedir_carta_a_mostrar(otros_pjs, self.listado_inicial)
<commit_msg>Update jugador.py<commit_after># pedidos = Conexion con Interfaz Jugador
# Dados = Conexion con Dados
# listado_inicial conexion con ListadoCartas

class Jugador(object):
    """Representa a un jugador manejado por un usuario.
    Todo el manejo para pedirle y mostrarle cosas al usuario se hace utilizando su atributo "pedidos"
    que se encarga de dichas tareas.
    En este modulo no puede haber ninguna funcion raw_input ni print, ni ninguna semejante."""

    def __init__(self, nombre, posicion_inicial, listado_inicial, dados, pedidos):
        """Recibe su nombre, una posicion inicial, un listado ya inicializado, los dados a usar
        y alguien que le permita hacerle pedidos al usuario, de la manera que corresponda."""
	#Notes : Nombre = String ; posicion_inicial integer, listado_inicial = lista, dados = lista de objetos
	self.nombre = nombre
	self.pos = posicion_inicial
	self.lista_cartas = listado_inicial
	self.dados = dados[:]
	self.pedidos = pedidos # Referencia a interfaz_jugador de cada jugador!

    def get_nombre(self):
        """Devuelve el nombre del jugador"""
        return self.nombre

    def __eq___(self, otro):
        """Verifica si un jugador es igual a otro jugador.
        Dos jugadores son iguales cuando tienen el mismo nombre"""
        return (self.nombre == otro.nombre)

    def asignar_carta(self, carta):
        """Se le asigna una carta a la mano del jugador. Este la marca como vista en su listado
        de cartas."""
		self.lista_cartas.sacar_carta(carta)

    def get_posicion(self):
        """Obtiene la posicion del jugador."""
		return self.pos

    def alguna_carta(self, jugada):
        """Se fija si el jugador tiene alguna de las cartas indicadas en la jugada.
        Parametros:
            - jugada: iterable con cartas.
        Salida: si tiene al menos una de las cartas, debe preguntarle al usuario cual
        prefiere mostrarle. Si no tiene ninguna, devuelve None."""
		cartas_a_mostrar = list( set(self.lista_cartas) & set(jugada) )

		if not cartas_a_mostrar
			return None
		else
			return self.pedidos.pedir_carta_a_mostrar(self.nombre, cartas_a_mostrar)

    def arriesgar(self):
        """Devuelve arriesgo del usuario (personaje, arma, jugador), o None si no desea arriesgarse."""
        return self.pedidos.preguntar_arriesgo()

    def mover(self, tablero):
        """Lanza los dados y se mueve en algun sentido por el tablero. Le muestra al usuario el resultado de
        haber lanzado los dados, y le pide el sentido en el que debe moverse."""
	self.pedidos.mostrar_dados( [ dados_user.lanzar() for dados_user in self.dados ] )
	self.pedidos.pedir_sentido()
	self.pos = movimiento

    def sugerir(self, tablero, otros_jugadores):
        """Si esta en algun lugar para hacer sugerencias, le pregunta al usuario si desea hacer una.
        En caso afirmativo, le muestra la mano al jugador, le muestra el listado de cartas que aun no vio,
        le pide la jugada, y le consulta al resto de los jugadores si tiene alguna
        de las cartas.
        Parametros:
            - tablero: tablero del juego.
            - otros_jugadores: un iterable con los demas jugadores, en el orden en el que se les debe consultar."""
        if tablero[self.pos_inicial] == None:
		return

	respuesta = self.pedidos.quiere_consultar(tablero[self.pos_inicial])

	if not respuesta:
		return

	self.pedidos.mostrar_mano(self.lista_cartas)
	self.lista_cartas.obtener_no_vistas()

	posible_pj = self.pedidos.pedir_personaje()
	posible_arma = self.pedidos.pedir_arma()
	posible_lugar = self.pedidos.pedir_lugar()
	jugada_final = [posible_pj, posible_arma, posible_lugar]

	for jugadores_en_partida in otros_jugadores:
		carta = jugadores_en_partida.alguna_carta(jugada_final)
		if carta == None:
			self.pedidos.mostrar_carta(jugadores_en_partida, carta)
		else
			self.pedidos.mostrar_no_hay_cartas()
<|endoftext|>


<commit_before># Import
import matplotlib.pyplot as plt
import numpy as np
from sklearn import cluster
from sklearn import datasets
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import scale

digits = datasets.load_digits()
data = scale(digits.data)

X_train, X_test, y_train, y_test, images_train, images_test = train_test_split(data, digits.target, digits.images,
                                                                               test_size=0.25, random_state=42)

n_samples, n_features = X_train.shape

print(n_samples)
print(n_features)
n_digits = len(np.unique(y_train))

clf = cluster.KMeans(init='k-means++', n_clusters=10, random_state=42)
clf.fit(X_train)

#  Visualization code
fig = plt.figure(figsize=(8, 3))

fig.suptitle('Cluster Center Images', fontsize=14, fontweight='bold')

# for i in range(10):
#     # Initialize subplots in a grid of 2X5, at i+1th position
#     ax = fig.add_subplot(2, 5, 1 + i)
#     # Display images
#     ax.imshow(clf.cluster_centers_[i].reshape((8, 8)), cmap=plt.cm.binary)
#     # Don't show the axes
#     plt.axis('off')
#
# plt.show()

y_pred = clf.predict(X_test)

print(y_pred[:100])

print(y_test[:100])

print(clf.cluster_centers_.shape)
<commit_msg>Visualise predicted results<commit_after># Import
import matplotlib.pyplot as plt
import numpy as np
from sklearn import cluster
from sklearn import datasets
from sklearn.cross_validation import train_test_split
from sklearn.manifold import Isomap
from sklearn.preprocessing import scale

digits = datasets.load_digits()
data = scale(digits.data)

X_train, X_test, y_train, y_test, images_train, images_test = train_test_split(data, digits.target, digits.images,
                                                                               test_size=0.25, random_state=42)

n_samples, n_features = X_train.shape

print(n_samples)
print(n_features)
n_digits = len(np.unique(y_train))

clf = cluster.KMeans(init='k-means++', n_clusters=10, random_state=42)
clf.fit(X_train)

#  Visualization code
# fig = plt.figure(figsize=(8, 3))
#
# fig.suptitle('Cluster Center Images', fontsize=14, fontweight='bold')
#
# for i in range(10):
#     # Initialize subplots in a grid of 2X5, at i+1th position
#     ax = fig.add_subplot(2, 5, 1 + i)
#     # Display images
#     ax.imshow(clf.cluster_centers_[i].reshape((8, 8)), cmap=plt.cm.binary)
#     # Don't show the axes
#     plt.axis('off')
#
# plt.show()

y_pred = clf.predict(X_test)

print(y_pred[:100])

print(y_test[:100])

print(clf.cluster_centers_.shape)

X_iso = Isomap(n_neighbors=10).fit_transform(X_train)

clusters = clf.fit_predict(X_train)

fig, ax = plt.subplots(1, 2, figsize=(8, 4))

fig.suptitle('Predicted Versus Training Labels', fontsize=14, fontweight='bold')
fig.subplots_adjust(top=0.85)

ax[0].scatter(X_iso[:, 0], X_iso[:, 1], c=clusters)
ax[0].set_title('Predicted Training Labels')
ax[1].scatter(X_iso[:, 0], X_iso[:, 1], c=y_train)
ax[1].set_title('Actual Training Labels')

plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/python3

import argparse, pickle, time, os, tempfile, shutil, subprocess, json, multiprocessing, datetime
from contextlib import contextmanager
from pathlib import Path

calibration_file_path = Path.home()/"aktuell/kamscan_calibration.pickle"

parser = argparse.ArgumentParser(description="Scan a document.")
parser.add_argument("--calibration", action="store_true", help="take a calibration image")
parser.add_argument("--mode", default="mono", choices={"gray", "color", "mono"}, help="colour mode of resulting pages")
parser.add_argument("filepath", type=Path, help="path to the PDF file for storing")
args = parser.parse_args()

def wait_for_excess_processes(processes, max_processes=multiprocessing.cpu_count()):
    while len({process for process in processes if process.poll() is None}) > max_processes:
        time.sleep(1)

def path_to_own_program(name):
    return (Path(__file__).parent/name).resolve()

class Camera:
    path = Path("/media/bronger/3937-6637/DCIM")

    def __init__(self):
        with self._camera_connected():
            self.paths = self._collect_paths()

    @contextmanager
    def _camera_connected(self):
        if not self.path.exists():
            print("Bitte Kamera einstöpseln.")
        while not self.path.exists():
            time.sleep(1)
        yield
        print("Bitte Kamera ausstöpseln.")
        while True:
            try:
                if not self.path.exists():
                    break
            except PermissionError as error:
                print(error)
            time.sleep(1)

    def _collect_paths(self):
        result = set()
        for root, __, filenames in os.walk(str(self.path)):
            for filename in filenames:
                if os.path.splitext(filename)[1] in {".JPG", ".ARW"}:
                    filepath = Path(root)/filename
                    result.add(filepath)
        return result

    @contextmanager
    def download(self):
        tempdir = Path(tempfile.mkdtemp())
        processes = set()
        print("Bitte Bilder machen.  Dann:")
        with self._camera_connected():
            paths = self._collect_paths()
            new_paths = paths - self.paths
            paths_with_timestamps = []
            for path in new_paths:
                output = subprocess.check_output(["exiv2", "-g", "Exif.Photo.DateTimeOriginal", str(path)]).decode().strip()
                paths_with_timestamps.append((datetime.datetime.strptime(output[-19:], "%Y:%m:%d %H:%M:%S"), path))
            paths_with_timestamps.sort()
            for i, path_with_timestamp in enumerate(paths_with_timestamps):
                path = path_with_timestamp[1]
                if path.suffix == ".ARW":
                    wait_for_excess_processes(processes)
                    dcraw_call = "dcraw -c -t 5 -o 0 -M -g 1 1"
                    if args.mode in {"grey", "mono"}:
                        dcraw_call += " -d"
                    process = subprocess.Popen(
                        [dcraw_call + " '{0}' > '{2}/{1:06}.ppm' && rm '{0}'".format(path, i, tempdir)], shell=True)
                    processes.add(process)
            wait_for_excess_processes(processes, max_processes=0)
            assert all(process.returncode == 0 for process in processes)
        yield tempdir
        shutil.rmtree(str(tempdir), ignore_errors=True)

camera = Camera()

class CorrectionData:

    def __init__(self):
        self.coordinates = 8 * [None]

    def coordinates_as_strings(self):
        return [str(coordinate) for coordinate in self.coordinates]

    def __repr__(self):
        return "links oben: {}, {}  rechts oben: {}, {}  links unten: {}, {}  rechts unten: {}, {}".format(*self.coordinates)

def analyze_scan(x, y, scaling, filepath, number_of_points):
    output = subprocess.check_output([str(path_to_own_program("analyze_scan.py")), str(x), str(y), str(scaling),
                                      str(filepath), str(number_of_points)]).decode()
    result = json.loads(output)
    return result

def analyze_calibration_image():
    with camera.download() as directory:
        filenames = os.listdir(str(directory))
        assert len(filenames) == 1
        filepath = directory/filenames[0]
        raw_points = analyze_scan(2000, 3000, 0.1, filepath, 4)
        points = [analyze_scan(x, y, 1, filepath, 1)[0] for x, y in raw_points]
    correction_data = CorrectionData()
    center_x = sum(point[0] for point in points) / len(points)
    center_y = sum(point[1] for point in points) / len(points)
    for point in points:
        if point[0] < center_x:
            if point[1] < center_y:
                correction_data.coordinates[0:2] = point
            else:
                correction_data.coordinates[4:6] = point
        else:
            if point[1] < center_y:
                correction_data.coordinates[2:4] = point
            else:
                correction_data.coordinates[6:8] = point
    return correction_data

def get_correction_data():
    correction_data = analyze_calibration_image()
    pickle.dump(correction_data, open(str(calibration_file_path), "wb"))
    return correction_data

if args.calibration:
    correction_data = get_correction_data()
else:
    try:
        correction_data = pickle.load(open(str(calibration_file_path), "rb"))
    except FileNotFoundError:
        correction_data = get_correction_data()

def process_image(filepath, out_filepath):
    x0, y0, width, height = json.loads(
        subprocess.check_output([str(path_to_own_program("undistort")), str(filepath)] +
                                correction_data.coordinates_as_strings()).decode())
    convert_call = ["convert", "-extract", "{}x{}+{}+{}".format(width, height, x0, y0), str(filepath)]
    if args.mode == "color":
        convert_call.extend(["-profile", "/home/bronger/.config/darktable/color/in/nex7_matrix.icc",
                             "-set", "colorspace", "XYZ", "-colorspace", "sRGB"])
    elif args.mode == "gray":
        convert_call.extend(["-colorspace", "gray"])
    elif args.mode == "mono":
        convert_call.extend(["-dither", "FloydSteinberg", "-compress", "group4"])
    convert_call.append(str(out_filepath))
    subprocess.check_call(convert_call)

with tempfile.TemporaryDirectory() as tempdir:
    basename = Path(tempdir)/args.filepath.stem
    pool = multiprocessing.Pool()
    results = set()
    with camera.download() as directory:
        processes = set()
        for i, filename in enumerate(os.listdir(str(directory))):
            filepath = directory/filename
            results.add(pool.apply_async(process_image, (filepath, "{}_{:04}.tif".format(basename, i))))
        pool.close()
        pool.join()
    for result in results:
        result.get()

    subprocess.check_call(["make_searchable_pdf.py", str(basename)])
    shutil.move(str(basename.with_suffix(".pdf")), str(args.filepath.parent))

subprocess.check_call(["evince", str(args.filepath)])
<commit_msg>Added error-check to parsing of output of analyze_scan.py.<commit_after>#!/usr/bin/python3

import argparse, pickle, time, os, tempfile, shutil, subprocess, json, multiprocessing, datetime
from contextlib import contextmanager
from pathlib import Path

calibration_file_path = Path.home()/"aktuell/kamscan_calibration.pickle"

parser = argparse.ArgumentParser(description="Scan a document.")
parser.add_argument("--calibration", action="store_true", help="take a calibration image")
parser.add_argument("--mode", default="mono", choices={"gray", "color", "mono"}, help="colour mode of resulting pages")
parser.add_argument("filepath", type=Path, help="path to the PDF file for storing")
args = parser.parse_args()

def wait_for_excess_processes(processes, max_processes=multiprocessing.cpu_count()):
    while len({process for process in processes if process.poll() is None}) > max_processes:
        time.sleep(1)

def path_to_own_program(name):
    return (Path(__file__).parent/name).resolve()

class Camera:
    path = Path("/media/bronger/3937-6637/DCIM")

    def __init__(self):
        with self._camera_connected():
            self.paths = self._collect_paths()

    @contextmanager
    def _camera_connected(self):
        if not self.path.exists():
            print("Bitte Kamera einstöpseln.")
        while not self.path.exists():
            time.sleep(1)
        yield
        print("Bitte Kamera ausstöpseln.")
        while True:
            try:
                if not self.path.exists():
                    break
            except PermissionError as error:
                print(error)
            time.sleep(1)

    def _collect_paths(self):
        result = set()
        for root, __, filenames in os.walk(str(self.path)):
            for filename in filenames:
                if os.path.splitext(filename)[1] in {".JPG", ".ARW"}:
                    filepath = Path(root)/filename
                    result.add(filepath)
        return result

    @contextmanager
    def download(self):
        tempdir = Path(tempfile.mkdtemp())
        processes = set()
        print("Bitte Bilder machen.  Dann:")
        with self._camera_connected():
            paths = self._collect_paths()
            new_paths = paths - self.paths
            paths_with_timestamps = []
            for path in new_paths:
                output = subprocess.check_output(["exiv2", "-g", "Exif.Photo.DateTimeOriginal", str(path)]).decode().strip()
                paths_with_timestamps.append((datetime.datetime.strptime(output[-19:], "%Y:%m:%d %H:%M:%S"), path))
            paths_with_timestamps.sort()
            for i, path_with_timestamp in enumerate(paths_with_timestamps):
                path = path_with_timestamp[1]
                if path.suffix == ".ARW":
                    wait_for_excess_processes(processes)
                    dcraw_call = "dcraw -c -t 5 -o 0 -M -g 1 1"
                    if args.mode in {"grey", "mono"}:
                        dcraw_call += " -d"
                    process = subprocess.Popen(
                        [dcraw_call + " '{0}' > '{2}/{1:06}.ppm' && rm '{0}'".format(path, i, tempdir)], shell=True)
                    processes.add(process)
            wait_for_excess_processes(processes, max_processes=0)
            assert all(process.returncode == 0 for process in processes)
        yield tempdir
        shutil.rmtree(str(tempdir), ignore_errors=True)

camera = Camera()

class CorrectionData:

    def __init__(self):
        self.coordinates = 8 * [None]

    def coordinates_as_strings(self):
        return [str(coordinate) for coordinate in self.coordinates]

    def __repr__(self):
        return "links oben: {}, {}  rechts oben: {}, {}  links unten: {}, {}  rechts unten: {}, {}".format(*self.coordinates)

def analyze_scan(x, y, scaling, filepath, number_of_points):
    output = subprocess.check_output([str(path_to_own_program("analyze_scan.py")), str(x), str(y), str(scaling),
                                      str(filepath), str(number_of_points)]).decode()
    try:
        result = json.loads(output)
    except json.decoder.JSONDecodeError:
        print("Invalid JSON: {}".repr(output))
        raise
    return result

def analyze_calibration_image():
    with camera.download() as directory:
        filenames = os.listdir(str(directory))
        assert len(filenames) == 1
        filepath = directory/filenames[0]
        raw_points = analyze_scan(2000, 3000, 0.1, filepath, 4)
        points = [analyze_scan(x, y, 1, filepath, 1)[0] for x, y in raw_points]
    correction_data = CorrectionData()
    center_x = sum(point[0] for point in points) / len(points)
    center_y = sum(point[1] for point in points) / len(points)
    for point in points:
        if point[0] < center_x:
            if point[1] < center_y:
                correction_data.coordinates[0:2] = point
            else:
                correction_data.coordinates[4:6] = point
        else:
            if point[1] < center_y:
                correction_data.coordinates[2:4] = point
            else:
                correction_data.coordinates[6:8] = point
    return correction_data

def get_correction_data():
    correction_data = analyze_calibration_image()
    pickle.dump(correction_data, open(str(calibration_file_path), "wb"))
    return correction_data

if args.calibration:
    correction_data = get_correction_data()
else:
    try:
        correction_data = pickle.load(open(str(calibration_file_path), "rb"))
    except FileNotFoundError:
        correction_data = get_correction_data()

def process_image(filepath, out_filepath):
    x0, y0, width, height = json.loads(
        subprocess.check_output([str(path_to_own_program("undistort")), str(filepath)] +
                                correction_data.coordinates_as_strings()).decode())
    convert_call = ["convert", "-extract", "{}x{}+{}+{}".format(width, height, x0, y0), str(filepath)]
    if args.mode == "color":
        convert_call.extend(["-profile", "/home/bronger/.config/darktable/color/in/nex7_matrix.icc",
                             "-set", "colorspace", "XYZ", "-colorspace", "sRGB"])
    elif args.mode == "gray":
        convert_call.extend(["-colorspace", "gray"])
    elif args.mode == "mono":
        convert_call.extend(["-dither", "FloydSteinberg", "-compress", "group4"])
    convert_call.append(str(out_filepath))
    subprocess.check_call(convert_call)

with tempfile.TemporaryDirectory() as tempdir:
    basename = Path(tempdir)/args.filepath.stem
    pool = multiprocessing.Pool()
    results = set()
    with camera.download() as directory:
        processes = set()
        for i, filename in enumerate(os.listdir(str(directory))):
            filepath = directory/filename
            results.add(pool.apply_async(process_image, (filepath, "{}_{:04}.tif".format(basename, i))))
        pool.close()
        pool.join()
    for result in results:
        result.get()

    subprocess.check_call(["make_searchable_pdf.py", str(basename)])
    shutil.move(str(basename.with_suffix(".pdf")), str(args.filepath.parent))

subprocess.check_call(["evince", str(args.filepath)])
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import smallsmilhandler

if __name__ == "__main__":

    comandos = sys.argv
    parser = make_parser()
    cHandler = smallsmilhandler.SmallSMILHandler()
    parser.setContentHandler(cHandler)

    if len(comandos) != 2:
        print ("Usage: python3 karaoke.py file.smil")
    else:
        parser.parse(open(comandos[1]))
        datos = cHandler.get_tags()
        total = ""
        for linea in datos:
            #lo primero que queremos el nombre mas el \t
            name = linea[0] + '\T'
            atributos = "" #Luego iremos mentiendo elementos
            #queremos es ir cogiendo los atributos tras coger el nombre e ir quitando blancos.
            #lina[1] (diccionario) como anteriormente atributo (primera parte del atributo)
            for atributo in linea[1].keys():
                if linea[1][atributo] != "":
                    elemento = linea[1][atributo]#sacamos el valor del atributo
                    atributos = atributos + atributo
                    atributos += '=' + elemento + '\T' #Unimos mas cosas a nuestra lista
            total = name + atributos + '\n'#nombre mas atributo completo con su valor
        print (total)
<commit_msg>me llega a coger src<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import smallsmilhandler

if __name__ == "__main__":

    comandos = sys.argv
    parser = make_parser()
    cHandler = smallsmilhandler.SmallSMILHandler()
    parser.setContentHandler(cHandler)

    if len(comandos) != 2:
        print ("Usage: python3 karaoke.py file.smil")
    else:
        parser.parse(open(comandos[1]))
        datos = cHandler.get_tags()
        total = ""
        for linea in datos:
            #lo primero que queremos el nombre mas el \t
            name = linea[0] + '\T'
            atributos = "" #Luego iremos mentiendo elementos
            #queremos es ir cogiendo los atributos tras coger el nombre e ir quitando blancos.
            #lina[1] (diccionario) como anteriormente atributo (primera parte del atributo)
            for atributo in linea[1].keys():
                if linea[1][atributo] != "":
                    elemento = linea[1][atributo]#sacamos el valor del atributo
                    #print(elemento)
                    atributos = atributos + atributo
                    atributos += '=' + elemento + '\T' #Unimos mas cosas a nuestra lista
            total += name + atributos + '\n'#nombre mas atributo completo con su valor 2 listas diferentes
        #print (total)

        for linea in datos:
            for atributo in linea[1].keys():
                #print(atributo)
                if atributo == "src":#ESto me lo hace bien
                    if linea[1][atributo] == "http://":
                        print(linea[1].atributo)
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import smallsmilhandler
import sys
import json
from urllib.request import urlretrieve

if __name__ == "__main__":
    try:
        fich = open(sys.argv[1], 'r')
        parser = make_parser()
        cHandler = smallsmilhandler.SmallSMILHandler()
        parser.setContentHandler(cHandler)
        parser.parse(fich)
        lista = cHandler.get_tags()
        archivo = open('karaoke.json','w')
        line = json.dumps(lista)
        archivo.write(line)

        for dicc in lista:
            for etiqueta in dicc:
                if dicc[etiqueta].find('http://') == 0:
                    url = dicc[etiqueta]
                    elemento = url[url.rfind('/') + 1:]
                    urlretrieve(url, elemento)
                    modifica = dicc[etiqueta].split('/')[-1]
                    dicc[etiqueta] = modifica

        for dicc in lista:
            print(dicc['name']),
            for i in dicc:
                if dicc[i] and i != 'name':
                    print("\t" + i + '="' + dicc[i] + '"')
    except IOError:
        sys.exit("Usage: python karaoke.py file.smil")
    except IndexError:
        sys.exit("Usage: python karaoke.py file.smil")
<commit_msg>karaoke final 1<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import smallsmilhandler
import sys
import json
from urllib.request import urlretrieve

class KaraokeLocal():
    def __init__(self, fich):
        parser = make_parser()
        cHandler = smallsmilhandler.SmallSMILHandler()
        parser.setContentHandler(cHandler)
        parser.parse(fich)
        self.lista = cHandler.get_tags()

    def __str__(self):
        elemento = ''
        for dicc in self.lista:
            elemento += dicc['name']
            for i in dicc:
                if dicc[i] and i != 'name':
                    elemento += "\t" + i + '="' + dicc[i] + '"'
            elemento +='\n'
        return elemento
    def to_json(self, fich):
        if fich != 'karaoke.json':
            archivo = open('karaoke.json','w')
            contenido = json.dumps(self.lista)
            archivo.write(contenido)

    def do_local(self):
         for dicc in self.lista:
            for etiqueta in dicc:
                if dicc[etiqueta].find('http://') == 0:
                    url = dicc[etiqueta]
                    elemento = url[url.rfind('/') + 1:]
                    urlretrieve(url, elemento)
                    modifica = dicc[etiqueta].split('/')[-1]
                    dicc[etiqueta] = modifica

if __name__ == "__main__":
    try:
        fich = open(sys.argv[1], 'r')

    except IOError:
        sys.exit("Usage: python karaoke.py file.smil")
    except IndexError:
        sys.exit("Usage: python karaoke.py file.smil")

    karaoke = KaraokeLocal(fich)
    print(karaoke)
    karaoke.to_json(fich)
    karaoke.do_local()
    karaoke.to_json(fich)
    print(karaoke)
<|endoftext|>