<commit_before># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Binary for generating predictions over a set of videos."""

import os
import time

import numpy
import tensorflow as tf

from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging

import eval_util
import losses
import readers
import utils

FLAGS = flags.FLAGS

if __name__ == '__main__':
  flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
                      "The directory to load the model files from.")
  flags.DEFINE_string("output_file", "",
                      "The file to save the predictions to.")
  flags.DEFINE_string(
      "input_data_pattern", "",
      "File glob defining the evaluation dataset in tensorflow.SequenceExample "
      "format. The SequenceExamples are expected to have an 'rgb' byte array "
      "sequence feature as well as a 'labels' int64 context feature.")

  # Model flags.
  flags.DEFINE_bool(
      "frame_features", False,
      "If set, then --eval_data_pattern must be frame-level features. "
      "Otherwise, --eval_data_pattern must be aggregated video-level "
      "features. The model must also be set appropriately (i.e. to read 3D "
      "batches VS 4D batches.")
  flags.DEFINE_integer(
      "batch_size", 8192,
      "How many examples to process per batch.")
  flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
                      "to use for training.")
  flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")

  # Other flags.
  flags.DEFINE_integer("num_readers", 1,
                       "How many threads to use for reading input files.")
  flags.DEFINE_integer("top_k", 20,
                       "How many predictions to output per video.")

def format_lines(video_ids, predictions, top_k):
  batch_size = len(video_ids)
  for video_index in range(batch_size):
    top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
    line = [(class_index, predictions[video_index][class_index])
            for class_index in top_indices]
  #  print("Type - Test :")
  #  print(type(video_ids[video_index]))
  #  print(video_ids[video_index].decode('utf-8'))
    line = sorted(line, key=lambda p: -p[1])
    yield video_ids[video_index].decode('utf-8') + "," + " ".join("%i %f" % pair
                                                  for pair in line) + "\n"

def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
  """Creates the section of the graph which reads the input data.

  Args:
    reader: A class which parses the input data.
    data_pattern: A 'glob' style path to the data files.
    batch_size: How many examples to process at a time.
    num_readers: How many I/O threads to use.

  Returns:
    A tuple containing the features tensor, labels tensor, and optionally a
    tensor containing the number of frames per video. The exact dimensions
    depend on the reader being used.

  Raises:
    IOError: If no files matching the given pattern were found.
  """
  with tf.name_scope("input"):
    files = gfile.Glob(data_pattern)
    if not files:
      raise IOError("Unable to find input files. data_pattern='" +
                    data_pattern + "'")
    logging.info("number of input files: " + str(len(files)))
    filename_queue = tf.train.string_input_producer(
        files, num_epochs=1, shuffle=False)
    examples_and_labels = [reader.prepare_reader(filename_queue)
                           for _ in range(num_readers)]

    video_id_batch, video_batch, unused_labels, num_frames_batch = (
        tf.train.batch_join(examples_and_labels,
                            batch_size=batch_size,
                            allow_smaller_final_batch = True,
                            enqueue_many=True))
    return video_id_batch, video_batch, num_frames_batch

def inference(reader, train_dir, data_pattern, out_file_location, batch_size, top_k):
  with tf.Session() as sess, gfile.Open(out_file_location, "w+") as out_file:
    video_id_batch, video_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
    latest_checkpoint = tf.train.latest_checkpoint(train_dir)
    if latest_checkpoint is None:
      raise Exception("unable to find a checkpoint at location: %s" % train_dir)
    else:
      meta_graph_location = latest_checkpoint + ".meta"
      logging.info("loading meta-graph: " + meta_graph_location)
    saver = tf.train.import_meta_graph(meta_graph_location, clear_devices=True)
    logging.info("restoring variables from " + latest_checkpoint)
    saver.restore(sess, latest_checkpoint)
    input_tensor = tf.get_collection("input_batch_raw")[0]
    num_frames_tensor = tf.get_collection("num_frames")[0]
    predictions_tensor = tf.get_collection("predictions")[0]

    # Workaround for num_epochs issue.
    def set_up_init_ops(variables):
      init_op_list = []
      for variable in list(variables):
        if "train_input" in variable.name:
          init_op_list.append(tf.assign(variable, 1))
          variables.remove(variable)
      init_op_list.append(tf.variables_initializer(variables))
      return init_op_list

    sess.run(set_up_init_ops(tf.get_collection_ref(
        tf.GraphKeys.LOCAL_VARIABLES)))

    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    num_examples_processed = 0
    start_time = time.time()
    out_file.write("VideoId,LabelConfidencePairs\n")

    try:
      while not coord.should_stop():
          video_id_batch_val, video_batch_val,num_frames_batch_val = sess.run([video_id_batch, video_batch, num_frames_batch])
          predictions_val, = sess.run([predictions_tensor], feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val})
          now = time.time()
          num_examples_processed += len(video_batch_val)
          num_classes = predictions_val.shape[1]
          logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
          for line in format_lines(video_id_batch_val, predictions_val, top_k):
            out_file.write(line)
          out_file.flush()

    except tf.errors.OutOfRangeError:
        logging.info('Done with inference. The output file was written to ' + out_file_location)
    finally:
        coord.request_stop()

    coord.join(threads)
    sess.close()

def main(unused_argv):
  logging.set_verbosity(tf.logging.INFO)

  # convert feature_names and feature_sizes to lists of values
  feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
      FLAGS.feature_names, FLAGS.feature_sizes)

  if FLAGS.frame_features:
    reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
                                            feature_sizes=feature_sizes)
  else:
    reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
                                                 feature_sizes=feature_sizes)

  if FLAGS.output_file is "":
    raise ValueError("'output_file' was not specified. "
      "Unable to continue with inference.")

  if FLAGS.input_data_pattern is "":
    raise ValueError("'input_data_pattern' was not specified. "
      "Unable to continue with inference.")

  inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
    FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)

if __name__ == "__main__":
  app.run()
<commit_msg>Remove commented out debug code.<commit_after># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Binary for generating predictions over a set of videos."""

import os
import time

import numpy
import tensorflow as tf

from tensorflow import app
from tensorflow import flags
from tensorflow import gfile
from tensorflow import logging

import eval_util
import losses
import readers
import utils

FLAGS = flags.FLAGS

if __name__ == '__main__':
  flags.DEFINE_string("train_dir", "/tmp/yt8m_model/",
                      "The directory to load the model files from.")
  flags.DEFINE_string("output_file", "",
                      "The file to save the predictions to.")
  flags.DEFINE_string(
      "input_data_pattern", "",
      "File glob defining the evaluation dataset in tensorflow.SequenceExample "
      "format. The SequenceExamples are expected to have an 'rgb' byte array "
      "sequence feature as well as a 'labels' int64 context feature.")

  # Model flags.
  flags.DEFINE_bool(
      "frame_features", False,
      "If set, then --eval_data_pattern must be frame-level features. "
      "Otherwise, --eval_data_pattern must be aggregated video-level "
      "features. The model must also be set appropriately (i.e. to read 3D "
      "batches VS 4D batches.")
  flags.DEFINE_integer(
      "batch_size", 8192,
      "How many examples to process per batch.")
  flags.DEFINE_string("feature_names", "mean_rgb", "Name of the feature "
                      "to use for training.")
  flags.DEFINE_string("feature_sizes", "1024", "Length of the feature vectors.")

  # Other flags.
  flags.DEFINE_integer("num_readers", 1,
                       "How many threads to use for reading input files.")
  flags.DEFINE_integer("top_k", 20,
                       "How many predictions to output per video.")

def format_lines(video_ids, predictions, top_k):
  batch_size = len(video_ids)
  for video_index in range(batch_size):
    top_indices = numpy.argpartition(predictions[video_index], -top_k)[-top_k:]
    line = [(class_index, predictions[video_index][class_index])
            for class_index in top_indices]
    line = sorted(line, key=lambda p: -p[1])
    yield video_ids[video_index].decode('utf-8') + "," + " ".join("%i %f" % pair
                                                  for pair in line) + "\n"

def get_input_data_tensors(reader, data_pattern, batch_size, num_readers=1):
  """Creates the section of the graph which reads the input data.

  Args:
    reader: A class which parses the input data.
    data_pattern: A 'glob' style path to the data files.
    batch_size: How many examples to process at a time.
    num_readers: How many I/O threads to use.

  Returns:
    A tuple containing the features tensor, labels tensor, and optionally a
    tensor containing the number of frames per video. The exact dimensions
    depend on the reader being used.

  Raises:
    IOError: If no files matching the given pattern were found.
  """
  with tf.name_scope("input"):
    files = gfile.Glob(data_pattern)
    if not files:
      raise IOError("Unable to find input files. data_pattern='" +
                    data_pattern + "'")
    logging.info("number of input files: " + str(len(files)))
    filename_queue = tf.train.string_input_producer(
        files, num_epochs=1, shuffle=False)
    examples_and_labels = [reader.prepare_reader(filename_queue)
                           for _ in range(num_readers)]

    video_id_batch, video_batch, unused_labels, num_frames_batch = (
        tf.train.batch_join(examples_and_labels,
                            batch_size=batch_size,
                            allow_smaller_final_batch = True,
                            enqueue_many=True))
    return video_id_batch, video_batch, num_frames_batch

def inference(reader, train_dir, data_pattern, out_file_location, batch_size, top_k):
  with tf.Session() as sess, gfile.Open(out_file_location, "w+") as out_file:
    video_id_batch, video_batch, num_frames_batch = get_input_data_tensors(reader, data_pattern, batch_size)
    latest_checkpoint = tf.train.latest_checkpoint(train_dir)
    if latest_checkpoint is None:
      raise Exception("unable to find a checkpoint at location: %s" % train_dir)
    else:
      meta_graph_location = latest_checkpoint + ".meta"
      logging.info("loading meta-graph: " + meta_graph_location)
    saver = tf.train.import_meta_graph(meta_graph_location, clear_devices=True)
    logging.info("restoring variables from " + latest_checkpoint)
    saver.restore(sess, latest_checkpoint)
    input_tensor = tf.get_collection("input_batch_raw")[0]
    num_frames_tensor = tf.get_collection("num_frames")[0]
    predictions_tensor = tf.get_collection("predictions")[0]

    # Workaround for num_epochs issue.
    def set_up_init_ops(variables):
      init_op_list = []
      for variable in list(variables):
        if "train_input" in variable.name:
          init_op_list.append(tf.assign(variable, 1))
          variables.remove(variable)
      init_op_list.append(tf.variables_initializer(variables))
      return init_op_list

    sess.run(set_up_init_ops(tf.get_collection_ref(
        tf.GraphKeys.LOCAL_VARIABLES)))

    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    num_examples_processed = 0
    start_time = time.time()
    out_file.write("VideoId,LabelConfidencePairs\n")

    try:
      while not coord.should_stop():
          video_id_batch_val, video_batch_val,num_frames_batch_val = sess.run([video_id_batch, video_batch, num_frames_batch])
          predictions_val, = sess.run([predictions_tensor], feed_dict={input_tensor: video_batch_val, num_frames_tensor: num_frames_batch_val})
          now = time.time()
          num_examples_processed += len(video_batch_val)
          num_classes = predictions_val.shape[1]
          logging.info("num examples processed: " + str(num_examples_processed) + " elapsed seconds: " + "{0:.2f}".format(now-start_time))
          for line in format_lines(video_id_batch_val, predictions_val, top_k):
            out_file.write(line)
          out_file.flush()

    except tf.errors.OutOfRangeError:
        logging.info('Done with inference. The output file was written to ' + out_file_location)
    finally:
        coord.request_stop()

    coord.join(threads)
    sess.close()

def main(unused_argv):
  logging.set_verbosity(tf.logging.INFO)

  # convert feature_names and feature_sizes to lists of values
  feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
      FLAGS.feature_names, FLAGS.feature_sizes)

  if FLAGS.frame_features:
    reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
                                            feature_sizes=feature_sizes)
  else:
    reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
                                                 feature_sizes=feature_sizes)

  if FLAGS.output_file is "":
    raise ValueError("'output_file' was not specified. "
      "Unable to continue with inference.")

  if FLAGS.input_data_pattern is "":
    raise ValueError("'input_data_pattern' was not specified. "
      "Unable to continue with inference.")

  inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
    FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)

if __name__ == "__main__":
  app.run()
<|endoftext|>


<commit_before># Copyright 2006 James Tauber and contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pyjamas import DOM

from pyjamas.ui.CellPanel import CellPanel
from pyjamas.ui import HasHorizontalAlignment
from pyjamas.ui import HasVerticalAlignment

class VerticalPanel(CellPanel):

    def __init__(self):
        CellPanel.__init__(self)

        self.horzAlign = HasHorizontalAlignment.ALIGN_LEFT
        self.vertAlign = HasVerticalAlignment.ALIGN_TOP

        DOM.setAttribute(self.getTable(), "cellSpacing", "0")
        DOM.setAttribute(self.getTable(), "cellPadding", "0")

    def add(self, widget):
        self.insert(widget, self.getWidgetCount())

    def getHorizontalAlignment(self):
        return self.horzAlign

    def getVerticalAlignment(self):
        return self.vertAlign

    def getWidget(self, index):
        return self.children[index]

    def getWidgetCount(self):
        return len(self.children)

    def getWidgetIndex(self, child):
        return self.children.index(child)

    def setWidget(self, index, widget):
        """Replace the widget at the given index with a new one"""
        existing = self.getWidget(index)
        if existing:
            self.remove(existing)
        self.insert(widget, index)

    def insert(self, widget, beforeIndex):
        widget.removeFromParent()

        tr = DOM.createTR()
        td = DOM.createTD()

        DOM.insertChild(self.getBody(), tr, beforeIndex)
        DOM.appendChild(tr, td)

        CellPanel.insert(self, widget, td, beforeIndex)

        self.setCellHorizontalAlignment(widget, self.horzAlign)
        self.setCellVerticalAlignment(widget, self.vertAlign)

    def remove(self, widget):
        if pyjslib.isNumber(widget):
            widget = self.getWidget(widget)

        if widget.getParent() != self:
            return False

        td = DOM.getParent(widget.getElement())
        tr = DOM.getParent(td)
        DOM.removeChild(self.getBody(), tr)

        CellPanel.remove(self, widget)
        return True

    def setHorizontalAlignment(self, align):
        self.horzAlign = align

    def setVerticalAlignment(self, align):
        self.vertAlign = align

<commit_msg>added missing import<commit_after># Copyright 2006 James Tauber and contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pyjamas import DOM

from pyjamas.ui.CellPanel import CellPanel
from pyjamas.ui import HasHorizontalAlignment
from pyjamas.ui import HasVerticalAlignment
import pyjslib

class VerticalPanel(CellPanel):

    def __init__(self):
        CellPanel.__init__(self)

        self.horzAlign = HasHorizontalAlignment.ALIGN_LEFT
        self.vertAlign = HasVerticalAlignment.ALIGN_TOP

        DOM.setAttribute(self.getTable(), "cellSpacing", "0")
        DOM.setAttribute(self.getTable(), "cellPadding", "0")

    def add(self, widget):
        self.insert(widget, self.getWidgetCount())

    def getHorizontalAlignment(self):
        return self.horzAlign

    def getVerticalAlignment(self):
        return self.vertAlign

    def getWidget(self, index):
        return self.children[index]

    def getWidgetCount(self):
        return len(self.children)

    def getWidgetIndex(self, child):
        return self.children.index(child)

    def setWidget(self, index, widget):
        """Replace the widget at the given index with a new one"""
        existing = self.getWidget(index)
        if existing:
            self.remove(existing)
        self.insert(widget, index)

    def insert(self, widget, beforeIndex):
        widget.removeFromParent()

        tr = DOM.createTR()
        td = DOM.createTD()

        DOM.insertChild(self.getBody(), tr, beforeIndex)
        DOM.appendChild(tr, td)

        CellPanel.insert(self, widget, td, beforeIndex)

        self.setCellHorizontalAlignment(widget, self.horzAlign)
        self.setCellVerticalAlignment(widget, self.vertAlign)

    def remove(self, widget):
        if pyjslib.isNumber(widget):
            widget = self.getWidget(widget)

        if widget.getParent() != self:
            return False

        td = DOM.getParent(widget.getElement())
        tr = DOM.getParent(td)
        DOM.removeChild(self.getBody(), tr)

        CellPanel.remove(self, widget)
        return True

    def setHorizontalAlignment(self, align):
        self.horzAlign = align

    def setVerticalAlignment(self, align):
        self.vertAlign = align

<|endoftext|>


<commit_before>"""
Test LpNorm cost
"""
import numpy
import theano
from theano import tensor as T
from nose.tools import raises

def test_shared_variables():
    '''
    LpNorm should handle shared variables.
    '''
    assert False

def test_symbolic_expressions_of_shared_variables():
    '''
    LpNorm should handle symbolic expressions of shared variables.
    '''
    assert False

@raises(Exception)
def test_symbolic_variables():
    '''
    LpNorm should not handle symbolic variables
    '''
    assert True

if __name__ == '__main__':
    test_shared_variables()
    test_symbolic_expressions_of_shared_variables()
    test_symbolic_variables()
<commit_msg>Add more code to LpNorm unit tests<commit_after>"""
Test LpNorm cost
"""
import os
from nose.tools import raises
from pylearn2.models.mlp import Linear
from pylearn2.models.mlp import Softmax
from pylearn2.models.mlp import MLP
from pylearn2.costs.cost import LpNorm
from pylearn2.datasets.cifar10 import CIFAR10
from pylearn2.training_algorithms.sgd import SGD
from pylearn2.termination_criteria import EpochCounter
from pylearn2.train import Train

def test_shared_variables():
    '''
    LpNorm should handle shared variables.
    '''
    model = MLP(
        layers=[Linear(dim=100, layer_name='linear', irange=1.0),
                Softmax(n_classes=10, layer_name='softmax', irange=1.0)],
        batch_size=100,
        nvis=3072
    )

    dataset = CIFAR10(which_set='train')

    cost = LpNorm(variables=model.get_params(), p=2)

    algorithm = SGD(
        learning_rate=0.01,
        cost=cost, batch_size=100,
        monitoring_dataset=dataset,
        termination_criterion=EpochCounter(1)
    )

    trainer = Train(
        dataset=dataset,
        model=model,
        algorithm=algorithm
    )

    trainer.main_loop()

    assert False

def test_symbolic_expressions_of_shared_variables():
    '''
    LpNorm should handle symbolic expressions of shared variables.
    '''
    assert False

@raises(Exception)
def test_symbolic_variables():
    '''
    LpNorm should not handle symbolic variables
    '''
    assert True

if __name__ == '__main__':
    test_shared_variables()
    test_symbolic_expressions_of_shared_variables()
    test_symbolic_variables()
<|endoftext|>


<commit_before>import argparse
import numpy as np
import matplotlib.pyplot as plt
from FAUSTPy import *

#######################################################
# set up command line arguments
#######################################################

parser = argparse.ArgumentParser()
parser.add_argument('-f', '--faustfloat',
                    dest="faustfloat",
                    default="float",
                    help="The value of FAUSTFLOAT.")
parser.add_argument('-p', '--path',
                    dest="faust_path",
                    default="",
                    help="The path of FAUSTFLOAT.")
parser.add_argument('-c', '--cflags',
                    dest="cflags",
                    default=[],
                    type=str.split,
                    help="Extra compiler flags")
parser.add_argument('-s', '--fs',
                    dest="fs",
                    default=48000,
                    type=int,
                    help="The sampling frequency")
args = parser.parse_args()

#######################################################
# initialise the FAUST object and get the default parameters
#######################################################

wrapper.FAUST_PATH = args.faust_path

dattorro = FAUST("dattorro_notch_cut_regalia.dsp", args.fs, args.faustfloat,
                 extra_compile_args=args.cflags)

#######################################################
# plot the frequency response with the default settings
#######################################################

audio = np.zeros((dattorro.dsp.num_in, args.fs), dtype=dattorro.dsp.dtype)
audio[:,0] = 1

out = dattorro.compute(audio)

print(audio)
print(out)

spec = np.fft.fft(out)[:,:args.fs/2]

fig = plt.figure()
p   = fig.add_subplot(1,1,1)
p.plot(20*np.log10(np.absolute(spec.T)+1e-8))

################
# show the plots
################

plt.show()

print("everything passes!")
<commit_msg>__main__.py: make the plot look nicer.<commit_after>import argparse
import numpy as np
import matplotlib.pyplot as plt
from FAUSTPy import *

#######################################################
# set up command line arguments
#######################################################

parser = argparse.ArgumentParser()
parser.add_argument('-f', '--faustfloat',
                    dest="faustfloat",
                    default="float",
                    help="The value of FAUSTFLOAT.")
parser.add_argument('-p', '--path',
                    dest="faust_path",
                    default="",
                    help="The path of FAUSTFLOAT.")
parser.add_argument('-c', '--cflags',
                    dest="cflags",
                    default=[],
                    type=str.split,
                    help="Extra compiler flags")
parser.add_argument('-s', '--fs',
                    dest="fs",
                    default=48000,
                    type=int,
                    help="The sampling frequency")
args = parser.parse_args()

#######################################################
# initialise the FAUST object and get the default parameters
#######################################################

wrapper.FAUST_PATH = args.faust_path

dattorro = FAUST("dattorro_notch_cut_regalia.dsp", args.fs, args.faustfloat,
                 extra_compile_args=args.cflags)

def_Q = dattorro.dsp.dattorro_notch_cut_regalia.Q
def_Gain = dattorro.dsp.dattorro_notch_cut_regalia.Gain
def_Freq = dattorro.dsp.dattorro_notch_cut_regalia.Center_Freq_

#######################################################
# plot the frequency response with the default settings
#######################################################

audio = np.zeros((dattorro.dsp.num_in, args.fs), dtype=dattorro.dsp.dtype)
audio[:,0] = 1

out = dattorro.compute(audio)

print(audio)
print(out)

spec = np.fft.fft(out)[:,:args.fs/2]

fig = plt.figure()
p   = fig.add_subplot(
    1,1,1,
    title="Frequency response with the default settings\n"
          "(Q={}, F={:.2f} Hz, G={:.0f} dB FS)".format(
              def_Q.zone, def_Freq.zone, 20*np.log10(def_Gain.zone+1e-8)
          ),
    xlabel="Frequency in Hz (log)",
    ylabel="Magnitude in dB FS",
    xscale="log"
)
p.plot(20*np.log10(np.absolute(spec.T)+1e-8))
p.legend(("Left channel", "Right channel"), loc="best")

################
# show the plots
################

plt.show()

print("everything passes!")
<|endoftext|>


<commit_before># Copyright (c) 2016 HuaWei, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from tempest import config
from tempest.lib.common.utils import data_utils
from tempest import test

from zaqar.tests.tempest_plugin.services.messaging.json import messaging_client

CONF = config.CONF

class BaseMessagingTest(test.BaseTestCase):

    """Base class for the Messaging (Zaqar) tests

    It is assumed that the following option is defined in the
    [service_available] section of etc/tempest.conf

        messaging as True
    """

    credentials = ['primary']

    @classmethod
    def skip_checks(cls):
        super(BaseMessagingTest, cls).skip_checks()
        if not CONF.service_available.zaqar:
            raise cls.skipException("Zaqar support is required")

    @classmethod
    def resource_setup(cls):
        super(BaseMessagingTest, cls).resource_setup()
        cls.messaging_cfg = CONF.messaging

    @classmethod
    def create_queue(cls, queue_name):
        """Wrapper utility that returns a test queue."""
        resp, body = cls.client.create_queue(queue_name)
        return resp, body

    @classmethod
    def delete_queue(cls, queue_name):
        """Wrapper utility that deletes a test queue."""
        resp, body = cls.client.delete_queue(queue_name)
        return resp, body

    @classmethod
    def list_queues(cls):
        """Wrapper utility that lists queues."""
        resp, body = cls.client.list_queues()
        return resp, body

    @classmethod
    def get_queue_stats(cls, queue_name):
        """Wrapper utility that returns the queue stats."""
        resp, body = cls.client.show_queue_stats(queue_name)
        return resp, body

    @classmethod
    def get_queue_metadata(cls, queue_name):
        """Wrapper utility that gets a queue metadata."""
        resp, body = cls.client.show_queue_metadata(queue_name)
        return resp, body

    @classmethod
    def set_queue_metadata(cls, queue_name, rbody):
        """Wrapper utility that sets the metadata of a queue."""
        resp, body = cls.client.set_queue_metadata(queue_name, rbody)
        return resp, body

    @classmethod
    def post_messages(cls, queue_name, rbody):
        """Wrapper utility that posts messages to a queue."""
        resp, body = cls.client.post_messages(queue_name, rbody)

        return resp, body

    @classmethod
    def list_messages(cls, queue_name):
        """Wrapper utility that lists the messages in a queue."""
        resp, body = cls.client.list_messages(queue_name)

        return resp, body

    @classmethod
    def delete_messages(cls, message_uri):
        """Wrapper utility that deletes messages."""
        resp, body = cls.client.delete_messages(message_uri)

        return resp, body

    @classmethod
    def post_claims(cls, queue_name, rbody, url_params=False):
        """Wrapper utility that claims messages."""
        resp, body = cls.client.post_claims(
            queue_name, rbody, url_params=False)

        return resp, body

    @classmethod
    def query_claim(cls, claim_uri):
        """Wrapper utility that gets a claim."""
        resp, body = cls.client.query_claim(claim_uri)

        return resp, body

    @classmethod
    def update_claim(cls, claim_uri, rbody):
        """Wrapper utility that updates a claim."""
        resp, body = cls.client.update_claim(claim_uri, rbody)

        return resp, body

    @classmethod
    def release_claim(cls, claim_uri):
        """Wrapper utility that deletes a claim."""
        resp, body = cls.client.delete_claim(claim_uri)

        return resp, body

    @classmethod
    def generate_message_body(cls, repeat=1):
        """Wrapper utility that sets the metadata of a queue."""
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        message_body = {key: value}

        rbody = ([{'body': message_body, 'ttl': message_ttl}] * repeat)
        return rbody

class BaseV1MessagingTest(BaseMessagingTest):
    """Base class for the Messaging (Zaqar) v1.0 tests."""
    @classmethod
    def setup_clients(cls):
        super(BaseV1MessagingTest, cls).setup_clients()
        cls.client = messaging_client.V1MessagingClient(
            cls.os.auth_provider,
            CONF.messaging.catalog_type,
            CONF.identity.region,
            build_interval=CONF.compute.build_interval,
            build_timeout=CONF.compute.build_timeout)

    @classmethod
    def check_queue_exists(cls, queue_name):
        """Wrapper utility that checks the existence of a test queue."""
        resp, body = cls.client.show_queue(queue_name)
        return resp, body

    @classmethod
    def check_queue_exists_head(cls, queue_name):
        """Wrapper utility checks the head of a queue via http HEAD."""
        resp, body = cls.client.head_queue(queue_name)
        return resp, body

class BaseV11MessagingTest(BaseMessagingTest):
    """Base class for the Messaging (Zaqar) v1.1 tests."""
    @classmethod
    def setup_clients(cls):
        super(BaseV11MessagingTest, cls).setup_clients()
        cls.client = messaging_client.V11MessagingClient(
            cls.os.auth_provider,
            CONF.messaging.catalog_type,
            CONF.identity.region,
            build_interval=CONF.compute.build_interval,
            build_timeout=CONF.compute.build_timeout)

    @classmethod
    def generate_message_body(cls, repeat=1):
        """Wrapper utility that sets the metadata of a queue."""
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        message_body = {key: value}

        body = ([{'body': message_body, 'ttl': message_ttl}] * repeat)
        rbody = {'messages': body}
        return rbody

class BaseV2MessagingTest(BaseMessagingTest):
    """Base class for the Messaging (Zaqar) v2 tests."""
    @classmethod
    def setup_clients(cls):
        super(BaseV2MessagingTest, cls).setup_clients()
        cls.client = messaging_client.V2MessagingClient(
            cls.os.auth_provider,
            CONF.messaging.catalog_type,
            CONF.identity.region,
            build_interval=CONF.compute.build_interval,
            build_timeout=CONF.compute.build_timeout)

    @classmethod
    def purge_queue(cls, queue_name, resource=None):
        resp, body = cls.client.purge_queue(
            queue_name, resource)
        return resp, body

    @classmethod
    def create_subscription(cls, queue_name, rbody):
        resp, body = cls.client.create_subscription(
            queue_name, rbody)
        return resp, body

    @classmethod
    def delete_subscription(cls, queue_name, subscription_id):
        resp, body = cls.client.delete_subscription(queue_name,
                                                    subscription_id)
        return resp, body

    @classmethod
    def list_subscription(cls, queue_name):
        resp, body = cls.client.list_subscription(queue_name)
        return resp, body

    @classmethod
    def show_subscription(cls, queue_name, subscription_id):
        resp, body = cls.client.show_subscription(queue_name, subscription_id)
        return resp, body

    @classmethod
    def update_subscription(cls, queue_name, subscription_id, rbody):
        resp, body = cls.client.update_subscription(queue_name,
                                                    subscription_id,
                                                    rbody)
        return resp, body

    @classmethod
    def generate_subscription_body(cls):
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        option_body = {key: value}
        subscribers = ['http://fake:8080', 'https://fake:8080',
                       'mailto:fake@123.com']
        rbody = [{'options': option_body, 'ttl': message_ttl,
                  'subscriber': subscriber} for subscriber in subscribers]
        return rbody

    @classmethod
    def generate_message_body(cls, repeat=1):
        """Wrapper utility that sets the metadata of a queue."""
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        message_body = {key: value}

        body = ([{'body': message_body, 'ttl': message_ttl}] * repeat)
        rbody = {'messages': body}
        return rbody
<commit_msg>Remove Deprecation tempest Warning<commit_after># Copyright (c) 2016 HuaWei, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from tempest import config
from tempest.lib.common.utils import data_utils
from tempest import test

from zaqar.tests.tempest_plugin.services.messaging.json import messaging_client

CONF = config.CONF

class BaseMessagingTest(test.BaseTestCase):

    """Base class for the Messaging (Zaqar) tests

    It is assumed that the following option is defined in the
    [service_available] section of etc/tempest.conf

        messaging as True
    """

    credentials = ['primary']

    @classmethod
    def skip_checks(cls):
        super(BaseMessagingTest, cls).skip_checks()
        if not CONF.service_available.zaqar:
            raise cls.skipException("Zaqar support is required")

    @classmethod
    def resource_setup(cls):
        super(BaseMessagingTest, cls).resource_setup()
        cls.messaging_cfg = CONF.messaging

    @classmethod
    def create_queue(cls, queue_name):
        """Wrapper utility that returns a test queue."""
        resp, body = cls.client.create_queue(queue_name)
        return resp, body

    @classmethod
    def delete_queue(cls, queue_name):
        """Wrapper utility that deletes a test queue."""
        resp, body = cls.client.delete_queue(queue_name)
        return resp, body

    @classmethod
    def list_queues(cls):
        """Wrapper utility that lists queues."""
        resp, body = cls.client.list_queues()
        return resp, body

    @classmethod
    def get_queue_stats(cls, queue_name):
        """Wrapper utility that returns the queue stats."""
        resp, body = cls.client.show_queue_stats(queue_name)
        return resp, body

    @classmethod
    def get_queue_metadata(cls, queue_name):
        """Wrapper utility that gets a queue metadata."""
        resp, body = cls.client.show_queue_metadata(queue_name)
        return resp, body

    @classmethod
    def set_queue_metadata(cls, queue_name, rbody):
        """Wrapper utility that sets the metadata of a queue."""
        resp, body = cls.client.set_queue_metadata(queue_name, rbody)
        return resp, body

    @classmethod
    def post_messages(cls, queue_name, rbody):
        """Wrapper utility that posts messages to a queue."""
        resp, body = cls.client.post_messages(queue_name, rbody)

        return resp, body

    @classmethod
    def list_messages(cls, queue_name):
        """Wrapper utility that lists the messages in a queue."""
        resp, body = cls.client.list_messages(queue_name)

        return resp, body

    @classmethod
    def delete_messages(cls, message_uri):
        """Wrapper utility that deletes messages."""
        resp, body = cls.client.delete_messages(message_uri)

        return resp, body

    @classmethod
    def post_claims(cls, queue_name, rbody, url_params=False):
        """Wrapper utility that claims messages."""
        resp, body = cls.client.post_claims(
            queue_name, rbody, url_params=False)

        return resp, body

    @classmethod
    def query_claim(cls, claim_uri):
        """Wrapper utility that gets a claim."""
        resp, body = cls.client.query_claim(claim_uri)

        return resp, body

    @classmethod
    def update_claim(cls, claim_uri, rbody):
        """Wrapper utility that updates a claim."""
        resp, body = cls.client.update_claim(claim_uri, rbody)

        return resp, body

    @classmethod
    def release_claim(cls, claim_uri):
        """Wrapper utility that deletes a claim."""
        resp, body = cls.client.delete_claim(claim_uri)

        return resp, body

    @classmethod
    def generate_message_body(cls, repeat=1):
        """Wrapper utility that sets the metadata of a queue."""
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        message_body = {key: value}

        rbody = ([{'body': message_body, 'ttl': message_ttl}] * repeat)
        return rbody

class BaseV1MessagingTest(BaseMessagingTest):
    """Base class for the Messaging (Zaqar) v1.0 tests."""
    @classmethod
    def setup_clients(cls):
        super(BaseV1MessagingTest, cls).setup_clients()
        cls.client = messaging_client.V1MessagingClient(
            cls.os_primary.auth_provider,
            CONF.messaging.catalog_type,
            CONF.identity.region,
            build_interval=CONF.compute.build_interval,
            build_timeout=CONF.compute.build_timeout)

    @classmethod
    def check_queue_exists(cls, queue_name):
        """Wrapper utility that checks the existence of a test queue."""
        resp, body = cls.client.show_queue(queue_name)
        return resp, body

    @classmethod
    def check_queue_exists_head(cls, queue_name):
        """Wrapper utility checks the head of a queue via http HEAD."""
        resp, body = cls.client.head_queue(queue_name)
        return resp, body

class BaseV11MessagingTest(BaseMessagingTest):
    """Base class for the Messaging (Zaqar) v1.1 tests."""
    @classmethod
    def setup_clients(cls):
        super(BaseV11MessagingTest, cls).setup_clients()
        cls.client = messaging_client.V11MessagingClient(
            cls.os_primary.auth_provider,
            CONF.messaging.catalog_type,
            CONF.identity.region,
            build_interval=CONF.compute.build_interval,
            build_timeout=CONF.compute.build_timeout)

    @classmethod
    def generate_message_body(cls, repeat=1):
        """Wrapper utility that sets the metadata of a queue."""
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        message_body = {key: value}

        body = ([{'body': message_body, 'ttl': message_ttl}] * repeat)
        rbody = {'messages': body}
        return rbody

class BaseV2MessagingTest(BaseMessagingTest):
    """Base class for the Messaging (Zaqar) v2 tests."""
    @classmethod
    def setup_clients(cls):
        super(BaseV2MessagingTest, cls).setup_clients()
        cls.client = messaging_client.V2MessagingClient(
            cls.os_primary.auth_provider,
            CONF.messaging.catalog_type,
            CONF.identity.region,
            build_interval=CONF.compute.build_interval,
            build_timeout=CONF.compute.build_timeout)

    @classmethod
    def purge_queue(cls, queue_name, resource=None):
        resp, body = cls.client.purge_queue(
            queue_name, resource)
        return resp, body

    @classmethod
    def create_subscription(cls, queue_name, rbody):
        resp, body = cls.client.create_subscription(
            queue_name, rbody)
        return resp, body

    @classmethod
    def delete_subscription(cls, queue_name, subscription_id):
        resp, body = cls.client.delete_subscription(queue_name,
                                                    subscription_id)
        return resp, body

    @classmethod
    def list_subscription(cls, queue_name):
        resp, body = cls.client.list_subscription(queue_name)
        return resp, body

    @classmethod
    def show_subscription(cls, queue_name, subscription_id):
        resp, body = cls.client.show_subscription(queue_name, subscription_id)
        return resp, body

    @classmethod
    def update_subscription(cls, queue_name, subscription_id, rbody):
        resp, body = cls.client.update_subscription(queue_name,
                                                    subscription_id,
                                                    rbody)
        return resp, body

    @classmethod
    def generate_subscription_body(cls):
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        option_body = {key: value}
        subscribers = ['http://fake:8080', 'https://fake:8080',
                       'mailto:fake@123.com']
        rbody = [{'options': option_body, 'ttl': message_ttl,
                  'subscriber': subscriber} for subscriber in subscribers]
        return rbody

    @classmethod
    def generate_message_body(cls, repeat=1):
        """Wrapper utility that sets the metadata of a queue."""
        message_ttl = data_utils.\
            rand_int_id(start=60, end=CONF.messaging.max_message_ttl)

        key = data_utils.arbitrary_string(size=20, base_text='MessagingKey')
        value = data_utils.arbitrary_string(size=20,
                                            base_text='MessagingValue')
        message_body = {key: value}

        body = ([{'body': message_body, 'ttl': message_ttl}] * repeat)
        rbody = {'messages': body}
        return rbody
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Main Controller"""
import datetime
import collections
import webhelpers.feedgenerator as feedgenerator
from sqlalchemy.orm.exc import NoResultFound
from tg import expose, flash, require, lurl, config
from tg import predicates
from tg import request, redirect, tmpl_context
from tg.decorators import paginate
from tg.exceptions import HTTPFound
from tg.i18n import ugettext as _, lazy_ugettext as l_
from tgext.admin.controller import AdminController
from tgext.admin.tgadminconfig import BootstrapTGAdminConfig as TGAdminConfig

from pyjobsweb import model
from pyjobsweb.controllers.error import ErrorController
from pyjobsweb.controllers.secure import SecureController
from pyjobsweb.lib.base import BaseController
from pyjobsweb.lib.helpers import slugify, get_job_url
from pyjobsweb.lib.stats import StatsQuestioner
from pyjobsweb.model import DBSession, Log
from pyjobsweb.model.data import JobOfferSQLAlchemy, SOURCES
from pyjobsweb.forms.ResearchForm import ResearchForm

__all__ = ['RootController']
existing_fields = (
    'title',
    'publication_datetime',
    'company',
    'company_url',
    'address',
    'description',
    'tags',
)

class RootController(BaseController):
    """
    The root controller for the pyjobsweb application.

    All the other controllers and WSGI applications should be mounted on this
    controller. For example::

        panel = ControlPanelController()
        another_app = AnotherWSGIApplication()

    Keep in mind that WSGI applications shouldn't be mounted directly: They
    must be wrapped around with :class:`tg.controllers.WSGIAppController`.

    """
    secc = SecureController()
    admin = AdminController(model, DBSession, config_type=TGAdminConfig)

    error = ErrorController()

    def _before(self, *args, **kw):
        tmpl_context.project_name = "Algoo"

    items_per_page = 20

    @expose('pyjobsweb.templates.jobs')
    @paginate('jobs', items_per_page=items_per_page)
    def index(self, query=None, radius=None, center=None, unit=None):
        if not query and not center and not radius:
            job_offers = JobOfferSQLAlchemy.get_all_job_offers()
        else:
            import pyjobsweb.lib.search_query as sq

            search_query = model.ElasticsearchQuery(0, self.items_per_page * 50)

            search_on = ['description', 'title']
            for q in query.split(','):
                search_query.builder.add_elem(sq.KeywordFilter(search_on, [q]))

            if center and radius:
                import geopy.exc
                try:
                    import geopy
                    geolocator = geopy.geocoders.Nominatim()
                    loc = geolocator.geocode(center)
                    center_point = sq.GeolocationFilter.Center(
                        loc.latitude, loc.longitude
                    )
                    radius = float(radius)
                    unit = sq.GeolocationFilter.UnitsEnum(unit)
                    search_query.builder.add_elem(
                        sq.GeolocationFilter(center_point, radius, unit)
                    )
                except AttributeError:
                    pass  # TODO : Mr proper, in case of impossible @ resolution
                except geopy.exc.GeopyError:
                    pass  # TODO : Mr proper
                except ValueError:
                    pass  # TODO : Mr proper

            ms = sq.Sort()
            ms.append(sq.DescSortStatement('publication_datetime'))
            search_query.builder.add_elem(ms)

            job_offers = search_query.execute_query()

        search_form = ResearchForm(action='/', method='POST').req()

        return dict(
            sources=SOURCES,
            jobs=job_offers,
            job_offer_search_form=search_form
        )

    @staticmethod
    def photon_query_builder(address):  # TODO: Export this in a package
        import urllib
        return u'http://photon.komoot.de/api/?q={}&lang=fr'\
            .format(urllib.quote(address.encode('utf-8')))

    @staticmethod
    def execute_query(url):
        import urllib2
        return urllib2.urlopen(url).read()

    @staticmethod
    def format_result(result):  # TODO: export this in a package
        import json
        results_dict = json.loads(result)

        france_results = list()
        other_results = list()

        features = results_dict['features']

        for qr in features:
            properties = qr['properties']
            address_elem = [
                'name', 'housenumber', 'street', 'postcode', 'state', 'country'
            ]
            address = dict()

            for e in address_elem:
                if e not in properties:
                    continue

                address[e] = properties[e].encode('utf-8')

            if address not in france_results and address not in other_results:
                if 'country' in address and address['country'] == 'France':
                    france_results.append(address)
                else:
                    other_results.append(address)

        results = list()
        results.extend(france_results)
        results.extend(other_results)

        return results

    @expose('json')
    def geocomplete(self, *args, **kwargs):
        if 'address' not in kwargs:
            return []

        req_url = self.photon_query_builder(kwargs['address'])
        raw_res = self.execute_query(req_url)
        results = self.format_result(raw_res)

        return dict(results=results)

    @expose()
    def rss(self, limit=50, source=None):
        """
        RSS feed of jobs
        :param source: source name
        :param limit: number of displayed jobs
        :return: RSS feed content
        """
        site_url = config.get('site.domain_base_url')
        feed = feedgenerator.Rss201rev2Feed(
            title=u"pyjobs : le job qu'il vous faut en python",
            link=site_url,
            description=u"Agrégation de jobs python",
            language=u"fr",
            feed_url=u"http://www.pyjobs.fr/rss?limit=%s" % limit
        )

        jobs = DBSession.query(JobOfferSQLAlchemy) \
            .order_by(JobOfferSQLAlchemy.publication_datetime.desc()) \
            .limit(limit)

        if source is not None:
            jobs = jobs.filter(JobOfferSQLAlchemy.source == source)

        for job in jobs:
            job_slug = slugify(job.title)
            feed.add_item(
                    title=job.title,
                    link=get_job_url(job.id, job_title=job.title, absolute=True),
                    description=job.description,
                    pubdate=job.publication_datetime,
                    unique_id="%s/job/%d/%s" % (site_url, job.id, job_slug)
            )

        return feed.writeString('utf-8')

    @expose('pyjobsweb.templates.job')
    def job(self, job_id, job_title=None, previous=None):
        """
        Job detail page
        :param job_id: Job identifier
        :param job_title: Job title (optional) for pretty url
        :return: dict
        """
        try:
            job = DBSession.query(JobOfferSQLAlchemy).filter_by(id=job_id).one()
        except NoResultFound:
            pass  # TODO: TubroGears 404 ?
        return dict(
                job=job,
                sources=SOURCES
        )

    @expose('pyjobsweb.templates.sources')
    def sources(self):
        sources_last_crawl = {}
        sorted_sources = collections.OrderedDict(sorted(SOURCES.items(), key=lambda x: x[1].label))
        for source_name in sorted_sources:
            try:
                sources_last_crawl[source_name] = DBSession.query(Log.datetime) \
                    .filter(Log.source == source_name) \
                    .order_by(Log.datetime.desc()) \
                    .limit(1)\
                    .one()[0]
            except NoResultFound:
                sources_last_crawl[source_name] = None

        return dict(
                sources=sorted_sources,
                existing_fields=existing_fields,
                sources_last_crawl=sources_last_crawl
        )

    @expose('pyjobsweb.templates.stats')
    def stats(self, since_months=4):
        stats = StatsQuestioner
        stats_questioner = stats(DBSession)
        date_from, date_to = stats.get_month_period(int(since_months))

        by_months = stats_questioner.by_complete_period(
            period=stats.PERIOD_MONTH,
            date_from=date_from,
            date_to=date_to,
        ).all()
        months = stats.extract(by_months, stats.FIELD_DATE)
        stats_month = stats.extract_stats(query_result=by_months, sources=SOURCES.keys())
        flat_month = stats.flat_query_by_y(
            query_result=by_months,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m')
        )

        by_weeks = stats_questioner.by_complete_period(
            stats.PERIOD_WEEK,
            date_from=date_from,
            date_to=date_to,
        ).all()
        weeks = stats.extract(by_weeks, stats.FIELD_DATE)
        stats_week = stats.extract_stats(query_result=by_weeks, sources=SOURCES.keys())
        flat_week = stats.flat_query_by_y(
            query_result=by_weeks,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m-%d')
        )

        return dict(
            sources=SOURCES,
            stats_month=stats_month,
            flat_month=flat_month,
            stats_week=stats_week,
            flat_week=flat_week,
            months=months,
            weeks=weeks,
            flat_x_field=stats.FIELDS[stats.FLAT_X_FIELD],
            flat_y_fields=SOURCES.keys(),
            sources_labels=[SOURCES[source].label for source in SOURCES]
        )

    @expose('pyjobsweb.templates.logs')
    def logs(self, source=None, last_days=1):

        logs_query = DBSession.query(Log)\
            .order_by(Log.datetime.desc())\
            .filter(Log.datetime >= datetime.datetime.now() + datetime.timedelta(days=-int(last_days)))\
            .filter(Log.message.in_(('CRAWL_LIST_START',
                                    'CRAWL_LIST_FINISHED',
                                    'ERROR_UNEXPECTED_END',
                                    'ERROR_CRAWNLING')))

        if source is not None:
            logs_query = logs_query.filter(Log.source == source)

        return dict(
            sources=SOURCES,
            logs=logs_query.all(),
            last_days=last_days
        )

    @expose('pyjobsweb.templates.about')
    def about(self):
        return dict()

    @expose('pyjobsweb.templates.index')
    @require(predicates.has_permission('manage', msg=l_('Only for managers')))
    def manage_permission_only(self, **kw):
        """Illustrate how a page for managers only works."""
        return dict(page='managers stuff')

    @expose('pyjobsweb.templates.index')
    @require(predicates.is_user('editor', msg=l_('Only for the editor')))
    def editor_user_only(self, **kw):
        """Illustrate how a page exclusive for the editor works."""
        return dict(page='editor stuff')

    @expose('pyjobsweb.templates.login')
    def login(self, came_from=lurl('/'), failure=None, login=''):
        """Start the user login."""
        if failure is not None:
            if failure == 'user-not-found':
                flash(_('User not found'), 'error')
            elif failure == 'invalid-password':
                flash(_('Invalid Password'), 'error')

        login_counter = request.environ.get('repoze.who.logins', 0)
        if failure is None and login_counter > 0:
            flash(_('Wrong credentials'), 'warning')

        return dict(page='login', login_counter=str(login_counter),
                    came_from=came_from, login=login)

    @expose()
    def post_login(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on successful
        authentication or redirect her back to the login page if login failed.

        """
        if not request.identity:
            login_counter = request.environ.get('repoze.who.logins', 0) + 1
            redirect('/login',
                     params=dict(came_from=came_from, __logins=login_counter))
        userid = request.identity['repoze.who.userid']
        flash(_('Welcome back, %s!') % userid)

        # Do not use tg.redirect with tg.url as it will add the mountpoint
        # of the application twice.
        return HTTPFound(location=came_from)

    @expose()
    def post_logout(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on logout and say
        goodbye as well.

        """
        flash(_('We hope to see you soon!'))
        return HTTPFound(location=came_from)
<commit_msg>Limit geolocation results to France only<commit_after># -*- coding: utf-8 -*-
"""Main Controller"""
import datetime
import collections
import webhelpers.feedgenerator as feedgenerator
from sqlalchemy.orm.exc import NoResultFound
from tg import expose, flash, require, lurl, config
from tg import predicates
from tg import request, redirect, tmpl_context
from tg.decorators import paginate
from tg.exceptions import HTTPFound
from tg.i18n import ugettext as _, lazy_ugettext as l_
from tgext.admin.controller import AdminController
from tgext.admin.tgadminconfig import BootstrapTGAdminConfig as TGAdminConfig

from pyjobsweb import model
from pyjobsweb.controllers.error import ErrorController
from pyjobsweb.controllers.secure import SecureController
from pyjobsweb.lib.base import BaseController
from pyjobsweb.lib.helpers import slugify, get_job_url
from pyjobsweb.lib.stats import StatsQuestioner
from pyjobsweb.model import DBSession, Log
from pyjobsweb.model.data import JobOfferSQLAlchemy, SOURCES
from pyjobsweb.forms.ResearchForm import ResearchForm

__all__ = ['RootController']
existing_fields = (
    'title',
    'publication_datetime',
    'company',
    'company_url',
    'address',
    'description',
    'tags',
)

class RootController(BaseController):
    """
    The root controller for the pyjobsweb application.

    All the other controllers and WSGI applications should be mounted on this
    controller. For example::

        panel = ControlPanelController()
        another_app = AnotherWSGIApplication()

    Keep in mind that WSGI applications shouldn't be mounted directly: They
    must be wrapped around with :class:`tg.controllers.WSGIAppController`.

    """
    secc = SecureController()
    admin = AdminController(model, DBSession, config_type=TGAdminConfig)

    error = ErrorController()

    def _before(self, *args, **kw):
        tmpl_context.project_name = "Algoo"

    items_per_page = 20

    @expose('pyjobsweb.templates.jobs')
    @paginate('jobs', items_per_page=items_per_page)
    def index(self, query=None, radius=None, center=None, unit=None):
        if not query and not center and not radius:
            job_offers = JobOfferSQLAlchemy.get_all_job_offers()
        else:
            import pyjobsweb.lib.search_query as sq

            search_query = model.ElasticsearchQuery(0, self.items_per_page * 50)

            search_on = ['description', 'title']
            for q in query.split(','):
                search_query.builder.add_elem(sq.KeywordFilter(search_on, [q]))

            if center and radius:
                import geopy.exc
                try:
                    import geopy
                    geolocator = geopy.geocoders.Nominatim()
                    loc = geolocator.geocode(center)
                    center_point = sq.GeolocationFilter.Center(
                        loc.latitude, loc.longitude
                    )
                    radius = float(radius)
                    unit = sq.GeolocationFilter.UnitsEnum(unit)
                    search_query.builder.add_elem(
                        sq.GeolocationFilter(center_point, radius, unit)
                    )
                except AttributeError:
                    pass  # TODO : Mr proper, in case of impossible @ resolution
                except geopy.exc.GeopyError:
                    pass  # TODO : Mr proper
                except ValueError:
                    pass  # TODO : Mr proper

            ms = sq.Sort()
            ms.append(sq.DescSortStatement('publication_datetime'))
            search_query.builder.add_elem(ms)

            job_offers = search_query.execute_query()

        search_form = ResearchForm(action='/', method='POST').req()

        return dict(
            sources=SOURCES,
            jobs=job_offers,
            job_offer_search_form=search_form
        )

    @staticmethod
    def photon_query_builder(address):  # TODO: Export this in a package
        import urllib
        return u'http://photon.komoot.de/api/?q={}&lang=fr'\
            .format(urllib.quote(address.encode('utf-8')))

    @staticmethod
    def execute_query(url):
        import urllib2
        return urllib2.urlopen(url).read()

    @staticmethod
    def format_result(result):  # TODO: export this in a package
        import json
        results_dict = json.loads(result)

        france_results = list()

        features = results_dict['features']

        for qr in features:
            properties = qr['properties']
            address_elem = [
                'name', 'housenumber', 'street', 'postcode', 'state', 'country'
            ]
            address = dict()

            for e in address_elem:
                if e not in properties:
                    continue

                address[e] = properties[e].encode('utf-8')

            if address not in france_results:
                if 'country' in address and address['country'] == 'France':
                    france_results.append(address)

        return france_results

    @expose('json')
    def geocomplete(self, *args, **kwargs):
        if 'address' not in kwargs:
            return []

        req_url = self.photon_query_builder(kwargs['address'])
        raw_res = self.execute_query(req_url)
        results = self.format_result(raw_res)

        return dict(results=results)

    @expose()
    def rss(self, limit=50, source=None):
        """
        RSS feed of jobs
        :param source: source name
        :param limit: number of displayed jobs
        :return: RSS feed content
        """
        site_url = config.get('site.domain_base_url')
        feed = feedgenerator.Rss201rev2Feed(
            title=u"pyjobs : le job qu'il vous faut en python",
            link=site_url,
            description=u"Agrégation de jobs python",
            language=u"fr",
            feed_url=u"http://www.pyjobs.fr/rss?limit=%s" % limit
        )

        jobs = DBSession.query(JobOfferSQLAlchemy) \
            .order_by(JobOfferSQLAlchemy.publication_datetime.desc()) \
            .limit(limit)

        if source is not None:
            jobs = jobs.filter(JobOfferSQLAlchemy.source == source)

        for job in jobs:
            job_slug = slugify(job.title)
            feed.add_item(
                    title=job.title,
                    link=get_job_url(job.id, job_title=job.title, absolute=True),
                    description=job.description,
                    pubdate=job.publication_datetime,
                    unique_id="%s/job/%d/%s" % (site_url, job.id, job_slug)
            )

        return feed.writeString('utf-8')

    @expose('pyjobsweb.templates.job')
    def job(self, job_id, job_title=None, previous=None):
        """
        Job detail page
        :param job_id: Job identifier
        :param job_title: Job title (optional) for pretty url
        :return: dict
        """
        try:
            job = DBSession.query(JobOfferSQLAlchemy).filter_by(id=job_id).one()
        except NoResultFound:
            pass  # TODO: TubroGears 404 ?
        return dict(
                job=job,
                sources=SOURCES
        )

    @expose('pyjobsweb.templates.sources')
    def sources(self):
        sources_last_crawl = {}
        sorted_sources = collections.OrderedDict(sorted(SOURCES.items(), key=lambda x: x[1].label))
        for source_name in sorted_sources:
            try:
                sources_last_crawl[source_name] = DBSession.query(Log.datetime) \
                    .filter(Log.source == source_name) \
                    .order_by(Log.datetime.desc()) \
                    .limit(1)\
                    .one()[0]
            except NoResultFound:
                sources_last_crawl[source_name] = None

        return dict(
                sources=sorted_sources,
                existing_fields=existing_fields,
                sources_last_crawl=sources_last_crawl
        )

    @expose('pyjobsweb.templates.stats')
    def stats(self, since_months=4):
        stats = StatsQuestioner
        stats_questioner = stats(DBSession)
        date_from, date_to = stats.get_month_period(int(since_months))

        by_months = stats_questioner.by_complete_period(
            period=stats.PERIOD_MONTH,
            date_from=date_from,
            date_to=date_to,
        ).all()
        months = stats.extract(by_months, stats.FIELD_DATE)
        stats_month = stats.extract_stats(query_result=by_months, sources=SOURCES.keys())
        flat_month = stats.flat_query_by_y(
            query_result=by_months,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m')
        )

        by_weeks = stats_questioner.by_complete_period(
            stats.PERIOD_WEEK,
            date_from=date_from,
            date_to=date_to,
        ).all()
        weeks = stats.extract(by_weeks, stats.FIELD_DATE)
        stats_week = stats.extract_stats(query_result=by_weeks, sources=SOURCES.keys())
        flat_week = stats.flat_query_by_y(
            query_result=by_weeks,
            sources=SOURCES.keys(),
            date_value_callback=lambda date: date.strftime('%Y-%m-%d')
        )

        return dict(
            sources=SOURCES,
            stats_month=stats_month,
            flat_month=flat_month,
            stats_week=stats_week,
            flat_week=flat_week,
            months=months,
            weeks=weeks,
            flat_x_field=stats.FIELDS[stats.FLAT_X_FIELD],
            flat_y_fields=SOURCES.keys(),
            sources_labels=[SOURCES[source].label for source in SOURCES]
        )

    @expose('pyjobsweb.templates.logs')
    def logs(self, source=None, last_days=1):

        logs_query = DBSession.query(Log)\
            .order_by(Log.datetime.desc())\
            .filter(Log.datetime >= datetime.datetime.now() + datetime.timedelta(days=-int(last_days)))\
            .filter(Log.message.in_(('CRAWL_LIST_START',
                                    'CRAWL_LIST_FINISHED',
                                    'ERROR_UNEXPECTED_END',
                                    'ERROR_CRAWNLING')))

        if source is not None:
            logs_query = logs_query.filter(Log.source == source)

        return dict(
            sources=SOURCES,
            logs=logs_query.all(),
            last_days=last_days
        )

    @expose('pyjobsweb.templates.about')
    def about(self):
        return dict()

    @expose('pyjobsweb.templates.index')
    @require(predicates.has_permission('manage', msg=l_('Only for managers')))
    def manage_permission_only(self, **kw):
        """Illustrate how a page for managers only works."""
        return dict(page='managers stuff')

    @expose('pyjobsweb.templates.index')
    @require(predicates.is_user('editor', msg=l_('Only for the editor')))
    def editor_user_only(self, **kw):
        """Illustrate how a page exclusive for the editor works."""
        return dict(page='editor stuff')

    @expose('pyjobsweb.templates.login')
    def login(self, came_from=lurl('/'), failure=None, login=''):
        """Start the user login."""
        if failure is not None:
            if failure == 'user-not-found':
                flash(_('User not found'), 'error')
            elif failure == 'invalid-password':
                flash(_('Invalid Password'), 'error')

        login_counter = request.environ.get('repoze.who.logins', 0)
        if failure is None and login_counter > 0:
            flash(_('Wrong credentials'), 'warning')

        return dict(page='login', login_counter=str(login_counter),
                    came_from=came_from, login=login)

    @expose()
    def post_login(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on successful
        authentication or redirect her back to the login page if login failed.

        """
        if not request.identity:
            login_counter = request.environ.get('repoze.who.logins', 0) + 1
            redirect('/login',
                     params=dict(came_from=came_from, __logins=login_counter))
        userid = request.identity['repoze.who.userid']
        flash(_('Welcome back, %s!') % userid)

        # Do not use tg.redirect with tg.url as it will add the mountpoint
        # of the application twice.
        return HTTPFound(location=came_from)

    @expose()
    def post_logout(self, came_from=lurl('/')):
        """
        Redirect the user to the initially requested page on logout and say
        goodbye as well.

        """
        flash(_('We hope to see you soon!'))
        return HTTPFound(location=came_from)
<|endoftext|>


<commit_before>from functools import total_ordering
import itertools as it

EXONIC_IMPACTS = set(["stop_gained",
                      "stop_lost",
                      "frameshift_variant",
                      "initiator_codon_variant",
                      "inframe_deletion",
                      "inframe_insertion",
                      "missense_variant",
                      "incomplete_terminal_codon_variant",
                      "stop_retained_variant",
                      "synonymous_variant",
                      "coding_sequence_variant",
                      "5_prime_UTR_variant",
                      "3_prime_UTR_variant",
                      "transcript_ablation",
                      "transcript_amplification",
                      "feature_elongation",
                      "feature_truncation"])

@total_ordering
class Effect(object):

    def __init__(self, effect_dict):
        # or maybe arg should be a dict for Effect()
        raise NotImplementedError

    def __le__(self, other):
        if self.pseudogene and not other.pseudogene:
            return True
        elif other.pseudogene and not self.pseudogene:
            return False
        if self.coding and not other.coding:
            return False
        elif other.coding and not self.coding:
            return True

        if self.severity != other.severity:
            return self.severity <= other.severity

        raise NotImplementedError
        # TODO: look at polyphen/sift?

    def __eq__(self, other):
        return self.effects == other.effects

    def __str__(self):
        # TODO: tab-delimited?
        raise NotImplementedError

    def __repr__(self):
        return "%s(%s-%s)" % (self.__class__.__name__, self.gene,
                self.consequence)

    @property
    def gene(self):
        raise NotImplementedError

    @property
    def transcript(self):
        raise NotImplementedError

    @property
    def exonic(self):
        raise NotImplementedError

    @property
    def coding(self):
        return True

    @property
    def lof(self):
        return True

    @property
    def aa_change(self):
        raise NotImplementedError

    @property
    def codon_change(self):
        raise NotImplementedError

    @property
    def severity(self):
        # higher is more severe. used for ordering.
        return 1

    @property
    def consequence(self):
        raise NotImplementedError

    @property
    def biotype(self):
        raise NotImplementedError

    @property
    def is_pseudogene(self): #bool
        raise NotImplementedError

class SnpEff(Effect):

    keys = [x.strip() for x in 'Allele | Annotation | Annotation_Impact | Gene_Name | Gene_ID | Feature_Type | Feature_ID | Transcript_BioType | Rank | HGVS.c | HGVS.p | cDNA.pos / cDNA.length | CDS.pos / CDS.length | AA.pos / AA.length | Distance | ERRORS / WARNINGS / INFO'.split("|")]

    def __init__(self, effect_string):
        assert not "," in effect_string
        assert not "=" in effect_string
        self.effect_string = effect_string
        self.effects = dict(it.izip(self.keys, (x.strip() for x in effect_string.split("|"))))

    @property
    def gene(self):
        return self.effects['Gene_Name'] or None

    @property
    def transcript(self):
        return self.effects['Feature_ID'] or None

    @property
    def exon(self):
        return self.effects['Rank']

    @property
    def consequence(self):
        if '&' in self.effects['Annotation']:
            return self.effects['Annotation'].split('&')
        return self.effects['Annotation']

    @property
    def biotype(self):
        return self.effects['Transcript_BioType']

    @property
    def alt(self):
        return self.effects['Allele']

    @property
    def is_pseudogene(self):
        return 'pseudogene' in self.effects['Transcript_BioType']

    @property
    def coding(self):
        # TODO: check start_gained and utr
        return self.exonic and not "utr" in self.consequence and self.consequence != "start_gained"

    @property
    def exonic(self):
        csqs = self.consequence
        if isinstance(csqs, basestring):
            csqs = [csqs]
        return any(csq in EXONIC_IMPACTS for csq in csqs) and self.effects['Transcript_BioType'] == 'protein_coding'

    # not defined in ANN field.
    aa_change = None
    sift = None
    polyphen = None

class VEP(Effect):
    keys = "Consequence|Codons|Amino_acids|Gene|SYMBOL|Feature|EXON|PolyPhen|SIFT|Protein_position|BIOTYPE".split("|")
    def __init__(self, effect_string, keys=None):
        assert not "," in effect_string
        assert not "=" in effect_string
        self.effect_string = effect_string
        if keys is not None: self.keys = keys

        self.effect_string = effect_string
        self.effects = dict(it.izip(self.keys, (x.strip() for x in effect_string.split("|"))))

    @property
    def gene(self):
        return self.effects['SYMBOL'] or self.effects['Gene']

    @property
    def transcript(self):
        return self.effects['Feature']

    @property
    def exon(self):
        return self.effects['EXON']

    @property
    def consequence(self):
        if '&' in self.effects['Consequence']:
            return self.effects['Consequence'].split('&')
        return self.effects['Consequence']

    @property
    def biotype(self):
        return self.effects['BIOTYPE']

    @property
    def alt(self):
        return self.effects.get('ALLELE')

    @property
    def is_pseudogene(self):
        return self.effects['BIOTYPE'] == 'processed_pseudogene'

    @property
    def coding(self):
        # what about start/stop_gained?
        return self.exonic and self.effect_name[1:] != "_prime_UTR_variant"

    def exonic(self):
        return self.consequence in EXONIC_IMPACTS and self.effects['BIOTYPE'] == 'protein_coding'

    @property
    def sift(self):
        return self.effects['SIFT']

    @property
    def polyphen(self):
        return self.effects['PolyPhen']

    @property
    def aa_change(self):
        return self.effects['Amino_acids']

if __name__ == "__main__":

    s = SnpEff("A|stop_gained|HIGH|C1orf170|ENSG00000187642|transcript|ENST00000433179|protein_coding|3/5|c.262C>T|p.Arg88*|262/3064|262/2091|88/696||")
    print s.effects
    print s.gene, s.transcript, s.consequence, s.is_pseudogene
    s = SnpEff("G|splice_donor_variant&intron_variant|HIGH|WASH7P|ENSG00000227232|transcript|ENST00000423562|unprocessed_pseudogene|6/9|n.822+2T>C||||||")
    print s.is_pseudogene
    s = SnpEff("G|missense_variant|MODERATE|OR4F5|ENSG00000186092|transcript|ENST00000335137|protein_coding|1/1|c.338T>G|p.Phe113Cys|338/918|338/918|113/305||")
    print s.coding, s.consequence, s.aa_change
<commit_msg>get all effects I can find<commit_after>from functools import total_ordering
import itertools as it

EXONIC_IMPACTS = set(["stop_gained",
                      "stop_lost",
                      "frameshift_variant",
                      "initiator_codon_variant",
                      "inframe_deletion",
                      "inframe_insertion",
                      "missense_variant",
                      "incomplete_terminal_codon_variant",
                      "stop_retained_variant",
                      "synonymous_variant",
                      "coding_sequence_variant",
                      "5_prime_UTR_variant",
                      "3_prime_UTR_variant",
                      "transcript_ablation",
                      "transcript_amplification",
                      "feature_elongation",
                      "feature_truncation"])

IMPACT_SEVERITY = dict([
    ('non_coding_exon_variant', 'LOW'),
    ('incomplete_terminal_codon_variant', 'LOW'),
    ('stop_retained_variant', 'LOW'),
    ('synonymous_variant', 'LOW'),
    ('coding_sequence_variant', 'LOW'),
    ('5_prime_UTR_variant', 'LOW'),
    ('3_prime_UTR_variant', 'LOW'),
    ('intron_variant', 'LOW'),
    ('NMD_transcript_variant', 'LOW'),
    ('nc_transcript_variant', 'LOW'),
    ('upstream_gene_variant', 'LOW'),
    ('downstream_gene_variant', 'LOW'),
    ('intergenic_variant', 'LOW'),
    ('transcript_ablation', 'LOW'),
    ('transcript_amplification', 'LOW'),
    ('feature_elongation', 'LOW'),
    ('feature_truncation', 'LOW'),

    ('inframe_deletion', 'MED'),
    ('inframe_insertion', 'MED'),
    ('missense_variant', 'MED'),
    ('splice_region_variant', 'MED'),
    ('mature_miRNA_variant', 'MED'),
    ('regulatory_region_variant', 'MED'),
    ('TF_binding_site_variant', 'MED'),
    ('regulatory_region_ablation', 'MED'),
    ('regulatory_region_amplification', 'MED'),
    ('TFBS_ablation', 'MED'),
    ('TFBS_amplification', 'MED'),

    ('splice_acceptor_variant', 'HIGH'),
    ('splice_donor_variant', 'HIGH'),
    ('stop_gained', 'HIGH'),
    ('stop_lost', 'HIGH'),
    ('frameshift_variant', 'HIGH'),
    ('initiator_codon_variant', 'HIGH'),

])

# these are taken from the snpeff manual.
# with MODIFIER => LOW and MEDIUM => MED
IMPACT_SEVERITY_SNPEFF = dict([
    ('chromosome_number_variation', 'HIGH'),
    ('exon_loss_variant', 'HIGH'),
    ('rare_amino_acid_variant', 'HIGH'),
    ('start_lost', 'HIGH'),
    ('5_prime_UTR_premature_start_codon_gain_variant', 'LOW'),
    ('conserved_intergenic_variant', 'LOW'),
    ('conserved_intron_variant', 'LOW'),
    ('exon_variant', 'LOW'),
    ('gene_variant', 'LOW'),
    ('intergenic_region', 'LOW'),
    ('intragenic_variant', 'LOW'),
    ('miRNA', 'LOW'),
    ('non_coding_transcript_exon_variant', 'LOW'),
    ('non_coding_transcript_variant', 'LOW'),
    ('start_retained', 'LOW'),
    ('transcript_variant', 'LOW'),
    ('3_prime_UTR_truncation+exon_loss', 'MED'),
    ('5_prime_UTR_truncation+exon_loss_variant', 'MED'),
    ('disruptive_inframe_deletion', 'MED'),
    ('disruptive_inframe_insertion', 'MED')
])

# http://uswest.ensembl.org/info/genome/variation/predicted_data.html#consequences
IMPACT_SEVERITY_VEP = dict([
    ('transcript_ablation', 'HIGH'),
    ('splice_acceptor_variant', 'HIGH'),
    ('splice_donor_variant', 'HIGH'),
    ('stop_gained', 'HIGH'),
    ('frameshift_variant', 'HIGH'),
    ('stop_lost', 'HIGH'),
    ('transcript_amplification', 'HIGH'),

    ('inframe_insertion', 'MED'),
    ('inframe_deletion', 'MED'),
    ('missense_variant', 'MED'),
    ('protein_altering_variant', 'MED'),
    ('regulatory_region_ablation', 'MED'),

    ('splice_region_variant', 'LOW'),
    ('incomplete_terminal_codon_variant', 'LOW'),
    ('start_lost', 'LOW'),
    ('stop_retained_variant', 'LOW'),
    ('synonymous_variant', 'LOW'),
    ('coding_sequence_variant', 'LOW'),
    ('mature_miRNA_variant', 'LOW'),
    ('5_prime_UTR_variant', 'LOW'),
    ('3_prime_UTR_variant', 'LOW'),
    ('non_coding_transcript_exon_variant', 'LOW'),
    ('intron_variant', 'LOW'),
    ('NMD_transcript_variant', 'LOW'),
    ('non_coding_transcript_variant', 'LOW'),
    ('upstream_gene_variant', 'LOW'),
    ('downstream_gene_variant', 'LOW'),
    ('TFBS_ablation', 'MODERATE'),
    ('TFBS_amplification', 'LOW'),
    ('TF_binding_site_variant', 'LOW'),
    ('regulatory_region_amplification', 'LOW'),
    ('feature_elongation', 'LOW'),
    ('regulatory_region_variant', 'LOW'),
    ('feature_truncation', 'LOW'),
    ('intergenic_variant', 'LOW'),
])

# I decided these myself.
IMPACT_SEVERITY_CUSTOM = dict([
    ('sequence_feature', 'LOW'),
    ('transcript', 'LOW'),  # ? snpEff

    # occurs with 'exon_loss' in snpEff
    ('3_prime_UTR_truncation', 'MED'),
    ('3_prime_UTR_truncation+exon_loss', 'MED'),
    ('3_prime_UTR_truncation+exon_loss_variant', 'MED'),
    ('exon_loss', 'MED'),

    ('5_prime_UTR_truncation', 'MED'),
    ('5_prime_UTR_truncation+exon_loss_variant', 'MED'),
    ('non_canonical_start_codon', 'LOW'),
    ('initiator_codon_variant', 'LOW'),
])

IMPACT_SEVERITY.update(IMPACT_SEVERITY_SNPEFF)
IMPACT_SEVERITY.update(IMPACT_SEVERITY_CUSTOM)
IMPACT_SEVERITY.update(IMPACT_SEVERITY_VEP)

@total_ordering
class Effect(object):

    def __init__(self, effect_dict):
        # or maybe arg should be a dict for Effect()
        raise NotImplementedError

    def __le__(self, other):
        if self.pseudogene and not other.pseudogene:
            return True
        elif other.pseudogene and not self.pseudogene:
            return False
        if self.coding and not other.coding:
            return False
        elif other.coding and not self.coding:
            return True

        if self.severity != other.severity:
            return self.severity <= other.severity

        raise NotImplementedError
        # TODO: look at polyphen/sift?

    def __eq__(self, other):
        return self.effects == other.effects

    def __str__(self):
        # TODO: tab-delimited?
        raise NotImplementedError

    def __repr__(self):
        return "%s(%s-%s)" % (self.__class__.__name__, self.gene,
                self.consequence)

    @property
    def gene(self):
        raise NotImplementedError

    @property
    def transcript(self):
        raise NotImplementedError

    @property
    def exonic(self):
        raise NotImplementedError

    @property
    def coding(self):
        return True

    @property
    def lof(self):
        return True

    @property
    def aa_change(self):
        raise NotImplementedError

    @property
    def codon_change(self):
        raise NotImplementedError

    @property
    def severity(self, lookup={'HIGH': 3, 'MED': 2, 'LOW': 1}, sev=IMPACT_SEVERITY):
        # higher is more severe. used for ordering.
        return max(lookup[IMPACT_SEVERITY[csq]] for csq in self.consequences)

    @property
    def impact_severity(self):
        return ['xxx', 'LOW', 'MEDIUM', 'HIGH'][self.severity]

    @property
    def consequence(self):
        raise NotImplementedError

    @property
    def biotype(self):
        raise NotImplementedError

    @property
    def is_pseudogene(self): #bool
        raise NotImplementedError

class SnpEff(Effect):

    keys = [x.strip() for x in 'Allele | Annotation | Annotation_Impact | Gene_Name | Gene_ID | Feature_Type | Feature_ID | Transcript_BioType | Rank | HGVS.c | HGVS.p | cDNA.pos / cDNA.length | CDS.pos / CDS.length | AA.pos / AA.length | Distance | ERRORS / WARNINGS / INFO'.split("|")]

    def __init__(self, effect_string):
        assert not "," in effect_string
        assert not "=" in effect_string
        self.effect_string = effect_string
        self.effects = dict(it.izip(self.keys, (x.strip() for x in effect_string.split("|"))))

    @property
    def gene(self):
        return self.effects['Gene_Name'] or None

    @property
    def transcript(self):
        return self.effects['Feature_ID'] or None

    @property
    def exon(self):
        return self.effects['Rank']

    @property
    def consequence(self):
        if '&' in self.effects['Annotation']:
            return self.effects['Annotation'].split('&')
        return self.effects['Annotation']

    @property
    def consequences(self):
        return self.effects['Annotation'].split('&')

    @property
    def biotype(self):
        return self.effects['Transcript_BioType']

    @property
    def alt(self):
        return self.effects['Allele']

    @property
    def is_pseudogene(self):
        return 'pseudogene' in self.effects['Transcript_BioType']

    @property
    def coding(self):
        # TODO: check start_gained and utr
        return self.exonic and not "utr" in self.consequence and not "start_gained" in self.consequence

    @property
    def exonic(self):
        csqs = self.consequence
        if isinstance(csqs, basestring):
            csqs = [csqs]
        return any(csq in EXONIC_IMPACTS for csq in csqs) and self.effects['Transcript_BioType'] == 'protein_coding'

    # not defined in ANN field.
    aa_change = None
    sift = None
    polyphen = None

class VEP(Effect):
    keys = "Consequence|Codons|Amino_acids|Gene|SYMBOL|Feature|EXON|PolyPhen|SIFT|Protein_position|BIOTYPE".split("|")
    def __init__(self, effect_string, keys=None):
        assert not "," in effect_string
        assert not "=" in effect_string
        self.effect_string = effect_string
        if keys is not None: self.keys = keys

        self.effect_string = effect_string
        self.effects = dict(it.izip(self.keys, (x.strip() for x in effect_string.split("|"))))

    @property
    def gene(self):
        return self.effects['SYMBOL'] or self.effects['Gene']

    @property
    def transcript(self):
        return self.effects['Feature']

    @property
    def exon(self):
        return self.effects['EXON']

    @property
    def consequence(self):
        if '&' in self.effects['Consequence']:
            return self.effects['Consequence'].split('&')
        return self.effects['Consequence']

    @property
    def consequences(self):
        return self.effects['Consequence'].split('&')

    @property
    def biotype(self):
        return self.effects['BIOTYPE']

    @property
    def alt(self):
        return self.effects.get('ALLELE')

    @property
    def is_pseudogene(self):
        return self.effects['BIOTYPE'] == 'processed_pseudogene'

    @property
    def coding(self):
        # what about start/stop_gained?
        return self.exonic and self.effect_name[1:] != "_prime_UTR_variant"

    def exonic(self):
        return any(csq in EXONIC_IMPACTS for csq in self.consequences) and self.effects['BIOTYPE'] == 'protein_coding'

    @property
    def sift(self):
        return self.effects['SIFT']

    @property
    def polyphen(self):
        return self.effects['PolyPhen']

    @property
    def aa_change(self):
        return self.effects['Amino_acids']

if __name__ == "__main__":

    s = SnpEff("A|stop_gained|HIGH|C1orf170|ENSG00000187642|transcript|ENST00000433179|protein_coding|3/5|c.262C>T|p.Arg88*|262/3064|262/2091|88/696||")
    print s.effects
    print s.gene, s.transcript, s.consequence, s.is_pseudogene
    s = SnpEff("G|splice_donor_variant&intron_variant|HIGH|WASH7P|ENSG00000227232|transcript|ENST00000423562|unprocessed_pseudogene|6/9|n.822+2T>C||||||")
    print s.is_pseudogene
    s = SnpEff("G|missense_variant|MODERATE|OR4F5|ENSG00000186092|transcript|ENST00000335137|protein_coding|1/1|c.338T>G|p.Phe113Cys|338/918|338/918|113/305||")
    print s.coding, s.consequence, s.aa_change
<|endoftext|>


<commit_before>import time
from numbers import Number

def timed_func(func):
    """Decorator for timing our traversal methods."""
    def timed(*args, **kwargs):
        start = time.time()
        result = func(*args, **kwargs)
        elapsed = time.time() - start
        return (result, elapsed)
    return timed

@timed_func
def insertion(_list):
    '''Sorts a list via the insertion method.'''
    if type(_list) is not list:
        raise TypeError('Entire list must be numbers')
    for i in range(1, len(_list)):

        key = _list[i]
        if not isinstance(key, Number):
            raise TypeError('Entire list must be numbers')
        position = i

        while position > 0 and _list[position-1] > key:
            _list[position] = _list[position-1]
            position = position-1

        _list[position] = key

    return _list

if __name__ == '__main__':
    lengths = [10, 100, 1000, 10000]
    times = []
    for x in lengths:
        output = insertion(range(x))
        times.append(output[1])
    print 'Best case scenario:'
    for length, tim in zip(lengths, times):
        print 'a list of length {} was sorted in {}'.format(length, tim)
    diff = []
    for x in range(len(times)-2):
        diff.append(times[x+1]/times[x])
    average = reduce(lambda x, y: x+y, diff) / len(diff)
    print 'As length increases by 10, time increases by {}'.format(average)

    lengths = [10, 100, 1000, 10000]
    times = []
    for x in lengths:
        output = insertion(range(x)[::-1])
        times.append(output[1])
    print 'Worse case scenario:'
    for length, tim in zip(lengths, times):
        print 'a list of length {} was sorted in {}'.format(length, tim)
    diff = []
    for x in range(len(times)-2):
        diff.append(times[x+1]/times[x])
    average = reduce(lambda x, y: x+y, diff) / len(diff)
    print 'As length increases by 10, time increases by {}'.format(average)

<commit_msg>Use tuple assignment in insertion sort. Change variable names.<commit_after>import time
from numbers import Number

def timed_func(func):
    """Decorator for timing our traversal methods."""
    def timed(*args, **kwargs):
        start = time.time()
        result = func(*args, **kwargs)
        elapsed = time.time() - start
        return (result, elapsed)
    return timed

@timed_func
def insertion(in_list):
    '''Sorts a list via the insertion method.
    Raises TypeError if not list or composed of Numbers.'''
    if type(in_list) is not list:
        raise TypeError('Entire list must be numbers')
    for i in range(1, len(in_list)):
        key = in_list[i]
        if not isinstance(key, Number):
            raise TypeError('Entire list must be numbers')
        position = i
        while position > 0 and in_list[position-1] > key:
            in_list[position], in_list[position-1] = in_list[position-1], in_list[position]
            position -= 1
    return in_list

if __name__ == '__main__':
    lengths = [10, 100, 1000, 10000]
    times = []
    for x in lengths:
        output = insertion(range(x))
        times.append(output[1])
    print 'Best case scenario:'
    for length, tim in zip(lengths, times):
        print 'a list of length {} was sorted in {}'.format(length, tim)
    diff = []
    for x in range(len(times)-2):
        diff.append(times[x+1]/times[x])
    average = reduce(lambda x, y: x+y, diff) / len(diff)
    print 'As length increases by 10, time increases by {}'.format(average)

    lengths = [10, 100, 1000, 10000]
    times = []
    for x in lengths:
        output = insertion(range(x)[::-1])
        times.append(output[1])
    print 'Worse case scenario:'
    for length, tim in zip(lengths, times):
        print 'a list of length {} was sorted in {}'.format(length, tim)
    diff = []
    for x in range(len(times)-2):
        diff.append(times[x+1]/times[x])
    average = reduce(lambda x, y: x+y, diff) / len(diff)
    print 'As length increases by 10, time increases by {}'.format(average)

<|endoftext|>


<commit_before>"""
==========================
PySpecKit HDF5 Reader
==========================

Routines for reading in spectra from HDF5 files.

Note: Current no routines for parsing HDF5 headers in classes.py.

.. moduleauthor:: Adam Ginsburg <adam.g.ginsburg@gmail.com>
.. moduleauthor:: Jordan Mirocha <mirochaj@gmail.com>
"""
try:
    import h5py
    h5OK = True
except ImportError:
    h5OK = False

import numpy as np
from .. import units

try:
    import astropy.io.fits as pyfits
except ImportError:
    import pyfits

def open_hdf5(filename, xaxkey = 'xarr', datakey = 'data', errkey = 'error'):
    """
    This reader expects three datasets to exist in the hdf5 file 'filename':
    'xarr', 'data', and 'error', by default.  Can specify other dataset names.
    """

    if not h5OK:
        print("WARNING: h5py not installed; cannot read hdf5 files.")
    else:
        f = h5py.File(filename, 'r')

        try: xarr = f[xaxkey].value
        except KeyError: print('Dataset \'%s\' not found.' % xaxkey)

        try: data = f[datakey].value
        except KeyError: print('Dataset \'%s\' not found.' % datakey)

        try: error = f[errkey].value
        except KeyError:
            print('Dataset \'%s\' not found.  Assuming uniform errors.' % errkey)
            error = np.ones_like(data)

        try:
            xunits = f[xaxkey].attrs['units']
            if xunits == '':
                xunits = 'unknown'
        except KeyError:
            xunits = 'unknown'

        try:
            yunits = f[datakey].attrs['units']
            if yunits == '':
                yunits = 'unknown'
        except KeyError:
            yunits = 'unknown'

        try:
            xtype = f[xaxkey].attrs['type']
            if xtype == '':
                xtype = xaxkey
        except KeyError:
            xtype = xaxkey

        try:
            ytype = f[datakey].attrs['type']
            if ytype == '':
                ytype = datakey
        except KeyError:
            ytype = datakey

        XAxis = units.SpectroscopicAxis(xarr, xunits)

        header = {'xunits': xunits, 'xtype': xtype, 'yunits': yunits, 'ytype': ytype}

        return data, error, XAxis, header

<commit_msg>import hdf5 more lazily to avoid import-time issues<commit_after>"""
==========================
PySpecKit HDF5 Reader
==========================

Routines for reading in spectra from HDF5 files.

Note: Current no routines for parsing HDF5 headers in classes.py.

.. moduleauthor:: Adam Ginsburg <adam.g.ginsburg@gmail.com>
.. moduleauthor:: Jordan Mirocha <mirochaj@gmail.com>
"""
import numpy as np
from .. import units

try:
    import astropy.io.fits as pyfits
except ImportError:
    import pyfits

def open_hdf5(filename, xaxkey = 'xarr', datakey = 'data', errkey = 'error'):
    """
    This reader expects three datasets to exist in the hdf5 file 'filename':
    'xarr', 'data', and 'error', by default.  Can specify other dataset names.
    """
    import h5py

    f = h5py.File(filename, 'r')

    try: xarr = f[xaxkey].value
    except KeyError: print('Dataset \'%s\' not found.' % xaxkey)

    try: data = f[datakey].value
    except KeyError: print('Dataset \'%s\' not found.' % datakey)

    try: error = f[errkey].value
    except KeyError:
        print('Dataset \'%s\' not found.  Assuming uniform errors.' % errkey)
        error = np.ones_like(data)

    try:
        xunits = f[xaxkey].attrs['units']
        if xunits == '':
            xunits = 'unknown'
    except KeyError:
        xunits = 'unknown'

    try:
        yunits = f[datakey].attrs['units']
        if yunits == '':
            yunits = 'unknown'
    except KeyError:
        yunits = 'unknown'

    try:
        xtype = f[xaxkey].attrs['type']
        if xtype == '':
            xtype = xaxkey
    except KeyError:
        xtype = xaxkey

    try:
        ytype = f[datakey].attrs['type']
        if ytype == '':
            ytype = datakey
    except KeyError:
        ytype = datakey

    XAxis = units.SpectroscopicAxis(xarr, xunits)

    header = {'xunits': xunits, 'xtype': xtype, 'yunits': yunits, 'ytype': ytype}

    return data, error, XAxis, header

<|endoftext|>