<commit_before>import os

import tensorflow as tf

class ValueFunctionDQN:
    def __init__(self, scope="MyValueFunctionEstimator", state_dim=2, n_actions=3, train_batch_size=64,
                 learning_rate=1e-4, hidden_layers_size=None, decay_lr=False, huber_loss=False, summaries_path=None,
                 reset_default_graph=False, checkpoints_dir=None):
        # Input check
        if hidden_layers_size is None:
            hidden_layers_size = [128, 64]  # Default ANN architecture
        assert len(hidden_layers_size) >= 1, "At least one hidden layer must be specified."

        # Support variables
        self.scope = scope
        self.layers_size = [state_dim] + hidden_layers_size + [n_actions]  # Size of all layers (including in & out)
        self.weights = []
        self.biases = []
        self.weights_old = []
        self.biases_old = []
        self.learning_rate = learning_rate
        self.train_batch_size = train_batch_size
        self.n_train_epochs = 0
        self.summaries_path = summaries_path
        self.train_writer = None
        self.checkpoints_dir = checkpoints_dir

        if reset_default_graph:
            tf.reset_default_graph()

        # Build Tensorflow graph
        with tf.variable_scope(self.scope):
            # Inputs, weights, biases and targets of the ANN
            self.x = tf.placeholder(tf.float32, shape=(None, state_dim), name="x")
            self.train_targets = tf.placeholder(tf.float32, shape=(None, n_actions), name="train_targets")

            for l in range(len(self.layers_size) - 1):
                self.weights.append(tf.get_variable(name="w" + str(l), shape=[self.layers_size[l],
                                                                              self.layers_size[l + 1]],
                                                    initializer=tf.contrib.layers.xavier_initializer()))
                self.biases.append(tf.get_variable(name="b" + str(l), shape=[self.layers_size[l + 1]],
                                                   initializer=tf.constant_initializer(0.0)))

                self.weights_old.append(tf.get_variable(name="w-" + str(l), shape=[self.layers_size[l],
                                                                                self.layers_size[l + 1]],
                                                        initializer=tf.contrib.layers.xavier_initializer()))
                self.biases_old.append(tf.get_variable(name="b-" + str(l), shape=[self.layers_size[l + 1]],
                                                       initializer=tf.constant_initializer(0.0)))

            if summaries_path is not None:
                with tf.name_scope('params_summaries'):
                    for l in range(len(self.layers_size) - 1):
                        self.variable_summaries(self.weights[l], "w" + str(l), histogram=True)
                        self.variable_summaries(self.biases[l], "b" + str(l), histogram=True)

            # Interconnection of the various ANN nodes
            self.prediction = self.model(self.x)
            self.prediction_with_old_params = self.model(self.x, use_old_params=True)

            # Training calculations
            if huber_loss:
                self.loss = self.huber_loss(self.train_targets, self.prediction)
            else:
                self.SE = tf.squared_difference(self.train_targets, self.prediction, name="SquaredError")
                self.loss = tf.reduce_mean(self.SE, name="loss")

            self.global_step = tf.Variable(0, trainable=False)
            if decay_lr:
                self.learning_rate = tf.train.exponential_decay(1e-4, self.global_step, 3000 * 200, 1e-5 / 1e-4)
            self.opt_op = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
            self.train_op = self.opt_op.minimize(self.loss, global_step=self.global_step)

            self.init_op = tf.global_variables_initializer()

            if self.summaries_path is not None:
                self.variable_summaries(self.loss, "loss", scalar_only=True)
                self.variable_summaries(self.learning_rate, "learning_rate", scalar_only=True)

        if self.checkpoints_dir is not None:
            var_list = []
            for l in range(len(self.layers_size) - 1):
                var_list.append(self.weights[l])
                var_list.append(self.biases[l])
            self.saver = tf.train.Saver(var_list, pad_step_number=True)

        if self.summaries_path is not None:
            self.merged_summaries = tf.summary.merge_all()
            self.summaries_path += "_{}".format(self.scope)
            if not os.path.exists(self.summaries_path):
                os.makedirs(self.summaries_path)
            self.train_writer = tf.summary.FileWriter(self.summaries_path, graph=tf.get_default_graph())
        else:
            self.merged_summaries = None

        self.session = None

    def model(self, x, use_old_params=False):
        z = []
        hidden = [x]
        for l in range(len(self.layers_size)-2):
            if use_old_params:
                z.append(tf.matmul(hidden[l], self.weights_old[l]) + self.biases_old[l])
            else:
                z.append(tf.matmul(hidden[l], self.weights[l]) + self.biases[l])
            hidden.append(tf.nn.relu(z[l], name="hidden_" + str(l + 1)))
        if use_old_params:
            z.append(tf.matmul(hidden[-1], self.weights_old[-1]) + self.biases_old[-1])
        else:
            z.append(tf.matmul(hidden[-1], self.weights[-1]) + self.biases[-1])

        if not use_old_params:
            if self.summaries_path is not None:
                with tf.name_scope('layers_summaries'):
                    for l in range(len(self.layers_size) - 1):
                        self.variable_summaries(z[l], "z" + str(l))
                        self.variable_summaries(hidden[l], "hidden" + str(l))

        return z[-1]  # Output layer has Identity units.

    @staticmethod
    def huber_loss(targets, predictions):
        error = targets - predictions
        fn_choice_maker1 = (tf.to_int32(tf.sign(error + 1)) + 1) / 2
        fn_choice_maker2 = (tf.to_int32(tf.sign(-error + 1)) + 1) / 2
        choice_maker_sqr = tf.to_float(tf.multiply(fn_choice_maker1, fn_choice_maker2))
        sqr_contrib = tf.multiply(choice_maker_sqr, tf.square(error)*0.5)
        abs_contrib = tf.abs(error)-0.5 - tf.multiply(choice_maker_sqr, tf.abs(error)-0.5)
        loss = tf.reduce_mean(sqr_contrib + abs_contrib)
        return loss

    def init_tf_session(self):
        if self.session is None:
            self.session = tf.Session()
            self.session.run(self.init_op)  # Global Variables Initializer (init op)

    def predict(self, states, use_old_params=False):
        self.init_tf_session()  # Make sure the Tensorflow session exists

        feed_dict = {self.x: states}
        if use_old_params:
            q = self.session.run(self.prediction_with_old_params, feed_dict=feed_dict)
        else:
            q = self.session.run(self.prediction, feed_dict=feed_dict)

        return q

    def train(self, states, targets):
        self.init_tf_session()  # Make sure the Tensorflow session exists

        feed_dict = {self.x: states, self.train_targets: targets}
        if self.summaries_path is not None and self.n_train_epochs % 2000 == 0:
            fetches = [self.loss, self.train_op, self.merged_summaries]
        else:
            fetches = [self.loss, self.train_op]

        values = self.session.run(fetches, feed_dict=feed_dict)

        if self.summaries_path is not None and self.n_train_epochs % 2000 == 0:
            self.train_writer.add_summary(values[2], global_step=self.n_train_epochs)

        if self.checkpoints_dir is not None and self.n_train_epochs % 40000 == 0:
            self.saver.save(self.session, self.checkpoints_dir, global_step=self.global_step)

        self.n_train_epochs += 1
        return values[0]

    @staticmethod
    def variable_summaries(var, name, histogram=False, scalar_only=False):
        """Attach a lot of summaries to a Tensor (for TensorBoard visualization)."""
        if scalar_only:
            tf.summary.scalar(name, var)
        else:
            mean = tf.reduce_mean(var)
            tf.summary.scalar(name+'_mean', mean)
            with tf.name_scope('stddev'):
                stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
            tf.summary.scalar(name+'_stddev', stddev)
            tf.summary.scalar(name+'_max', tf.reduce_max(var))
            tf.summary.scalar(name+'_min', tf.reduce_min(var))
            if histogram:
                tf.summary.histogram(name+'_histogram', var)

    def update_old_params(self):
        self.init_tf_session()  # Make sure the Tensorflow session exists

        update_ops = []
        for l in range(len(self.layers_size) - 1):
            update_ops.append(self.weights_old[l].assign(self.weights[l]))
            update_ops.append(self.biases_old[l].assign(self.biases[l]))

        self.session.run(update_ops)

    def close_summary_file(self):
        self.train_writer.close()
<commit_msg>Delete value_functions.py<commit_after><|endoftext|>


<commit_before>import cgi

from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext import db

class Hooker(db.Model):
  phone = db.IntegerProperty()
  name = db.StringProperty()
  age = db.IntegerProperty()
  gender = db.StringProperty()
  longitude = db.FloatProperty()
  latitude = db.FloatProperty()
  date = db.DateTimeProperty(auto_now_add=True)

class Pimp(webapp.RequestHandler):
  def post(self):
    hooker = Hooker()
    hooker.phone = int(self.request.get('phone'))
    hooker.name = self.request.get('name')
    hooker.age = int(self.request.get('age'))
    hooker.gender = self.request.get('gender')
    hooker.longitude = float(self.request.get('longitude'))
    hooker.latitude = float(self.request.get('latitude'))
    hooker.put()
    self.redirect('/')

class MainPage(webapp.RequestHandler):
  def get(self):
    self.response.out.write('')
    self.response.out.write('
Hookers
')
    hookers = Hooker.gql("ORDER BY date DESC LIMIT 10")
    for hooker in hookers:
      self.response.out.write(hooker.name)
      self.response.out.write("/")
      self.response.out.write(hooker.age)
      self.response.out.write("/")
      self.response.out.write(hooker.gender)
      self.response.out.write("
")
    self.response.out.write("""
          <form action="/hereiam" method="post">

<input name="phone" value="9548168827" />

<input name="name" value="Ted" />

<input name="age" value="36" />

<input name="gender" value="m" />

<input name="longitude" value="-80.4038" />

<input name="latitude" value="26.1353"/>

<input type="submit" value="Hook Me Up">

          </form>""")
    self.response.out.write('')

application = webapp.WSGIApplication(
                                     [('/', MainPage),
                                      ('/hereiam', Pimp)],
                                     debug=True)

def main():
  run_wsgi_app(application)

if __name__ == "__main__":
  main()<commit_msg>moved listing into submission form where it belongs and filter out self<commit_after>import cgi

from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext import db

class Hooker(db.Model):
  phone = db.IntegerProperty()
  name = db.StringProperty()
  age = db.IntegerProperty()
  gender = db.StringProperty()
  longitude = db.FloatProperty()
  latitude = db.FloatProperty()
  date = db.DateTimeProperty(auto_now_add=True)

class Pimp(webapp.RequestHandler):
  def post(self):
    freshmeat = Hooker()
    freshmeat.phone = int(self.request.get('phone'))
    freshmeat.name = self.request.get('name')
    freshmeat.age = int(self.request.get('age'))
    freshmeat.gender = self.request.get('gender')
    freshmeat.longitude = float(self.request.get('longitude'))
    freshmeat.latitude = float(self.request.get('latitude'))
    freshmeat.put()
    hookers = Hooker.gql("ORDER BY date DESC LIMIT 10")
    for hooker in hookers:
      if freshmeat.phone != hooker.phone:
        self.response.out.write(hooker.name)
        self.response.out.write("/")
        self.response.out.write(hooker.age)
        self.response.out.write("/")
        self.response.out.write(hooker.gender)
        self.response.out.write("
")

class MainPage(webapp.RequestHandler):
  def get(self):
    self.response.out.write("""

          <form action="/hereiam" method="post">

<input name="phone" value="9548168827" />

<input name="name" value="Ted" />

<input name="age" value="36" />

<input name="gender" value="m" />

<input name="longitude" value="-80.4038" />

<input name="latitude" value="26.1353"/>

<input type="submit" value="Hook Me Up">

          </form>

          """)

application = webapp.WSGIApplication(
                                     [('/', MainPage),
                                     ('/hereiam', Pimp)],
                                     debug=True)

def main():
  run_wsgi_app(application)

if __name__ == "__main__":
  main()<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
from sqlalchemy import Column, ForeignKey, Integer, String, LargeBinary
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine

Base = declarative_base()

class Site(Base):
  __tablename__ = 'site'
  id = Column(Integer, primary_key=True)
  name = Column(String(250))
  hostname = Column(String(250))
  mangas = relationship(
        'Manga',
        secondary='site_manga_link'
  )

class Manga(Base):
  __tablename__ = 'manga'
  id = Column(Integer, primary_key=True)
  name = Column(String(250))
  sites = relationship(
        Site,
        secondary='site_manga_link'
  )

# because the same manga can be on different sites
class SiteMangaLink(Base):
    __tablename__ = 'site_manga_link'
    site_id = Column(Integer, ForeignKey('site.id'), primary_key=True)
    manga_id = Column(Integer, ForeignKey('manga.id'), primary_key=True)

class Chapter(Base):
  __tablename__ = 'chapter'
__table_args__ = (
      UniqueConstraint('site_id', 'manga_id', 'num'),
  )
  id = Column(Integer, primary_key=True)
  num = Column(Integer) # chapter number in the serie
  site_id = Column(Integer, ForeignKey('site.id'))
  manga_id = Column(Integer, ForeignKey('manga.id'))
  url = Column(String(2048), nullable=False)
  name = Column(String(250), nullable=False)
  done = Column(Boolean()) # if we download the chapter or not

  manga = relationship(Manga)
  site = relationship(site)

class Image(Base):
  __tablename__ = 'image'
  id = Column(Integer, primary_key=True)
  url = Column(String(2048), nullable=False)
  img = Column(LargeBinary())
  num = Column(Integer, nullable=False) # page number in the chapter
  chapter_id = Column(Integer, ForeignKey('chapter.id'))

  chapter = relationship(Chapter)

# Create an engine that stores data in the local directory's
# sqlalchemy_example.db file.
engine = create_engine('sqlite:///store.db')

# Create all tables in the engine. This is equivalent to "Create Table"
# statements in raw SQL.
Base.metadata.create_all(engine)
<commit_msg>Improve storage module<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
from contextlib import contextmanager
from sqlalchemy import Column, ForeignKey, Integer, String, LargeBinary, UniqueConstraint, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

Base = declarative_base()

__DB = None
__SESSION_MAKER = None

def create_db(file_name='mdg.store', force=False):
  global __DB
  if __DB is not None and force is False:
    # TODO: create some real exceptions
    raise Exception('Engine already exist !')
  db = __DB = create_engine('sqlite:///' + file_name)
  # Create all tables in the engine. This is equivalent to "Create Table"
  # statements in raw SQL.
  Base.metadata.create_all(db)
  return db

def get_db():
  return __DB

def get_session():
  global __SESSION_MAKER
  db = get_db()
  if __SESSION_MAKER is None:
    sm =__SESSION_MAKER = sessionmaker()
    sm.configure(bind=db)

  return __SESSION_MAKER()

def remove_db():
  global __DB, __SESSION_MAKER
  __DB = None
  __SESSION_MAKER = None

@contextmanager
def session_scope():
    """Provide a transactional scope around a series of operations."""
    session = get_session()
    try:
        yield session
        session.commit()
    except:
        session.rollback()
        raise
    finally:
        session.close()

def run_my_program():
    with session_scope() as session:
        ThingOne().go(session)
        ThingTwo().go(session)

################################################################################
# Model
################################################################################
class Site(Base):
  __tablename__ = 'site'
  id = Column(Integer, primary_key=True)
  name = Column(String(250))
  hostname = Column(String(250))

  mangas = relationship(
        'Manga',
        secondary='site_manga_link'
  )

  @classmethod
  def find_with_hostname(cls, hn):
    with session_scope() as s:
      result = s.query(Site).filter(Site.hostname==hn).all()
      result_l = len(result)
      if result_l > 1:
        # too much result
        return None
      elif result_l == 1:
        # ok return
        return result[0]
      else:
        # no result
        return None

class Manga(Base):
  __tablename__ = 'manga'
  id = Column(Integer, primary_key=True)
  name = Column(String(250))
  sites = relationship(
        Site,
        secondary='site_manga_link'
  )

# because the same manga can be on different sites
class SiteMangaLink(Base):
    __tablename__ = 'site_manga_link'
    site_id = Column(Integer, ForeignKey('site.id'), primary_key=True)
    manga_id = Column(Integer, ForeignKey('manga.id'), primary_key=True)

class Chapter(Base):
  __tablename__ = 'chapter'
  __table_args__ = (
      UniqueConstraint('site_id', 'manga_id', 'num'),
  )
  id = Column(Integer, primary_key=True)
  num = Column(Integer) # chapter number in the serie
  site_id = Column(Integer, ForeignKey('site.id'))
  manga_id = Column(Integer, ForeignKey('manga.id'))
  url = Column(String(2048), nullable=False)
  name = Column(String(250), nullable=False)
  done = Column(Boolean()) # if we download the chapter or not

  manga = relationship(Manga)
  site = relationship(Site)
  images = relationship('Image')

class Image(Base):
  __tablename__ = 'image'
  id = Column(Integer, primary_key=True)
  url = Column(String(2048), nullable=False)
  img = Column(LargeBinary())
  num = Column(Integer, nullable=False) # page number in the chapter
  chapter_id = Column(Integer, ForeignKey('chapter.id'))
  type_mime = Column(String(50), nullable=False)

  chapter = relationship(Chapter)
<|endoftext|>


<commit_before>#!/usr/bin/env python

#    Copyright (C) 2001  Jeff Epler  <jepler@unpythonic.dhs.org>
#
#    This program can be distributed under the terms of the GNU LGPL.
#    See the file COPYING.
#

import os, sys
from errno import *
from stat import *
try:
    import fuse
    from fuse import Fuse
except ImportError:
    print >> sys.stderr, """
! If you are trying the Python example filesystem from
! the fuse-python source directory, without installation,
! you are suggested to link or copy build/lib.*/_fusemodule.so
! to the root of the source tree.
"""
    raise

if not hasattr(fuse, '__version__'):
    raise RuntimeError, \
        "your fuse-py doesn't know of fuse.__version__, probably it's too old."

import thread
class Xmp(Fuse):

    root = '/'

    def __init__(self, *args, **kw):

        Fuse.__init__(self, *args, **kw)

        if 0:
            print "xmp.py:Xmp:mountpoint: %s" % repr(self.mountpoint)
            print "xmp.py:Xmp:unnamed mount options: %s" % self.optlist
            print "xmp.py:Xmp:named mount options: %s" % self.optdict

        # do stuff to set up your filesystem here, if you want
        #thread.start_new_thread(self.mythread, ())
        pass

    def mythread(self):

        """
        The beauty of the FUSE python implementation is that with the python interp
        running in foreground, you can have threads
        """
        print "mythread: started"
        #while 1:
        #    time.sleep(120)
        #    print "mythread: ticking"

    def getattr(self, path):
        return os.lstat(self.root + path)

    def readlink(self, path):
        return os.readlink(self.root + path)

    def getdir(self, path):
        return map(lambda x: (x,0), os.listdir(self.root + path))

    def unlink(self, path):
        return os.unlink(self.root + path)

    def rmdir(self, path):
        return os.rmdir(self.root + path)

    def symlink(self, path, path1):
        return os.symlink(path, self.root + path1)

    def rename(self, path, path1):
        return os.rename(self.root + path, self.root + path1)

    def link(self, path, path1):
        return os.link(self.root + path, self.root + path1)

    def chmod(self, path, mode):
        return os.chmod(self.root + path, mode)

    def chown(self, path, user, group):
        return os.chown(self.root + path, user, group)

    def truncate(self, path, size):
        f = open(self.root + path, "w+")
        return f.truncate(size)

    def mknod(self, path, mode, dev):
        """ Python has no os.mknod, so we can only do some things """
        if S_ISREG(mode):
            open(self.root + path, "w")
        else:
            return -EINVAL

    def mkdir(self, path, mode):
        return os.mkdir(self.root + path, mode)

    def utime(self, path, times):
        return os.utime(self.root + path, times)

    def open(self, path, flags):
        #print "xmp.py:Xmp:open: %s" % path
        os.close(os.open(self.root + path, flags))
        return 0

    def read(self, path, length, offset):
        #print "xmp.py:Xmp:read: %s" % path
        f = open(self.root + path, "r")
        f.seek(offset)
        return f.read(length)

    def write(self, path, buf, off):
        #print "xmp.py:Xmp:write: %s" % path
        f = open(self.root + path, "r+")
        f.seek(off)
        f.write(buf)
        return len(buf)

    def release(self, path, flags):
        print "xmp.py:Xmp:release: %s %s" % (path, flags)
        return 0

    def statfs(self):
        """
        Should return a tuple with the following 6 elements:
            - blocksize - size of file blocks, in bytes
            - totalblocks - total number of blocks in the filesystem
            - freeblocks - number of free blocks
            - totalfiles - total number of file inodes
            - freefiles - nunber of free file inodes

        Feel free to set any of the above values to 0, which tells
        the kernel that the info is not available.
        """
        print "xmp.py:Xmp:statfs: returning fictitious values"
        blocks_size = 1024
        blocks = 100000
        blocks_free = 25000
        files = 100000
        files_free = 60000
        namelen = 80
        return (blocks_size, blocks, blocks - blocks_free, blocks_free, files, files_free, namelen)

    def fsync(self, path, isfsyncfile):
        print "xmp.py:Xmp:fsync: path=%s, isfsyncfile=%s" % (self.root + path, isfsyncfile)
        return 0

if __name__ == '__main__':

    usage="""
Userspace nullfs-alike: mirror the filesystem tree from some point on.

""" + Fuse.fusage

    server = Xmp(version="%prog " + fuse.__version__,
                 usage=usage),
                 dash_s_do='setsingle')

    server.parser.add_option(mountopt="root", metavar="PATH", default='/', type=str,
                             help="mirror filesystem from under PATH [default: %default]")
    server.parse(values=server, errex=1)

    try:
        os.stat(server.root)
    except OSError:
        print >> sys.stderr, "can't stat root of underlying filesystem"
        sys.exit(1)

    server.main()
<commit_msg>fix typo in xmp.py<commit_after>#!/usr/bin/env python

#    Copyright (C) 2001  Jeff Epler  <jepler@unpythonic.dhs.org>
#
#    This program can be distributed under the terms of the GNU LGPL.
#    See the file COPYING.
#

import os, sys
from errno import *
from stat import *
try:
    import fuse
    from fuse import Fuse
except ImportError:
    print >> sys.stderr, """
! If you are trying the Python example filesystem from
! the fuse-python source directory, without installation,
! you are suggested to link or copy build/lib.*/_fusemodule.so
! to the root of the source tree.
"""
    raise

if not hasattr(fuse, '__version__'):
    raise RuntimeError, \
        "your fuse-py doesn't know of fuse.__version__, probably it's too old."

import thread
class Xmp(Fuse):

    root = '/'

    def __init__(self, *args, **kw):

        Fuse.__init__(self, *args, **kw)

        if 0:
            print "xmp.py:Xmp:mountpoint: %s" % repr(self.mountpoint)
            print "xmp.py:Xmp:unnamed mount options: %s" % self.optlist
            print "xmp.py:Xmp:named mount options: %s" % self.optdict

        # do stuff to set up your filesystem here, if you want
        #thread.start_new_thread(self.mythread, ())
        pass

    def mythread(self):

        """
        The beauty of the FUSE python implementation is that with the python interp
        running in foreground, you can have threads
        """
        print "mythread: started"
        #while 1:
        #    time.sleep(120)
        #    print "mythread: ticking"

    def getattr(self, path):
        return os.lstat(self.root + path)

    def readlink(self, path):
        return os.readlink(self.root + path)

    def getdir(self, path):
        return map(lambda x: (x,0), os.listdir(self.root + path))

    def unlink(self, path):
        return os.unlink(self.root + path)

    def rmdir(self, path):
        return os.rmdir(self.root + path)

    def symlink(self, path, path1):
        return os.symlink(path, self.root + path1)

    def rename(self, path, path1):
        return os.rename(self.root + path, self.root + path1)

    def link(self, path, path1):
        return os.link(self.root + path, self.root + path1)

    def chmod(self, path, mode):
        return os.chmod(self.root + path, mode)

    def chown(self, path, user, group):
        return os.chown(self.root + path, user, group)

    def truncate(self, path, size):
        f = open(self.root + path, "w+")
        return f.truncate(size)

    def mknod(self, path, mode, dev):
        """ Python has no os.mknod, so we can only do some things """
        if S_ISREG(mode):
            open(self.root + path, "w")
        else:
            return -EINVAL

    def mkdir(self, path, mode):
        return os.mkdir(self.root + path, mode)

    def utime(self, path, times):
        return os.utime(self.root + path, times)

    def open(self, path, flags):
        #print "xmp.py:Xmp:open: %s" % path
        os.close(os.open(self.root + path, flags))
        return 0

    def read(self, path, length, offset):
        #print "xmp.py:Xmp:read: %s" % path
        f = open(self.root + path, "r")
        f.seek(offset)
        return f.read(length)

    def write(self, path, buf, off):
        #print "xmp.py:Xmp:write: %s" % path
        f = open(self.root + path, "r+")
        f.seek(off)
        f.write(buf)
        return len(buf)

    def release(self, path, flags):
        print "xmp.py:Xmp:release: %s %s" % (path, flags)
        return 0

    def statfs(self):
        """
        Should return a tuple with the following 6 elements:
            - blocksize - size of file blocks, in bytes
            - totalblocks - total number of blocks in the filesystem
            - freeblocks - number of free blocks
            - totalfiles - total number of file inodes
            - freefiles - nunber of free file inodes

        Feel free to set any of the above values to 0, which tells
        the kernel that the info is not available.
        """
        print "xmp.py:Xmp:statfs: returning fictitious values"
        blocks_size = 1024
        blocks = 100000
        blocks_free = 25000
        files = 100000
        files_free = 60000
        namelen = 80
        return (blocks_size, blocks, blocks - blocks_free, blocks_free, files, files_free, namelen)

    def fsync(self, path, isfsyncfile):
        print "xmp.py:Xmp:fsync: path=%s, isfsyncfile=%s" % (self.root + path, isfsyncfile)
        return 0

if __name__ == '__main__':

    usage="""
Userspace nullfs-alike: mirror the filesystem tree from some point on.

""" + Fuse.fusage

    server = Xmp(version="%prog " + fuse.__version__,
                 usage=usage,
                 dash_s_do='setsingle')

    server.parser.add_option(mountopt="root", metavar="PATH", default='/', type=str,
                             help="mirror filesystem from under PATH [default: %default]")
    server.parse(values=server, errex=1)

    try:
        os.stat(server.root)
    except OSError:
        print >> sys.stderr, "can't stat root of underlying filesystem"
        sys.exit(1)

    server.main()
<|endoftext|>


<commit_before># Module for reading and parsing Scintilla.iface file

def sanitiseLine(line):
	if line[-1:] == '\n': line = line[:-1]
	if line.find("##") != -1:
		line = line[:line.find("##")]
	line = line.strip()
	return line

def decodeFunction(featureVal):
	retType, rest = featureVal.split(" ", 1)
	nameIdent, params = rest.split("(")
	name, value = nameIdent.split("=")
	params, rest = params.split(")")
	param1, param2 = params.split(",")[0:2]
	return retType, name, value, param1, param2

def decodeEvent(featureVal):
	retType, rest = featureVal.split(" ", 1)
	nameIdent, params = rest.split("(")
	name, value = nameIdent.split("=")
	return retType, name, value

def decodeParam(p):
	param = p.strip()
	type = ""
	name = ""
	value = ""
	if " " in param:
		type, nv = param.split(" ")
		if "=" in nv:
			name, value = nv.split("=")
		else:
			name = nv
	return type, name, value

class Face:

	def __init__(self):
		self.order = []
		self.features = {}
		self.values = {}
		self.events = {}

	def ReadFromFile(self, name):
		currentCategory = ""
		currentComment = []
		currentCommentFinished = 0
		file = open(name)
		for line in file.readlines():
			line = sanitiseLine(line)
			if line:
				if line[0] == "#":
					if line[1] == " ":
						if currentCommentFinished:
							currentComment = []
							currentCommentFinished = 0
						currentComment.append(line[2:])
				else:
					currentCommentFinished = 1
					featureType, featureVal = line.split(" ", 1)
					if featureType in ["fun", "get", "set"]:
						retType, name, value, param1, param2 = decodeFunction(featureVal)
						p1 = decodeParam(param1)
						p2 = decodeParam(param2)
						self.features[name] = {
							"FeatureType": featureType,
							"ReturnType": retType,
							"Value": value,
							"Param1Type": p1[0], "Param1Name": p1[1], "Param1Value": p1[2],
							"Param2Type": p2[0],	"Param2Name": p2[1], "Param2Value": p2[2],
							"Category": currentCategory, "Comment": currentComment
						}
						if value in self.values:
							raise "Duplicate value " + value + " " + name
						self.values[value] = 1
						self.order.append(name)
					elif featureType == "evt":
						retType, name, value = decodeEvent(featureVal)
						self.features[name] = {
							"FeatureType": featureType,
							"ReturnType": retType,
							"Value": value,
							"Category": currentCategory, "Comment": currentComment
						}
						if value in self.events:
							raise "Duplicate event " + value + " " + name
						self.events[value] = 1
						self.order.append(name)
					elif featureType == "cat":
						currentCategory = featureVal
					elif featureType == "val":
						name, value = featureVal.split("=", 1)
						self.features[name] = {
							"FeatureType": featureType,
							"Category": currentCategory,
							"Value": value }
						self.order.append(name)
					elif featureType == "enu" or featureType == "lex":
						name, value = featureVal.split("=", 1)
						self.features[name] = {
							"FeatureType": featureType,
							"Category": currentCategory,
							"Value": value }
						self.order.append(name)

<commit_msg>Show error if no '=' in definition.<commit_after># Module for reading and parsing Scintilla.iface file

def sanitiseLine(line):
	if line[-1:] == '\n': line = line[:-1]
	if line.find("##") != -1:
		line = line[:line.find("##")]
	line = line.strip()
	return line

def decodeFunction(featureVal):
	retType, rest = featureVal.split(" ", 1)
	nameIdent, params = rest.split("(")
	name, value = nameIdent.split("=")
	params, rest = params.split(")")
	param1, param2 = params.split(",")[0:2]
	return retType, name, value, param1, param2

def decodeEvent(featureVal):
	retType, rest = featureVal.split(" ", 1)
	nameIdent, params = rest.split("(")
	name, value = nameIdent.split("=")
	return retType, name, value

def decodeParam(p):
	param = p.strip()
	type = ""
	name = ""
	value = ""
	if " " in param:
		type, nv = param.split(" ")
		if "=" in nv:
			name, value = nv.split("=")
		else:
			name = nv
	return type, name, value

class Face:

	def __init__(self):
		self.order = []
		self.features = {}
		self.values = {}
		self.events = {}

	def ReadFromFile(self, name):
		currentCategory = ""
		currentComment = []
		currentCommentFinished = 0
		file = open(name)
		for line in file.readlines():
			line = sanitiseLine(line)
			if line:
				if line[0] == "#":
					if line[1] == " ":
						if currentCommentFinished:
							currentComment = []
							currentCommentFinished = 0
						currentComment.append(line[2:])
				else:
					currentCommentFinished = 1
					featureType, featureVal = line.split(" ", 1)
					if featureType in ["fun", "get", "set"]:
						retType, name, value, param1, param2 = decodeFunction(featureVal)
						p1 = decodeParam(param1)
						p2 = decodeParam(param2)
						self.features[name] = {
							"FeatureType": featureType,
							"ReturnType": retType,
							"Value": value,
							"Param1Type": p1[0], "Param1Name": p1[1], "Param1Value": p1[2],
							"Param2Type": p2[0],	"Param2Name": p2[1], "Param2Value": p2[2],
							"Category": currentCategory, "Comment": currentComment
						}
						if value in self.values:
							raise "Duplicate value " + value + " " + name
						self.values[value] = 1
						self.order.append(name)
					elif featureType == "evt":
						retType, name, value = decodeEvent(featureVal)
						self.features[name] = {
							"FeatureType": featureType,
							"ReturnType": retType,
							"Value": value,
							"Category": currentCategory, "Comment": currentComment
						}
						if value in self.events:
							raise "Duplicate event " + value + " " + name
						self.events[value] = 1
						self.order.append(name)
					elif featureType == "cat":
						currentCategory = featureVal
					elif featureType == "val":
						try:
							name, value = featureVal.split("=", 1)
						except ValueError:
							print("Failure %s" % featureVal)
							raise
						self.features[name] = {
							"FeatureType": featureType,
							"Category": currentCategory,
							"Value": value }
						self.order.append(name)
					elif featureType == "enu" or featureType == "lex":
						name, value = featureVal.split("=", 1)
						self.features[name] = {
							"FeatureType": featureType,
							"Category": currentCategory,
							"Value": value }
						self.order.append(name)

<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite"""

from nova.virt import xenapi_conn
from nova.virt.xenapi import fake
from nova.virt.xenapi import volume_utils
from nova.virt.xenapi import vm_utils
from nova.virt.xenapi import vmops
from nova import utils

def stubout_instance_snapshot(stubs):
    @classmethod
    def fake_fetch_image(cls, session, instance_id, image, user, project,
                         type):
        # Stubout wait_for_task
        def fake_wait_for_task(self, task, id):
            class FakeEvent:

                def send(self, value):
                    self.rv = value

                def wait(self):
                    return self.rv

            done = FakeEvent()
            self._poll_task(id, task, done)
            rv = done.wait()
            return rv

        def fake_loop(self):
            pass

        stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task',
                  fake_wait_for_task)

        stubs.Set(xenapi_conn.XenAPISession, '_stop_loop', fake_loop)

        from nova.virt.xenapi.fake import create_vdi
        name_label = "instance-%s" % instance_id
        #TODO: create fake SR record
        sr_ref = "fakesr"
        vdi_ref = create_vdi(name_label=name_label, read_only=False,
                             sr_ref=sr_ref, sharable=False)
        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
        vdi_uuid = vdi_rec['uuid']
        return vdi_uuid

    stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image)

    def fake_parse_xmlrpc_value(val):
        return val

    stubs.Set(xenapi_conn, '_parse_xmlrpc_value', fake_parse_xmlrpc_value)

    def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref,
                              original_parent_uuid):
        from nova.virt.xenapi.fake import create_vdi
        name_label = "instance-%s" % instance_id
        #TODO: create fake SR record
        sr_ref = "fakesr"
        vdi_ref = create_vdi(name_label=name_label, read_only=False,
                             sr_ref=sr_ref, sharable=False)
        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
        vdi_uuid = vdi_rec['uuid']
        return vdi_uuid

    stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image)

    def fake_parse_xmlrpc_value(val):
        return val

    stubs.Set(xenapi_conn, '_parse_xmlrpc_value', fake_parse_xmlrpc_value)

    def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref,
                              original_parent_uuid):
        #TODO(sirp): Should we actually fake out the data here
        return "fakeparent"

    stubs.Set(vm_utils, 'wait_for_vhd_coalesce', fake_wait_for_vhd_coalesce)

def stubout_session(stubs, cls):
    """Stubs out two methods from XenAPISession"""
    def fake_import(self):
        """Stubs out get_imported_xenapi of XenAPISession"""
        fake_module = 'nova.virt.xenapi.fake'
        from_list = ['fake']
        return __import__(fake_module, globals(), locals(), from_list, -1)

    stubs.Set(xenapi_conn.XenAPISession, '_create_session',
                       lambda s, url: cls(url))
    stubs.Set(xenapi_conn.XenAPISession, 'get_imported_xenapi',
                       fake_import)

def stub_out_get_target(stubs):
    """Stubs out _get_target in volume_utils"""
    def fake_get_target(volume_id):
        return (None, None)

    stubs.Set(volume_utils, '_get_target', fake_get_target)

def stubout_get_this_vm_uuid(stubs):
    def f():
        vms = [rec['uuid'] for ref, rec
               in fake.get_all_records('VM').iteritems()
               if rec['is_control_domain']]
        return vms[0]
    stubs.Set(vm_utils, 'get_this_vm_uuid', f)

def stubout_stream_disk(stubs):
    def f(_1, _2, _3, _4):
        pass
    stubs.Set(vm_utils, '_stream_disk', f)

def stubout_is_vdi_pv(stubs):
    def f(_1):
        return False
    stubs.Set(vm_utils, '_is_vdi_pv', f)

def stubout_loopingcall_start(stubs):
    def fake_start(self, interval, now=True):
        self.f(*self.args, **self.kw)
    stubs.Set(utils.LoopingCall, 'start', fake_start)

class FakeSessionForVMTests(fake.SessionBase):
    """ Stubs out a XenAPISession for VM tests """
    def __init__(self, uri):
        super(FakeSessionForVMTests, self).__init__(uri)

    def host_call_plugin(self, _1, _2, _3, _4, _5):
        sr_ref = fake.get_all('SR')[0]
        vdi_ref = fake.create_vdi('', False, sr_ref, False)
        vdi_rec = fake.get_record('VDI', vdi_ref)
        return '<string>%s</string>' % vdi_rec['uuid']

    def VM_start(self, _1, ref, _2, _3):
        vm = fake.get_record('VM', ref)
        if vm['power_state'] != 'Halted':
            raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted',
                                  vm['power_state']])
        vm['power_state'] = 'Running'
        vm['is_a_template'] = False
        vm['is_control_domain'] = False

    def VM_snapshot(self, session_ref, vm_ref, label):
        status = "Running"
        template_vm_ref = fake.create_vm(label, status, is_a_template=True,
            is_control_domain=False)

        sr_ref = "fakesr"
        template_vdi_ref = fake.create_vdi(label, read_only=True,
            sr_ref=sr_ref, sharable=False)

        template_vbd_ref = fake.create_vbd(template_vm_ref, template_vdi_ref)
        return template_vm_ref

    def VDI_destroy(self, session_ref, vdi_ref):
        fake.destroy_vdi(vdi_ref)

    def VM_destroy(self, session_ref, vm_ref):
        fake.destroy_vm(vm_ref)

    def SR_scan(self, session_ref, sr_ref):
        pass

    def VDI_set_name_label(self, session_ref, vdi_ref, name_label):
        pass

def stub_out_vm_methods(stubs):
    def fake_shutdown(self, inst, vm, method="clean"):
        pass

    def fake_acquire_bootlock(self, vm):
        pass

    def fake_release_bootlock(self, vm):
        pass

    def fake_spawn_rescue(self, inst):
        inst._rescue = False

    stubs.Set(vmops.VMOps, "_shutdown", fake_shutdown)
    stubs.Set(vmops.VMOps, "_acquire_bootlock", fake_acquire_bootlock)
    stubs.Set(vmops.VMOps, "_release_bootlock", fake_release_bootlock)
    stubs.Set(vmops.VMOps, "spawn_rescue", fake_spawn_rescue)

class FakeSessionForVolumeTests(fake.SessionBase):
    """ Stubs out a XenAPISession for Volume tests """
    def __init__(self, uri):
        super(FakeSessionForVolumeTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        valid_vdi = False
        refs = fake.get_all('VDI')
        for ref in refs:
            rec = fake.get_record('VDI', ref)
            if rec['uuid'] == uuid:
                valid_vdi = True
        if not valid_vdi:
            raise fake.Failure([['INVALID_VDI', 'session', self._session]])

class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests):
    """ Stubs out a XenAPISession for Volume tests: it injects failures """
    def __init__(self, uri):
        super(FakeSessionForVolumeFailedTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        # This is for testing failure
        raise fake.Failure([['INVALID_VDI', 'session', self._session]])

    def PBD_unplug(self, _1, ref):
        rec = fake.get_record('PBD', ref)
        rec['currently-attached'] = False

    def SR_forget(self, _1, ref):
        pass

class FakeSessionForMigrationTests(fake.SessionBase):
    """Stubs out a XenAPISession for Migration tests"""
    def __init__(self, uri):
        super(FakeSessionForMigrationTests, self).__init__(uri)

    def VDI_get_by_uuid(*args):
        return 'hurr'

    def VDI_resize_online(*args):
        pass

    def VM_start(self, _1, ref, _2, _3):
        vm = fake.get_record('VM', ref)
        if vm['power_state'] != 'Halted':
            raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted',
                                  vm['power_state']])
        vm['power_state'] = 'Running'
        vm['is_a_template'] = False
        vm['is_control_domain'] = False

def stub_out_migration_methods(stubs):
    def fake_get_snapshot(self, instance):
        return 'vm_ref', dict(image='foo', snap='bar')

    @classmethod
    def fake_get_vdi(cls, session, vm_ref):
        vdi_ref = fake.create_vdi(name_label='derp', read_only=False,
                             sr_ref='herp', sharable=False)
        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
        return vdi_ref, {'uuid': vdi_rec['uuid'], }

    def fake_shutdown(self, inst, vm, hard=True):
        pass

    @classmethod
    def fake_sr(cls, session, *args):
        pass

    @classmethod
    def fake_get_sr_path(cls, *args):
        return "fake"

    def fake_destroy(*args, **kwargs):
        pass

    def fake_reset_network(*args, **kwargs):
        pass

    stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
    stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr)
    stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr)
    stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot)
    stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi)
    stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None)
    stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path)
    stubs.Set(vmops.VMOps, 'reset_network', fake_reset_network)
    stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown)
<commit_msg>remove stubbing of XenAPISession.wait_for_task for xenapi tests as it doesn't need to be faked.  Also removed duplicate code that stubbed xenapi_conn._parse_xmlrpc_value.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite"""

from nova.virt import xenapi_conn
from nova.virt.xenapi import fake
from nova.virt.xenapi import volume_utils
from nova.virt.xenapi import vm_utils
from nova.virt.xenapi import vmops
from nova import utils

def stubout_instance_snapshot(stubs):
    @classmethod
    def fake_fetch_image(cls, session, instance_id, image, user, project,
                         type):
        from nova.virt.xenapi.fake import create_vdi
        name_label = "instance-%s" % instance_id
        #TODO: create fake SR record
        sr_ref = "fakesr"
        vdi_ref = create_vdi(name_label=name_label, read_only=False,
                             sr_ref=sr_ref, sharable=False)
        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
        vdi_uuid = vdi_rec['uuid']
        return vdi_uuid

    stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image)

    def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref,
                              original_parent_uuid):
        from nova.virt.xenapi.fake import create_vdi
        name_label = "instance-%s" % instance_id
        #TODO: create fake SR record
        sr_ref = "fakesr"
        vdi_ref = create_vdi(name_label=name_label, read_only=False,
                             sr_ref=sr_ref, sharable=False)
        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
        vdi_uuid = vdi_rec['uuid']
        return vdi_uuid

    stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image)

    def fake_parse_xmlrpc_value(val):
        return val

    stubs.Set(xenapi_conn, '_parse_xmlrpc_value', fake_parse_xmlrpc_value)

    def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref,
                              original_parent_uuid):
        #TODO(sirp): Should we actually fake out the data here
        return "fakeparent"

    stubs.Set(vm_utils, 'wait_for_vhd_coalesce', fake_wait_for_vhd_coalesce)

def stubout_session(stubs, cls):
    """Stubs out two methods from XenAPISession"""
    def fake_import(self):
        """Stubs out get_imported_xenapi of XenAPISession"""
        fake_module = 'nova.virt.xenapi.fake'
        from_list = ['fake']
        return __import__(fake_module, globals(), locals(), from_list, -1)

    stubs.Set(xenapi_conn.XenAPISession, '_create_session',
                       lambda s, url: cls(url))
    stubs.Set(xenapi_conn.XenAPISession, 'get_imported_xenapi',
                       fake_import)

def stub_out_get_target(stubs):
    """Stubs out _get_target in volume_utils"""
    def fake_get_target(volume_id):
        return (None, None)

    stubs.Set(volume_utils, '_get_target', fake_get_target)

def stubout_get_this_vm_uuid(stubs):
    def f():
        vms = [rec['uuid'] for ref, rec
               in fake.get_all_records('VM').iteritems()
               if rec['is_control_domain']]
        return vms[0]
    stubs.Set(vm_utils, 'get_this_vm_uuid', f)

def stubout_stream_disk(stubs):
    def f(_1, _2, _3, _4):
        pass
    stubs.Set(vm_utils, '_stream_disk', f)

def stubout_is_vdi_pv(stubs):
    def f(_1):
        return False
    stubs.Set(vm_utils, '_is_vdi_pv', f)

def stubout_loopingcall_start(stubs):
    def fake_start(self, interval, now=True):
        self.f(*self.args, **self.kw)
    stubs.Set(utils.LoopingCall, 'start', fake_start)

class FakeSessionForVMTests(fake.SessionBase):
    """ Stubs out a XenAPISession for VM tests """
    def __init__(self, uri):
        super(FakeSessionForVMTests, self).__init__(uri)

    def host_call_plugin(self, _1, _2, _3, _4, _5):
        sr_ref = fake.get_all('SR')[0]
        vdi_ref = fake.create_vdi('', False, sr_ref, False)
        vdi_rec = fake.get_record('VDI', vdi_ref)
        return '<string>%s</string>' % vdi_rec['uuid']

    def VM_start(self, _1, ref, _2, _3):
        vm = fake.get_record('VM', ref)
        if vm['power_state'] != 'Halted':
            raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted',
                                  vm['power_state']])
        vm['power_state'] = 'Running'
        vm['is_a_template'] = False
        vm['is_control_domain'] = False

    def VM_snapshot(self, session_ref, vm_ref, label):
        status = "Running"
        template_vm_ref = fake.create_vm(label, status, is_a_template=True,
            is_control_domain=False)

        sr_ref = "fakesr"
        template_vdi_ref = fake.create_vdi(label, read_only=True,
            sr_ref=sr_ref, sharable=False)

        template_vbd_ref = fake.create_vbd(template_vm_ref, template_vdi_ref)
        return template_vm_ref

    def VDI_destroy(self, session_ref, vdi_ref):
        fake.destroy_vdi(vdi_ref)

    def VM_destroy(self, session_ref, vm_ref):
        fake.destroy_vm(vm_ref)

    def SR_scan(self, session_ref, sr_ref):
        pass

    def VDI_set_name_label(self, session_ref, vdi_ref, name_label):
        pass

def stub_out_vm_methods(stubs):
    def fake_shutdown(self, inst, vm, method="clean"):
        pass

    def fake_acquire_bootlock(self, vm):
        pass

    def fake_release_bootlock(self, vm):
        pass

    def fake_spawn_rescue(self, inst):
        inst._rescue = False

    stubs.Set(vmops.VMOps, "_shutdown", fake_shutdown)
    stubs.Set(vmops.VMOps, "_acquire_bootlock", fake_acquire_bootlock)
    stubs.Set(vmops.VMOps, "_release_bootlock", fake_release_bootlock)
    stubs.Set(vmops.VMOps, "spawn_rescue", fake_spawn_rescue)

class FakeSessionForVolumeTests(fake.SessionBase):
    """ Stubs out a XenAPISession for Volume tests """
    def __init__(self, uri):
        super(FakeSessionForVolumeTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        valid_vdi = False
        refs = fake.get_all('VDI')
        for ref in refs:
            rec = fake.get_record('VDI', ref)
            if rec['uuid'] == uuid:
                valid_vdi = True
        if not valid_vdi:
            raise fake.Failure([['INVALID_VDI', 'session', self._session]])

class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests):
    """ Stubs out a XenAPISession for Volume tests: it injects failures """
    def __init__(self, uri):
        super(FakeSessionForVolumeFailedTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        # This is for testing failure
        raise fake.Failure([['INVALID_VDI', 'session', self._session]])

    def PBD_unplug(self, _1, ref):
        rec = fake.get_record('PBD', ref)
        rec['currently-attached'] = False

    def SR_forget(self, _1, ref):
        pass

class FakeSessionForMigrationTests(fake.SessionBase):
    """Stubs out a XenAPISession for Migration tests"""
    def __init__(self, uri):
        super(FakeSessionForMigrationTests, self).__init__(uri)

    def VDI_get_by_uuid(*args):
        return 'hurr'

    def VDI_resize_online(*args):
        pass

    def VM_start(self, _1, ref, _2, _3):
        vm = fake.get_record('VM', ref)
        if vm['power_state'] != 'Halted':
            raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted',
                                  vm['power_state']])
        vm['power_state'] = 'Running'
        vm['is_a_template'] = False
        vm['is_control_domain'] = False

def stub_out_migration_methods(stubs):
    def fake_get_snapshot(self, instance):
        return 'vm_ref', dict(image='foo', snap='bar')

    @classmethod
    def fake_get_vdi(cls, session, vm_ref):
        vdi_ref = fake.create_vdi(name_label='derp', read_only=False,
                             sr_ref='herp', sharable=False)
        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
        return vdi_ref, {'uuid': vdi_rec['uuid'], }

    def fake_shutdown(self, inst, vm, hard=True):
        pass

    @classmethod
    def fake_sr(cls, session, *args):
        pass

    @classmethod
    def fake_get_sr_path(cls, *args):
        return "fake"

    def fake_destroy(*args, **kwargs):
        pass

    def fake_reset_network(*args, **kwargs):
        pass

    stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
    stubs.Set(vm_utils.VMHelper, 'scan_default_sr', fake_sr)
    stubs.Set(vm_utils.VMHelper, 'scan_sr', fake_sr)
    stubs.Set(vmops.VMOps, '_get_snapshot', fake_get_snapshot)
    stubs.Set(vm_utils.VMHelper, 'get_vdi_for_vm_safely', fake_get_vdi)
    stubs.Set(xenapi_conn.XenAPISession, 'wait_for_task', lambda x, y, z: None)
    stubs.Set(vm_utils.VMHelper, 'get_sr_path', fake_get_sr_path)
    stubs.Set(vmops.VMOps, 'reset_network', fake_reset_network)
    stubs.Set(vmops.VMOps, '_shutdown', fake_shutdown)
<|endoftext|>


<commit_before>import unittest

import os
import simplejson

import socorro.lib.filesystem as soc_filesys

import socorro.unittest.testlib.util as tutil

import socorro.cron.dailyUrl as dailyUrl

jsonData = [
  # [testMount_index,uuid,json dict]
  [0,'abcd289c-a4e0-496a-a124-beb6d2100125', {
  "ProductName": "Firefox",
  "Version": "3.6",
  "EMCheckCompatibility": "true",
  "Add-ons": "oma@me.com:1.1.2,opa@me.com:6.0.13",
  "Throttleable": "1",
  "Theme": "classic\/1.0",
  "StartupTime": "1264455098",
  "timestamp": 1264455122.84,
  "ReleaseChannel": "release"
  }],
  [1,'deadbea7-3141-4ec4-96bc-5a8212100125', {
  "ProductName": "Firefox",
  "Version": "3.6",
  "EMCheckCompatibility": "false",
  "Add-ons": "oma@me.com:1.1.2,opa@me.com:6.0.13",
  "Throttleable": "0",
  "Theme": "classic\/1.0",
  "StartupTime": "1264455098",
  "timestamp": 1264455122.84,
  "ReleaseChannel": "release"
  }],
  [2,'deaf36ac-2615-42a1-8ede-e98e12100125', {
  "ProductName": "Firefox",
  "Version": "3.6",
  "Add-ons": "oma@me.com:1.1.2,opa@me.com:6.0.13",
  "Theme": "classic\/1.0",
  "StartupTime": "1264455098",
  "timestamp": 1264455122.84,
  "ReleaseChannel": "release"
  }],
]

def setup_module():
  tutil.nosePrintModule(__file__)

class testFormatter:
  def __init__(self,initialData = []):
    self.data = initialData

  def writerow(self,aList):
    self.data.append(aList)

class TestDailyUrl(unittest.TestCase):
  def setUp(self):
    try:
      shutil.rmtree(self.testDir)
    except:
      pass
    self.testDir = os.path.join(os.path.dirname(__file__),'TEST_DIR')
    soc_filesys.makedirs(self.testDir)
    self.testMounts =['mount0','foo%(S)sbar%(S)sbaz%(S)s..%(S)sboo'%{'S':os.path.sep},'moo%(S)syou'%{'S':os.path.sep}]
    for d in jsonData:
      dir = os.path.join(self.testDir,self.testMounts[d[0]])
      jsonFile = dailyUrl.pathFromUuidAndMount(dir,d[1],'json')
      jsonDir = os.path.dirname(jsonFile)
      soc_filesys.makedirs(jsonDir)
      fh = open(jsonFile,'w')
      simplejson.dump(d[2],fh)

  def tearDown(self):
    try:
      shutil.rmtree(self.testDir)
    except:
      pass

  def testPathFromUuidAndMount(self):
    uuids = ['0bba61c5-dfc3-43e7-dead-8afd20071025', '0bba929f-8721-460c-dead-a43c20071025', '0b9ff107-8672-4aac-dead-b2bd20081225']
    suffix = 'suff'
    expected = [os.path.sep.join([self.testMounts[0],'0b','ba',uuids[0]+'.'+suffix]),
                os.path.sep.join([self.testMounts[1],'0b','ba',uuids[1]+'.'+suffix]),
                os.path.sep.join([self.testMounts[2],'0b','9f',uuids[2]+'.'+suffix]),
                ]
    for i in range(len(uuids)):
      got = dailyUrl.pathFromUuidAndMount(self.testMounts[i],uuids[i],suffix)
      assert expected[i] == got, 'Expected %s but got %s'%(expected[i],got)

  def testGetJson(self):
    config = {'rawFileMountPoints': ' '.join([os.path.join(self.testDir,x) for x in self.testMounts])}
    for jd in jsonData:
      x = dailyUrl.getJson(config,jd[1])
      assert x, 'expected a real json file, but got %s'%(x)
    x = dailyUrl.getJson(config,'aaaaffff-aeae-6666-7777-babe02100125')
    assert not x, 'expected None, got %s'%(x)

  def testGetColumnHeader(self):
    assert ['addons_checked'] == dailyUrl.getColumnHeader([])

  def testAppendDetailsFromJson(self):
    config = {'rawFileMountPoints': ' '.join([os.path.join(self.testDir,x) for x in self.testMounts])}
    cList = []
    dailyUrl.appendDetailsFromJson(config,cList,'aaaaffff-aeae-6666-7777-babe02100125')
    assert [] == cList, 'expected empty list, got %s'%(str(cList))
    expected = [['checked'],['not'],['unknown']]
    for i in range(len(jsonData)):
      jd = jsonData[i]
      cList = []
      dailyUrl.appendDetailsFromJson(config,cList,jd[1])
      assert expected[i] == cList, 'Expected %s, got %s'%(str(expected[i]),str(cList))

  def testWriteRowToInternalAndExternalFiles(self):
    iF = testFormatter([])
    pF = testFormatter([])
    data = [
      ['0','http://url',2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'', 18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ]
    iExpected = [
      ['0','http://url',2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'', 18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ]
    pExpected = [
      ['0','URL removed',2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'',18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'', 18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'',18],
      ]
    for d in data:
      dailyUrl.writeRowToInternalAndExternalFiles(iF,pF,d)

    for i in range(len(iF.data)):
      assert iExpected[i] == iF.data[i]
    for i in range(len(pF.data)):
      assert pExpected[i] == pF.data[i]
<commit_msg>Add test of addonsChecked<commit_after>import unittest

import os
import simplejson

import socorro.lib.filesystem as soc_filesys

import socorro.unittest.testlib.util as tutil
import socorro.unittest.testlib.loggerForTest as tLogger

import socorro.cron.dailyUrl as dailyUrl

dailyUrl.logger = tLogger.TestingLogger()

jsonData = [
  # [testMount_index,uuid,json dict]
  [0,'abcd289c-a4e0-496a-a124-beb6d2100125', {
  "ProductName": "Firefox",
  "Version": "3.6",
  "EMCheckCompatibility": "true",
  "Add-ons": "oma@me.com:1.1.2,opa@me.com:6.0.13",
  "Throttleable": "1",
  "Theme": "classic\/1.0",
  "StartupTime": "1264455098",
  "timestamp": 1264455122.84,
  "ReleaseChannel": "release"
  }],
  [1,'deadbea7-3141-4ec4-96bc-5a8212100125', {
  "ProductName": "Firefox",
  "Version": "3.6",
  "EMCheckCompatibility": "false",
  "Add-ons": "oma@me.com:1.1.2,opa@me.com:6.0.13",
  "Throttleable": "0",
  "Theme": "classic\/1.0",
  "StartupTime": "1264455098",
  "timestamp": 1264455122.84,
  "ReleaseChannel": "release"
  }],
  [2,'deaf36ac-2615-42a1-8ede-e98e12100125', {
  "ProductName": "Firefox",
  "Version": "3.6",
  "Add-ons": "oma@me.com:1.1.2,opa@me.com:6.0.13",
  "Theme": "classic\/1.0",
  "StartupTime": "1264455098",
  "timestamp": 1264455122.84,
  "ReleaseChannel": "release"
  }],
]

def setup_module():
  tutil.nosePrintModule(__file__)

class testFormatter:
  def __init__(self,initialData = []):
    self.data = initialData

  def writerow(self,aList):
    self.data.append(aList)

class TestDailyUrl(unittest.TestCase):
  def setUp(self):
    try:
      shutil.rmtree(self.testDir)
    except:
      pass
    self.testDir = os.path.join(os.path.dirname(__file__),'TEST_DIR')
    soc_filesys.makedirs(self.testDir)
    self.testMounts =['mount0','foo%(S)sbar%(S)sbaz%(S)s..%(S)sboo'%{'S':os.path.sep},'moo%(S)syou'%{'S':os.path.sep}]
    for d in jsonData:
      dir = os.path.join(self.testDir,self.testMounts[d[0]])
      jsonFile = dailyUrl.pathFromUuidAndMount(dir,d[1],'json')
      jsonDir = os.path.dirname(jsonFile)
      soc_filesys.makedirs(jsonDir)
      fh = open(jsonFile,'w')
      simplejson.dump(d[2],fh)

  def tearDown(self):
    try:
      shutil.rmtree(self.testDir)
    except:
      pass

  def testPathFromUuidAndMount(self):
    uuids = ['0bba61c5-dfc3-43e7-dead-8afd20071025', '0bba929f-8721-460c-dead-a43c20071025', '0b9ff107-8672-4aac-dead-b2bd20081225']
    suffix = 'suff'
    expected = [os.path.sep.join([self.testMounts[0],'0b','ba',uuids[0]+'.'+suffix]),
                os.path.sep.join([self.testMounts[1],'0b','ba',uuids[1]+'.'+suffix]),
                os.path.sep.join([self.testMounts[2],'0b','9f',uuids[2]+'.'+suffix]),
                ]
    for i in range(len(uuids)):
      got = dailyUrl.pathFromUuidAndMount(self.testMounts[i],uuids[i],suffix)
      assert expected[i] == got, 'Expected %s but got %s'%(expected[i],got)

  def testGetJson(self):
    config = {'rawFileMountPoints': ' '.join([os.path.join(self.testDir,x) for x in self.testMounts])}
    for jd in jsonData:
      x = dailyUrl.getJson(config,jd[1])
      assert x, 'expected a real json file, but got %s'%(x)
    x = dailyUrl.getJson(config,'aaaaffff-aeae-6666-7777-babe02100125')
    assert not x, 'expected None, got %s'%(x)

  def testGetColumnHeader(self):
    assert ['addons_checked'] == dailyUrl.getColumnHeader([])

  def testAppendDetailsFromJson(self):
    config = {'rawFileMountPoints': ' '.join([os.path.join(self.testDir,x) for x in self.testMounts])}
    cList = []
    dailyUrl.appendDetailsFromJson(config,cList,'aaaaffff-aeae-6666-7777-babe02100125')
    assert [] == cList, 'expected empty list, got %s'%(str(cList))
    expected = [['checked'],['not'],['unknown']]
    for i in range(len(jsonData)):
      jd = jsonData[i]
      cList = []
      dailyUrl.appendDetailsFromJson(config,cList,jd[1])
      assert expected[i] == cList, 'Expected %s, got %s'%(str(expected[i]),str(cList))

  def testWriteRowToInternalAndExternalFiles(self):
    iF = testFormatter([])
    pF = testFormatter([])
    data = [
      ['0','http://url',2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'', 18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ]
    iExpected = [
      ['0','http://url',2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'', 18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'email@a.b',18],
      ]
    pExpected = [
      ['0','URL removed',2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'',18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'', 18],
      ['0','', 2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,'',18],
      ]
    for d in data:
      dailyUrl.writeRowToInternalAndExternalFiles(iF,pF,d)

    for i in range(len(iF.data)):
      assert iExpected[i] == iF.data[i]
    for i in range(len(pF.data)):
      assert pExpected[i] == pF.data[i]

  def testAddonsChecked(self):
    cases = [
      ['1', 'checked'],
      ['0', 'checked'],
      [0, 'not'],
      [1, 'checked'],
      ['false', 'not'],
      ['true', 'checked'],
      ['FALSE', 'not'],
      ['TRUE', 'checked'],
      [[],'not'],
      [{}, 'not'],
      [('toot'),'checked'],
     ]
    for arg,expected in cases:
      got = dailyUrl.addonsChecked(arg)
      assert expected == got, 'Got %s, expected %s'%(got,expected)
<|endoftext|>


<commit_before>import numpy as np
import cPickle
import load_data
import mscentipede
import argparse
import time, pdb

def learn_model(options):

    # load motif sites
    motif_handle = load_data.ZipFile(options.motif_file)
    locations = motif_handle.read()
    motif_handle.close()
    if np.any([len(loc)<5 for loc in locations]):
        print "Error: ensure all rows in motif instance file contain same number of columns"
        sys.exit(1)
    try:
        scores = np.array([loc[4:] for loc in locations]).astype('float')
    except ValueError:
        print "Error: column 5 and higher should all be numeric values."
        sys.exit(1)

    # load read data
    bam_handles = [load_data.BamFile(bam_file) for bam_file in options.bam_files]
    count_data = np.array([bam_handle.get_read_counts(locations, width=max([200,options.window])) \
        for bam_handle in bam_handles])
    ig = [handle.close() for handle in bam_handles]
    total_counts = np.sum(count_data, 2).T

    # extract reads within specified window size
    if options.window<200:
        counts = np.array([np.hstack((count[:,100-options.window/2:100+options.window/2], \
            count[:,300-options.window/2:300+options.window/2])).T \
            for count in count_data]).T
    else:
        counts = np.array([count.T for count in count_data]).T

    # specify background
    if options.model=='msCentipede':
        background_counts = np.ones((1,2*options.window,1), dtype=float)
    elif options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
        bam_handle = load_data.BamFile(options.bam_file_genomicdna)
        bg_count_data = np.array([bam_handle.get_read_counts(locations, width=options.window)])
        bam_handle.close()
        background_counts = np.array([count.T for count in bg_count_data]).T

    # estimate model parameters
    footprint_model, count_model, prior = mscentipede.estimate_optimal_model(counts, total_counts, scores, \
        background_counts, options.model, options.restarts, options.mintol)

    # save model parameter estimates
    model_handle = open(options.model_file, 'w')
    cPickle.Pickler(model_handle,protocol=2).dump(footprint_model)
    cPickle.Pickler(model_handle,protocol=2).dump(count_model)
    cPickle.Pickler(model_handle,protocol=2).dump(prior)
    model_handle.close()

def infer_binding(options):

    # load the model
    handle = open(options.model_file, "r")
    footprint_model = cPickle.load(handle)
    count_model = cPickle.load(handle)
    prior = cPickle.load(handle)
    handle.close()

    # load motifs
    motif_handle = load_data.ZipFile(options.motif_file)

    # open read data handles
    bam_handles = [load_data.BamFile(bam_file) for bam_file in options.bam_files]

    # open background data handles
    if options.model=='msCentipede':
        background_counts = np.ones((1,2*options.window,1), dtype=float)
    elif options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
        bg_handle = load_data.BamFile(options.bam_file_genomicdna)

    # check number of motif sites
    pipe = load_data.subprocess.Popen("zcat %s | wc -l"%options.motif_file, \
        stdout=load_data.subprocess.PIPE, shell=True)
    Ns = int(pipe.communicate()[0].strip())
    loops = Ns/options.batch+1

    # open gzip file to save inference
    handle = load_data.gzip.open(options.posterior_file, "wb")
    header = ['Chr','Start','Stop','Strand','LogPosOdds','LogPriorOdds','MultLikeRatio','NegBinLikeRatio']
    handle.write('\t'.join(header)+'\n')

    for n in xrange(loops):
        starttime = time.time()
        locations = motif_handle.read(batch=options.batch)

        count_data = np.array([bam_handle.get_read_counts(locations, width=max([200,options.window])) \
            for bam_handle in bam_handles])
        total_counts = np.sum(count_data, 2).T

        scores = np.array([loc[4:] for loc in locations]).astype('float')

        # extract reads within specified window size
        if options.window<200:
            counts = np.array([np.hstack((count[:,100-options.window/2:100+options.window/2], \
                count[:,300-options.window/2:300+options.window/2])).T \
                for count in count_data]).T
        else:
            counts = np.array([count.T for count in count_data]).T

        # specify background
        if options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
            bg_count_data = np.array([bg_handle.get_read_counts(locations, width=options.window)])
            background_counts = np.array([count.T for count in bg_count_data]).T

        logodds = mscentipede.decode(counts, total_counts, scores, background_counts, \
            footprint_model, count_model, prior, options.model)

        ignore = [loc.extend(['%.4f'%p for p in pos[:4]])
            for loc,pos in zip(locations,logodds)]
        ignore = [handle.write('\t'.join(map(str,elem))+'\n') for elem in locations]
        print len(locations), time.time()-starttime

    handle.close()
    if options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
        bg_handle.close()
    ig = [handle.close() for handle in bam_handles]

def parse_args():

    parser = argparse.ArgumentParser(description="""runs msCentipede, to "
        "infer posterior of transcription factor binding, given a set of motif locations.""")

    parser.add_argument("--mode",
                        choices=("learn","infer"),
                        default="learn",
                        help="specify whether msCentipede is used to learn model parameters "
                        " or infer factor binding ")

    parser.add_argument("--infer",
                        default=False,
                        action='store_true',
                        help="call the subroutine to compute binding posteriors, given "
                        "msCentipede model parameters")

    parser.add_argument("--model",
                        choices=("msCentipede", "msCentipede_flexbg", "msCentipede_flexbgmean"),
                        default="msCentipede",
                        help="models differ in how they capture background rate of chromatin accessibility")

    parser.add_argument("--restarts",
                        type=int,
                        default=1,
                        help="number of re-runs of the algorithm")

    parser.add_argument("--mintol",
                        type=float,
                        default=1e-6,
                        help="convergence criterion for change in per-site marginal likelihood")

    parser.add_argument("--model_file",
                        type=str,
                        default=None,
                        help="file name to store the model parameters")

    parser.add_argument("--posterior_file",
                        type=str,
                        default=None,
                        help="file name to store the posterior odds ratio, and "
                        "likelihood ratios for each model component, at each motif. ")

    parser.add_argument("--window",
                        type=int,
                        default=128,
                        help="size of window around the motif, where chromatin "
                        "accessibility profile is used for inferring transcription "
                        "factor binding.")

    parser.add_argument("--batch",
                        type=int,
                        default=10000,
                        help="number of motifs to use for learning model parameters; "
                        " also, number of motifs to decode at a time.")

    parser.add_argument("motif_file",
                        action="store",
                        help="name of a gzipped text file containing "
                        " positional information and other attributes for motifs "
                        " of a transcription factor. Columns of the file should be as follows. "
                        " Chromosome Start End Strand PWM_Score [Attribute_1 Attribute_2 ...] "
                        " additional attributes are optional.")

    parser.add_argument("bam_files",
                        action="store",
                        nargs="+",
                        help="comma-separated list of bam files "
                        " from a chromatin accessibility assay ")

    parser.add_argument("--bam_file_genomicdna",
                        action="store",
                        default=None,
                        help="bam file from a chromatin accessibility assay on genomic DNA")

    options = parser.parse_args()

    # if no motif file is provided, throw an error
    if options.motif_file is None:
        parser.error("Need to provide a file of motifs for a transcription factor")

    # if no model file is provided, create a `default` model file name
    if options.model_file is None:
        options.model_file = options.motif_file.split('.')[0]+"_model_parameters.pkl"

    # if no posterior file is provided, create a `default` posterior file name
    if options.posterior_file is None:
        options.posterior_file = options.motif_file.split('.')[0]+"_binding_posterior.txt.gz"

    # make sure model file exists, before trying to run inference
    if options.mode=='infer':
        try:
            handle = open(options.model_file, 'r')
            handle.close()
        except IOError:
            parser.error("Need to provide the file where model parameters are saved")

    if options.model in ['msCentipede-flexbgmean','msCentipede-flexbg'] and options.bam_file_genomicdna is None:
        parser.error("Need to provide a bam file containing chromatin accessibility \
            data in genomic DNA, if the model is specified to be \
            msCentipede-flexbgmean or msCentipede-flexbg")

    return options

def main():

    options = parse_args()

    if options.mode=='learn':
        learn_model(options)

    elif options.mode=='infer':
        infer_binding(options)

if __name__=="__main__":

    main()
<commit_msg>minor edits<commit_after>import numpy as np
import cPickle
import load_data
import mscentipede
import argparse
import time, pdb

def learn_model(options):

    # load motif sites
    motif_handle = load_data.ZipFile(options.motif_file)
    locations = motif_handle.read()
    motif_handle.close()
    if np.any([len(loc)<5 for loc in locations]):
        print "Error: ensure all rows in motif instance file contain same number of columns"
        sys.exit(1)
    try:
        scores = np.array([loc[4:] for loc in locations]).astype('float')
    except ValueError:
        print "Error: column 5 and higher should all be numeric values."
        sys.exit(1)

    # load read data
    bam_handles = [load_data.BamFile(bam_file) for bam_file in options.bam_files]
    count_data = np.array([bam_handle.get_read_counts(locations, width=max([200,options.window])) \
        for bam_handle in bam_handles])
    ig = [handle.close() for handle in bam_handles]
    total_counts = np.sum(count_data, 2).T

    # extract reads within specified window size
    if options.window<200:
        counts = np.array([np.hstack((count[:,100-options.window/2:100+options.window/2], \
            count[:,300-options.window/2:300+options.window/2])).T \
            for count in count_data]).T
    else:
        counts = np.array([count.T for count in count_data]).T

    # specify background
    if options.model=='msCentipede':
        background_counts = np.ones((1,2*options.window,1), dtype=float)
    elif options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
        bam_handle = load_data.BamFile(options.bam_file_genomicdna)
        bg_count_data = np.array([bam_handle.get_read_counts(locations, width=options.window)])
        bam_handle.close()
        background_counts = np.array([count.T for count in bg_count_data]).T

    # estimate model parameters
    footprint_model, count_model, prior = mscentipede.estimate_optimal_model(counts, total_counts, scores, \
        background_counts, options.model, options.restarts, options.mintol)

    # save model parameter estimates
    model_handle = open(options.model_file, 'w')
    cPickle.Pickler(model_handle,protocol=2).dump(footprint_model)
    cPickle.Pickler(model_handle,protocol=2).dump(count_model)
    cPickle.Pickler(model_handle,protocol=2).dump(prior)
    model_handle.close()

def infer_binding(options):

    # load the model
    handle = open(options.model_file, "r")
    footprint_model = cPickle.load(handle)
    count_model = cPickle.load(handle)
    prior = cPickle.load(handle)
    handle.close()

    # load motifs
    motif_handle = load_data.ZipFile(options.motif_file)

    # open read data handles
    bam_handles = [load_data.BamFile(bam_file) for bam_file in options.bam_files]

    # open background data handles
    if options.model=='msCentipede':
        background_counts = np.ones((1,2*options.window,1), dtype=float)
    elif options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
        bg_handle = load_data.BamFile(options.bam_file_genomicdna)

    # check number of motif sites
    pipe = load_data.subprocess.Popen("zcat %s | wc -l"%options.motif_file, \
        stdout=load_data.subprocess.PIPE, shell=True)
    Ns = int(pipe.communicate()[0].strip())
    loops = Ns/options.batch+1

    # open gzip file to save inference
    handle = load_data.gzip.open(options.posterior_file, "wb")
    header = ['Chr','Start','Stop','Strand','LogPosOdds','LogPriorOdds','MultLikeRatio','NegBinLikeRatio']
    handle.write('\t'.join(header)+'\n')

    for n in xrange(loops):
        starttime = time.time()
        locations = motif_handle.read(batch=options.batch)

        count_data = np.array([bam_handle.get_read_counts(locations, width=max([200,options.window])) \
            for bam_handle in bam_handles])
        total_counts = np.sum(count_data, 2).T

        scores = np.array([loc[4:] for loc in locations]).astype('float')

        # extract reads within specified window size
        if options.window<200:
            counts = np.array([np.hstack((count[:,100-options.window/2:100+options.window/2], \
                count[:,300-options.window/2:300+options.window/2])).T \
                for count in count_data]).T
        else:
            counts = np.array([count.T for count in count_data]).T

        # specify background
        if options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
            bg_count_data = np.array([bg_handle.get_read_counts(locations, width=options.window)])
            background_counts = np.array([count.T for count in bg_count_data]).T

        logodds = mscentipede.decode(counts, total_counts, scores, background_counts, \
            footprint_model, count_model, prior, options.model)

        ignore = [loc.extend(['%.4f'%p for p in pos[:4]])
            for loc,pos in zip(locations,logodds)]
        ignore = [handle.write('\t'.join(map(str,elem))+'\n') for elem in locations]
        print len(locations), time.time()-starttime

    handle.close()
    if options.model in ['msCentipede_flexbgmean','msCentipede_flexbg']:
        bg_handle.close()
    ig = [handle.close() for handle in bam_handles]

def parse_args():

    parser = argparse.ArgumentParser(description="""runs msCentipede, to "
        "infer posterior of transcription factor binding, given a set of motif locations.""")

    parser.add_argument("--mode",
                        choices=("learn","infer"),
                        default="learn",
                        help="specify whether msCentipede is used to learn model parameters "
                        " or infer factor binding ")

    parser.add_argument("--infer",
                        default=False,
                        action='store_true',
                        help="call the subroutine to compute binding posteriors, given "
                        "msCentipede model parameters")

    parser.add_argument("--model",
                        choices=("msCentipede", "msCentipede_flexbg", "msCentipede_flexbgmean"),
                        default="msCentipede",
                        help="models differ in how they capture background rate of chromatin accessibility")

    parser.add_argument("--restarts",
                        type=int,
                        default=1,
                        help="number of re-runs of the algorithm")

    parser.add_argument("--mintol",
                        type=float,
                        default=1e-6,
                        help="convergence criterion for change in per-site marginal likelihood")

    parser.add_argument("--model_file",
                        type=str,
                        default=None,
                        help="file name to store the model parameters")

    parser.add_argument("--posterior_file",
                        type=str,
                        default=None,
                        help="file name to store the posterior odds ratio, and "
                        "likelihood ratios for each model component, at each motif. ")

    parser.add_argument("--window",
                        type=int,
                        default=128,
                        help="size of window around the motif, where chromatin "
                        "accessibility profile is used for inferring transcription "
                        "factor binding.")

    parser.add_argument("--batch",
                        type=int,
                        default=10000,
                        help="number of motifs to use for learning model parameters; "
                        " also, number of motifs to decode at a time.")

    parser.add_argument("motif_file",
                        action="store",
                        help="name of a gzipped text file containing "
                        " positional information and other attributes for motifs "
                        " of a transcription factor. Columns of the file should be as follows. "
                        " Chromosome Start End Strand PWM_Score [Attribute_1 Attribute_2 ...] "
                        " additional attributes are optional.")

    parser.add_argument("bam_files",
                        action="store",
                        nargs="+",
                        help="comma-separated list of bam files "
                        " from a chromatin accessibility assay ")

    parser.add_argument("--bam_file_genomicdna",
                        action="store",
                        default=None,
                        help="bam file from a chromatin accessibility assay on genomic DNA")

    options = parser.parse_args()

    # if no motif file is provided, throw an error
    if options.motif_file is None:
        parser.error("Need to provide a file of motifs for a transcription factor")

    # if no model file is provided, create a `default` model file name
    if options.model_file is None:
        options.model_file = options.motif_file.split('.')[0]+"_model_parameters.pkl"

    # if no posterior file is provided, create a `default` posterior file name
    if options.posterior_file is None:
        options.posterior_file = options.motif_file.split('.')[0]+"_binding_posterior.txt.gz"

    # make sure model file exists, before trying to run inference
    if options.mode=='infer':
        try:
            handle = open(options.model_file, 'r')
            handle.close()
        except IOError:
            parser.error("Need to provide the file where model parameters are saved")

    if options.model in ['msCentipede_flexbgmean','msCentipede_flexbg'] and options.bam_file_genomicdna is None:
        parser.error("Need to provide a bam file containing chromatin accessibility \
            data in genomic DNA, if the model is specified to be \
            msCentipede-flexbgmean or msCentipede-flexbg")

    return options

def main():

    options = parse_args()

    if options.mode=='learn':
        learn_model(options)

    elif options.mode=='infer':
        infer_binding(options)

if __name__=="__main__":

    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
from __future__ import unicode_literals
import matplotlib.pyplot as plt
from errno import EEXIST
from os import makedirs

from os.path import exists, dirname

from constants import STEPS, DELTA_T, CONTROLLER, RESULTS_DIRECTORY, TRAJECTORY
from trajectory import create_trajectory

class Plotter:
    def __init__(self):
        trajectory = create_trajectory()
        self.t = [i * DELTA_T for i in range(STEPS)]
        self.x_ref = [trajectory.get_position_at(i * DELTA_T).x for i in range(STEPS)]
        self.y_ref = [trajectory.get_position_at(i * DELTA_T).y for i in range(STEPS)]
        self.x = []
        self.y = []
        self.theta = []
        self.theta_ref = []
        self.v_c = []
        self.w_c = []

        self.fig_part_0, self.plots_part_0 = plt.subplots(2, 2, sharex=True)
        self.fig_part_1 = plt.figure()
        self.plots_part_1 = [
            plt.subplot(221),
            plt.subplot(223),
            plt.subplot(122),
        ]

        self.fig_part_2, self.plots_part_2 = plt.subplots(1, 2)

        self.LINE_WIDTH = 2
        self.FIGURE_TITLE_SIZE = 21
        self.PLOT_TITLE_SIZE = 19
        self.PLOT_AXIS_LABEL_SIZE = 17

    def add_point(self, pose):
        self.x.append(pose.position.x)
        self.y.append(pose.position.y)

    def decorate_plot(self, plot, title, x_label, y_label):
        plot.set_title(title, fontsize=self.PLOT_TITLE_SIZE)
        plot.set_xlabel(x_label, fontsize=self.PLOT_AXIS_LABEL_SIZE)
        plot.set_ylabel(y_label, fontsize=self.PLOT_AXIS_LABEL_SIZE)
        plot.legend(loc=0)
        plot.grid()

    def plot_results(self):
        zeros = [0 for _ in range(STEPS)]
        e_x = [(b_i - a_i) for a_i , b_i in zip(self.x, self.x_ref)]
        e_y = [(b_i - a_i) for a_i , b_i in zip(self.y, self.y_ref)]
        self.plots_part_0[0, 0].plot(self.t, self.x_ref, 'r--', label=r'$x_{ref}$', lw=self.LINE_WIDTH)
        self.plots_part_0[0, 0].plot(self.t, self.x, 'b', label=r'$x$')
        self.plots_part_0[0, 1].plot(self.t, zeros, 'r--', label=r'$e=0$', lw=self.LINE_WIDTH)
        self.plots_part_0[0, 1].plot(self.t, e_x, 'b', label=r'$x_{error}$')
        self.plots_part_0[1, 0].plot(self.t, self.y_ref, 'r--', label=r'$y_{ref}$', lw=self.LINE_WIDTH)
        self.plots_part_0[1, 0].plot(self.t, self.y, 'b', label=r'$y$')
        self.plots_part_0[1, 1].plot(self.t, zeros, 'r--', label=r'$e=0$', lw=self.LINE_WIDTH)
        self.plots_part_0[1, 1].plot(self.t, e_y, 'b', label=r'$y_{error}$')

        e_theta = [(b_i - a_i) for a_i , b_i in zip(self.theta, self.theta_ref)]
        self.plots_part_1[0].plot(self.t, self.theta_ref, 'r--', label=r'$\theta_{ref}$', lw=self.LINE_WIDTH)
        self.plots_part_1[0].plot(self.t, self.theta, 'b', label=r'$\theta$')
        self.plots_part_1[1].plot(self.t, zeros, 'r--', label=r'$e=0$', lw=self.LINE_WIDTH)
        self.plots_part_1[1].plot(self.t, e_theta, 'b', label=r'$\theta_{error}$')
        self.plots_part_1[2].plot(self.x_ref, self.y_ref, 'r--', label=r'${\rm reference}$', lw=self.LINE_WIDTH)
        self.plots_part_1[2].plot(self.x, self.y, 'b', label=r'${\rm followed}$')

        self.plots_part_2[0].plot(self.t, self.v_c, 'b', label=r'$v_{c}$', lw=self.LINE_WIDTH)
        self.plots_part_2[1].plot(self.t, self.w_c, 'b', label=r'$\omega_{c}$', lw=self.LINE_WIDTH)

        self.decorate_plot(self.plots_part_0[0, 0], r'$x\ {\rm and}\ x_{ref}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$x[{\rm m}]$')
        self.decorate_plot(self.plots_part_0[0, 1], r'$x_{error}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$x_{error}[{\rm m}]$')
        self.decorate_plot(self.plots_part_0[1, 0], r'$y\ {\rm and}\ y_{ref}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$y[{\rm m}]$')
        self.decorate_plot(self.plots_part_0[1, 1], r'$y_{error}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$y_{error}[{\rm m}]$')

        self.decorate_plot(self.plots_part_1[0], r'$\theta,\ \theta_{ref}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$\theta[{\rm rad}]$')
        self.decorate_plot(self.plots_part_1[1], r'$\theta_{error}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$\theta_{error}[{\rm rad}]$')
        self.decorate_plot(self.plots_part_1[2], r'${\rm followed\ trajectory\ vs\ reference\ trajectory}$',
                           r'$x[{\rm m}]$', r'$y[{\rm m}]$')

        self.decorate_plot(self.plots_part_2[0], r'$v_{c}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$v_{c}[{\rm m/s}]$')
        self.decorate_plot(self.plots_part_2[1], r'$\omega_{c}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$\omega_{c}[{\rm rad/s}]$')

        title = ''
        if CONTROLLER == 'euler':
            title = r'${\rm Euler\ method\ controller}\ $'
        elif CONTROLLER == 'pid':
            title = r'${\rm PID\ controller}\ $'

        self.fig_part_0.suptitle(title + r'${\rm results - }\ x\ {\rm and}\ y$', fontsize=self.FIGURE_TITLE_SIZE)
        self.fig_part_1.suptitle(title + r'${\rm results - }\ \theta\ {\rm and\ trajectory}$',
                                 fontsize=self.FIGURE_TITLE_SIZE)
        self.fig_part_2.suptitle(title + r'${\rm results - }\ v_{c}\ {\rm and}\ \omega_{c}$',
                                 fontsize=self.FIGURE_TITLE_SIZE)

        plt.show()

    def export_results(self):
        data_to_export = {
            't.txt': self.t,
            'x.txt': self.x,
            'x_ref.txt': self.x_ref,
            'y.txt': self.y,
            'y_ref.txt': self.y_ref,
            'theta.txt': self.theta,
            'theta_ref.txt': self.theta_ref,
            'v_c.txt': self.v_c,
            'w_c.txt':self.w_c,
        }

        path = RESULTS_DIRECTORY + CONTROLLER + '/' + TRAJECTORY + '/'
        for file_name, a_list in data_to_export.items():
            self.export_list(path + file_name, a_list)

    def export_list(self, path_to_file, a_list):
        if not exists(dirname(path_to_file)):
            try:
                makedirs(dirname(path_to_file))
            except OSError as exc:  # Guard against race condition
                if exc.errno != EEXIST:
                    raise

        with open(path_to_file, 'w') as file:
            for e in a_list:
                file.write(str(e) + '\n')
<commit_msg>Share x-axis<commit_after>#!/usr/bin/env python
# coding=utf-8
from __future__ import unicode_literals
import matplotlib.pyplot as plt
from errno import EEXIST
from os import makedirs

from os.path import exists, dirname

from constants import STEPS, DELTA_T, CONTROLLER, RESULTS_DIRECTORY, TRAJECTORY
from trajectory import create_trajectory

class Plotter:
    def __init__(self):
        trajectory = create_trajectory()
        self.t = [i * DELTA_T for i in range(STEPS)]
        self.x_ref = [trajectory.get_position_at(i * DELTA_T).x for i in range(STEPS)]
        self.y_ref = [trajectory.get_position_at(i * DELTA_T).y for i in range(STEPS)]
        self.x = []
        self.y = []
        self.theta = []
        self.theta_ref = []
        self.v_c = []
        self.w_c = []

        self.fig_part_0, self.plots_part_0 = plt.subplots(2, 2, sharex=True)
        # self.fig_part_1 = plt.figure()
        self.fig_part_1, self.plots_part_1 = plt.subplots(2, 2, sharex = True)
        #     plt.subplot(221),
        #     plt.subplot(223),
        #     plt.subplot(122),
        # ]

        self.fig_part_2, self.plots_part_2 = plt.subplots(1, 2)

        self.LINE_WIDTH = 2
        self.FIGURE_TITLE_SIZE = 21
        self.PLOT_TITLE_SIZE = 19
        self.PLOT_AXIS_LABEL_SIZE = 17

    def add_point(self, pose):
        self.x.append(pose.position.x)
        self.y.append(pose.position.y)

    def decorate_plot(self, plot, title, x_label, y_label):
        plot.set_title(title, fontsize=self.PLOT_TITLE_SIZE)
        plot.set_xlabel(x_label, fontsize=self.PLOT_AXIS_LABEL_SIZE)
        plot.set_ylabel(y_label, fontsize=self.PLOT_AXIS_LABEL_SIZE)
        plot.legend(loc=0)
        plot.grid()

    def plot_results(self):
        zeros = [0 for _ in range(STEPS)]
        e_x = [(b_i - a_i) for a_i , b_i in zip(self.x, self.x_ref)]
        e_y = [(b_i - a_i) for a_i , b_i in zip(self.y, self.y_ref)]
        self.plots_part_0[0, 0].plot(self.t, self.x_ref, 'r--', label=r'$x_{ref}$', lw=self.LINE_WIDTH)
        self.plots_part_0[0, 0].plot(self.t, self.x, 'b', label=r'$x$')
        self.plots_part_0[0, 1].plot(self.t, zeros, 'r--', label=r'$e=0$', lw=self.LINE_WIDTH)
        self.plots_part_0[0, 1].plot(self.t, e_x, 'b', label=r'$x_{error}$')
        self.plots_part_0[1, 0].plot(self.t, self.y_ref, 'r--', label=r'$y_{ref}$', lw=self.LINE_WIDTH)
        self.plots_part_0[1, 0].plot(self.t, self.y, 'b', label=r'$y$')
        self.plots_part_0[1, 1].plot(self.t, zeros, 'r--', label=r'$e=0$', lw=self.LINE_WIDTH)
        self.plots_part_0[1, 1].plot(self.t, e_y, 'b', label=r'$y_{error}$')

        self.plots_part_2[0].plot(self.t, self.v_c, 'b', label=r'$v_{c}$', lw=self.LINE_WIDTH)
        self.plots_part_2[1].plot(self.t, self.w_c, 'b', label=r'$\omega_{c}$', lw=self.LINE_WIDTH)

        e_theta = [(b_i - a_i) for a_i, b_i in zip(self.theta, self.theta_ref)]
        self.plots_part_1[0, 0].plot(self.t, self.theta_ref, 'r--', label=r'$\theta_{ref}$', lw=self.LINE_WIDTH)
        self.plots_part_1[0, 0].plot(self.t, self.theta, 'b', label=r'$\theta$')
        self.plots_part_1[1, 0].plot(self.t, zeros, 'r--', label=r'$e=0$', lw=self.LINE_WIDTH)
        self.plots_part_1[1, 0].plot(self.t, e_theta, 'b', label=r'$\theta_{error}$')

        plt.figure(self.fig_part_1.number)
        trajectory_plot = plt.subplot(122)
        trajectory_plot.plot(self.x_ref, self.y_ref, 'r--', label=r'${\rm reference}$', lw=self.LINE_WIDTH)
        trajectory_plot.plot(self.x, self.y, 'b', label=r'${\rm followed}$')

        self.decorate_plot(self.plots_part_0[0, 0], r'$x\ {\rm and}\ x_{ref}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$x[{\rm m}]$')
        self.decorate_plot(self.plots_part_0[0, 1], r'$x_{error}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$x_{error}[{\rm m}]$')
        self.decorate_plot(self.plots_part_0[1, 0], r'$y\ {\rm and}\ y_{ref}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$y[{\rm m}]$')
        self.decorate_plot(self.plots_part_0[1, 1], r'$y_{error}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$y_{error}[{\rm m}]$')

        self.decorate_plot(self.plots_part_1[0, 0], r'$\theta,\ \theta_{ref}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$\theta[{\rm rad}]$')
        self.decorate_plot(self.plots_part_1[1, 0], r'$\theta_{error}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$\theta_{error}[{\rm rad}]$')
        self.decorate_plot(trajectory_plot, r'${\rm followed\ trajectory\ vs\ reference\ trajectory}$',
                           r'$x[{\rm m}]$', r'$y[{\rm m}]$')

        self.decorate_plot(self.plots_part_2[0], r'$v_{c}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$v_{c}[{\rm m/s}]$')
        self.decorate_plot(self.plots_part_2[1], r'$\omega_{c}\ {\rm vs}\ t$', r'$t[{\rm s}]$', r'$\omega_{c}[{\rm rad/s}]$')

        title = ''
        if CONTROLLER == 'euler':
            title = r'${\rm Euler\ method\ controller}\ $'
        elif CONTROLLER == 'pid':
            title = r'${\rm PID\ controller}\ $'

        self.fig_part_0.suptitle(title + r'${\rm results - }\ x\ {\rm and}\ y$', fontsize=self.FIGURE_TITLE_SIZE)
        self.fig_part_2.suptitle(title + r'${\rm results - }\ v_{c}\ {\rm and}\ \omega_{c}$',
                                 fontsize=self.FIGURE_TITLE_SIZE)
        self.fig_part_1.suptitle(title + r'${\rm results - }\ \theta\ {\rm and\ trajectory}$',
                                 fontsize=self.FIGURE_TITLE_SIZE)

        plt.show()

    def export_results(self):
        data_to_export = {
            't.txt': self.t,
            'x.txt': self.x,
            'x_ref.txt': self.x_ref,
            'y.txt': self.y,
            'y_ref.txt': self.y_ref,
            'theta.txt': self.theta,
            'theta_ref.txt': self.theta_ref,
            'v_c.txt': self.v_c,
            'w_c.txt':self.w_c,
        }

        path = RESULTS_DIRECTORY + CONTROLLER + '/' + TRAJECTORY + '/'
        for file_name, a_list in data_to_export.items():
            self.export_list(path + file_name, a_list)

    def export_list(self, path_to_file, a_list):
        if not exists(dirname(path_to_file)):
            try:
                makedirs(dirname(path_to_file))
            except OSError as exc:  # Guard against race condition
                if exc.errno != EEXIST:
                    raise

        with open(path_to_file, 'w') as file:
            for e in a_list:
                file.write(str(e) + '\n')
<|endoftext|>


<commit_before><commit_msg>Skipping sshd tests because those will be fixed in PR 588.<commit_after><|endoftext|>