<commit_before># Copyright (C) 2010 Peter Teichman

import logging
import os
import re
import readline
import sys
import time

from brain import Brain

from cmdparse import Command

_DEFAULT_BRAIN_FILENAME = "cobe.brain"

log = logging.getLogger("cobe")

class InitCommand(Command):
    def __init__(self):
        Command.__init__(self, "init", summary="Initialize a new brain")

        self.add_option("", "--force", action="store_true")
        self.add_option("", "--order", type="int", default=5)

    def run(self, options, args):
        filename = _DEFAULT_BRAIN_FILENAME

        if os.path.exists(filename):
            if options.force:
                os.remove(filename)
            else:
                log.error("%s already exists!", filename)
                return

        Brain.init(filename, options.order)

def progress_generator(filename):
    s = os.stat(filename)
    size_left = s.st_size

    fd = open(filename)
    for line in fd.xreadlines():
        size_left = size_left - len(line)
        progress = 100 * (1. - (float(size_left) / float(s.st_size)))

        yield line, progress

    fd.close()

class LearnCommand(Command):
    def __init__(self):
        Command.__init__(self, "learn", summary="Learn a file of text")

    def run(self, options, args):
        if len(args) != 1:
            log.error("usage: learn <text file>")
            return

        b = Brain(_DEFAULT_BRAIN_FILENAME)

        for filename in args:
            now = time.time()
            print filename

            count = 0
            for line, progress in progress_generator(filename):
                show_progress = ((count % 100) == 0)

                if show_progress:
                    elapsed = time.time() - now
                    sys.stdout.write("\r%.0f%% (%d/s)" % (progress,
                                                          count/elapsed))
                    sys.stdout.flush()

                b.learn(line.strip(), commit=show_progress)
                count = count + 1

            elapsed = time.time() - now
            print "\r100%% (%d/s)" % (count/elapsed)

class LearnIrcLogCommand(Command):
    def __init__(self):
        Command.__init__(self, "learn-irc-log", summary="Learn an irc log")
        self.add_option("-i", "--ignore-nick",
                        action="append", dest="ignored_nicks",
                        help="Ignore an IRC nick (can be specified multiple times)")

    def run(self, options, args):
        if len(args) != 1:
            log.error("usage: learn-irc-log <irc log file>")
            return

        b = Brain(_DEFAULT_BRAIN_FILENAME)

        for filename in args:
            now = time.time()
            print filename

            count = 0
            for line, progress in progress_generator(filename):
                show_progress = ((count % 100) == 0)

                if show_progress:
                    elapsed = time.time() - now
                    sys.stdout.write("\r%.0f%% (%d/s)" % (progress,
                                                          count/elapsed))
                    sys.stdout.flush()

                msg = self._parse_irc_message(line.strip(),
                                              options.ignored_nicks)
                if msg:
                    b.learn(msg, commit=show_progress)
                    count = count + 1

            elapsed = time.time() - now
            print "\r100%% (%d/s)" % (count/elapsed)

    def _parse_irc_message(self, msg, ignored_nicks=None):
        # only match lines of the form "HH:MM <nick> message"
        match = re.match("\d+:\d+\s+<(.+?)>\s+(.*)", msg)
        if not match:
            return None

        nick = match.group(1)
        msg = match.group(2)

        if ignored_nicks is not None and nick in ignored_nicks:
            return None

        # strip "username: " at the beginning of messages
        msg = re.sub("^\S+[,:]\s+", "", msg)

        # strip kibot style '"asdf" --user, 06-oct-09' quotes
        msg = re.sub("\"(.*)\" --\S+,\s+\d+-\S+-\d+",
                     lambda m: m.group(1), msg)

        return msg

class ConsoleCommand(Command):
    def __init__(self):
        Command.__init__(self, "console", summary="Speak with Cobe.")

    def run(self, options, args):
        b = Brain(_DEFAULT_BRAIN_FILENAME)

        while True:
            try:
                cmd = raw_input("> ")
            except EOFError:
                print
                sys.exit(0)

            b.learn(cmd)
            print b.reply(cmd).capitalize().encode("utf-8")
<commit_msg>Allow learning of multiple files in sequence<commit_after># Copyright (C) 2010 Peter Teichman

import logging
import os
import re
import readline
import sys
import time

from brain import Brain

from cmdparse import Command

_DEFAULT_BRAIN_FILENAME = "cobe.brain"

log = logging.getLogger("cobe")

class InitCommand(Command):
    def __init__(self):
        Command.__init__(self, "init", summary="Initialize a new brain")

        self.add_option("", "--force", action="store_true")
        self.add_option("", "--order", type="int", default=5)

    def run(self, options, args):
        filename = _DEFAULT_BRAIN_FILENAME

        if os.path.exists(filename):
            if options.force:
                os.remove(filename)
            else:
                log.error("%s already exists!", filename)
                return

        Brain.init(filename, options.order)

def progress_generator(filename):
    s = os.stat(filename)
    size_left = s.st_size

    fd = open(filename)
    for line in fd.xreadlines():
        size_left = size_left - len(line)
        progress = 100 * (1. - (float(size_left) / float(s.st_size)))

        yield line, progress

    fd.close()

class LearnCommand(Command):
    def __init__(self):
        Command.__init__(self, "learn", summary="Learn a file of text")

    def run(self, options, args):
        if len(args) == 0:
            log.error("usage: learn <text file>")
            return

        b = Brain(_DEFAULT_BRAIN_FILENAME)

        for filename in args:
            now = time.time()
            print filename

            count = 0
            for line, progress in progress_generator(filename):
                show_progress = ((count % 100) == 0)

                if show_progress:
                    elapsed = time.time() - now
                    sys.stdout.write("\r%.0f%% (%d/s)" % (progress,
                                                          count/elapsed))
                    sys.stdout.flush()

                b.learn(line.strip(), commit=show_progress)
                count = count + 1

            elapsed = time.time() - now
            print "\r100%% (%d/s)" % (count/elapsed)

class LearnIrcLogCommand(Command):
    def __init__(self):
        Command.__init__(self, "learn-irc-log", summary="Learn an irc log")
        self.add_option("-i", "--ignore-nick",
                        action="append", dest="ignored_nicks",
                        help="Ignore an IRC nick (can be specified multiple times)")

    def run(self, options, args):
        if len(args) == 0:
            log.error("usage: learn-irc-log <irc log file>")
            return

        b = Brain(_DEFAULT_BRAIN_FILENAME)

        for filename in args:
            now = time.time()
            print filename

            count = 0
            for line, progress in progress_generator(filename):
                show_progress = ((count % 100) == 0)

                if show_progress:
                    elapsed = time.time() - now
                    sys.stdout.write("\r%.0f%% (%d/s)" % (progress,
                                                          count/elapsed))
                    sys.stdout.flush()

                msg = self._parse_irc_message(line.strip(),
                                              options.ignored_nicks)
                if msg:
                    b.learn(msg, commit=show_progress)
                    count = count + 1

            elapsed = time.time() - now
            print "\r100%% (%d/s)" % (count/elapsed)

    def _parse_irc_message(self, msg, ignored_nicks=None):
        # only match lines of the form "HH:MM <nick> message"
        match = re.match("\d+:\d+\s+<(.+?)>\s+(.*)", msg)
        if not match:
            return None

        nick = match.group(1)
        msg = match.group(2)

        if ignored_nicks is not None and nick in ignored_nicks:
            return None

        # strip "username: " at the beginning of messages
        msg = re.sub("^\S+[,:]\s+", "", msg)

        # strip kibot style '"asdf" --user, 06-oct-09' quotes
        msg = re.sub("\"(.*)\" --\S+,\s+\d+-\S+-\d+",
                     lambda m: m.group(1), msg)

        return msg

class ConsoleCommand(Command):
    def __init__(self):
        Command.__init__(self, "console", summary="Speak with Cobe.")

    def run(self, options, args):
        b = Brain(_DEFAULT_BRAIN_FILENAME)

        while True:
            try:
                cmd = raw_input("> ")
            except EOFError:
                print
                sys.exit(0)

            b.learn(cmd)
            print b.reply(cmd).capitalize().encode("utf-8")
<|endoftext|>


<commit_before>import argparse

from flask import Flask, jsonify, request
from flask import json
from gevent.pywsgi import WSGIServer

app = Flask(__name__)
site_data = {}
data_file_path = 'us_data.json'

def load_data():
    with open(data_file_path, 'r') as data_file:
        global site_data
        site_data = json.loads(data_file.read())

def get_site_data(system_name, body_name):
    global site_data
    if system_name is None and body_name is None:
        return site_data
    for site in site_data:
        if system_name.lower() == site['system'].lower() and body_name.lower() == site['body'].lower():
            return site
    return site_data

@app.route('/v1/unknown-site')
def unknown_site_data():
    system_name = request.args.get('system')
    body_name = request.args.get('body')
    return_site_data = get_site_data(system_name=system_name, body_name=body_name)
    return jsonify(
        {
            'data': return_site_data
        }
    )

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--debug',
        dest='debug',
        action='store_true',
        default=False,
        help='Run in debug mode')
    args = parser.parse_args()
    load_data()
    port = 10000
    if args.debug:
        app.run(host='127.0.0.1', port=port, debug=True, threaded=True)
    else:
        http_server = WSGIServer(('127.0.0.1', port), app)
        http_server.serve_forever()
<commit_msg>Fixed incorrect route name<commit_after>import argparse

from flask import Flask, jsonify, request
from flask import json
from gevent.pywsgi import WSGIServer

app = Flask(__name__)
site_data = {}
data_file_path = 'us_data.json'

def load_data():
    with open(data_file_path, 'r') as data_file:
        global site_data
        site_data = json.loads(data_file.read())

def get_site_data(system_name, body_name):
    global site_data
    if system_name is None and body_name is None:
        return site_data
    for site in site_data:
        if system_name.lower() == site['system'].lower() and body_name.lower() == site['body'].lower():
            return site
    return site_data

@app.route('/v1/unknown-sites')
def unknown_site_data():
    system_name = request.args.get('system')
    body_name = request.args.get('body')
    return_site_data = get_site_data(system_name=system_name, body_name=body_name)
    return jsonify(
        {
            'data': return_site_data
        }
    )

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--debug',
        dest='debug',
        action='store_true',
        default=False,
        help='Run in debug mode')
    args = parser.parse_args()
    load_data()
    port = 10000
    if args.debug:
        app.run(host='127.0.0.1', port=port, debug=True, threaded=True)
    else:
        http_server = WSGIServer(('127.0.0.1', port), app)
        http_server.serve_forever()
<|endoftext|>


<commit_before>#!/usr/bin/python
#    Copyright 2013, 2014 Joshua Charles Campbell, Alex Wilson, Eddie Santos
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from unnaturalcode.ucUtil import *
from unnaturalcode.unnaturalCode import *
from logging import debug, info, warning, error
import urllib
import re

allWhitespace = re.compile('^\s+$')
whitespace = re.compile('\s')

class genericLexeme(ucLexeme):

    @classmethod
    def stringify_build(cls, t, v):
        """
        Stringify a lexeme: produce a string describing it. For MITLM the
        string can't contain any whitespace.
        """
        if t is None:
            t = "UNKNOWN"
        if len(v) > 20:
            v = v[0:20]
        # quote all whitespace, colons, backslashes, control and unicode, and quotes
        v = urllib.quote(v, "!#$%&()*+,-./;<=>?@[]^_`{|}~")
        assert(re.match('\w+$', t) is not none), "Lexeme type should be a word."
        return t + ":" + v

class genericSource(ucSource):

    lexemeClass = genericLexeme

    def deLex(self):
        line = 1
        col = 0
        src = ""
        for l in self:
            for i in range(line, l.start.line):
                src += os.linesep
                col = 0
                line += 1
            for i in range(col, l.start.col):
                src += " "
                col += 1
            src += l.val
            col += len(l.val)
            nls = l.val.count(os.linesep)
            if (nls > 0):
                line += nls
                col = len(l.val.splitlines().pop())
        return src

    def scrubbed(self):
        """Clean up generic source code removing extra whitespace tokens and comments"""
        ls = copy(self)
        assert len(ls)
        i = 0
        r = []
        for i in range(0, len(ls)):
            #if allWhitespace.match(ls[i].val) and i < len(ls)-1 and allWhitespace.match(ls[i+1]):
                #continue
            if allWhitespace.match(ls[i].val):
                continue
            else:
                val = ls[i].val
                val = whitespace.sub("", val)
                r.append(ls[i].__class__((ls[i][0], val, ls[i][2], ls[i][3], ls[i][4])))
        assert len(r)
        return genericSource(r)

<commit_msg>Make assertion error more verbose<commit_after>#!/usr/bin/python
#    Copyright 2013, 2014 Joshua Charles Campbell, Alex Wilson, Eddie Santos
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

from unnaturalcode.ucUtil import *
from unnaturalcode.unnaturalCode import *
from logging import debug, info, warning, error
import urllib
import re

allWhitespace = re.compile('^\s+$')
whitespace = re.compile('\s')

class genericLexeme(ucLexeme):

    @classmethod
    def stringify_build(cls, t, v):
        """
        Stringify a lexeme: produce a string describing it. For MITLM the
        string can't contain any whitespace.
        """
        if t is None:
            t = "UNKNOWN"
        if len(v) > 20:
            v = v[0:20]
        # quote all whitespace, colons, backslashes, control and unicode, and quotes
        v = urllib.quote(v, "!#$%&()*+,-./;<=>?@[]^_`{|}~")
        assert(re.match('\w+$', t) is not none), "Lexeme type should be a word %s %s." % (t, v)
        return t + ":" + v

class genericSource(ucSource):

    lexemeClass = genericLexeme

    def deLex(self):
        line = 1
        col = 0
        src = ""
        for l in self:
            for i in range(line, l.start.line):
                src += os.linesep
                col = 0
                line += 1
            for i in range(col, l.start.col):
                src += " "
                col += 1
            src += l.val
            col += len(l.val)
            nls = l.val.count(os.linesep)
            if (nls > 0):
                line += nls
                col = len(l.val.splitlines().pop())
        return src

    def scrubbed(self):
        """Clean up generic source code removing extra whitespace tokens and comments"""
        ls = copy(self)
        assert len(ls)
        i = 0
        r = []
        for i in range(0, len(ls)):
            #if allWhitespace.match(ls[i].val) and i < len(ls)-1 and allWhitespace.match(ls[i+1]):
                #continue
            if allWhitespace.match(ls[i].val):
                continue
            else:
                val = ls[i].val
                val = whitespace.sub("", val)
                r.append(ls[i].__class__((ls[i][0], val, ls[i][2], ls[i][3], ls[i][4])))
        assert len(r)
        return genericSource(r)

<|endoftext|>


<commit_before>""" The aim of this file is to contain all the function
and the main which have to do with emotion recognition, especially
with the Kanade database."""

import glob
import argparse
# import DimensionalityReduction
import cPickle as pickle
from sklearn import cross_validation

import os
import fnmatch
import matplotlib.image as io
import matplotlib.pyplot as plt
import numpy as np
from skimage.transform import resize
from skimage import io
from skimage import color
import os

import facedetection

import deepbelief as db
import restrictedBoltzmannMachine as rbm
from common import *

parser = argparse.ArgumentParser(description='digit recognition')
parser.add_argument('--rbmnesterov', dest='rbmnesterov',action='store_true', default=False,
                    help=("if true, rbms are trained using nesterov momentum"))
parser.add_argument('--save',dest='save',action='store_true', default=False,
                    help="if true, the network is serialized and saved")
parser.add_argument('--train',dest='train',action='store_true', default=False,
                    help=("if true, the network is trained from scratch from the"
                          "traning data"))
parser.add_argument('--rbm', dest='rbm',action='store_true', default=False,
                    help=("if true, the code for traning an rbm on the data is run"))
parser.add_argument('--db', dest='db',action='store_true', default=False,
                    help=("if true, the code for traning a deepbelief net on the"
                          "data is run"))
parser.add_argument('--trainSize', type=int, default=10000,
                    help='the number of tranining cases to be considered')
parser.add_argument('--testSize', type=int, default=1000,
                    help='the number of testing cases to be considered')
parser.add_argument('netFile', help="file where the serialized network should be saved")
parser.add_argument('--nesterov', dest='nesterov',action='store_true', default=False,
                    help=("if true, the deep belief net is trained using nesterov momentum"))
parser.add_argument('--rmsprop', dest='rmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the deep belief net."))
parser.add_argument('--rbmrmsprop', dest='rbmrmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the rbms."))
parser.add_argument('--cv', dest='cv',action='store_true', default=False,
                    help=("if true, do cross validation"))
parser.add_argument('--facedetection', dest='facedetection',action='store_true', default=False,
                    help=("if true, do face detection"))
parser.add_argument('--maxEpochs', type=int, default=1000,
                    help='the maximum number of supervised epochs')
parser.add_argument('--validation',dest='validation',action='store_true', default=False,
                    help="if true, the network is trained using a validation set")

# DEBUG mode?
parser.add_argument('--debug', dest='debug',action='store_false', default=False,
                    help=("if true, the deep belief net is ran in DEBUG mode"))

# Get the arguments of the program
args = parser.parse_args()

# Set the debug mode in the deep belief net
db.DEBUG = args.debug

SMALL_SIZE = ((40, 30))

def rbmEmotions(big=False, reconstructRandom=False):
  data, labels = readKanade(big)
  print "data.shape"
  print data.shape

  trainData = data[0:-1, :]

  # Train the network
  if args.train:
    # The number of hidden units is taken from a deep learning tutorial
    # The data are the values of the images have to be normalized before being
    # presented to the network
    nrVisible = len(data[0])
    nrHidden = 800
    # use 1 dropout to test the rbm for now
    net = rbm.RBM(nrVisible, nrHidden, 0.01, 1, 1, rmsprop=args.rbmrmsprop,
                  nesterov=args.rbmnesterov)
    net.train(trainData)
    t = visualizeWeights(net.weights.T, SMALL_SIZE, (10,10))
  else:
    # Take the saved network and use that for reconstructions
    f = open(args.netFile, "rb")
    t = pickle.load(f)
    net = pickle.load(f)
    f.close()

  # get a random image and see it looks like
  # if reconstructRandom:
  #   test = np.random.random_sample(test.shape)

  # Show the initial image first
  test = data[-1, :]
  print "test.shape"
  print test.shape

  plt.imshow(vectorToImage(test, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('initialface.png', transparent=True)

  recon = net.reconstruct(test.reshape(1, test.shape[0]))
  print recon.shape

  plt.imshow(vectorToImage(recon, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('reconstructface.png', transparent=True)

  # Show the weights and their form in a tile fashion
  # Plot the weights
  plt.imshow(t, cmap=plt.cm.gray)
  plt.axis('off')
  if args.rbmrmsprop:
    st='rmsprop'
  else:
    st = 'simple'
  plt.savefig('weights' + st + '.png', transparent=True)
  print "done"

  if args.save:
    f = open(args.netFile, "wb")
    pickle.dump(t, f)
    pickle.dump(net, f)

"""
  Arguments:
    big: should the big or small images be used?
"""
def deepBeliefKanadeCV(big=False):
  data, labels = readKanade(big)

  data, labels = shuffle(data, labels)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # TODO: try boosting for CV in order to increase the number of folds
  params =[(0.1, 0.1) , (0.1, 0.05), (0.05, 0.01), (0.05, 0.05)]
  unsupervisedData = buildUnsupervisedDataSet()

  kf = cross_validation.KFold(n=len(data), k=len(params))
  bestCorrect = 0
  bestProbs = 0

  fold = 0
  for train, test in kf:

    trainData = data[train]
    trainLabels = labels[train]

    # TODO: this might require more thought
    net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
               unsupervisedLearningRate=params[fold][0],
               supervisedLearningRate=params[fold][1],
               nesterovMomentum=args.nesterov,
               rbmNesterovMomentum=args.rbmnesterov,
               rmsprop=args.rmsprop,
               hiddenDropout=0.5,
               rbmHiddenDropout=0.5,
               visibleDropout=0.8,
               rbmVisibleDropout=1)

    net.train(trainData, trainLabels,
              maxEpochs=args.maxEpochs,
              validation=args.validation,
              unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0
    errorCases = []

    for i in xrange(len(test)):
      print "predicted"
      print "probs"
      print probs[i]
      print predicted[i]
      print "actual"
      actual = actualLabels[i]
      print np.argmax(actual)
      if predicted[i] == np.argmax(actual):
        correct += 1
      else:
        errorCases.append(i)

    print "correct for " + str(params[fold])
    print correct

    if bestCorrect < correct:
      bestCorrect = correct
      bestParam = params[fold]
      bestProbs = correct * 1.0 / len(test)

    fold += 1

  print "bestParam"
  print bestParam

  print "bestProbs"
  print bestProbs

# TODO: shuffle training data for minibatches
def deepBeliefKanade(big=False):
  data, labels = readKanade(big, None)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=5)
  for train, test in kf:
    break

  trainData = data[train]
  trainLabels = labels[train]

  # TODO: this might require more thought
  net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
             unsupervisedLearningRate=0.05,
             # is this not a bad learning rate?
             supervisedLearningRate=0.01,
             nesterovMomentum=args.nesterov,
             rbmNesterovMomentum=args.rbmnesterov,
             rmsprop=args.rmsprop,
             hiddenDropout=0.5,
             rbmHiddenDropout=0.5,
             visibleDropout=0.8,
             rbmVisibleDropout=1)

  unsupervisedData = buildUnsupervisedDataSet()

  net.train(trainData, trainLabels, maxEpochs=args.maxEpochs,
            validation=args.validation, unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    print "predicted"
    print "probs"
    print probs[i]
    print predicted[i]
    print "actual"
    actual = actualLabels[i]
    print np.argmax(actual)
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0/ len(test)

def buildUnsupervisedDataSet():
  return np.vstack((
    # readCroppedYale(),
    readAttData(),
    readJaffe(),
    # readNottingham(),
    readAberdeen()))

def readKanade(big=False, folds=None):
  if big:
    files = glob.glob('kanade_150*.pickle')
  else:
    files = glob.glob('kanade_f*.pickle')

  if not folds:
    folds = range(1, 6)

  # Read the data from them. Sort out the files that do not have
  # the folds that we want
  # TODO: do this better (with regex in the file name)
  # DO not reply on the order returned
  files = [ files[x -1] for x in folds]

  data = np.array([])
  labels = np.array([])
  # TODO: do LDA on the training data

  # TODO: do proper CV in which you use 4 folds for training and one for testing
  # at that time
  dataFolds = []
  labelFolds = []
  for filename in files:
    with open(filename, "rb") as  f:
      # Sort out the labels from the data
      # TODO: run the readKanade again tomorrow and change these idnices here
      dataAndLabels = pickle.load(f)
      foldData = dataAndLabels[:, 0:-1]
      print "foldData.shape"
      print foldData.shape
      foldLabels = dataAndLabels[:,-1]
      dataFolds.append(foldData)
      foldLabels = np.array(map(int, foldLabels))

      vectorLabels = labelsToVectors(foldLabels -1, 7)
      labelFolds.append(vectorLabels)

      print "foldLabels.shape"
      print vectorLabels.shape

  data =  np.vstack(tuple(dataFolds))
  labels = np.vstack(tuple(labelFolds))
  return data, labels

# TODO: get big, small as argument in order to be able to fit the resizing
def readCroppedYale():
  # PATH = "/data/mcr10/yaleb/CroppedYale"
  PATH = "/home/aela/uni/project/CroppedYale"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  # Filter out the ones that containt "amyes bient"
  imageFiles = [ x for x in imageFiles if not "Ambient" in x]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)

    images += [img.reshape(-1)]

  return np.array(images)

def readAttData():
  PATH = "/data/mcr10/att"
  # PATH = "/home/aela/uni/project/code/pics/cambrdige_pics"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)
    images += [img.reshape(-1)]

  return np.array(images)

def readAndCrop(path, extension, doRecognition, isColoured=False):
  dirforres = "detection-cropped"
  pathForCropped = os.path.join(path, dirforres)

  if doRecognition:
    if not os.path.exists(pathForCropped):
      os.makedirs(pathForCropped)

    imageFiles = [(os.path.join(dirpath, f), f)
      for dirpath, dirnames, files in os.walk(path)
      for f in fnmatch.filter(files, '*.' + extension)]

    images = []

    for fullPath, shortPath in imageFiles:
      # Do not do this for already cropped images
      if pathForCropped in fullPath:
        continue

      print fullPath
      img = io.imread(fullPath)
      if isColoured:
        img = color.rgb2gray(img)
        img = np.array(img * 255, dtype='uint8')

      face = facedetection.cropFace(img)
      if not face == None:
        # Only do the resizing once you are done with the cropping of the faces
        face = resize(face, SMALL_SIZE)
        # Check that you are always saving them in the right format
        print "face.min"
        print face.min()

        print "face.max"
        print face.max()

        assert face.min() >=0 and face.max() <=1
        images += [face.reshape(-1)]

        # Save faces as files
        croppedFileName = os.path.join(pathForCropped, shortPath)
        io.imsave(croppedFileName, face)

  else:
    images = []
    imageFiles = [os.path.join(dirpath, f)
      for dirpath, dirnames, files in os.walk(pathForCropped)
      for f in fnmatch.filter(files, '*.' + extension)]

    for f in imageFiles:
      img = io.imread(f)
      if not (img.min() >=0 and img.max()<=1.000008):
        img = img / 255.0
      images += [img.reshape(-1)]

  print len(images)
  return np.array(images)

# This needs some thought: remove the cropped folder from path?

def readJaffe():
  PATH = "/data/mcr10/jaffe"
  # PATH = "/home/aela/uni/project/jaffe"
  return readAndCrop(PATH , "tiff", args.facedetection, isColoured=False)

def readNottingham():
  PATH = "/home/aela/uni/project/nottingham"
  # PATH = "/data/mcr10/nottingham"
  return readAndCrop(PATH, "gif", args.facedetection, isColoured=False)

def readAberdeen():
  PATH = "/data/mcr10/Aberdeen"
  # PATH = "/home/aela/uni/project/Aberdeen"
  return readAndCrop(PATH, "jpg", args.facedetection, isColoured=True)

def main():
  # deepBeliefKanade()

  # readNottingham()
  # readCroppedYale()
  # readJaffe()
  # readAttData()
  # readAberdeen()
  if args.rbm:
    rbmEmotions()
  elif args.cv:
    deepBeliefKanadeCV()
  elif args.db:
    deepBeliefKanade()

# You can also group the emotions into positive and negative to see
# if you can get better results (probably yes)
if __name__ == '__main__':
  import random
  print "FIXING RANDOMNESS"
  random.seed(6)
  np.random.seed(6)
  main()<commit_msg>trying to fix the emotions reading cropped files<commit_after>""" The aim of this file is to contain all the function
and the main which have to do with emotion recognition, especially
with the Kanade database."""

import glob
import argparse
# import DimensionalityReduction
import cPickle as pickle
from sklearn import cross_validation

import os
import fnmatch
import matplotlib.image as io
import matplotlib.pyplot as plt
import numpy as np
from skimage.transform import resize
from skimage import io
from skimage import color
import os

import facedetection

import deepbelief as db
import restrictedBoltzmannMachine as rbm
from common import *

parser = argparse.ArgumentParser(description='digit recognition')
parser.add_argument('--rbmnesterov', dest='rbmnesterov',action='store_true', default=False,
                    help=("if true, rbms are trained using nesterov momentum"))
parser.add_argument('--save',dest='save',action='store_true', default=False,
                    help="if true, the network is serialized and saved")
parser.add_argument('--train',dest='train',action='store_true', default=False,
                    help=("if true, the network is trained from scratch from the"
                          "traning data"))
parser.add_argument('--rbm', dest='rbm',action='store_true', default=False,
                    help=("if true, the code for traning an rbm on the data is run"))
parser.add_argument('--db', dest='db',action='store_true', default=False,
                    help=("if true, the code for traning a deepbelief net on the"
                          "data is run"))
parser.add_argument('--trainSize', type=int, default=10000,
                    help='the number of tranining cases to be considered')
parser.add_argument('--testSize', type=int, default=1000,
                    help='the number of testing cases to be considered')
parser.add_argument('netFile', help="file where the serialized network should be saved")
parser.add_argument('--nesterov', dest='nesterov',action='store_true', default=False,
                    help=("if true, the deep belief net is trained using nesterov momentum"))
parser.add_argument('--rmsprop', dest='rmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the deep belief net."))
parser.add_argument('--rbmrmsprop', dest='rbmrmsprop',action='store_true', default=False,
                    help=("if true, rmsprop is used when training the rbms."))
parser.add_argument('--cv', dest='cv',action='store_true', default=False,
                    help=("if true, do cross validation"))
parser.add_argument('--facedetection', dest='facedetection',action='store_true', default=False,
                    help=("if true, do face detection"))
parser.add_argument('--maxEpochs', type=int, default=1000,
                    help='the maximum number of supervised epochs')
parser.add_argument('--validation',dest='validation',action='store_true', default=False,
                    help="if true, the network is trained using a validation set")

# DEBUG mode?
parser.add_argument('--debug', dest='debug',action='store_false', default=False,
                    help=("if true, the deep belief net is ran in DEBUG mode"))

# Get the arguments of the program
args = parser.parse_args()

# Set the debug mode in the deep belief net
db.DEBUG = args.debug

SMALL_SIZE = ((40, 30))

def rbmEmotions(big=False, reconstructRandom=False):
  data, labels = readKanade(big)
  print "data.shape"
  print data.shape

  trainData = data[0:-1, :]

  # Train the network
  if args.train:
    # The number of hidden units is taken from a deep learning tutorial
    # The data are the values of the images have to be normalized before being
    # presented to the network
    nrVisible = len(data[0])
    nrHidden = 800
    # use 1 dropout to test the rbm for now
    net = rbm.RBM(nrVisible, nrHidden, 0.01, 1, 1, rmsprop=args.rbmrmsprop,
                  nesterov=args.rbmnesterov)
    net.train(trainData)
    t = visualizeWeights(net.weights.T, SMALL_SIZE, (10,10))
  else:
    # Take the saved network and use that for reconstructions
    f = open(args.netFile, "rb")
    t = pickle.load(f)
    net = pickle.load(f)
    f.close()

  # get a random image and see it looks like
  # if reconstructRandom:
  #   test = np.random.random_sample(test.shape)

  # Show the initial image first
  test = data[-1, :]
  print "test.shape"
  print test.shape

  plt.imshow(vectorToImage(test, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('initialface.png', transparent=True)

  recon = net.reconstruct(test.reshape(1, test.shape[0]))
  print recon.shape

  plt.imshow(vectorToImage(recon, SMALL_SIZE), cmap=plt.cm.gray)
  plt.axis('off')
  plt.savefig('reconstructface.png', transparent=True)

  # Show the weights and their form in a tile fashion
  # Plot the weights
  plt.imshow(t, cmap=plt.cm.gray)
  plt.axis('off')
  if args.rbmrmsprop:
    st='rmsprop'
  else:
    st = 'simple'
  plt.savefig('weights' + st + '.png', transparent=True)
  print "done"

  if args.save:
    f = open(args.netFile, "wb")
    pickle.dump(t, f)
    pickle.dump(net, f)

"""
  Arguments:
    big: should the big or small images be used?
"""
def deepBeliefKanadeCV(big=False):
  data, labels = readKanade(big)

  data, labels = shuffle(data, labels)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # TODO: try boosting for CV in order to increase the number of folds
  params =[(0.1, 0.1) , (0.1, 0.05), (0.05, 0.01), (0.05, 0.05)]
  unsupervisedData = buildUnsupervisedDataSet()

  kf = cross_validation.KFold(n=len(data), k=len(params))
  bestCorrect = 0
  bestProbs = 0

  fold = 0
  for train, test in kf:

    trainData = data[train]
    trainLabels = labels[train]

    # TODO: this might require more thought
    net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
               unsupervisedLearningRate=params[fold][0],
               supervisedLearningRate=params[fold][1],
               nesterovMomentum=args.nesterov,
               rbmNesterovMomentum=args.rbmnesterov,
               rmsprop=args.rmsprop,
               hiddenDropout=0.5,
               rbmHiddenDropout=0.5,
               visibleDropout=0.8,
               rbmVisibleDropout=1)

    net.train(trainData, trainLabels,
              maxEpochs=args.maxEpochs,
              validation=args.validation,
              unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0
    errorCases = []

    for i in xrange(len(test)):
      print "predicted"
      print "probs"
      print probs[i]
      print predicted[i]
      print "actual"
      actual = actualLabels[i]
      print np.argmax(actual)
      if predicted[i] == np.argmax(actual):
        correct += 1
      else:
        errorCases.append(i)

    print "correct for " + str(params[fold])
    print correct

    if bestCorrect < correct:
      bestCorrect = correct
      bestParam = params[fold]
      bestProbs = correct * 1.0 / len(test)

    fold += 1

  print "bestParam"
  print bestParam

  print "bestProbs"
  print bestProbs

# TODO: shuffle training data for minibatches
def deepBeliefKanade(big=False):
  data, labels = readKanade(big, None)

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=5)
  for train, test in kf:
    break

  trainData = data[train]
  trainLabels = labels[train]

  # TODO: this might require more thought
  net = db.DBN(5, [1200, 1500, 1500, 1500, 7],
             unsupervisedLearningRate=0.05,
             # is this not a bad learning rate?
             supervisedLearningRate=0.01,
             nesterovMomentum=args.nesterov,
             rbmNesterovMomentum=args.rbmnesterov,
             rmsprop=args.rmsprop,
             hiddenDropout=0.5,
             rbmHiddenDropout=0.5,
             visibleDropout=0.8,
             rbmVisibleDropout=1)

  unsupervisedData = buildUnsupervisedDataSet()

  net.train(trainData, trainLabels, maxEpochs=args.maxEpochs,
            validation=args.validation, unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    print "predicted"
    print "probs"
    print probs[i]
    print predicted[i]
    print "actual"
    actual = actualLabels[i]
    print np.argmax(actual)
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0/ len(test)

def buildUnsupervisedDataSet():
  return np.vstack((
    # readCroppedYale(),
    readAttData(),
    readJaffe(),
    # readNottingham(),
    readAberdeen()))

def readKanade(big=False, folds=None):
  if big:
    files = glob.glob('kanade_150*.pickle')
  else:
    files = glob.glob('kanade_f*.pickle')

  if not folds:
    folds = range(1, 6)

  # Read the data from them. Sort out the files that do not have
  # the folds that we want
  # TODO: do this better (with regex in the file name)
  # DO not reply on the order returned
  files = [ files[x -1] for x in folds]

  data = np.array([])
  labels = np.array([])
  # TODO: do LDA on the training data

  # TODO: do proper CV in which you use 4 folds for training and one for testing
  # at that time
  dataFolds = []
  labelFolds = []
  for filename in files:
    with open(filename, "rb") as  f:
      # Sort out the labels from the data
      # TODO: run the readKanade again tomorrow and change these idnices here
      dataAndLabels = pickle.load(f)
      foldData = dataAndLabels[:, 0:-1]
      print "foldData.shape"
      print foldData.shape
      foldLabels = dataAndLabels[:,-1]
      dataFolds.append(foldData)
      foldLabels = np.array(map(int, foldLabels))

      vectorLabels = labelsToVectors(foldLabels -1, 7)
      labelFolds.append(vectorLabels)

      print "foldLabels.shape"
      print vectorLabels.shape

  data =  np.vstack(tuple(dataFolds))
  labels = np.vstack(tuple(labelFolds))
  return data, labels

# TODO: get big, small as argument in order to be able to fit the resizing
def readCroppedYale():
  # PATH = "/data/mcr10/yaleb/CroppedYale"
  PATH = "/home/aela/uni/project/CroppedYale"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  # Filter out the ones that containt "amyes bient"
  imageFiles = [ x for x in imageFiles if not "Ambient" in x]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)

    images += [img.reshape(-1)]

  return np.array(images)

def readAttData():
  PATH = "/data/mcr10/att"
  # PATH = "/home/aela/uni/project/code/pics/cambrdige_pics"

  imageFiles = [os.path.join(dirpath, f)
    for dirpath, dirnames, files in os.walk(PATH)
    for f in fnmatch.filter(files, '*.pgm')]

  images = []
  for f in imageFiles:
    img = io.imread(f)
    img = resize(img, SMALL_SIZE)
    images += [img.reshape(-1)]

  return np.array(images)

def readAndCrop(path, extension, doRecognition, isColoured=False):
  dirforres = "detection-cropped"
  pathForCropped = os.path.join(path, dirforres)

  if doRecognition:
    if not os.path.exists(pathForCropped):
      os.makedirs(pathForCropped)

    imageFiles = [(os.path.join(dirpath, f), f)
      for dirpath, dirnames, files in os.walk(path)
      for f in fnmatch.filter(files, '*.' + extension)]

    images = []

    for fullPath, shortPath in imageFiles:
      # Do not do this for already cropped images
      if pathForCropped in fullPath:
        continue

      print fullPath
      img = io.imread(fullPath)
      if isColoured:
        img = color.rgb2gray(img)
        img = np.array(img * 255, dtype='uint8')

      face = facedetection.cropFace(img)
      if not face == None:
        # Only do the resizing once you are done with the cropping of the faces
        face = resize(face, SMALL_SIZE)
        # Check that you are always saving them in the right format
        print "face.min"
        print face.min()

        print "face.max"
        print face.max()

        assert face.min() >=0 and face.max() <=1
        images += [face.reshape(-1)]

        # Save faces as files
        croppedFileName = os.path.join(pathForCropped, shortPath)
        io.imsave(croppedFileName, face)

  else:
    images = []
    imageFiles = [os.path.join(dirpath, f)
      for dirpath, dirnames, files in os.walk(pathForCropped)
      for f in fnmatch.filter(files, '*.' + extension)]

    for f in imageFiles:
      img = io.imread(f)
      if type(img[0,0]) == np.uint8:
        print "rescaling unit"
      # if not (img.min() >=0.0 and img.max()<=1.000008):
        img = img / 255.0
      images += [img.reshape(-1)]

  print len(images)
  return np.array(images)

# This needs some thought: remove the cropped folder from path?

def readJaffe():
  PATH = "/data/mcr10/jaffe"
  # PATH = "/home/aela/uni/project/jaffe"
  return readAndCrop(PATH , "tiff", args.facedetection, isColoured=False)

def readNottingham():
  PATH = "/home/aela/uni/project/nottingham"
  # PATH = "/data/mcr10/nottingham"
  return readAndCrop(PATH, "gif", args.facedetection, isColoured=False)

def readAberdeen():
  PATH = "/data/mcr10/Aberdeen"
  # PATH = "/home/aela/uni/project/Aberdeen"
  return readAndCrop(PATH, "jpg", args.facedetection, isColoured=True)

def main():
  # deepBeliefKanade()

  # readNottingham()
  # readCroppedYale()
  # readJaffe()
  # readAttData()
  # readAberdeen()
  if args.rbm:
    rbmEmotions()
  elif args.cv:
    deepBeliefKanadeCV()
  elif args.db:
    deepBeliefKanade()

# You can also group the emotions into positive and negative to see
# if you can get better results (probably yes)
if __name__ == '__main__':
  import random
  print "FIXING RANDOMNESS"
  random.seed(6)
  np.random.seed(6)
  main()<|endoftext|>


<commit_before>import os
import warnings

import nibabel as nb
import numpy as np

from ...utils.misc import package_check
from ...utils.filemanip import split_filename

try:
    package_check('nipy')
except Exception, e:
    warnings.warn('nipy not installed')
else:
    from nipy.labs.mask import compute_mask
    from nipy.algorithms.registration import FmriRealign4d as FR4d
    from nipy import save_image

from ..base import (TraitedSpec, BaseInterface, traits,
                    BaseInterfaceInputSpec, isdefined, File,
                    InputMultiPath, OutputMultiPath)

class ComputeMaskInputSpec(BaseInterfaceInputSpec):
    mean_volume = File(exists=True, mandatory=True,
    desc="mean EPI image, used to compute the threshold for the mask")
    reference_volume = File(exists=True,
                       desc=("reference volume used to compute the mask. "
                             "If none is give, the mean volume is used."))
    m = traits.Float(desc="lower fraction of the histogram to be discarded")
    M = traits.Float(desc="upper fraction of the histogram to be discarded")
    cc = traits.Bool(desc="Keep only the largest connected component")

class ComputeMaskOutputSpec(TraitedSpec):
    brain_mask = File(exists=True)

class ComputeMask(BaseInterface):
    input_spec = ComputeMaskInputSpec
    output_spec = ComputeMaskOutputSpec

    def _run_interface(self, runtime):

        args = {}
        for key in [k for k, _ in self.inputs.items()
                    if k not in BaseInterfaceInputSpec().trait_names()]:
            value = getattr(self.inputs, key)
            if isdefined(value):
                if key in ['mean_volume', 'reference_volume']:
                    nii = nb.load(value)
                    value = nii.get_data()
                args[key] = value

        brain_mask = compute_mask(**args)
        _, name, ext = split_filename(self.inputs.mean_volume)
        self._brain_mask_path = os.path.abspath("%s_mask.%s" % (name, ext))
        nb.save(nb.Nifti1Image(brain_mask.astype(np.uint8),
                nii.get_affine()), self._brain_mask_path)

        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs["brain_mask"] = self._brain_mask_path
        return outputs

class FmriRealign4dInputSpec(BaseInterfaceInputSpec):

    in_file = InputMultiPath(exists=True,
                             mandatory=True,
                             desc="File to realign")
    tr = traits.Float(desc="TR in seconds",
                      mandatory=True)
    slice_order = traits.Either(traits.List(traits.Int),
                  traits.Enum("ascending", "descending"),
                  mandatory=True, desc='slice order')
    interleaved = traits.Bool(desc="True if interleaved",
                  mandatory=True)
    tr_slices = traits.Float(desc="TR slices")
    start = traits.Float(0.0, usedefault=True,
                         desc="time offset into TR to align slices to")
    time_interp = traits.Bool(True, usedefault=True,
                    desc="Assume smooth changes across time e.g., fmri series")
    loops = traits.Int(5, usedefault=True, desc="loops within each run")
    between_loops = traits.Int(5, usedefault=True, desc="loops used to \
                                                          realign different \
                                                          runs")
    speedup = traits.Int(5, usedefault=True, desc="successive image \
                                                    sub-sampling factors \
                                                    for acceleration")

class FmriRealign4dOutputSpec(TraitedSpec):

    out_file = OutputMultiPath(File(exists=True), desc="Realigned files")
    par_file = OutputMultiPath(File(exists=True), desc="Motion parameter files")

class FmriRealign4d(BaseInterface):
    """Simultaneous motion and slice timing correction algorithm

    This interface wraps nipy's FmriRealign4d algorithm [1]_.

    Examples
    --------
    >>> from nipype.interfaces.nipy.preprocess import FmriRealign4d
    >>> realigner = FmriRealign4d()
    >>> realigner.inputs.in_file = ['functional.nii']
    >>> realigner.inputs.tr = 2
    >>> realigner.inputs.slice_order = 'ascending'
    >>> realigner.inputs.interleaved = True
    >>> res = realigner.run() # doctest: +SKIP

    References
    ----------
    .. [1] Roche A. A four-dimensional registration algorithm with \
       application to joint correction of motion and slice timing \
       in fMRI. IEEE Trans Med Imaging. 2011 Aug;30(8):1546-54. DOI_.

    .. _DOI: http://dx.doi.org/10.1109/TMI.2011.2131152

    """

    input_spec = FmriRealign4dInputSpec
    output_spec = FmriRealign4dOutputSpec
    keywords = ['slice timing', 'motion correction']

    def _run_interface(self, runtime):

        all_ims = []

        for image in self.inputs.in_file:
            im = nb.load(image)
            im.affine = im.get_affine()
            all_ims.append(im)

        if not isdefined(self.inputs.tr_slices):
            TR_slices = None
        else:
            TR_slices = self.inputs.tr_slices

        R = FR4d(all_ims, tr=self.inputs.tr,
            slice_order=self.inputs.slice_order,
            interleaved=self.inputs.interleaved,
            tr_slices=TR_slices,
            time_interp=self.inputs.time_interp,
            start=self.inputs.start)

        R.estimate(loops=self.inputs.loops,
                   between_loops=self.inputs.between_loops,
                   speedup=self.inputs.speedup)

        corr_run = R.resample()
        self._out_file_path = []
        self._par_file_path = []

        for j, corr in enumerate(corr_run):
            self._out_file_path.append(os.path.abspath('corr_%s.nii.gz' %
            (split_filename(self.inputs.in_file[j])[1])))
            save_image(corr, self._out_file_path[j])

            self._par_file_path.append(os.path.abspath('%s.par' %
            (os.path.split(self.inputs.in_file[j])[1])))
            mfile = open(self._par_file_path[j], 'w')
            motion = R._transforms[j]
            #output a .par file that looks like fsl.mcflirt's .par file
            for i, mo in enumerate(motion):
                params = ['%.10f' % item for item in np.hstack((mo.rotation,
                                                             mo.translation))]
                string = ' '.join(params) + '\n'
                mfile.write(string)
            mfile.close()

        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['out_file'] = self._out_file_path
        outputs['par_file'] = self._par_file_path
        return outputs
<commit_msg>fix: removed interleaved option<commit_after>import os
import warnings

import nibabel as nb
import numpy as np

from ...utils.misc import package_check
from ...utils.filemanip import split_filename

try:
    package_check('nipy')
except Exception, e:
    warnings.warn('nipy not installed')
else:
    from nipy.labs.mask import compute_mask
    from nipy.algorithms.registration import FmriRealign4d as FR4d
    from nipy import save_image

from ..base import (TraitedSpec, BaseInterface, traits,
                    BaseInterfaceInputSpec, isdefined, File,
                    InputMultiPath, OutputMultiPath)

class ComputeMaskInputSpec(BaseInterfaceInputSpec):
    mean_volume = File(exists=True, mandatory=True,
    desc="mean EPI image, used to compute the threshold for the mask")
    reference_volume = File(exists=True,
                       desc=("reference volume used to compute the mask. "
                             "If none is give, the mean volume is used."))
    m = traits.Float(desc="lower fraction of the histogram to be discarded")
    M = traits.Float(desc="upper fraction of the histogram to be discarded")
    cc = traits.Bool(desc="Keep only the largest connected component")

class ComputeMaskOutputSpec(TraitedSpec):
    brain_mask = File(exists=True)

class ComputeMask(BaseInterface):
    input_spec = ComputeMaskInputSpec
    output_spec = ComputeMaskOutputSpec

    def _run_interface(self, runtime):

        args = {}
        for key in [k for k, _ in self.inputs.items()
                    if k not in BaseInterfaceInputSpec().trait_names()]:
            value = getattr(self.inputs, key)
            if isdefined(value):
                if key in ['mean_volume', 'reference_volume']:
                    nii = nb.load(value)
                    value = nii.get_data()
                args[key] = value

        brain_mask = compute_mask(**args)
        _, name, ext = split_filename(self.inputs.mean_volume)
        self._brain_mask_path = os.path.abspath("%s_mask.%s" % (name, ext))
        nb.save(nb.Nifti1Image(brain_mask.astype(np.uint8),
                nii.get_affine()), self._brain_mask_path)

        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs["brain_mask"] = self._brain_mask_path
        return outputs

class FmriRealign4dInputSpec(BaseInterfaceInputSpec):

    in_file = InputMultiPath(exists=True,
                             mandatory=True,
                             desc="File to realign")
    tr = traits.Float(desc="TR in seconds",
                      mandatory=True)
    slice_order = traits.Either(traits.List(traits.Int),
                  traits.Enum("ascending", "descending"),
                  mandatory=True, desc='slice order')
    tr_slices = traits.Float(desc="TR slices")
    start = traits.Float(0.0, usedefault=True,
                         desc="time offset into TR to align slices to")
    time_interp = traits.Bool(True, usedefault=True,
                    desc="Assume smooth changes across time e.g., fmri series")
    loops = traits.Int(5, usedefault=True, desc="loops within each run")
    between_loops = traits.Int(5, usedefault=True, desc="loops used to \
                                                          realign different \
                                                          runs")
    speedup = traits.Int(5, usedefault=True, desc="successive image \
                                                    sub-sampling factors \
                                                    for acceleration")

class FmriRealign4dOutputSpec(TraitedSpec):

    out_file = OutputMultiPath(File(exists=True), desc="Realigned files")
    par_file = OutputMultiPath(File(exists=True), desc="Motion parameter files")

class FmriRealign4d(BaseInterface):
    """Simultaneous motion and slice timing correction algorithm

    This interface wraps nipy's FmriRealign4d algorithm [1]_.

    Examples
    --------
    >>> from nipype.interfaces.nipy.preprocess import FmriRealign4d
    >>> realigner = FmriRealign4d()
    >>> realigner.inputs.in_file = ['functional.nii']
    >>> realigner.inputs.tr = 2
    >>> realigner.inputs.slice_order = 'ascending'
    >>> res = realigner.run() # doctest: +SKIP

    References
    ----------
    .. [1] Roche A. A four-dimensional registration algorithm with \
       application to joint correction of motion and slice timing \
       in fMRI. IEEE Trans Med Imaging. 2011 Aug;30(8):1546-54. DOI_.

    .. _DOI: http://dx.doi.org/10.1109/TMI.2011.2131152

    """

    input_spec = FmriRealign4dInputSpec
    output_spec = FmriRealign4dOutputSpec
    keywords = ['slice timing', 'motion correction']

    def _run_interface(self, runtime):

        all_ims = []

        for image in self.inputs.in_file:
            im = nb.load(image)
            im.affine = im.get_affine()
            all_ims.append(im)

        if not isdefined(self.inputs.tr_slices):
            TR_slices = None
        else:
            TR_slices = self.inputs.tr_slices

        R = FR4d(all_ims, tr=self.inputs.tr,
            slice_order=self.inputs.slice_order,
            tr_slices=TR_slices,
            time_interp=self.inputs.time_interp,
            start=self.inputs.start)

        R.estimate(loops=self.inputs.loops,
                   between_loops=self.inputs.between_loops,
                   speedup=self.inputs.speedup)

        corr_run = R.resample()
        self._out_file_path = []
        self._par_file_path = []

        for j, corr in enumerate(corr_run):
            self._out_file_path.append(os.path.abspath('corr_%s.nii.gz' %
            (split_filename(self.inputs.in_file[j])[1])))
            save_image(corr, self._out_file_path[j])

            self._par_file_path.append(os.path.abspath('%s.par' %
            (os.path.split(self.inputs.in_file[j])[1])))
            mfile = open(self._par_file_path[j], 'w')
            motion = R._transforms[j]
            #output a .par file that looks like fsl.mcflirt's .par file
            for i, mo in enumerate(motion):
                params = ['%.10f' % item for item in np.hstack((mo.rotation,
                                                             mo.translation))]
                string = ' '.join(params) + '\n'
                mfile.write(string)
            mfile.close()

        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['out_file'] = self._out_file_path
        outputs['par_file'] = self._par_file_path
        return outputs
<|endoftext|>


<commit_before>from nipype.interfaces import afni

from nose.tools import assert_equal, assert_raises

def test_To3d():
    cmd = afni.To3d()
    yield assert_equal, cmd.cmdline, 'to3d'
    # datatype
    cmd = afni.To3d(datatype='anat')
    yield assert_equal, cmd.cmdline, 'to3d -anat'
    cmd = afni.To3d(datatype='epan')
    yield assert_equal, cmd.cmdline, 'to3d -epan'
    # datum
    cmd = afni.To3d()
    cmd.inputs.datum = 'float'
    yield assert_equal, cmd.cmdline, 'to3d -datum float'
    # session
    cmd = afni.To3d()
    cmd.inputs.session = '/home/bobama'
    yield assert_equal, cmd.cmdline, 'to3d -session /home/bobama'
    # prefix
    cmd = afni.To3d(prefix='foo.nii.gz')
    yield assert_equal, cmd.cmdline, 'to3d -prefix foo.nii.gz'
    # infiles
    cmd = afni.To3d(infiles='/data/*.dcm')
    yield assert_equal, cmd.cmdline, 'to3d /data/*.dcm'
    # skip_outliers
    cmd = afni.To3d(skip_outliers=True)
    yield assert_equal, cmd.cmdline, 'to3d -skip_outliers'
    # assume_dicom_mosaic
    cmd = afni.To3d(assume_dicom_mosaic=True)
    yield assert_equal, cmd.cmdline, 'to3d -assume_dicom_mosaic'

    # Test slice time params
    cmd = afni.To3d()
    td = dict(slice_order='zt', nz=12, nt=150, TR=2000, tpattern='alt+z')
    cmd.inputs.time_dependencies = td
    yield assert_equal, cmd.cmdline, 'to3d -time:zt 12 150 2000 alt+z'

    cmd = afni.To3d()
    td = dict(slice_order='tz', nt=150, nz=12, TR=2000, tpattern='alt+z')
    cmd.inputs.time_dependencies = td
    yield assert_equal, cmd.cmdline, 'to3d -time:tz 150 12 2000 alt+z'

    # These tests fill fail because they do not specify all required
    # args for the time_dependencies
    # dict(slice_order='zt', nz=12, nt=150, TR=2000, tpattern='alt+z')
    cmd = afni.To3d()
    cmd.inputs.time_dependencies = dict()
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # only slice_order
    cmd.inputs.time_dependencies = dict(slice_order='zt')
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # only slice_order
    cmd.inputs.time_dependencies = dict(slice_order='tz')
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order and nz
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12)
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order, nz, nt
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150)
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order, nz, nt, TR
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150,
                                        TR=2000)
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order, nz, nt, tpattern
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150,
                                        tpattern='alt+z')
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # provide too many parameters for time_dependencies
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150,
                                        tpattern='alt+z', TR=2000,
                                        username='foo')
    yield assert_raises, AttributeError, getattr, cmd, 'cmdline'

<commit_msg>Add tests for 3drefit<commit_after>from nipype.testing import *

from nipype.interfaces import afni

def test_To3d():
    cmd = afni.To3d()
    yield assert_equal, cmd.cmdline, 'to3d'
    # datatype
    cmd = afni.To3d(datatype='anat')
    yield assert_equal, cmd.cmdline, 'to3d -anat'
    cmd = afni.To3d(datatype='epan')
    yield assert_equal, cmd.cmdline, 'to3d -epan'
    # datum
    cmd = afni.To3d()
    cmd.inputs.datum = 'float'
    yield assert_equal, cmd.cmdline, 'to3d -datum float'
    # session
    cmd = afni.To3d()
    cmd.inputs.session = '/home/bobama'
    yield assert_equal, cmd.cmdline, 'to3d -session /home/bobama'
    # prefix
    cmd = afni.To3d(prefix='foo.nii.gz')
    yield assert_equal, cmd.cmdline, 'to3d -prefix foo.nii.gz'
    # infiles
    cmd = afni.To3d(infiles='/data/*.dcm')
    yield assert_equal, cmd.cmdline, 'to3d /data/*.dcm'
    # skip_outliers
    cmd = afni.To3d(skip_outliers=True)
    yield assert_equal, cmd.cmdline, 'to3d -skip_outliers'
    # assume_dicom_mosaic
    cmd = afni.To3d(assume_dicom_mosaic=True)
    yield assert_equal, cmd.cmdline, 'to3d -assume_dicom_mosaic'

    # Test slice time params
    cmd = afni.To3d()
    td = dict(slice_order='zt', nz=12, nt=150, TR=2000, tpattern='alt+z')
    cmd.inputs.time_dependencies = td
    yield assert_equal, cmd.cmdline, 'to3d -time:zt 12 150 2000 alt+z'

    cmd = afni.To3d()
    td = dict(slice_order='tz', nt=150, nz=12, TR=2000, tpattern='alt+z')
    cmd.inputs.time_dependencies = td
    yield assert_equal, cmd.cmdline, 'to3d -time:tz 150 12 2000 alt+z'

    # These tests fill fail because they do not specify all required
    # args for the time_dependencies
    # dict(slice_order='zt', nz=12, nt=150, TR=2000, tpattern='alt+z')
    cmd = afni.To3d()
    cmd.inputs.time_dependencies = dict()
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # only slice_order
    cmd.inputs.time_dependencies = dict(slice_order='zt')
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # only slice_order
    cmd.inputs.time_dependencies = dict(slice_order='tz')
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order and nz
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12)
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order, nz, nt
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150)
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order, nz, nt, TR
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150,
                                        TR=2000)
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # slice_order, nz, nt, tpattern
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150,
                                        tpattern='alt+z')
    yield assert_raises, KeyError, getattr, cmd, 'cmdline'
    # provide too many parameters for time_dependencies
    cmd.inputs.time_dependencies = dict(slice_order='zt', nz=12, nt=150,
                                        tpattern='alt+z', TR=2000,
                                        username='foo')
    yield assert_raises, AttributeError, getattr, cmd, 'cmdline'

def test_Threedrefit():
    cmd = afni.Threedrefit()
    yield assert_equal, cmd.cmdline, '3drefit'
    # deoblique
    cmd = afni.Threedrefit()
    cmd.inputs.deoblique = True
    yield assert_equal, cmd.cmdline, '3drefit -deoblique'
    # xorigin
    cmd = afni.Threedrefit()
    cmd.inputs.xorigin = 12.34
    yield assert_equal, cmd.cmdline, '3drefit -xorigin 12.34'
    # yorigin
    cmd = afni.Threedrefit()
    cmd.inputs.yorigin = 12.34
    yield assert_equal, cmd.cmdline, '3drefit -yorigin 12.34'
    # zorigin
    cmd = afni.Threedrefit()
    cmd.inputs.zorigin = 12.34
    yield assert_equal, cmd.cmdline, '3drefit -zorigin 12.34'

<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
from builtins import open

import os
from tempfile import mkstemp, mkdtemp
import warnings

from ...testing import assert_equal, assert_true, assert_false, TempFATFS
from ...utils.filemanip import (save_json, load_json,
                                    fname_presuffix, fnames_presuffix,
                                    hash_rename, check_forhash,
                                    copyfile, copyfiles,
                                    filename_to_list, list_to_filename,
                                    split_filename, get_related_files)

import numpy as np

def test_split_filename():
    res = split_filename('foo.nii')
    yield assert_equal, res, ('', 'foo', '.nii')
    res = split_filename('foo.nii.gz')
    yield assert_equal, res, ('', 'foo', '.nii.gz')
    res = split_filename('/usr/local/foo.nii.gz')
    yield assert_equal, res, ('/usr/local', 'foo', '.nii.gz')
    res = split_filename('../usr/local/foo.nii')
    yield assert_equal, res, ('../usr/local', 'foo', '.nii')
    res = split_filename('/usr/local/foo.a.b.c.d')
    yield assert_equal, res, ('/usr/local', 'foo.a.b.c', '.d')
    res = split_filename('/usr/local/')
    yield assert_equal, res, ('/usr/local', '', '')

def test_fname_presuffix():
    fname = 'foo.nii'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii'
    fname += '.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp', use_ext=False)
    yield assert_equal, pth, '/tmp/pre_foo_post'

def test_fnames_presuffix():
    fnames = ['foo.nii', 'bar.nii']
    pths = fnames_presuffix(fnames, 'pre_', '_post', '/tmp')
    yield assert_equal, pths, ['/tmp/pre_foo_post.nii', '/tmp/pre_bar_post.nii']

def test_hash_rename():
    new_name = hash_rename('foobar.nii', 'abc123')
    yield assert_equal, new_name, 'foobar_0xabc123.nii'
    new_name = hash_rename('foobar.nii.gz', 'abc123')
    yield assert_equal, new_name, 'foobar_0xabc123.nii.gz'

def test_check_forhash():
    fname = 'foobar'
    orig_hash = '_0x4323dbcefdc51906decd8edcb3327943'
    hashed_name = ''.join((fname, orig_hash, '.nii'))
    result, hash = check_forhash(hashed_name)
    yield assert_true, result
    yield assert_equal, hash, [orig_hash]
    result, hash = check_forhash('foobar.nii')
    yield assert_false, result
    yield assert_equal, hash, None

def _temp_analyze_files():
    """Generate temporary analyze file pair."""
    fd, orig_img = mkstemp(suffix='.img')
    orig_hdr = orig_img[:-4] + '.hdr'
    fp = open(orig_hdr, 'w+')
    fp.close()
    return orig_img, orig_hdr

def test_copyfile():
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    new_img = os.path.join(pth, 'newfile.img')
    new_hdr = os.path.join(pth, 'newfile.hdr')
    copyfile(orig_img, new_img)
    yield assert_true, os.path.exists(new_img)
    yield assert_true, os.path.exists(new_hdr)
    os.unlink(new_img)
    os.unlink(new_hdr)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_copyfile_true():
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    new_img = os.path.join(pth, 'newfile.img')
    new_hdr = os.path.join(pth, 'newfile.hdr')
    # Test with copy=True
    copyfile(orig_img, new_img, copy=True)
    yield assert_true, os.path.exists(new_img)
    yield assert_true, os.path.exists(new_hdr)
    os.unlink(new_img)
    os.unlink(new_hdr)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_copyfiles():
    orig_img1, orig_hdr1 = _temp_analyze_files()
    orig_img2, orig_hdr2 = _temp_analyze_files()
    pth, fname = os.path.split(orig_img1)
    new_img1 = os.path.join(pth, 'newfile.img')
    new_hdr1 = os.path.join(pth, 'newfile.hdr')
    pth, fname = os.path.split(orig_img2)
    new_img2 = os.path.join(pth, 'secondfile.img')
    new_hdr2 = os.path.join(pth, 'secondfile.hdr')
    newfiles = copyfiles([orig_img1, orig_img2], [new_img1, new_img2])
    yield assert_true, os.path.exists(new_img1)
    yield assert_true, os.path.exists(new_hdr1)
    yield assert_true, os.path.exists(new_img2)
    yield assert_true, os.path.exists(new_hdr2)
    # cleanup
    os.unlink(orig_img1)
    os.unlink(orig_hdr1)
    os.unlink(orig_img2)
    os.unlink(orig_hdr2)
    os.unlink(new_img1)
    os.unlink(new_hdr1)
    os.unlink(new_img2)
    os.unlink(new_hdr2)

def test_linkchain():
    if os.name is not 'posix':
        return
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    new_img1 = os.path.join(pth, 'newfile1.img')
    new_hdr1 = os.path.join(pth, 'newfile1.hdr')
    new_img2 = os.path.join(pth, 'newfile2.img')
    new_hdr2 = os.path.join(pth, 'newfile2.hdr')
    new_img3 = os.path.join(pth, 'newfile3.img')
    new_hdr3 = os.path.join(pth, 'newfile3.hdr')
    copyfile(orig_img, new_img1)
    yield assert_true, os.path.islink(new_img1)
    yield assert_true, os.path.islink(new_hdr1)
    copyfile(new_img1, new_img2, copy=True)
    yield assert_false, os.path.islink(new_img2)
    yield assert_false, os.path.islink(new_hdr2)
    yield assert_false, os.path.samefile(orig_img, new_img2)
    yield assert_false, os.path.samefile(orig_hdr, new_hdr2)
    copyfile(new_img1, new_img3, copy=True, use_hardlink=True)
    yield assert_false, os.path.islink(new_img3)
    yield assert_false, os.path.islink(new_hdr3)
    yield assert_true, os.path.samefile(orig_img, new_img3)
    yield assert_true, os.path.samefile(orig_hdr, new_hdr3)

    # Test that re-copying leaves files
    stat1 = os.stat(new_img1)
    stat2 = os.stat(new_img2)
    stat3 = os.stat(new_img3)
    # Same symlink
    copyfile(orig_img, new_img1)
    # Hash matches
    copyfile(new_img1, new_img2, copy=True)
    # Hardlinks
    copyfile(new_img1, new_img3, copy=True, use_hardlink=True)
    yield assert_equal, stat1, os.stat(new_img1)
    yield assert_equal, stat2, os.stat(new_img2)
    yield assert_equal, stat3, os.stat(new_img3)

    os.unlink(new_img1)
    os.unlink(new_hdr1)
    os.unlink(new_img2)
    os.unlink(new_hdr2)
    os.unlink(new_img3)
    os.unlink(new_hdr3)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_copyfallback():
    if os.name is not 'posix':
        return
    orig_img, orig_hdr = _temp_analyze_files()
    pth, imgname = os.path.split(orig_img)
    pth, hdrname = os.path.split(orig_hdr)
    try:
        fatfs = TempFATFS()
    except (IOError, OSError):
        warnings.warn('Fuse mount failed. copyfile fallback tests skipped.')
    else:
        with fatfs as fatdir:
            tgt_img = os.path.join(fatdir, imgname)
            tgt_hdr = os.path.join(fatdir, hdrname)
            for copy in (True, False):
                for use_hardlink in (True, False):
                    copyfile(orig_img, tgt_img, copy=copy,
                             use_hardlink=use_hardlink)
                    yield assert_true, os.path.exists(tgt_img)
                    yield assert_true, os.path.exists(tgt_hdr)
                    yield assert_false, os.path.islink(tgt_img)
                    yield assert_false, os.path.islink(tgt_hdr)
                    yield assert_false, os.path.samefile(orig_img, tgt_img)
                    yield assert_false, os.path.samefile(orig_hdr, tgt_hdr)
                    os.unlink(tgt_img)
                    os.unlink(tgt_hdr)
    finally:
        os.unlink(orig_img)
        os.unlink(orig_hdr)

def test_filename_to_list():
    x = filename_to_list('foo.nii')
    yield assert_equal, x, ['foo.nii']
    x = filename_to_list(['foo.nii'])
    yield assert_equal, x, ['foo.nii']
    x = filename_to_list(('foo', 'bar'))
    yield assert_equal, x, ['foo', 'bar']
    x = filename_to_list(12.34)
    yield assert_equal, x, None

def test_list_to_filename():
    x = list_to_filename(['foo.nii'])
    yield assert_equal, x, 'foo.nii'
    x = list_to_filename(['foo', 'bar'])
    yield assert_equal, x, ['foo', 'bar']

def test_json():
    # Simple roundtrip test of json files, just a sanity check.
    adict = dict(a='one', c='three', b='two')
    fd, name = mkstemp(suffix='.json')
    save_json(name, adict)  # save_json closes the file
    new_dict = load_json(name)
    os.unlink(name)
    yield assert_equal, sorted(adict.items()), sorted(new_dict.items())

def test_related_files():
    file1 = '/path/test.img'
    file2 = '/path/test.hdr'
    file3 = '/path/test.BRIK'
    file4 = '/path/test.HEAD'
    file5 = '/path/foo.nii'

    spm_files1 = get_related_files(file1)
    spm_files2 = get_related_files(file2)
    afni_files1 = get_related_files(file3)
    afni_files2 = get_related_files(file4)
    yield assert_equal, len(spm_files1), 3
    yield assert_equal, len(spm_files2), 3
    yield assert_equal, len(afni_files1), 2
    yield assert_equal, len(afni_files2), 2
    yield assert_equal, len(get_related_files(file5)), 1

    yield assert_true, '/path/test.hdr' in spm_files1
    yield assert_true, '/path/test.img' in spm_files1
    yield assert_true, '/path/test.mat' in spm_files1
    yield assert_true, '/path/test.hdr' in spm_files2
    yield assert_true, '/path/test.img' in spm_files2
    yield assert_true, '/path/test.mat' in spm_files2
    yield assert_true, '/path/test.BRIK' in afni_files1
    yield assert_true, '/path/test.HEAD' in afni_files1
    yield assert_true, '/path/test.BRIK' in afni_files2
    yield assert_true, '/path/test.HEAD' in afni_files2
<commit_msg>TEST: Separate recopy test, err_msg for debugging<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
from builtins import open

import os
from tempfile import mkstemp, mkdtemp
import warnings

from ...testing import assert_equal, assert_true, assert_false, TempFATFS
from ...utils.filemanip import (save_json, load_json,
                                    fname_presuffix, fnames_presuffix,
                                    hash_rename, check_forhash,
                                    copyfile, copyfiles,
                                    filename_to_list, list_to_filename,
                                    split_filename, get_related_files)

import numpy as np

def test_split_filename():
    res = split_filename('foo.nii')
    yield assert_equal, res, ('', 'foo', '.nii')
    res = split_filename('foo.nii.gz')
    yield assert_equal, res, ('', 'foo', '.nii.gz')
    res = split_filename('/usr/local/foo.nii.gz')
    yield assert_equal, res, ('/usr/local', 'foo', '.nii.gz')
    res = split_filename('../usr/local/foo.nii')
    yield assert_equal, res, ('../usr/local', 'foo', '.nii')
    res = split_filename('/usr/local/foo.a.b.c.d')
    yield assert_equal, res, ('/usr/local', 'foo.a.b.c', '.d')
    res = split_filename('/usr/local/')
    yield assert_equal, res, ('/usr/local', '', '')

def test_fname_presuffix():
    fname = 'foo.nii'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii'
    fname += '.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp', use_ext=False)
    yield assert_equal, pth, '/tmp/pre_foo_post'

def test_fnames_presuffix():
    fnames = ['foo.nii', 'bar.nii']
    pths = fnames_presuffix(fnames, 'pre_', '_post', '/tmp')
    yield assert_equal, pths, ['/tmp/pre_foo_post.nii', '/tmp/pre_bar_post.nii']

def test_hash_rename():
    new_name = hash_rename('foobar.nii', 'abc123')
    yield assert_equal, new_name, 'foobar_0xabc123.nii'
    new_name = hash_rename('foobar.nii.gz', 'abc123')
    yield assert_equal, new_name, 'foobar_0xabc123.nii.gz'

def test_check_forhash():
    fname = 'foobar'
    orig_hash = '_0x4323dbcefdc51906decd8edcb3327943'
    hashed_name = ''.join((fname, orig_hash, '.nii'))
    result, hash = check_forhash(hashed_name)
    yield assert_true, result
    yield assert_equal, hash, [orig_hash]
    result, hash = check_forhash('foobar.nii')
    yield assert_false, result
    yield assert_equal, hash, None

def _temp_analyze_files():
    """Generate temporary analyze file pair."""
    fd, orig_img = mkstemp(suffix='.img')
    orig_hdr = orig_img[:-4] + '.hdr'
    fp = open(orig_hdr, 'w+')
    fp.close()
    return orig_img, orig_hdr

def test_copyfile():
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    new_img = os.path.join(pth, 'newfile.img')
    new_hdr = os.path.join(pth, 'newfile.hdr')
    copyfile(orig_img, new_img)
    yield assert_true, os.path.exists(new_img)
    yield assert_true, os.path.exists(new_hdr)
    os.unlink(new_img)
    os.unlink(new_hdr)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_copyfile_true():
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    new_img = os.path.join(pth, 'newfile.img')
    new_hdr = os.path.join(pth, 'newfile.hdr')
    # Test with copy=True
    copyfile(orig_img, new_img, copy=True)
    yield assert_true, os.path.exists(new_img)
    yield assert_true, os.path.exists(new_hdr)
    os.unlink(new_img)
    os.unlink(new_hdr)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_copyfiles():
    orig_img1, orig_hdr1 = _temp_analyze_files()
    orig_img2, orig_hdr2 = _temp_analyze_files()
    pth, fname = os.path.split(orig_img1)
    new_img1 = os.path.join(pth, 'newfile.img')
    new_hdr1 = os.path.join(pth, 'newfile.hdr')
    pth, fname = os.path.split(orig_img2)
    new_img2 = os.path.join(pth, 'secondfile.img')
    new_hdr2 = os.path.join(pth, 'secondfile.hdr')
    newfiles = copyfiles([orig_img1, orig_img2], [new_img1, new_img2])
    yield assert_true, os.path.exists(new_img1)
    yield assert_true, os.path.exists(new_hdr1)
    yield assert_true, os.path.exists(new_img2)
    yield assert_true, os.path.exists(new_hdr2)
    # cleanup
    os.unlink(orig_img1)
    os.unlink(orig_hdr1)
    os.unlink(orig_img2)
    os.unlink(orig_hdr2)
    os.unlink(new_img1)
    os.unlink(new_hdr1)
    os.unlink(new_img2)
    os.unlink(new_hdr2)

def test_linkchain():
    if os.name is not 'posix':
        return
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    new_img1 = os.path.join(pth, 'newfile1.img')
    new_hdr1 = os.path.join(pth, 'newfile1.hdr')
    new_img2 = os.path.join(pth, 'newfile2.img')
    new_hdr2 = os.path.join(pth, 'newfile2.hdr')
    new_img3 = os.path.join(pth, 'newfile3.img')
    new_hdr3 = os.path.join(pth, 'newfile3.hdr')
    copyfile(orig_img, new_img1)
    yield assert_true, os.path.islink(new_img1)
    yield assert_true, os.path.islink(new_hdr1)
    copyfile(new_img1, new_img2, copy=True)
    yield assert_false, os.path.islink(new_img2)
    yield assert_false, os.path.islink(new_hdr2)
    yield assert_false, os.path.samefile(orig_img, new_img2)
    yield assert_false, os.path.samefile(orig_hdr, new_hdr2)
    copyfile(new_img1, new_img3, copy=True, use_hardlink=True)
    yield assert_false, os.path.islink(new_img3)
    yield assert_false, os.path.islink(new_hdr3)
    yield assert_true, os.path.samefile(orig_img, new_img3)
    yield assert_true, os.path.samefile(orig_hdr, new_hdr3)

    os.unlink(new_img1)
    os.unlink(new_hdr1)
    os.unlink(new_img2)
    os.unlink(new_hdr2)
    os.unlink(new_img3)
    os.unlink(new_hdr3)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_recopy():
    # Re-copying with the same parameters on an unchanged file should be
    # idempotent
    #
    # Test for copying from regular files and symlinks
    orig_img, orig_hdr = _temp_analyze_files()
    pth, fname = os.path.split(orig_img)
    img_link = os.path.join(pth, 'imglink.img')
    hdr_link = os.path.join(pth, 'imglink.hdr')
    new_img = os.path.join(pth, 'newfile.img')
    new_hdr = os.path.join(pth, 'newfile.hdr')
    copyfile(orig_img, img_link)
    for copy in (True, False):
        for use_hardlink in (True, False):
            copyfile(orig_img, new_img, copy=copy, use_hardlink=use_hardlink)
            img_stat = os.stat(new_img)
            hdr_stat = os.stat(new_hdr)
            copyfile(orig_img, new_img, copy=copy, use_hardlink=use_hardlink)
            err_msg = "Regular - OS: {}; Copy: {}; Hardlink: {}".format(
                os.name, copy, use_hardlink)
            yield assert_equal, img_stat, os.stat(new_img), err_msg
            yield assert_equal, hdr_stat, os.stat(new_hdr), err_msg
            os.unlink(new_img)
            os.unlink(new_hdr)

            copyfile(img_link, new_img, copy=copy, use_hardlink=use_hardlink)
            img_stat = os.stat(new_img)
            hdr_stat = os.stat(new_hdr)
            copyfile(img_link, new_img, copy=copy, use_hardlink=use_hardlink)
            err_msg = "Symlink - OS: {}; Copy: {}; Hardlink: {}".format(
                os.name, copy, use_hardlink)
            yield assert_equal, img_stat, os.stat(new_img), err_msg
            yield assert_equal, hdr_stat, os.stat(new_hdr), err_msg
            os.unlink(new_img)
            os.unlink(new_hdr)
    os.unlink(img_link)
    os.unlink(hdr_link)
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_copyfallback():
    if os.name is not 'posix':
        return
    orig_img, orig_hdr = _temp_analyze_files()
    pth, imgname = os.path.split(orig_img)
    pth, hdrname = os.path.split(orig_hdr)
    try:
        fatfs = TempFATFS()
    except (IOError, OSError):
        warnings.warn('Fuse mount failed. copyfile fallback tests skipped.')
    else:
        with fatfs as fatdir:
            tgt_img = os.path.join(fatdir, imgname)
            tgt_hdr = os.path.join(fatdir, hdrname)
            for copy in (True, False):
                for use_hardlink in (True, False):
                    copyfile(orig_img, tgt_img, copy=copy,
                             use_hardlink=use_hardlink)
                    yield assert_true, os.path.exists(tgt_img)
                    yield assert_true, os.path.exists(tgt_hdr)
                    yield assert_false, os.path.islink(tgt_img)
                    yield assert_false, os.path.islink(tgt_hdr)
                    yield assert_false, os.path.samefile(orig_img, tgt_img)
                    yield assert_false, os.path.samefile(orig_hdr, tgt_hdr)
                    os.unlink(tgt_img)
                    os.unlink(tgt_hdr)
    finally:
        os.unlink(orig_img)
        os.unlink(orig_hdr)

def test_filename_to_list():
    x = filename_to_list('foo.nii')
    yield assert_equal, x, ['foo.nii']
    x = filename_to_list(['foo.nii'])
    yield assert_equal, x, ['foo.nii']
    x = filename_to_list(('foo', 'bar'))
    yield assert_equal, x, ['foo', 'bar']
    x = filename_to_list(12.34)
    yield assert_equal, x, None

def test_list_to_filename():
    x = list_to_filename(['foo.nii'])
    yield assert_equal, x, 'foo.nii'
    x = list_to_filename(['foo', 'bar'])
    yield assert_equal, x, ['foo', 'bar']

def test_json():
    # Simple roundtrip test of json files, just a sanity check.
    adict = dict(a='one', c='three', b='two')
    fd, name = mkstemp(suffix='.json')
    save_json(name, adict)  # save_json closes the file
    new_dict = load_json(name)
    os.unlink(name)
    yield assert_equal, sorted(adict.items()), sorted(new_dict.items())

def test_related_files():
    file1 = '/path/test.img'
    file2 = '/path/test.hdr'
    file3 = '/path/test.BRIK'
    file4 = '/path/test.HEAD'
    file5 = '/path/foo.nii'

    spm_files1 = get_related_files(file1)
    spm_files2 = get_related_files(file2)
    afni_files1 = get_related_files(file3)
    afni_files2 = get_related_files(file4)
    yield assert_equal, len(spm_files1), 3
    yield assert_equal, len(spm_files2), 3
    yield assert_equal, len(afni_files1), 2
    yield assert_equal, len(afni_files2), 2
    yield assert_equal, len(get_related_files(file5)), 1

    yield assert_true, '/path/test.hdr' in spm_files1
    yield assert_true, '/path/test.img' in spm_files1
    yield assert_true, '/path/test.mat' in spm_files1
    yield assert_true, '/path/test.hdr' in spm_files2
    yield assert_true, '/path/test.img' in spm_files2
    yield assert_true, '/path/test.mat' in spm_files2
    yield assert_true, '/path/test.BRIK' in afni_files1
    yield assert_true, '/path/test.HEAD' in afni_files1
    yield assert_true, '/path/test.BRIK' in afni_files2
    yield assert_true, '/path/test.HEAD' in afni_files2
<|endoftext|>


<commit_before>import os
from tempfile import mkstemp

from nipype.testing import assert_equal, assert_true, assert_false
from nipype.utils.filemanip import (save_json, load_json, loadflat,
                                    fname_presuffix, fnames_presuffix,
                                    hash_rename, check_forhash)

import numpy as np

def test_fname_presuffix():
    fname = 'foo.nii'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii'
    fname += '.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp', use_ext=False)
    yield assert_equal, pth, '/tmp/pre_foo_post'

def test_fnames_presuffix():
    fnames = ['foo.nii', 'bar.nii']
    pths = fnames_presuffix(fnames, 'pre_', '_post', '/tmp')
    yield assert_equal, pths, ['/tmp/pre_foo_post.nii', '/tmp/pre_bar_post.nii']

def test_hash_rename():
    new_name = hash_rename('foobar.nii', 'abc123')
    assert_equal(new_name, 'foobar_0xabc123.nii')

def test_check_forhash():
    fname = 'foobar'
    orig_hash = '_0x4323dbcefdc51906decd8edcb3327943'
    #hashed_name = 'foobar_0x4323dbcefdc51906decd8edcb3327943.nii'
    hashed_name = ''.join((fname, orig_hash, '.nii'))
    result, hash = check_forhash(hashed_name)
    yield assert_true, result
    yield assert_equal, hash, [orig_hash]
    result, hash = check_forhash('foobar.nii')
    yield assert_false, result
    yield assert_equal, hash, None

def test_json():
    # Simple roundtrip test of json files, just a sanity check.
    adict = dict(a='one', c='three', b='two')
    fd, name = mkstemp(suffix='.json')
    save_json(name, adict) # save_json closes the file
    new_dict = load_json(name)
    os.unlink(name)
    yield assert_equal, sorted(adict.items()), sorted(new_dict.items())

def test_loadflat():
    alist = [dict(a='one', c='three', b='two'),
             dict(a='one', c='three', b='two')]
    fd, name = mkstemp(suffix='.npz')
    np.savez(name,a=alist)
    aloaded = loadflat(name)['a']
    os.unlink(name)
    yield assert_equal, len(aloaded), 2
    yield assert_equal, sorted(aloaded[0].items()), sorted(alist[0].items())
    adict = dict(a='one', c='three', b='two')
    fd, name = mkstemp(suffix='.npz')
    np.savez(name,a=adict)
    aloaded = loadflat(name)['a']
    os.unlink(name)
    yield assert_true, isinstance(aloaded, dict)
    yield assert_equal, sorted(aloaded.items()), sorted(adict.items())

<commit_msg>Add test for copyfile.<commit_after>import os
from tempfile import mkstemp

from nipype.testing import assert_equal, assert_true, assert_false
from nipype.utils.filemanip import (save_json, load_json, loadflat,
                                    fname_presuffix, fnames_presuffix,
                                    hash_rename, check_forhash,
                                    copyfile)

import numpy as np

def test_fname_presuffix():
    fname = 'foo.nii'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii'
    fname += '.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp')
    yield assert_equal, pth, '/tmp/pre_foo_post.nii.gz'
    pth = fname_presuffix(fname, 'pre_', '_post', '/tmp', use_ext=False)
    yield assert_equal, pth, '/tmp/pre_foo_post'

def test_fnames_presuffix():
    fnames = ['foo.nii', 'bar.nii']
    pths = fnames_presuffix(fnames, 'pre_', '_post', '/tmp')
    yield assert_equal, pths, ['/tmp/pre_foo_post.nii', '/tmp/pre_bar_post.nii']

def test_hash_rename():
    new_name = hash_rename('foobar.nii', 'abc123')
    assert_equal(new_name, 'foobar_0xabc123.nii')

def test_check_forhash():
    fname = 'foobar'
    orig_hash = '_0x4323dbcefdc51906decd8edcb3327943'
    hashed_name = ''.join((fname, orig_hash, '.nii'))
    result, hash = check_forhash(hashed_name)
    yield assert_true, result
    yield assert_equal, hash, [orig_hash]
    result, hash = check_forhash('foobar.nii')
    yield assert_false, result
    yield assert_equal, hash, None

def test_copyfile():
    fd, orig_img = mkstemp(suffix = '.img')
    orig_hdr = orig_img[:-4] + '.hdr'
    fp = file(orig_hdr, 'w+')
    fp.close()
    pth, fname = os.path.split(orig_img)
    new_img = os.path.join(pth, 'newfile.img')
    new_hdr = os.path.join(pth, 'newfile.hdr')
    copyfile(orig_img, new_img)
    yield assert_true, os.path.exists(new_img)
    yield assert_true, os.path.exists(new_hdr)
    os.unlink(new_img)
    os.unlink(new_hdr)
    # Test with copy=True
    copyfile(orig_img, new_img, copy=True)
    yield assert_true, os.path.exists(new_img)
    yield assert_true, os.path.exists(new_hdr)
    os.unlink(new_img)
    os.unlink(new_hdr)
    # final cleanup
    os.unlink(orig_img)
    os.unlink(orig_hdr)

def test_json():
    # Simple roundtrip test of json files, just a sanity check.
    adict = dict(a='one', c='three', b='two')
    fd, name = mkstemp(suffix='.json')
    save_json(name, adict) # save_json closes the file
    new_dict = load_json(name)
    os.unlink(name)
    yield assert_equal, sorted(adict.items()), sorted(new_dict.items())

def test_loadflat():
    alist = [dict(a='one', c='three', b='two'),
             dict(a='one', c='three', b='two')]
    fd, name = mkstemp(suffix='.npz')
    np.savez(name,a=alist)
    aloaded = loadflat(name)['a']
    os.unlink(name)
    yield assert_equal, len(aloaded), 2
    yield assert_equal, sorted(aloaded[0].items()), sorted(alist[0].items())
    adict = dict(a='one', c='three', b='two')
    fd, name = mkstemp(suffix='.npz')
    np.savez(name,a=adict)
    aloaded = loadflat(name)['a']
    os.unlink(name)
    yield assert_true, isinstance(aloaded, dict)
    yield assert_equal, sorted(aloaded.items()), sorted(adict.items())

<|endoftext|>