<commit_before># -*- coding: utf-8 -*-
import os
import sys
import docopt
import logging
import traceback
import mongoengine
from nfq.tweemanager.reportermanager import generateReports, gen_list_of_weeks
from nfq.tweemanager.version import __version__
from nfq.tweemanager.settings import cfgmanager
from nfq.tweemanager.getoldtweets import setTweetCriteria, getoldtweetsGenerator
from nfq.tweemanager.searchlistenerAPI import listenertweets, searchtweets
from nfq.tweemanager.tools import StdoutTweetProcessor, FileTweetProcessor, MongoTweetProcessor

TweetProcessor = None

def tweemanager():
    """ tweemanager:
    NFQ Solutions: this package is at beta-rc stage.

    Usage:
        tweemanager genconfig [options]
        tweemanager reporting [(--cfgfile <cfgfile> | --cfgjsonstr <cfgjsonstr>)] [options]
        tweemanager (listener|searchtweets|getoldtweets)
                    [(--cfgfile <cfgfile> | --cfgjsonstr <cfgjsonstr>)]
                    [options]
        tweemanager --version

    Options:
        --logfile <logfile>                 log file name:
        --loglevel <loglevel>               loglevel and command line verbosity:
                                                CRITICAL
                                                ERROR
                                                WARNING
                                                INFO
                                                DEBUG
                                                NOTSET
        -c <cfgfile> --cfgfile <cfgfile>    Set a config file to be used.
        --cfgjsonstr <cfgjsonstr>           Set a config json string to be used.
        -o <fout> --output <fout>           Set the output file/database.
                                            If not set it will use stdout.

    Complements:
        -h --help                           Show this screen.
        --version                           Show version.

    """
    #
    # Arguments Handling and Validation
    #
    args = docopt.docopt(tweemanager.__doc__, version=__version__)
    #
    # Arguments Handling Done
    #
    # #########################################################################
    #
    # Config Handling
    #
    config = cfgmanager()
    if args['--cfgjsonstr']:
        # given a jsonstring:
        print('config via json string')
        config.readfromjsonstr(args['--cfgjsonstr'])
    elif args['--cfgfile']:
        # given a cfgfile:
        cfgfile, cfgfile_extension = os.path.splitext(args['--cfgfile'])
        if cfgfile_extension.lower() == '.json':
            print('.json')
            config.readfromjsonfile(args['--cfgfile'])
        elif cfgfile_extension.lower() == '.ini' or cfgfile_extension.lower() == '.cfg':
            print('.ini/.cfg')
            config.readfromfile(args['--cfgfile'])
        else:
            print(Exception("ERROR: unkown file extension for given file '{0}{1}'".format(cfgfile, cfgfile_extension)))
            sys.exit(0)
    else:
        # if no configuration is given it will use tweem.cfg that will be created
        # at runtime directory.
        try:
            # check if file exists isn
            args['--cfgfile'] = "tweem.json"
            config.readfromjsonfile(args['--cfgfile'])
        except:
            args['--cfgfile'] = "tweem.cfg"
            config.readfromfile(args['--cfgfile'])
    cfgmanager.setconfigassettings(config._sections)
    #
    # Config Handling
    #
    # #########################################################################
    #
    # command genconfig
    #
    if args['genconfig']:
        print('genconfig command activated')
        with open('tweeconfig.cfg', 'w') as tweeconfig:
            tweeconfig.write(
"""[TwitterAPIcredentials]
consumer_key = ""
consumer_secret = ""
access_key = ""
access_secret = ""

[ListenerSpecs]
usersarray = [""]
trackarray = [""]

[SearchSpecs]
searchquery = [""]
maxtweets = ""

[GOTSpecs]
username = ""
since = ""
until = ""
querysearch = ["]
maxtweets = 100

[TextPatterns]
patternstoexclude = []
patternstoinclude = []
langtoinclude = []
alertwords = []

[MongoDBSpecs]
host = ""
repocollname = ""

[ElasticSpecs]
host = ""
index = ""
"""
                )
        print('Done')
        print('Check the configuration file. If some fields are not going to use, delete them')
        return
    #
    # command genconfig Done
    #
    # #########################################################################
    #
    # logging basic config
    #
    loglevels = {
        "CRITICAL": logging.CRITICAL,
        "ERROR": logging.ERROR,
        "WARNING": logging.WARNING,
        "INFO": logging.INFO,
        "DEBUG": logging.DEBUG,
        "NOTSET": logging.NOTSET
    }
    # Build default using config file:
    if (sys.version_info) >= (3, 4):
        cfgit = cfgmanager.LogSpecs.items()
    else:
        cfgit = cfgmanager.LogSpecs.iteritems()
    for key, value in cfgit:
        if key == 'loglevel':
            if not value or value is None:
                cfgmanager.LogSpecs['loglevel'] = 'INFO'
        if key == 'logfile':
            if not value or value is None:
                cfgmanager.LogSpecs['logfile'] = None
    # overwritting if command line options:
    if args['--loglevel']:
        cfgmanager.LogSpecs['loglevel'] = args['--loglevel']
    if args['--logfile']:
        cfgmanager.LogSpecs['logfile'] = args['--logfile']
    # start the logger:
    logging.basicConfig(
        level=loglevels[cfgmanager.LogSpecs['loglevel']],
        format='%(asctime)s - %(levelname)s - %(funcName)s - %(message)s',
        filename=cfgmanager.LogSpecs['logfile']
    )
    logging.info("logging configuration done")
    logging.debug("using configuration {}".format(dict(cfgmanager.LogSpecs)))
    #
    # logging basic config Done
    #
    # #########################################################################
    #
    # TweetHandler
    #
    # Setting output
    if args['--output']:
        if args['--output'] == 'mongodb':
            logging.info('Setting output to {}'.format(args['--output']))
            mongoengine.connect(host=cfgmanager.MongoDBSpecs['host'])
            TweetProcessor = MongoTweetProcessor
        else:
            logging.info('Setting default output to {}'.format(args['--output']))
            FileTweetProcessor.OUTPUT_FILE = args['--output']
            TweetProcessor = FileTweetProcessor
    else:
        logging.info('Setting default output to stdout')
        TweetProcessor = StdoutTweetProcessor
    # Setting output done
    # Setting TextPatterns
    logging.info("Setting TextPatterns")
    logging.debug("using TextPatterns {}".format(dict(cfgmanager.TextPatterns)))
    TweetProcessor.PATTERNSTOEXCLUDE = cfgmanager.TextPatterns['patternstoexclude']
    TweetProcessor.PATTERNSTOINCLUDE = cfgmanager.TextPatterns['patternstoinclude']
    TweetProcessor.LANGTOINCLUDE = cfgmanager.TextPatterns['langtoinclude']
    logging.info("Setting TextPatterns Done")
    # Setting TextPatterns Done
    # #########################################################################
    #
    # command listener
    #
    if args['listener']:
        try:
            logging.info('listener command activated')
            listenertweets(TweetProcessor, cfgmanager.ListenerSpecs['trackarray'])
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            formatted_traceback = traceback.format_exception(exc_type, exc_value, exc_traceback)
            for traceback_line in formatted_traceback:
                logging.critical(traceback_line)
            logging.critical('Error in listener command')
        return
    # #########################################################################
    #
    # command searchtweets
    #
    if args['searchtweets']:
        try:
            logging.info('searchtweets command selected')
            searchtweets(TweetProcessor, cfgmanager.SearchSpecs['searchquery'], maxtweets=cfgmanager.SearchSpecs['maxtweets'])
            logging.info('searchtweets command Done')
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            formatted_traceback = traceback.format_exception(exc_type, exc_value, exc_traceback)
            for traceback_line in formatted_traceback:
                logging.critical(traceback_line)
            logging.critical('Error in searchtweets command')
        finally:
            return
    #
    # command searchtweets Done
    #
    # #########################################################################
    #
    # command getoldtweets
    #

    if args['getoldtweets']:
        # Getting the information to use it in the loop
        StartDate = cfgmanager.GOTSpecs['since']
        EndDate = cfgmanager.GOTSpecs['until']
        user = cfgmanager.GOTSpecs['username']
        query = cfgmanager.GOTSpecs['querysearch']
        maxtweets = cfgmanager.GOTSpecs['maxtweets']

        for values in gen_list_of_weeks(StartDate, EndDate):
            Start = values['start']
            End = values['end']
            try:
                logging.info('getoldtweets command selected')
                tweetCriteria = setTweetCriteria(
                    username=user,
                    since=Start,
                    until=End,
                    querySearch=query,
                    maxTweets=maxtweets)
                for tweet in getoldtweetsGenerator(tweetCriteria):
                    posttweet = TweetProcessor(tweet)
                    posttweet.sendtooutput()
                logging.info('getoldtweets command Done')
            except Exception as e:
                print("")
                print(e)
                logging.critical('Error in getoldtweets command')

    #
    # command getoldtweets Done
    #
    # #########################################################################
    #
    # command reporting
    #

    if args['reporting']:
        logging.info('reporting command selected')
        # mongoengine.connect(host=cfgmanager.MongoDBSpecs['host'])
        try:
            host = cfgmanager.MongoDBSpecs['host']
            name_collection = cfgmanager.MongoDBSpecs['repocollname']
            from_got = cfgmanager.MongoDBSpecs['fromgot']
            classifier = cfgmanager.MongoDBSpecs['classifier']
            print('Classifier =', classifier)
            if str(from_got) == 'True':
                from_got = True
                logging.info('Setting from_got option to True')
            else:
                from_got = False

            if args['--output']:

                if args['--output'] == 'mongodb':
                    logging.info('Setting output to {}'.format(args['--output']))
                    out = 'mongodb'
                    outname = 'Reports_' + name_collection
                else:
                    logging.info('Setting default output to {}'.format(args['--output']))
                    out = 'json'
                    outname = args['--output']
            else:
                logging.info('Setting default output to stdout')
                out = 'stdout'
                outname = None

            alertwords = cfgmanager.TextPatterns['alertwords']
            print('Alert words to find:',alertwords)

            generateReports(host=host, alertwords=alertwords, name_collection=name_collection,
                            output=out, output_name=outname, fromgot=from_got, classifier=classifier)

        except mongoengine.ConnectionError:
            logging.error('check if connection to mongo is defined')

        logging.info('reporting command Done')
        return
    #
    # command reporting Done
    #
    # #########################################################################
    #
    # command importToMongo
    #
    if args.get('importToMongo'):
        logging.info('importToMongo command selected')
        return
    #
    # command importToMongo Done
    #
    # #########################################################################
    #
    # command dumpFromMongo
    #
    if args.get('dumpCorpusFromMongo'):
        logging.info('dumpCorpusFromMongo command selected')
        logging.debug('Not implemented')
        return
    #
    # command dumpFromMongo Done
    #
    # #########################################################################
    #
    # command importToElastic
    #
    if args.get('importToElastic'):
        logging.info('importToElastic command selected')
        logging.debug('Not implemented')
        return
    #
    # command importToElastic Done
    #
    # #########################################################################
    #
    # command mongo2elastic
    #
    if args.get('mongo2elastic'):
        logging.info('mongo2elastic command selected')
        logging.debug('Not implemented')
        return
    #
    # command mongo2elastic Done
    #
    # #########################################################################

if __name__ == '__main__':
    sys.exit(tweemanager())
<commit_msg>Functios renamed following the stylo guide<commit_after># -*- coding: utf-8 -*-
import os
import sys
import docopt
import logging
import traceback
import mongoengine
from nfq.tweemanager.reportermanager import generate_reports, gen_list_of_weeks
from nfq.tweemanager.version import __version__
from nfq.tweemanager.settings import cfgmanager
from nfq.tweemanager.getoldtweets import setTweetCriteria, getoldtweetsGenerator
from nfq.tweemanager.searchlistenerAPI import listenertweets, searchtweets
from nfq.tweemanager.tools import StdoutTweetProcessor, FileTweetProcessor, MongoTweetProcessor

TweetProcessor = None

def tweemanager():
    """ tweemanager:
    NFQ Solutions: this package is at beta-rc stage.

    Usage:
        tweemanager genconfig [options]
        tweemanager reporting [(--cfgfile <cfgfile> | --cfgjsonstr <cfgjsonstr>)] [options]
        tweemanager (listener|searchtweets|getoldtweets)
                    [(--cfgfile <cfgfile> | --cfgjsonstr <cfgjsonstr>)]
                    [options]
        tweemanager --version

    Options:
        --logfile <logfile>                 log file name:
        --loglevel <loglevel>               loglevel and command line verbosity:
                                                CRITICAL
                                                ERROR
                                                WARNING
                                                INFO
                                                DEBUG
                                                NOTSET
        -c <cfgfile> --cfgfile <cfgfile>    Set a config file to be used.
        --cfgjsonstr <cfgjsonstr>           Set a config json string to be used.
        -o <fout> --output <fout>           Set the output file/database.
                                            If not set it will use stdout.

    Complements:
        -h --help                           Show this screen.
        --version                           Show version.

    """
    #
    # Arguments Handling and Validation
    #
    args = docopt.docopt(tweemanager.__doc__, version=__version__)
    #
    # Arguments Handling Done
    #
    # #########################################################################
    #
    # Config Handling
    #
    config = cfgmanager()
    if args['--cfgjsonstr']:
        # given a jsonstring:
        print('config via json string')
        config.readfromjsonstr(args['--cfgjsonstr'])
    elif args['--cfgfile']:
        # given a cfgfile:
        cfgfile, cfgfile_extension = os.path.splitext(args['--cfgfile'])
        if cfgfile_extension.lower() == '.json':
            print('.json')
            config.readfromjsonfile(args['--cfgfile'])
        elif cfgfile_extension.lower() == '.ini' or cfgfile_extension.lower() == '.cfg':
            print('.ini/.cfg')
            config.readfromfile(args['--cfgfile'])
        else:
            print(Exception("ERROR: unkown file extension for given file '{0}{1}'".format(cfgfile, cfgfile_extension)))
            sys.exit(0)
    else:
        # if no configuration is given it will use tweem.cfg that will be created
        # at runtime directory.
        try:
            # check if file exists isn
            args['--cfgfile'] = "tweem.json"
            config.readfromjsonfile(args['--cfgfile'])
        except:
            args['--cfgfile'] = "tweem.cfg"
            config.readfromfile(args['--cfgfile'])
    cfgmanager.setconfigassettings(config._sections)
    #
    # Config Handling
    #
    # #########################################################################
    #
    # command genconfig
    #
    if args['genconfig']:
        print('genconfig command activated')
        with open('tweeconfig.cfg', 'w') as tweeconfig:
            tweeconfig.write(
"""[TwitterAPIcredentials]
consumer_key = ""
consumer_secret = ""
access_key = ""
access_secret = ""

[ListenerSpecs]
usersarray = [""]
trackarray = [""]

[SearchSpecs]
searchquery = [""]
maxtweets = ""

[GOTSpecs]
username = ""
since = ""
until = ""
querysearch = ["]
maxtweets = 100

[TextPatterns]
patternstoexclude = []
patternstoinclude = []
langtoinclude = []
alertwords = []

[MongoDBSpecs]
host = ""
repocollname = ""

[ElasticSpecs]
host = ""
index = ""
"""
                )
        print('Done')
        print('Check the configuration file. If some fields are not going to use, delete them')
        return
    #
    # command genconfig Done
    #
    # #########################################################################
    #
    # logging basic config
    #
    loglevels = {
        "CRITICAL": logging.CRITICAL,
        "ERROR": logging.ERROR,
        "WARNING": logging.WARNING,
        "INFO": logging.INFO,
        "DEBUG": logging.DEBUG,
        "NOTSET": logging.NOTSET
    }
    # Build default using config file:
    if (sys.version_info) >= (3, 4):
        cfgit = cfgmanager.LogSpecs.items()
    else:
        cfgit = cfgmanager.LogSpecs.iteritems()
    for key, value in cfgit:
        if key == 'loglevel':
            if not value or value is None:
                cfgmanager.LogSpecs['loglevel'] = 'INFO'
        if key == 'logfile':
            if not value or value is None:
                cfgmanager.LogSpecs['logfile'] = None
    # overwritting if command line options:
    if args['--loglevel']:
        cfgmanager.LogSpecs['loglevel'] = args['--loglevel']
    if args['--logfile']:
        cfgmanager.LogSpecs['logfile'] = args['--logfile']
    # start the logger:
    logging.basicConfig(
        level=loglevels[cfgmanager.LogSpecs['loglevel']],
        format='%(asctime)s - %(levelname)s - %(funcName)s - %(message)s',
        filename=cfgmanager.LogSpecs['logfile']
    )
    logging.info("logging configuration done")
    logging.debug("using configuration {}".format(dict(cfgmanager.LogSpecs)))
    #
    # logging basic config Done
    #
    # #########################################################################
    #
    # TweetHandler
    #
    # Setting output
    if args['--output']:
        if args['--output'] == 'mongodb':
            logging.info('Setting output to {}'.format(args['--output']))
            mongoengine.connect(host=cfgmanager.MongoDBSpecs['host'])
            TweetProcessor = MongoTweetProcessor
        else:
            logging.info('Setting default output to {}'.format(args['--output']))
            FileTweetProcessor.OUTPUT_FILE = args['--output']
            TweetProcessor = FileTweetProcessor
    else:
        logging.info('Setting default output to stdout')
        TweetProcessor = StdoutTweetProcessor
    # Setting output done
    # Setting TextPatterns
    logging.info("Setting TextPatterns")
    logging.debug("using TextPatterns {}".format(dict(cfgmanager.TextPatterns)))
    TweetProcessor.PATTERNSTOEXCLUDE = cfgmanager.TextPatterns['patternstoexclude']
    TweetProcessor.PATTERNSTOINCLUDE = cfgmanager.TextPatterns['patternstoinclude']
    TweetProcessor.LANGTOINCLUDE = cfgmanager.TextPatterns['langtoinclude']
    logging.info("Setting TextPatterns Done")
    # Setting TextPatterns Done
    # #########################################################################
    #
    # command listener
    #
    if args['listener']:
        try:
            logging.info('listener command activated')
            listenertweets(TweetProcessor, cfgmanager.ListenerSpecs['trackarray'])
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            formatted_traceback = traceback.format_exception(exc_type, exc_value, exc_traceback)
            for traceback_line in formatted_traceback:
                logging.critical(traceback_line)
            logging.critical('Error in listener command')
        return
    # #########################################################################
    #
    # command searchtweets
    #
    if args['searchtweets']:
        try:
            logging.info('searchtweets command selected')
            searchtweets(TweetProcessor, cfgmanager.SearchSpecs['searchquery'], maxtweets=cfgmanager.SearchSpecs['maxtweets'])
            logging.info('searchtweets command Done')
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            formatted_traceback = traceback.format_exception(exc_type, exc_value, exc_traceback)
            for traceback_line in formatted_traceback:
                logging.critical(traceback_line)
            logging.critical('Error in searchtweets command')
        finally:
            return
    #
    # command searchtweets Done
    #
    # #########################################################################
    #
    # command getoldtweets
    #

    if args['getoldtweets']:
        # Getting the information to use it in the loop
        StartDate = cfgmanager.GOTSpecs['since']
        EndDate = cfgmanager.GOTSpecs['until']
        user = cfgmanager.GOTSpecs['username']
        query = cfgmanager.GOTSpecs['querysearch']
        maxtweets = cfgmanager.GOTSpecs['maxtweets']

        for values in gen_list_of_weeks(StartDate, EndDate):
            Start = values['start']
            End = values['end']
            try:
                logging.info('getoldtweets command selected')
                tweetCriteria = setTweetCriteria(
                    username=user,
                    since=Start,
                    until=End,
                    querySearch=query,
                    maxTweets=maxtweets)
                for tweet in getoldtweetsGenerator(tweetCriteria):
                    posttweet = TweetProcessor(tweet)
                    posttweet.sendtooutput()
                logging.info('getoldtweets command Done')
            except Exception as e:
                print("")
                print(e)
                logging.critical('Error in getoldtweets command')

    #
    # command getoldtweets Done
    #
    # #########################################################################
    #
    # command reporting
    #

    if args['reporting']:
        logging.info('reporting command selected')
        # mongoengine.connect(host=cfgmanager.MongoDBSpecs['host'])
        try:
            host = cfgmanager.MongoDBSpecs['host']
            name_collection = cfgmanager.MongoDBSpecs['repocollname']
            from_got = cfgmanager.MongoDBSpecs['fromgot']
            classifier = cfgmanager.MongoDBSpecs['classifier']
            print('Classifier =', classifier)
            if str(from_got) == 'True':
                from_got = True
                logging.info('Setting from_got option to True')
            else:
                from_got = False

            if args['--output']:

                if args['--output'] == 'mongodb':
                    logging.info('Setting output to {}'.format(args['--output']))
                    out = 'mongodb'
                    outname = 'Reports_' + name_collection
                else:
                    logging.info('Setting default output to {}'.format(args['--output']))
                    out = 'json'
                    outname = args['--output']
            else:
                logging.info('Setting default output to stdout')
                out = 'stdout'
                outname = None

            alertwords = cfgmanager.TextPatterns['alertwords']
            print('Alert words to find:',alertwords)

            generate_reports(host=host, alertwords=alertwords, name_collection=name_collection,
                            output=out, output_name=outname, fromgot=from_got, classifier=classifier)

        except mongoengine.ConnectionError:
            logging.error('check if connection to mongo is defined')

        logging.info('reporting command Done')
        return
    #
    # command reporting Done
    #
    # #########################################################################
    #
    # command importToMongo
    #
    if args.get('importToMongo'):
        logging.info('importToMongo command selected')
        return
    #
    # command importToMongo Done
    #
    # #########################################################################
    #
    # command dumpFromMongo
    #
    if args.get('dumpCorpusFromMongo'):
        logging.info('dumpCorpusFromMongo command selected')
        logging.debug('Not implemented')
        return
    #
    # command dumpFromMongo Done
    #
    # #########################################################################
    #
    # command importToElastic
    #
    if args.get('importToElastic'):
        logging.info('importToElastic command selected')
        logging.debug('Not implemented')
        return
    #
    # command importToElastic Done
    #
    # #########################################################################
    #
    # command mongo2elastic
    #
    if args.get('mongo2elastic'):
        logging.info('mongo2elastic command selected')
        logging.debug('Not implemented')
        return
    #
    # command mongo2elastic Done
    #
    # #########################################################################

if __name__ == '__main__':
    sys.exit(tweemanager())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os

########################################
# Configuration, please edit
########################################

# post_pages contains (wildcard, destination, template, use_in_feed) tuples.
#
# The wildcard is used to generate a list of reSt source files
# (whatever/thing.txt).
# That fragment must have an associated metadata file (whatever/thing.meta),
# and opcionally translated files (example for spanish, with code "es"):
#     whatever/thing.txt.es and whatever/thing.meta.es
#
# From those files, a set of HTML fragment files will be generated:
# cache/whatever/thing.html (and maybe cache/whatever/thing.html.es)
#
# These files are combinated with the template to produce rendered
# pages, which will be placed at
# output / TRANSLATIONS[lang] / destination / pagename.html
#
# where "pagename" is specified in the metadata file.
#
# if use_in_feed is True, then those posts will be added to the site's
# rss feeds.
#

post_pages = (
    ("posts/*.txt", "posts", "post.tmpl", True),
    ("stories/*.txt", "stories", "story.tmpl", False),
)

# One or more folders containing files to be copied as-is into the output.
# The format is a dictionary of "source" "relative destination".
# Default is:
# FILES_FOLDERS = {'files': '' }
# Which means copy 'files' into 'output'

# A mapping of languages to file-extensions that represent that language.
# Feel free to add or delete extensions to any list, but don't add any new
# compilers unless you write the interface for it yourself.
#
# 'rest' is reStructuredText
# 'markdown' is MarkDown
# 'html' assumes the file is html and just copies it
#post_compilers = {
#    "rest": ('.txt', '.rst'),
#    "markdown": ('.md', '.mdown', '.markdown')
#    "html": ('.html', '.htm')
#    }

# Nikola is multilingual!
#
# Currently supported languages are:
#   English -> en
#   Greek -> gr
#   German -> de
#   French -> fr
#   Russian -> ru
#   Spanish -> es
#
# If you want to use Nikola with a non-supported language you have to provide
# a module containing the necessary translations
# (p.e. look at the modules at: ./nikola/data/themes/default/messages/fr.py).
# If a specific post is not translated to a language, then the version
# in the default language will be shown instead.

# What is the default language?
DEFAULT_LANG = "en"

# What other languages do you have?
# The format is {"translationcode" : "path/to/translation" }
# the path will be used as a prefix for the generated pages location
TRANSLATIONS = {
    "en": "",
    #"gr": "./gr",
    #"de": "./de",
    #"fr": "./fr",
    #"ru": "./ru",
    #"es": "./es",
    }

# Paths for different autogenerated bits. These are combined with the
# translation paths.

# Final locations are:
# output / TRANSLATION[lang] / TAG_PATH / index.html (list of tags)
# output / TRANSLATION[lang] / TAG_PATH / tag.html (list of posts for a tag)
# output / TRANSLATION[lang] / TAG_PATH / tag.xml (RSS feed for a tag)
TAG_PATH = "categories"

# If TAG_PAGES_ARE_INDEXES is set to True, each tag's page will contain
# the posts themselves. If set to False, it will be just a list of links.
TAG_PAGES_ARE_INDEXES = True

# Final location is output / TRANSLATION[lang] / INDEX_PATH / index-*.html
INDEX_PATH = ""
# Final locations for the archives are:
# output / TRANSLATION[lang] / ARCHIVE_PATH / ARCHIVE_FILENAME
# output / TRANSLATION[lang] / ARCHIVE_PATH / YEAR / index.html
ARCHIVE_PATH = ""
ARCHIVE_FILENAME = "archive.html"
# Final locations are:
# output / TRANSLATION[lang] / RSS_PATH / rss.xml
RSS_PATH = ""

# Slug the Tag URL easier for users to type, special characters are
# often removed or replaced as well.
SLUG_TAG_PATH = True

# A list of redirection tuples, [("foo/from.html", "/bar/to.html")].
#
# A HTML file will be created in output/foo/from.html that redirects
# to the "/bar/to.html" URL. notice that the "from" side MUST be a
# relative URL.
#
# If you don't need any of these, just set to []

REDIRECTIONS = []

# Commands to execute to deploy. Can be anything, for example,
# you may use rsync:
# "rsync -rav output/* joe@my.site:/srv/www/site"
# And then do a backup, or ping pingomatic.
# To do manual deployment, set it to []
DEPLOY_COMMANDS = []

# Where the output site should be located
# If you don't use an absolute path, it will be considered as relative
# to the location of conf.py

OUTPUT_FOLDER = 'output'

# Filters to apply to the output.
# A directory where the keys are either: a file extensions, or
# a tuple of file extensions.
#
# And the value is a list of commands to be applied in order.
#
# Each command must be either:
#
# A string containing a '%s' which will
# be replaced with a filename. The command *must* produce output
# in place.
#
# Or:
#
# A python callable, which will be called with the filename as
# argument.
#
# By default, there are no filters.
FILTERS = {
#    ".jpg": ["jpegoptim --strip-all -m75 -v %s"],
}

##############################################################################
# Image Gallery Options
##############################################################################

# Galleries are folders in galleries/
# Final location of galleries will be output / GALLERY_PATH / gallery_name
GALLERY_PATH = "galleries"
THUMBNAIL_SIZE = 180
MAX_IMAGE_SIZE = 1280
USE_FILENAME_AS_TITLE = True

##############################################################################
# HTML fragments and diverse things that are used by the templates
##############################################################################

# Data about this site
BLOG_AUTHOR = "Your Name"
BLOG_TITLE = "Demo Site"
BLOG_URL = "http://nikola.ralsina.com.ar"
BLOG_EMAIL = "joe@demo.site"
BLOG_DESCRIPTION = "This is a demo site for Nikola."

# Data about post-per-page indexes
INDEXES_TITLE = ""  # If this is empty, the default is BLOG_TITLE
INDEXES_PAGES = ""  # If this is empty, the default is 'old posts page %d' translated

# Name of the theme to use. Themes are located in themes/theme_name
THEME = 'site'

# Show only teasers in the index pages? Defaults to False.
# INDEX_TEASERS = False

# A HTML fragment describing the license, for the sidebar.
# I recomment using the Creative Commons' wizard:
# http://creativecommons.org/choose/
LICENSE = """

"""

# A small copyright notice for the page footer (in HTML)
CONTENT_FOOTER = u'Contents © 2012 Example Joe'

# To enable comments via Disqus, you need to create a forum at
# http://disqus.com, and set DISQUS_FORUM to the short name you selected.
# If you want to disable comments, set it to False.
DISQUS_FORUM = "nikolademo"

# Enable Addthis social buttons?
# Defaults to true
# ADD_THIS_BUTTONS = True

# Modify the number of Post per Index Page
# Defaults to 10
# INDEX_DISPLAY_POST_COUNT = 10

# RSS_LINK is a HTML fragment to link the RSS or Atom feeds. If set to None,
# the base.tmpl will use the feed Nikola generates. However, you may want to
# change it for a feedburner feed or something else.
RSS_LINK = None

# A search form to search this site, for the sidebar. You can use a google
# custom search (http://www.google.com/cse/)
# Or a duckduckgo search: https://duckduckgo.com/search_box.html
# This example should work for pretty much any site we generate.
SEARCH_FORM = ""
# This search form is better for the "site" theme where it
# appears on the navigation bar
#SEARCH_FORM = """
#
#<form method="get" id="search" action="http://duckduckgo.com/" """\
#"""class="navbar-form pull-left">
#<input type="hidden" name="sites" value="%s"/>
#<input type="hidden" name="k8" value="#444444"/>
#<input type="hidden" name="k9" value="#D51920"/>
#<input type="hidden" name="kt" value="h"/>
#<input type="text" name="q" maxlength="255" """\
#"""placeholder="Search…" class="span2" style="margin-top: 4px;"/>
#<input type="submit" value="DuckDuckGo Search" style="visibility: hidden;" />
#</form>
#
#""" % BLOG_URL

# Google analytics or whatever else you use. Added to the bottom of
# in the default template (base.tmpl).
ANALYTICS = """
    """

# Put in global_context things you want available on all your templates.
# It can be anything, data, functions, modules, etc.
GLOBAL_CONTEXT = {
    'analytics': ANALYTICS,
    'blog_author': BLOG_AUTHOR,
    'blog_title': BLOG_TITLE,
    'blog_url': BLOG_URL,
    'blog_desc': BLOG_DESCRIPTION,
    'translations': TRANSLATIONS,
    'license': LICENSE,
    'search_form': SEARCH_FORM,
    'disqus_forum': DISQUS_FORUM,
    'content_footer': CONTENT_FOOTER,
    'rss_path': RSS_PATH,
    'rss_link': RSS_LINK,
    # Locale-dependent links for the sidebar
    # You should provide a key-value pair for each used language.
    'sidebar_links': {
        'en': (
            ('/' + os.path.join(ARCHIVE_PATH, ARCHIVE_FILENAME), 'Archives'),
            ('/categories/index.html', 'Tags'),
            ('/stories/about-nikola.html', 'About Nikola'),
            ('/stories/handbook.html', 'The Nikola Handbook'),
            ('http://nikola.ralsina.com.ar', 'Powered by Nikola!'),
            ),
        }
    }
<commit_msg>added it to the translation list in the sample conf<commit_after># -*- coding: utf-8 -*-

import os

########################################
# Configuration, please edit
########################################

# post_pages contains (wildcard, destination, template, use_in_feed) tuples.
#
# The wildcard is used to generate a list of reSt source files
# (whatever/thing.txt).
# That fragment must have an associated metadata file (whatever/thing.meta),
# and opcionally translated files (example for spanish, with code "es"):
#     whatever/thing.txt.es and whatever/thing.meta.es
#
# From those files, a set of HTML fragment files will be generated:
# cache/whatever/thing.html (and maybe cache/whatever/thing.html.es)
#
# These files are combinated with the template to produce rendered
# pages, which will be placed at
# output / TRANSLATIONS[lang] / destination / pagename.html
#
# where "pagename" is specified in the metadata file.
#
# if use_in_feed is True, then those posts will be added to the site's
# rss feeds.
#

post_pages = (
    ("posts/*.txt", "posts", "post.tmpl", True),
    ("stories/*.txt", "stories", "story.tmpl", False),
)

# One or more folders containing files to be copied as-is into the output.
# The format is a dictionary of "source" "relative destination".
# Default is:
# FILES_FOLDERS = {'files': '' }
# Which means copy 'files' into 'output'

# A mapping of languages to file-extensions that represent that language.
# Feel free to add or delete extensions to any list, but don't add any new
# compilers unless you write the interface for it yourself.
#
# 'rest' is reStructuredText
# 'markdown' is MarkDown
# 'html' assumes the file is html and just copies it
#post_compilers = {
#    "rest": ('.txt', '.rst'),
#    "markdown": ('.md', '.mdown', '.markdown')
#    "html": ('.html', '.htm')
#    }

# Nikola is multilingual!
#
# Currently supported languages are:
#   English -> en
#   Greek -> gr
#   German -> de
#   French -> fr
#   Russian -> ru
#   Spanish -> es
#   Italian -> it
#
# If you want to use Nikola with a non-supported language you have to provide
# a module containing the necessary translations
# (p.e. look at the modules at: ./nikola/data/themes/default/messages/fr.py).
# If a specific post is not translated to a language, then the version
# in the default language will be shown instead.

# What is the default language?
DEFAULT_LANG = "en"

# What other languages do you have?
# The format is {"translationcode" : "path/to/translation" }
# the path will be used as a prefix for the generated pages location
TRANSLATIONS = {
    "en": "",
    #"gr": "./gr",
    #"de": "./de",
    #"fr": "./fr",
    #"ru": "./ru",
    #"es": "./es",
    }

# Paths for different autogenerated bits. These are combined with the
# translation paths.

# Final locations are:
# output / TRANSLATION[lang] / TAG_PATH / index.html (list of tags)
# output / TRANSLATION[lang] / TAG_PATH / tag.html (list of posts for a tag)
# output / TRANSLATION[lang] / TAG_PATH / tag.xml (RSS feed for a tag)
TAG_PATH = "categories"

# If TAG_PAGES_ARE_INDEXES is set to True, each tag's page will contain
# the posts themselves. If set to False, it will be just a list of links.
TAG_PAGES_ARE_INDEXES = True

# Final location is output / TRANSLATION[lang] / INDEX_PATH / index-*.html
INDEX_PATH = ""
# Final locations for the archives are:
# output / TRANSLATION[lang] / ARCHIVE_PATH / ARCHIVE_FILENAME
# output / TRANSLATION[lang] / ARCHIVE_PATH / YEAR / index.html
ARCHIVE_PATH = ""
ARCHIVE_FILENAME = "archive.html"
# Final locations are:
# output / TRANSLATION[lang] / RSS_PATH / rss.xml
RSS_PATH = ""

# Slug the Tag URL easier for users to type, special characters are
# often removed or replaced as well.
SLUG_TAG_PATH = True

# A list of redirection tuples, [("foo/from.html", "/bar/to.html")].
#
# A HTML file will be created in output/foo/from.html that redirects
# to the "/bar/to.html" URL. notice that the "from" side MUST be a
# relative URL.
#
# If you don't need any of these, just set to []

REDIRECTIONS = []

# Commands to execute to deploy. Can be anything, for example,
# you may use rsync:
# "rsync -rav output/* joe@my.site:/srv/www/site"
# And then do a backup, or ping pingomatic.
# To do manual deployment, set it to []
DEPLOY_COMMANDS = []

# Where the output site should be located
# If you don't use an absolute path, it will be considered as relative
# to the location of conf.py

OUTPUT_FOLDER = 'output'

# Filters to apply to the output.
# A directory where the keys are either: a file extensions, or
# a tuple of file extensions.
#
# And the value is a list of commands to be applied in order.
#
# Each command must be either:
#
# A string containing a '%s' which will
# be replaced with a filename. The command *must* produce output
# in place.
#
# Or:
#
# A python callable, which will be called with the filename as
# argument.
#
# By default, there are no filters.
FILTERS = {
#    ".jpg": ["jpegoptim --strip-all -m75 -v %s"],
}

##############################################################################
# Image Gallery Options
##############################################################################

# Galleries are folders in galleries/
# Final location of galleries will be output / GALLERY_PATH / gallery_name
GALLERY_PATH = "galleries"
THUMBNAIL_SIZE = 180
MAX_IMAGE_SIZE = 1280
USE_FILENAME_AS_TITLE = True

##############################################################################
# HTML fragments and diverse things that are used by the templates
##############################################################################

# Data about this site
BLOG_AUTHOR = "Your Name"
BLOG_TITLE = "Demo Site"
BLOG_URL = "http://nikola.ralsina.com.ar"
BLOG_EMAIL = "joe@demo.site"
BLOG_DESCRIPTION = "This is a demo site for Nikola."

# Data about post-per-page indexes
INDEXES_TITLE = ""  # If this is empty, the default is BLOG_TITLE
INDEXES_PAGES = ""  # If this is empty, the default is 'old posts page %d' translated

# Name of the theme to use. Themes are located in themes/theme_name
THEME = 'site'

# Show only teasers in the index pages? Defaults to False.
# INDEX_TEASERS = False

# A HTML fragment describing the license, for the sidebar.
# I recomment using the Creative Commons' wizard:
# http://creativecommons.org/choose/
LICENSE = """

"""

# A small copyright notice for the page footer (in HTML)
CONTENT_FOOTER = u'Contents © 2012 Example Joe'

# To enable comments via Disqus, you need to create a forum at
# http://disqus.com, and set DISQUS_FORUM to the short name you selected.
# If you want to disable comments, set it to False.
DISQUS_FORUM = "nikolademo"

# Enable Addthis social buttons?
# Defaults to true
# ADD_THIS_BUTTONS = True

# Modify the number of Post per Index Page
# Defaults to 10
# INDEX_DISPLAY_POST_COUNT = 10

# RSS_LINK is a HTML fragment to link the RSS or Atom feeds. If set to None,
# the base.tmpl will use the feed Nikola generates. However, you may want to
# change it for a feedburner feed or something else.
RSS_LINK = None

# A search form to search this site, for the sidebar. You can use a google
# custom search (http://www.google.com/cse/)
# Or a duckduckgo search: https://duckduckgo.com/search_box.html
# This example should work for pretty much any site we generate.
SEARCH_FORM = ""
# This search form is better for the "site" theme where it
# appears on the navigation bar
#SEARCH_FORM = """
#
#<form method="get" id="search" action="http://duckduckgo.com/" """\
#"""class="navbar-form pull-left">
#<input type="hidden" name="sites" value="%s"/>
#<input type="hidden" name="k8" value="#444444"/>
#<input type="hidden" name="k9" value="#D51920"/>
#<input type="hidden" name="kt" value="h"/>
#<input type="text" name="q" maxlength="255" """\
#"""placeholder="Search…" class="span2" style="margin-top: 4px;"/>
#<input type="submit" value="DuckDuckGo Search" style="visibility: hidden;" />
#</form>
#
#""" % BLOG_URL

# Google analytics or whatever else you use. Added to the bottom of
# in the default template (base.tmpl).
ANALYTICS = """
    """

# Put in global_context things you want available on all your templates.
# It can be anything, data, functions, modules, etc.
GLOBAL_CONTEXT = {
    'analytics': ANALYTICS,
    'blog_author': BLOG_AUTHOR,
    'blog_title': BLOG_TITLE,
    'blog_url': BLOG_URL,
    'blog_desc': BLOG_DESCRIPTION,
    'translations': TRANSLATIONS,
    'license': LICENSE,
    'search_form': SEARCH_FORM,
    'disqus_forum': DISQUS_FORUM,
    'content_footer': CONTENT_FOOTER,
    'rss_path': RSS_PATH,
    'rss_link': RSS_LINK,
    # Locale-dependent links for the sidebar
    # You should provide a key-value pair for each used language.
    'sidebar_links': {
        'en': (
            ('/' + os.path.join(ARCHIVE_PATH, ARCHIVE_FILENAME), 'Archives'),
            ('/categories/index.html', 'Tags'),
            ('/stories/about-nikola.html', 'About Nikola'),
            ('/stories/handbook.html', 'The Nikola Handbook'),
            ('http://nikola.ralsina.com.ar', 'Powered by Nikola!'),
            ),
        }
    }
<|endoftext|>


<commit_before>"""The fsl module provides classes for interfacing with the `FSL
<http://www.fmrib.ox.ac.uk/fsl/index.html>`_ command line tools.  This
was written to work with FSL version 4.1.4.

Examples
--------
See the docstrings of the individual classes for examples.

"""

import os
from glob import glob
import warnings

import numpy as np

from nipype.interfaces.fsl.base import FSLCommand, FSLInfo, NEW_FSLCommand,\
    FSLTraitedSpec
from nipype.utils.filemanip import list_to_filename
from nipype.interfaces.base import Bunch, traits, TraitedSpec,\
    OutputMultiPath, File
from nipype.utils.misc import isdefined
from nipype.utils.docparse import get_doc

warn = warnings.warn
warnings.filterwarnings('always', category=UserWarning)

class SmoothInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", position=0, mandatory=True)
    fwhm = traits.Float(argstr="-kernel gauss %f -fmean", position=1,
                            mandatory=True)
    outfile = File(argstr="%s", position=2, genfile=True)

class SmoothOutputSpec(TraitedSpec):
    smoothedimage = File(exists=True)

class Smooth(NEW_FSLCommand):
    '''Use fslmaths to smooth the image

    This is dumb, of course - we should use nipy for such things! But it is a
    step along the way to get the "standard" FSL pipeline in place.

    This is meant to be a throwaway class, so it's not currently very robust.
    Effort would be better spent integrating basic numpy into nipype'''

    input_spec = SmoothInputSpec
    output_spec = SmoothOutputSpec
    _cmd = 'fslmaths'

    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()['smoothedimage']
        return None

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['smoothedimage'] = self._gen_fname(self.inputs.infile,
                                self.inputs.outfile, suffix='_smooth')
        return outputs

    def _format_arg(self, name, trait_spec, value):
        if name == 'fwhm':
            # ohinds: convert fwhm to stddev
            return super(Smooth, self)._format_arg(name, trait_spec, float(value) / np.sqrt(8 * np.log(2)))
        else:
            return super(Smooth, self)._format_arg(name, trait_spec, value)

class MergeInputSpec(FSLTraitedSpec):
    infiles = traits.List(File(exists=True), argstr="%s", position=2, mandatory=True)
    dimension = traits.Enum('t', 'x', 'y', 'z', argstr="-%s", position=0,
                            desc="dimension along which the file will be merged",
                            mandatory=True)
    outfile = File(argstr="%s", position=1, genfile=True)

class MergeOutputSpec(TraitedSpec):
    outfile = File(exists=True)

class Merge(NEW_FSLCommand):
    """Use fslmerge to concatenate images
    """

    _cmd = 'fslmerge'
    input_spec = MergeInputSpec
    output_spec = MergeOutputSpec

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['outfile'] = self._gen_fname(self.inputs.infiles[0],
                                self.inputs.outfile, suffix='_merged')
        return outputs
    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()[name]
        return None

class ExtractRoiInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", position=0, desc="input file", mandatory=True)
    outfile = File(argstr="%s", position=1, desc="output file", genfile=True)
    xmin = traits.Float(argstr="%f", position=2)
    xsize = traits.Float(argstr="%f", position=3)
    ymin = traits.Float(argstr="%f", position=4)
    ysize = traits.Float(argstr="%f", position=5)
    zmin = traits.Float(argstr="%f", position=6)
    zsize = traits.Float(argstr="%f", position=7)
    tmin = traits.Int(argstr="%d", position=8)
    tsize = traits.Int(argstr="%d", position=9)

class ExtractRoiOutputSpec(TraitedSpec):
    outfile = File(exists=True)

class ExtractRoi(NEW_FSLCommand):
    """Uses FSL Fslroi command to extract region of interest (ROI)
    from an image.

    You can a) take a 3D ROI from a 3D data set (or if it is 4D, the
    same ROI is taken from each time point and a new 4D data set is
    created), b) extract just some time points from a 4D data set, or
    c) control time and space limits to the ROI.  Note that the
    arguments are minimum index and size (not maximum index).  So to
    extract voxels 10 to 12 inclusive you would specify 10 and 3 (not
    10 and 12).

    >>> from nipype.interfaces import fsl
    >>> fslroi = fsl.ExtractRoi(infile='foo.nii', outfile='bar.nii', \
                                tmin=0, tsize=1)
    >>> fslroi.cmdline
    'fslroi foo.nii bar.nii 0 1'
    """

    _cmd = 'fslroi'
    input_spec = ExtractRoiInputSpec
    output_spec = ExtractRoiOutputSpec

    def _list_outputs(self):
        """Create a Bunch which contains all possible files generated
        by running the interface.  Some files are always generated, others
        depending on which ``inputs`` options are set.

        Returns
        -------
        outputs : Bunch object
            Bunch object containing all possible files generated by
            interface object.

            If None, file was not generated
            Else, contains path, filename of generated outputfile

        """
        outputs = self._outputs().get()
        outputs['outfile'] = self._gen_fname(self.inputs.infile,
                                self.inputs.outfile, suffix='_roi')
        return outputs

    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()[name]
        return None

class SplitInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", position = 0, desc="input filename")
    outbasename = traits.Str(argstr="%s", position=1, desc="outputs prefix")
    dimension = traits.Enum('t','x','y','z', argstr="-%s", position=2, desc="dimension along which the file will be split")

class SplitOutputSpec(TraitedSpec):
    outfiles = OutputMultiPath(File(exists=True))

class Split(NEW_FSLCommand):
    """Uses FSL Fslsplit command to separate a volume into images in
    time, x, y or z dimension.
    """
    _cmd = 'fslsplit'
    input_spec = SplitInputSpec
    output_spec = SplitOutputSpec

    def _list_outputs(self):
        """Create a Bunch which contains all possible files generated
        by running the interface.  Some files are always generated, others
        depending on which ``inputs`` options are set.

        Returns
        -------
        outputs : Bunch object
            Bunch object containing all possible files generated by
            interface object.

            If None, file was not generated
            Else, contains path, filename of generated outputfile

        """
        outputs = self._outputs().get()
        type, ext = FSLInfo.outputtype()
        outbase = 'vol*'
        if isdefined(self.inputs.outbasename):
            outbase = '%s*' % self.inputs.outbasename
        outputs['outfiles'] = sorted(glob(os.path.join(os.getcwd(),
                                                    outbase + ext)))
        return outputs

class ImageMaths(FSLCommand):
    """Use FSL fslmaths command to allow mathematical manipulation of images
    """
    opt_map = {}

    @property
    def cmd(self):
        """sets base command, immutable"""
        return 'fslmaths'

    def inputs_help(self):
        """Print command line documentation for fslmaths."""
        print get_doc(self.cmd, self.opt_map, trap_error=False)

    def _populate_inputs(self):
        self.inputs = Bunch(infile=None,
                            infile2=None,
                            outfile=None,
                            optstring=None,
                            suffix=None,  # ohinds: outfile suffix
                            outdatatype=None)  # ohinds: change outdatatype

    def _get_outfile(self):
        suffix = '_maths'  # ohinds: build suffix
        if self.inputs.suffix:
            suffix = self.inputs.suffix
        return self._gen_fname(self.inputs.infile,
                                  self.inputs.outfile,
                                  suffix=suffix)

    def _parse_inputs(self):
        """validate fsl fslmaths options"""

        # Add infile and outfile to the args if they are specified
        allargs = []
        if self.inputs.infile:
            allargs.insert(0, list_to_filename(self.inputs.infile))
            self.outfile = self._get_outfile()
        if self.inputs.optstring:
            allargs.insert(1, self.inputs.optstring)

        if self.inputs.infile2:
            allargs.insert(2, list_to_filename(self.inputs.infile2))
            allargs.insert(3, self.outfile)
        else:
            allargs.insert(2, self.outfile)

        if self.inputs.outdatatype:  # ohinds: assign odt
            allargs.append('-odt ' + self.inputs.outdatatype)

        return allargs

    def run(self, infile=None, infile2=None, outfile=None, **inputs):
        """Execute the command.
        >>> from nipype.interfaces import fsl
        >>> import os
        >>> maths = fsl.ImageMaths(infile='foo.nii', optstring= '-add 5', \
                                   outfile='foo_maths.nii')
        >>> maths.cmdline
        'fslmaths foo.nii -add 5 foo_maths.nii'

        """

        if infile:
            self.inputs.infile = infile
        if infile2:
            self.inputs.infile = infile2
        if not self.inputs.infile:
            raise AttributeError('Fslmaths requires an input file')
        if outfile:
            self.inputs.outfile = outfile
        self.inputs.update(**inputs)
        return super(ImageMaths, self).run()

    def outputs_help(self):
        """
        Parameters
        ----------
        (all default to None and are unset)

        outfile : /path/to/outfile
            path and filename to computed image
        """
        print self.outputs_help.__doc__

    def outputs(self):
        """Returns a :class:`nipype.interfaces.base.Bunch` with outputs

        Parameters
        ----------
        (all default to None and are unset)

            outfile : string,file
                path/name of file of fslmaths image
        """
        outputs = Bunch(outfile=None)
        return outputs

    def aggregate_outputs(self):
        """Create a Bunch which contains all possible files generated
        by running the interface.  Some files are always generated, others
        depending on which ``inputs`` options are set.

        Returns
        -------
        outputs : Bunch object
            Bunch object containing all possible files generated by
            interface object.

            If None, file was not generated
            Else, contains path, filename of generated outputfile

        """
        outputs = self.outputs()
        outputs.outfile = glob(self._get_outfile())[0]
        return outputs
<commit_msg>traited fsl.IamgeMaths<commit_after>"""The fsl module provides classes for interfacing with the `FSL
<http://www.fmrib.ox.ac.uk/fsl/index.html>`_ command line tools.  This
was written to work with FSL version 4.1.4.

Examples
--------
See the docstrings of the individual classes for examples.

"""

import os
from glob import glob
import warnings

import numpy as np

from nipype.interfaces.fsl.base import FSLCommand, FSLInfo, NEW_FSLCommand,\
    FSLTraitedSpec
from nipype.utils.filemanip import list_to_filename
from nipype.interfaces.base import Bunch, traits, TraitedSpec,\
    OutputMultiPath, File
from nipype.utils.misc import isdefined
from nipype.utils.docparse import get_doc

warn = warnings.warn
warnings.filterwarnings('always', category=UserWarning)

class SmoothInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", position=0, mandatory=True)
    fwhm = traits.Float(argstr="-kernel gauss %f -fmean", position=1,
                            mandatory=True)
    outfile = File(argstr="%s", position=2, genfile=True)

class SmoothOutputSpec(TraitedSpec):
    smoothedimage = File(exists=True)

class Smooth(NEW_FSLCommand):
    '''Use fslmaths to smooth the image

    This is dumb, of course - we should use nipy for such things! But it is a
    step along the way to get the "standard" FSL pipeline in place.

    This is meant to be a throwaway class, so it's not currently very robust.
    Effort would be better spent integrating basic numpy into nipype'''

    input_spec = SmoothInputSpec
    output_spec = SmoothOutputSpec
    _cmd = 'fslmaths'

    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()['smoothedimage']
        return None

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['smoothedimage'] = self._gen_fname(self.inputs.infile,
                                self.inputs.outfile, suffix='_smooth')
        return outputs

    def _format_arg(self, name, trait_spec, value):
        if name == 'fwhm':
            # ohinds: convert fwhm to stddev
            return super(Smooth, self)._format_arg(name, trait_spec, float(value) / np.sqrt(8 * np.log(2)))
        else:
            return super(Smooth, self)._format_arg(name, trait_spec, value)

class MergeInputSpec(FSLTraitedSpec):
    infiles = traits.List(File(exists=True), argstr="%s", position=2, mandatory=True)
    dimension = traits.Enum('t', 'x', 'y', 'z', argstr="-%s", position=0,
                            desc="dimension along which the file will be merged",
                            mandatory=True)
    outfile = File(argstr="%s", position=1, genfile=True)

class MergeOutputSpec(TraitedSpec):
    outfile = File(exists=True)

class Merge(NEW_FSLCommand):
    """Use fslmerge to concatenate images
    """

    _cmd = 'fslmerge'
    input_spec = MergeInputSpec
    output_spec = MergeOutputSpec

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['outfile'] = self._gen_fname(self.inputs.infiles[0],
                                self.inputs.outfile, suffix='_merged')
        return outputs

    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()[name]
        return None

class ExtractRoiInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", position=0, desc="input file", mandatory=True)
    outfile = File(argstr="%s", position=1, desc="output file", genfile=True)
    xmin = traits.Float(argstr="%f", position=2)
    xsize = traits.Float(argstr="%f", position=3)
    ymin = traits.Float(argstr="%f", position=4)
    ysize = traits.Float(argstr="%f", position=5)
    zmin = traits.Float(argstr="%f", position=6)
    zsize = traits.Float(argstr="%f", position=7)
    tmin = traits.Int(argstr="%d", position=8)
    tsize = traits.Int(argstr="%d", position=9)

class ExtractRoiOutputSpec(TraitedSpec):
    outfile = File(exists=True)

class ExtractRoi(NEW_FSLCommand):
    """Uses FSL Fslroi command to extract region of interest (ROI)
    from an image.

    You can a) take a 3D ROI from a 3D data set (or if it is 4D, the
    same ROI is taken from each time point and a new 4D data set is
    created), b) extract just some time points from a 4D data set, or
    c) control time and space limits to the ROI.  Note that the
    arguments are minimum index and size (not maximum index).  So to
    extract voxels 10 to 12 inclusive you would specify 10 and 3 (not
    10 and 12).

    >>> from nipype.interfaces import fsl
    >>> fslroi = fsl.ExtractRoi(infile='foo.nii', outfile='bar.nii', \
                                tmin=0, tsize=1)
    >>> fslroi.cmdline
    'fslroi foo.nii bar.nii 0 1'
    """

    _cmd = 'fslroi'
    input_spec = ExtractRoiInputSpec
    output_spec = ExtractRoiOutputSpec

    def _list_outputs(self):
        """Create a Bunch which contains all possible files generated
        by running the interface.  Some files are always generated, others
        depending on which ``inputs`` options are set.

        Returns
        -------
        outputs : Bunch object
            Bunch object containing all possible files generated by
            interface object.

            If None, file was not generated
            Else, contains path, filename of generated outputfile

        """
        outputs = self._outputs().get()
        outputs['outfile'] = self._gen_fname(self.inputs.infile,
                                self.inputs.outfile, suffix='_roi')
        return outputs

    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()[name]
        return None

class SplitInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", position = 0, desc="input filename")
    outbasename = traits.Str(argstr="%s", position=1, desc="outputs prefix")
    dimension = traits.Enum('t','x','y','z', argstr="-%s", position=2, desc="dimension along which the file will be split")

class SplitOutputSpec(TraitedSpec):
    outfiles = OutputMultiPath(File(exists=True))

class Split(NEW_FSLCommand):
    """Uses FSL Fslsplit command to separate a volume into images in
    time, x, y or z dimension.
    """
    _cmd = 'fslsplit'
    input_spec = SplitInputSpec
    output_spec = SplitOutputSpec

    def _list_outputs(self):
        """Create a Bunch which contains all possible files generated
        by running the interface.  Some files are always generated, others
        depending on which ``inputs`` options are set.

        Returns
        -------
        outputs : Bunch object
            Bunch object containing all possible files generated by
            interface object.

            If None, file was not generated
            Else, contains path, filename of generated outputfile

        """
        outputs = self._outputs().get()
        type, ext = FSLInfo.outputtype()
        outbase = 'vol*'
        if isdefined(self.inputs.outbasename):
            outbase = '%s*' % self.inputs.outbasename
        outputs['outfiles'] = sorted(glob(os.path.join(os.getcwd(),
                                                    outbase + ext)))
        return outputs

class ImageMathsInputSpec(FSLTraitedSpec):
    infile = File(exists=True, argstr="%s", mandatory=True, position=0)
    infile2 = File(exists=True, argstr="%s", position=2)
    outfile = File(argstr="%s", position=3, genfile=True)
    optstring = traits.Str(argstr="%s", mandatory=True, position=1,
                           desc="string defining the operation, i. e. -add")
    suffix = traits.Str(argstr="%s", desc="outfile suffix")
    outdatatype = traits.Enum('char', 'short', 'int', 'float', 'double',
                              'input', argstr="-odt %s", position=4,
                              desc="output datatype, one of (char, short, int, float, double, input)")

class ImageMathsOutputSpec(TraitedSpec):
    outfile = File(exists=True)

class ImageMaths(NEW_FSLCommand):
    """Use FSL fslmaths command to allow mathematical manipulation of images
    Example:
    >>> from nipype.interfaces import fsl
    >>> import os
    >>> maths = fsl.ImageMaths(infile='foo.nii', optstring= '-add 5', \
                               outfile='foo_maths.nii')
    >>> maths.cmdline
    'fslmaths foo.nii -add 5 foo_maths.nii'

    """
    input_spec = ImageMathsInputSpec
    output_spec = ImageMathsOutputSpec

    _cmd = 'fslmaths'

    def _gen_filename(self, name):
        if name == 'outfile':
            return self._list_outputs()[name]
        return None

    def _list_outputs(self):
        suffix = '_maths'  # ohinds: build suffix
        if isdefined(self.inputs.suffix):
            suffix = self.inputs.suffix
        outputs = self._outputs().get()
        outputs['outfile'] = self._gen_fname(self.inputs.infile,
                                self.inputs.outfile, suffix=suffix)
        return outputs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2018-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import sys
import collections
import numpy
from openquake.baselib import sap
from openquake.calculators.extract import Extractor
from openquake.calculators import views

def getdata(what, calc_ids, samplesites):
    extractors = [Extractor(calc_id) for calc_id in calc_ids]
    extractor = extractors[0]
    sitecol = extractor.get('sitecol')
    oq = extractor.oqparam
    imtls = oq.imtls
    poes = oq.poes
    if len(sitecol) > samplesites:
        numpy.random.seed(samplesites)
        sids = numpy.random.choice(len(sitecol), samplesites, replace=False)
    else:  # keep all sites
        sids = sitecol['sids']
    arrays = [extractor.get(what + '?kind=mean').mean[sids]]
    extractor.close()
    for extractor in extractors[1:]:
        oq = extractor.oqparam
        numpy.testing.assert_equal(extractor.get('sitecol').array, sitecol)
        if what == 'hcurves':
            numpy.testing.assert_equal(oq.imtls.array, imtls.array)
        elif what == 'hmaps':
            numpy.testing.assert_equal(oq.poes, poes)
        arrays.append(extractor.get(what + '?kind=mean').mean[sids])
        extractor.close()
    return sids, imtls, poes, numpy.array(arrays)  # shape (C, N, L)

def get_diff_idxs(array, rtol, atol):
    """
    Given an array with (C, N, L) values, being the first the reference value,
    compute the relative differences and discard the one below the tolerance.
    :returns: indices where there are sensible differences.
    """
    C, N, L = array.shape
    diff_idxs = set()  # indices of the sites with differences
    for c in range(1, C):
        for n in range(N):
            if not numpy.allclose(array[c, n], array[0, n], rtol, atol):
                diff_idxs.add(n)
    return numpy.fromiter(diff_idxs, int)

@sap.script
def compare(what, imt, calc_ids, files, samplesites=100, rtol=.1, atol=1E-4):
    """
    Compare the hazard curves or maps of two or more calculations
    """
    sids, imtls, poes, arrays = getdata(what, calc_ids, samplesites)
    try:
        levels = imtls[imt]
    except KeyError:
        sys.exit(
            '%s not found. The available IMTs are %s' % (imt, list(imtls)))
    imt2idx = {imt: i for i, imt in enumerate(imtls)}
    head = ['site_id'] if files else ['site_id', 'calc_id']
    if what == 'hcurves':
        array_imt = arrays[:, :, imtls(imt)]
        header = head + ['%.5f' % lvl for lvl in levels]
    else:  # hmaps
        array_imt = arrays[:, :, imt2idx[imt]]
        header = head + [str(poe) for poe in poes]
    rows = collections.defaultdict(list)
    diff_idxs = get_diff_idxs(array_imt, rtol, atol)
    if len(diff_idxs) == 0:
        print('There are no differences within the tolerance of %d%%' %
              (rtol * 100))
        return
    arr = array_imt.transpose(1, 0, 2)  # shape (N, C, L)
    for sid, array in sorted(zip(sids[diff_idxs], arr[diff_idxs])):
        for calc_id, cols in zip(calc_ids, array):
            if files:
                rows[calc_id].append([sid] + list(cols))
            else:
                rows['all'].append([sid, calc_id] + list(cols))
    if files:
        fdict = {calc_id: open('%s.txt' % calc_id, 'w')
                 for calc_id in calc_ids}
        for calc_id, f in fdict.items():
            f.write(views.rst_table(rows[calc_id], header))
            print('Generated %s' % f.name)
    else:
        print(views.rst_table(rows['all'], header))

compare.arg('what', 'hmaps or hcurves', choices={'hmaps', 'hcurves'})
compare.arg('imt', 'intensity measure type to compare')
compare.arg('calc_ids', 'calculation IDs', type=int, nargs='+')
compare.flg('files', 'write the results in multiple files')
compare.opt('samplesites', 'number of sites to sample', type=int)
compare.opt('rtol', 'relative tolerance', type=float)
compare.opt('atol', 'absolute tolerance', type=float)
<commit_msg>Small fix in oq compare [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2018-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import sys
import collections
import numpy
from openquake.baselib import sap
from openquake.calculators.extract import Extractor
from openquake.calculators import views

def getdata(what, calc_ids, samplesites):
    extractors = [Extractor(calc_id) for calc_id in calc_ids]
    extractor = extractors[0]
    sitecol = extractor.get('sitecol')
    oq = extractor.oqparam
    imtls = oq.imtls
    poes = oq.poes
    if len(sitecol) > samplesites:
        numpy.random.seed(samplesites)
        sids = numpy.random.choice(len(sitecol), samplesites, replace=False)
    else:  # keep all sites
        sids = sitecol['sids']
    arrays = [extractor.get(what + '?kind=mean').mean[sids]]
    extractor.close()
    for extractor in extractors[1:]:
        oq = extractor.oqparam
        numpy.testing.assert_equal(
            extractor.get('sitecol').array, sitecol.array)
        if what == 'hcurves':
            numpy.testing.assert_equal(oq.imtls.array, imtls.array)
        elif what == 'hmaps':
            numpy.testing.assert_equal(oq.poes, poes)
        arrays.append(extractor.get(what + '?kind=mean').mean[sids])
        extractor.close()
    return sids, imtls, poes, numpy.array(arrays)  # shape (C, N, L)

def get_diff_idxs(array, rtol, atol):
    """
    Given an array with (C, N, L) values, being the first the reference value,
    compute the relative differences and discard the one below the tolerance.
    :returns: indices where there are sensible differences.
    """
    C, N, L = array.shape
    diff_idxs = set()  # indices of the sites with differences
    for c in range(1, C):
        for n in range(N):
            if not numpy.allclose(array[c, n], array[0, n], rtol, atol):
                diff_idxs.add(n)
    return numpy.fromiter(diff_idxs, int)

@sap.script
def compare(what, imt, calc_ids, files, samplesites=100, rtol=.1, atol=1E-4):
    """
    Compare the hazard curves or maps of two or more calculations
    """
    sids, imtls, poes, arrays = getdata(what, calc_ids, samplesites)
    try:
        levels = imtls[imt]
    except KeyError:
        sys.exit(
            '%s not found. The available IMTs are %s' % (imt, list(imtls)))
    imt2idx = {imt: i for i, imt in enumerate(imtls)}
    head = ['site_id'] if files else ['site_id', 'calc_id']
    if what == 'hcurves':
        array_imt = arrays[:, :, imtls(imt)]
        header = head + ['%.5f' % lvl for lvl in levels]
    else:  # hmaps
        array_imt = arrays[:, :, imt2idx[imt]]
        header = head + [str(poe) for poe in poes]
    rows = collections.defaultdict(list)
    diff_idxs = get_diff_idxs(array_imt, rtol, atol)
    if len(diff_idxs) == 0:
        print('There are no differences within the tolerance of %d%%' %
              (rtol * 100))
        return
    arr = array_imt.transpose(1, 0, 2)  # shape (N, C, L)
    for sid, array in sorted(zip(sids[diff_idxs], arr[diff_idxs])):
        for calc_id, cols in zip(calc_ids, array):
            if files:
                rows[calc_id].append([sid] + list(cols))
            else:
                rows['all'].append([sid, calc_id] + list(cols))
    if files:
        fdict = {calc_id: open('%s.txt' % calc_id, 'w')
                 for calc_id in calc_ids}
        for calc_id, f in fdict.items():
            f.write(views.rst_table(rows[calc_id], header))
            print('Generated %s' % f.name)
    else:
        print(views.rst_table(rows['all'], header))

compare.arg('what', 'hmaps or hcurves', choices={'hmaps', 'hcurves'})
compare.arg('imt', 'intensity measure type to compare')
compare.arg('calc_ids', 'calculation IDs', type=int, nargs='+')
compare.flg('files', 'write the results in multiple files')
compare.opt('samplesites', 'number of sites to sample', type=int)
compare.opt('rtol', 'relative tolerance', type=float)
compare.opt('atol', 'absolute tolerance', type=float)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import glob
from datetime import date
from collections import namedtuple
import requests
import rdflib
from rdflib.extras import infixowl
from lxml import etree
from utils import add_hierarchy, makeGraph, rowParse
from IPython import embed

PScheme = namedtuple('PScheme', ['curie', 'name', 'species', 'devstage', 'citation'])
PScheme('ilx:something','some parcellation scheme concept','NCBITaxon:1234','adult','Bob Jones said so')

PARC_SUPER = ('ilx:ilx_brain_parcellation_concept', 'Brain parcellation scheme concept')
TODAY = date.isoformat(date.today())

def make_scheme(scheme, parent=PARC_SUPER[0]):  # ick...
    out = [
        (scheme.curie, rdflib.RDF.type, rdflib.OWL.Class),
        (scheme.curie, rdflib.RDFS.label, scheme.name),
        (scheme.curie, rdflib.RDFS.subClassOf, parent),
        (scheme.curie, 'ilx:has_taxon', scheme.species),
        (scheme.curie, 'ilx:has_developmental_stage', scheme.devstage),
        (scheme.curie, 'OBOANN:definingCitation', scheme.citation),
    ]
    return out

def add_scheme(graph, scheme, parent=None):
    if not parent:
        [graph.add_node(*triple) for triple in make_scheme(scheme)]
    else:
        [graph.add_node(*triple) for triple in make_scheme(scheme, parent)]

def parcellation_schemes(ontids):
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/'
    filename = 'parcellation'
    ontid = ONT_PATH + filename + '.ttl'
    PREFIXES = {
        'ilx':'http://uri.interlex.org/base/',
    }
    new_graph = makeGraph(filename, PREFIXES)
    new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
    new_graph.add_node(ontid, rdflib.RDFS.label, 'NIF collected parcellation schemes ontology')
    new_graph.add_node(ontid, rdflib.RDFS.comment, 'Brain parcellation schemes as represented by root concepts.')
    new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)

    for import_id in sorted(ontids):
        new_graph.add_node(ontid, rdflib.OWL.imports, import_id)
    new_graph.add_node(PARC_SUPER[0], rdflib.RDF.type, rdflib.OWL.Class)
    new_graph.add_node(PARC_SUPER[0], rdflib.RDFS.label, PARC_SUPER[1])
    new_graph.write(delay=True)

def mouse_brain_atlas():
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/generated/'
    filename = 'mbaslim'
    ontid = ONT_PATH + filename + '.ttl'
    PREFIXES = {
        'ilx':'http://uri.interlex.org/base/',
        'obo':'http://purl.obolibrary.org/obo/',
        'OBOANN':'http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#',  # FIXME needs to die a swift death
        'MBA':'http://api.brain-map.org/api/v2/data/Structure/',
        'owl':'http://www.w3.org/2002/07/owl#',  # this should autoadd for prefixes but doesnt!?
    }
    new_graph = makeGraph(filename, PREFIXES)
    new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
    new_graph.add_node(ontid, rdflib.RDFS.label, 'Allen Mouse Brain Atlas Ontology')
    new_graph.add_node(ontid, rdflib.RDFS.comment, 'This file is automatically generated from the Allen Brain Atlas API')
    new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)
    meta = PScheme('ilx:allen_brain_parc_region',
                   'Allen Mouse Brain Atlas parcellation concept',
                   'NCBITaxon:10090',
                   'adult P56',
                   'http://help.brain-map.org/download/attachments/2818169/AllenReferenceAtlas_v2_2011.pdf?version=1&modificationDate=1319667383440')  # yay no doi! wat
    add_scheme(new_graph, meta)
    #superclass = rdflib.URIRef('http://uri.interlex.org/base/ilx_allen_brain_parc_region')
    #new_graph.add_node(superclass, rdflib.RDFS.label, 'Allen Mouse Brain Atlas brain region')
    #new_graph.add_node(superclass, rdflib.RDFS.subClassOf, PARC_SUPER[0])
    c_prefix, c_suffix = meta.curie.split(':')
    superclass = new_graph.namespaces[c_prefix][c_suffix]

    aba_map = {
        'acronym':new_graph.namespaces['OBOANN']['acronym'],  # FIXME all this is BAD WAY
        #'id':namespaces['ABA'],
        'name':rdflib.RDFS.label,
        #'parent_structure_id':rdflib.RDFS['subClassOf'],
        'safe_name':new_graph.namespaces['OBOANN']['synonym'],
    }

    def aba_trips(node_d):
        output = []
        parent = 'MBA:' + str(node_d['id'])  # FIXME HRM what happens if we want to change ABA:  OH LOOK
        for key, edge in sorted(aba_map.items()):
            value = node_d[key]
            if not value:
                continue
            elif key == 'safe_name' and value == node_d['name']:
                continue  # don't duplicate labels as synonyms
            output.append( (parent, edge, value) )
        return output

    root = 997  # for actual parts of the brain
    url = 'http://api.brain-map.org/api/v2/tree_search/Structure/997.json?descendants=true'
    resp = requests.get(url).json()
    for node_d in resp['msg']:
        if node_d['id'] == 997:  # FIXME need a better place to document this :/
            node_d['name'] = 'allen mouse brain atlas parcellation root'
            node_d['safe_name'] = 'allen mouse brain atlas parcellation root'
            node_d['acronym'] = 'mbaroot'
        ident = new_graph.namespaces['MBA'][str(node_d['id'])]
        cls = infixowl.Class(ident, graph=new_graph.g)
        cls.subClassOf = [superclass]
        parent = node_d['parent_structure_id']
        if parent:
            parent = new_graph.namespaces['MBA'][str(parent)]
            #add_hierarchy(new_graph.g, parent, rdflib.URIRef('http://uri.interlex.org/base/proper_part_of'), cls)
            add_hierarchy(new_graph.g, parent, rdflib.URIRef('http://purl.obolibrary.org/obo/BFO_0000050'), cls)

        for t in aba_trips(node_d):
            new_graph.add_node(*t)

    new_graph.write(delay=True)
    return ontid

class cocomac(rowParse):
    superclass = rdflib.URIRef('http://uri.interlex.org/base/ilx_cocomac_parc_region')
    def __init__(self, graph, rows, header):
        self.g = graph
        super().__init__(rows, header)#, order=[0])

    def ID(self, value):
        self.identifier = 'cocomac:' + value  # safe because reset every row (ish)
        self.g.add_node(self.identifier, rdflib.RDF.type, rdflib.OWL.Class)
        self.g.add_node(self.identifier, rdflib.RDFS.subClassOf, self.superclass)

    def Key(self, value):
        pass

    def Summary(self, value):
        pass

    def Acronym(self, value):
        self.g.add_node(self.identifier, 'OBOANN:acronym', value)

    def FullName(self, value):
        self.g.add_node(self.identifier, rdflib.RDFS.label, value)

    def LegacyID(self, value):
        if value:  # FIXME should fix in add_node
            self.g.add_node(self.identifier, 'OBOANN:acronym', value)

    def BrainInfoID(self, value):
        pass

def cocomac_make():
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/generated/'
    filename = 'cocomacslim'
    ontid = ONT_PATH + filename + '.ttl'
    PREFIXES = {
        'ilx':'http://uri.interlex.org/base/',
        'OBOANN':'http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#',  # FIXME needs to die a swift death
        'cocomac':'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT%20*%20from%20BrainMaps_BrainSiteAcronyms%20where%20ID=',  # looking for better options
    }
    new_graph = makeGraph(filename, PREFIXES)
    new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
    new_graph.add_node(ontid, rdflib.RDFS.label, 'CoCoMac terminology')
    new_graph.add_node(ontid, rdflib.RDFS.comment, 'This file is automatically generated from the CoCoMac database on the terms from BrainMaps_BrainSiteAcronyms.')
    new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)
    meta = PScheme(cocomac.superclass,
                   'CoCoMac terminology parcellation concept',
                   'NCBITaxon:9544',
                   'various',
                   'problem')  # problems detected :/
    add_scheme(new_graph, meta)
    #new_graph.add_node(cocomac.superclass, rdflib.RDFS.label, 'CoCoMac terminology brain region')
    #new_graph.add_node(cocomac.superclass, rdflib.RDFS.subClassOf, PARC_SUPER[0])

    #url = 'http://cocomac.g-node.org/services/search_wizard.php?T=BrainMaps_BrainSiteAcronyms&x0=&limit=3000&page=1&format=json'
    #resp = json.loads(requests.get(url).json())  # somehow everything is double escaped :x
    url = 'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT * from BrainMaps_BrainSiteAcronyms;&format=json'
    #url = 'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT%20*%20from%20BrainMaps_BrainSiteAcronyms;&format=json'
    #tab_name = resp['resultTable']
    #table = resp['tables'][tab_name]
    table = requests.get(url).json()
    fields = table['fields']
    data = table['data']
    #rows = sorted(data.values())
    cocomac(new_graph, data.values(), fields)

    new_graph.write(delay=True)
    return ontid

def fmri_atlases():
    ATLAS_PATH = '/usr/share/fsl/data/atlases/'
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/generated/'

    # ingest the structured xml files and get the name of the atlas/parcellation scheme
    # ingest the labels
    # for each xml files/set of nii files generate a ttl file

    ontids = []
    for xmlfile in glob.glob(ATLAS_PATH + '*.xml'):
        tree = etree.parse(xmlfile)
        name = tree.xpath('header//name')[0].text

        filename = os.path.splitext(os.path.basename(xmlfile))[0]
        ontid = ONT_PATH + filename + '.ttl'
        PREFIXES = {
            '':ontid+'/',
            'ilx':'http://uri.interlex.org/base/',
            'skos':'http://www.w3.org/2004/02/skos/core#',
            'NCBITaxon':'http://purl.obolibrary.org/obo/NCBITaxon_',
            'OBOANN':'http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#',  # FIXME needs to die a swift death
        }
        new_graph = makeGraph(filename, PREFIXES)
        new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
        new_graph.add_node(ontid, rdflib.RDFS.label, name)
        new_graph.add_node(ontid, rdflib.RDFS.comment, 'This file is automatically generated from the %s file in the FSL atlas collection.' % xmlfile)
        new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)

        meta = PScheme('ilx:placeholder_' + name.replace(' ','_'),
                       name + ' parcellation concept',
                       'NCBITaxon:9606',
                       'adult',
                       'http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Atlases')  # problems detected :/
        add_scheme(new_graph, meta)

        sn = tree.xpath('header//shortname')
        if sn:
            new_graph.add_node(ontid, rdflib.namespace.SKOS.altLabel, sn[0].text)

        #tree.xpath('header//shortname').text
        for node in tree.xpath('data//label'):
            id_ = ':' + node.get('index')
            label = node.text
            new_graph.add_node(id_, rdflib.RDFS.subClassOf, meta.curie)
            new_graph.add_node(id_, rdflib.RDF.type, rdflib.OWL.Class)
            new_graph.add_node(id_, rdflib.RDFS.label, label)
        #print([(l.get('index'),l.text) for l in tree.xpath('data//label')])
        new_graph.write(delay=True)
        ontids.append(ontid)

    #embed()
    return ontids

def main():
    oper = makeGraph('', {})
    oper.reset_writeloc()
    fs = fmri_atlases()
    c = cocomac_make()
    m = mouse_brain_atlas()
    fs.extend([c, m])
    parcellation_schemes(fs)
    oper.owlapi_conversion()

    # make a protege catalog file to simplify life
    uriline = '  <uri id="User Entered Import Resolution" name="{ontid}" uri="{filename}"/>'
    xmllines = ['<?xml version="1.0" encoding="UTF-8" standalone="no"?>',
    '<catalog prefer="public" xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog">',] + \
    [uriline.format(ontid=f, filename=f.rsplit('/',1)[-1]) for f in fs] + \
    ['  <group id="Folder Repository, directory=, recursive=true, Auto-Update=true, version=2" prefer="public" xml:base=""/>',
    '</catalog>',]
    xml = '\n'.join(xmllines)
    with open('/tmp/catalog-v001.xml','wt') as f:
        f.write(xml)

    # be sure to run
    # find -name '*.ttl.ttl' -exec sh -c 'a=$(echo "$0" | sed -r "s/.ttl$//") && mv "$0" "$a"' {}  \;
    # to move the converted files

if __name__ == '__main__':
    main()

<commit_msg>updates to prefixes and one temp identifier for parcellations<commit_after>#!/usr/bin/env python3

import os
import glob
from datetime import date
from collections import namedtuple
import requests
import rdflib
from rdflib.extras import infixowl
from lxml import etree
from utils import add_hierarchy, makeGraph, rowParse
from IPython import embed

PScheme = namedtuple('PScheme', ['curie', 'name', 'species', 'devstage', 'citation'])
PScheme('ilx:something','some parcellation scheme concept','NCBITaxon:1234','adult','Bob Jones said so')

PARC_SUPER = ('ilx:ilx_brain_parcellation_scheme_concept', 'Brain parcellation scheme concept')
TODAY = date.isoformat(date.today())

def make_scheme(scheme, parent=PARC_SUPER[0]):  # ick...
    out = [
        (scheme.curie, rdflib.RDF.type, rdflib.OWL.Class),
        (scheme.curie, rdflib.RDFS.label, scheme.name),
        (scheme.curie, rdflib.RDFS.subClassOf, parent),
        (scheme.curie, 'ilx:has_taxon', scheme.species),
        (scheme.curie, 'ilx:has_developmental_stage', scheme.devstage),
        (scheme.curie, 'OBOANN:definingCitation', scheme.citation),
    ]
    return out

def add_scheme(graph, scheme, parent=None):
    if not parent:
        [graph.add_node(*triple) for triple in make_scheme(scheme)]
    else:
        [graph.add_node(*triple) for triple in make_scheme(scheme, parent)]

def parcellation_schemes(ontids):
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/'
    filename = 'parcellation'
    ontid = ONT_PATH + filename + '.ttl'
    PREFIXES = {
        'ilx':'http://uri.interlex.org/base/',
    }
    new_graph = makeGraph(filename, PREFIXES)
    new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
    new_graph.add_node(ontid, rdflib.RDFS.label, 'NIF collected parcellation schemes ontology')
    new_graph.add_node(ontid, rdflib.RDFS.comment, 'Brain parcellation schemes as represented by root concepts.')
    new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)

    for import_id in sorted(ontids):
        new_graph.add_node(ontid, rdflib.OWL.imports, import_id)
    new_graph.add_node(PARC_SUPER[0], rdflib.RDF.type, rdflib.OWL.Class)
    new_graph.add_node(PARC_SUPER[0], rdflib.RDFS.label, PARC_SUPER[1])
    new_graph.write(delay=True)

def mouse_brain_atlas():
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/generated/'
    filename = 'mbaslim'
    ontid = ONT_PATH + filename + '.ttl'
    PREFIXES = {
        'ilx':'http://uri.interlex.org/base/',
        #'obo':'http://purl.obolibrary.org/obo/',
        'BFO':'http://purl.obolibrary.org/obo/BFO_',
        'OBOANN':'http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#',  # FIXME needs to die a swift death
        'MBA':'http://api.brain-map.org/api/v2/data/Structure/',
        'owl':'http://www.w3.org/2002/07/owl#',  # this should autoadd for prefixes but doesnt!?
        'NCBITaxon':'http://purl.obolibrary.org/obo/NCBITaxon_',
    }
    new_graph = makeGraph(filename, PREFIXES)
    new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
    new_graph.add_node(ontid, rdflib.RDFS.label, 'Allen Mouse Brain Atlas Ontology')
    new_graph.add_node(ontid, rdflib.RDFS.comment, 'This file is automatically generated from the Allen Brain Atlas API')
    new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)
    meta = PScheme('ilx:ilx_allen_brain_parc_region',
                   'Allen Mouse Brain Atlas parcellation concept',
                   'NCBITaxon:10090',
                   'adult P56',
                   'http://help.brain-map.org/download/attachments/2818169/AllenReferenceAtlas_v2_2011.pdf?version=1&modificationDate=1319667383440')  # yay no doi! wat
    add_scheme(new_graph, meta)
    #superclass = rdflib.URIRef('http://uri.interlex.org/base/ilx_allen_brain_parc_region')
    #new_graph.add_node(superclass, rdflib.RDFS.label, 'Allen Mouse Brain Atlas brain region')
    #new_graph.add_node(superclass, rdflib.RDFS.subClassOf, PARC_SUPER[0])
    c_prefix, c_suffix = meta.curie.split(':')
    superclass = new_graph.namespaces[c_prefix][c_suffix]

    aba_map = {
        'acronym':new_graph.namespaces['OBOANN']['acronym'],  # FIXME all this is BAD WAY
        #'id':namespaces['ABA'],
        'name':rdflib.RDFS.label,
        #'parent_structure_id':rdflib.RDFS['subClassOf'],
        'safe_name':new_graph.namespaces['OBOANN']['synonym'],
    }

    def aba_trips(node_d):
        output = []
        parent = 'MBA:' + str(node_d['id'])  # FIXME HRM what happens if we want to change ABA:  OH LOOK
        for key, edge in sorted(aba_map.items()):
            value = node_d[key]
            if not value:
                continue
            elif key == 'safe_name' and value == node_d['name']:
                continue  # don't duplicate labels as synonyms
            output.append( (parent, edge, value) )
        return output

    root = 997  # for actual parts of the brain
    url = 'http://api.brain-map.org/api/v2/tree_search/Structure/997.json?descendants=true'
    resp = requests.get(url).json()
    for node_d in resp['msg']:
        if node_d['id'] == 997:  # FIXME need a better place to document this :/
            node_d['name'] = 'allen mouse brain atlas parcellation root'
            node_d['safe_name'] = 'allen mouse brain atlas parcellation root'
            node_d['acronym'] = 'mbaroot'
        ident = new_graph.namespaces['MBA'][str(node_d['id'])]
        cls = infixowl.Class(ident, graph=new_graph.g)
        cls.subClassOf = [superclass]
        parent = node_d['parent_structure_id']
        if parent:
            parent = new_graph.namespaces['MBA'][str(parent)]
            #add_hierarchy(new_graph.g, parent, rdflib.URIRef('http://uri.interlex.org/base/proper_part_of'), cls)
            add_hierarchy(new_graph.g, parent, rdflib.URIRef('http://purl.obolibrary.org/obo/BFO_0000050'), cls)

        for t in aba_trips(node_d):
            new_graph.add_node(*t)

    new_graph.write(delay=True)
    return ontid

class cocomac(rowParse):
    superclass = rdflib.URIRef('http://uri.interlex.org/base/ilx_cocomac_parc_region')
    def __init__(self, graph, rows, header):
        self.g = graph
        super().__init__(rows, header)#, order=[0])

    def ID(self, value):
        self.identifier = 'cocomac:' + value  # safe because reset every row (ish)
        self.g.add_node(self.identifier, rdflib.RDF.type, rdflib.OWL.Class)
        self.g.add_node(self.identifier, rdflib.RDFS.subClassOf, self.superclass)

    def Key(self, value):
        pass

    def Summary(self, value):
        pass

    def Acronym(self, value):
        self.g.add_node(self.identifier, 'OBOANN:acronym', value)

    def FullName(self, value):
        self.g.add_node(self.identifier, rdflib.RDFS.label, value)

    def LegacyID(self, value):
        if value:  # FIXME should fix in add_node
            self.g.add_node(self.identifier, 'OBOANN:acronym', value)

    def BrainInfoID(self, value):
        pass

def cocomac_make():
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/generated/'
    filename = 'cocomacslim'
    ontid = ONT_PATH + filename + '.ttl'
    PREFIXES = {
        'ilx':'http://uri.interlex.org/base/',
        'OBOANN':'http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#',  # FIXME needs to die a swift death
        'cocomac':'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT%20*%20from%20BrainMaps_BrainSiteAcronyms%20where%20ID=',  # looking for better options
        'NCBITaxon':'http://purl.obolibrary.org/obo/NCBITaxon_',
    }
    new_graph = makeGraph(filename, PREFIXES)
    new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
    new_graph.add_node(ontid, rdflib.RDFS.label, 'CoCoMac terminology')
    new_graph.add_node(ontid, rdflib.RDFS.comment, 'This file is automatically generated from the CoCoMac database on the terms from BrainMaps_BrainSiteAcronyms.')
    new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)
    meta = PScheme(cocomac.superclass,
                   'CoCoMac terminology parcellation concept',
                   'NCBITaxon:9544',
                   'various',
                   'problem')  # problems detected :/
    add_scheme(new_graph, meta)
    #new_graph.add_node(cocomac.superclass, rdflib.RDFS.label, 'CoCoMac terminology brain region')
    #new_graph.add_node(cocomac.superclass, rdflib.RDFS.subClassOf, PARC_SUPER[0])

    #url = 'http://cocomac.g-node.org/services/search_wizard.php?T=BrainMaps_BrainSiteAcronyms&x0=&limit=3000&page=1&format=json'
    #resp = json.loads(requests.get(url).json())  # somehow everything is double escaped :x
    url = 'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT * from BrainMaps_BrainSiteAcronyms;&format=json'
    #url = 'http://cocomac.g-node.org/services/custom_sql_query.php?sql=SELECT%20*%20from%20BrainMaps_BrainSiteAcronyms;&format=json'
    #tab_name = resp['resultTable']
    #table = resp['tables'][tab_name]
    table = requests.get(url).json()
    fields = table['fields']
    data = table['data']
    #rows = sorted(data.values())
    cocomac(new_graph, data.values(), fields)

    new_graph.write(delay=True)
    return ontid

def fmri_atlases():
    ATLAS_PATH = '/usr/share/fsl/data/atlases/'
    ONT_PATH = 'http://ontology.neuinfo.org/NIF/ttl/generated/'

    # ingest the structured xml files and get the name of the atlas/parcellation scheme
    # ingest the labels
    # for each xml files/set of nii files generate a ttl file

    ontids = []
    for xmlfile in glob.glob(ATLAS_PATH + '*.xml'):
        tree = etree.parse(xmlfile)
        name = tree.xpath('header//name')[0].text

        filename = os.path.splitext(os.path.basename(xmlfile))[0]
        ontid = ONT_PATH + filename + '.ttl'
        PREFIXES = {
            '':ontid+'/',
            'ilx':'http://uri.interlex.org/base/',
            'skos':'http://www.w3.org/2004/02/skos/core#',
            'NCBITaxon':'http://purl.obolibrary.org/obo/NCBITaxon_',
            'OBOANN':'http://ontology.neuinfo.org/NIF/Backend/OBO_annotation_properties.owl#',  # FIXME needs to die a swift death
        }
        new_graph = makeGraph(filename, PREFIXES)
        new_graph.add_node(ontid, rdflib.RDF.type, rdflib.OWL.Ontology)
        new_graph.add_node(ontid, rdflib.RDFS.label, name)
        new_graph.add_node(ontid, rdflib.RDFS.comment, 'This file is automatically generated from the %s file in the FSL atlas collection.' % xmlfile)
        new_graph.add_node(ontid, rdflib.OWL.versionInfo, TODAY)

        meta = PScheme('ilx:placeholder_' + name.replace(' ','_'),
                       name + ' parcellation concept',
                       'NCBITaxon:9606',
                       'adult',
                       'http://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Atlases')  # problems detected :/
        add_scheme(new_graph, meta)

        sn = tree.xpath('header//shortname')
        if sn:
            new_graph.add_node(ontid, rdflib.namespace.SKOS.altLabel, sn[0].text)

        #tree.xpath('header//shortname').text
        for node in tree.xpath('data//label'):
            id_ = ':' + node.get('index')
            label = node.text
            new_graph.add_node(id_, rdflib.RDFS.subClassOf, meta.curie)
            new_graph.add_node(id_, rdflib.RDF.type, rdflib.OWL.Class)
            new_graph.add_node(id_, rdflib.RDFS.label, label)
        #print([(l.get('index'),l.text) for l in tree.xpath('data//label')])
        new_graph.write(delay=True)
        ontids.append(ontid)

    #embed()
    return ontids

def main():
    oper = makeGraph('', {})
    oper.reset_writeloc()
    fs = fmri_atlases()
    c = cocomac_make()
    m = mouse_brain_atlas()
    fs.extend([c, m])
    parcellation_schemes(fs)
    oper.owlapi_conversion()

    # make a protege catalog file to simplify life
    uriline = '  <uri id="User Entered Import Resolution" name="{ontid}" uri="{filename}"/>'
    xmllines = ['<?xml version="1.0" encoding="UTF-8" standalone="no"?>',
    '<catalog prefer="public" xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog">',] + \
    [uriline.format(ontid=f, filename=f.rsplit('/',1)[-1]) for f in fs] + \
    ['  <group id="Folder Repository, directory=, recursive=true, Auto-Update=true, version=2" prefer="public" xml:base=""/>',
    '</catalog>',]
    xml = '\n'.join(xmllines)
    with open('/tmp/catalog-v001.xml','wt') as f:
        f.write(xml)

    # be sure to run
    # find -name '*.ttl.ttl' -exec sh -c 'a=$(echo "$0" | sed -r "s/.ttl$//") && mv "$0" "$a"' {}  \;
    # to move the converted files

if __name__ == '__main__':
    main()

<|endoftext|>