<commit_before>#!/usr/bin/env python
#
# nConvert v1.1
# Author: ashwini@majestik.net
#
#
# This utility will convert a simple, human-readable Contacts CSV to a format that's
# compatible with Nokia PC Suite. Had to write a utility to do this because standard
# Microsoft Outlook CSV exports weren't reading in Nokia PC Suite. Figured out it was because
# NPCS wants all these extraneous fields and utf-16 encoding. Obviously.
#
# Format your Contacts CSV like this: "First Name","Last Name","Mobile Phone"
#
# Just run 'python nconvert.py' to start.
#

__author__ = 'ashwini@majestik.net'

print "nConvert v1.1 - Utility to convert CSV to a Nokia-CSV format for import in Nokia PC Suite\r\n"

inputfile = raw_input('Please enter the name of your CSV file (e.g. my_contacts.csv): ')

try:
    contacts = open(inputfile, 'rU')

except IOError as e:
    print "Unable to open \"{0}\" - {1})".format(inputfile, e)

else:
    filter = contacts.read()
    filter = filter.split("\n")
    contacts.close()

    # Remove header from source if exists
    for hfield in ["First name", "name", "Name"]:
        if hfield in filter[0]: del filter[0]

    # Figure out name of output file
    if "\\" in inputfile:
        filename = inputfile.split("\\")[-1]
        outputfile = inputfile.replace(filename, "Nokia-"+filename)
    elif "/" in inputfile:
        filename = inputfile.split("/")[-1]
        outputfile = inputfile.replace(filename, "Nokia-"+filename)
    else:
        filename = inputfile
        outputfile = "Nokia-"+inputfile

    # Create output file, write the Nokia header, encode in UTF-16
    contacts_out = open(outputfile, 'wb')
    header = "\"Title\",\"First name\",\"Middle name\",\"Last name\",\"Suffix\",\"Job title\",\"Company\",\"Birthday\",\"SIP address\",\"Push-to-talk\",\"Share view\",\"User ID\",\"Notes\",\"General mobile\",\"General phone\",\"General email\",\"General fax\",\"General video call\",\"General web address\",\"General VOIP address\",\"General P.O.Box\",\"General extension\",\"General street\",\"General postal/ZIP code\",\"General city\",\"General state/province\",\"General country/region\",\"Home mobile\",\"Home phone\",\"Home email\",\"Home fax\",\"Home video call\",\"Home web address\",\"Home VOIP address\",\"Home P.O.Box\",\"Home extension\",\"Home street\",\"Home postal/ZIP code\",\"Home city\",\"Home state/province\",\"Home country/region\",\"Business mobile\",\"Business phone\",\"Business email\",\"Business fax\",\"Business video call\",\"Business web address\",\"Business VOIP address\",\"Business P.O.Box\",\"Business extension\",\"Business street\",\"Business postal/ZIP code\",\"Business city\",\"Business state/province\",\"Business country/region\",\"\"{0}".format("\r\n")
    header = header.encode('utf-16')
    contacts_out.write(header)
    ContactCount = 0

    # Convert all contacts from source CSV to Nokia friendly output CSV
    for row in filter:
        try:
            field = row.split(",")

            # Clean up the phone number field
            for symbol in ["-", " ", "."]:
                if symbol in field[2]: field[2] = field[2].replace(symbol, "")

            new_row = "\"\",{0},\"\",{1},\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",{2},\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\"{3}".format(field[0], field[1], field[2],"\r\n")

            new_row = new_row.encode('utf-16')
            ContactCount += 1
            contacts_out.write(new_row)
        except IndexError:
            contacts_out.close()
            break

    print "\r\nSuccessfully created '{0}' with {1} total contacts. Go ahead and import that!".format(outputfile, ContactCount)

<commit_msg>source1<commit_after>#!/usr/bin/env python
#
# nConvert v1.1
# Author: ashwini@majestik.net
#
#
# This utility will convert a simple, human-readable Contacts CSV to a format that's
# compatible with Nokia PC Suite. Had to write a utility to do this because standard
# Microsoft Outlook CSV exports weren't reading in Nokia PC Suite. Figured out it was because
# NPCS wants all these extraneous fields and utf-16 encoding. Obviously. :)
#
# Format your Contacts CSV like this: "First Name","Last Name","Mobile Phone"
#
# Just run 'python nconvert.py' to start.
#

__author__ = 'ashwini@majestik.net'

print "nConvert v1.1 - Utility to convert CSV to a Nokia-CSV format for import in Nokia PC Suite\r\n"

inputfile = raw_input('Please enter the name of your CSV file (e.g. my_contacts.csv): ')

try:
    contacts = open(inputfile, 'rU')

except IOError as e:
    print "Unable to open \"{0}\" - {1})".format(inputfile, e)

else:
    filter = contacts.read()
    filter = filter.split("\n")
    contacts.close()

    # Remove header from source if exists
    for hfield in ["First name", "name", "Name"]:
        if hfield in filter[0]: del filter[0]

    # Figure out name of output file
    if "\\" in inputfile:
        filename = inputfile.split("\\")[-1]
        outputfile = inputfile.replace(filename, "Nokia-"+filename)
    elif "/" in inputfile:
        filename = inputfile.split("/")[-1]
        outputfile = inputfile.replace(filename, "Nokia-"+filename)
    else:
        filename = inputfile
        outputfile = "Nokia-"+inputfile

    # Create output file, write the Nokia header, encode in UTF-16
    contacts_out = open(outputfile, 'wb')
    header = "\"Title\",\"First name\",\"Middle name\",\"Last name\",\"Suffix\",\"Job title\",\"Company\",\"Birthday\",\"SIP address\",\"Push-to-talk\",\"Share view\",\"User ID\",\"Notes\",\"General mobile\",\"General phone\",\"General email\",\"General fax\",\"General video call\",\"General web address\",\"General VOIP address\",\"General P.O.Box\",\"General extension\",\"General street\",\"General postal/ZIP code\",\"General city\",\"General state/province\",\"General country/region\",\"Home mobile\",\"Home phone\",\"Home email\",\"Home fax\",\"Home video call\",\"Home web address\",\"Home VOIP address\",\"Home P.O.Box\",\"Home extension\",\"Home street\",\"Home postal/ZIP code\",\"Home city\",\"Home state/province\",\"Home country/region\",\"Business mobile\",\"Business phone\",\"Business email\",\"Business fax\",\"Business video call\",\"Business web address\",\"Business VOIP address\",\"Business P.O.Box\",\"Business extension\",\"Business street\",\"Business postal/ZIP code\",\"Business city\",\"Business state/province\",\"Business country/region\",\"\"{0}".format("\r\n")
    header = header.encode('utf-16')
    contacts_out.write(header)
    ContactCount = 0

    # Convert all contacts from source CSV to Nokia friendly output CSV
    for row in filter:
        try:
            field = row.split(",")

            # Clean up the phone number field
            for symbol in ["-", " ", "."]:
                if symbol in field[2]: field[2] = field[2].replace(symbol, "")

            new_row = "\"\",{0},\"\",{1},\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",{2},\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\"{3}".format(field[0], field[1], field[2],"\r\n")

            new_row = new_row.encode('utf-16')
            ContactCount += 1
            contacts_out.write(new_row)
        except IndexError:
            contacts_out.close()
            break

    print "\r\nSuccessfully created '{0}' with {1} total contacts. Go ahead and import that!".format(outputfile, ContactCount)

<|endoftext|>


<commit_before>import net
import sim
import edb as edb_

import numpy as np

__doc__ = """Training loop. Should be made executable from the terminal/commandline with a config file."""

# TODO Argparse

def fit(models, env, edb, config, verbose=True):
    """
    :type: model: list or tuple
    :param models: Tuple of (Q-Network, Target-Network).

    :type env: sim.Simulator
    :param env: Environment object.

    :type edb: edb.ExperienceDB
    :param edb: Experience database.

    :type config: dict or str
    :param config: Configuration dictionary or path to a YAML configuration file.

    :type verbose: str
    :param verbose: quack quack.
    """

    # Make print function
    def _print(msg):
        if verbose:
            print(msg)

    # Read config
    if isinstance(config, str):
        config = readconfigfile(config)

    # Read networks
    model, targetmodel = models

    # # Check models
    assert hasattr(targetmodel, 'baggage'), "Target model must have baggage with target parameter update function."
    assert "updatetargetparams" in targetmodel.baggage.keys(), "Target parameter update function 'updatetargetparams' " \
                                                               "must be in targetmodel's baggage."
    assert model.savedir is not None, "Model must have a place to save parameters."

    _print("[+] Tests passed.")

    # Prepare loop variables
    # Episode counter
    episodecount = 0
    # Iteration counter
    itercount = -1
    # Epoch counter
    gamecount = 0

    # Begin loop
    while True:
        # Update loop variables
        itercount += 1

        # Check if loop must be broken
        if itercount > config['maxitercount']:
            _print("[-] Maximum number of iterations reached, aborting training...")
            break

        if gamecount > config['maxgamecount']:
            _print("[-] Maximum number of epochs reached, aborting training...")
            break

        # Start new game
        env.newgame()
        gamecount += 1
        isterminal = False
        gameclock = 0

        _print("| New Game |")

        # Sample initial state from environment
        state = env.getstate()

        while not isterminal:
            # ----- [GET XP] -----
            # Q is a tensor which can be squeezed to a vector, which can then be argmax-ed to get an action.
            # Select an action (1 - eps) greedily
            if config['greed'] >= np.random.uniform(low=0., high=1.):
                # Sample Q from network
                Q = model.classifier(state)
                # Find action to take
                action = Q.squeeze().argmax()
                _print("| [T = {}] Q-Values: {} |".format(gameclock, Q.squeeze()))
                _print("| [T = {}] Taking action {} greedily. |".format(gameclock, action))
            else:
                action = np.random.randint(low=0, high=config['numactions'])
                _print("| [T = {}] Taking action {} randomly. |".format(gameclock, action))

            # See what the environment thinks about this action
            reward, newstate, isterminal = env.getresponse(action)
            # Log to experience database
            edb.log(state, action, reward, newstate, isterminal)
            # Set state to newstate
            state = newstate

            # ----- [LEARN FROM XP] -----
            # Fetch new batch from experience database
            x, yt = edb.batcherbatcher(targetnetwork=targetmodel, gamma=config['gamma'],
                                       batchsize=config['batchsize'])

            try:
                # Train on batch
                out = model.classifiertrainer(x, yt)
                # Update target network parameters
                targetmodel.baggage["updatetargetparams"](params=model.params,
                                                          decay=config['targetnetworkparamdecay'])

            except Exception as e:
                # Print diagnostics
                _print("[-] Exception raised while training model. Printing Diagnostics:")
                diagnostics = {'yt.shape': yt.shape,
                               'x.shape': x.shape,
                               'len(edb.db)': len(edb.db)}
                for diagitem, diagres in diagnostics.items():
                    _print(">>> {} = {}".format(diagitem, diagres))
                # Make a substitute out such that training could continue
                out = {'C': 'Error', 'L': 'Error', 'yt': yt}

            # Print
            echomsg = "| Cost: {C} || Loss: {L} || Target: {targ} |".format(C=out['C'], L=out['L'], targ=yt)
            _print(echomsg)
            _print("------------------------")

            # Increment counters
            gameclock += 1

        # Check if the game was won
        gamewon = env.getreward() == 1.
        _print("| [{}] Game {} won. |".format(*{True: ('+', 'was'), False: ('-', 'was not')}[gamewon]))
        _print("\n")
        # Save parameters if required
        if itercount % config['saveevery'] == 0:
            _print("[+] Saving model parameters...\n")
            model.save('--iter-{}-routine'.format(itercount))
            targetmodel.save('--iter-{}-routine'.format(itercount))

    return model

def readconfigfile(path):
    """
    Read configuration file from path.

    :type path: str
    :param path: Path to configuration yaml file.
    """
    assert path.endswith('.yml') or path.endswith('.save'), "Configuration file must be a YAML file or a Pickled " \
                                                            "dictionary object with '.save' extension."
    # TODO Parse config file
    pass

def main(configpath):
    """Pretend this function doesn't exist."""

    # FIXME: Debug and prettify.
    if not __debug__:
        return

    # ---Build Environment---
    # Build videoframes
    videopath = '/media/data/nrahaman/DeepTrack/Data/movies/1'
    vf = sim.VideoFrames(videopath)
    # Build track
    trackpath = '/home/nrahaman/Python/Antipasti/Projects/DeepTrack/tracks/1/ctrax_results.mat'
    tr = sim.Track(trackpath)
    # Build Sim
    env = sim.FlySimulator(vf, tr)

    # ---Build EDB---
    ed = edb_.ExperienceDB(maxsize=100)

    # ---Build models---
    models = net.simple()

    savedirmodel = '/media/data/nrahaman/NotNeuro/tracky/Try1/Value/Weights/'
    savedirtarget = '/media/data/nrahaman/NotNeuro/tracky/Try1/Target/Weights/'
    models[0].savedir = savedirmodel
    models[1].savedir = savedirtarget

    # ---Train---
    # Build config
    config = {'batchsize': 1,
              'gamma': 0.9,
              'targetnetworkparamdecay': 0.9,
              'maxitercount': 1000000,
              'maxgamecount': 2,
              'saveevery': 500,
              'numactions': 5,
              'greed': 0.9}

    trainedmodel = fit(models, env, ed, config)
    trainedmodel.save('--final')

if __name__ == '__main__':
    main(None)
<commit_msg>- bug fix<commit_after>import net
import sim
import edb as edb_

import numpy as np

__doc__ = """Training loop. Should be made executable from the terminal/commandline with a config file."""

# TODO Argparse

def fit(models, env, edb, config, verbose=True):
    """
    :type: model: list or tuple
    :param models: Tuple of (Q-Network, Target-Network).

    :type env: sim.Simulator
    :param env: Environment object.

    :type edb: edb.ExperienceDB
    :param edb: Experience database.

    :type config: dict or str
    :param config: Configuration dictionary or path to a YAML configuration file.

    :type verbose: str
    :param verbose: quack quack.
    """

    # Make print function
    def _print(msg):
        if verbose:
            print(msg)

    # Read config
    if isinstance(config, str):
        config = readconfigfile(config)

    # Read networks
    model, targetmodel = models

    # # Check models
    assert hasattr(targetmodel, 'baggage'), "Target model must have baggage with target parameter update function."
    assert "updatetargetparams" in targetmodel.baggage.keys(), "Target parameter update function 'updatetargetparams' " \
                                                               "must be in targetmodel's baggage."
    assert model.savedir is not None, "Model must have a place to save parameters."

    _print("[+] Tests passed.")

    # Prepare loop variables
    # Episode counter
    episodecount = 0
    # Iteration counter
    itercount = -1
    # Epoch counter
    gamecount = 0

    # Begin loop
    while True:
        # Update loop variables
        itercount += 1

        # Check if loop must be broken
        if itercount > config['maxitercount']:
            _print("[-] Maximum number of iterations reached, aborting training...")
            break

        if gamecount > config['maxgamecount']:
            _print("[-] Maximum number of epochs reached, aborting training...")
            break

        # Start new game
        env.newgame()
        gamecount += 1
        isterminal = False
        gameclock = 0

        _print("| New Game |")

        # Sample initial state from environment
        state = env.getstate()

        while not isterminal:
            # ----- [GET XP] -----
            # Q is a tensor which can be squeezed to a vector, which can then be argmax-ed to get an action.
            # Select an action (1 - eps) greedily
            if config['greed'] >= np.random.uniform(low=0., high=1.):
                # Sample Q from network
                Q = model.classifier(state)
                # Find action to take
                action = Q.squeeze().argmax()
                _print("| [T = {}] Q-Values: {} |".format(gameclock, Q.squeeze()))
                _print("| [T = {}] Taking action {} greedily. |".format(gameclock, action))
            else:
                action = np.random.randint(low=0, high=config['numactions'])
                _print("| [T = {}] Taking action {} randomly. |".format(gameclock, action))

            # See what the environment thinks about this action
            reward, newstate, isterminal = env.getresponse(action)
            # Log to experience database
            edb.log(state, action, reward, newstate, isterminal)
            # Set state to newstate
            state = newstate

            # ----- [LEARN FROM XP] -----
            # Fetch new batch from experience database
            x, yt = edb.batcherbatcher(targetnetwork=targetmodel, gamma=config['gamma'],
                                       batchsize=config['batchsize'])

            try:
                # Train on batch
                out = model.classifiertrainer(x, yt)
                # Update target network parameters
                targetmodel.baggage["updatetargetparams"](params=model.params,
                                                          decay=config['targetnetworkparamdecay'])

            except Exception as e:
                # Print diagnostics
                _print("[-] Exception raised while training model. Printing Diagnostics:")
                diagnostics = {'yt.shape': yt.shape,
                               'x.shape': x.shape,
                               'len(edb.db)': len(edb.db)}
                for diagitem, diagres in diagnostics.items():
                    _print(">>> {} = {}".format(diagitem, diagres))
                # Make a substitute out such that training could continue
                out = {'C': 'Error', 'L': 'Error', 'yt': yt}

            # Print
            echomsg = "| Cost: {C} || Loss: {L} || Target: {targ} |".format(C=out['C'], L=out['L'], targ=yt)
            _print(echomsg)
            _print("------------------------")

            # Increment counters
            gameclock += 1
            itercount += 1

        # Check if the game was won
        gamewon = env.getreward() == 1.
        _print("| [{}] Game {} won. |".format(*{True: ('+', 'was'), False: ('-', 'was not')}[gamewon]))
        _print("\n")
        # Save parameters if required
        if itercount % config['saveevery'] == 0:
            _print("[+] Saving model parameters at iteration {}...\n".format(itercount))
            model.save('--iter-{}-routine'.format(itercount))
            targetmodel.save('--iter-{}-routine'.format(itercount))

    return model

def readconfigfile(path):
    """
    Read configuration file from path.

    :type path: str
    :param path: Path to configuration yaml file.
    """
    assert path.endswith('.yml') or path.endswith('.save'), "Configuration file must be a YAML file or a Pickled " \
                                                            "dictionary object with '.save' extension."
    # TODO Parse config file
    pass

def main(configpath):
    """Pretend this function doesn't exist."""

    # FIXME: Debug and prettify.
    if not __debug__:
        return

    # ---Build Environment---
    # Build videoframes
    videopath = '/media/data/nrahaman/DeepTrack/Data/movies/1'
    vf = sim.VideoFrames(videopath)
    # Build track
    trackpath = '/home/nrahaman/Python/Antipasti/Projects/DeepTrack/tracks/1/ctrax_results.mat'
    tr = sim.Track(trackpath)
    # Build Sim
    env = sim.FlySimulator(vf, tr)

    # ---Build EDB---
    ed = edb_.ExperienceDB(maxsize=100)

    # ---Build models---
    models = net.simple()

    savedirmodel = '/media/data/nrahaman/NotNeuro/tracky/Try1/Value/Weights/'
    savedirtarget = '/media/data/nrahaman/NotNeuro/tracky/Try1/Target/Weights/'
    models[0].savedir = savedirmodel
    models[1].savedir = savedirtarget

    # ---Train---
    # Build config
    config = {'batchsize': 1,
              'gamma': 0.9,
              'targetnetworkparamdecay': 0.9,
              'maxitercount': 1000000,
              'maxgamecount': 100000,
              'saveevery': 500,
              'numactions': 5,
              'greed': 0.9}

    trainedmodel = fit(models, env, ed, config)
    trainedmodel.save('--final')

if __name__ == '__main__':
    main(None)
<|endoftext|>


<commit_before>"""
This module contains support functions and convenience methods used
throughout symfit. Some are used predominantly internally, others are
designed for users.
"""
from functools import wraps
from collections import OrderedDict
import sys
import warnings

import numpy as np
from sympy.utilities.lambdify import lambdify
import sympy
from sympy.tensor import IndexedBase
from sympy import Symbol

from symfit.core.argument import Parameter, Variable

if sys.version_info >= (3,0):
    import inspect as inspect_sig
else:
    import funcsigs as inspect_sig

def seperate_symbols(func):
    """
    Seperate the symbols in symbolic function func. Return them in alphabetical
    order.

    :param func: scipy symbolic function.
    :return: (vars, params), a tuple of all variables and parameters, each
        sorted in alphabetical order.
    :raises TypeError: only symfit Variable and Parameter are allowed, not sympy
        Symbols.
    """
    params = []
    vars = []
    for symbol in func.free_symbols:
        if isinstance(symbol, Parameter):
            params.append(symbol)
        elif isinstance(symbol, (Variable, IndexedBase, Symbol)):
            vars.append(symbol)
        else:
            raise TypeError('model contains an unknown symbol type, {}'.format(type(symbol)))
    params.sort(key=lambda symbol: symbol.name)
    vars.sort(key=lambda symbol: symbol.name)
    return vars, params

def sympy_to_py(func, vars, params):
    """
    Turn a symbolic expression into a Python lambda function,
    which has the names of the variables and parameters as it's argument names.

    :param func: sympy expression
    :param vars: variables in this model
    :param params: parameters in this model
    :return: lambda function to be used for numerical evaluation of the model. Ordering of the arguments will be vars
        first, then params.
    """
    return lambdify((vars + params), func, modules='numpy', dummify=False)

def sympy_to_scipy(func, vars, params):
    """
    Convert a symbolic expression to one scipy digs. Not used by ``symfit`` any more.

    :param func: sympy expression
    :param vars: variables
    :param params: parameters
    :return: Scipy-style function to be used for numerical evaluation of the model.
    """
    lambda_func = sympy_to_py(func, vars, params)
    def f(x, p):
        """
        Scipy style function.

        :param x: list of arrays, NxM
        :param p: tuple of parameter values.
        """
        x = np.atleast_2d(x)
        y = [x[i] for i in range(len(x))] if len(x[0]) else []
        try:
            ans = lambda_func(*(y + list(p)))
        except TypeError:
            # Possibly this is a constant function in which case it only has Parameters.
            ans = lambda_func(*list(p))# * np.ones(x_shape)
        return ans

    return f

def variables(names):
    """
    Convenience function for the creation of multiple variables.

    :param names: string of variable names. Should be comma seperated.
        Example: x, y = variables('x, y')
    """
    return [Variable(name=name.strip()) for name in names.split(',')]

def parameters(names):
    """
    Convenience function for the creation of multiple parameters.

    :param names: string of parameter names. Should be comma seperated.
        Example: a, b = parameters('a, b')
    """
    return [Parameter(name=name.strip()) for name in names.split(',')]

def cache(func):
    """
    Decorator function that gets a method as its input and either buffers the input,
    or returns the buffered output. Used in conjunction with properties to take away
    the standard buffering logic.

    :param func:
    :return:
    """
    @wraps(func)
    def new_func(self):
        try:
            return getattr(self, '_{}'.format(func.__name__))
        except AttributeError:
            setattr(self, '_{}'.format(func.__name__), func(self))
            return getattr(self, '_{}'.format(func.__name__))

    return new_func

def jacobian(expr, symbols):
    """
    Derive a symbolic expr w.r.t. each symbol in symbols. This returns a symbolic jacobian vector.

    :param expr: A sympy Expr.
    :param symbols: The symbols w.r.t. which to derive.
    """
    jac = []
    for symbol in symbols:
        # Differentiate to every param
        f = sympy.diff(expr, symbol)
        jac.append(f)
    return jac

def key2str(target):
    """
    In ``symfit`` there are many dicts with symbol: value pairs.
    These can not be used immediately as \*\*kwargs, even though this would make
    a lot of sense from the context.
    This function wraps such dict to make them usable as \*\*kwargs immidiately.

    :param target: dict to be made save
    :return: dict of str(symbol): value pairs.
    """
    return {str(symbol): value for symbol, value in target.items()}

class RequiredKeyword(object):
    """ Flag variable to indicate that this is a required keyword. """

class RequiredKeywordError(Exception):
    """ Error raised in case a keyword-only argument is not treated as such. """

class keywordonly(object):
    """
    Decorator class which wraps a python 2 function into one with keyword-only arguments.

    Example::

      @keywordonly(floor=True)
      def f(x, **kwargs):
          floor = kwargs.pop('floor')
          return np.floor(x**2) if floor else x**2

    This decorator is not much more than::

      floor = kwargs.pop('floor') if 'floor' in kwargs else True

    However, I prefer it's usage because:

    - it's clear from reading the function declaration there is an option to provide this
      argument. The information on possible keywords is where you'd expect it to be.
    - you're guaranteed that the pop works.
    - Perhaps in the future I will make this inspect-compatible so then we come full circle.

    Please note that this decorator needs a ** argument on the wrapped function
    in order to work.
    """
    def __init__(self, **kwonly_arguments):
        self.kwonly_arguments = kwonly_arguments
        self.required_keywords = {kw: value for kw, value in kwonly_arguments.items() if value is RequiredKeyword}
        self.optional_keywords = {kw: value for kw, value in kwonly_arguments.items() if value is not RequiredKeyword}

    def __call__(self, func):
        sig = inspect_sig.signature(func)
        # A var keyword has to be found for this to be correct.
        for param in sig.parameters.values():
            if param.kind == param.VAR_KEYWORD:
                break
        else:
            raise RequiredKeywordError(
                'The keywordonly decorator requires the function to '
                'accept a **kwargs argument.'
            )

        @wraps(func)
        def wrapped_func(*args, **kwargs):
            """
            :param args: args used to call the function
            :param kwargs: kwargs used to call the function
            :return: Wrapped function which behaves like it has keyword-only arguments.
            :raises: ``RequiredKeywordError`` if not all required keywords were specified.
            """
            for kw in self.required_keywords:
                if kw not in kwargs:
                    raise RequiredKeywordError(
                        'Keyword `{}` is a required keyword. '
                        'Please provide a value.'.format(kw)
                    )
            else:  # All required keywords were provided. Assign defaults.
                for kw, default in self.optional_keywords.items():
                    if kw not in kwargs:
                        kwargs[kw] = default
                else: # All defaults assigned where needed. Call func!
                    return func(*args, **kwargs)
        return wrapped_func

class deprecated(object):
    """
    Decorator to raise a DeprecationWarning.
    """
    def __init__(self, replacement=None):
        """
        :param replacement: The function which should now be used instead.
        """
        self.replacement = replacement

    def __call__(self, func):
        @wraps(func)
        def deprecated_func(*args, **kwargs):
            warnings.warn(DeprecationWarning(
                '`{}` has been deprecated.'.format(func.__name__)
                + ' Use `{}` instead.'.format(self.replacement)) if self.replacement else ''
            )
            return func(*args, **kwargs)
        return deprecated_func

class D(sympy.Derivative):
    """
    Convenience wrapper for ``sympy.Derivative``. Used most notably in defining
    ``ODEModel``'s.
    """
<commit_msg>Updated keywordonly such that all tests for it work in py2. However, all other tests fail.<commit_after>"""
This module contains support functions and convenience methods used
throughout symfit. Some are used predominantly internally, others are
designed for users.
"""
from functools import wraps
from collections import OrderedDict
import sys
import warnings

import numpy as np
from sympy.utilities.lambdify import lambdify
import sympy
from sympy.tensor import IndexedBase
from sympy import Symbol

from symfit.core.argument import Parameter, Variable

if sys.version_info >= (3,0):
    import inspect as inspect_sig
else:
    import funcsigs as inspect_sig

def seperate_symbols(func):
    """
    Seperate the symbols in symbolic function func. Return them in alphabetical
    order.

    :param func: scipy symbolic function.
    :return: (vars, params), a tuple of all variables and parameters, each
        sorted in alphabetical order.
    :raises TypeError: only symfit Variable and Parameter are allowed, not sympy
        Symbols.
    """
    params = []
    vars = []
    for symbol in func.free_symbols:
        if isinstance(symbol, Parameter):
            params.append(symbol)
        elif isinstance(symbol, (Variable, IndexedBase, Symbol)):
            vars.append(symbol)
        else:
            raise TypeError('model contains an unknown symbol type, {}'.format(type(symbol)))
    params.sort(key=lambda symbol: symbol.name)
    vars.sort(key=lambda symbol: symbol.name)
    return vars, params

def sympy_to_py(func, vars, params):
    """
    Turn a symbolic expression into a Python lambda function,
    which has the names of the variables and parameters as it's argument names.

    :param func: sympy expression
    :param vars: variables in this model
    :param params: parameters in this model
    :return: lambda function to be used for numerical evaluation of the model. Ordering of the arguments will be vars
        first, then params.
    """
    return lambdify((vars + params), func, modules='numpy', dummify=False)

def sympy_to_scipy(func, vars, params):
    """
    Convert a symbolic expression to one scipy digs. Not used by ``symfit`` any more.

    :param func: sympy expression
    :param vars: variables
    :param params: parameters
    :return: Scipy-style function to be used for numerical evaluation of the model.
    """
    lambda_func = sympy_to_py(func, vars, params)
    def f(x, p):
        """
        Scipy style function.

        :param x: list of arrays, NxM
        :param p: tuple of parameter values.
        """
        x = np.atleast_2d(x)
        y = [x[i] for i in range(len(x))] if len(x[0]) else []
        try:
            ans = lambda_func(*(y + list(p)))
        except TypeError:
            # Possibly this is a constant function in which case it only has Parameters.
            ans = lambda_func(*list(p))# * np.ones(x_shape)
        return ans

    return f

def variables(names):
    """
    Convenience function for the creation of multiple variables.

    :param names: string of variable names. Should be comma seperated.
        Example: x, y = variables('x, y')
    """
    return [Variable(name=name.strip()) for name in names.split(',')]

def parameters(names):
    """
    Convenience function for the creation of multiple parameters.

    :param names: string of parameter names. Should be comma seperated.
        Example: a, b = parameters('a, b')
    """
    return [Parameter(name=name.strip()) for name in names.split(',')]

def cache(func):
    """
    Decorator function that gets a method as its input and either buffers the input,
    or returns the buffered output. Used in conjunction with properties to take away
    the standard buffering logic.

    :param func:
    :return:
    """
    @wraps(func)
    def new_func(self):
        try:
            return getattr(self, '_{}'.format(func.__name__))
        except AttributeError:
            setattr(self, '_{}'.format(func.__name__), func(self))
            return getattr(self, '_{}'.format(func.__name__))

    return new_func

def jacobian(expr, symbols):
    """
    Derive a symbolic expr w.r.t. each symbol in symbols. This returns a symbolic jacobian vector.

    :param expr: A sympy Expr.
    :param symbols: The symbols w.r.t. which to derive.
    """
    jac = []
    for symbol in symbols:
        # Differentiate to every param
        f = sympy.diff(expr, symbol)
        jac.append(f)
    return jac

def key2str(target):
    """
    In ``symfit`` there are many dicts with symbol: value pairs.
    These can not be used immediately as \*\*kwargs, even though this would make
    a lot of sense from the context.
    This function wraps such dict to make them usable as \*\*kwargs immidiately.

    :param target: dict to be made save
    :return: dict of str(symbol): value pairs.
    """
    return {str(symbol): value for symbol, value in target.items()}

class RequiredKeyword(object):
    """ Flag variable to indicate that this is a required keyword. """

class RequiredKeywordError(Exception):
    """ Error raised in case a keyword-only argument is not treated as such. """

class keywordonly(object):
    """
    Decorator class which wraps a python 2 function into one with keyword-only arguments.

    Example::

      @keywordonly(floor=True)
      def f(x, **kwargs):
          floor = kwargs.pop('floor')
          return np.floor(x**2) if floor else x**2

    This decorator is not much more than::

      floor = kwargs.pop('floor') if 'floor' in kwargs else True

    However, I prefer it's usage because:

    - it's clear from reading the function declaration there is an option to provide this
      argument. The information on possible keywords is where you'd expect it to be.
    - you're guaranteed that the pop works.
    - Perhaps in the future I will make this inspect-compatible so then we come full circle.

    Please note that this decorator needs a ** argument on the wrapped function
    in order to work.
    """
    def __init__(self, **kwonly_arguments):
        self.kwonly_arguments = kwonly_arguments
        # Mark which are required
        self.required_keywords = {
            kw for kw, value in kwonly_arguments.items() if value is RequiredKeyword
        }
        # Transform all into keywordonly inspect.Parameter objects.
        self.keywordonly_parameters = OrderedDict(
            (kw, inspect_sig.Parameter(kw,
                                       kind=inspect_sig.Parameter.KEYWORD_ONLY,
                                       default=value)
             )
            for kw, value in kwonly_arguments.items()
        )

    def __call__(self, func):
        sig = inspect_sig.signature(func)
        params = []
        # A var keyword has to be found for this to be correct.
        for name, param in sig.parameters.items():
            if param.kind == param.VAR_KEYWORD:
                # Keyword only's go before the **kwargs parameter.
                params.extend(self.keywordonly_parameters.values())
                params.append(param)
                break
            params.append(param)
        else:
            raise RequiredKeywordError(
                'The keywordonly decorator requires the function to '
                'accept a **kwargs argument.'
            )
        # Update signature
        sig = sig.replace(parameters=params)
        func.__signature__ = sig

        @wraps(func)
        def wrapped_func(*args, **kwargs):
            """
            :param args: args used to call the function
            :param kwargs: kwargs used to call the function
            :return: Wrapped function which behaves like it has keyword-only arguments.
            :raises: ``RequiredKeywordError`` if not all required keywords were specified.
            """
            bound_args = func.__signature__.bind(*args, **kwargs)
            # Apply defaults
            for param in sig.parameters.values():
                if param.name not in bound_args.arguments:
                    if param.default is RequiredKeyword:
                        raise RequiredKeywordError(
                            'Keyword `{}` is a required keyword. '
                            'Please provide a value.'.format(param.name)
                        )
                    bound_args.arguments[param.name] = param.default
            return func(**bound_args.arguments)
        return wrapped_func

class deprecated(object):
    """
    Decorator to raise a DeprecationWarning.
    """
    def __init__(self, replacement=None):
        """
        :param replacement: The function which should now be used instead.
        """
        self.replacement = replacement

    def __call__(self, func):
        @wraps(func)
        def deprecated_func(*args, **kwargs):
            warnings.warn(DeprecationWarning(
                '`{}` has been deprecated.'.format(func.__name__)
                + ' Use `{}` instead.'.format(self.replacement)) if self.replacement else ''
            )
            return func(*args, **kwargs)
        return deprecated_func

class D(sympy.Derivative):
    """
    Convenience wrapper for ``sympy.Derivative``. Used most notably in defining
    ``ODEModel``'s.
    """
<|endoftext|>


<commit_before>from flask import Flask, redirect, render_template, request
import db
import message
import random
import read
import delete

app = Flask(__name__)

def check():
    c = db.messages.find("messages", "all")
    messages = []
    for x in c:
        if 'to' in x:
            if x['to'] == addr:
                messages.append(x)
    return messages

@app.route("/")
def index():
    try:
        message = check()
        message.reverse()
        return render_template("index.html", messages=message, num=str(len(message)), addr=addr)
    except:
        return "You seem to be a new node, ByteMail needs some time to setup, reload the page in 5 minutes or so."
@app.route("/sent/")
def sent():
    try:
        c = db.sent.find("sent", "all")
        message = []
        for x in c:
            message.append(x)
        message.reverse()
        return render_template("sent.html", messages=message, num=str(len(check())), addr=addr)
    except:
        return "<script>alert('You have not sent any messages yet.');window.location = '/';</script>"
@app.route("/read/<id>")
def read_(id):
    data = read.read(id, addr).split()
    id = data[1]
    date = data[3]
    time = data[4]
    title = data[8]
    from_ = data[6]
    message = ' '.join(data[9:])
    num = check()
    return render_template("read.html", num=str(len(num)), id=id, time=time, from_=from_, message=message, title=title, addr=addr)

@app.route("/send/", methods=['GET', 'POST'])
def send():
    num = check()
    if request.method == 'POST':
        to = request.form['to']
        if len(to) != 32:
            check_ = db.addressdb.find("addresses", "all")
            for x in check_:
                for y in x:
                    if y == to:
                        to = x[y]
        title = request.form['title']
        msg = request.form['message']
        check_ = message.send_msg(msg, title, to, addr)
        check_ = """<script>alert("{0}");window.location = '/';</script>""".format(check_)
        return check_

    return render_template("send.html", addr=addr, num=str(len(num)))

@app.route("/delete/<id>")
def delete_(id):
    check = delete.send_delete(id, addr)
    return redirect("/")

def run():
    global addr
    addr = db.data.find("data", "all")[0]['addr']
    app.run(port=5334)

if __name__ == "__main__":
    app.run(debug=True, port=5044)
<commit_msg>0.4.1<commit_after>from flask import Flask, redirect, render_template, request
import db
import message
import random
import read
import delete

app = Flask(__name__)

def check():
    c = db.messages.find("messages", "all")
    messages = []
    for x in c:
        if 'to' in x:
            if x['to'] == addr:
                messages.append(x)
    return messages

@app.route("/")
def index():
    message = check()
    message.reverse()
    return render_template("index.html", messages=message, num=str(len(message)), addr=addr)

@app.route("/sent/")
def sent():
    try:
        c = db.sent.find("sent", "all")
        message = []
        for x in c:
            message.append(x)
        message.reverse()
        return render_template("sent.html", messages=message, num=str(len(check())), addr=addr)
    except:
        return "<script>alert('You have not sent any messages yet.');window.location = '/';</script>"
@app.route("/read/<id>")
def read_(id):
    data = read.read(id, addr).split()
    id = data[1]
    date = data[3]
    time = data[4]
    title = data[8]
    from_ = data[6]
    message = ' '.join(data[9:])
    num = check()
    return render_template("read.html", num=str(len(num)), id=id, time=time, from_=from_, message=message, title=title, addr=addr)

@app.route("/send/", methods=['GET', 'POST'])
def send():
    num = check()
    if request.method == 'POST':
        to = request.form['to']
        if len(to) != 32:
            check_ = db.addressdb.find("addresses", "all")
            for x in check_:
                for y in x:
                    if y == to:
                        to = x[y]
        title = request.form['title']
        msg = request.form['message']
        check_ = message.send_msg(msg, title, to, addr)
        check_ = """<script>alert("{0}");window.location = '/';</script>""".format(check_)
        return check_

    return render_template("send.html", addr=addr, num=str(len(num)))

@app.route("/delete/<id>")
def delete_(id):
    check = delete.send_delete(id, addr)
    return redirect("/")

def run():
    global addr
    addr = db.data.find("data", "all")[0]['addr']
    app.run(port=5334)

if __name__ == "__main__":
    app.run(debug=True, port=5044)
<|endoftext|>


<commit_before>import os.path
from lxml import etree
from datetime import *
import time
from datetime import datetime
from elasticsearch import Elasticsearch
import glob
import json

def json2es():
    es = Elasticsearch()
    es.indices.create(index='my-index', ignore=400)
    es.index(index="my-index", doc_type="test-type", id=42, body={"any": "data", "timestamp": datetime.now()})
    print(es.get(index="my-index", doc_type="test-type", id=42)['_source'])

def xml2esjson(importpath, exportpath):
    total_record = 0
    print(f"importpath={importpath} exportpath={exportpath}")

    namespaces = {'ns': 'http://www.openarchives.org/OAI/2.0/',
                  'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
                  'rdfs': "http://www.w3.org/2000/01/rdf-schema#",
                  'dc': 'http://purl.org/dc/elements/1.1/',
                  'dcterms': 'http://purl.org/dc/terms/',
                  'foaf': 'http://xmlns.com/foaf/0.1/',
                  'owl': 'http://www.w3.org/2002/07/owl#'}

    files = glob.glob(importpath)
    for filename in files:
        tree = etree.parse(filename)
        root = tree.getroot()
        records = root.xpath('//ns:record', namespaces=namespaces)
        print(f"readfile name={filename} size={len(records)}")
        total_record += len(records)

        json_list = []

        for r in records:
            j = {}

            # 2-3-1
            x = r.find(".//dcterms:identifier[@rdf:datatype='http://ndl.go.jp/dcndl/terms/JPNO']",
                       namespaces=namespaces)
            j['jpno'] = x.text

            # 2-8
            x = r.find('.//dcterms:title', namespaces=namespaces)
            j['dcterms_title'] = x.text

            xl = r.findall('.//dc:title', namespaces=namespaces)

            # 2-42
            xl = r.findall('.//dcterms:creator', namespaces=namespaces)

            # 2-46
            xl = r.findall('.//dc:creator', namespaces=namespaces)
            j['creators'] = [x.text for x in xl]

            # print(j)
            json_list.append(j)

        # end of xml file
        jsonfilename = f"{filename}.json"
        writepath = os.path.join(exportpath, jsonfilename)
        jsontext = json.dumps(json_list, sort_keys=True, ensure_ascii=False, indent=2)
        with open(writepath, 'w') as f:
            f.write(jsontext)

    # ===
    print(f"total_record=#{total_record}")

if __name__ == '__main__':
    print(f"@start time={datetime.now().strftime('%Y/%m/%d %H:%M:%S')}")

    importpath = "D:\\data\\xml\\ndl_oaipmh_2013-06-22-11.xml"
    exportpath = "D:\\data\\json"

    xml2esjson(importpath, exportpath)

    print(f"@finish time={datetime.now().strftime('%Y/%m/%d %H:%M:%S')}")
<commit_msg>実装中。dcterms:subject に対応。<commit_after>import os.path
from lxml import etree
from datetime import *
import time
from datetime import datetime
from elasticsearch import Elasticsearch
import glob
import json
import re
from urllib.parse import urlparse

def json2es():
    es = Elasticsearch()
    es.indices.create(index='my-index', ignore=400)
    es.index(index="my-index", doc_type="test-type", id=42, body={"any": "data", "timestamp": datetime.now()})
    print(es.get(index="my-index", doc_type="test-type", id=42)['_source'])

def xml2esjson(importpath, exportpath):
    total_record = 0
    print(f"importpath={importpath} exportpath={exportpath}")

    ns = {'ns': 'http://www.openarchives.org/OAI/2.0/',
                  'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
                  'rdfs': "http://www.w3.org/2000/01/rdf-schema#",
                  'dc': 'http://purl.org/dc/elements/1.1/',
                  'dcterms': 'http://purl.org/dc/terms/',
          'dcndl': 'http://ndl.go.jp/dcndl/terms/',
                  'foaf': 'http://xmlns.com/foaf/0.1/',
                  'owl': 'http://www.w3.org/2002/07/owl#'}

    files = glob.glob(importpath)
    for filename in files:
        tree = etree.parse(filename)
        root = tree.getroot()
        records = root.xpath('//ns:record', namespaces=ns)
        print(f"readfile name={filename} size={len(records)}")
        total_record += len(records)

        json_list = []

        for r in records:
            j = {}

            # 2-3-1
            x = r.find(".//dcterms:identifier[@rdf:datatype='http://ndl.go.jp/dcndl/terms/JPNO']",
                       namespaces=ns)
            j['jpno'] = x.text

            # 2-6-1
            xl = r.findall(".//dcterms:identifier[@rdf:datatype='http://ndl.go.jp/dcndl/terms/ISBN']",
                           namespaces=ns)
            j['isbn'] = [x.text for x in xl]

            # 2-8
            x = r.find('.//dcterms:title', namespaces=ns)
            j['dcterms_title'] = x.text

            xl = r.findall('.//dc:title', namespaces=ns)
            xla = []
            for x in xl:
                xla.append({"value": r.find('.//rdf:Description/rdf:value', namespaces=ns).text,
                            "transcription": r.find('.//rdf:Description/dcndl:transcription', namespaces=ns).text
                            })
            j['dc_titles'] = xla

            # 2-42
            xl = r.findall('.//dcterms:creator', namespaces=ns)

            # 2-46
            xl = r.findall('.//dc:creator', namespaces=ns)
            j['creators'] = [x.text for x in xl]

            # 2-81-n/2-82-n
            # 1st:2-81-n
            xla = []
            xl = r.findall(".//dcterms:subject", namespaces=ns)
            for x in xl:
                for k, v in x.attrib.items():
                    if k == "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource":
                        # print(f"match {v}")
                        o = urlparse(v)
                        m = re.match(r"/class/(\w+)/(\w+)", o.path)
                        if m:
                            # print(m.group(1), m.group(2))
                            xla.append({m.group(1): m.group(2)})

                            # print(type(x.attrib))
                        #                m = re.search("'(.*)': '(.*)'", x.attrib)

            j['classes'] = xla
            # 2nd:2-82-n

            # print(j)
            json_list.append(j)

        # end of xml file
        fn = os.path.basename(filename)
        name, ext = os.path.splitext(fn)
        jsonfilename = f"{name}.json"
        writepath = os.path.join(exportpath, jsonfilename)

        print(f"writepath={writepath}")

        json_text = json.dumps(json_list, sort_keys=True, ensure_ascii=False, indent=2)

        with open(writepath, 'w', encoding='utf8') as f:
            f.write(json_text)

    # ===
    print(f"total_record={total_record}")

if __name__ == '__main__':
    print(f"@start time={datetime.now().strftime('%Y/%m/%d %H:%M:%S')}")

    importpath = "D:\\data\\xml\\ndl_oaipmh_2013-06-22-4789.xml"
    exportpath = "D:\\data\\json"

    xml2esjson(importpath, exportpath)

    print(f"@finish time={datetime.now().strftime('%Y/%m/%d %H:%M:%S')}")
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2016 NeuroData (http://neurodata.io)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# graph.py
# Repackaged by Derek Pisner on 2019-02-19
# Originally created by Greg Kiar on 2016-01-27.
# Email: dpisner@utexas.edu

from __future__ import print_function
import warnings
warnings.simplefilter("ignore")
from itertools import combinations, product
from collections import defaultdict
import numpy as np
import networkx as nx
import nibabel as nib
import ndmg
import time
import os

class graph_tools(object):
    def __init__(self, rois, tracks, affine, namer, attr=None, sens="dwi"):
        """
        Initializes the graph with nodes corresponding to the number of ROIs

        **Positional Arguments:**
                rois:
                    - Set of ROIs as either an array or niftii file)
                attr:
                    - Node or graph attributes. Can be a list. If 1 dimensional
                      will be interpretted as a graph attribute. If N
                      dimensional will be interpretted as node attributes. If
                      it is any other dimensional, it will be ignored.
                sens: string, Default to 'dmri'
                    - sensor of acquisition. Can be 'dmri' or 'fmri'
        """
        self.edge_dict = defaultdict(int)
	self.roi_file = rois
	self.roi_img = nib.load(self.roi_file)
        self.rois = self.roi_img.get_data().astype('int')
        n_ids = np.unique(self.rois)
        self.n_ids = n_ids[n_ids > 0]
        self.N = len(self.n_ids)
        self.modal = sens
	self.tracks = tracks
	self.affine = affine
	self.namer = namer
        pass

    def make_regressors(self, attr=None):
	from nilearn.image import new_img_like
        import pandas as pd

        def get_parcel_list(parlistfile):
            bna_img = nib.load(parlistfile)
            bna_data = np.round(bna_img.get_data(),1)
            ##Get an array of unique parcels
            bna_data_for_coords_uniq = np.unique(bna_data)
            ##Number of parcels:
            par_max = len(bna_data_for_coords_uniq) - 1
            bna_data = bna_data.astype('int16')
            img_stack = []
            for idx in range(1, par_max+1):
                roi_img = bna_data == bna_data_for_coords_uniq[idx].astype('int16')
                roi_img = roi_img.astype('int16')
                img_stack.append(roi_img)
            img_stack = np.array(img_stack).astype('int16')
            img_list = []
            for idy in range(par_max):
                roi_img_nifti = new_img_like(bna_img, img_stack[idy])
                img_list.append(roi_img_nifti)
            return(img_list)

        img_list = get_parcel_list(self.roi_file)

        print('Compiling voxel coordinates of all streamlines...')
        points = []
        for _, streamline in enumerate(self.tracks):
            point_set = np.round(streamline).astype('int')
            for point in point_set:
                points.append(tuple(point))

        print('Counting fiber waytotals and # voxels for each ROI...')
        self.fibers = {}
        self.rois = {}
        img_ix = 0
        for roi_img in img_list[1:]:
            roi_data = roi_img.get_data().astype('int')
            img_ix = img_ix + 1
            self.rois[img_ix] = np.sum(roi_data.astype('bool'))
            for point in set(points):
                try:
                    loc = self.roi_data[point[0], point[1], point[2]]
                except:
                    loc = None
                if loc is not None:
                    try:
                        self.fibers[img_ix] = self.fibers[img_ix] + 1
                    except:
                        self.fibers[img_ix] = 1
                else:
                    continue
	    try:
		self.fibers[img_ix]
	    except:
		self.fibers[img_ix] = 0
            print(str(img_ix) + ': Fibers - ' + str(self.fibers[img_ix]) + '  Voxels - ' + str(self.rois[img_ix]))

        self.df_regressors = pd.DataFrame({'fiber_count':pd.Series(self.fibers), 'roi_size':pd.Series(self.rois)})
	self.df_out = self.namer.name_derivative(self.namer.dirs['output']['fiber'], "roi_regressors.csv")
	self.df_regressors.to_csv(self.df_out, sep='\t')

        return

    def make_graph_old(self, attr=None):
        """
        Takes streamlines and produces a graph
        **Positional Arguments:**
                streamlines:
                    - Fiber streamlines either file or array in a dipy EuDX
                      or compatible format.
        """
	print('Building connectivity matrix...')
	self.g = nx.Graph(name="Generated by NeuroData's MRI Graphs (ndmg)",
                          date=time.asctime(time.localtime()),
                          source="http://m2g.io",
                          region="brain",
                          sensor=self.modal,
                          ecount=0,
                          vcount=len(self.n_ids)
                          )
        print(self.g.graph)

        [self.g.add_node(ids) for ids in self.n_ids]

        nlines = np.shape(self.tracks)[0]
        print("# of Streamlines: " + str(nlines))

        for idx, streamline in enumerate(self.tracks):
            if (idx % int(nlines*0.05)) == 0:
                print(idx)

            points = np.round(streamline).astype(int)
            p = set()
            for point in points:
                try:
                    loc = self.rois[point[0], point[1], point[2]]
                except IndexError:
                    pass
                else:
                    pass

                if loc:
                    p.add(loc)

            edges = combinations(p, 2)
            for edge in edges:
                lst = tuple([int(node) for node in edge])
                self.edge_dict[tuple(sorted(lst))] += 1

        edge_list = [(k[0], k[1], v) for k, v in self.edge_dict.items()]
        self.g.add_weighted_edges_from(edge_list)
	return self.g

    def cor_graph(self, timeseries, attr=None):
        """
        Takes timeseries and produces a correlation matrix

        **Positional Arguments:**
            timeseries:
                -the timeseries file to extract correlation for
                dimensions are [numrois]x[numtimesteps]
        """
        ts = timeseries[0]
        rois = timeseries[1]
        print("Estimating absolute correlation matrix for {} ROIs...".format(len(rois)))
        self.g = np.abs(np.corrcoef(ts))  # calculate abs pearson correlation
        self.g = np.nan_to_num(self.g).astype(object)
        self.n_ids = rois
        return self.g

    def get_graph(self):
        """
        Returns the graph object created
        """
        try:
            return self.g
        except AttributeError:
            print("Error: the graph has not yet been defined.")
            pass

    def as_matrix(self):
        """
        Returns the graph as a matrix.
        """
        g = self.get_graph()
        return nx.to_numpy_matrix(g, nodelist=np.sort(g.nodes()).tolist())

    def save_graph(self, graphname, fmt='igraph'):
        """
        Saves the graph to disk

        **Positional Arguments:**

                graphname:
                    - Filename for the graph
        """
        self.g.graph['ecount'] = nx.number_of_edges(self.g)
        self.g = nx.convert_node_labels_to_integers(self.g, first_label=1)
	print(self.g.graph)
        if fmt == 'edgelist':
            nx.write_weighted_edgelist(self.g, graphname, encoding='utf-8')
        elif fmt == 'gpickle':
            nx.write_gpickle(self.g, graphname)
        elif fmt == 'graphml':
            nx.write_graphml(self.g, graphname)
	elif fmt == 'txt':
	    np.savetxt(graphname, nx.to_numpy_matrix(self.g))
        elif fmt == 'npy':
            np.save(graphname, nx.to_numpy_matrix(self.g))
	elif fmt == 'igraph':
	    if self.modal == 'dwi':
                nx.write_weighted_edgelist(self.g, graphname, delimiter=" ", encoding='utf-8')
            elif self.modal == 'func':
                np.savetxt(graphname, self.g, comments='', delimiter=',', header=','.join([str(n) for n in self.n_ids]))
            else:
                raise ValueError("Unsupported Modality.")
        else:
            raise ValueError('Only edgelist, gpickle, and graphml currently supported')
        pass

    def summary(self):
        """
        User friendly wrapping and display of graph properties
        """
        print("\nGraph Summary:")
        print(nx.info(self.g))
        pass
<commit_msg>Add fiber count/roi voxel count regressor stage<commit_after>#!/usr/bin/env python

# Copyright 2016 NeuroData (http://neurodata.io)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# graph.py
# Repackaged by Derek Pisner on 2019-02-19
# Originally created by Greg Kiar on 2016-01-27.
# Email: dpisner@utexas.edu

from __future__ import print_function
import warnings
warnings.simplefilter("ignore")
from itertools import combinations, product
from collections import defaultdict
import numpy as np
import networkx as nx
import nibabel as nib
import ndmg
import time
import os

class graph_tools(object):
    def __init__(self, rois, tracks, affine, namer, attr=None, sens="dwi"):
        """
        Initializes the graph with nodes corresponding to the number of ROIs

        **Positional Arguments:**
                rois:
                    - Set of ROIs as either an array or niftii file)
                attr:
                    - Node or graph attributes. Can be a list. If 1 dimensional
                      will be interpretted as a graph attribute. If N
                      dimensional will be interpretted as node attributes. If
                      it is any other dimensional, it will be ignored.
                sens: string, Default to 'dmri'
                    - sensor of acquisition. Can be 'dmri' or 'fmri'
        """
        self.edge_dict = defaultdict(int)
	self.roi_file = rois
	self.roi_img = nib.load(self.roi_file)
        self.rois = self.roi_img.get_data().astype('int')
        n_ids = np.unique(self.rois)
        self.n_ids = n_ids[n_ids > 0]
        self.N = len(self.n_ids)
        self.modal = sens
	self.tracks = tracks
	self.affine = affine
	self.namer = namer
        pass

    def make_regressors(self, attr=None):
	from nilearn.image import new_img_like
        import pandas as pd

        def get_parcel_list(parlistfile):
            bna_img = nib.load(parlistfile)
            bna_data = np.round(bna_img.get_data(),1)
            ##Get an array of unique parcels
            bna_data_for_coords_uniq = np.unique(bna_data)
            ##Number of parcels:
            par_max = len(bna_data_for_coords_uniq) - 1
            bna_data = bna_data.astype('int16')
            img_stack = []
            for idx in range(1, par_max+1):
                roi_img = bna_data == bna_data_for_coords_uniq[idx].astype('int16')
                roi_img = roi_img.astype('int16')
                img_stack.append(roi_img)
            img_stack = np.array(img_stack).astype('int16')
            img_list = []
            for idy in range(par_max):
                roi_img_nifti = new_img_like(bna_img, img_stack[idy])
                img_list.append(roi_img_nifti)
            return(img_list)

        img_list = get_parcel_list(self.roi_file)

        print('Compiling voxel coordinates of all streamlines...')
        points = []
        for _, streamline in enumerate(self.tracks):
            point_set = np.round(streamline).astype('int')
            for point in point_set:
                points.append(tuple(point))

        print('Counting fiber waytotals and # voxels for each ROI...')
        fibers = {}
        rois = {}
        img_ix = 0
        for img in img_list[1:]:
            data = img.get_data()
            img_ix = img_ix + 1
            rois[img_ix] = np.sum(data.astype('bool'))
            for point in set(points):
                try:
                    loc = data[point[0], point[1], point[2]]
                except IndexError:
                    loc = None
                    pass
                if loc:
                    try:
                        fibers[img_ix] = fibers[img_ix] + 1
                    except:
                        fibers[img_ix] = 1
                else:
                    continue
            try:
                fibers[img_ix]
            except:
                fibers[img_ix] = 0
            print(str(img_ix) + ': Fibers - ' + str(fibers[img_ix]) + '  Voxels: ' + str(rois[img_ix]))

        self.df_regressors = pd.DataFrame({'fiber_count':pd.Series(fibers), 'roi_size':pd.Series(rois)})
	self.df_out = self.namer.name_derivative(self.namer.dirs['output']['fiber'], "roi_regressors.csv")
	self.df_regressors.to_csv(self.df_out, sep='\t')

        return

    def make_graph_old(self, attr=None):
        """
        Takes streamlines and produces a graph
        **Positional Arguments:**
                streamlines:
                    - Fiber streamlines either file or array in a dipy EuDX
                      or compatible format.
        """
	print('Building connectivity matrix...')
	self.g = nx.Graph(name="Generated by NeuroData's MRI Graphs (ndmg)",
                          date=time.asctime(time.localtime()),
                          source="http://m2g.io",
                          region="brain",
                          sensor=self.modal,
                          ecount=0,
                          vcount=len(self.n_ids)
                          )
        print(self.g.graph)

        [self.g.add_node(ids) for ids in self.n_ids]

        nlines = np.shape(self.tracks)[0]
        print("# of Streamlines: " + str(nlines))

        for idx, streamline in enumerate(self.tracks):
            if (idx % int(nlines*0.05)) == 0:
                print(idx)

            points = np.round(streamline).astype(int)
            p = set()
            for point in points:
                try:
                    loc = self.rois[point[0], point[1], point[2]]
                except IndexError:
                    pass
                else:
                    pass

                if loc:
                    p.add(loc)

            edges = combinations(p, 2)
            for edge in edges:
                lst = tuple([int(node) for node in edge])
                self.edge_dict[tuple(sorted(lst))] += 1

        edge_list = [(k[0], k[1], v) for k, v in self.edge_dict.items()]
        self.g.add_weighted_edges_from(edge_list)
	return self.g

    def cor_graph(self, timeseries, attr=None):
        """
        Takes timeseries and produces a correlation matrix

        **Positional Arguments:**
            timeseries:
                -the timeseries file to extract correlation for
                dimensions are [numrois]x[numtimesteps]
        """
        ts = timeseries[0]
        rois = timeseries[1]
        print("Estimating absolute correlation matrix for {} ROIs...".format(len(rois)))
        self.g = np.abs(np.corrcoef(ts))  # calculate abs pearson correlation
        self.g = np.nan_to_num(self.g).astype(object)
        self.n_ids = rois
        return self.g

    def get_graph(self):
        """
        Returns the graph object created
        """
        try:
            return self.g
        except AttributeError:
            print("Error: the graph has not yet been defined.")
            pass

    def as_matrix(self):
        """
        Returns the graph as a matrix.
        """
        g = self.get_graph()
        return nx.to_numpy_matrix(g, nodelist=np.sort(g.nodes()).tolist())

    def save_graph(self, graphname, fmt='igraph'):
        """
        Saves the graph to disk

        **Positional Arguments:**

                graphname:
                    - Filename for the graph
        """
        self.g.graph['ecount'] = nx.number_of_edges(self.g)
        self.g = nx.convert_node_labels_to_integers(self.g, first_label=1)
	print(self.g.graph)
        if fmt == 'edgelist':
            nx.write_weighted_edgelist(self.g, graphname, encoding='utf-8')
        elif fmt == 'gpickle':
            nx.write_gpickle(self.g, graphname)
        elif fmt == 'graphml':
            nx.write_graphml(self.g, graphname)
	elif fmt == 'txt':
	    np.savetxt(graphname, nx.to_numpy_matrix(self.g))
        elif fmt == 'npy':
            np.save(graphname, nx.to_numpy_matrix(self.g))
	elif fmt == 'igraph':
	    if self.modal == 'dwi':
                nx.write_weighted_edgelist(self.g, graphname, delimiter=" ", encoding='utf-8')
            elif self.modal == 'func':
                np.savetxt(graphname, self.g, comments='', delimiter=',', header=','.join([str(n) for n in self.n_ids]))
            else:
                raise ValueError("Unsupported Modality.")
        else:
            raise ValueError('Only edgelist, gpickle, and graphml currently supported')
        pass

    def summary(self):
        """
        User friendly wrapping and display of graph properties
        """
        print("\nGraph Summary:")
        print(nx.info(self.g))
        pass
<|endoftext|>


<commit_before># Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from contracts import new_contract

import collections
import datetime
import libvirt
import sqlalchemy
import neat.db
# import novaclient

new_contract('long', lambda x: isinstance(x, (int, long)))
new_contract('deque', collections.deque)
new_contract('function', lambda x: hasattr(x, '__call__'))
new_contract('datetime', datetime.datetime)
new_contract('virConnect', libvirt.virConnect)
new_contract('virDomain', libvirt.virDomain)
new_contract('Table', sqlalchemy.Table)
new_contract('Database', neat.db.Database)
# new_contract('Client', novaclient.v1_1.client.Client)
<commit_msg>Fixed a problem with imports in contracts_extra.py<commit_after># Copyright 2012 Anton Beloglazov
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from contracts import new_contract

new_contract('long', lambda x: isinstance(x, (int, long)))
new_contract('function', lambda x: hasattr(x, '__call__'))

import collections
new_contract('deque', collections.deque)

import datetime
new_contract('datetime', datetime.datetime)

import libvirt
new_contract('virConnect', libvirt.virConnect)
new_contract('virDomain', libvirt.virDomain)

import sqlalchemy
new_contract('Table', sqlalchemy.Table)

import neat.db
new_contract('Database', neat.db.Database)
<|endoftext|>