<commit_before>import unittest
from axelrod import Game, Actions
import axelrod.payoff as ap

C, D = Actions.C, Actions.D

class TestPayoff(unittest.TestCase):

    @classmethod
    def setUpClass(cls):

        cls.players = ('Alternator', 'TitForTat', 'Random')

        cls.interactions = {
            (0, 0): [(C, C), (D, D), (C, C), (D, D), (C, C)],
            (0, 1): [(C, C), (D, C), (C, D), (D, C), (C, D)],
            (0, 2): [(C, C), (D, C), (C, D), (D, C), (C, D)],
            (1, 1): [(C, C), (C, C), (C, C), (C, C), (C, C)],
            (1, 2): [(C, D), (D, D), (D, C), (C, D), (D, C)],
            (2, 2): [(D, C), (D, D), (D, C), (D, D), (D, D)]

        }

        cls.expected_payoff_matrix = [
            [11, 13, 13],
            [13, 15, 11],
            [13, 11, 13]
        ]

        cls.expected_payoff = [
            [[11.0, 11.0], [13, 13], [15, 12]],
            [[13, 13], [15.0, 15.0], [14, 7]],
            [[10, 12], [14, 12], [8.0, 14.5]],
        ]

        cls.expected_scores = [
            [28, 25],
            [27, 20],
            [24, 24]
        ]

        cls.expected_normalised_scores = [
            [2.8, 2.5],
            [2.7, 2.0],
            [2.4, 2.4]
        ]

        cls.expected_ranking = [0, 2, 1]
        cls.expected_ranked_names = ['Alternator', 'Random', 'TitForTat']

        cls.expected_normalised_payoff = [
            [2.2, 2.6, 2.7],
            [2.6, 3.0, 2.1],
            [2.2, 2.6, 2.25]
        ]
        cls.expected_payoff_stddevs = [
            [0.0, 0.0, 0.3],
            [0.0, 0.0, 0.7],
            [0.2, 0.2, 0.65]
        ]

        cls.expected_wins = [[2, 0], [0, 0], [0, 2]]

        cls.diff_means = [
            [ 0. ,  0. ,  0.5],
            [ 0. ,  0. , -0.5],
            [-0.5,  0.5,  0. ]
        ]

        cls.expected_score_diffs = [
            [0.0, 0.0, 0.0, 0.0, 1.0, 0.0],
            [0.0, 0.0, 0.0, 0.0, 0.0, -1.0],
            [-1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
        ]

    def test_nplayers(self):
        nplayers = ap.nplayers(self.interactions)
        self.assertEqual(nplayers, 5)

    def test_payoff_matrix(self):
        payoff_matrix = ap.payoff_matrix(self.interactions, 3, Game())
        self.assertEqual(payoff_matrix, self.expected_payoff_matrix)

    def test_interaction_payoff(self):
        payoff = ap.interaction_payoff(self.interactions[(2, 2)], Game())
        self.assertEqual(payoff, (13, 3))

    def test_scores_matrix(self):
        scores = ap.normalised_scores(self.interactions, Game())
        self.assertEqual(scores, self.expected_scores)

    def test_scores(self):
        scores = ap.scores(self.expected_payoff)
        self.assertEqual(scores, self.expected_scores)

    def test_normalised_scores(self):
        scores = ap.normalised_scores(self.expected_scores, 5)
        self.assertEqual(scores, self.expected_normalised_scores)

    def test_ranking(self):
        ranking = ap.ranking(self.expected_scores)
        self.assertEqual(ranking, self.expected_ranking)

    def test_ranked_names(self):
        ranked_names = ap.ranked_names(
            self.players, self.expected_ranking,)
        self.assertEqual(ranked_names, self.expected_ranked_names)

    def test_payoff_diffs_means(self):
        #payoff_matrix = ap.payoff_matrix(self.interactions, Game(), 3, 5)
        diff_means = ap.payoff_diffs_means(self.expected_payoff, 5)
        self.assertEqual(diff_means, self.diff_means)

    def test_score_diffs(self):
        #payoff_matrix = ap.payoff_matrix(self.interactions, Game(), 3, 5)
        score_diffs = ap.score_diffs(self.expected_payoff, 5)
        self.assertEqual(score_diffs, self.expected_score_diffs)

    @staticmethod
    def round_matrix(matrix, precision):
        return [[round(x, precision) for x in row] for row in matrix]

    def test_normalised_payoff(self):
        averages, stddevs = ap.normalised_payoff(self.expected_payoff, 5)
        self.assertEqual(
            self.round_matrix(averages, 2), self.expected_normalised_payoff)
        self.assertEqual(
            self.round_matrix(stddevs, 2), self.expected_payoff_stddevs)

    def test_winning_player(self):
        test_players = (8, 4)
        test_payoffs = (34, 44)
        winner = ap.winning_player(test_players, test_payoffs)
        self.assertEqual(winner, 4)
        test_payoffs = (54, 44)
        winner = ap.winning_player(test_players, test_payoffs)
        self.assertEqual(winner, 8)
        test_payoffs = (34, 34)
        winner = ap.winning_player(test_players, test_payoffs)
        self.assertEqual(winner, None)

    def test_wins(self):
        wins = ap.wins(self.expected_payoff)
        self.assertEqual(wins, self.expected_wins)
<commit_msg>fixes up test_scores_matrix<commit_after>import unittest
from axelrod import Game, Actions
import axelrod.payoff as ap

C, D = Actions.C, Actions.D

class TestPayoff(unittest.TestCase):

    @classmethod
    def setUpClass(cls):

        cls.players = ('Alternator', 'TitForTat', 'Random')

        cls.interactions = {
            (0, 0): [(C, C), (D, D), (C, C), (D, D), (C, C)],
            (0, 1): [(C, C), (D, C), (C, D), (D, C), (C, D)],
            (0, 2): [(C, C), (D, C), (C, D), (D, C), (C, D)],
            (1, 1): [(C, C), (C, C), (C, C), (C, C), (C, C)],
            (1, 2): [(C, D), (D, D), (D, C), (C, D), (D, C)],
            (2, 2): [(D, C), (D, D), (D, C), (D, D), (D, D)]

        }

        cls.expected_payoff_matrix = [
            [11, 13, 13],
            [13, 15, 11],
            [13, 11, 13]
        ]

        cls.expected_payoff = [
            [[11.0, 11.0], [13, 13], [15, 12]],
            [[13, 13], [15.0, 15.0], [14, 7]],
            [[10, 12], [14, 12], [8.0, 14.5]],
        ]

        cls.expected_scores = [
            [28, 25],
            [27, 20],
            [24, 24]
        ]

        cls.expected_normalised_scores = [
            [2.8, 2.5],
            [2.7, 2.0],
            [2.4, 2.4]
        ]

        cls.expected_ranking = [0, 2, 1]
        cls.expected_ranked_names = ['Alternator', 'Random', 'TitForTat']

        cls.expected_normalised_payoff = [
            [2.2, 2.6, 2.7],
            [2.6, 3.0, 2.1],
            [2.2, 2.6, 2.25]
        ]
        cls.expected_payoff_stddevs = [
            [0.0, 0.0, 0.3],
            [0.0, 0.0, 0.7],
            [0.2, 0.2, 0.65]
        ]

        cls.expected_wins = [[2, 0], [0, 0], [0, 2]]

        cls.diff_means = [
            [ 0. ,  0. ,  0.5],
            [ 0. ,  0. , -0.5],
            [-0.5,  0.5,  0. ]
        ]

        cls.expected_score_diffs = [
            [0.0, 0.0, 0.0, 0.0, 1.0, 0.0],
            [0.0, 0.0, 0.0, 0.0, 0.0, -1.0],
            [-1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
        ]

    def test_nplayers(self):
        nplayers = ap.nplayers(self.interactions)
        self.assertEqual(nplayers, 3)

    def test_payoff_matrix(self):
        payoff_matrix = ap.payoff_matrix(self.interactions, 3, Game())
        self.assertEqual(payoff_matrix, self.expected_payoff_matrix)

    def test_interaction_payoff(self):
        payoff = ap.interaction_payoff(self.interactions[(2, 2)], Game())
        self.assertEqual(payoff, (13, 3))

    def test_scores_matrix(self):
        scores = ap.scores_matrix(self.interactions, Game())
        self.assertEqual(scores, self.expected_scores)

    def test_scores(self):
        scores = ap.scores(self.expected_payoff)
        self.assertEqual(scores, self.expected_scores)

    def test_normalised_scores(self):
        scores = ap.normalised_scores(self.expected_scores, 5)
        self.assertEqual(scores, self.expected_normalised_scores)

    def test_ranking(self):
        ranking = ap.ranking(self.expected_scores)
        self.assertEqual(ranking, self.expected_ranking)

    def test_ranked_names(self):
        ranked_names = ap.ranked_names(
            self.players, self.expected_ranking,)
        self.assertEqual(ranked_names, self.expected_ranked_names)

    def test_payoff_diffs_means(self):
        #payoff_matrix = ap.payoff_matrix(self.interactions, Game(), 3, 5)
        diff_means = ap.payoff_diffs_means(self.expected_payoff, 5)
        self.assertEqual(diff_means, self.diff_means)

    def test_score_diffs(self):
        #payoff_matrix = ap.payoff_matrix(self.interactions, Game(), 3, 5)
        score_diffs = ap.score_diffs(self.expected_payoff, 5)
        self.assertEqual(score_diffs, self.expected_score_diffs)

    @staticmethod
    def round_matrix(matrix, precision):
        return [[round(x, precision) for x in row] for row in matrix]

    def test_normalised_payoff(self):
        averages, stddevs = ap.normalised_payoff(self.expected_payoff, 5)
        self.assertEqual(
            self.round_matrix(averages, 2), self.expected_normalised_payoff)
        self.assertEqual(
            self.round_matrix(stddevs, 2), self.expected_payoff_stddevs)

    def test_winning_player(self):
        test_players = (8, 4)
        test_payoffs = (34, 44)
        winner = ap.winning_player(test_players, test_payoffs)
        self.assertEqual(winner, 4)
        test_payoffs = (54, 44)
        winner = ap.winning_player(test_players, test_payoffs)
        self.assertEqual(winner, 8)
        test_payoffs = (34, 34)
        winner = ap.winning_player(test_players, test_payoffs)
        self.assertEqual(winner, None)

    def test_wins(self):
        wins = ap.wins(self.expected_payoff)
        self.assertEqual(wins, self.expected_wins)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  Copyright (C) Inkton 2016 <thebird@nest.yt>#
## -*- python -*-
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

import errno, os, sys
import argparse

from datetime import datetime, date, time
from subprocess import Popen, PIPE, STDOUT

from thing import FailedValidation
from cloud import Cloud
from auth import Auth

class Deployment(Cloud):

    def __init__(self, auth):
        super(Deployment, self).__init__('deployment', auth)

    def new_copy(self, object):
        new_entity = Deployment(self.auth)
        new_entity.__dict__.update(object)
        return new_entity

    def parse_command(self, subparsers):
        cmd_parser = subparsers.add_parser(self.subject, help='Manage the deployment')
        app_cmd_parsers = cmd_parser.add_subparsers(dest='dep_command', help='Deployment commands')

        app_cmd_parsers.add_parser('pull', help='Download the deployment')
        app_cmd_parsers.add_parser('push', help='Publish the source and deploy')
        app_cmd_parsers.add_parser('restore', help='Restore the project')
        app_cmd_parsers.add_parser('clear', help='Clear the project')
        app_cmd_parsers.add_parser('clean', help='Clean the project')
        app_cmd_parsers.add_parser('build', help='Build the project')
        app_cmd_parsers.add_parser('uncache', help='Clear cache')

    def exec_command(self, args):
        self.set_log(args.log)
        cmd_handled = False
        if args.command == self.subject:
            self.log("handle deployment command")
            cmd_handled = True
            if (args.dep_command == 'pull'):
                self.pull()
            elif (args.dep_command == 'push'):
                self.push()
            elif (args.app_command == 'restore'):
                self.restore()
            elif (args.app_command == 'build'):
                self.build()
            elif (args.app_command == 'clear'):
                self.clear()
            elif (args.app_command == 'clean'):
                self.clean()
            elif (args.app_command == 'uncache'):
                self.clear_cache()
            else:
                cmd_handled = False
            self.end_cmd()
        return cmd_handled

    def pull(self):
        try:
            self.os_exec("rm -rf " + self.get_source_target_folder())
            self.os_exec("mkdir -p " + self.get_source_target_folder())

            if not os.path.isdir("/tmp/source"):
                self.os_exec("git clone "
                + self.get_contact_id() + "@"
                + self.get_hostname() + ":repository.git /tmp/source")

            rsync_cmd = "/usr/bin/rsync -avzr "

            if self.get_platform_tag() == "api" or self.get_platform_tag() == "mvc":
                self.os_exec("rm -rf " + self.get_source_shared_folder())
                self.os_exec("mkdir -p " + self.get_source_shared_folder())
                self.os_exec("rsync -r /tmp/source/ " + self.get_source_shared_folder())
                self.os_exec("cd " + self.get_source_shared_folder() + " && git checkout " + self.get_source_shared_git_branch())
                self.os_exec(rsync_cmd + " --exclude=.git/ --exclude=bin --exclude=obj --timeout=120 --progress -e 'ssh' nest:" + self.get_source_shared_folder() + "/ " + self.get_source_shared_folder() + "/ ")
                self.os_exec(rsync_cmd + " --timeout=120 --progress -e 'ssh' nest:/var/app/log /var/app")
                self.os_exec(rsync_cmd + " --timeout=60 --progress -e 'ssh' nest:/var/app/source/" + self.get_app_tag_capitalized() + ".sln /var/app/source")
                self.os_exec(rsync_cmd + " --timeout=60 --progress -e 'ssh' nest:/var/app/app.nest /var/app")
                self.os_exec(rsync_cmd + " --timeout=60 --progress -e 'ssh' nest:/var/app/app.json /var/app")
                self.os_exec(rsync_cmd + " --exclude=nest/.git --timeout=120 --progress -e 'ssh' nest:/var/app/nest /var/app")

            self.os_exec("rsync -r /tmp/source/ " + self.get_source_target_folder())
            self.os_exec("cd " + self.get_source_target_folder() + " && git checkout " + self.get_source_target_git_branch())
            self.os_exec(rsync_cmd + " --exclude=.git --exclude=bin --exclude=obj --timeout=120 --progress -e 'ssh' nest:" + self.get_source_target_folder() + "/ " + self.get_source_target_folder() + "/")

            self.setup_git()
        except Exception as e:
                print(e)

    def push(self):
        try:
            rsync_cmd = "/usr/bin/rsync -avzr "
            self.os_exec("rsync -r /tmp/source/ " + self.get_source_shared_folder())
            self.os_exec(rsync_cmd + " --exclude=.git --exclude=bin --exclude=obj --timeout=120 --progress " + self.get_source_shared_folder() + "/ -e 'ssh' nest:" + self.get_source_shared_folder() + "/ ")
            self.os_exec(rsync_cmd + " --timeout=60 --progress /var/app/app.nest -e 'ssh' nest:/var/app")
            self.os_exec(rsync_cmd + " --timeout=60 --progress /var/app/app.json -e 'ssh' nest:/var/app")
            self.os_exec(rsync_cmd + " --exclude=nest/.git --timeout=120 --progress /var/app/nest -e 'ssh' nest:/var/app")

            self.os_exec(rsync_cmd + "--exclude=.git --exclude=bin --exclude=obj --timeout=120 --progress " + self.get_source_target_folder() + "/ -e 'ssh' nest:" + self.get_source_target_folder() + "/")
            self.os_exec("ssh nest 'deploy'", False)
        except Exception as e:
                print(e)

    def restore(self):
    	self.log("test restore")
        self.os_exec("dotnet restore --packages /var/app/packages " + self.get_source_target_folder(), False)

    def build(self):
    	self.log("build")
        self.os_exec("dotnet build " + self.get_source_target_folder() + " -c Debug ", False)

    def clean(self):
    	self.log("clean")
        self.os_exec("dotnet clean " + self.get_source_target_folder())

    def clear(self):
        self.log("remove build folders")
        self.os_exec("rm -rf " + self.get_source_target_folder() + "/obj")
        self.os_exec("rm -rf " + self.get_source_target_folder() + "/bin")

    def setup_git_for_user(self, user):
        self.log("setup git for user " + user)
        git_config = "sudo su - "+ user +" -c \"git config --file " + self.get_source_target_folder() + "/.git/config "
        self.os_exec(git_config + " user.email "+os.environ['NEST_CONTACT_EMAIL']+ "\"" )
        self.os_exec(git_config + " user.name "+os.environ['NEST_CONTACT_ID']+ "\"" )
        self.os_exec(git_config + " core.autocrlf false\"")
        self.os_exec(git_config + " push.default simple\"")

    def setup_git(self):
    	self.log("setup git")
    	self.setup_git_for_user(os.environ['NEST_CONTACT_ID'])
<commit_msg>fixed deployment commnad header<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  Copyright (C) Inkton 2016 <thebird@nest.yt>#
## -*- python -*-
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

import errno, os, sys
import argparse

from datetime import datetime, date, time
from subprocess import Popen, PIPE, STDOUT

from thing import FailedValidation
from cloud import Cloud
from auth import Auth

class Deployment(Cloud):

    def __init__(self, auth):
        super(Deployment, self).__init__('deployment', auth)

    def new_copy(self, object):
        new_entity = Deployment(self.auth)
        new_entity.__dict__.update(object)
        return new_entity

    def parse_command(self, subparsers):
        cmd_parser = subparsers.add_parser(self.subject, help='Manage the deployment')
        app_cmd_parsers = cmd_parser.add_subparsers(dest='dep_command', help='Deployment commands')

        app_cmd_parsers.add_parser('pull', help='Download the deployment')
        app_cmd_parsers.add_parser('push', help='Publish the source and deploy')
        app_cmd_parsers.add_parser('restore', help='Restore the project')
        app_cmd_parsers.add_parser('clear', help='Clear the project')
        app_cmd_parsers.add_parser('clean', help='Clean the project')
        app_cmd_parsers.add_parser('build', help='Build the project')
        app_cmd_parsers.add_parser('uncache', help='Clear cache')

    def exec_command(self, args):
        self.set_log(args.log)
        cmd_handled = False
        if args.command == self.subject:
            self.log("handle deployment command")
            cmd_handled = True
            if (args.dep_command == 'pull'):
                self.pull()
            elif (args.dep_command == 'push'):
                self.push()
            elif (args.dep_command == 'restore'):
                self.restore()
            elif (args.dep_command == 'build'):
                self.build()
            elif (args.dep_command == 'clear'):
                self.clear()
            elif (args.dep_command == 'clean'):
                self.clean()
            elif (args.dep_command == 'uncache'):
                self.clear_cache()
            else:
                cmd_handled = False
            self.end_cmd()
        return cmd_handled

    def pull(self):
        try:
            self.os_exec("rm -rf " + self.get_source_target_folder())
            self.os_exec("mkdir -p " + self.get_source_target_folder())

            if not os.path.isdir("/tmp/source"):
                self.os_exec("git clone "
                + self.get_contact_id() + "@"
                + self.get_hostname() + ":repository.git /tmp/source")

            rsync_cmd = "/usr/bin/rsync -avzr "

            if self.get_platform_tag() == "api" or self.get_platform_tag() == "mvc":
                self.os_exec("rm -rf " + self.get_source_shared_folder())
                self.os_exec("mkdir -p " + self.get_source_shared_folder())
                self.os_exec("rsync -r /tmp/source/ " + self.get_source_shared_folder())
                self.os_exec("cd " + self.get_source_shared_folder() + " && git checkout " + self.get_source_shared_git_branch())
                self.os_exec(rsync_cmd + " --exclude=.git/ --exclude=bin --exclude=obj --timeout=120 --progress -e 'ssh' nest:" + self.get_source_shared_folder() + "/ " + self.get_source_shared_folder() + "/ ")
                self.os_exec(rsync_cmd + " --timeout=120 --progress -e 'ssh' nest:/var/app/log /var/app")
                self.os_exec(rsync_cmd + " --timeout=60 --progress -e 'ssh' nest:/var/app/source/" + self.get_app_tag_capitalized() + ".sln /var/app/source")
                self.os_exec(rsync_cmd + " --timeout=60 --progress -e 'ssh' nest:/var/app/app.nest /var/app")
                self.os_exec(rsync_cmd + " --timeout=60 --progress -e 'ssh' nest:/var/app/app.json /var/app")
                self.os_exec(rsync_cmd + " --exclude=nest/.git --timeout=120 --progress -e 'ssh' nest:/var/app/nest /var/app")

            self.os_exec("rsync -r /tmp/source/ " + self.get_source_target_folder())
            self.os_exec("cd " + self.get_source_target_folder() + " && git checkout " + self.get_source_target_git_branch())
            self.os_exec(rsync_cmd + " --exclude=.git --exclude=bin --exclude=obj --timeout=120 --progress -e 'ssh' nest:" + self.get_source_target_folder() + "/ " + self.get_source_target_folder() + "/")

            self.setup_git()
        except Exception as e:
                print(e)

    def push(self):
        try:
            rsync_cmd = "/usr/bin/rsync -avzr "
            self.os_exec("rsync -r /tmp/source/ " + self.get_source_shared_folder())
            self.os_exec(rsync_cmd + " --exclude=.git --exclude=bin --exclude=obj --timeout=120 --progress " + self.get_source_shared_folder() + "/ -e 'ssh' nest:" + self.get_source_shared_folder() + "/ ")
            self.os_exec(rsync_cmd + " --timeout=60 --progress /var/app/app.nest -e 'ssh' nest:/var/app")
            self.os_exec(rsync_cmd + " --timeout=60 --progress /var/app/app.json -e 'ssh' nest:/var/app")
            self.os_exec(rsync_cmd + " --exclude=nest/.git --timeout=120 --progress /var/app/nest -e 'ssh' nest:/var/app")

            self.os_exec(rsync_cmd + "--exclude=.git --exclude=bin --exclude=obj --timeout=120 --progress " + self.get_source_target_folder() + "/ -e 'ssh' nest:" + self.get_source_target_folder() + "/")
            self.os_exec("ssh nest 'deploy'", False)
        except Exception as e:
                print(e)

    def restore(self):
    	self.log("test restore")
        self.os_exec("dotnet restore --packages /var/app/packages " + self.get_source_target_folder(), False)

    def build(self):
    	self.log("build")
        self.os_exec("dotnet build " + self.get_source_target_folder() + " -c Debug ", False)

    def clean(self):
    	self.log("clean")
        self.os_exec("dotnet clean " + self.get_source_target_folder())

    def clear(self):
        self.log("remove build folders")
        self.os_exec("rm -rf " + self.get_source_target_folder() + "/obj")
        self.os_exec("rm -rf " + self.get_source_target_folder() + "/bin")

    def setup_git_for_user(self, user):
        self.log("setup git for user " + user)
        git_config = "sudo su - "+ user +" -c \"git config --file " + self.get_source_target_folder() + "/.git/config "
        self.os_exec(git_config + " user.email "+os.environ['NEST_CONTACT_EMAIL']+ "\"" )
        self.os_exec(git_config + " user.name "+os.environ['NEST_CONTACT_ID']+ "\"" )
        self.os_exec(git_config + " core.autocrlf false\"")
        self.os_exec(git_config + " push.default simple\"")

    def setup_git(self):
    	self.log("setup git")
    	self.setup_git_for_user(os.environ['NEST_CONTACT_ID'])
<|endoftext|>


<commit_before>from os.path import join
from copy import deepcopy

import mne
import numpy as np
from joblib import Memory
from scipy.signal import tukey

MNE_DATA = mne.datasets.somato.data_path()

mem = Memory(location='.', verbose=0)

@mem.cache(ignore=['n_jobs'])
def load_data(dataset="somato", n_splits=10, sfreq=None, epoch=None,
              filter_params=[2., None], return_array=True, n_jobs=1):
    """Load and prepare the somato dataset for multiCSC

    Parameters
    ----------
    dataset : str in {'somato', 'sample'}
        Dataset to load.
    n_splits : int
        Split the signal in n_split signals of same length before returning it.
        If epoch is provided, the signal is instead splitted according to the
        epochs and this option is not followed.
    sfreq : float
        Sampling frequency of the signal. The data are resampled to match it.
    epoch : tuple or None
        If set to a tuple, extract epochs from the raw data, using
        t_min=epoch[0] and t_max=epoch[1]. Else, use the raw signal, divided
        in n_splits chunks.
    filter_params : tuple of length 2
        Boundaries of filtering, e.g. (2, None), (30, 40), (None, 40).
    return_array : boolean
        If True, return an NumPy array, instead of mne objects.
    n_jobs : int
        Number of jobs that can be used for preparing (filtering) the data.

    Returns
    -------
    X : array, shape (n_splits, n_channels, n_times)
        The loaded dataset.
    info : dict
        MNE dictionary of information about recording settings.
    """

    pick_types_epoch = dict(meg='grad', eeg=False, eog=True, stim=False)
    pick_types_final = dict(meg='grad', eeg=False, eog=False, stim=False)

    subjects_dir = join(MNE_DATA, "subjects")
    if dataset == 'somato':
        data_path = join(MNE_DATA, 'MEG', 'somato')
        file_name = join(data_path, 'sef_raw_sss.fif')
        raw = mne.io.read_raw_fif(file_name, preload=True)
        raw.notch_filter(np.arange(50, 101, 50), n_jobs=n_jobs)
        event_id = 1

        # Dipole fit information
        file_trans = join(data_path, "sef_raw_sss-trans.fif")
        file_bem = join(subjects_dir, 'somato', 'bem',
                        'somato-5120-bem-sol.fif')

    elif dataset == 'sample':
        data_path = mne.datasets.sample.data_path()
        data_path = join(data_path, 'MEG', 'sample')
        file_name = join(data_path, 'sample_audvis_raw.fif')
        raw = mne.io.read_raw_fif(file_name, preload=True)
        raw.notch_filter(np.arange(60, 181, 60), n_jobs=n_jobs)
        event_id = [1, 2, 3, 4]

        # Dipole fit information
        cov = join(data_path, 'sample_audvis-cov.fif')
        file_trans = join(data_path, 'sample_audvis_raw-trans.fif')
        file_bem = join(subjects_dir, 'sample', 'bem',
                        'sample-5120-bem-sol.fif')

    else:
        raise ValueError('Unknown parameter dataset=%s.' % (dataset, ))
    raw.filter(*filter_params, n_jobs=n_jobs)

    baseline = (None, 0)
    events = mne.find_events(raw, stim_channel='STI 014')
    events = mne.pick_events(events, include=event_id)

    # compute the covariance matrix for somato
    if dataset == "somato":
        picks_cov = mne.pick_types(raw.info, **pick_types_epoch)
        epochs_cov = mne.Epochs(raw, events, event_id, tmin=-4, tmax=0,
                                picks=picks_cov, baseline=baseline,
                                reject=dict(grad=4000e-13, eog=350e-6),
                                preload=True)
        epochs_cov.pick_types(**pick_types_final)
        cov = mne.compute_covariance(epochs_cov)

    if epoch:
        t_min, t_max = epoch

        picks = mne.pick_types(raw.info, **pick_types_epoch)
        epochs = mne.Epochs(raw, events, event_id, t_min, t_max, picks=picks,
                            baseline=baseline, reject=dict(
                                grad=4000e-13, eog=350e-6), preload=True)
        epochs.pick_types(**pick_types_final)
        info = epochs.info
        if sfreq is not None:
            epochs = epochs.resample(sfreq, npad='auto', n_jobs=n_jobs)

        if return_array:
            X = epochs.get_data()

    else:
        events[:, 0] -= raw.first_samp
        raw.pick_types(**pick_types_final)
        info = raw.info

        if sfreq is not None:
            raw, events = raw.resample(sfreq, events=events, npad='auto',
                                       n_jobs=n_jobs)

        if return_array:
            X = raw.get_data()
            n_channels, n_times = X.shape
            n_times = n_times // n_splits
            X = X[:, :n_splits * n_times]
            X = X.reshape(n_channels, n_splits, n_times).swapaxes(0, 1)

    # Deep copy before modifying info to avoid issues when saving EvokedArray
    info = deepcopy(info)
    info['events'] = events
    info['event_id'] = event_id
    info['subject'] = dataset
    info['subjects_dir'] = subjects_dir

    info['cov'] = cov
    info['file_bem'] = file_bem
    info['file_trans'] = file_trans

    if return_array:
        n_splits, n_channels, n_times = X.shape
        X *= tukey(n_times, alpha=0.1)[None, None, :]
        X /= np.std(X)
        return X, info
    elif epoch:
        return epoch, info
    else:
        return raw, info
<commit_msg>FIX data path in loader<commit_after>from os.path import join
from copy import deepcopy

import mne
import numpy as np
from joblib import Memory
from scipy.signal import tukey

mem = Memory(location='.', verbose=0)

@mem.cache(ignore=['n_jobs'])
def load_data(dataset="somato", n_splits=10, sfreq=None, epoch=None,
              filter_params=[2., None], return_array=True, n_jobs=1):
    """Load and prepare the somato dataset for multiCSC

    Parameters
    ----------
    dataset : str in {'somato', 'sample'}
        Dataset to load.
    n_splits : int
        Split the signal in n_split signals of same length before returning it.
        If epoch is provided, the signal is instead splitted according to the
        epochs and this option is not followed.
    sfreq : float
        Sampling frequency of the signal. The data are resampled to match it.
    epoch : tuple or None
        If set to a tuple, extract epochs from the raw data, using
        t_min=epoch[0] and t_max=epoch[1]. Else, use the raw signal, divided
        in n_splits chunks.
    filter_params : tuple of length 2
        Boundaries of filtering, e.g. (2, None), (30, 40), (None, 40).
    return_array : boolean
        If True, return an NumPy array, instead of mne objects.
    n_jobs : int
        Number of jobs that can be used for preparing (filtering) the data.

    Returns
    -------
    X : array, shape (n_splits, n_channels, n_times)
        The loaded dataset.
    info : dict
        MNE dictionary of information about recording settings.
    """

    pick_types_epoch = dict(meg='grad', eeg=False, eog=True, stim=False)
    pick_types_final = dict(meg='grad', eeg=False, eog=False, stim=False)

    if dataset == 'somato':
        data_path = mne.datasets.somato.data_path()
        subjects_dir = join(data_path, "subjects")
        data_dir = join(data_path, 'MEG', 'somato')
        file_name = join(data_dir, 'sef_raw_sss.fif')
        raw = mne.io.read_raw_fif(file_name, preload=True)
        raw.notch_filter(np.arange(50, 101, 50), n_jobs=n_jobs)
        event_id = 1

        # Dipole fit information
        cov = None  # see below
        file_trans = join(data_dir, "sef_raw_sss-trans.fif")
        file_bem = join(subjects_dir, 'somato', 'bem',
                        'somato-5120-bem-sol.fif')

    elif dataset == 'sample':
        data_path = mne.datasets.sample.data_path()
        subjects_dir = join(data_path, "subjects")
        data_dir = join(data_path, 'MEG', 'sample')
        file_name = join(data_dir, 'sample_audvis_raw.fif')
        raw = mne.io.read_raw_fif(file_name, preload=True)
        raw.notch_filter(np.arange(60, 181, 60), n_jobs=n_jobs)
        event_id = [1, 2, 3, 4]

        # Dipole fit information
        cov = join(data_dir, 'sample_audvis-cov.fif')
        file_trans = join(data_dir, 'sample_audvis_raw-trans.fif')
        file_bem = join(subjects_dir, 'sample', 'bem',
                        'sample-5120-bem-sol.fif')

    else:
        raise ValueError('Unknown parameter dataset=%s.' % (dataset, ))
    raw.filter(*filter_params, n_jobs=n_jobs)

    baseline = (None, 0)
    events = mne.find_events(raw, stim_channel='STI 014')
    events = mne.pick_events(events, include=event_id)

    # compute the covariance matrix for somato
    if dataset == "somato":
        picks_cov = mne.pick_types(raw.info, **pick_types_epoch)
        epochs_cov = mne.Epochs(raw, events, event_id, tmin=-4, tmax=0,
                                picks=picks_cov, baseline=baseline,
                                reject=dict(grad=4000e-13, eog=350e-6),
                                preload=True)
        epochs_cov.pick_types(**pick_types_final)
        cov = mne.compute_covariance(epochs_cov)

    if epoch:
        t_min, t_max = epoch

        picks = mne.pick_types(raw.info, **pick_types_epoch)
        epochs = mne.Epochs(raw, events, event_id, t_min, t_max, picks=picks,
                            baseline=baseline, reject=dict(
                                grad=4000e-13, eog=350e-6), preload=True)
        epochs.pick_types(**pick_types_final)
        info = epochs.info
        if sfreq is not None:
            epochs = epochs.resample(sfreq, npad='auto', n_jobs=n_jobs)

        if return_array:
            X = epochs.get_data()

    else:
        events[:, 0] -= raw.first_samp
        raw.pick_types(**pick_types_final)
        info = raw.info

        if sfreq is not None:
            raw, events = raw.resample(sfreq, events=events, npad='auto',
                                       n_jobs=n_jobs)

        if return_array:
            X = raw.get_data()
            n_channels, n_times = X.shape
            n_times = n_times // n_splits
            X = X[:, :n_splits * n_times]
            X = X.reshape(n_channels, n_splits, n_times).swapaxes(0, 1)

    # Deep copy before modifying info to avoid issues when saving EvokedArray
    info = deepcopy(info)
    info['events'] = events
    info['event_id'] = event_id
    info['subject'] = dataset
    info['subjects_dir'] = subjects_dir

    info['cov'] = cov
    info['file_bem'] = file_bem
    info['file_trans'] = file_trans

    if return_array:
        n_splits, n_channels, n_times = X.shape
        X *= tukey(n_times, alpha=0.1)[None, None, :]
        X /= np.std(X)
        return X, info
    elif epoch:
        return epoch, info
    else:
        return raw, info
<|endoftext|>


<commit_before>import numpy as np

def _checked(f):
    def wrapper(*args, **kwargs):
        return f(*args, **kwargs).reshape((-1, 1))
    return wrapper

def _length(a):
    return np.linalg.norm(a, axis=1)

def _normalize(a):
    return a / _length(a)

def _dot(a, b):
    return np.sum(a * b, axis=1)

def sphere(center, radius):
    @_checked
    def f(p):
        return _length(p - center) - radius
    return f

def box(size):
    size = np.array(size) / 2
    @_checked
    def f(p):
        q = np.abs(p) - size
        return _length(np.maximum(q, 0)) + np.minimum(np.amax(q, axis=1), 0)
    return f

def aabb(a, b):
    a = np.array(a)
    b = np.array(b)
    size = b - a
    offset = a + size / 2
    return translate(offset, box(size))

def round_box(size, radius):
    size = np.array(size) / 2
    @_checked
    def f(p):
        q = np.abs(p) - size
        return _length(np.maximum(q, 0)) + np.minimum(np.amax(q, axis=1), 0) - radius
    return f

def torus(r1, r2):
    @_checked
    def f(p):
        xz = p[:,[0,2]]
        y = p[:,1]
        a = _length(xz) - r1
        b = _length(np.stack([a, y], axis=1)) - r2
        return b
    return f

def capsule(a, b, radius):
    a = np.array(a)
    b = np.array(b)
    @_checked
    def f(p):
        pa = p - a
        ba = b - a
        h = np.clip(np.dot(pa, ba) / np.dot(ba, ba), 0, 1).reshape((-1, 1))
        return _length(pa - np.multiply(ba, h)) - radius
    return f

def capped_cylinder(a, b, radius):
    a = np.array(a)
    b = np.array(b)
    @_checked
    def f(p):
        ba = b - a
        pa = p - a
        baba = np.dot(ba, ba)
        paba = np.dot(pa, ba).reshape((-1, 1))
        x = _length(pa * baba - ba * paba) - radius * baba
        y = np.abs(paba - baba * 0.5) - baba * 0.5
        x = x.reshape((-1, 1))
        y = y.reshape((-1, 1))
        x2 = x * x
        y2 = y * y * baba
        d = np.where(
            np.maximum(x, y) < 0,
            -np.minimum(x2, y2),
            np.where(x > 0, x2, 0) + np.where(y > 0, y2, 0))
        return np.sign(d) * np.sqrt(np.abs(d)) / baba
    return f

def rounded_cylinder(ra, rb, h):
    @_checked
    def f(p):
        d = np.stack([
            _length(p[:,[0,2]]) - 2 * ra + rb,
            np.abs(p[:,1]) - h], axis=-1)
        return (
            np.minimum(np.maximum(d[:,0], d[:,1]), 0) +
            _length(np.maximum(d, 0)) - rb)
    return f

def capped_cone(a, b, ra, rb):
    a = np.array(a)
    b = np.array(b)
    @_checked
    def f(p):
        rba = rb - ra
        baba = np.dot(b - a, b - a)
        papa = _dot(p - a, p - a)
        paba = np.dot(p - a, b - a) / baba
        x = np.sqrt(papa - paba * paba * baba)
        cax = np.maximum(0, x - np.where(paba < 0.5, ra, rb))
        cay = np.abs(paba - 0.5) - 0.5
        k = rba * rba + baba
        f = np.clip((rba * (x - ra) + paba * baba) / k, 0, 1)
        cbx = x - ra - f * rba
        cby = paba - f
        s = np.where(np.logical_and(cbx < 0, cay < 0), -1, 1)
        return s * np.sqrt(np.minimum(
            cax * cax + cay * cay * baba,
            cbx * cbx + cby * cby * baba))
    return f

def ellipsoid(size):
    size = np.array(size)
    @_checked
    def f(p):
        k0 = _length(p / size)
        k1 = _length(p / (size * size))
        return k0 * (k0 - 1) / k1
    return f

def pyramid(h):
    @_checked
    def f(p):
        a = np.abs(p[:,[0,2]]) - 0.5
        w = a[:,1] > a[:,0]
        a[w] = a[:,[1,0]][w]
        px = a[:,0]
        py = p[:,1]
        pz = a[:,1]
        m2 = h * h + 0.25
        qx = pz
        qy = h * py - 0.5 * px
        qz = h * px + 0.5 * py
        s = np.maximum(-qx, 0)
        t = np.clip((qy - 0.5 * pz) / (m2 + 0.25), 0, 1)
        a = m2 * (qx + s) ** 2 + qy * qy
        b = m2 * (qx + 0.5 * t) ** 2 + (qy - m2 * t) ** 2
        d2 = np.where(
            np.minimum(qy, -qx * m2 - qy * 0.5) > 0,
            0, np.minimum(a, b))
        return np.sqrt((d2 + qz * qz) / m2) * np.sign(np.maximum(qz, -py))
    return f

def octahedron(s):
    @_checked
    def f(p):
        return (np.sum(np.abs(p), axis=1) - s) * 0.57735027
    return f

def union(a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            d1 = np.minimum(d1, d2)
        return d1
    return f

def difference(a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            d1 = np.maximum(d1, -d2)
        return d1
    return f

def intersection(a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            d1 = np.maximum(d1, d2)
        return d1
    return f

def smooth_union(k, a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            h = np.clip(0.5 + 0.5 * (d2 - d1) / k, 0, 1)
            m = d2 + (d1 - d2) * h
            d1 = m - k * h * (1 - h)
        return d1
    return f

def smooth_difference(k, a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            h = np.clip(0.5 - 0.5 * (d2 + d1) / k, 0, 1)
            m = d1 + (-d2 - d1) * h
            d1 = m + k * h * (1 - h)
        return d1
    return f

def smooth_intersection(k, a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            h = np.clip(0.5 - 0.5 * (d2 - d1) / k, 0, 1)
            m = d2 + (d1 - d2) * h
            d1 = m + k * h * (1 - h)
        return d1
    return f

def blend(k, a, b):
    @_checked
    def f(p):
        return k * b(p) + (1 - k) * a(p)
    return f

def translate(offset, sdf):
    @_checked
    def f(p):
        return sdf(p - offset)
    return f

def scale(factor, sdf):
    try:
        x, y, z = factor
    except TypeError:
        x = y = z = factor
    @_checked
    def f(p):
        return sdf(p / (x, y, z))
    return f

def rotate(vector, angle, sdf):
    x, y, z = _normalize(np.array(vector).reshape(1, -1))[0]
    s = np.sin(angle)
    c = np.cos(angle)
    m = 1 - c
    matrix = np.array([
        [m*x*x + c, m*x*y + z*s, m*z*x - y*s],
        [m*x*y - z*s, m*y*y + c, m*y*z + x*s],
        [m*z*x + y*s, m*y*z - x*s, m*z*z + c],
    ]).T
    @_checked
    def f(p):
        return sdf(np.dot(p, matrix))
    return f

def repeat(count, spacing, sdf):
    count = np.array(count)
    spacing = np.array(spacing)
    @_checked
    def f(p):
        q = p - spacing * np.clip(np.round(p / spacing), -count, count)
        return sdf(q)
    return f

def elongate(size, sdf):
    @_checked
    def f(p):
        q = np.abs(p) - size
        x = q[:,0].reshape((-1, 1))
        y = q[:,1].reshape((-1, 1))
        z = q[:,2].reshape((-1, 1))
        w = np.minimum(np.maximum(x, np.maximum(y, z)), 0)
        return sdf(np.maximum(q, 0)) + w
    return f

def twist(k, sdf):
    @_checked
    def f(p):
        x = p[:,0]
        y = p[:,1]
        z = p[:,2]
        c = np.cos(k * z)
        s = np.sin(k * z)
        x2 = c * x - s * y
        y2 = s * x + c * y
        z2 = z
        return sdf(np.stack([x2, y2, z2], axis=-1))
    return f

def bend(k, sdf):
    @_checked
    def f(p):
        x = p[:,0]
        y = p[:,1]
        z = p[:,2]
        c = np.cos(k * x)
        s = np.sin(k * x)
        x2 = c * x - s * y
        y2 = s * x + c * y
        z2 = z
        return sdf(np.stack([x2, y2, z2], axis=-1))
    return f

def onion(thickness, sdf):
    @_checked
    def f(p):
        return np.abs(sdf(p)) - thickness
    return f
<commit_msg>platonic solids<commit_after>import numpy as np

def _checked(f):
    def wrapper(*args, **kwargs):
        return f(*args, **kwargs).reshape((-1, 1))
    return wrapper

def _length(a):
    return np.linalg.norm(a, axis=1)

def _normalize(a):
    return a / np.linalg.norm(a)

def _dot(a, b):
    return np.sum(a * b, axis=1)

def sphere(center, radius):
    @_checked
    def f(p):
        return _length(p - center) - radius
    return f

def box(size):
    size = np.array(size) / 2
    @_checked
    def f(p):
        q = np.abs(p) - size
        return _length(np.maximum(q, 0)) + np.minimum(np.amax(q, axis=1), 0)
    return f

def aabb(a, b):
    a = np.array(a)
    b = np.array(b)
    size = b - a
    offset = a + size / 2
    return translate(offset, box(size))

def round_box(size, radius):
    size = np.array(size) / 2
    @_checked
    def f(p):
        q = np.abs(p) - size
        return _length(np.maximum(q, 0)) + np.minimum(np.amax(q, axis=1), 0) - radius
    return f

def torus(r1, r2):
    @_checked
    def f(p):
        xz = p[:,[0,2]]
        y = p[:,1]
        a = _length(xz) - r1
        b = _length(np.stack([a, y], axis=1)) - r2
        return b
    return f

def capsule(a, b, radius):
    a = np.array(a)
    b = np.array(b)
    @_checked
    def f(p):
        pa = p - a
        ba = b - a
        h = np.clip(np.dot(pa, ba) / np.dot(ba, ba), 0, 1).reshape((-1, 1))
        return _length(pa - np.multiply(ba, h)) - radius
    return f

def capped_cylinder(a, b, radius):
    a = np.array(a)
    b = np.array(b)
    @_checked
    def f(p):
        ba = b - a
        pa = p - a
        baba = np.dot(ba, ba)
        paba = np.dot(pa, ba).reshape((-1, 1))
        x = _length(pa * baba - ba * paba) - radius * baba
        y = np.abs(paba - baba * 0.5) - baba * 0.5
        x = x.reshape((-1, 1))
        y = y.reshape((-1, 1))
        x2 = x * x
        y2 = y * y * baba
        d = np.where(
            np.maximum(x, y) < 0,
            -np.minimum(x2, y2),
            np.where(x > 0, x2, 0) + np.where(y > 0, y2, 0))
        return np.sign(d) * np.sqrt(np.abs(d)) / baba
    return f

def rounded_cylinder(ra, rb, h):
    @_checked
    def f(p):
        d = np.stack([
            _length(p[:,[0,2]]) - 2 * ra + rb,
            np.abs(p[:,1]) - h], axis=-1)
        return (
            np.minimum(np.maximum(d[:,0], d[:,1]), 0) +
            _length(np.maximum(d, 0)) - rb)
    return f

def capped_cone(a, b, ra, rb):
    a = np.array(a)
    b = np.array(b)
    @_checked
    def f(p):
        rba = rb - ra
        baba = np.dot(b - a, b - a)
        papa = _dot(p - a, p - a)
        paba = np.dot(p - a, b - a) / baba
        x = np.sqrt(papa - paba * paba * baba)
        cax = np.maximum(0, x - np.where(paba < 0.5, ra, rb))
        cay = np.abs(paba - 0.5) - 0.5
        k = rba * rba + baba
        f = np.clip((rba * (x - ra) + paba * baba) / k, 0, 1)
        cbx = x - ra - f * rba
        cby = paba - f
        s = np.where(np.logical_and(cbx < 0, cay < 0), -1, 1)
        return s * np.sqrt(np.minimum(
            cax * cax + cay * cay * baba,
            cbx * cbx + cby * cby * baba))
    return f

def ellipsoid(size):
    size = np.array(size)
    @_checked
    def f(p):
        k0 = _length(p / size)
        k1 = _length(p / (size * size))
        return k0 * (k0 - 1) / k1
    return f

def pyramid(h):
    @_checked
    def f(p):
        a = np.abs(p[:,[0,2]]) - 0.5
        w = a[:,1] > a[:,0]
        a[w] = a[:,[1,0]][w]
        px = a[:,0]
        py = p[:,1]
        pz = a[:,1]
        m2 = h * h + 0.25
        qx = pz
        qy = h * py - 0.5 * px
        qz = h * px + 0.5 * py
        s = np.maximum(-qx, 0)
        t = np.clip((qy - 0.5 * pz) / (m2 + 0.25), 0, 1)
        a = m2 * (qx + s) ** 2 + qy * qy
        b = m2 * (qx + 0.5 * t) ** 2 + (qy - m2 * t) ** 2
        d2 = np.where(
            np.minimum(qy, -qx * m2 - qy * 0.5) > 0,
            0, np.minimum(a, b))
        return np.sqrt((d2 + qz * qz) / m2) * np.sign(np.maximum(qz, -py))
    return f

def union(a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            d1 = np.minimum(d1, d2)
        return d1
    return f

def difference(a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            d1 = np.maximum(d1, -d2)
        return d1
    return f

def intersection(a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            d1 = np.maximum(d1, d2)
        return d1
    return f

def smooth_union(k, a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            h = np.clip(0.5 + 0.5 * (d2 - d1) / k, 0, 1)
            m = d2 + (d1 - d2) * h
            d1 = m - k * h * (1 - h)
        return d1
    return f

def smooth_difference(k, a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            h = np.clip(0.5 - 0.5 * (d2 + d1) / k, 0, 1)
            m = d1 + (-d2 - d1) * h
            d1 = m + k * h * (1 - h)
        return d1
    return f

def smooth_intersection(k, a, *bs):
    @_checked
    def f(p):
        d1 = a(p)
        for b in bs:
            d2 = b(p)
            h = np.clip(0.5 - 0.5 * (d2 - d1) / k, 0, 1)
            m = d2 + (d1 - d2) * h
            d1 = m + k * h * (1 - h)
        return d1
    return f

def blend(k, a, b):
    @_checked
    def f(p):
        return k * b(p) + (1 - k) * a(p)
    return f

def translate(offset, sdf):
    @_checked
    def f(p):
        return sdf(p - offset)
    return f

def scale(factor, sdf):
    try:
        x, y, z = factor
    except TypeError:
        x = y = z = factor
    @_checked
    def f(p):
        return sdf(p / (x, y, z))
    return f

def rotate(vector, angle, sdf):
    x, y, z = _normalize(vector)
    s = np.sin(angle)
    c = np.cos(angle)
    m = 1 - c
    matrix = np.array([
        [m*x*x + c, m*x*y + z*s, m*z*x - y*s],
        [m*x*y - z*s, m*y*y + c, m*y*z + x*s],
        [m*z*x + y*s, m*y*z - x*s, m*z*z + c],
    ]).T
    @_checked
    def f(p):
        return sdf(np.dot(p, matrix))
    return f

def repeat(count, spacing, sdf):
    count = np.array(count)
    spacing = np.array(spacing)
    @_checked
    def f(p):
        q = p - spacing * np.clip(np.round(p / spacing), -count, count)
        return sdf(q)
    return f

def elongate(size, sdf):
    @_checked
    def f(p):
        q = np.abs(p) - size
        x = q[:,0].reshape((-1, 1))
        y = q[:,1].reshape((-1, 1))
        z = q[:,2].reshape((-1, 1))
        w = np.minimum(np.maximum(x, np.maximum(y, z)), 0)
        return sdf(np.maximum(q, 0)) + w
    return f

def twist(k, sdf):
    @_checked
    def f(p):
        x = p[:,0]
        y = p[:,1]
        z = p[:,2]
        c = np.cos(k * z)
        s = np.sin(k * z)
        x2 = c * x - s * y
        y2 = s * x + c * y
        z2 = z
        return sdf(np.stack([x2, y2, z2], axis=-1))
    return f

def bend(k, sdf):
    @_checked
    def f(p):
        x = p[:,0]
        y = p[:,1]
        z = p[:,2]
        c = np.cos(k * x)
        s = np.sin(k * x)
        x2 = c * x - s * y
        y2 = s * x + c * y
        z2 = z
        return sdf(np.stack([x2, y2, z2], axis=-1))
    return f

def onion(thickness, sdf):
    @_checked
    def f(p):
        return np.abs(sdf(p)) - thickness
    return f

def tetrahedron(r):
    @_checked
    def f(p):
        x = p[:,0]
        y = p[:,1]
        z = p[:,2]
        return (np.maximum(np.abs(x + y) - z, np.abs(x - y) + z) - 1) / np.sqrt(3)
    return f

def cube(r):
    @_checked
    def f(p):
        q = np.abs(p) - r
        return _length(np.maximum(q, 0)) + np.minimum(np.amax(q, axis=1), 0)
    return f

def octahedron(r):
    @_checked
    def f(p):
        return (np.sum(np.abs(p), axis=1) - r) * np.tan(np.radians(30))
    return f

def dodecahedron(r):
    x, y, z = _normalize(((1 + np.sqrt(5)) / 2, 1, 0))
    @_checked
    def f(p):
        p = np.abs(p / r)
        a = np.dot(p, (x, y, z))
        b = np.dot(p, (z, x, y))
        c = np.dot(p, (y, z, x))
        q = (np.maximum(np.maximum(a, b), c) - x) * r
        return q
    return f

def icosahedron(r):
    x, y, z = _normalize(((np.sqrt(5) + 3) / 2, 1, 0))
    w = np.sqrt(3) / 3
    @_checked
    def f(p):
        p = np.abs(p / r)
        a = np.dot(p, (x, y, z))
        b = np.dot(p, (z, x, y))
        c = np.dot(p, (y, z, x))
        d = np.dot(p, (w, w, w)) - x
        return np.maximum(np.maximum(np.maximum(a, b), c) - x, d) * r
    return f
<|endoftext|>


<commit_before># pylint: disable=line-too-long, no-member

import datetime
import json
import time

from django.conf import settings
from django.template.loader import render_to_string
from django.utils import timezone

from ..models import DataPoint

def extract_secondary_identifier(properties):
    if 'reading-type' in properties:
        return properties['reading-type']

    return None

def generator_name(identifier): # pylint: disable=unused-argument
    return 'Google Fit'

def compile_frequency_visualization(identifier, points, folder): # pylint: disable=unused-argument
    now = timezone.now()

    now = now.replace(second=0, microsecond=0)

    remainder = now.minute % 10

    now = now.replace(minute=(now.minute - remainder))

    start = now - datetime.timedelta(days=2)

    points = points.filter(created__lte=now, created__gte=start).order_by('created')

    end = start + datetime.timedelta(seconds=600)
    point_index = 0
    point_count = points.count()

    point = None

    if point_count > 0:
        point = points[point_index]

    timestamp_counts = {}

    keys = []

    while start < now:
        timestamp = str(time.mktime(start.timetuple()))

        keys.append(timestamp)

        timestamp_counts[timestamp] = 0

        while point is not None and point.created < end and point_index < (point_count - 1):
            timestamp_counts[timestamp] += 1

            point_index += 1

            point = points[point_index]

        start = end
        end = start + datetime.timedelta(seconds=600)

    timestamp_counts['keys'] = keys

    with open(folder + '/timestamp-counts.json', 'w') as outfile:
        json.dump(timestamp_counts, outfile, indent=2)

# def compile_report(generator, sources): # pylint: disable=too-many-locals
#    timestamp = arrow.get()
#    filename = tempfile.gettempdir() + '/pdk_export_' + str(timestamp.seconds) + '.' + \
#               str(timestamp.seconds / 1e6) + '.zip'
#
#    with ZipFile(filename, 'w') as export_file:
#        for secondary_identifier in SECONDARY_FIELDS:
#            secondary_filename = tempfile.gettempdir() + '/' + generator + '-' + \
#                                 secondary_identifier + '.txt'
#
#            with open(secondary_filename, 'w') as outfile:
#                writer = csv.writer(outfile, delimiter='\t')
#
#                columns = [
#                    'Source',
#                    'Created Timestamp',
#                    'Created Date',
#                ]
#
#                for column in SECONDARY_FIELDS[secondary_identifier]:
#                    columns.append(column)
#
#                writer.writerow(columns)
#
#                for source in sources:
#                    points = DataPoint.objects.filter(source=source, generator_identifier=generator, secondary_identifier=secondary_identifier).order_by('source', 'created') # pylint: disable=no-member,line-too-long
#
#                    index = 0
#                    count = points.count()
#
#                    while index < count:
#                        for point in points[index:(index + 5000)]:
#                            row = []
#
#                            row.append(point.source)
#                            row.append(calendar.timegm(point.created.utctimetuple()))
#                            row.append(point.created.isoformat())
#
#                            properties = point.fetch_properties()
#
#                            for column in SECONDARY_FIELDS[secondary_identifier]:
#                                if column in properties:
#                                    row.append(properties[column])
#                                else:
#                                    row.append('')
#
#                            writer.writerow(row)
#
#                        index += 5000
#
#            export_file.write(secondary_filename, secondary_filename.split('/')[-1])
#
#    return filename
<commit_msg>Code cleanup<commit_after># pylint: disable=line-too-long, no-member

import datetime
import json
import time

from django.utils import timezone

def extract_secondary_identifier(properties):
    if 'reading-type' in properties:
        return properties['reading-type']

    return None

def generator_name(identifier): # pylint: disable=unused-argument
    return 'Google Fit'

def compile_frequency_visualization(identifier, points, folder): # pylint: disable=unused-argument
    now = timezone.now()

    now = now.replace(second=0, microsecond=0)

    remainder = now.minute % 10

    now = now.replace(minute=(now.minute - remainder))

    start = now - datetime.timedelta(days=2)

    points = points.filter(created__lte=now, created__gte=start).order_by('created')

    end = start + datetime.timedelta(seconds=600)
    point_index = 0
    point_count = points.count()

    point = None

    if point_count > 0:
        point = points[point_index]

    timestamp_counts = {}

    keys = []

    while start < now:
        timestamp = str(time.mktime(start.timetuple()))

        keys.append(timestamp)

        timestamp_counts[timestamp] = 0

        while point is not None and point.created < end and point_index < (point_count - 1):
            timestamp_counts[timestamp] += 1

            point_index += 1

            point = points[point_index]

        start = end
        end = start + datetime.timedelta(seconds=600)

    timestamp_counts['keys'] = keys

    with open(folder + '/timestamp-counts.json', 'w') as outfile:
        json.dump(timestamp_counts, outfile, indent=2)

# def compile_report(generator, sources): # pylint: disable=too-many-locals
#    timestamp = arrow.get()
#    filename = tempfile.gettempdir() + '/pdk_export_' + str(timestamp.seconds) + '.' + \
#               str(timestamp.seconds / 1e6) + '.zip'
#
#    with ZipFile(filename, 'w') as export_file:
#        for secondary_identifier in SECONDARY_FIELDS:
#            secondary_filename = tempfile.gettempdir() + '/' + generator + '-' + \
#                                 secondary_identifier + '.txt'
#
#            with open(secondary_filename, 'w') as outfile:
#                writer = csv.writer(outfile, delimiter='\t')
#
#                columns = [
#                    'Source',
#                    'Created Timestamp',
#                    'Created Date',
#                ]
#
#                for column in SECONDARY_FIELDS[secondary_identifier]:
#                    columns.append(column)
#
#                writer.writerow(columns)
#
#                for source in sources:
#                    points = DataPoint.objects.filter(source=source, generator_identifier=generator, secondary_identifier=secondary_identifier).order_by('source', 'created') # pylint: disable=no-member,line-too-long
#
#                    index = 0
#                    count = points.count()
#
#                    while index < count:
#                        for point in points[index:(index + 5000)]:
#                            row = []
#
#                            row.append(point.source)
#                            row.append(calendar.timegm(point.created.utctimetuple()))
#                            row.append(point.created.isoformat())
#
#                            properties = point.fetch_properties()
#
#                            for column in SECONDARY_FIELDS[secondary_identifier]:
#                                if column in properties:
#                                    row.append(properties[column])
#                                else:
#                                    row.append('')
#
#                            writer.writerow(row)
#
#                        index += 5000
#
#            export_file.write(secondary_filename, secondary_filename.split('/')[-1])
#
#    return filename
<|endoftext|>


<commit_before># Copyright 2018 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

import requests.exceptions

from subiquitycore.async_helpers import (
    schedule_task,
    )
from subiquitycore.controller import (
    BaseController,
    Skip,
    )

from subiquity.models.snaplist import SnapSelection
from subiquity.ui.views.snaplist import SnapListView

log = logging.getLogger('subiquity.controllers.snaplist')

class SnapdSnapInfoLoader:

    def __init__(self, model, snapd, store_section):
        self.model = model
        self.store_section = store_section

        self.main_task = None
        self.snap_list_fetched = False
        self.failed = False

        self.snapd = snapd
        self.pending_info_snaps = []
        self.tasks = {}  # {snap:task}

    def start(self):
        log.debug("loading list of snaps")
        self.main_task = schedule_task(self._start())

    async def _start(self):
        task = self.tasks[None] = schedule_task(self._load_list())
        await task
        self.pending_snaps = self.model.get_snap_list()
        log.debug("fetched list of %s snaps", len(self.pending_snaps))
        while self.pending_snaps:
            snap = self.pending_snaps.pop(0)
            task = self.tasks[snap] = schedule_task(
                self._fetch_info_for_snap(snap))
            await task

    async def _load_list(self):
        try:
            result = await self.snapd.get(
                'v2/find', section=self.store_section)
        except requests.exceptions.RequestException:
            log.exception("loading list of snaps failed")
            self.failed = True
            return
        self.model.load_find_data(result)
        self.snap_list_fetched = True

    def stop(self):
        if self.main_task is not None:
            self.main_task.cancel()

    async def _fetch_info_for_snap(self, snap):
        log.debug('starting fetch for %s', snap.name)
        try:
            data = await self.snapd.get('v2/find', name=snap.name)
        except requests.exceptions.RequestException:
            log.exception("loading snap info failed")
            # XXX something better here?
            return
        log.debug('got data for %s', snap.name)
        self.model.load_info_data(data)

    def get_snap_list_task(self):
        return self.tasks[None]

    def get_snap_info_task(self, snap):
        if snap not in self.tasks:
            if snap in self.pending_snaps:
                self.pending_snaps.remove(snap)
            self.tasks[snap] = schedule_task(self._fetch_info_for_snap(snap))
        return self.tasks[snap]

class SnapListController(BaseController):

    model_name = "snaplist"
    signals = [
        ('snapd-network-change', 'snapd_network_changed'),
    ]

    def _make_loader(self):
        return SnapdSnapInfoLoader(
            self.model, self.app.snapd,
            self.opts.snap_section)

    def __init__(self, app):
        super().__init__(app)
        self.loader = self._make_loader()

    def snapd_network_changed(self):
        # If the loader managed to load the list of snaps, the
        # network must basically be working.
        if self.loader.snap_list_fetched:
            return
        else:
            self.loader.stop()
        self.loader = self._make_loader()
        self.loader.start()

    def start_ui(self):
        if self.loader.failed or not self.app.base_model.network.has_network:
            # If loading snaps failed or the network is disabled, skip the
            # screen.
            self.signal.emit_signal("installprogress:snap-config-done")
            raise Skip()
        if 'snaps' in self.answers:
            to_install = {}
            for snap_name, selection in self.answers['snaps'].items():
                to_install[snap_name] = SnapSelection(**selection)
            self.done(to_install)
            return
        self.ui.set_body(SnapListView(self.model, self))

    def get_snap_list_task(self):
        return self.loader.get_snap_list_task()

    def get_snap_info_task(self, snap):
        return self.loader.get_snap_info_task(snap)

    def done(self, snaps_to_install):
        log.debug(
            "SnapListController.done next_screen snaps_to_install=%s",
            snaps_to_install)
        self.model.set_installed_list(snaps_to_install)
        self.configured()
        self.app.next_screen()

    def cancel(self, sender=None):
        self.app.prev_screen()
<commit_msg>missed a signal in #613<commit_after># Copyright 2018 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import logging

import requests.exceptions

from subiquitycore.async_helpers import (
    schedule_task,
    )
from subiquitycore.controller import (
    BaseController,
    Skip,
    )

from subiquity.models.snaplist import SnapSelection
from subiquity.ui.views.snaplist import SnapListView

log = logging.getLogger('subiquity.controllers.snaplist')

class SnapdSnapInfoLoader:

    def __init__(self, model, snapd, store_section):
        self.model = model
        self.store_section = store_section

        self.main_task = None
        self.snap_list_fetched = False
        self.failed = False

        self.snapd = snapd
        self.pending_info_snaps = []
        self.tasks = {}  # {snap:task}

    def start(self):
        log.debug("loading list of snaps")
        self.main_task = schedule_task(self._start())

    async def _start(self):
        task = self.tasks[None] = schedule_task(self._load_list())
        await task
        self.pending_snaps = self.model.get_snap_list()
        log.debug("fetched list of %s snaps", len(self.pending_snaps))
        while self.pending_snaps:
            snap = self.pending_snaps.pop(0)
            task = self.tasks[snap] = schedule_task(
                self._fetch_info_for_snap(snap))
            await task

    async def _load_list(self):
        try:
            result = await self.snapd.get(
                'v2/find', section=self.store_section)
        except requests.exceptions.RequestException:
            log.exception("loading list of snaps failed")
            self.failed = True
            return
        self.model.load_find_data(result)
        self.snap_list_fetched = True

    def stop(self):
        if self.main_task is not None:
            self.main_task.cancel()

    async def _fetch_info_for_snap(self, snap):
        log.debug('starting fetch for %s', snap.name)
        try:
            data = await self.snapd.get('v2/find', name=snap.name)
        except requests.exceptions.RequestException:
            log.exception("loading snap info failed")
            # XXX something better here?
            return
        log.debug('got data for %s', snap.name)
        self.model.load_info_data(data)

    def get_snap_list_task(self):
        return self.tasks[None]

    def get_snap_info_task(self, snap):
        if snap not in self.tasks:
            if snap in self.pending_snaps:
                self.pending_snaps.remove(snap)
            self.tasks[snap] = schedule_task(self._fetch_info_for_snap(snap))
        return self.tasks[snap]

class SnapListController(BaseController):

    model_name = "snaplist"
    signals = [
        ('snapd-network-change', 'snapd_network_changed'),
    ]

    def _make_loader(self):
        return SnapdSnapInfoLoader(
            self.model, self.app.snapd,
            self.opts.snap_section)

    def __init__(self, app):
        super().__init__(app)
        self.loader = self._make_loader()

    def snapd_network_changed(self):
        # If the loader managed to load the list of snaps, the
        # network must basically be working.
        if self.loader.snap_list_fetched:
            return
        else:
            self.loader.stop()
        self.loader = self._make_loader()
        self.loader.start()

    def start_ui(self):
        if self.loader.failed or not self.app.base_model.network.has_network:
            # If loading snaps failed or the network is disabled, skip the
            # screen.
            self.configured()
            raise Skip()
        if 'snaps' in self.answers:
            to_install = {}
            for snap_name, selection in self.answers['snaps'].items():
                to_install[snap_name] = SnapSelection(**selection)
            self.done(to_install)
            return
        self.ui.set_body(SnapListView(self.model, self))

    def get_snap_list_task(self):
        return self.loader.get_snap_list_task()

    def get_snap_info_task(self, snap):
        return self.loader.get_snap_info_task(snap)

    def done(self, snaps_to_install):
        log.debug(
            "SnapListController.done next_screen snaps_to_install=%s",
            snaps_to_install)
        self.model.set_installed_list(snaps_to_install)
        self.configured()
        self.app.next_screen()

    def cancel(self, sender=None):
        self.app.prev_screen()
<|endoftext|>


<commit_before># coding=utf-8
from Framework.Controller import Controller
from Database.Controllers.Disciplina import Disciplina as BDDisciplina
from Models.Disciplina.RespostaListar import RespostaListar
from Database.Models.Disciplina import Disciplina as ModelDisciplina

class Disciplina(Controller):

	def Listar(self,pedido_listar):
		return RespostaListar(BDDisciplina().pegarDisciplinas("WHERE id_departamento = %s AND nome LIKE %s LIMIT %s OFFSET %s",(str(pedido_listar.getIdDepartamento()),"%"+pedido_listar.getNome().replace(' ','%')+"%",str(pedido_listar.getQuantidade()),(str(pedido_listar.getQuantidade()*pedido_listar.getPagina())))))

	def Ver(self, pedido_ver):
		return RespostaVer(BDDisciplina().pegarDiscina("WHERE id = %s ", (pedido_ver.getId())))

	def Cadastrar(self,pedido_cadastrar):
		disciplina = ModelDisciplin()
		disciplina.setNome(pedido_cadastrar.getNome())
		disciplina.setCodigo(pedido_cadastrar.getCodigo())
		return RespostaCadastrar(BDDisciplina().inserirDisciplina(disciplina))

	def Editar(self,pedido_editar):
		disciplina = BDDisciplina().pegarDisciplina("WHERE id = %s ", (pedido_editar.getId()))
		disciplina.setNome(pedido_editar.getNome())
		disciplina.setCodigo(pedido_editar.getCodigo())
		BDDisciplina().alterarDisciplina(disciplina)
		return RespostaEditar("Disciplina Editado com sucesso!")

	def Deletar(self,pedido_deletar):
		disciplina = BDDisciplina().pegarDisciplina("WHERE id = %s ", (pedido_deletar.getId()))
		BDDisciplina().removerDisciplina(disciplina)
		return RespostaDeletar("Disciplina Removido com sucesso!")
<commit_msg>consertando controllers disciplina<commit_after># coding=utf-8
from Framework.Controller import Controller
from Database.Controllers.Disciplina import Disciplina as BDDisciplina
from Models.Disciplina.RespostaListar import RespostaListar
from Database.Models.Disciplina import Disciplina as ModelDisciplina

class Disciplina(Controller):

	def Listar(self,pedido_listar):
		return RespostaListar(BDDisciplina().pegarDisciplinas("WHERE id_departamento = %s AND nome LIKE %s LIMIT %s OFFSET %s",(str(pedido_listar.getIdDepartamento()),"%"+pedido_listar.getNome().replace(' ','%')+"%",str(pedido_listar.getQuantidade()),(str(pedido_listar.getQuantidade()*pedido_listar.getPagina())))))

	def Ver(self, pedido_ver):
		return RespostaVer(BDDisciplina().pegarDiscina("WHERE id = %s ", (str(pedido_ver.getId()),)))

	def Cadastrar(self,pedido_cadastrar):
		disciplina = ModelDisciplin()
		disciplina.setNome(pedido_cadastrar.getNome())
		disciplina.setCodigo(pedido_cadastrar.getCodigo())
		return RespostaCadastrar(BDDisciplina().inserirDisciplina(disciplina))

	def Editar(self,pedido_editar):
		disciplina = BDDisciplina().pegarDisciplina("WHERE id = %s ", (str(pedido_editar.getId()),))
		disciplina.setNome(pedido_editar.getNome())
		disciplina.setCodigo(pedido_editar.getCodigo())
		BDDisciplina().alterarDisciplina(disciplina)
		return RespostaEditar("Disciplina Editado com sucesso!")

	def Deletar(self,pedido_deletar):
		disciplina = BDDisciplina().pegarDisciplina("WHERE id = %s ", (str(pedido_deletar.getId()),))
		BDDisciplina().removerDisciplina(disciplina)
		return RespostaDeletar("Disciplina Removido com sucesso!")
<|endoftext|>


<commit_before>import os, sys, re
import argparse
import numpy as np
import h5py
import skimage.io
import skimage.transform
from sklearn.preprocessing import LabelBinarizer
from keras.applications import InceptionV3
from keras.applications.imagenet_utils import preprocess_input
from keras.preprocessing import image
from keras.models import Model
from keras.layers import Dropout, Flatten, Input

# Command line arguments: image in-path, feature out-path, extension for output
parser = argparse.ArgumentParser(description='Perform InceptionV3-ImageNet feature extraction on images.')

parser.add_argument(nargs='?', type=str, dest='img_path',
                    default='./images', action='store')
parser.add_argument(nargs='?', type=str, dest='out_path',
                    default='./output/features', action='store')
parser.add_argument(nargs='?', type=str, dest='ext',
                    default='hdf5', action='store')
parser.add_argument(nargs='?', type=bool, dest='compressed',
					default=True, action='store')
args = parser.parse_args()

def extract_multi():

    # Load dataset (any image-based dataset)
    matches = [(re.match(r'^(([a-zA-Z]+)\d+\.png)', fname), path)
    		for path, dirs, files in os.walk(''+str(args.img_path))
    		for fname in files]
    patches = [skimage.transform.resize( # resize image to (256, 256)
    	skimage.io.imread(os.path.join(path, match.group(1))), # open each image
	    (256, 256)) for match, path in matches if match]

    # Preprocess for InceptionV3
    patches = preprocess_input(np.array(patches))

    # Construct model (using ImageNet weights)
    inceptionV3 = InceptionV3(weights = "imagenet", include_top = False,
        input_shape = patches[0].shape)

    # Isolate pre-softmax outputs
    x = inceptionV3.output

    # Experimental - flatten to 2d for CSV
    if args.ext == "csv":
		x = Flatten()(x)
		x = Dropout(0.5)(x)

	# Construct extractor model
    extractor = Model(inputs=[inceptionV3.input], outputs=[x])

    # Extract features with Model.predict()
    features = extractor.predict(x=patches, batch_size=2)

    # TODO: get rid of zero-padding
    # TODO: concatenate individual patches to individual 1d arrays?

    return features

# HARD CODED TEST STUFF BELOW
features = extract_multi()
print(features.shape)
extension = str(args.ext)
if extension == "hdf5":
	# (Recommended, default) save --> .hdf
	f = h5py.File(""+str(args.out_path)+".hdf5", "w")
	hdf = f.create_dataset(name=str(args.out_path), data=features)
elif extension == "npy": # god please don't actually do this
	fname = "" + str(args.out_path)
	np.save(file=fname, allow_pickle=True, arr=features)
else:
	# TODO: add Flatten()(features) layer --> reduce to 2d array
	if args.compressed == True:
		extension += ".gz"
	fname = "" + str(args.out_path) + "." + extension
	np.savetxt(fname=fname, X=features, fmt='%1.6f') # %fmt is broken?
# TODO: npz for the optional list of concat. 1d arrays
# END HARD CODED TEST STUFF

import gc; gc.collect()<commit_msg>whitespace<commit_after>import os, sys, re
import argparse
import numpy as np
import h5py
import skimage.io
import skimage.transform
from sklearn.preprocessing import LabelBinarizer
from keras.applications import InceptionV3
from keras.applications.imagenet_utils import preprocess_input
from keras.preprocessing import image
from keras.models import Model
from keras.layers import Dropout, Flatten, Input

# Command line arguments: image in-path, feature out-path, extension for output
parser = argparse.ArgumentParser(description='Perform InceptionV3-ImageNet feature extraction on images.')

parser.add_argument(nargs='?', type=str, dest='img_path',
                    default='./images', action='store')
parser.add_argument(nargs='?', type=str, dest='out_path',
                    default='./output/features', action='store')
parser.add_argument(nargs='?', type=str, dest='ext',
                    default='hdf5', action='store')
parser.add_argument(nargs='?', type=bool, dest='compressed',
					default=True, action='store')
args = parser.parse_args()

def extract_multi():

    # Load dataset (any image-based dataset)
    matches = [(re.match(r'^(([a-zA-Z]+)\d+\.png)', fname), path)
    		for path, dirs, files in os.walk(''+str(args.img_path))
    		for fname in files]
    patches = [skimage.transform.resize( # resize image to (256, 256)
    	skimage.io.imread(os.path.join(path, match.group(1))), # open each image
	    (256, 256)) for match, path in matches if match]

    # Preprocess for InceptionV3
    patches = preprocess_input(np.array(patches))

    # Construct model (using ImageNet weights)
    inceptionV3 = InceptionV3(weights = "imagenet", include_top = False,
        input_shape = patches[0].shape)

    # Isolate pre-softmax outputs
    x = inceptionV3.output

    # Experimental - flatten to 2d for CSV
    if args.ext == "csv":
		x = Flatten()(x)
		x = Dropout(0.5)(x)

	# Construct extractor model
    extractor = Model(inputs=[inceptionV3.input], outputs=[x])

    # Extract features with Model.predict()
    features = extractor.predict(x=patches, batch_size=2)

    # TODO: get rid of zero-padding
    # TODO: concatenate individual patches to individual 1d arrays?

    return features

# HARD CODED TEST STUFF BELOW
features = extract_multi()
print(features.shape)
extension = str(args.ext)
if extension == "hdf5":
	# (Recommended, default) save --> .hdf
	f = h5py.File(""+str(args.out_path)+".hdf5", "w")
	hdf = f.create_dataset(name=str(args.out_path), data=features)
elif extension == "npy": # god please don't actually do this
	fname = "" + str(args.out_path)
	np.save(file=fname, allow_pickle=True, arr=features)
else:
	# TODO: add Flatten()(features) layer --> reduce to 2d array
	if args.compressed == True:
		extension += ".gz"
	fname = "" + str(args.out_path) + "." + extension
	np.savetxt(fname=fname, X=features, fmt='%1.6f') # %fmt is broken?
# TODO: npz for the optional list of concat. 1d arrays
# END HARD CODED TEST STUFF

import gc; gc.collect()<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

#: major, minor, patch version for Spack, in a tuple
spack_version_info = (0, 15, 4)

#: String containing Spack version joined with .'s
spack_version = '.'.join(str(v) for v in spack_version_info)

__all__ = ['spack_version_info', 'spack_version']
<commit_msg>bump version number to 0.16.0<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

#: major, minor, patch version for Spack, in a tuple
spack_version_info = (0, 16, 0)

#: String containing Spack version joined with .'s
spack_version = '.'.join(str(v) for v in spack_version_info)

__all__ = ['spack_version_info', 'spack_version']
<|endoftext|>


<commit_before>"""
Helpers for working with traitlet-based classes.

In particular, these helpers help with instantiation of HasTraits object
hierarchies from dictionaries and lists of trait names.
"""
import re
import warnings
import six

import pandas as pd
import numpy as np

try:
    from pandas.api.types import infer_dtype
except ImportError: # Pandas before 0.20.0
    from pandas.lib import infer_dtype

import traitlets as T

from ..utils import infer_vegalite_type

TYPECODE_MAP = {'ordinal': 'O',
                'nominal': 'N',
                'quantitative': 'Q',
                'temporal': 'T'}

INV_TYPECODE_MAP = {v: k for k, v in TYPECODE_MAP.items()}

def infer_keywords(cls, *args, **kwargs):
    """Utility to initialize a HasTraits object from args and kwargs

    Arguments are converted to keyword arguments by inferring the keyword
    from their type.
    Keyword arguments are converted to the correct Instance class
    if required.
    """
    # Import here to avoid circular imports
    from .schema import jstraitlets as jst

    # TODO: can we make this more efficient & less fragile?
    def is_simple_union(trait):
        """Return True if trait is, e.g. Union(Class, List(Class))"""
        return (isinstance(trait, T.Union) and
                len(trait.trait_types) == 2 and
                isinstance(trait.trait_types[0], T.Instance) and
                isinstance(trait.trait_types[1], T.List) and
                trait.trait_types[0].klass == trait.trait_types[1]._trait.klass)

    # TODO: make this less brittle
    def get_class(trait):
        if isinstance(trait, T.Instance):
            return trait.klass
        elif is_simple_union(trait):
            return trait.trait_types[0].klass
        else:
            return None

    traits = cls.class_traits()
    classes = {n: get_class(t) for n, t in traits.items()}

    # Turn all keyword arguments to the appropriate class
    for name, arg in kwargs.items():
        Trait = classes.get(name, None)
        if Trait is not None and not isinstance(arg, Trait):
            try:
                kwargs[name] = Trait(arg)
            except (TypeError, T.TraitError):
                pass  # errors will handled by traitlets below

    # find forward/backward mapping among unique classes
    name_to_trait = {}
    while classes:
        name, trait = classes.popitem()
        if trait is jst.undefined:
            continue
        if trait not in set.union(set(classes.values()),
                                  set(name_to_trait.values())):
            name_to_trait[name] = trait
    trait_to_name = {t: n for n, t in name_to_trait.items()}

    # Update all arguments
    for arg in args:
        # find highest-level class in the method resolution order that appears
        # in the dict:
        names = (trait_to_name.get(cls, None) for cls in arg.__class__.__mro__)
        name = next((n for n in names if n), None)
        if name is None:
            raise ValueError("{0}: Unable to infer argument name for {1}"
                             "".format(cls, arg))
        elif name in kwargs:
            raise ValueError("{0}: {1} specified both by arg and kwarg"
                             "".format(cls, name))

        else:
            kwargs[name] = arg
    return kwargs

def update_traits(obj, **kwargs):
    """Convenience routine to call set_trait() with a dictionary of values"""
    for key, val in kwargs.items():
        obj.set_trait(key, val)
    return obj

def update_inferred_traits(obj, *args, **kwargs):
    """Infer types from positional arguments and update traits of obj"""
    kwargs = infer_keywords(obj, *args, **kwargs)
    return update_traits(obj, **kwargs)

def update_subtraits(obj, attrs, *args, **kwargs):
    """Recursively update sub-traits without overwriting other traits"""
    # Import here to avoid circular imports
    from .schema import jstraitlets as jst

    if not (args or kwargs):
        return obj
    if isinstance(attrs, six.string_types):
        attrs = (attrs,)
    if len(attrs) == 0:
        update_inferred_traits(obj, *args, **kwargs)
    else:
        attr = attrs[0]
        trait = getattr(obj, attr)  # error here if attr is not present
        if trait is jst.undefined:
            trait = obj.traits()[attr].klass()
        setattr(obj, attr, update_subtraits(trait, attrs[1:], *args, **kwargs))
    return obj

def construct_shorthand(field=None, aggregate=None, type=None):
    """Construct a shorthand representation.

    See also: parse_shorthand"""
    # Import here to avoid circular imports
    from .schema import jstraitlets as jst

    if field is jst.undefined or field is None:
        return ''

    sh = field

    if aggregate is not jst.undefined and aggregate is not None:
        sh = '{0}({1})'.format(aggregate, sh)

    if type is not jst.undefined and type is not None:
        type = TYPECODE_MAP.get(type, type)
        if type not in INV_TYPECODE_MAP:
            raise ValueError('Unrecognized Type: {0}'.format(type))
        sh = '{0}:{1}'.format(sh, type)

    return sh

def parse_shorthand(shorthand):
    """
    Parse the shorthand expression for aggregation, field, and type.

    These are of the form:

    - "col_name"
    - "col_name:O"
    - "average(col_name)"
    - "average(col_name):O"

    Parameters
    ----------
    shorthand: str
        Shorthand string

    Returns
    -------
    D : dict
        Dictionary containing the field, aggregate, and typecode
    """
    if not shorthand:
        return {}

    # Must import this here to avoid circular imports
    from .schema import AggregateOp
    valid_aggregates = AggregateOp().values
    valid_typecodes = list(TYPECODE_MAP) + list(INV_TYPECODE_MAP)

    # build regular expressions
    units = dict(field='(?P<field>.*)',
                 type='(?P<type>{0})'.format('|'.join(valid_typecodes)),
                 aggregate='(?P<aggregate>{0})'.format('|'.join(valid_aggregates)))
    patterns = [r'{field}',
                r'{field}:{type}',
                r'{aggregate}\({field}\)',
                r'{aggregate}\({field}\):{type}']
    regexps = (re.compile('\A' + p.format(**units) + '\Z', re.DOTALL)
               for p in patterns[::-1])

    # find matches depending on valid fields passed
    match = next(exp.match(shorthand).groupdict() for exp in regexps
                 if exp.match(shorthand))

    # Use short form of the type expression
    typ = match.get('type', None)
    if typ:
        match['type'] = INV_TYPECODE_MAP.get(typ, typ)
    return match
<commit_msg>refactor traitlet_utils<commit_after>"""
Helpers for working with traitlet-based classes.

In particular, these helpers help with instantiation of HasTraits object
hierarchies from dictionaries and lists of trait names.
"""
import re
import warnings
import six

import pandas as pd
import numpy as np

try:
    from pandas.api.types import infer_dtype
except ImportError: # Pandas before 0.20.0
    from pandas.lib import infer_dtype

import traitlets as T

from ..utils import infer_vegalite_type

TYPECODE_MAP = {'ordinal': 'O',
                'nominal': 'N',
                'quantitative': 'Q',
                'temporal': 'T'}

INV_TYPECODE_MAP = {v: k for k, v in TYPECODE_MAP.items()}

def channel_type_dict(cls):
    """Create a dict of trait names to channel type"""

    traits = cls.class_traits()

    def is_union_of_trait_and_list(trait):
        return (isinstance(trait, T.Union) and
                len(trait.trait_types) == 2 and
                isinstance(trait.trait_types[0], T.Instance) and
                isinstance(trait.trait_types[1], T.List) and
                trait.trait_types[0].klass == trait.trait_types[1]._trait.klass)

    def get_class(trait):
        if isinstance(trait, T.Instance):
            return trait.klass
        elif is_union_of_trait_and_list(trait):
            return trait.trait_types[0].klass
        else:
            return None

    classes = {n: get_class(t) for n, t in traits.items()}
    return {n:t for n, t in classes.items() if t is not None}

def infer_keywords(cls, *args, **kwargs):
    """Utility to initialize a HasTraits object from args and kwargs

    Arguments are converted to keyword arguments by inferring the keyword
    from their type.
    Keyword arguments are converted to the correct Instance class
    if required.
    """
    # Import here to avoid circular imports
    from .schema import jstraitlets as jst

    # For all traits, extract the channel type.
    traits = cls.class_traits()
    trait_classes = channel_type_dict(cls)

    # Turn all keyword arguments to the appropriate class
    for name, arg in kwargs.items():
        Trait = trait_classes.get(name, None)
        if Trait is not None and not isinstance(arg, Trait):
            try:
                kwargs[name] = Trait(arg)
            except (TypeError, T.TraitError):
                pass  # errors will handled by traitlets below

    # find forward/backward mapping among unique classes
    name_to_trait = {}
    while trait_classes:
        name, trait = trait_classes.popitem()
        if trait is jst.undefined:
            continue
        if trait not in set.union(set(trait_classes.values()),
                                  set(name_to_trait.values())):
            name_to_trait[name] = trait
    trait_to_name = {t: n for n, t in name_to_trait.items()}

    # Update all arguments
    for arg in args:
        # find highest-level class in the method resolution order that appears
        # in the dict:
        names = (trait_to_name.get(cls, None) for cls in arg.__class__.__mro__)
        name = next((n for n in names if n), None)
        if name is None:
            raise ValueError("{0}: Unable to infer argument name for {1}"
                             "".format(cls, arg))
        elif name in kwargs:
            raise ValueError("{0}: {1} specified both by arg and kwarg"
                             "".format(cls, name))

        else:
            kwargs[name] = arg
    return kwargs

def update_traits(obj, **kwargs):
    """Convenience routine to call set_trait() with a dictionary of values"""
    for key, val in kwargs.items():
        obj.set_trait(key, val)
    return obj

def update_inferred_traits(obj, *args, **kwargs):
    """Infer types from positional arguments and update traits of obj"""
    kwargs = infer_keywords(obj, *args, **kwargs)
    return update_traits(obj, **kwargs)

def update_subtraits(obj, attrs, *args, **kwargs):
    """Recursively update sub-traits without overwriting other traits"""
    # Import here to avoid circular imports
    from .schema import jstraitlets as jst

    if not (args or kwargs):
        return obj
    if isinstance(attrs, six.string_types):
        attrs = (attrs,)
    if len(attrs) == 0:
        update_inferred_traits(obj, *args, **kwargs)
    else:
        attr = attrs[0]
        trait = getattr(obj, attr)  # error here if attr is not present
        if trait is jst.undefined:
            trait = obj.traits()[attr].klass()
        setattr(obj, attr, update_subtraits(trait, attrs[1:], *args, **kwargs))
    return obj

def construct_shorthand(field=None, aggregate=None, type=None):
    """Construct a shorthand representation.

    See also: parse_shorthand"""
    # Import here to avoid circular imports
    from .schema import jstraitlets as jst

    if field is jst.undefined or field is None:
        return ''

    sh = field

    if aggregate is not jst.undefined and aggregate is not None:
        sh = '{0}({1})'.format(aggregate, sh)

    if type is not jst.undefined and type is not None:
        type = TYPECODE_MAP.get(type, type)
        if type not in INV_TYPECODE_MAP:
            raise ValueError('Unrecognized Type: {0}'.format(type))
        sh = '{0}:{1}'.format(sh, type)

    return sh

def parse_shorthand(shorthand):
    """
    Parse the shorthand expression for aggregation, field, and type.

    These are of the form:

    - "col_name"
    - "col_name:O"
    - "average(col_name)"
    - "average(col_name):O"

    Parameters
    ----------
    shorthand: str
        Shorthand string

    Returns
    -------
    D : dict
        Dictionary containing the field, aggregate, and typecode
    """
    if not shorthand:
        return {}

    # Must import this here to avoid circular imports
    from .schema import AggregateOp
    valid_aggregates = AggregateOp().values
    valid_typecodes = list(TYPECODE_MAP) + list(INV_TYPECODE_MAP)

    # build regular expressions
    units = dict(field='(?P<field>.*)',
                 type='(?P<type>{0})'.format('|'.join(valid_typecodes)),
                 aggregate='(?P<aggregate>{0})'.format('|'.join(valid_aggregates)))
    patterns = [r'{field}',
                r'{field}:{type}',
                r'{aggregate}\({field}\)',
                r'{aggregate}\({field}\):{type}']
    regexps = (re.compile('\A' + p.format(**units) + '\Z', re.DOTALL)
               for p in patterns[::-1])

    # find matches depending on valid fields passed
    match = next(exp.match(shorthand).groupdict() for exp in regexps
                 if exp.match(shorthand))

    # Use short form of the type expression
    typ = match.get('type', None)
    if typ:
        match['type'] = INV_TYPECODE_MAP.get(typ, typ)
    return match
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""

Pyle allows you to use Python as a replacement for command line tools such
as `sed` or `perl`. It reads its standard input and evaluates each line with
the expression specified, outputting the results on standard out.
Optionally, it can operate on a list of filenames instead in which case each
file is read and processed in order. The variables `line`, representing the
current input line being processed, and `words`, representing the current
line split by whitespace, are available to the expression. In addition the
`re` module is available.

"""

import argparse
import cStringIO as StringIO
import re
import sys
import traceback

STANDARD_MODULES = ['re']

def pyle_evaluate(command=None, modules=None, inplace=False, files=None):
    eval_globals = {}

    modules = STANDARD_MODULES + (modules or [])
    for module_arg in modules:
        for module in module_arg.strip().split(","):
            module = module.strip()
            if module:
                eval_globals[module] = __import__(module)

    if not command:
        # Default 'do nothing' program
        command = 'line'

    files = files or ['-']
    for file in files:
        if file == '-':
            file = sys.stdin

        out_buf = sys.stdout if not inplace else StringIO.StringIO()

        with (open(file, 'rb') if not hasattr(file, 'read') else file) as in_file:
            for line in in_file.readlines():
                line = line[:-1]
                words = [word.strip() for word in re.split(r'\s+', line) if word]
                eval_locals = {'line': line, 'words': words}
                try:
                    out_line = eval(command, eval_globals, eval_locals)
                except Exception:
                    traceback.print_exc(None, sys.stderr)
                else:
                    # If the result is something list-like or iterable, output each item space separated.
                    if out_line is not None and not isinstance(out_line, str):
                        try:
                            out_line = ' '.join(out_line)
                        except:
                            # Guess it wasn't a list after all.
                            out_line = unicode(out_line)

                    out_line = out_line or u''
                    out_buf.write(out_line + u'\n')
        if inplace:
            with open(file, 'wb') as out_file:
                out_file.write(out_buf.getvalue())
            out_buf.close()

def pyle(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("-m", "--modules", dest="modules", action='append',
        help="import MODULE before evaluation. May be specified more than once.")
    parser.add_argument("-i", "--inplace", dest="inplace", action='store_true', default=False,
        help="edit files in place. When used with file name arguments, the files will be replaced by the output of the evaluation")
    parser.add_argument("-e", "--expression", dest="expression",
        help="the statement to evaluate on each line")
    parser.add_argument('files', nargs='*',
        help="files to read as input. If used with --inplace, the files will be replaced with the output")

    args = parser.parse_args() if not argv else parser.parse_args(argv)

    pyle_evaluate(args.expression, args.modules, args.inplace, args.files)

if __name__ == '__main__':
    pyle()
<commit_msg>Make line numbers and filenames available to expressions.<commit_after>#!/usr/bin/env python

"""

Pyle allows you to use Python as a replacement for command line tools such
as `sed` or `perl`. It reads its standard input and evaluates each line with
the expression specified, outputting the results on standard out.
Optionally, it can operate on a list of filenames instead in which case each
file is read and processed in order. The variables `line`, representing the
current input line being processed, `words`, representing the current
line split by whitespace, `num`, the 0 index line number in the current
file, and `filename`, the name of the current file, are available to the
expression. In addition the `re` module is available.

"""

import argparse
import cStringIO as StringIO
import re
import sys
import traceback

STANDARD_MODULES = ['re']

def pyle_evaluate(command=None, modules=None, inplace=False, files=None):
    eval_globals = {}

    modules = STANDARD_MODULES + (modules or [])
    for module_arg in modules:
        for module in module_arg.strip().split(","):
            module = module.strip()
            if module:
                eval_globals[module] = __import__(module)

    if not command:
        # Default 'do nothing' program
        command = 'line'

    files = files or ['-']
    for file in files:
        if file == '-':
            file = sys.stdin

        out_buf = sys.stdout if not inplace else StringIO.StringIO()

        with (open(file, 'rb') if not hasattr(file, 'read') else file) as in_file:
            for num, line in enumerate(in_file.readlines()):
                line = line[:-1]
                words = [word.strip() for word in re.split(r'\s+', line) if word]
                eval_locals = {'line': line, 'words': words, 'filename': in_file.name, 'num': num}
                try:
                    out_line = eval(command, eval_globals, eval_locals)
                except Exception:
                    traceback.print_exc(None, sys.stderr)
                else:
                    # If the result is something list-like or iterable, output each item space separated.
                    if out_line is not None and not isinstance(out_line, str):
                        try:
                            out_line = ' '.join(out_line)
                        except:
                            # Guess it wasn't a list after all.
                            out_line = unicode(out_line)

                    out_line = out_line or u''
                    out_buf.write(out_line + u'\n')
        if inplace:
            with open(file, 'wb') as out_file:
                out_file.write(out_buf.getvalue())
            out_buf.close()

def pyle(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("-m", "--modules", dest="modules", action='append',
        help="import MODULE before evaluation. May be specified more than once.")
    parser.add_argument("-i", "--inplace", dest="inplace", action='store_true', default=False,
        help="edit files in place. When used with file name arguments, the files will be replaced by the output of the evaluation")
    parser.add_argument("-e", "--expression", dest="expression",
        help="the statement to evaluate on each line")
    parser.add_argument('files', nargs='*',
        help="files to read as input. If used with --inplace, the files will be replaced with the output")

    args = parser.parse_args() if not argv else parser.parse_args(argv)

    pyle_evaluate(args.expression, args.modules, args.inplace, args.files)

if __name__ == '__main__':
    pyle()
<|endoftext|>