<commit_before>import inspect
import sys

import MySQLdb
import pymw
import pymw.interfaces

import artgraph.plugins
import artgraph.relationship

from artgraph.node import NodeTypes
from artgraph.node import Node

class Miner(object):
    nodes = []
    master = None
    task_queue = []
    db = None
    plugins = {}

    def __init__(self, debug=False):
        mwinterface = pymw.interfaces.GenericInterface()
        self.master = pymw.PyMW_Master(mwinterface, delete_files=not debug)
        self.db = MySQLdb.connect(read_default_file="./my.cnf", read_default_group="client_artistgraph")

        self.load_plugins(debug)

    def load_plugins(self, debug):
        for m in inspect.getmembers(artgraph.plugins, inspect.ismodule):
            for plugin_name, plugin in inspect.getmembers(m[1], lambda (x): inspect.isclass(x) and issubclass(x, artgraph.plugins.Plugin)):
                node_type = plugin.get_target_node_type()

                if debug:
                    sys.stderr.write("Loading %s...\n" % plugin_name)

                if node_type:
                    # Workaround for enum __eq__ failing
                    node_type = str(node_type)

                    if node_type in self.plugins:
                        self.plugins[node_type].append(plugin)
                    else:
                        self.plugins[node_type] = [plugin]

    def mine(self, artist):
        first_node = Node(artist, NodeTypes.ARTIST)
        self.add_node(first_node)
        self.mine_internal(first_node)
        (finished_task, new_relationships) = self.master.get_result()

        while new_relationships:
            for n in new_relationships:
                new_node = n.get_predicate()
                old_nodes = list(x for x in self.nodes if x == new_node)

                if len(old_nodes) == 0:
                    self.add_node(new_node)
                    self.mine_internal(new_node)
                else:
                    new_node.set_id(old_nodes[0].get_id())

                self.add_relationship(n)

            (finished_task, new_relationships) = self.master.get_result()

        self.db.close()

    def mine_internal(self, current_node, level=0, parent=None, relationship=None):
        self.nodes.append(current_node)

        for p in self.plugins[str(current_node.get_type())]:
            plugin = p(current_node)
            self.task_queue.append(self.master.submit_task(plugin.get_nodes, input_data=(plugin,), modules=("artgraph",), data_files=("my.cnf",)))

    def add_node(self, node):
        cursor = self.db.cursor()

        # For some reason, enum to enum __eq__ is not working here
        if str(node.get_type()) == str(NodeTypes.ARTIST):
            cursor.execute("""
            INSERT INTO `artist` (`stageName`)
            VALUES  (%s)
            """, (node.get_title()))
            node.set_id(self.db.insert_id())

        cursor.close()
        self.db.commit()

    def add_relationship(self, relationship):
        cursor = self.db.cursor()

        if relationship.__class__ == artgraph.relationship.AssociatedActRelationship:
                cursor.execute("""
                INSERT INTO assoc_artist (artistID, assoc_ID)
                VALUES (%s, %s)
                """, (relationship.get_subject().get_id(), relationship.get_predicate().get_id()))

        cursor.close()
        self.db.commit()
<commit_msg>Fixed a bug in recursive calls of the mining method, which uncovers a bug in PyMW<commit_after>import inspect
import sys

import MySQLdb
import pymw
import pymw.interfaces

import artgraph.plugins
import artgraph.relationship

from artgraph.node import NodeTypes
from artgraph.node import Node

class Miner(object):
    nodes = []
    master = None
    task_queue = []
    db = None
    plugins = {}

    def __init__(self, debug=False):
        mwinterface = pymw.interfaces.GenericInterface()
        self.master = pymw.PyMW_Master(mwinterface, delete_files=not debug)
        self.db = MySQLdb.connect(read_default_file="./my.cnf", read_default_group="client_artistgraph")

        self.load_plugins(debug)

    def load_plugins(self, debug):
        for m in inspect.getmembers(artgraph.plugins, inspect.ismodule):
            for plugin_name, plugin in inspect.getmembers(m[1], lambda (x): inspect.isclass(x) and issubclass(x, artgraph.plugins.Plugin)):
                node_type = plugin.get_target_node_type()

                if debug:
                    sys.stderr.write("Loading %s...\n" % plugin_name)

                if node_type:
                    # Workaround for enum __eq__ failing
                    node_type = str(node_type)

                    if node_type in self.plugins:
                        self.plugins[node_type].append(plugin)
                    else:
                        self.plugins[node_type] = [plugin]

    def mine(self, artist):
        first_node = Node(artist, NodeTypes.ARTIST)
        self.add_node(first_node)
        self.mine_internal(first_node)
        (finished_task, new_relationships) = self.master.get_result()

        while finished_task:
            for n in new_relationships:
                new_node = n.get_predicate()
                old_nodes = list(x for x in self.nodes if x == new_node)

                if len(old_nodes) == 0:
                    self.add_node(new_node)
                    self.mine_internal(new_node)
                else:
                    new_node.set_id(old_nodes[0].get_id())

                self.add_relationship(n)

            (finished_task, new_relationships) = self.master.get_result()

        self.db.close()

    def mine_internal(self, current_node, level=0, parent=None, relationship=None):
        self.nodes.append(current_node)

        for p in self.plugins[str(current_node.get_type())]:
            plugin = p(current_node)
            self.task_queue.append(self.master.submit_task(plugin.get_nodes, input_data=(plugin,), modules=("artgraph",), data_files=("my.cnf",)))

    def add_node(self, node):
        cursor = self.db.cursor()

        # For some reason, enum to enum __eq__ is not working here
        if str(node.get_type()) == str(NodeTypes.ARTIST):
            cursor.execute("""
            INSERT INTO `artist` (`stageName`)
            VALUES  (%s)
            """, (node.get_title()))
            node.set_id(self.db.insert_id())

        cursor.close()
        self.db.commit()

    def add_relationship(self, relationship):
        cursor = self.db.cursor()

        if relationship.__class__ == artgraph.relationship.AssociatedActRelationship:
                cursor.execute("""
                INSERT INTO assoc_artist (artistID, assoc_ID)
                VALUES (%s, %s)
                """, (relationship.get_subject().get_id(), relationship.get_predicate().get_id()))

        cursor.close()
        self.db.commit()
<|endoftext|>


<commit_before>"""functions used for syntax analysis"""
from datetime import datetime
from pathlib import Path

import numpy as np
import pandas as pd

HERE = Path(__file__).parent

def date_from_cbin_filename(cbin_filename):
    """extracts date as a string from a .cbin filename
    and converts to a datetime object

    Parameters
    ----------
    cbin_filename : str
        path to a .cbin file

    Returns
    -------
    dt : datetime
        convert from string using strptime function
    """
    name = Path(cbin_filename).name
    splitname = name.split('.')
    animal_id_date = splitname[0]
    animal_id, date, time = animal_id_date.split('_')
    dt = date + '-' + time
    dt = datetime.strptime(dt, '%d%m%y-%H%M')
    return dt

FIELDS_SYNTAX = [
    'date',
    'time',
    'cbin_filename',
    'label',
    'label_plus_one',
    'label_minus_one',
    'label_minus_two',
]

def make_df_trans_probs(vds_list, start_label='S', end_label='E'):
    """make a dataframe that can be used to compute transition probabilities
    between birdsong syllables.

    Takes a list of vak Datasets; function expects each vocalization in the
    dataset to be from .cbin audio files.

    Parameters
    ----------
    vds_list : list
        of vak.Dataset.
    start_label : str
        Label that represents start of a sequence, e.g. a song bout. Default is 'S'.
    end_label : str
        Label that represents end of a sequence. Default is 'E'.

    Returns
    -------
    df : pandas.Dataframe
        with the following columns
            date : datetime
                extracted from audio filename.
                Used to determine transition probabilities on a certain day,
                and for resampling analysis where the dates are shuffled.
            cbin_filename : str
                filename of audio file associated with annotation.
            label : str
                String label for current label of interest.
                For example, label 'a' in transition 'a' -> 'b'.
            label_plus_one : str
                Label that follows current label.
                For exapmle, label 'b' in transition 'a' -> 'b'.
            label_minus_one : str
                Label that preceded current label.
            label_minus_two : str
                Label that preceded label before current label.
    """
    df_dict = {field: [] for field in FIELDS_SYNTAX}
    for vds in vds_list:
        for voc in vds.voc_list:
            cbin_filename = voc.audio_path
            datetime = date_from_cbin_filename(cbin_filename)
            labels = voc.annot.labels.tolist()
            labels = [start_label] + labels + [end_label]
            len_labels = len(labels)
            for ind, label in enumerate(labels):
                if ind == 0:
                    label_minus_one = None
                    label_minus_two = None
                    label_plus_one = labels[ind + 1]
                elif ind == 1:
                    label_minus_one = labels[0]
                    label_minus_two = None
                    label_plus_one = labels[ind + 1]
                elif ind == (len_labels - 1):  # i.e., last index
                    label_minus_one = labels[ind - 1]
                    label_minus_two = labels[ind - 2]
                    label_plus_one = None
                else:
                    label_minus_one = labels[ind - 1]
                    label_minus_two = labels[ind - 2]
                    label_plus_one = labels[ind + 1]
                df_dict['date'].append(datetime.date())
                df_dict['time'].append(datetime.time())
                df_dict['cbin_filename'].append(cbin_filename)
                df_dict['label'].append(label)
                df_dict['label_plus_one'].append(label_plus_one)
                df_dict['label_minus_one'].append(label_minus_one)
                df_dict['label_minus_two'].append(label_minus_two)
    df = pd.DataFrame.from_dict(df_dict)
    return df

def get_trans_prob(df, date, label, label_plus_one):
    """

    Parameters
    ----------
    df
    date
    label
    label_plus_one

    Returns
    -------

    """
    df_date = df[df['date'] == date]
    label_count = len(
        df_date[df_date['label'] == label].index
    )
    trans_count = len(
        df_date[(df_date['label'] == label) & (df_date['label_plus_one'] == label_plus_one)].index
    )
    p = trans_count / label_count
    return p

def make_trans_mat(df, min_p=0.01):
    """

    Parameters
    ----------
    df
    min_p

    Returns
    -------

    """
    labels = df['label'].unique()
    num_labels = labels.shape[0]
    trans_mat = np.zeros((num_labels, num_labels))
    for row, label in enumerate(labels):
        if label == 'E':
            continue
        else:
            for col, label_plus_one in enumerate(labels):
                p = get_trans_prob(df, day, label, label_plus_one)
                if p > min_p:
                    trans_mat[row, col] = p
                else:
                    trans_mat[row, col] = 0.
    # adjust so all rows sum to 1
    row_sums = trans_mat.sum(axis = 1)
    trans_mat = trans_mat[row_sums != 0.0, :]
    row_sums = trans_mat.sum(axis = 1)
    trans_mat = trans_mat / row_sums[:, np.newaxis]
    return trans_mat

def find_branch_points(trans_mat, labels):
    """given a 2-d transition matrix, find branch points, i.e. labels
    that can transition to more than one label

    Parameters
    ----------
    trans_mat : numpy.ndarray
        2-d matrix where rows represent probabilities of transitioning
        from some label to whichever label is indexed by the column
        where the probability appears.
    labels : list
        of str, unique set of labels. The index of each label
        must correspond to the row in trans_mat representing the
        probabilities of transitioning from that label to others.

    Returns
    -------
    branch_point_ind : numpy.ndarray
        indices of rows that are branch points in trans_mat.
    branch_point_lbl : list
        of labels. Labels for indices in branch_point_ind.
    """
    branch_point_ind = []
    branch_point_lbl = []
    for ind, (row, label) in enumerate(zip(trans_mat, labels)):
        ps = np.nonzero(row)[0]
        if ps.shape[0] > 1:
            branch_point_ind.append(ind)
            branch_point_lbl.append(label)
    if branch_point_ind:  # is not empty
        branch_point_ind = np.asarray(branch_point_ind)
    return branch_point_ind, branch_point_lbl

<commit_msg>rewrite date_from_cbin_filename to be less fragile<commit_after>"""functions used for syntax analysis"""
from datetime import datetime
from pathlib import Path

import numpy as np
import pandas as pd

HERE = Path(__file__).parent

def date_from_cbin_filename(cbin_filename):
    """extracts date as a string from a .cbin filename
    and converts to a datetime object

    Parameters
    ----------
    cbin_filename : str
        path to a .cbin file

    Returns
    -------
    dt : datetime
        convert from string using strptime function

    Examples
    --------
    >>> cbin_filename = 'gy6or6_baseline_220312_0836.3.cbin'
    >>> dt = date_from_cbin_filename(cbin_filename)
    """
    name = Path(cbin_filename).name
    splitname = name.split('.')  # splits into: stem, file number, extension
    splitname = splitname[0]  # just keep the stem
    splitname = splitname.split('_')  # further split stem; last two will be date + time
    date, time = splitname[-2:]
    dt = date + '-' + time
    dt = datetime.strptime(dt, '%d%m%y-%H%M')
    return dt

FIELDS_SYNTAX = [
    'date',
    'time',
    'cbin_filename',
    'label',
    'label_plus_one',
    'label_minus_one',
    'label_minus_two',
]

def make_df_trans_probs(vds_list, start_label='S', end_label='E'):
    """make a dataframe that can be used to compute transition probabilities
    between birdsong syllables.

    Takes a list of vak Datasets; function expects each vocalization in the
    dataset to be from .cbin audio files.

    Parameters
    ----------
    vds_list : list
        of vak.Dataset.
    start_label : str
        Label that represents start of a sequence, e.g. a song bout. Default is 'S'.
    end_label : str
        Label that represents end of a sequence. Default is 'E'.

    Returns
    -------
    df : pandas.Dataframe
        with the following columns
            date : datetime
                extracted from audio filename.
                Used to determine transition probabilities on a certain day,
                and for resampling analysis where the dates are shuffled.
            cbin_filename : str
                filename of audio file associated with annotation.
            label : str
                String label for current label of interest.
                For example, label 'a' in transition 'a' -> 'b'.
            label_plus_one : str
                Label that follows current label.
                For exapmle, label 'b' in transition 'a' -> 'b'.
            label_minus_one : str
                Label that preceded current label.
            label_minus_two : str
                Label that preceded label before current label.
    """
    df_dict = {field: [] for field in FIELDS_SYNTAX}
    for vds in vds_list:
        for voc in vds.voc_list:
            cbin_filename = voc.audio_path
            datetime = date_from_cbin_filename(cbin_filename)
            labels = voc.annot.labels.tolist()
            labels = [start_label] + labels + [end_label]
            len_labels = len(labels)
            for ind, label in enumerate(labels):
                if ind == 0:
                    label_minus_one = None
                    label_minus_two = None
                    label_plus_one = labels[ind + 1]
                elif ind == 1:
                    label_minus_one = labels[0]
                    label_minus_two = None
                    label_plus_one = labels[ind + 1]
                elif ind == (len_labels - 1):  # i.e., last index
                    label_minus_one = labels[ind - 1]
                    label_minus_two = labels[ind - 2]
                    label_plus_one = None
                else:
                    label_minus_one = labels[ind - 1]
                    label_minus_two = labels[ind - 2]
                    label_plus_one = labels[ind + 1]
                df_dict['date'].append(datetime.date())
                df_dict['time'].append(datetime.time())
                df_dict['cbin_filename'].append(cbin_filename)
                df_dict['label'].append(label)
                df_dict['label_plus_one'].append(label_plus_one)
                df_dict['label_minus_one'].append(label_minus_one)
                df_dict['label_minus_two'].append(label_minus_two)
    df = pd.DataFrame.from_dict(df_dict)
    return df

def get_trans_prob(df, date, label, label_plus_one):
    """

    Parameters
    ----------
    df
    date
    label
    label_plus_one

    Returns
    -------

    """
    df_date = df[df['date'] == date]
    label_count = len(
        df_date[df_date['label'] == label].index
    )
    trans_count = len(
        df_date[(df_date['label'] == label) & (df_date['label_plus_one'] == label_plus_one)].index
    )
    p = trans_count / label_count
    return p

def make_trans_mat(df, min_p=0.01):
    """

    Parameters
    ----------
    df
    min_p

    Returns
    -------

    """
    labels = df['label'].unique()
    num_labels = labels.shape[0]
    trans_mat = np.zeros((num_labels, num_labels))
    for row, label in enumerate(labels):
        if label == 'E':
            continue
        else:
            for col, label_plus_one in enumerate(labels):
                p = get_trans_prob(df, day, label, label_plus_one)
                if p > min_p:
                    trans_mat[row, col] = p
                else:
                    trans_mat[row, col] = 0.
    # adjust so all rows sum to 1
    row_sums = trans_mat.sum(axis = 1)
    trans_mat = trans_mat[row_sums != 0.0, :]
    row_sums = trans_mat.sum(axis = 1)
    trans_mat = trans_mat / row_sums[:, np.newaxis]
    return trans_mat

def find_branch_points(trans_mat, labels):
    """given a 2-d transition matrix, find branch points, i.e. labels
    that can transition to more than one label

    Parameters
    ----------
    trans_mat : numpy.ndarray
        2-d matrix where rows represent probabilities of transitioning
        from some label to whichever label is indexed by the column
        where the probability appears.
    labels : list
        of str, unique set of labels. The index of each label
        must correspond to the row in trans_mat representing the
        probabilities of transitioning from that label to others.

    Returns
    -------
    branch_point_ind : numpy.ndarray
        indices of rows that are branch points in trans_mat.
    branch_point_lbl : list
        of labels. Labels for indices in branch_point_ind.
    """
    branch_point_ind = []
    branch_point_lbl = []
    for ind, (row, label) in enumerate(zip(trans_mat, labels)):
        ps = np.nonzero(row)[0]
        if ps.shape[0] > 1:
            branch_point_ind.append(ind)
            branch_point_lbl.append(label)
    if branch_point_ind:  # is not empty
        branch_point_ind = np.asarray(branch_point_ind)
    return branch_point_ind, branch_point_lbl

<|endoftext|>


<commit_before><commit_msg>Get basic safety radius back<commit_after><|endoftext|>


<commit_before>import io
import sys
import inspect
import argparse
import binwalk.common
from binwalk.compat import *

class ModuleOption(object):
	'''
	A container class that allows modules to declare command line options.
	'''

	def __init__(self, kwargs={}, nargs=0, priority=0, description="", short="", long="", type=str, dtype=""):
		'''
		Class constructor.

		@kwargs      - A dictionary of kwarg key-value pairs affected by this command line option.
		@nargs       - The number of arguments this option accepts (only 1 or 0 is currently supported).
		@priority    - A value from 0 to 100. Higher priorities will override kwarg values set by lower priority options.
		@description - A description to be displayed in the help output.
		@short       - The short option to use (optional).
		@long        - The long option to use (if None, this option will not be displayed in help output).
		@type        - The accepted data type (one of: io.FileIO/argparse.FileType/binwalk.common.BlockFile, list, str, int, float).
		@dtype       - The accepted data type, as displayed in the help output.

		Returns None.
		'''
		self.kwargs = kwargs
		self.nargs = nargs
		self.priority = priority
		self.description = description
		self.short = short
		self.long = long
		self.type = type
		self.dtype = str(dtype)

		if not self.dtype and self.type:
			self.dtype = str(self.type)

class ModuleKwarg(object):
		'''
		A container class allowing modules to specify their expected __init__ kwarg(s).
		'''

		def __init__(self, name="", default=None, description=""):
			'''
			Class constructor.

			@name        - Kwarg name.
			@default     - Default kwarg value.
			@description - Description string.

			Return None.
			'''
			self.name = name
			self.default = default
			self.description = description

class Result(object):
	'''
	Generic class for storing and accessing scan results.
	'''

	def __init__(self, **kwargs):
		'''
		Class constructor.

		@offset      - The file offset of the result.
		@description - The result description, as displayed to the user.
		@file        - The file object of the scanned file.
		@valid       - Set to True if the result if value, False if invalid.
		@display     - Set to True to display the result to the user, False to hide it.

		Provide additional kwargs as necessary.
		Returns None.
		'''
		self.offset = 0
		self.description = ''
		self.file = None
		self.valid = True
		self.display = True

		for (k, v) in iterator(kwargs):
			setattr(self, k, v)

class Error(Result):
	'''
	A subclass of binwalk.module.Result.
	'''

	def __init__(self, **kwargs):
		'''
		Accepts all the same kwargs as binwalk.module.Result, but the following are also added:

		@exception - In case of an exception, this is the exception object.

		Returns None.
		'''
		self.exception = None
		Result.__init__(self, **kwargs)

		if self.exception:
			sys.stderr.write("Exception: " + str(self.exception) + "\n")
		elif self.description:
			sys.stderr.write("Error: " + self.description + "\n")

class Module(object):
	'''
	All module classes must be subclassed from this.
	'''
	# The module name, as displayed in help output
	NAME = ""

	# A list of binwalk.module.ModuleOption command line options
	CLI = []

	# A list of binwalk.module.ModuleKwargs accepted by __init__
	KWARGS = []

	# A dictionary of module dependencies; all modules depend on binwalk.modules.configuration.Configuration
	DEPENDS = {}

	# Format string for printing the header during a scan
	HEADER_FORMAT = "%s\n"

	# Format string for printing each result during a scan
	RESULT_FORMAT = "%.8d      %s\n"

	# The header to print during a scan.
	# Set to None to not print a header.
	# Note that this will be formatted per the HEADER_FORMAT format string.
	HEADER = ["OFFSET      DESCRIPTION"]

	# The attribute names to print during a scan, as provided to the self.results method.
	# Set to None to not print any results.
	# Note that these will be formatted per the RESULT_FORMAT format string.
	RESULT = ['offset', 'description']

	def __init__(self, dependency=False, **kwargs):
		# TODO: Instantiate plugins object
		# self.plugins = x
		self.errors = []
		self.results = []

		process_kwargs(self, kwargs)

		# If the module was loaded as a dependency, don't display or log any results
		if dependency:
			self.config.display.quiet = True
			self.config.display.log = None

		try:
			self.load()
		except KeyboardInterrupt as e:
			raise e
		except Exception as e:
			self.error(exception=e)

	def load(self):
		'''
		Invoked at module load time.
		May be overridden by the module sub-class.
		'''
		return None

	def init(self):
		'''
		Invoked prior to self.run.
		May be overridden by the module sub-class.

		Returns None.
		'''
		return None

	def run(self):
		'''
		Executes the main module routine.
		Must be overridden by the module sub-class.

		Returns True on success, False on failure.
		'''
		return False

	def validate(self, r):
		'''
		Validates the result.
		May be overridden by the module sub-class.

		@r - The result, an instance of binwalk.module.Result.

		Returns None.
		'''
		r.valid = True
		return None

	def _plugins_pre_scan(self):
		# plugins(self)
		return None

	def _plugins_post_scan(self):
		# plugins(self)
		return None

	def _plugins_callback(self, r):
		return None

	def _build_display_args(self, r):
		args = []

		if self.RESULT:
			if type(self.RESULT) != type([]):
				result = [self.RESULT]
			else:
				result = self.RESULT

			for name in result:
				args.append(getattr(r, name))

		return args

	def result(self, **kwargs):
		'''
		Validates a result, stores it in self.results and prints it.

		Accepts the same kwargs as the binwalk.module.Result class.

		Returns None.
		'''
		r = Result(**kwargs)

		self.validate(r)
		self._plugins_callback(r)

		if r.valid:
			self.results.append(r)
			if r.display:
				display_args = self._build_display_args(r)
				if display_args:
					self.config.display.result(*display_args)

	def error(self, **kwargs):
		'''
		Stores the specified error in self.errors.

		Accepts the same kwargs as the binwalk.module.Error class.

		Returns None.
		'''
		e = Error(**kwargs)
		self.errors.append(e)

	def header(self):
		self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
		if type(self.HEADER) == type([]):
			self.config.display.header(*self.HEADER)
		elif self.HEADER:
			self.config.display.header(self.HEADER)

	def footer(self):
		self.config.display.footer()

	def main(self):
		'''
		Responsible for calling self.init, initializing self.config.display, and calling self.run.

		Returns the value returned from self.run.
		'''
		try:
			self.init()
		except KeyboardInterrupt as e:
			raise e
		except Exception as e:
			self.error(exception=e)
			return False

		self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)

		self._plugins_pre_scan()

		try:
			retval = self.run()
		except KeyboardInterrupt as e:
			raise e
		except Exception as e:
			self.error(exception=e)
			return False

		self._plugins_post_scan()

		return retval

class Modules(object):
	'''
	Main class used for running and managing modules.
	'''

	def __init__(self, argv=sys.argv[1:]):
		'''
		Class constructor.

		@argv  - List of command line options. Must not include the program name (sys.argv[0]).

		Returns None.
		'''
		self.arguments = argv
		self.dependency_results = {}

	def list(self, attribute="run"):
		'''
		Finds all modules with the specified attribute.

		@attribute - The desired module attribute.

		Returns a list of modules that contain the specified attribute.
		'''
		modules = []

		for (name, module) in inspect.getmembers(binwalk.modules):
			if inspect.isclass(module) and hasattr(module, attribute):
				modules.append(module)

		return modules

	def help(self):
		help_string = ""

		for obj in self.list(attribute="CLI"):
			if obj.CLI:
				help_string += "\n%s Options:\n" % obj.NAME

				for module_option in obj.CLI:
					if module_option.long:
						long_opt = '--' + module_option.long

						if module_option.nargs > 0:
							optargs = "=%s" % module_option.dtype
						else:
							optargs = ""

						if module_option.short:
							short_opt = "-" + module_option.short + ","
						else:
							short_opt = "   "

						fmt = "    %%s %%s%%-%ds%%s\n" % (32-len(long_opt))
						help_string += fmt % (short_opt, long_opt, optargs, module_option.description)

		return help_string

	def execute(self):
		run_modules = []
		for module in self.list():
			if self.run(module):
				run_modules.append(module)
		return run_modules

	def run(self, module):
		obj = self.load(module)

		if isinstance(obj, binwalk.module.Module) and obj.enabled:
			try:
				obj.main()
			except AttributeError as e:
				sys.stderr.write("WARNING: " + str(e) + "\n")
				obj = None

		return obj

	def load(self, module):
		kwargs = self.argv(module, argv=self.arguments)
		kwargs.update(self.dependencies(module))
		return module(**kwargs)

	def dependencies(self, module):
		kwargs = {}

		if hasattr(module, "DEPENDS"):
			for (kwarg, mod) in iterator(module.DEPENDS):
				if mod == module:
					continue

				if not has_key(self.dependency_results, mod):
					# TODO: What to do if a dependency fails?
					self.dependency_results[mod] = self.run(mod)
				kwargs[kwarg] = self.dependency_results[mod]

		return kwargs

	def argv(self, module, argv=sys.argv[1:]):
		'''
		Processes argv for any options specific to the specified module.

		@module - The module to process argv for.
		@argv   - A list of command line arguments (excluding argv[0]).

		Returns a dictionary of kwargs for the specified module.
		'''
		kwargs = {}
		last_priority = {}
		longs = []
		shorts = ""
		parser = argparse.ArgumentParser(add_help=False)

		if hasattr(module, "CLI"):

			for module_option in module.CLI:
				if not module_option.long:
					continue

				if module_option.nargs == 0:
					action = 'store_true'
				else:
					action = None

				if module_option.short:
					parser.add_argument('-' + module_option.short, '--' + module_option.long, action=action, dest=module_option.long)
				else:
					parser.add_argument('--' + module_option.long, action=action, dest=module_option.long)

			args, unknown = parser.parse_known_args(argv)
			args = args.__dict__

			for module_option in module.CLI:

				if module_option.type in [io.FileIO, argparse.FileType, binwalk.common.BlockFile]:

					for k in get_keys(module_option.kwargs):
						kwargs[k] = []
						for unk in unknown:
							if not unk.startswith('-'):
								kwargs[k].append(unk)

				elif has_key(args, module_option.long) and args[module_option.long] not in [None, False]:

					i = 0
					for (name, value) in iterator(module_option.kwargs):
						if not has_key(last_priority, name) or last_priority[name] <= module_option.priority:
							if module_option.nargs > i:
								value = args[module_option.long]
								i += 1

							last_priority[name] = module_option.priority

							# Do this manually as argparse doesn't seem to be able to handle hexadecimal values
							if module_option.type == int:
								kwargs[name] = int(value, 0)
							elif module_option.type == float:
								kwargs[name] = float(value)
							elif module_option.type == dict:
								if not has_key(kwargs, name):
									kwargs[name] = {}
								kwargs[name][len(kwargs[name])] = value
							elif module_option.type == list:
								if not has_key(kwargs, name):
									kwargs[name] = []
								kwargs[name].append(value)
							else:
								kwargs[name] = value
		else:
			raise Exception("binwalk.module.Modules.argv: %s has no attribute 'CLI'" % str(module))

		if not has_key(kwargs, 'enabled'):
			kwargs['enabled'] = False

		return kwargs

	def kwargs(self, module, kwargs):
		'''
		Processes a module's kwargs. All modules should use this for kwarg processing.

		@module - An instance of the module (e.g., self)
		@kwargs - The kwargs passed to the module

		Returns None.
		'''
		if hasattr(module, "KWARGS"):
			for module_argument in module.KWARGS:
				if has_key(kwargs, module_argument.name):
					arg_value = kwargs[module_argument.name]
				else:
					arg_value = module_argument.default

				setattr(module, module_argument.name, arg_value)

			for (k, v) in iterator(kwargs):
				if not hasattr(module, k):
					setattr(module, k, v)
		else:
			raise Exception("binwalk.module.Modules.process_kwargs: %s has no attribute 'KWARGS'" % str(module))

def process_kwargs(obj, kwargs):
	'''
	Convenience wrapper around binwalk.module.Modules.kwargs.

	@obj    - The class object (an instance of a sub-class of binwalk.module.Module).
	@kwargs - The kwargs provided to the object's __init__ method.

	Returns None.
	'''
	return Modules().kwargs(obj, kwargs)

def show_help(fd=sys.stdout):
	'''
	Convenience wrapper around binwalk.module.Modules.help.

	@fd - An object with a write method (e.g., sys.stdout, sys.stderr, etc).

	Returns None.
	'''
	fd.write(Modules().help())

<commit_msg>Added Configuration module as a dependency for other modules<commit_after>import io
import sys
import inspect
import argparse
import binwalk.common
from binwalk.compat import *

class ModuleOption(object):
	'''
	A container class that allows modules to declare command line options.
	'''

	def __init__(self, kwargs={}, nargs=0, priority=0, description="", short="", long="", type=str, dtype=""):
		'''
		Class constructor.

		@kwargs      - A dictionary of kwarg key-value pairs affected by this command line option.
		@nargs       - The number of arguments this option accepts (only 1 or 0 is currently supported).
		@priority    - A value from 0 to 100. Higher priorities will override kwarg values set by lower priority options.
		@description - A description to be displayed in the help output.
		@short       - The short option to use (optional).
		@long        - The long option to use (if None, this option will not be displayed in help output).
		@type        - The accepted data type (one of: io.FileIO/argparse.FileType/binwalk.common.BlockFile, list, str, int, float).
		@dtype       - The accepted data type, as displayed in the help output.

		Returns None.
		'''
		self.kwargs = kwargs
		self.nargs = nargs
		self.priority = priority
		self.description = description
		self.short = short
		self.long = long
		self.type = type
		self.dtype = str(dtype)

		if not self.dtype and self.type:
			self.dtype = str(self.type)

class ModuleKwarg(object):
		'''
		A container class allowing modules to specify their expected __init__ kwarg(s).
		'''

		def __init__(self, name="", default=None, description=""):
			'''
			Class constructor.

			@name        - Kwarg name.
			@default     - Default kwarg value.
			@description - Description string.

			Return None.
			'''
			self.name = name
			self.default = default
			self.description = description

class Result(object):
	'''
	Generic class for storing and accessing scan results.
	'''

	def __init__(self, **kwargs):
		'''
		Class constructor.

		@offset      - The file offset of the result.
		@description - The result description, as displayed to the user.
		@file        - The file object of the scanned file.
		@valid       - Set to True if the result if value, False if invalid.
		@display     - Set to True to display the result to the user, False to hide it.

		Provide additional kwargs as necessary.
		Returns None.
		'''
		self.offset = 0
		self.description = ''
		self.file = None
		self.valid = True
		self.display = True

		for (k, v) in iterator(kwargs):
			setattr(self, k, v)

class Error(Result):
	'''
	A subclass of binwalk.module.Result.
	'''

	def __init__(self, **kwargs):
		'''
		Accepts all the same kwargs as binwalk.module.Result, but the following are also added:

		@exception - In case of an exception, this is the exception object.

		Returns None.
		'''
		self.exception = None
		Result.__init__(self, **kwargs)

		if self.exception:
			sys.stderr.write("Exception: " + str(self.exception) + "\n")
		elif self.description:
			sys.stderr.write("Error: " + self.description + "\n")

class Module(object):
	'''
	All module classes must be subclassed from this.
	'''
	# The module name, as displayed in help output
	NAME = ""

	# A list of binwalk.module.ModuleOption command line options
	CLI = []

	# A list of binwalk.module.ModuleKwargs accepted by __init__
	KWARGS = []

	# A dictionary of module dependencies; all modules depend on binwalk.modules.configuration.Configuration
	DEPENDS = {}

	# Format string for printing the header during a scan
	HEADER_FORMAT = "%s\n"

	# Format string for printing each result during a scan
	RESULT_FORMAT = "%.8d      %s\n"

	# The header to print during a scan.
	# Set to None to not print a header.
	# Note that this will be formatted per the HEADER_FORMAT format string.
	HEADER = ["OFFSET      DESCRIPTION"]

	# The attribute names to print during a scan, as provided to the self.results method.
	# Set to None to not print any results.
	# Note that these will be formatted per the RESULT_FORMAT format string.
	RESULT = ['offset', 'description']

	def __init__(self, dependency=False, **kwargs):
		# TODO: Instantiate plugins object
		# self.plugins = x
		self.errors = []
		self.results = []

		process_kwargs(self, kwargs)

		# If the module was loaded as a dependency, don't display or log any results
		if dependency:
			self.config.display.quiet = True
			self.config.display.log = None

		try:
			self.load()
		except KeyboardInterrupt as e:
			raise e
		except Exception as e:
			self.error(exception=e)

	def load(self):
		'''
		Invoked at module load time.
		May be overridden by the module sub-class.
		'''
		return None

	def init(self):
		'''
		Invoked prior to self.run.
		May be overridden by the module sub-class.

		Returns None.
		'''
		return None

	def run(self):
		'''
		Executes the main module routine.
		Must be overridden by the module sub-class.

		Returns True on success, False on failure.
		'''
		return False

	def validate(self, r):
		'''
		Validates the result.
		May be overridden by the module sub-class.

		@r - The result, an instance of binwalk.module.Result.

		Returns None.
		'''
		r.valid = True
		return None

	def _plugins_pre_scan(self):
		# plugins(self)
		return None

	def _plugins_post_scan(self):
		# plugins(self)
		return None

	def _plugins_callback(self, r):
		return None

	def _build_display_args(self, r):
		args = []

		if self.RESULT:
			if type(self.RESULT) != type([]):
				result = [self.RESULT]
			else:
				result = self.RESULT

			for name in result:
				args.append(getattr(r, name))

		return args

	def result(self, **kwargs):
		'''
		Validates a result, stores it in self.results and prints it.

		Accepts the same kwargs as the binwalk.module.Result class.

		Returns None.
		'''
		r = Result(**kwargs)

		self.validate(r)
		self._plugins_callback(r)

		if r.valid:
			self.results.append(r)
			if r.display:
				display_args = self._build_display_args(r)
				if display_args:
					self.config.display.result(*display_args)

	def error(self, **kwargs):
		'''
		Stores the specified error in self.errors.

		Accepts the same kwargs as the binwalk.module.Error class.

		Returns None.
		'''
		e = Error(**kwargs)
		self.errors.append(e)

	def header(self):
		self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
		if type(self.HEADER) == type([]):
			self.config.display.header(*self.HEADER)
		elif self.HEADER:
			self.config.display.header(self.HEADER)

	def footer(self):
		self.config.display.footer()

	def main(self):
		'''
		Responsible for calling self.init, initializing self.config.display, and calling self.run.

		Returns the value returned from self.run.
		'''
		try:
			self.init()
		except KeyboardInterrupt as e:
			raise e
		except Exception as e:
			self.error(exception=e)
			return False

		self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)

		self._plugins_pre_scan()

		try:
			retval = self.run()
		except KeyboardInterrupt as e:
			raise e
		except Exception as e:
			self.error(exception=e)
			return False

		self._plugins_post_scan()

		return retval

class Modules(object):
	'''
	Main class used for running and managing modules.
	'''

	def __init__(self, argv=sys.argv[1:]):
		'''
		Class constructor.

		@argv  - List of command line options. Must not include the program name (sys.argv[0]).

		Returns None.
		'''
		self.arguments = argv
		self.dependency_results = {}

	def list(self, attribute="run"):
		'''
		Finds all modules with the specified attribute.

		@attribute - The desired module attribute.

		Returns a list of modules that contain the specified attribute.
		'''
		modules = []

		for (name, module) in inspect.getmembers(binwalk.modules):
			if inspect.isclass(module) and hasattr(module, attribute):
				modules.append(module)

		return modules

	def help(self):
		help_string = ""

		for obj in self.list(attribute="CLI"):
			if obj.CLI:
				help_string += "\n%s Options:\n" % obj.NAME

				for module_option in obj.CLI:
					if module_option.long:
						long_opt = '--' + module_option.long

						if module_option.nargs > 0:
							optargs = "=%s" % module_option.dtype
						else:
							optargs = ""

						if module_option.short:
							short_opt = "-" + module_option.short + ","
						else:
							short_opt = "   "

						fmt = "    %%s %%s%%-%ds%%s\n" % (32-len(long_opt))
						help_string += fmt % (short_opt, long_opt, optargs, module_option.description)

		return help_string + "\n"

	def execute(self):
		run_modules = []
		for module in self.list():
			if self.run(module):
				run_modules.append(module)
		return run_modules

	def run(self, module):
		obj = self.load(module)

		if isinstance(obj, binwalk.module.Module) and obj.enabled:
			try:
				obj.main()
			except AttributeError as e:
				sys.stderr.write("WARNING: " + str(e) + "\n")
				obj = None

		return obj

	def load(self, module):
		kwargs = self.argv(module, argv=self.arguments)
		kwargs.update(self.dependencies(module))
		return module(**kwargs)

	def dependencies(self, module):
		kwargs = {}

		if hasattr(module, "DEPENDS"):
			for (kwarg, mod) in iterator(module.DEPENDS):
				if mod == module:
					continue

				if not has_key(self.dependency_results, mod):
					# TODO: What to do if a dependency fails?
					self.dependency_results[mod] = self.run(mod)
				kwargs[kwarg] = self.dependency_results[mod]

		return kwargs

	def argv(self, module, argv=sys.argv[1:]):
		'''
		Processes argv for any options specific to the specified module.

		@module - The module to process argv for.
		@argv   - A list of command line arguments (excluding argv[0]).

		Returns a dictionary of kwargs for the specified module.
		'''
		kwargs = {}
		last_priority = {}
		longs = []
		shorts = ""
		parser = argparse.ArgumentParser(add_help=False)

		if hasattr(module, "CLI"):

			for module_option in module.CLI:
				if not module_option.long:
					continue

				if module_option.nargs == 0:
					action = 'store_true'
				else:
					action = None

				if module_option.short:
					parser.add_argument('-' + module_option.short, '--' + module_option.long, action=action, dest=module_option.long)
				else:
					parser.add_argument('--' + module_option.long, action=action, dest=module_option.long)

			args, unknown = parser.parse_known_args(argv)
			args = args.__dict__

			for module_option in module.CLI:

				if module_option.type in [io.FileIO, argparse.FileType, binwalk.common.BlockFile]:

					for k in get_keys(module_option.kwargs):
						kwargs[k] = []
						for unk in unknown:
							if not unk.startswith('-'):
								kwargs[k].append(unk)

				elif has_key(args, module_option.long) and args[module_option.long] not in [None, False]:

					i = 0
					for (name, value) in iterator(module_option.kwargs):
						if not has_key(last_priority, name) or last_priority[name] <= module_option.priority:
							if module_option.nargs > i:
								value = args[module_option.long]
								i += 1

							last_priority[name] = module_option.priority

							# Do this manually as argparse doesn't seem to be able to handle hexadecimal values
							if module_option.type == int:
								kwargs[name] = int(value, 0)
							elif module_option.type == float:
								kwargs[name] = float(value)
							elif module_option.type == dict:
								if not has_key(kwargs, name):
									kwargs[name] = {}
								kwargs[name][len(kwargs[name])] = value
							elif module_option.type == list:
								if not has_key(kwargs, name):
									kwargs[name] = []
								kwargs[name].append(value)
							else:
								kwargs[name] = value
		else:
			raise Exception("binwalk.module.Modules.argv: %s has no attribute 'CLI'" % str(module))

		if not has_key(kwargs, 'enabled'):
			kwargs['enabled'] = False

		return kwargs

	def kwargs(self, module, kwargs):
		'''
		Processes a module's kwargs. All modules should use this for kwarg processing.

		@module - An instance of the module (e.g., self)
		@kwargs - The kwargs passed to the module

		Returns None.
		'''
		if hasattr(module, "KWARGS"):
			for module_argument in module.KWARGS:
				if has_key(kwargs, module_argument.name):
					arg_value = kwargs[module_argument.name]
				else:
					arg_value = module_argument.default

				setattr(module, module_argument.name, arg_value)

			for (k, v) in iterator(kwargs):
				if not hasattr(module, k):
					setattr(module, k, v)
		else:
			raise Exception("binwalk.module.Modules.process_kwargs: %s has no attribute 'KWARGS'" % str(module))

def process_kwargs(obj, kwargs):
	'''
	Convenience wrapper around binwalk.module.Modules.kwargs.

	@obj    - The class object (an instance of a sub-class of binwalk.module.Module).
	@kwargs - The kwargs provided to the object's __init__ method.

	Returns None.
	'''
	return Modules().kwargs(obj, kwargs)

def show_help(fd=sys.stdout):
	'''
	Convenience wrapper around binwalk.module.Modules.help.

	@fd - An object with a write method (e.g., sys.stdout, sys.stderr, etc).

	Returns None.
	'''
	fd.write(Modules().help())

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sat Aug 13 22:42:08 2016

@author: Tian Zhou
"""

import os,sys

def processOneFile(fn):
    cnt=0;
    accuracy=0.0
    precision=0.0
    recall=0.0
    f1=0.0
    with open(fn) as fin:
        header=fin.readline()[:-1].split('\t')
        pACC=header.index('accuracy')
        pPRE=header.index('precision')
        pREC=header.index('recall')
        pF1=header.index('f1')
        for line in fin:
            line=line[:-1].split('\t')
            accuracy+=float(line[pACC])
            precision+=float(line[pPRE])
            recall+=float(line[pREC])
            f1+=float(line[pF1])
            cnt+=1
    if cnt!=0:
        accuracy/=cnt
        precision/=cnt
        recall/=cnt
        f1/=cnt
    return (cnt, accuracy, precision, recall, f1)

def output(outFn, data):
    fout=open(outFn, 'w')
    if not fout:
        print('cannot open output file')
        return
    fout.write('\t'.join(['filename','count','accuracy','precision','recall','f1'])+'\n')
    for fn,t in sorted(data.items()):
        fout.write(fn+'\t'+'\t'.join([str(v) if isinstance(v,int) else '%.6f'%v for v in t]))
        fout.write('\n')

def main(folder, outFn):
    if not folder.endswith('\\') and not folder.endswith('/'):
        folder+='/'
    data={}
    print('loading')
    for fn in os.listdir(folder):
        if fn.startswith('res-') and fn.endswith('.txt') and os.path.isfile(folder+fn):
            print('  loading: '+fn)
            data[fn]=processOneFile(folder+fn)
    print('loaded: '+str(len(data)))
    print('outputting')
    output(outFn, data)
    print('done')

if __name__=='__main__':
    if len(sys.argv)!=3:
        print('usage: <folder of test result files> <output file name>')
        exit()
    folder=sys.argv[1]
    outFn=sys.argv[2]
    main(folder,outFn)
<commit_msg>support frequence-based evaluator result summary<commit_after># -*- coding: utf-8 -*-
"""
Created on Sat Aug 13 22:42:08 2016

@author: Tian Zhou
"""

import os,sys

def processOneFile(filename, content):
    cnt=0;
    accuracy=0.0
    precision=0.0
    recall=0.0
    f1=0.0
    fp=0.0
    fn=0.0
    sd=0.0
    sr=0.0
    with open(filename) as fin:
        header=fin.readline()[:-1].split('\t')
        pACC=header.index('accuracy')
        pPRE=header.index('precision')
        pREC=header.index('recall')
        pF1=header.index('f1')
        pfp=header.index('fre-pos')
        pfn=header.index('fre-neg')
        psd=header.index('score-diff')
        psr=header.index('score-ratio')
        for line in fin:
            line=line[:-1].split('\t')
            accuracy+=float(line[pACC])
            precision+=float(line[pPRE])
            recall+=float(line[pREC])
            f1+=float(line[pF1])
            fp+=float(line[pfp])
            fn+=float(line[pfn])
            sd+=float(line[psd])
            sr+=float(line[psr])
            cnt+=1
    if cnt!=0:
        accuracy/=cnt
        precision/=cnt
        recall/=cnt
        f1/=cnt
        fp/=cnt
        fn/=cnt
        sd/=cnt
        sr/=cnt
    return (cnt, accuracy, precision, recall, f1, fp, fn, sd, sr)

def output(outFn, data, header, nDig):
    fout=open(outFn, 'w')
    if not fout:
        print('cannot open output file')
        return
    pointFmt='%.'+str(nDig)+'f'
    fout.write('\t'.join(header)+'\n')
    for fn,t in sorted(data.items()):
        fout.write(fn+'\t'+'\t'.join([str(v) if isinstance(v,int) else pointFmt%v for v in t]))
        fout.write('\n')

def main(folder, outFn, nDig):
    if not folder.endswith('\\') and not folder.endswith('/'):
        folder+='/'
    header=['filename','count','accuracy','precision','recall','f1','fre-pos','fre-neg','score-diff','score-ratio']
    data={}
    print('loading')
    for fn in os.listdir(folder):
        if fn.startswith('res-') and fn.endswith('.txt') and os.path.isfile(folder+fn):
            print('  loading: '+fn)
            data[fn]=processOneFile(folder+fn, header)
    print('loaded: '+str(len(data)))
    print('outputting')
    output(outFn, data, header, nDig)
    print('done')

if __name__=='__main__':
    if len(sys.argv)<3 or len(sys.argv)>4:
        print('usage: <folder of test result files> <output file name> [# of digit for result (def: 3)]')
        print('\tFile names should start with "res-" and end with ".txt"')
        exit()
    folder=sys.argv[1]
    outFn=sys.argv[2]
    nDig=int(sys.argv[3]) if len(sys.argv)==4 else 3
    main(folder,outFn,nDig)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
Created on 27/06/2013

@authors: Andrés Felipe Calderón andres.calderon@correlibre.org
          Sebastián Ortiz neoecos@gmail.com

@license:  GNU AFFERO GENERAL PUBLIC LICENSE

Caliope Storage is the base of Caliope's Framework
Copyright (C) 2013 Fundación Correlibre

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
import os
import getopt
import sys
import json

from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
from flask import Flask, render_template, send_from_directory
from jinja2 import FileSystemLoader

from gevent import monkey;

from api.views import api
from server_notifications.views import server_notifications
from file_uploader.views import file_uploader

#: Gevent to patch all TCP/IP connections
monkey.patch_all()

app = Flask(__name__)
app.secret_key = os.urandom(24)
app.debug = True

app.register_blueprint(api, url_prefix='/api')
app.register_blueprint(server_notifications, url_prefix='/event_from_server')
app.register_blueprint(file_uploader, url_prefix='/upload')

@app.route('/')
def index():
        return render_template('index.html')

@app.route('/<path:filename>')
def custom_static(filename):
    return send_from_directory(app.config['STATIC_PATH'], filename)

def main(argv):
    configfile = "conf/caliope_server.json"
    try:
        opts, args = getopt.getopt(argv, "hc:", ["help", "config="])
    except getopt.GetoptError:
      print 'runserver.py -c <configfile>'
      sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            print 'runserver.py -c <configfile>'
            sys.exit()
        elif opt in ("-c", "--config"):
            configfile = arg

    try:
        config = json.loads(open(configfile).read())
    except IOError:
        print "Error: can\'t find config file or read data"
    else:

        if 'port' in config:
            port = int(config['port'])
        else:
            port = 8000

        if 'static' in config:
            app.config['STATIC_PATH'] = config['static']
        else:
            app.config['STATIC_PATH'] = "."

        print "=============================="
        print "listening at port : " + str(port)
        print "static base directory : " + app.config['STATIC_PATH']
        print "=============================="

        app.jinja_loader = FileSystemLoader(os.path.join(".", app.config['STATIC_PATH']))
        http_server = WSGIServer(('', port), app, handler_class=WebSocketHandler)
        http_server.serve_forever()

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Small refactor, including header, imports and some pep8 warnings<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@authors: Andrés Felipe Calderón andres.calderon@correlibre.org
          Sebastián Ortiz neoecos@gmail.com

@license:  GNU AFFERO GENERAL PUBLIC LICENSE

Caliope Server is the web server of Caliope's Framework
Copyright (C) 2013 Fundación Correlibre

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
#system, and standard library
import os
import getopt
import sys
import json

#gevent
from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
from gevent import monkey

#flask
from flask import Flask, render_template, send_from_directory
from jinja2 import FileSystemLoader

#Blueprints
from api.views import api
from server_notifications.views import server_notifications
from file_uploader.views import file_uploader

#: Gevent to patch all TCP/IP connections
monkey.patch_all()

app = Flask(__name__)
app.secret_key = os.urandom(24)
app.debug = True

app.register_blueprint(api, url_prefix='/api')
app.register_blueprint(server_notifications, url_prefix='/event_from_server')
app.register_blueprint(file_uploader, url_prefix='/upload')

@app.route('/')
def index():
        return render_template('index.html')

@app.route('/<path:filename>')
def custom_static(filename):
    return send_from_directory(app.config['STATIC_PATH'], filename)

def main(argv):
    configfile = "conf/caliope_server.json"
    try:
        opts, args = getopt.getopt(argv, "hc:", ["help", "config="])
    except getopt.GetoptError:
        print 'caliope_server.py -c <configfile>'
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            print 'caliope_server.py -c <configfile>'
            sys.exit()
        elif opt in ("-c", "--config"):
            configfile = arg
    try:
        config = json.loads(open(configfile).read())
    except IOError:
        print "Error: can\'t find config file or read data"
    else:
        if 'port' in config:
            port = int(config['port'])
        else:
            port = 8000

        if 'static' in config:
            app.config['STATIC_PATH'] = config['static']
        else:
            app.config['STATIC_PATH'] = "."
        print "=============================="
        print "listening at port : " + str(port)
        print "static base directory : " + app.config['STATIC_PATH']
        print "=============================="

        app.jinja_loader = FileSystemLoader(os.path.join(".",
                                            app.config['STATIC_PATH']))
        http_server = WSGIServer(('', port), app, handler_class=WebSocketHandler)
        http_server.serve_forever()

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python2.6
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""

__all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
           "BooleanArgument"]

import logging
from resource_management.core.exceptions import Fail, InvalidArgument
from resource_management.core.environment import Environment

class ResourceArgument(object):
  def __init__(self, default=None, required=False, allow_override=False):
    self.required = False # Prevents the initial validate from failing
    if hasattr(default, '__call__'):
      self.default = default
    else:
      self.default = self.validate(default)
    self.required = required
    self.allow_override = allow_override

  def validate(self, value):
    if self.required and value is None:
      raise InvalidArgument("Required argument %s missing" % self.name)
    return value

class ForcedListArgument(ResourceArgument):
  def validate(self, value):
    value = super(ForcedListArgument, self).validate(value)
    if not isinstance(value, (tuple, list)):
      value = [value]
    return value

class BooleanArgument(ResourceArgument):
  def validate(self, value):
    value = super(BooleanArgument, self).validate(value)
    if not value in (True, False):
      raise InvalidArgument(
        "Expected a boolean for %s received %r" % (self.name, value))
    return value

class Accessor(object):
  def __init__(self, name):
    self.name = name

  def __get__(self, obj, cls):
    try:
      return obj.arguments[self.name]
    except KeyError:
      val = obj._arguments[self.name].default
      if hasattr(val, '__call__'):
        val = val(obj)
      return val

  def __set__(self, obj, value):
    obj.arguments[self.name] = obj._arguments[self.name].validate(value)

class ResourceMetaclass(type):
  # def __new__(cls, name, bases, attrs):
  #     super_new = super(ResourceMetaclass, cls).__new__
  #     return super_new(cls, name, bases, attrs)

  def __init__(mcs, _name, bases, attrs):
    mcs._arguments = getattr(bases[0], '_arguments', {}).copy()
    for key, value in list(attrs.items()):
      if isinstance(value, ResourceArgument):
        value.name = key
        mcs._arguments[key] = value
        setattr(mcs, key, Accessor(key))

class Resource(object):
  __metaclass__ = ResourceMetaclass

  log = logging.getLogger("resource_management.resource")

  action = ForcedListArgument(default="nothing")
  ignore_failures = BooleanArgument(default=False)
  not_if = ResourceArgument() # pass command e.g. not_if = ('ls','/root/jdk')
  only_if = ResourceArgument() # pass command
  initial_wait = ResourceArgument() # in seconds

  actions = ["nothing"]

  def __new__(cls, name, env=None, provider=None, **kwargs):
    if isinstance(name, list):
      while len(name) != 1:
        cls(name.pop(0), env, provider, **kwargs)

      name = name[0]

    env = env or Environment.get_instance()
    provider = provider or getattr(cls, 'provider', None)

    r_type = cls.__name__
    if r_type not in env.resources:
      env.resources[r_type] = {}
    if name not in env.resources[r_type]:
      obj = super(Resource, cls).__new__(cls)
      env.resources[r_type][name] = obj
      env.resource_list.append(obj)
      return obj

    obj = env.resources[r_type][name]
    if obj.provider != provider:
      raise Fail("Duplicate resource %r with a different provider %r != %r" % (
      obj, provider, obj.provider))

    obj.override(**kwargs)
    return obj

  def __init__(self, name, env=None, provider=None, **kwargs):
    if isinstance(name, list):
      name = name.pop(0)

    if hasattr(self, 'name'):
      return

    self.env = env or Environment.get_instance()
    self.name = name

    self.provider = provider or getattr(self, 'provider', None)

    self.arguments = {}
    for key, value in kwargs.items():
      try:
        arg = self._arguments[key]
      except KeyError:
        raise Fail("%s received unsupported argument %s" % (self, key))
      else:
        try:
          self.arguments[key] = arg.validate(value)
        except InvalidArgument, exc:
          raise InvalidArgument("%s %s" % (self, exc))

    Resource.log.debug("New resource %s: %s" % (self, self.arguments))

    if not self.env.test_mode:
      self.env.run()

  def validate(self):
    pass

  def override(self, **kwargs):
    for key, value in kwargs.items():
      try:
        arg = self._arguments[key]
      except KeyError:
        raise Fail("%s received unsupported argument %s" % (self, key))
      else:
        if value != self.arguments.get(key):
          if not arg.allow_override:
            raise Fail(
              "%s doesn't allow overriding argument '%s'" % (self, key))

          try:
            self.arguments[key] = arg.validate(value)
          except InvalidArgument, exc:
            raise InvalidArgument("%s %s" % (self, exc))
    self.validate()

  def __repr__(self):
    return "%s['%s']" % (self.__class__.__name__, self.name)

  def __unicode__(self):
    return u"%s['%s']" % (self.__class__.__name__, self.name)

  def __getstate__(self):
    return dict(
      name=self.name,
      provider=self.provider,
      arguments=self.arguments,
      env=self.env,
    )

  def __setstate__(self, state):
    self.name = state['name']
    self.provider = state['provider']
    self.arguments = state['arguments']
    self.env = state['env']

    Resource.log = logging.getLogger("resource_management.resource")

    self.validate()
<commit_msg>AMBARI-3993. RMF Remove resource inheritance and enable same name for the resources (Andrew Onischuk via dlysnichenko)<commit_after>#!/usr/bin/env python2.6
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""

__all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
           "BooleanArgument"]

import logging
from resource_management.core.exceptions import Fail, InvalidArgument
from resource_management.core.environment import Environment

class ResourceArgument(object):
  def __init__(self, default=None, required=False):
    self.required = False # Prevents the initial validate from failing
    if hasattr(default, '__call__'):
      self.default = default
    else:
      self.default = self.validate(default)
    self.required = required

  def validate(self, value):
    if self.required and value is None:
      raise InvalidArgument("Required argument %s missing" % self.name)
    return value

class ForcedListArgument(ResourceArgument):
  def validate(self, value):
    value = super(ForcedListArgument, self).validate(value)
    if not isinstance(value, (tuple, list)):
      value = [value]
    return value

class BooleanArgument(ResourceArgument):
  def validate(self, value):
    value = super(BooleanArgument, self).validate(value)
    if not value in (True, False):
      raise InvalidArgument(
        "Expected a boolean for %s received %r" % (self.name, value))
    return value

class Accessor(object):
  def __init__(self, name):
    self.name = name

  def __get__(self, obj, cls):
    try:
      return obj.arguments[self.name]
    except KeyError:
      val = obj._arguments[self.name].default
      if hasattr(val, '__call__'):
        val = val(obj)
      return val

  def __set__(self, obj, value):
    obj.arguments[self.name] = obj._arguments[self.name].validate(value)

class ResourceMetaclass(type):
  # def __new__(cls, name, bases, attrs):
  #     super_new = super(ResourceMetaclass, cls).__new__
  #     return super_new(cls, name, bases, attrs)

  def __init__(mcs, _name, bases, attrs):
    mcs._arguments = getattr(bases[0], '_arguments', {}).copy()
    for key, value in list(attrs.items()):
      if isinstance(value, ResourceArgument):
        value.name = key
        mcs._arguments[key] = value
        setattr(mcs, key, Accessor(key))

class Resource(object):
  __metaclass__ = ResourceMetaclass

  log = logging.getLogger("resource_management.resource")

  action = ForcedListArgument(default="nothing")
  ignore_failures = BooleanArgument(default=False)
  not_if = ResourceArgument() # pass command e.g. not_if = ('ls','/root/jdk')
  only_if = ResourceArgument() # pass command
  initial_wait = ResourceArgument() # in seconds

  actions = ["nothing"]

  def __new__(cls, name, env=None, provider=None, **kwargs):
    if isinstance(name, list):
      while len(name) != 1:
        cls(name.pop(0), env, provider, **kwargs)

      name = name[0]

    env = env or Environment.get_instance()
    provider = provider or getattr(cls, 'provider', None)

    r_type = cls.__name__
    if r_type not in env.resources:
      env.resources[r_type] = {}

    obj = super(Resource, cls).__new__(cls)
    env.resources[r_type][name] = obj
    env.resource_list.append(obj)
    return obj

  def __init__(self, name, env=None, provider=None, **kwargs):
    if isinstance(name, list):
      name = name.pop(0)

    if hasattr(self, 'name'):
      return

    self.env = env or Environment.get_instance()
    self.name = name

    self.provider = provider or getattr(self, 'provider', None)

    self.arguments = {}
    for key, value in kwargs.items():
      try:
        arg = self._arguments[key]
      except KeyError:
        raise Fail("%s received unsupported argument %s" % (self, key))
      else:
        try:
          self.arguments[key] = arg.validate(value)
        except InvalidArgument, exc:
          raise InvalidArgument("%s %s" % (self, exc))

    Resource.log.debug("New resource %s: %s" % (self, self.arguments))

    if not self.env.test_mode:
      self.env.run()

  def validate(self):
    pass

  def __repr__(self):
    return "%s['%s']" % (self.__class__.__name__, self.name)

  def __unicode__(self):
    return u"%s['%s']" % (self.__class__.__name__, self.name)

  def __getstate__(self):
    return dict(
      name=self.name,
      provider=self.provider,
      arguments=self.arguments,
      env=self.env,
    )

  def __setstate__(self, state):
    self.name = state['name']
    self.provider = state['provider']
    self.arguments = state['arguments']
    self.env = state['env']

    Resource.log = logging.getLogger("resource_management.resource")

    self.validate()
<|endoftext|>


<commit_before>
# coding: utf-8

# In this exercise we will decode orientation using data collected for the Cognitive Neuroscience module in 2017.  The subject performed a task that manipulated whether attention was placed towards the left or right visual field, or with no attentional focus (control condition).
#
# Notes from Dan:
#
# *v1_tseries are the time series files, as voxel * volume matrices
# *v1_r2 are the variance explained per voxel by the FIR model with three conditions for task=0/1/2
# design is a long form matrix (rows are individual events, first column are volumes and second column trial type) indicating the volume at which the different trial types occurred, 0 = neutral task (press button when stimulus cross changes color), 1 = attend left side and detect the direction of rotation, 2 = attend right side and detect the direction of rotation
#
# Stimulus was two gabor patches left and right of fixation flickering at 5 Hz.
#
# fixate: 500 ms
# task cue: 500 ms
# ISI: 1000 ms
# stimulus: 4000 ms
# change+resp: 1500 ms
# var ITI: uniform distribution between 2500 and 9500 ms
#
# From staring at the deconvolutions it seems like the optimal time window to be looking at to separate trial types starts basically with the stimulus at 2 s, you start seeing task/neutral separation immediately and then by ~3-4 s you start seeing separation by left/right in most voxels.
#
# ##### Load data
#
# First we load the data files.

# In[308]:

import os,json,glob,pickle
import numpy,pandas
import nibabel
import sklearn.multiclass
from sklearn.svm import SVC
import sklearn.metrics
import sklearn.model_selection
import sklearn.preprocessing
import scipy.stats,scipy.io
import random
import seaborn
get_ipython().magic('matplotlib inline')
import matplotlib.pyplot as plt

datadir='data'

print('using data from %s'%datadir)
lv1_ts=scipy.io.loadmat(os.path.join(datadir,'lv1_tseries.mat'))['lv1']
rv1_ts=scipy.io.loadmat(os.path.join(datadir,'rv1_tseries.mat'))['rv1']
# scale the data so that we don't need to bother with intercept in the model
lv1_ts=sklearn.preprocessing.scale(lv1_ts.T)
rv1_ts=sklearn.preprocessing.scale(rv1_ts.T)
tsdata={'leftV1':lv1_ts,'rightV1':rv1_ts}

desmtx=scipy.io.loadmat(os.path.join(datadir,'design.mat'))['design']

ntrials=desmtx.shape[0]
ntp,nvox=lv1_ts.shape
print(ntrials,'trials')
print(nvox,'voxels')
print(ntp,'timepoints')
lv1_ts.shape

# In[160]:

# Reproduce the deconvolution analysis using an FIR model
# the onset times are in volumes, so we just use tr=1
# use 20-second window

def make_fir_model(onsets,tslength,hrflength=48,tr=1):
    """
    generate an FIR model design matrix
    this only works for a single condition
    """

    X=numpy.zeros((tslength,int(hrflength/tr)))
    for i in range(hrflength):
        for o in onsets:
            try:
                X[o+i,i]=1
            except:
                pass
    return X

desmtx_df=pandas.DataFrame(desmtx,columns=['condition','onset'])
onsets={}
onsets['neutral']=desmtx_df.query('condition==0').onset.values
onsets['attendleft']=desmtx_df.query('condition==1').onset.values
onsets['attendright']=desmtx_df.query('condition==2').onset.values

left_fir=make_fir_model(onsets['attendleft'],ntp)
right_fir=make_fir_model(onsets['attendright'],ntp)
neutral_fir=make_fir_model(onsets['neutral'],ntp)
fir=numpy.hstack((left_fir,right_fir,neutral_fir))

# show the design matrix
plt.imshow(fir[:400,:])
plt.axis('auto')
print(fir.shape)

# In[161]:

# estimate the model

beta_hat_left=numpy.linalg.inv(fir.T.dot(fir)).dot(fir.T).dot(lv1_ts)
beta_hat_right=numpy.linalg.inv(fir.T.dot(fir)).dot(fir.T).dot(rv1_ts)
plt.figure(figsize=(12,6))
plt.subplot(1,2,1)
plt.plot(beta_hat_left[:48].mean(1))
plt.plot(beta_hat_left[48:96].mean(1))
plt.plot(beta_hat_left[96:144].mean(1))
plt.legend(['attend left','attend right','neutral'])
plt.title('Left V1')
plt.subplot(1,2,2)
plt.plot(beta_hat_right[:48].mean(1))
plt.plot(beta_hat_right[48:96].mean(1))
plt.plot(beta_hat_right[96:144].mean(1))
plt.legend(['attend left','attend right','neutral'])
plt.title('Right V1')

pred_left=fir.dot(beta_hat_left)

# In[162]:

# check fit of the model over first 500 timepoints
plt.figure(figsize=(14,4))
plt.plot(sklearn.preprocessing.scale(lv1_ts.mean(1)[:500]))
plt.plot(sklearn.preprocessing.scale(rv1_ts.mean(1)[:500]))
meanpred=sklearn.preprocessing.scale(pred_left.mean(1))

plt.plot(meanpred[:500])
pred_left.mean(1).shape

# #### Classification analysis
#
# Now let's fit a classifier using balanced 8-fold crossvalidation. For now we only include attention trials.  We will fit the classifier at each time point along the trial timecourse.  We use a nested crossvalidation loop to determine the classifier parameters for each dataset.

# In[338]:

def run_classifier(data,labels, shuffle=False,nfolds=8,scale=True,
                  clf=None):
    """
    run classifier for a single dataset
    """
    features=data
    if scale:
        features=sklearn.preprocessing.scale(features)
    if shuffle:
        numpy.random.shuffle(labels)
    if not clf:
        clf=sklearn.svm.SVC(C=C)
    skf = sklearn.model_selection.StratifiedKFold(8)
    pred=numpy.zeros(labels.shape[0])
    for train, test in skf.split(features,labels):
        clf.fit(features[train,:],labels[train])
        pred[test]=clf.predict(features[test,:])
    acc=sklearn.metrics.accuracy_score(labels, pred)
    return acc

def get_accuracy_timeseries(tsdata,labels_attend,onsets,shuffle=False,clf=clf,window=40,
                           voxels=None):
    """
    iterate over timepoints
    """
    acc=numpy.zeros(window)
    for tp in range(window):
        # pull out data for each trial/timepoint
        if voxels is None:
            data=numpy.zeros((len(labels_attend),tsdata['leftV1'].shape[1] + tsdata['rightV1'].shape[1]))
        else:
            data=numpy.zeros((len(labels_attend),tsdata[voxels+'V1'].shape[1]))

        ctr=0
        for cond in ['attendleft','attendright']:
            for ons in onsets[cond]:
                if voxels is None:
                    data[ctr,:]=numpy.hstack((tsdata['leftV1'][ons+tp,:],tsdata['rightV1'][ons+tp,:]))
                else:
                    data[ctr,:]=tsdata[voxels+'V1'][ons+tp,:]

                ctr+=1
        acc[tp]=run_classifier(data,labels_attend,clf=clf,shuffle=shuffle)
    return acc

labels_attend=numpy.array([i for i in labels if i > 0])

#clf=sklearn.linear_model.LogisticRegressionCV(penalty='l1',solver='liblinear')
#clf=sklearn.svm.SVC(C=1)
tuned_parameters = [{'C': [0.5,1,5,10,20]}]
clf = sklearn.model_selection.GridSearchCV(sklearn.svm.SVC(C=1,kernel='linear'), tuned_parameters, cv=5)

acc_all=get_accuracy_timeseries(tsdata,labels_attend,onsets,clf=clf)
acc_left=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='left',clf=clf)
acc_right=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='right',clf=clf)

# #### Plot the results

# In[339]:

plt.figure(figsize=(14,5))
plt.subplot(1,3,1)
plt.plot(numpy.arange(0,20,0.5),acc_all)
plt.axis([0,20,0,1])
plt.plot([0,20],[0.5,0.5],'k--')
plt.title('All voxels')
plt.xlabel('Timepoints (0.5 secs each)')
plt.ylabel('Pecent classification accuracy')

plt.subplot(1,3,2)
plt.plot(numpy.arange(0,20,0.5),acc_left)
plt.axis([0,20,0,1])
plt.plot([0,20],[0.5,0.5],'k--')
plt.title('Left V1')
plt.xlabel('Timepoints (0.5 secs each)')
plt.ylabel('Pecent classification accuracy')

plt.subplot(1,3,3)
plt.plot(numpy.arange(0,20,0.5),acc_right)
plt.axis([0,20,0,1])
plt.plot([0,20],[0.5,0.5],'k--')
plt.title('Right V1')
plt.xlabel('Timepoints (0.5 secs each)')
plt.ylabel('Pecent classification accuracy')

# Now let's run it with the labels shuffled 100 times to see how good these results are compared to chance.  This will take a few minutes to complete. For a real analysis one would want to do this many more times (up to ~5000) in order for the distribution of extreme values to stabilize.

# In[ ]:

# if the saved results already exist then just reload them, to save time
if os.path.exists('shuffled_accuracy.pkl'):
    acc_all_rand,acc_left_rand,acc_right_rand,clf=pickle.load(open('shuffled_accuracy.pkl','rb'))
else:
    acc_all_rand=numpy.zeros((100,40))
    acc_left_rand=numpy.zeros((100,40))
    acc_right_rand=numpy.zeros((100,40))

    for i in range(100):
        acc_all_rand[i,:]=get_accuracy_timeseries(tsdata,labels_attend,onsets,shuffle=True,clf=clf)
        acc_left_rand[i,:]=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='left',shuffle=True,clf=clf)
        acc_right_rand[i,:]=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='right',shuffle=True,clf=clf)
    pickle.dump((acc_all_rand,acc_left_rand,acc_right_rand,clf),open('shuffled_accuracy.pkl','wb'))

# Now we plot those results alongside the true classification results, adding an asterisk at the timepoints where the observed accuracy is greater than the 99th percentile of the random accuracies.

# In[ ]:

rand_percentile=99 # percent cutoff for randomization

plt.figure(figsize=(14,5))
plt.subplot(1,3,1)
plt.plot(numpy.arange(0,20,0.5),acc_all)
plt.axis([0,20,0,1])
plt.plot([0,20],[0.5,0.5],'k--')
plt.title('All voxels')
plt.xlabel('Timepoints (0.5 secs each)')
plt.ylabel('Pecent classification accuracy')
for i in range(100):
    plt.plot(numpy.arange(0,20,0.5),acc_all_rand[i,:],'r',linewidth=0.1)
cutoff=numpy.zeros(40)
for i in range(40):
    cutoff[i]=scipy.stats.scoreatpercentile(acc_all_rand[:,i],rand_percentile)
    if acc_all[i]>cutoff[i]:
        plt.text(i/2,0.9,'*')

plt.subplot(1,3,2)
plt.plot(numpy.arange(0,20,0.5),acc_left)
plt.axis([0,20,0,1])
plt.plot([0,20],[0.5,0.5],'k--')
plt.title('Left V1')
plt.xlabel('Timepoints (0.5 secs each)')
plt.ylabel('Pecent classification accuracy')
for i in range(100):
    plt.plot(numpy.arange(0,20,0.5),acc_left_rand[i,:],'r',linewidth=0.1)
cutoff=numpy.zeros(40)
for i in range(40):
    cutoff[i]=scipy.stats.scoreatpercentile(acc_left_rand[:,i],rand_percentile)
    if acc_left[i]>cutoff[i]:
        plt.text(i/2,0.9,'*')

plt.subplot(1,3,3)
plt.plot(numpy.arange(0,20,0.5),acc_right)
plt.axis([0,20,0,1])
plt.plot([0,20],[0.5,0.5],'k--')
plt.title('Right V1')
plt.xlabel('Timepoints (0.5 secs each)')
plt.ylabel('Pecent classification accuracy')
for i in range(100):
    plt.plot(numpy.arange(0,20,0.5),acc_right_rand[i,:],'r',linewidth=0.1)
cutoff=numpy.zeros(40)
for i in range(40):
    cutoff[i]=scipy.stats.scoreatpercentile(acc_right_rand[:,i],rand_percentile)
    if acc_right[i]>cutoff[i]:
        plt.text(i/2,0.9,'*')

# In[ ]:

<commit_msg>removed plotting<commit_after>
# coding: utf-8

# In this exercise we will decode orientation using data collected for the Cognitive Neuroscience module in 2017.  The subject performed a task that manipulated whether attention was placed towards the left or right visual field, or with no attentional focus (control condition).
#
# Notes from Dan:
#
# *v1_tseries are the time series files, as voxel * volume matrices
# *v1_r2 are the variance explained per voxel by the FIR model with three conditions for task=0/1/2
# design is a long form matrix (rows are individual events, first column are volumes and second column trial type) indicating the volume at which the different trial types occurred, 0 = neutral task (press button when stimulus cross changes color), 1 = attend left side and detect the direction of rotation, 2 = attend right side and detect the direction of rotation
#
# Stimulus was two gabor patches left and right of fixation flickering at 5 Hz.
#
# fixate: 500 ms
# task cue: 500 ms
# ISI: 1000 ms
# stimulus: 4000 ms
# change+resp: 1500 ms
# var ITI: uniform distribution between 2500 and 9500 ms
#
# From staring at the deconvolutions it seems like the optimal time window to be looking at to separate trial types starts basically with the stimulus at 2 s, you start seeing task/neutral separation immediately and then by ~3-4 s you start seeing separation by left/right in most voxels.
#
# ##### Load data
#
# First we load the data files.

# In[308]:

import os,json,glob,pickle
import numpy,pandas
import nibabel
import sklearn.multiclass
from sklearn.svm import SVC
import sklearn.metrics
import sklearn.model_selection
import sklearn.preprocessing
import scipy.stats,scipy.io
import random
import seaborn

datadir='data'

print('using data from %s'%datadir)
lv1_ts=scipy.io.loadmat(os.path.join(datadir,'lv1_tseries.mat'))['lv1']
rv1_ts=scipy.io.loadmat(os.path.join(datadir,'rv1_tseries.mat'))['rv1']
# scale the data so that we don't need to bother with intercept in the model
lv1_ts=sklearn.preprocessing.scale(lv1_ts.T)
rv1_ts=sklearn.preprocessing.scale(rv1_ts.T)
tsdata={'leftV1':lv1_ts,'rightV1':rv1_ts}

desmtx=scipy.io.loadmat(os.path.join(datadir,'design.mat'))['design']

ntrials=desmtx.shape[0]
ntp,nvox=lv1_ts.shape
print(ntrials,'trials')
print(nvox,'voxels')
print(ntp,'timepoints')
lv1_ts.shape

# In[160]:

# Reproduce the deconvolution analysis using an FIR model
# the onset times are in volumes, so we just use tr=1
# use 20-second window

def make_fir_model(onsets,tslength,hrflength=48,tr=1):
    """
    generate an FIR model design matrix
    this only works for a single condition
    """

    X=numpy.zeros((tslength,int(hrflength/tr)))
    for i in range(hrflength):
        for o in onsets:
            try:
                X[o+i,i]=1
            except:
                pass
    return X

desmtx_df=pandas.DataFrame(desmtx,columns=['condition','onset'])
onsets={}
onsets['neutral']=desmtx_df.query('condition==0').onset.values
onsets['attendleft']=desmtx_df.query('condition==1').onset.values
onsets['attendright']=desmtx_df.query('condition==2').onset.values

left_fir=make_fir_model(onsets['attendleft'],ntp)
right_fir=make_fir_model(onsets['attendright'],ntp)
neutral_fir=make_fir_model(onsets['neutral'],ntp)
fir=numpy.hstack((left_fir,right_fir,neutral_fir))

# show the design matrix
plt.imshow(fir[:400,:])
plt.axis('auto')
print(fir.shape)

# In[161]:

# estimate the model

beta_hat_left=numpy.linalg.inv(fir.T.dot(fir)).dot(fir.T).dot(lv1_ts)
beta_hat_right=numpy.linalg.inv(fir.T.dot(fir)).dot(fir.T).dot(rv1_ts)

pred_left=fir.dot(beta_hat_left)

# In[162]:

# check fit of the model over first 500 timepoints
meanpred=sklearn.preprocessing.scale(pred_left.mean(1))

plt.plot(meanpred[:500])
pred_left.mean(1).shape

# #### Classification analysis
#
# Now let's fit a classifier using balanced 8-fold crossvalidation. For now we only include attention trials.  We will fit the classifier at each time point along the trial timecourse.  We use a nested crossvalidation loop to determine the classifier parameters for each dataset.

# In[338]:

def run_classifier(data,labels, shuffle=False,nfolds=8,scale=True,
                  clf=None):
    """
    run classifier for a single dataset
    """
    features=data
    if scale:
        features=sklearn.preprocessing.scale(features)
    if shuffle:
        numpy.random.shuffle(labels)
    if not clf:
        clf=sklearn.svm.SVC(C=C)
    skf = sklearn.model_selection.StratifiedKFold(8)
    pred=numpy.zeros(labels.shape[0])
    for train, test in skf.split(features,labels):
        clf.fit(features[train,:],labels[train])
        pred[test]=clf.predict(features[test,:])
    acc=sklearn.metrics.accuracy_score(labels, pred)
    return acc

def get_accuracy_timeseries(tsdata,labels_attend,onsets,shuffle=False,clf=clf,window=40,
                           voxels=None):
    """
    iterate over timepoints
    """
    acc=numpy.zeros(window)
    for tp in range(window):
        # pull out data for each trial/timepoint
        if voxels is None:
            data=numpy.zeros((len(labels_attend),tsdata['leftV1'].shape[1] + tsdata['rightV1'].shape[1]))
        else:
            data=numpy.zeros((len(labels_attend),tsdata[voxels+'V1'].shape[1]))

        ctr=0
        for cond in ['attendleft','attendright']:
            for ons in onsets[cond]:
                if voxels is None:
                    data[ctr,:]=numpy.hstack((tsdata['leftV1'][ons+tp,:],tsdata['rightV1'][ons+tp,:]))
                else:
                    data[ctr,:]=tsdata[voxels+'V1'][ons+tp,:]

                ctr+=1
        acc[tp]=run_classifier(data,labels_attend,clf=clf,shuffle=shuffle)
    return acc

labels_attend=numpy.array([i for i in labels if i > 0])

#clf=sklearn.linear_model.LogisticRegressionCV(penalty='l1',solver='liblinear')
#clf=sklearn.svm.SVC(C=1)
tuned_parameters = [{'C': [0.5,1,5,10,20]}]
clf = sklearn.model_selection.GridSearchCV(sklearn.svm.SVC(C=1,kernel='linear'), tuned_parameters, cv=5)

acc_all=get_accuracy_timeseries(tsdata,labels_attend,onsets,clf=clf)
acc_left=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='left',clf=clf)
acc_right=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='right',clf=clf)

# #### Plot the results

# In[339]:

# Now let's run it with the labels shuffled 100 times to see how good these results are compared to chance.  This will take a few minutes to complete. For a real analysis one would want to do this many more times (up to ~5000) in order for the distribution of extreme values to stabilize.

# In[ ]:

# if the saved results already exist then just reload them, to save time
if os.path.exists('shuffled_accuracy.pkl'):
    acc_all_rand,acc_left_rand,acc_right_rand,clf=pickle.load(open('shuffled_accuracy.pkl','rb'))
else:
    acc_all_rand=numpy.zeros((100,40))
    acc_left_rand=numpy.zeros((100,40))
    acc_right_rand=numpy.zeros((100,40))

    for i in range(100):
        acc_all_rand[i,:]=get_accuracy_timeseries(tsdata,labels_attend,onsets,shuffle=True,clf=clf)
        acc_left_rand[i,:]=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='left',shuffle=True,clf=clf)
        acc_right_rand[i,:]=get_accuracy_timeseries(tsdata,labels_attend,onsets,voxels='right',shuffle=True,clf=clf)
    pickle.dump((acc_all_rand,acc_left_rand,acc_right_rand,clf),open('shuffled_accuracy.pkl','wb'))

# Now we plot those results alongside the true classification results, adding an asterisk at the timepoints where the observed accuracy is greater than the 99th percentile of the random accuracies.

# In[ ]:

rand_percentile=99 # percent cutoff for randomization
<|endoftext|>