<commit_before># Load in core dependencies
import os
import shutil
import subprocess
import tempfile
import unittest

# Load in 3rd party dependencies
from jinja2 import Template

# Set up constants
__dir__ = os.path.dirname(os.path.abspath(__file__))

# Set up helper fn
def template(tmpl_path):
    """ Decorator that templates the returned content. """
    # Pre-emptively read in the template
    tmpl = None
    with open(tmpl_path) as f:
        tmpl = Template(f.read())

    # Define our templating wrapper fn
    def decorator_fn(fn):
        def templator_fn(*args, **kwargs):
            # Run the normal function
            data = fn(*args, **kwargs)

            # Render the info
            return tmpl.render(**data)
        return templator_fn
    return decorator_fn

class Base(object):
    # Determine the plugins directory
    plugin_test_dir = os.path.expanduser('~/.config/sublime-text-2/Packages/sublime-plugin-tests-tmp')

    @classmethod
    def ensure_plugin_test_dir(cls):
        # If the plugin test directory does not exist, create it
        if not os.path.exists(cls.plugin_test_dir):
            os.makedirs(cls.plugin_test_dir)

    @classmethod
    def ensure_launcher(cls):
        # Ensure the plugin test directory exists
        cls.ensure_plugin_test_dir()

        # If command.py doesn't exist, copy it
        orig_command_path = __dir__ + '/command.py'
        dest_command_path = cls.plugin_test_dir + '/command.py'
        if not os.path.exists(dest_command_path):
            shutil.copyfile(orig_command_path, dest_command_path)
        else:
        # Otherwise...
            # If there are updates for command.py
            expected_command = None
            with open(orig_command_path) as f:
                expected_command = f.read()
            actual_command = None
            with open(dest_command_path) as f:
                actual_command = f.read()
            if expected_command != actual_command:
                # Update the file
                shutil.copyfile(orig_command_path, dest_command_path)

                # and notify the user we must restart Sublime
                raise Exception('We had to update the test launcher plugin. You must close or restart Sublime to continue testing.')

        # TODO: Use similar copy model minus the exception
        # TODO: If we overwrite utils, be sure to wait so that changes for import get picked up
        if not os.path.exists(cls.plugin_test_dir + '/utils'):
            shutil.copytree(__dir__ + '/utils', cls.plugin_test_dir + '/utils')

        # Notify the user that the launcher exists
        return True

    @classmethod
    def _run_test(cls, test_str):
        # Guarantee there is an output directory and launcher
        cls.ensure_launcher()

        # Reserve an output file
        output_file = tempfile.mkstemp()[1]

        # Template plugin
        plugin_runner = None
        with open(__dir__ + '/templates/plugin_runner.py') as f:
            runner_template = Template(f.read())
            plugin_runner = runner_template.render(output_file=output_file)

        # Output plugin_runner to directory
        with open(cls.plugin_test_dir + '/plugin_runner.py', 'w') as f:
            f.write(plugin_runner)

        # Output test to directory
        with open(cls.plugin_test_dir + '/plugin.py', 'w') as f:
            f.write(test_str)

        # Start a subprocess to run the plugin
        # TODO: We might want a development mode (runs commands inside local sublime window) and a testing mode (calls out to Vagrant box)
        # TODO: or at least 2 plugin hooks, one for CLI based testing and one for internal dev
        # TODO: Rename tmp_test command
        subprocess.call(['sublime_text', '--command', 'tmp_test'])

        # TODO: How does this work if `tmp_test` is theoretically run in parallel

        # Read in the output
        with open(output_file) as f:
            # Read, parse, and return the result
            result = f.read()
            result_lines = result.split('\n')
            return {
                'raw_result': result,
                'success': result_lines[0] == 'SUCCESS',
                'meta_info': '\n'.join(result_lines[1:])
            }

class TestCase(unittest.TestCase, Base):
    def __call__(self, result=None):
        # For each test
        loader = unittest.TestLoader()
        for test_name in loader.getTestCaseNames(self.__class__):
            # Wrap the function
            test_fn = getattr(self, test_name)
            print test_name
            wrapped_test = self._wrap_test(test_fn)
            setattr(self, test_name, wrapped_test)

        # Call the original function
        unittest.TestCase.__call__(self, result)

    def _wrap_test(self, test_fn):
        # Generate a wrapped function
        def wrapped_fn(*args, **kwargs):
            # Get the test info
            test_str = test_fn(*args, **kwargs)

            # Run the test and process the result
            result = self._run_test(test_str)
            success = result['success']
            failure_reason = result['meta_info'] or 'Test failed'

            # Assert we were successful
            self.assertTrue(success, failure_reason)

        # Return the wrapped function
        return wrapped_fn

<commit_msg>Trying to track down quadruple calls but no luck<commit_after># Load in core dependencies
import os
import shutil
import subprocess
import tempfile
import unittest

# Load in 3rd party dependencies
from jinja2 import Template

# Set up constants
__dir__ = os.path.dirname(os.path.abspath(__file__))

# Set up helper fn
def template(tmpl_path):
    """ Decorator that templates the returned content. """
    # Pre-emptively read in the template
    tmpl = None
    with open(tmpl_path) as f:
        tmpl = Template(f.read())

    # Define our templating wrapper fn
    def decorator_fn(fn):
        def templator_fn(*args, **kwargs):
            # Run the normal function
            data = fn(*args, **kwargs)

            # Render the info
            return tmpl.render(**data)
        return templator_fn
    return decorator_fn

class Base(object):
    # Determine the plugins directory
    plugin_test_dir = os.path.expanduser('~/.config/sublime-text-2/Packages/sublime-plugin-tests-tmp')

    @classmethod
    def ensure_plugin_test_dir(cls):
        # If the plugin test directory does not exist, create it
        if not os.path.exists(cls.plugin_test_dir):
            os.makedirs(cls.plugin_test_dir)

    @classmethod
    def ensure_launcher(cls):
        # Ensure the plugin test directory exists
        cls.ensure_plugin_test_dir()

        # If command.py doesn't exist, copy it
        orig_command_path = __dir__ + '/command.py'
        dest_command_path = cls.plugin_test_dir + '/command.py'
        if not os.path.exists(dest_command_path):
            shutil.copyfile(orig_command_path, dest_command_path)
        else:
        # Otherwise...
            # If there are updates for command.py
            expected_command = None
            with open(orig_command_path) as f:
                expected_command = f.read()
            actual_command = None
            with open(dest_command_path) as f:
                actual_command = f.read()
            if expected_command != actual_command:
                # Update the file
                shutil.copyfile(orig_command_path, dest_command_path)

                # and notify the user we must restart Sublime
                raise Exception('We had to update the test launcher plugin. You must close or restart Sublime to continue testing.')

        # TODO: Use similar copy model minus the exception
        # TODO: If we overwrite utils, be sure to wait so that changes for import get picked up
        if not os.path.exists(cls.plugin_test_dir + '/utils'):
            shutil.copytree(__dir__ + '/utils', cls.plugin_test_dir + '/utils')

        # Notify the user that the launcher exists
        return True

    @classmethod
    def _run_test(cls, test_str):
        # Guarantee there is an output directory and launcher
        cls.ensure_launcher()

        # Reserve an output file
        output_file = tempfile.mkstemp()[1]

        # Template plugin
        plugin_runner = None
        with open(__dir__ + '/templates/plugin_runner.py') as f:
            runner_template = Template(f.read())
            plugin_runner = runner_template.render(output_file=output_file)

        # Output plugin_runner to directory
        with open(cls.plugin_test_dir + '/plugin_runner.py', 'w') as f:
            f.write(plugin_runner)

        # Output test to directory
        with open(cls.plugin_test_dir + '/plugin.py', 'w') as f:
            f.write(test_str)

        # Start a subprocess to run the plugin
        # TODO: We might want a development mode (runs commands inside local sublime window) and a testing mode (calls out to Vagrant box)
        # TODO: or at least 2 plugin hooks, one for CLI based testing and one for internal dev
        # TODO: Rename tmp_test command
        subprocess.call(['sublime_text', '--command', 'tmp_test'])

        # TODO: How does this work if `tmp_test` is theoretically run in parallel

        # Read in the output
        with open(output_file) as f:
            # Read, parse, and return the result
            result = f.read()
            result_lines = result.split('\n')
            return {
                'raw_result': result,
                'success': result_lines[0] == 'SUCCESS',
                'meta_info': '\n'.join(result_lines[1:])
            }

class TestCase(unittest.TestCase, Base):
    def __init__(self, result=None):
        # For each test
        loader = unittest.TestLoader()
        print 'a'
        for test_name in loader.getTestCaseNames(self.__class__):
            # Wrap the function
            test_fn = getattr(self, test_name)
            print test_name
            wrapped_test = self._wrap_test(test_fn)
            setattr(self, test_name, wrapped_test)

        # Call the original function
        unittest.TestCase.__init__(self, result)

    def _wrap_test(self, test_fn):
        # Generate a wrapped function
        def wrapped_fn(*args, **kwargs):
            # Get the test info
            test_str = test_fn(*args, **kwargs)

            # Run the test and process the result
            result = self._run_test(test_str)
            success = result['success']
            failure_reason = result['meta_info'] or 'Test failed'

            # Assert we were successful
            self.assertTrue(success, failure_reason)

        # Return the wrapped function
        return wrapped_fn

<|endoftext|>


<commit_before>from sklearn import datasets

from sklearn.cluster import KMeans
# import numpy
import traceback
from submissions.Ottenlips import billionaires

from submissions.Ottenlips import billionaires

class DataFrame:
    data = []
    feature_names = []
    target = []
    target_names = []

bill = DataFrame()

list_of_billionaire = billionaires.get_billionaires()

def billtarget(age):
    if age<40:
        return 1
    else:
        return 0

for billionaires in list_of_billionaire:
    # print(billionaires['wealth']['type'])
    # print(billionaires)
    bill.target.append(billionaires['demographics']['age'])
    # bill.target.append(billionaires['wealth']['how']['inherited'])
    bill.data.append([
         float( billionaires['wealth']['worth in billions']),
        #float(billionaires['demographics']['age']),
          # float(billionaires['location']['gdp']),
         float(billionaires['rank']),
    ])

bill.feature_names = [
    # 'age',
    'wealth'
    # 'gdp of origin country',
    'rank',
]

bill.target_names = [
    'Young',
    'Old',
]

'''
Make a customn classifier,
'''
km = KMeans(
    n_clusters=15,
     # max_iter=3000,
    #  n_init=10,
    # init='k-means++',
    #  algorithm='auto',
    #  precompute_distances='auto',
    # tol=1e-4,
    #  n_jobs=-1,
    #  random_state=numpy.RandomState,
    #  verbose=1,
    #  copy_x=True,
)

billScaled = DataFrame()

def setupScales(grid):
    global min, max
    min = list(grid[0])
    max = list(grid[0])
    for row in range(1, len(grid)):
        for col in range(len(grid[row])):
            cell = grid[row][col]
            if cell < min[col]:
                min[col] = cell
            if cell > max[col]:
                max[col] = cell

def scaleGrid(grid):
    newGrid = []
    for row in range(len(grid)):
        newRow = []
        for col in range(len(grid[row])):
            try:
                cell = grid[row][col]
                scaled = (cell - min[col]) \
                         / (max[col] - min[col])
                newRow.append(scaled)
            except:
                pass
        newGrid.append(newRow)
    return newGrid

setupScales(bill.data)
billScaled.data = scaleGrid(bill.data)
billScaled.feature_names = bill.feature_names
billScaled.target = bill.target
billScaled.target_names = bill.target_names

Examples = {
    'Billdefault': {
        'frame': bill,
    },

   'BillKMClassifier': {
        'frame': bill,
        'kmeans': km,
},

}

<commit_msg>results make sense now, but there is not enough feature data to predict the target<commit_after>from sklearn import datasets

from sklearn.cluster import KMeans
# import numpy
import traceback
from submissions.Ottenlips import billionaires

from submissions.Ottenlips import billionaires

class DataFrame:
    data = []
    feature_names = []
    target = []
    target_names = []

bill = DataFrame()

list_of_billionaire = billionaires.get_billionaires()

def billTarget(string):
    if(billionaires['demographics']['age']<30 and billionaires['demographics']['age'] != -1):
        return 1
    elif(billionaires['demographics']['age'] != -1):
        return 0
    else:
        return 2

for billionaires in list_of_billionaire:
    # print(billionaires['wealth']['type'])
    # print(billionaires)
    bill.target.append(billionaires['demographics']['age'])
    # bill.target.append(billionaires['wealth']['how']['inherited'])
    bill.data.append([
         float( billionaires['wealth']['worth in billions']),
        #float(billionaires['demographics']['age']),
          # float(billionaires['location']['gdp']),
         float(billionaires['rank']),
    ])

bill.feature_names = [
    # 'age',
    'wealth'
    # 'gdp of origin country',
    'rank',
]

bill.target_names = [
    'old',
    'young',
    'age not listed'
]

'''
Make a customn classifier,
'''
km = KMeans(
    n_clusters=3,
      # max_iter=3000,
     # n_init=1,
     init='k-means++',
    #  algorithm='auto',
      precompute_distances='auto',
    #  tol=1e-4,
    #  n_jobs=-1,
    #  random_state=numpy.RandomState,
    #  verbose=1,
      copy_x=True,
)

billScaled = DataFrame()

def setupScales(grid):
    global min, max
    min = list(grid[0])
    max = list(grid[0])
    for row in range(1, len(grid)):
        for col in range(len(grid[row])):
            cell = grid[row][col]
            if cell < min[col]:
                min[col] = cell
            if cell > max[col]:
                max[col] = cell

def scaleGrid(grid):
    newGrid = []
    for row in range(len(grid)):
        newRow = []
        for col in range(len(grid[row])):
            try:
                cell = grid[row][col]
                scaled = (cell - min[col]) \
                         / (max[col] - min[col])
                newRow.append(scaled)
            except:
                pass
        newGrid.append(newRow)
    return newGrid

setupScales(bill.data)
billScaled.data = scaleGrid(bill.data)
billScaled.feature_names = bill.feature_names
billScaled.target = bill.target
billScaled.target_names = bill.target_names

Examples = {
    'Billdefault': {
        'frame': bill,
    },

   'BillKMClassifier': {
        'frame': bill,
        'kmeans': km,
},

}

<|endoftext|>


<commit_before>from collections import defaultdict
from simuvex.s_ref import RefTypes, SimRegRead, SimRegWrite, SimTmpRead, SimTmpWrite, SimMemRead, SimMemWrite
from simuvex import SimIRSB, SimProcedure
import logging

l = logging.getLogger("angr.ddg")
l.setLevel(logging.DEBUG)

class DDG(object):
    def __init__(self, cfg, entry_point):
        self._cfg = cfg
        self._entry_point = entry_point

        self._ddg = defaultdict(lambda:defaultdict(set))
        self._symbolic_mem_ops = set()

    def debug_print(self):
        l.debug(self._ddg)

    def _trace_source(self, init_run, init_ref):
        '''
        Trace the sources (producers) of a given ref
        '''
        l.debug("Tracing source for symbolic memory dep %s of %s", init_ref, init_run)
        sources = set()
        # Memorization
        traced = {}

        init_stmt_id = init_ref.stmt_idx
        init_reg_deps = set()
        init_tmp_deps = set()
        real_ref = init_run.statements[init_stmt_id].refs[-1]
        init_reg_deps |= set(list(real_ref.data_reg_deps))
        init_tmp_deps |= set(list(real_ref.data_tmp_deps))
        if type(real_ref) == SimMemWrite:
            init_reg_deps |= set(list(real_ref.addr_reg_deps))
            init_tmp_deps |= set(list(real_ref.addr_tmp_deps))
        stack = [(init_run, init_stmt_id, init_reg_deps, init_tmp_deps)]
        while len(stack) > 0:
            run, stmt_id, reg_deps, tmp_deps = stack.pop()
            l.debug("Traversing %s", run)
            # Added to traced set
            traced[run.addr] = reg_deps
            reg_dep_to_stmt_id = {}

            if isinstance(run, SimIRSB):
                irsb = run
                if stmt_id == -1:
                    stmt_ids = range(len(irsb.statements))
                else:
                    stmt_ids = range(stmt_id + 1)
                stmt_ids.reverse()

                for stmt_id in stmt_ids:
                    stmt = irsb.statements[stmt_id]
                    if len(stmt.refs) > 0:
                        real_ref = stmt.refs[-1]
                        if type(real_ref) == SimRegWrite and real_ref.offset in reg_deps:
                            reg_dep_to_stmt_id[real_ref.offset] = stmt_id
                            reg_deps.remove(real_ref.offset)
                            reg_deps |= set(list(real_ref.data_reg_deps))
                            tmp_deps |= set(list(real_ref.data_tmp_deps))
                        elif type(real_ref) == SimTmpWrite and real_ref.tmp in tmp_deps:
                            tmp_deps.remove(real_ref.tmp)
                            reg_deps |= set(list(real_ref.data_reg_deps))
                            tmp_deps |= set(list(real_ref.data_tmp_deps))
            elif isinstance(run, SimProcedure):
                refs = run.refs()
                refs.reverse()
                for ref in refs:
                    if type(ref) == SimRegWrite and ref.offset in reg_deps:
                        reg_dep_to_stmt_id[ref.offset] = -1
                        reg_deps.remove(ref.offset)
                        reg_deps |= set(list(ref.data_reg_deps))
                        tmp_deps |= set(list(ref.data_tmp_deps))
                    elif type(ref) == SimTmpWrite and ref.tmp in tmp_deps:
                        tmp_deps.remove(ref.tmp)
                        reg_deps |= set(list(ref.data_reg_deps))
                        tmp_deps |= set(list(ref.data_tmp_deps))

            if len(reg_deps) > 0:
                predecessors = self._cfg.get_predecessors(run)
                for s in predecessors:
                    tpl = (s.addr, reg_deps)
                    if s.addr not in traced:
                        stack.append((s, -1, reg_deps.copy(), set()))
                    else:
                        old_reg_deps = traced[s.addr]
                        if not reg_deps.issubset(old_reg_deps):
                            new_reg_deps = reg_deps.copy()
                            new_reg_deps |= old_reg_deps
                            stack.append((s, -1, new_reg_deps, set()))
            for reg, stmt_id in reg_dep_to_stmt_id.items():
                sources.add((run.addr, stmt_id))
            if len(reg_deps) == 0 or len(predecessors) == 0:
                # This is the end of the u-d chain
                # FIXME: What if there is a loop, and len(successors) never == 0?
                for reg in reg_deps:
                    if reg not in reg_dep_to_stmt_id:
                        # It's never been written before
                        # We use a fake run.addr and stmt_id so that we don't lose them
                        l.debug("Register %d has never been assigned a value before.", reg)
                        sources.add((-1 * reg, -1))
                    else:
                        # sources.add((run.addr, reg_dep_to_stmt_id[reg]))
                        # They have already been added to sources
                        pass

        return sources

    def _solve_symbolic_mem_operations(self):
        '''
        We try to resolve symbolic memory operations in the following manner:
        For each memory operation, trace the pointer from its root producers.
        And then relate each memory read with all memory writes that has shared
        producers with it.
        It's imprecise and could be over-approximating, but it's better than
        losing dependencies.
        '''
        # irsb, stmt_id => list of tuples (irsb, ref)
        mem_read_producers = defaultdict(list)
        mem_write_producers = defaultdict(list)
        for irsb, ref in self._symbolic_mem_ops:
            sources = self._trace_source(irsb, ref)
            for src_irsb_addr, stmt_id in sources:
                if type(ref) == SimMemRead:
                    mem_read_producers[(src_irsb_addr, stmt_id)].append((irsb.addr, ref))
                elif type(ref) == SimMemWrite:
                    mem_write_producers[(src_irsb_addr, stmt_id)].append((irsb.addr, ref))

        for tpl, lst_writes in mem_write_producers.items():
            if tpl in mem_read_producers:
                lst_reads = mem_read_producers[tpl]
                for read_irsb, read_ref in lst_reads:
                    for write_irsb, write_ref in lst_writes:
                        self._ddg[read_irsb][read_ref.stmt_idx].add((write_irsb, write_ref.stmt_idx))
        import ipdb
        ipdb.set_trace()

    def construct(self):
        worklist = set()
        # Added the first container into the worklist
        initial_container = AddrToRefContainer(self._cfg.get_irsb((None, None, self._entry_point)), defaultdict(set))
        worklist.add(initial_container)
        analyzed_runs = set()
        analyzed_containers = {}
        while len(worklist) > 0:
            container = worklist.pop()
            run = container.run
            # If we updated our addr_to_ref map, we should set redo_flag to
            # True, then all of its successors will be reanalyzed
            redo_flag = container.reanalyze_successors
            if isinstance(run, SimIRSB):
                irsb = run
                l.debug("Running %s", irsb)
                # Simulate the execution of this irsb.
                # For MemWriteRef, fill the addr_to_ref dict with every single concretizable
                # memory address, and ignore those symbolic ones
                # For MemReadRef, get its related MemoryWriteRef from our dict
                # TODO: Is it possible to trace memory operations even if the memory is not
                # concretizable itself?
                statements = irsb.statements
                for i in range(len(statements)):
                    stmt = statements[i]
                    refs = stmt.refs
                    if len(refs) > 0:
                        real_ref = refs[len(refs) - 1]
                        if type(real_ref) == SimMemWrite:
                            addr = real_ref.addr
                            if not addr.is_symbolic():
                                concrete_addr = addr.any()
                                tpl = (irsb.addr, i)
                                if tpl not in container.addr_to_ref[concrete_addr]:
                                    container.addr_to_ref[concrete_addr].add(tpl)
                                    redo_flag = True
                            else:
                                self._symbolic_mem_ops.add((irsb, real_ref))
                    for ref in refs:
                        if type(ref) == SimMemRead:
                            addr = ref.addr
                            if not addr.is_symbolic():
                                concrete_addr = addr.any()
                                if concrete_addr in container.addr_to_ref:
                                    self._ddg[irsb.addr][i] |= (container.addr_to_ref[concrete_addr])
                            else:
                                self._symbolic_mem_ops.add((irsb, ref))
            elif isinstance(run, SimProcedure):
                sim_proc = run
                l.debug("Running %s", sim_proc)
                refs = sim_proc.refs()
                for ref in refs:
                    if isinstance(ref, SimMemRead):
                        addr = ref.addr
                        if not addr.is_symbolic():
                            concrete_addr = addr.any()
                            # print sim_proc
                            # print "0x%08x" % concrete_addr
                            if concrete_addr in container.addr_to_ref:
                                self._ddg[sim_proc.addr][-1] |= (container.addr_to_ref[concrete_addr])
                                # print "In list"
                        else:
                            self._symbolic_mem_ops.add((sim_proc, ref))
                    elif isinstance(ref, SimMemWrite):
                        addr = ref.addr
                        if not addr.is_symbolic():
                            concrete_addr = addr.any()
                            tpl = (sim_proc.addr, -1)
                            if tpl not in container.addr_to_ref[concrete_addr]:
                                container.addr_to_ref[concrete_addr].add((sim_proc.addr, -1))
                                redo_flag = True
                        else:
                            self._symbolic_mem_ops.add((sim_proc, ref))

            analyzed_runs.add(run)
            analyzed_containers[run] = container

            # Get successors of the current irsb,
            successors = self._cfg.get_successors(run)
            if redo_flag:
                for successor in successors:
                    if successor in analyzed_runs:
                        analyzed_runs.remove(successor)
            # ... and add them to our worklist with a shallow copy of the addr_to_ref dict
            for successor in successors:
                if successor not in analyzed_runs:
                    # Let's see if there is any changes in the addresses dict
                    new_addr_to_ref = container.addr_to_ref.copy()
                    new_container = None
                    if successor in analyzed_containers:
                        old_container = analyzed_containers[successor]
                        new_container = AddrToRefContainer(successor, old_container.addr_to_ref, reanalyze_successors=True)
                        if not new_container.combine(container.addr_to_ref):
                            # There isn't any new addresses
                            continue
                    if new_container is None:
                        new_container = AddrToRefContainer(successor, container.addr_to_ref.copy())
                    worklist.add(new_container)

        self._solve_symbolic_mem_operations()
        import ipdb
        ipdb.set_trace()

class AddrToRefContainer(object):
    def __init__(self, run, addr_to_ref, reanalyze_successors=False):
        self.run = run
        self.addr_to_ref = addr_to_ref
        self.reanalyze_successors = reanalyze_successors

    def combine(self, new_addr_to_ref):
        altered = False
        for k, v in new_addr_to_ref.items():
            if k not in self.addr_to_ref:
                self.addr_to_ref[k] = v.copy()
                altered = True
            else:
                if not v.issubset(self.addr_to_ref[k]):
                    altered = True
                    self.addr_to_ref[k] |= v
        return altered
<commit_msg>Removed debugging breaks in ddg.py.<commit_after>from collections import defaultdict
from simuvex.s_ref import RefTypes, SimRegRead, SimRegWrite, SimTmpRead, SimTmpWrite, SimMemRead, SimMemWrite
from simuvex import SimIRSB, SimProcedure
import logging

l = logging.getLogger("angr.ddg")
l.setLevel(logging.DEBUG)

class DDG(object):
    def __init__(self, cfg, entry_point):
        self._cfg = cfg
        self._entry_point = entry_point

        self._ddg = defaultdict(lambda:defaultdict(set))
        self._symbolic_mem_ops = set()

    def debug_print(self):
        l.debug(self._ddg)

    def _trace_source(self, init_run, init_ref):
        '''
        Trace the sources (producers) of a given ref
        '''
        l.debug("Tracing source for symbolic memory dep %s of %s", init_ref, init_run)
        sources = set()
        # Memorization
        traced = {}

        init_stmt_id = init_ref.stmt_idx
        init_reg_deps = set()
        init_tmp_deps = set()
        real_ref = init_run.statements[init_stmt_id].refs[-1]
        init_reg_deps |= set(list(real_ref.data_reg_deps))
        init_tmp_deps |= set(list(real_ref.data_tmp_deps))
        if type(real_ref) == SimMemWrite:
            init_reg_deps |= set(list(real_ref.addr_reg_deps))
            init_tmp_deps |= set(list(real_ref.addr_tmp_deps))
        stack = [(init_run, init_stmt_id, init_reg_deps, init_tmp_deps)]
        while len(stack) > 0:
            run, stmt_id, reg_deps, tmp_deps = stack.pop()
            l.debug("Traversing %s", run)
            # Added to traced set
            traced[run.addr] = reg_deps
            reg_dep_to_stmt_id = {}

            if isinstance(run, SimIRSB):
                irsb = run
                if stmt_id == -1:
                    stmt_ids = range(len(irsb.statements))
                else:
                    stmt_ids = range(stmt_id + 1)
                stmt_ids.reverse()

                for stmt_id in stmt_ids:
                    stmt = irsb.statements[stmt_id]
                    if len(stmt.refs) > 0:
                        real_ref = stmt.refs[-1]
                        if type(real_ref) == SimRegWrite and real_ref.offset in reg_deps:
                            reg_dep_to_stmt_id[real_ref.offset] = stmt_id
                            reg_deps.remove(real_ref.offset)
                            reg_deps |= set(list(real_ref.data_reg_deps))
                            tmp_deps |= set(list(real_ref.data_tmp_deps))
                        elif type(real_ref) == SimTmpWrite and real_ref.tmp in tmp_deps:
                            tmp_deps.remove(real_ref.tmp)
                            reg_deps |= set(list(real_ref.data_reg_deps))
                            tmp_deps |= set(list(real_ref.data_tmp_deps))
            elif isinstance(run, SimProcedure):
                refs = run.refs()
                refs.reverse()
                for ref in refs:
                    if type(ref) == SimRegWrite and ref.offset in reg_deps:
                        reg_dep_to_stmt_id[ref.offset] = -1
                        reg_deps.remove(ref.offset)
                        reg_deps |= set(list(ref.data_reg_deps))
                        tmp_deps |= set(list(ref.data_tmp_deps))
                    elif type(ref) == SimTmpWrite and ref.tmp in tmp_deps:
                        tmp_deps.remove(ref.tmp)
                        reg_deps |= set(list(ref.data_reg_deps))
                        tmp_deps |= set(list(ref.data_tmp_deps))

            if len(reg_deps) > 0:
                predecessors = self._cfg.get_predecessors(run)
                for s in predecessors:
                    tpl = (s.addr, reg_deps)
                    if s.addr not in traced:
                        stack.append((s, -1, reg_deps.copy(), set()))
                    else:
                        old_reg_deps = traced[s.addr]
                        if not reg_deps.issubset(old_reg_deps):
                            new_reg_deps = reg_deps.copy()
                            new_reg_deps |= old_reg_deps
                            stack.append((s, -1, new_reg_deps, set()))
            for reg, stmt_id in reg_dep_to_stmt_id.items():
                sources.add((run.addr, stmt_id))
            if len(reg_deps) == 0 or len(predecessors) == 0:
                # This is the end of the u-d chain
                # FIXME: What if there is a loop, and len(successors) never == 0?
                for reg in reg_deps:
                    if reg not in reg_dep_to_stmt_id:
                        # It's never been written before
                        # We use a fake run.addr and stmt_id so that we don't lose them
                        l.debug("Register %d has never been assigned a value before.", reg)
                        sources.add((-1 * reg, -1))
                    else:
                        # sources.add((run.addr, reg_dep_to_stmt_id[reg]))
                        # They have already been added to sources
                        pass

        return sources

    def _solve_symbolic_mem_operations(self):
        '''
        We try to resolve symbolic memory operations in the following manner:
        For each memory operation, trace the pointer from its root producers.
        And then relate each memory read with all memory writes that has shared
        producers with it.
        It's imprecise and could be over-approximating, but it's better than
        losing dependencies.
        '''
        # irsb, stmt_id => list of tuples (irsb, ref)
        mem_read_producers = defaultdict(list)
        mem_write_producers = defaultdict(list)
        for irsb, ref in self._symbolic_mem_ops:
            sources = self._trace_source(irsb, ref)
            for src_irsb_addr, stmt_id in sources:
                if type(ref) == SimMemRead:
                    mem_read_producers[(src_irsb_addr, stmt_id)].append((irsb.addr, ref))
                elif type(ref) == SimMemWrite:
                    mem_write_producers[(src_irsb_addr, stmt_id)].append((irsb.addr, ref))

        for tpl, lst_writes in mem_write_producers.items():
            if tpl in mem_read_producers:
                lst_reads = mem_read_producers[tpl]
                for read_irsb, read_ref in lst_reads:
                    for write_irsb, write_ref in lst_writes:
                        self._ddg[read_irsb][read_ref.stmt_idx].add((write_irsb, write_ref.stmt_idx))

    def construct(self):
        worklist = set()
        # Added the first container into the worklist
        initial_container = AddrToRefContainer(self._cfg.get_irsb((None, None, self._entry_point)), defaultdict(set))
        worklist.add(initial_container)
        analyzed_runs = set()
        analyzed_containers = {}
        while len(worklist) > 0:
            container = worklist.pop()
            run = container.run
            # If we updated our addr_to_ref map, we should set redo_flag to
            # True, then all of its successors will be reanalyzed
            redo_flag = container.reanalyze_successors
            if isinstance(run, SimIRSB):
                irsb = run
                l.debug("Running %s", irsb)
                # Simulate the execution of this irsb.
                # For MemWriteRef, fill the addr_to_ref dict with every single concretizable
                # memory address, and ignore those symbolic ones
                # For MemReadRef, get its related MemoryWriteRef from our dict
                # TODO: Is it possible to trace memory operations even if the memory is not
                # concretizable itself?
                statements = irsb.statements
                for i in range(len(statements)):
                    stmt = statements[i]
                    refs = stmt.refs
                    if len(refs) > 0:
                        real_ref = refs[len(refs) - 1]
                        if type(real_ref) == SimMemWrite:
                            addr = real_ref.addr
                            if not addr.is_symbolic():
                                concrete_addr = addr.any()
                                tpl = (irsb.addr, i)
                                if tpl not in container.addr_to_ref[concrete_addr]:
                                    container.addr_to_ref[concrete_addr].add(tpl)
                                    redo_flag = True
                            else:
                                self._symbolic_mem_ops.add((irsb, real_ref))
                    for ref in refs:
                        if type(ref) == SimMemRead:
                            addr = ref.addr
                            if not addr.is_symbolic():
                                concrete_addr = addr.any()
                                if concrete_addr in container.addr_to_ref:
                                    self._ddg[irsb.addr][i] |= (container.addr_to_ref[concrete_addr])
                            else:
                                self._symbolic_mem_ops.add((irsb, ref))
            elif isinstance(run, SimProcedure):
                sim_proc = run
                l.debug("Running %s", sim_proc)
                refs = sim_proc.refs()
                for ref in refs:
                    if isinstance(ref, SimMemRead):
                        addr = ref.addr
                        if not addr.is_symbolic():
                            concrete_addr = addr.any()
                            # print sim_proc
                            # print "0x%08x" % concrete_addr
                            if concrete_addr in container.addr_to_ref:
                                self._ddg[sim_proc.addr][-1] |= (container.addr_to_ref[concrete_addr])
                                # print "In list"
                        else:
                            self._symbolic_mem_ops.add((sim_proc, ref))
                    elif isinstance(ref, SimMemWrite):
                        addr = ref.addr
                        if not addr.is_symbolic():
                            concrete_addr = addr.any()
                            tpl = (sim_proc.addr, -1)
                            if tpl not in container.addr_to_ref[concrete_addr]:
                                container.addr_to_ref[concrete_addr].add((sim_proc.addr, -1))
                                redo_flag = True
                        else:
                            self._symbolic_mem_ops.add((sim_proc, ref))

            analyzed_runs.add(run)
            analyzed_containers[run] = container

            # Get successors of the current irsb,
            successors = self._cfg.get_successors(run)
            if redo_flag:
                for successor in successors:
                    if successor in analyzed_runs:
                        analyzed_runs.remove(successor)
            # ... and add them to our worklist with a shallow copy of the addr_to_ref dict
            for successor in successors:
                if successor not in analyzed_runs:
                    # Let's see if there is any changes in the addresses dict
                    new_addr_to_ref = container.addr_to_ref.copy()
                    new_container = None
                    if successor in analyzed_containers:
                        old_container = analyzed_containers[successor]
                        new_container = AddrToRefContainer(successor, old_container.addr_to_ref, reanalyze_successors=True)
                        if not new_container.combine(container.addr_to_ref):
                            # There isn't any new addresses
                            continue
                    if new_container is None:
                        new_container = AddrToRefContainer(successor, container.addr_to_ref.copy())
                    worklist.add(new_container)

        self._solve_symbolic_mem_operations()

class AddrToRefContainer(object):
    def __init__(self, run, addr_to_ref, reanalyze_successors=False):
        self.run = run
        self.addr_to_ref = addr_to_ref
        self.reanalyze_successors = reanalyze_successors

    def combine(self, new_addr_to_ref):
        altered = False
        for k, v in new_addr_to_ref.items():
            if k not in self.addr_to_ref:
                self.addr_to_ref[k] = v.copy()
                altered = True
            else:
                if not v.issubset(self.addr_to_ref[k]):
                    altered = True
                    self.addr_to_ref[k] |= v
        return altered
<|endoftext|>


<commit_before># Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Jsonnet rules for Bazel."""

_JSONNET_FILETYPE = FileType([".jsonnet"])

def _setup_deps(deps):
  """Collects source files and import flags of transitive dependencies.

  Args:
    deps: List of deps labels from ctx.attr.deps.

  Returns:
    Returns a struct containing the following fields:
      transitive_sources: List of Files containing sources of transitive
          dependencies
      imports: List of Strings containing import flags set by transitive
          dependency targets.
  """
  transitive_sources = set(order="compile")
  imports = set()
  for dep in deps:
    transitive_sources += dep.transitive_jsonnet_files
    imports += ["%s/%s" % (dep.label.package, im) for im in dep.imports]

  return struct(
      transitive_sources = transitive_sources,
      imports = imports)

def _jsonnet_library_impl(ctx):
  """Implementation of the jsonnet_library rule."""
  depinfo = _setup_deps(ctx.attr.deps)
  sources = depinfo.transitive_sources + ctx.files.srcs
  imports = depinfo.imports + ctx.attr.imports
  return struct(files = set(),
                transitive_jsonnet_files = sources,
                imports = imports)

def _jsonnet_toolchain(ctx):
  return struct(
      jsonnet_path = ctx.file._jsonnet.path,
      imports = ["-J %s" % ctx.file._std.dirname])

def _jsonnet_to_json_impl(ctx):
  """Implementation of the jsonnet_to_json rule."""
  depinfo = _setup_deps(ctx.attr.deps)
  toolchain = _jsonnet_toolchain(ctx)
  jsonnet_vars = ctx.attr.vars
  jsonnet_code_vars = ctx.attr.code_vars
  command = (
      [
          "set -e;",
          toolchain.jsonnet_path,
      ] +
      ["-J %s/%s" % (ctx.label.package, im) for im in ctx.attr.imports] +
      ["-J %s" % im for im in depinfo.imports] +
      toolchain.imports +
      ["-J ."] +
      ["--var '%s'='%s'"
          % (var, jsonnet_vars[var]) for var in jsonnet_vars.keys()] +
      ["--code-var '%s'='%s'"
          % (var, jsonnet_code_vars[var]) for var in jsonnet_vars.keys()])

  outputs = []
  # If multiple_outputs is set to true, then jsonnet will be invoked with the
  # -m flag for multiple outputs. Otherwise, jsonnet will write the resulting
  # JSON to stdout, which is redirected into a single JSON output file.
  if len(ctx.attr.outs) > 1 or ctx.attr.multiple_outputs:
    output_json_files = [ctx.new_file(ctx.configuration.bin_dir, out.name)
                         for out in ctx.attr.outs]
    outputs += output_json_files
    command += ["-m", output_json_files[0].dirname, ctx.file.src.path]
  else:
    if len(ctx.attr.outs) > 1:
      fail("Only one file can be specified in outs if multiple_outputs is " +
           "not set.")

    compiled_json = ctx.new_file(ctx.configuration.bin_dir,
                                 ctx.attr.outs[0].name)
    outputs += [compiled_json]
    command += [ctx.file.src.path, "-o", compiled_json.path]

  compile_inputs = (
      [ctx.file.src, ctx.file._jsonnet, ctx.file._std] +
      list(depinfo.transitive_sources))

  ctx.action(
      inputs = compile_inputs,
      outputs = outputs,
      mnemonic = "Jsonnet",
      command = " ".join(command),
      use_default_shell_env = True,
      progress_message = "Compiling Jsonnet to JSON for " + ctx.label.name);

_EXIT_CODE_COMPARE_COMMAND = """
EXIT_CODE=$?
EXPECTED_EXIT_CODE=%d
if [ $EXIT_CODE -ne $EXPECTED_EXIT_CODE ] ; then
  echo "FAIL (exit code): %s"
  echo "Expected: $EXPECTED_EXIT_CODE"
  echo "Actual: $EXIT_CODE"
  echo "Output: $OUTPUT"
  exit 1
fi
"""

_DIFF_COMMAND = """
GOLDEN=$(cat %s)
if [ "$OUTPUT" != "$GOLDEN" ]; then
  echo "FAIL (output mismatch): %s"
  echo "Diff:"
  diff <(echo $GOLDEN) <(echo $OUTPUT)
  echo "Expected: $GOLDEN"
  echo "Actual: $OUTPUT"
  exit 1
fi
"""

_REGEX_DIFF_COMMAND = """
GOLDEN_REGEX=$(cat %s)
if [[ ! "$OUTPUT" =~ $GOLDEN_REGEX ]]; then
  echo "FAIL (regex mismatch): %s"
  echo "Output: $OUTPUT"
  exit 1
fi
"""

def _jsonnet_to_json_test_impl(ctx):
  """Implementation of the jsonnet_to_json_test rule."""
  depinfo = _setup_deps(ctx.attr.deps)
  toolchain = _jsonnet_toolchain(ctx)

  golden_files = []
  if ctx.file.golden:
    golden_files += [ctx.file.golden]
    if ctx.attr.regex:
      diff_command = _REGEX_DIFF_COMMAND % (ctx.file.golden.short_path,
                                           ctx.label.name)
    else:
      diff_command = _DIFF_COMMAND % (ctx.file.golden.short_path,
                                     ctx.label.name)

  jsonnet_vars = ctx.attr.vars
  jsonnet_code_vars = ctx.attr.code_vars
  jsonnet_command = " ".join(
      ["OUTPUT=$(%s" % ctx.file._jsonnet.short_path] +
      ["-J %s/%s" % (ctx.label.package, im) for im in ctx.attr.imports] +
      ["-J %s" % im for im in depinfo.imports] +
      toolchain.imports +
      ["-J ."] +
      ["--var %s=%s"
          % (var, jsonnet_vars[var]) for var in jsonnet_vars.keys()] +
      ["--code-var %s=%s"
          % (var, jsonnet_code_vars[var]) for var in jsonnet_vars.keys()] +
      [
          ctx.file.src.path,
          "2>&1)",
      ])

  command = "\n".join([
      "#!/bin/bash",
      jsonnet_command,
      _EXIT_CODE_COMPARE_COMMAND % (ctx.attr.error, ctx.label.name),
      diff_command])

  ctx.file_action(output = ctx.outputs.executable,
                  content = command,
                  executable = True);

  test_inputs = (
      [ctx.file.src, ctx.file._jsonnet, ctx.file._std] +
      golden_files +
      list(depinfo.transitive_sources))

  return struct(
      runfiles = ctx.runfiles(files = test_inputs, collect_data = True))

_jsonnet_common_attrs = {
    "deps": attr.label_list(providers = ["transitive_jsonnet_files"],
                            allow_files = False),
    "imports": attr.string_list(),
    "_jsonnet": attr.label(
        default = Label("//tools/build_defs/jsonnet:jsonnet"),
        executable = True,
        single_file = True),
    "_std": attr.label(default = Label("//tools/build_defs/jsonnet:std"),
                       single_file = True),
}

_jsonnet_library_attrs = {
    "srcs": attr.label_list(allow_files = _JSONNET_FILETYPE),
}

jsonnet_library = rule(
    _jsonnet_library_impl,
    attrs = _jsonnet_library_attrs + _jsonnet_common_attrs,
)

_jsonnet_compile_attrs = {
    "src": attr.label(allow_files = _JSONNET_FILETYPE,
                      single_file = True),
    "vars": attr.string_dict(),
    "code_vars": attr.string_dict(),
}

_jsonnet_to_json_attrs = _jsonnet_compile_attrs + {
    "outs": attr.output_list(mandatory = True),
    "multiple_outputs": attr.bool(),
}

jsonnet_to_json = rule(
    _jsonnet_to_json_impl,
    attrs = _jsonnet_to_json_attrs + _jsonnet_common_attrs,
)

_jsonnet_to_json_test_attrs = _jsonnet_compile_attrs + {
    "golden": attr.label(allow_files = True, single_file = True),
    "error": attr.int(),
    "regex": attr.bool(),
}

jsonnet_to_json_test = rule(
    _jsonnet_to_json_test_impl,
    attrs = _jsonnet_to_json_test_attrs + _jsonnet_common_attrs,
    executable = True,
    test = True,
)
<commit_msg>[jsonnet] Fix jsonnet_to_json rule to read code_vars from code_vars instead of vars.<commit_after># Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Jsonnet rules for Bazel."""

_JSONNET_FILETYPE = FileType([".jsonnet"])

def _setup_deps(deps):
  """Collects source files and import flags of transitive dependencies.

  Args:
    deps: List of deps labels from ctx.attr.deps.

  Returns:
    Returns a struct containing the following fields:
      transitive_sources: List of Files containing sources of transitive
          dependencies
      imports: List of Strings containing import flags set by transitive
          dependency targets.
  """
  transitive_sources = set(order="compile")
  imports = set()
  for dep in deps:
    transitive_sources += dep.transitive_jsonnet_files
    imports += ["%s/%s" % (dep.label.package, im) for im in dep.imports]

  return struct(
      transitive_sources = transitive_sources,
      imports = imports)

def _jsonnet_library_impl(ctx):
  """Implementation of the jsonnet_library rule."""
  depinfo = _setup_deps(ctx.attr.deps)
  sources = depinfo.transitive_sources + ctx.files.srcs
  imports = depinfo.imports + ctx.attr.imports
  return struct(files = set(),
                transitive_jsonnet_files = sources,
                imports = imports)

def _jsonnet_toolchain(ctx):
  return struct(
      jsonnet_path = ctx.file._jsonnet.path,
      imports = ["-J %s" % ctx.file._std.dirname])

def _jsonnet_to_json_impl(ctx):
  """Implementation of the jsonnet_to_json rule."""
  depinfo = _setup_deps(ctx.attr.deps)
  toolchain = _jsonnet_toolchain(ctx)
  jsonnet_vars = ctx.attr.vars
  jsonnet_code_vars = ctx.attr.code_vars
  command = (
      [
          "set -e;",
          toolchain.jsonnet_path,
      ] +
      ["-J %s/%s" % (ctx.label.package, im) for im in ctx.attr.imports] +
      ["-J %s" % im for im in depinfo.imports] +
      toolchain.imports +
      ["-J ."] +
      ["--var '%s'='%s'"
          % (var, jsonnet_vars[var]) for var in jsonnet_vars.keys()] +
      ["--code-var '%s'='%s'"
          % (var, jsonnet_code_vars[var]) for var in jsonnet_code_vars.keys()])

  outputs = []
  # If multiple_outputs is set to true, then jsonnet will be invoked with the
  # -m flag for multiple outputs. Otherwise, jsonnet will write the resulting
  # JSON to stdout, which is redirected into a single JSON output file.
  if len(ctx.attr.outs) > 1 or ctx.attr.multiple_outputs:
    output_json_files = [ctx.new_file(ctx.configuration.bin_dir, out.name)
                         for out in ctx.attr.outs]
    outputs += output_json_files
    command += ["-m", output_json_files[0].dirname, ctx.file.src.path]
  else:
    if len(ctx.attr.outs) > 1:
      fail("Only one file can be specified in outs if multiple_outputs is " +
           "not set.")

    compiled_json = ctx.new_file(ctx.configuration.bin_dir,
                                 ctx.attr.outs[0].name)
    outputs += [compiled_json]
    command += [ctx.file.src.path, "-o", compiled_json.path]

  compile_inputs = (
      [ctx.file.src, ctx.file._jsonnet, ctx.file._std] +
      list(depinfo.transitive_sources))

  ctx.action(
      inputs = compile_inputs,
      outputs = outputs,
      mnemonic = "Jsonnet",
      command = " ".join(command),
      use_default_shell_env = True,
      progress_message = "Compiling Jsonnet to JSON for " + ctx.label.name);

_EXIT_CODE_COMPARE_COMMAND = """
EXIT_CODE=$?
EXPECTED_EXIT_CODE=%d
if [ $EXIT_CODE -ne $EXPECTED_EXIT_CODE ] ; then
  echo "FAIL (exit code): %s"
  echo "Expected: $EXPECTED_EXIT_CODE"
  echo "Actual: $EXIT_CODE"
  echo "Output: $OUTPUT"
  exit 1
fi
"""

_DIFF_COMMAND = """
GOLDEN=$(cat %s)
if [ "$OUTPUT" != "$GOLDEN" ]; then
  echo "FAIL (output mismatch): %s"
  echo "Diff:"
  diff <(echo $GOLDEN) <(echo $OUTPUT)
  echo "Expected: $GOLDEN"
  echo "Actual: $OUTPUT"
  exit 1
fi
"""

_REGEX_DIFF_COMMAND = """
GOLDEN_REGEX=$(cat %s)
if [[ ! "$OUTPUT" =~ $GOLDEN_REGEX ]]; then
  echo "FAIL (regex mismatch): %s"
  echo "Output: $OUTPUT"
  exit 1
fi
"""

def _jsonnet_to_json_test_impl(ctx):
  """Implementation of the jsonnet_to_json_test rule."""
  depinfo = _setup_deps(ctx.attr.deps)
  toolchain = _jsonnet_toolchain(ctx)

  golden_files = []
  if ctx.file.golden:
    golden_files += [ctx.file.golden]
    if ctx.attr.regex:
      diff_command = _REGEX_DIFF_COMMAND % (ctx.file.golden.short_path,
                                           ctx.label.name)
    else:
      diff_command = _DIFF_COMMAND % (ctx.file.golden.short_path,
                                     ctx.label.name)

  jsonnet_vars = ctx.attr.vars
  jsonnet_code_vars = ctx.attr.code_vars
  jsonnet_command = " ".join(
      ["OUTPUT=$(%s" % ctx.file._jsonnet.short_path] +
      ["-J %s/%s" % (ctx.label.package, im) for im in ctx.attr.imports] +
      ["-J %s" % im for im in depinfo.imports] +
      toolchain.imports +
      ["-J ."] +
      ["--var %s=%s"
          % (var, jsonnet_vars[var]) for var in jsonnet_vars.keys()] +
      ["--code-var %s=%s"
          % (var, jsonnet_code_vars[var]) for var in jsonnet_code_vars.keys()] +
      [
          ctx.file.src.path,
          "2>&1)",
      ])

  command = "\n".join([
      "#!/bin/bash",
      jsonnet_command,
      _EXIT_CODE_COMPARE_COMMAND % (ctx.attr.error, ctx.label.name),
      diff_command])

  ctx.file_action(output = ctx.outputs.executable,
                  content = command,
                  executable = True);

  test_inputs = (
      [ctx.file.src, ctx.file._jsonnet, ctx.file._std] +
      golden_files +
      list(depinfo.transitive_sources))

  return struct(
      runfiles = ctx.runfiles(files = test_inputs, collect_data = True))

_jsonnet_common_attrs = {
    "deps": attr.label_list(providers = ["transitive_jsonnet_files"],
                            allow_files = False),
    "imports": attr.string_list(),
    "_jsonnet": attr.label(
        default = Label("//tools/build_defs/jsonnet:jsonnet"),
        executable = True,
        single_file = True),
    "_std": attr.label(default = Label("//tools/build_defs/jsonnet:std"),
                       single_file = True),
}

_jsonnet_library_attrs = {
    "srcs": attr.label_list(allow_files = _JSONNET_FILETYPE),
}

jsonnet_library = rule(
    _jsonnet_library_impl,
    attrs = _jsonnet_library_attrs + _jsonnet_common_attrs,
)

_jsonnet_compile_attrs = {
    "src": attr.label(allow_files = _JSONNET_FILETYPE,
                      single_file = True),
    "vars": attr.string_dict(),
    "code_vars": attr.string_dict(),
}

_jsonnet_to_json_attrs = _jsonnet_compile_attrs + {
    "outs": attr.output_list(mandatory = True),
    "multiple_outputs": attr.bool(),
}

jsonnet_to_json = rule(
    _jsonnet_to_json_impl,
    attrs = _jsonnet_to_json_attrs + _jsonnet_common_attrs,
)

_jsonnet_to_json_test_attrs = _jsonnet_compile_attrs + {
    "golden": attr.label(allow_files = True, single_file = True),
    "error": attr.int(),
    "regex": attr.bool(),
}

jsonnet_to_json_test = rule(
    _jsonnet_to_json_test_impl,
    attrs = _jsonnet_to_json_test_attrs + _jsonnet_common_attrs,
    executable = True,
    test = True,
)
<|endoftext|>


<commit_before>"""
The Kamitani paper: reconstruction of visual stimuli
======================================================

"""

# Options
# =======

remove_rest_period = True
foveal_focus_radius = None  # 2.5 is a good value if set
multi_scale = True          # Not compatible with foveal focal radius
voxel_restriction = 'roi'   # could also be 'V1' or None
generate_video = 'test.mp4'

###############################################################################
from matplotlib import pyplot as plt
from sklearn.feature_selection import SelectKBest, f_classif
### Load Kamitani dataset #####################################################
from nisl import datasets
dataset = datasets.fetch_kamitani()
X_random = dataset.data_random
X_figure = dataset.data_figure
y_random = dataset.target_random
y_figure = dataset.target_figure

### Preprocess data ###########################################################
import numpy as np
from scipy import signal

# Detrend data on each session independently
print "detrending data"
for s in range(20):
    X_random[s] = signal.detrend(X_random[s], axis=0)

for s in range(12):
    X_figure[s] = signal.detrend(X_figure[s], axis=0)

X_train = np.hstack(X_random).T
y_train = np.hstack(y_random).T
X_test = np.hstack(X_figure).T
y_test = np.hstack(y_figure).T

n_pixels = y_train.shape[1]
n_features = X_train.shape[1]

# Remove rest period
if remove_rest_period:
    X_train = X_train[y_train[:, 0] != -1]
    y_train = y_train[y_train[:, 0] != -1]
    X_test = X_test[y_test[:, 0] != -1]
    y_test = y_test[y_test[:, 0] != -1]

# Take only the foveal part (radius is custom)
if foveal_focus_radius:
    from numpy import linalg
    c = (4.5, 4.5)
    radius = foveal_focus_radius
    y_mask = np.ones(100, dtype='bool')
    for i in range(10):
        for j in range(10):
            y_mask[i * 10 + j] = (linalg.norm((c[0] - i, c[1] - j)) <= radius)
    n_pixels = y_mask.sum()
    y_train = y_train[:, y_mask]
    y_test = y_test[:, y_mask]
    # Show the mask
    # plt.imshow(np.reshape(y_mask, [10, 10]), cmap=plt.cm.gray,
    #         interpolation='nearest')
    # plt.show()

# Compute scaled images

if multi_scale:
    y_shape = (10, 10)
    y_rows, y_cols = y_shape
    height_transform = np.repeat((np.identity(y_cols / 2) * 0.5), 2, 1)
    width_transform = np.repeat((np.identity(y_rows / 2) * 0.5), 2, 0)

    y_train_tall = np.array(
        [np.dot(height_transform, np.reshape(m, y_shape)).flatten()
        for m in y_train])
    y_train_large = np.array(
        [np.dot(np.reshape(m, y_shape), width_transform).flatten()
        for m in y_train])
    y_train_big = np.array([np.dot(height_transform,
        np.dot(np.reshape(m, y_shape), width_transform)).flatten()
        for m in y_train])

    # We add them to original data
    y_train = np.concatenate((y_train, y_train_tall,
            y_train_large, y_train_big), axis=1)

# Keep V1 only
if voxel_restriction:
    X_mask = np.zeros(n_features, dtype='bool')
    rv = dataset.roi_volInd
    if voxel_restriction == 'V1':
        volInd = np.unique(
            np.concatenate((rv[0, 2], rv[0, 3], rv[4, 2], rv[4, 3])))
    elif voxel_restriction == 'roi':
        volInd = np.unique(np.array([x for y in rv.flatten() for x in y]))
    else:
        print 'Error: unknown voxel restriction'
        exit
    mask_ind = []
    for i in volInd:
        mask_ind.append(np.where(dataset.volInd == i)[0])
    mask_ind = np.array([y for x in mask_ind for y in x])
    X_mask[mask_ind] = True

    X_train = X_train[:, X_mask]
    X_test = X_test[:, X_mask]
    n_features = X_train.shape[1]

# Feature selection analysis

"""
def roi_stat(indices):
    # get ROI names
    names = dataset.roi_name[0:8, 2:4].flatten()
    roi_indices = dataset.roi_volInd[0:8, 2:4].flatten()
    names = dataset.roi_name[:, 0:2].flatten()
    roi_indices = dataset.roi_volInd[:, 0:2].flatten()
    data_indices = []
    for i, roi_ind in enumerate(roi_indices):
        roi_ind = roi_ind.squeeze()
        data_ind = []
        for p in roi_ind:
            data_ind.append(np.where(dataset.volInd == p)[0])
        data_indices.append(np.array([y for x in data_ind for y in x]))

    count = np.zeros(names.shape)
    for ind in indices:
        for i, data_ind in enumerate(data_indices):
            count[i] += (np.where(np.unique(data_ind) == ind)[0].size != 0)
    return (names, count)

from sklearn.svm import SVC
feature_selection = SelectKBest(f_classif, k=100)

feature_selection = RFE(SVC(kernel='linear', C=1.), n_features_to_select=100)

roi_features = []
for i in range(n_features):
    feature_selection.fit(X_train, y_train[i])
    n, c = roi_stat(np.where(feature_selection.get_support())[0])
    roi_features.append(c)

rf = np.array(roi_features)
plt.figure(1)
for i, nn in enumerate(n):
    plt.subplot(6, 4, i + 1)
    plt.axis('off')
    plt.title(nn)
    plt.imshow(np.reshape(rf[:, i], [10, 10]), cmap=plt.cm.hot,
                      interpolation='nearest', vmin=0, vmax=100)

plt.show()

for i, nn in enumerate(n):
    print nn[0] + " : %d" % c[i]
"""

### Prediction function #######################################################

from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
import copy

"""
def make_pipelines(feature_selection, clf, X_train, y_train, n_features):
    pipelines = []
    pipeline_ref = Pipeline([('fs', feature_selection), ('clf', clf)])
    for i in range(n_features):
        print "Count %d of %d" % ((i + 1), n_features)
        pipeline = copy.deepcopy(pipeline_ref)
        pipeline.fit(X_train, y_train[i, :])
        pipelines.append(pipeline)
    return pipelines

def predict(pipelines, X_test):
    preds = []
    for i, x_test in enumerate(X_test):
        pred = []
        for p in pipelines:
            pred.append(p.predict(x_test))
        pred = np.array(pred)
        preds.append(pred.squeeze())
    return preds
"""

"""
    f_classif 100 + SVC classique : 65%

pipelines = []

for i in range(n_features):
    print "Count %d of %d" % ((i + 1), n_features)
    clf = SVC(kernel='linear', C=1.)
    feature_selection = SelectKBest(f_classif, k=50)
    anova_svc = Pipeline([('anova', feature_selection), ('svc', clf)])
    anova_svc.fit(X_train, y_train[i, :])
    pipelines.append(anova_svc)

"""
"""
    f_classif 100 + Ridge
"""

from sklearn.linear_model import OrthogonalMatchingPursuit as OMP

clf = OMP(n_nonzero_coefs=20)

clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)

"""
clf.fit(X_train, y_train_tall.T)
y_pred_tall = clf.predict(X_test)

clf.fit(X_train, y_train_large.T)
y_pred_large = clf.predict(X_test)

clf.fit(X_train, y_train_big.T)
y_pred_big = clf.predict(X_test)
"""

"""
    Sparse SVC (tres long, 3 heures) :

from sklearn.svm.sparse import SVC
acc = np.zeros(n_features)

for i in range(100):
    print "Count %d of %d" % ((i + 1), n_features)
    clf = SVC(kernel='linear', C=1.)
    clf.fit(X_train, y_train[i, :])
    score = 0
    for j, y in enumerate(y_test.T):
        score += clf.predict(X_test[j, :]) == y[i]
    acc[i] = score

"""

"""
    Sparse SVC sur V1 :
    -------------------

from sklearn.svm.sparse import SVC
acc = []
clfs = []

for i in range(n_features):
    print "Count %d of %d" % ((i + 1), n_features)
    clf = SVC(kernel='linear', C=1.)
    clf.fit(X_train_v1, y_train[i, :])
    clfs.append(clf)

"""

"""
    f_classif 100 + sparse SVC
    --------------------------

from sklearn.svm.sparse import SVC

for i in range(n_features):
    print "Count %d of %d" % ((i + 1), n_features)
    clf = SVC(kernel='linear', C=1.)
    feature_selection = SelectKBest(f_classif, k=100)
    anova_svc = Pipeline([('anova', feature_selection), ('svc', clf)])
    anova_svc.fit(X_train, y_train[i, :])
    pipelines.append(anova_svc)

"""

"""
    f_classif 100 + Ridge
    --------------------------

from sklearn.linear_model import RidgeClassifier

for i in range(n_features):
    print "Count %d of %d" % ((i + 1), n_features)
    clf = RidgeClassifier(alpha=10.)
    feature_selection = SelectKBest(f_classif, k=100)
    anova_clf = Pipeline([('anova', feature_selection), ('clf', clf)])
    anova_clf.fit(X_train, y_train[i, :])
    pipelines.append(anova_clf)
"""

"""
# Learn
pipes = make_pipelines(SelectKBest(f_classif, k=100),
        SVC(kernel='linear', C=1.), X_train, y_train, n_features)

# Predict
y_pred = predict(pipes, X_test)
"""

### Prediction ################################################################

# Revert scaled images if needed
if multi_scale:
    y_preds = np.split(y_pred, [n_pixels, 1.5 * n_pixels,
            2 * n_pixels, 2.25 * n_pixels], axis=1)

    y_pred = y_preds[0]
    y_pred_tall = np.array([np.dot(height_transform.T * 2,
        np.reshape(m, (5, 10))).flatten() for m in y_preds[1]])
    y_pred_large = np.array([np.dot(np.reshape(m, (10, 5)),
        width_transform.T * 2).flatten() for m in y_preds[2]])
    y_pred_big = [np.dot(height_transform.T * 2, np.reshape(m, (5, 5)))
        for m in y_preds[3]]
    y_pred_big = np.array([np.dot(np.reshape(m, (10, 5)),
        width_transform.T * 2).flatten() for m in y_pred_big])

    y_pred = (.25 * y_pred + .25 * y_pred_tall + .25 * y_pred_large
        + .25 * y_pred_big)

"""
preds = []
for i, x_test in enumerate(X_test):
    pred = []
    for p in pipelines:
        pred.append(p.predict(x_test))
    pred = np.array(pred)
    preds.append(pred.squeeze())

# Visualize results
y_pred = np.zeros(y_mask.shape)
y_pred[y_mask] = acc
plt.imshow(np.reshape(y_pred, [10, 10]), cmap=plt.cm.gray,
        interpolation='nearest')
plt.show()

print "Result : %d" % np.mean(acc)
"""

if generate_video:
    from matplotlib import animation
    fig = plt.figure()
    sp1 = plt.subplot(121)
    sp1.axis('off')
    sp2 = plt.subplot(122)
    sp2.axis('off')
    ims = []
    for i, t in enumerate(y_pred):
        ims.append((
            sp1.imshow(np.reshape(y_test[i], (10, 10)), cmap=plt.cm.gray,
            interpolation='nearest'),
            sp2.imshow(np.reshape(t, (10, 10)), cmap=plt.cm.gray,
            interpolation='nearest')))

    im_ani = animation.ArtistAnimation(fig, ims, interval=1000)
    im_ani.save(generate_video)
<commit_msg>Optionize every algorithm. More clever multiscale (take into account overlapping patches)<commit_after># Options
# =======

# Preprocessing
remove_rest_period = True
foveal_focus_radius = None  # 2.5 is a good value if set
multi_scale = False          # Not compatible with foveal focal radius
voxel_restriction = 'roi'   # could also be 'V1' or None
preprocessing = 'detrend'

# Learning
learn_fusion_params = False  # Learn fusion params with LinearRegression
# Available classifiers :
# - anova_svc: f_classif 50 features + linear SVC (C = 1.)
# - ridge: ridge regression
# - omp: Orthgonal Matching Pursuit (n_nonzero_coefs=20)
# - anova_ridge: f_classif 50 features + ridge regression
# - lassolars
# - bayesianridge: Bayesian Ridge
classifier = 'lassolars'

# Output
generate_video = 'testLL.mp4'

### Imports ###################################################################

from matplotlib import pyplot as plt
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.pipeline import Pipeline

### Load Kamitani dataset #####################################################
from nisl import datasets
dataset = datasets.fetch_kamitani_data()
X_random = dataset.data_random
X_figure = dataset.data_figure
y_random = dataset.target_random
y_figure = dataset.target_figure

### Preprocess data ###########################################################
import numpy as np
from scipy import signal

print "Preprocessing data"

# Detrend data on each session independently
if preprocessing == "detrend":
    for x in X_random:
        x[:] = signal.detrend(x, axis=1)
    for x in X_figure:
        x[:] = signal.detrend(x, axis=1)
elif preprocessing == "normalize":
    for x in X_random:
        x[:] = signal.detrend(x, axis=1)
    for x in X_figure:
        x[:] = signal.detrend(x, axis=1)

#for s in range(20):
#    X_random[s] -= X_random[s].mean()

#for s in range(12):
#    X_figure[s] -= X_figure[s].mean()

X_train = np.hstack(X_random).T
y_train = np.hstack(y_random).astype(np.float).T
X_test = np.hstack(X_figure).T
y_test = np.hstack(y_figure).astype(np.float).T

n_pixels = y_train.shape[1]
n_features = X_train.shape[1]

# Remove rest period
if remove_rest_period:
    X_train = X_train[y_train[:, 0] != -1]
    y_train = y_train[y_train[:, 0] != -1]
    X_test = X_test[y_test[:, 0] != -1]
    y_test = y_test[y_test[:, 0] != -1]

# Take only the foveal part (radius is custom)
if foveal_focus_radius:
    from numpy import linalg
    c = (4.5, 4.5)
    radius = foveal_focus_radius
    y_mask = np.ones(100, dtype='bool')
    for i in range(10):
        for j in range(10):
            y_mask[i * 10 + j] = (linalg.norm((c[0] - i, c[1] - j)) <= radius)
    n_pixels = y_mask.sum()
    y_train = y_train[:, y_mask]
    y_test = y_test[:, y_mask]
    # Show the mask
    # plt.imshow(np.reshape(y_mask, [10, 10]), cmap=plt.cm.gray,
    #         interpolation='nearest')
    # plt.show()

# Compute scaled images

if multi_scale:
    y_shape = (10, 10)
    y_rows, y_cols = y_shape
    height_transform = np.zeros((y_rows - 1, y_rows))
    for i in range(y_rows - 1):
        height_transform[i, i] = .5
        height_transform[i, i + 1] = .5
    width_transform = np.zeros((y_cols, y_cols - 1))
    for i in range(y_cols - 1):
        width_transform[i, i] = .5
        width_transform[i + 1, i] = .5

    width_backward = width_transform.T
    width_backward[0, 0] = 1
    width_backward[y_cols - 2, y_cols - 1] = 1

    height_backward = height_transform.T
    height_backward[0, 0] = 1
    height_backward[y_rows - 1, y_rows - 2] = 1

    y_train_tall = np.array(
        [np.dot(height_transform, np.reshape(m, y_shape)).flatten()
        for m in y_train])
    y_train_large = np.array(
        [np.dot(np.reshape(m, y_shape), width_transform).flatten()
        for m in y_train])
    y_train_big = np.array([np.dot(height_transform,
        np.dot(np.reshape(m, y_shape), width_transform)).flatten()
        for m in y_train])

    # We add them to original data
    y_train = np.concatenate((y_train, y_train_tall,
            y_train_large, y_train_big), axis=1)

# Filter voxels to take all ROIs or just V1
if voxel_restriction:
    X_mask = np.zeros(n_features, dtype='bool')
    rv = dataset.roi_volInd
    if voxel_restriction == 'V1':
        volInd = np.unique(
            np.concatenate((rv[0, 2], rv[0, 3], rv[4, 2], rv[4, 3])))
    elif voxel_restriction == 'roi':
        volInd = np.unique(np.array([x for y in rv.flatten() for x in y]))
    else:
        print 'Error: unknown voxel restriction'
        exit
    mask_ind = []
    for i in volInd:
        mask_ind.append(np.where(dataset.volInd == i)[0])
    mask_ind = np.array([y for x in mask_ind for y in x])
    X_mask[mask_ind] = True

    X_train = X_train[:, X_mask]
    X_test = X_test[:, X_mask]
    n_features = X_train.shape[1]

# Try to learn coefficients to merge images
if multi_scale and learn_fusion_params:
    from sklearn.linear_model import LinearRegression
    clf = LinearRegression()
    clf.fit(X_train, y_train)

    t_pred = clf.predict(X_train)
    t_preds = np.split(t_pred, [n_pixels, 2 * n_pixels - y_rows,
        3 * n_pixels - y_rows - y_cols, 4 * n_pixels - y_rows - y_cols + 1],
        axis=1)

    t_pred = t_preds[0]
    t_pred_tall = np.array([np.dot(height_backward,
        np.reshape(m, (9, 10))).flatten() for m in t_preds[1]])
    t_pred_large = np.array([np.dot(np.reshape(m, (10, 9)),
        width_backward).flatten() for m in t_preds[2]])
    t_pred_big = [np.dot(height_backward, np.reshape(m, (9, 9)))
        for m in t_preds[3]]
    t_pred_big = np.array([np.dot(np.reshape(m, (10, 9)),
        width_backward).flatten() for m in t_pred_big])

    fusions = []
    from sklearn.linear_model import LinearRegression
    for i, t in enumerate(t_pred.T):
        tX = np.column_stack((t_pred[:, i], t_pred_tall[:, i],
            t_pred_large[:, i], t_pred_big[:, i]))
        f = LinearRegression()
        f.fit(tX, y_train[:, i])
        fusions.append(f.coef_)

    fusions = np.array(fusions)
    fusions = (fusions.T / np.sum(fusions, axis=1)).T

# Feature selection analysis
"""
def roi_stat(indices):
    # get ROI names
    names = dataset.roi_name[0:8, 2:4].flatten()
    roi_indices = dataset.roi_volInd[0:8, 2:4].flatten()
    names = dataset.roi_name[:, 0:2].flatten()
    roi_indices = dataset.roi_volInd[:, 0:2].flatten()
    data_indices = []
    for i, roi_ind in enumerate(roi_indices):
        roi_ind = roi_ind.squeeze()
        data_ind = []
        for p in roi_ind:
            data_ind.append(np.where(dataset.volInd == p)[0])
        data_indices.append(np.array([y for x in data_ind for y in x]))

    count = np.zeros(names.shape)
    for ind in indices:
        for i, data_ind in enumerate(data_indices):
            count[i] += (np.where(np.unique(data_ind) == ind)[0].size != 0)
    return (names, count)

from sklearn.svm import SVC
feature_selection = SelectKBest(f_classif, k=100)

feature_selection = RFE(SVC(kernel='linear', C=1.), n_features_to_select=100)

roi_features = []
for i in range(n_features):
    feature_selection.fit(X_train, y_train[i])
    n, c = roi_stat(np.where(feature_selection.get_support())[0])
    roi_features.append(c)

rf = np.array(roi_features)
plt.figure(1)
for i, nn in enumerate(n):
    plt.subplot(6, 4, i + 1)
    plt.axis('off')
    plt.title(nn)
    plt.imshow(np.reshape(rf[:, i], [10, 10]), cmap=plt.cm.hot,
                      interpolation='nearest', vmin=0, vmax=100)

plt.show()

"""

### Prediction function #######################################################

print "Learning"
"""
import copy

def make_pipelines(feature_selection, clf, X_train, y_train, n_features):
    pipelines = []
    pipeline_ref = Pipeline([('fs', feature_selection), ('clf', clf)])
    for i in range(n_features):
        print "Count %d of %d" % ((i + 1), n_features)
        pipeline = copy.deepcopy(pipeline_ref)
        pipeline.fit(X_train, y_train[i, :])
        pipelines.append(pipeline)
    return pipelines

def predict(pipelines, X_test):
    preds = []
    for i, x_test in enumerate(X_test):
        pred = []
        for p in pipelines:
            pred.append(p.predict(x_test))
        pred = np.array(pred)
        preds.append(pred.squeeze())
    return preds
"""

# f_classif + SVC classique : 65%
if classifier == "anova_svc":
    unique_classifier = False

    from sklearn.svm import SVC
    clfs = []

    for i, pixel_time_serie in enumerate(y_train.T):
        print "Count %d of %d" % ((i + 1), y_train.shape[1])
        clf = SVC(kernel='linear', C=1.)
        feature_selection = SelectKBest(f_classif, k=50)
        anova_svc = Pipeline([('anova', feature_selection), ('svc', clf)])
        anova_svc.fit(X_train, pixel_time_serie)
        clfs.append(anova_svc)

# Ridge
elif classifier == "ridge":
    unique_classifier = True
    from sklearn.linear_model import RidgeCV
    clf = RidgeCV()
    clf.fit(X_train, y_train)

# f_classif + Ridge
elif classifier == 'anova_ridge':
    unique_classifier = False
    from sklearn.linear_model import RidgeCV
    clfs = []
    for i, pixel_time_serie in enumerate(y_train.T):
        print "Count %d of %d" % ((i + 1), y_train.shape[1])
        clf = RidgeCV()
        feature_selection = SelectKBest(f_classif, k=50)
        anova_clf = Pipeline([('anova', feature_selection), ('clf', clf)])
        anova_clf.fit(X_train, pixel_time_serie)
        clfs.append(anova_clf)

# OMP
elif classifier == "omp":
    unique_classifier = True
    from sklearn.linear_model import OrthogonalMatchingPursuit as OMP
    clf = OMP(n_nonzero_coefs=20)
    clf.fit(X_train, y_train)

# LassoLars
elif classifier == "lassolars":
    unique_classifier = False
    from sklearn.linear_model import LassoLarsCV

    clfs = []
    for i, pixel_time_serie in enumerate(y_train.T):
        print "Count %d of %d" % ((i + 1), y_train.shape[1])
        clf = LassoLarsCV()
        clf.fit(X_train, pixel_time_serie)
        clfs.append(clf)

# f_classif 100 + sparse SVC
elif classifier == "anova_sparsesvc":
    unique_classifier = False
    from sklearn.svm.sparse import SVC
    clfs = []

    for i, pixel_time_serie in enumerate(y_train.T):
        print "Count %d of %d" % ((i + 1), y_train.shape[1])
        clf = SVC(kernel='linear', C=1.)
        feature_selection = SelectKBest(f_classif, k=50)
        anova_svc = Pipeline([('anova', feature_selection), ('svc', clf)])
        anova_svc.fit(X_train, pixel_time_serie)
        clfs.append(anova_svc)

# Bayesian Ridge
elif classifier == "bayesianridge":
    unique_classifier = False
    from sklearn.linear_model import BayesianRidge
    clfs = []
    for i, pixel_time_serie in enumerate(y_train.T):
        print "Count %d of %d" % ((i + 1), y_train.shape[1])
        clf = BayesianRidge(normalize=False, n_iter=1000)
        feature_selection = SelectKBest(f_classif, k=500)
        anova_svc = Pipeline([('anova', feature_selection), ('svc', clf)])
        anova_svc.fit(X_train, pixel_time_serie)
        clfs.append(anova_svc)

"""
# Learn
pipes = make_pipelines(SelectKBest(f_classif, k=100),
        SVC(kernel='linear', C=1.), X_train, y_train, n_features)

# Predict
y_pred = predict(pipes, X_test)
"""

### Prediction ################################################################

print "Calculating scores and outputs"

# Different processing for algorithms handling multiple outputs and those who
# do not
if unique_classifier:
    y_pred = clf.predict(X_test)
else:
    y_pred = []
    for i, x_test in enumerate(X_test):
        pred = []
        for c in clfs:
            pred.append(c.predict(x_test))
        pred = np.array(pred)
        y_pred.append(pred.squeeze())
    y_pred = np.array(y_pred)

# Revert scaled images if needed
if multi_scale:
    y_preds = np.split(y_pred, [n_pixels, 2 * n_pixels - y_rows,
        3 * n_pixels - y_rows - y_cols,
        4 * n_pixels - 2 * y_rows - 2 * y_cols + 1], axis=1)

    y_pred = y_preds[0]
    y_pred_tall = np.array([np.dot(height_backward,
        np.reshape(m, (9, 10))).flatten() for m in y_preds[1]])
    y_pred_large = np.array([np.dot(np.reshape(m, (10, 9)),
        width_backward).flatten() for m in y_preds[2]])
    y_pred_big = [np.dot(height_backward, np.reshape(m, (9, 9)))
        for m in y_preds[3]]
    y_pred_big = np.array([np.dot(np.reshape(m, (10, 9)),
        width_backward).flatten() for m in y_pred_big])

    if learn_fusion_params:
        y_pred = np.array([y_pred.T, y_pred_tall.T, y_pred_large.T,
            y_pred_big.T])
        y_pred = np.sum(y_pred.T * fusions, axis=2)
    else:
        y_pred = (.25 * y_pred + .25 * y_pred_tall + .25 * y_pred_large
            + .25 * y_pred_big)

"""

# Visualize results
y_pred = np.zeros(y_mask.shape)
y_pred[y_mask] = acc
plt.imshow(np.reshape(y_pred, [10, 10]), cmap=plt.cm.gray,
        interpolation='nearest')
plt.show()

print "Result : %d" % np.mean(acc)
"""

"""
Show brains !
"""

if generate_video:
    from matplotlib import animation
    fig = plt.figure()
    sp1 = plt.subplot(131)
    sp1.axis('off')
    sp2 = plt.subplot(132)
    sp2.axis('off')
    sp3 = plt.subplot(133)
    sp3.axis('off')
    ims = []
    for i, t in enumerate(y_pred):
        ims.append((
            sp1.imshow(np.reshape(y_test[i], (10, 10)), cmap=plt.cm.gray,
            interpolation='nearest'),
            sp2.imshow(np.reshape(t, (10, 10)), cmap=plt.cm.gray,
            interpolation='nearest'),
            sp3.imshow(np.reshape(t > 0.5, (10, 10)), cmap=plt.cm.gray,
            interpolation='nearest')))

    im_ani = animation.ArtistAnimation(fig, ims, interval=1000)
    im_ani.save(generate_video)
<|endoftext|>


<commit_before># Filename: main.py
# Module:	main
# Date:		4th August 2004
# Author:	James Mills, prologic at shortcircuit dot net dot au

"""Main

This is the main module. Everything starts from here
after the command-line options have been parsed and
passed to here by the cli module.
"""

import os
import signal
import optparse

from pymills.utils import getProgName, \
		writePID, daemonize

from core import Core
from env import Environment
from kdb import __name__ as systemName
from kdb import __version__ as systemVersion

USAGE = """"%prog [options] <endPath> <command>

commands:
  start    Start %prog
  stop     Stop %prog
  rehash   Rehash (reload environment)
  initenv  Create a new empty environment for %prog.
  upgrade  Upgrade an existing environment."""

VERSION = "%prog v" + systemVersion

def parse_options():
	"""parse_options() -> opts, args

	Parse and command-line options given returning both
	the parsed options and arguments.
	"""

	parser = optparse.OptionParser(usage=USAGE, version=VERSION)

	parser.add_option("-n", "--no-fork",
			action="store_true", default=False, dest="nofork",
			help="Don't fork to background")

	(opts, args) = parser.parse_args()
	if len(args) != 2:
		parser.print_help()
		raise SystemExit, 1
	return opts, args

def run():
	"""run() -> None

	Parse all command-line arguments and options
	determine what command to run. The environment
	path must be the first argument specified.
	If no valid command is given as the second
	argument, an error is printed and the system
	is terminated with an error code of 1.
	"""

	opts, args = parse_options()

	envPath = args[0]
	command = args[1].upper()

	if command == "START":
		start(envPath, not opts.nofork)
	elif command == "STOP":
		stop(envPath)
	elif command == "RESTART":
		restart(envPath)
	elif command == "REHASH":
		rehash(envPath)
	elif command == "INITENV":
		initEnv(envPath)
	elif command == "UPGRADE":
		upgrade(envPath)
	else:
		print "ERROR: Invalid Command %s" % args[0]
		raise SystemExit, 1

def start(envPath, daemon=True):
	"""start(envPath, daemon=True) -> None

	Start the system. Check if the given envPath is
	valid and doesn't need upgrading. Daemonize if the
	daemon option (default is True) is given. Write the
	pid of this process and run the core.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)
	if env.needsUpgrade():
		print "ERROR: %s Environment '%s' needs upgrading" % (
				systemName, envPath)
		print "Run: %s %s upgrade" % (getProgName, envPath)
		raise SystemExit, 1

	print "-- Starting %s...\n" % systemName

	if daemon:
		daemonize()

	writePID(env.config.get(systemName, "pidfile") % env.path)

	core = Core(env.event, env)
	core.run()
	pidfile = env.config.get(systemName, "pidfile") % env.path
	os.remove(pidfile)

def stop(envPath):
	"""stop(envPath) -> None

	Stop the system by sending the KILL signal to the
	pid found in the environment given by envPath.
	Check if the given environment is valid before
	attempting this. If an error occurs while trying
	to do this, the error is printed and exitcode 1
	is returned.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)

	try:
		os.kill(int(open(env.config.get(
			systemName, "pidfile") % env.path).read()),
			signal.SIGTERM)
		print "-- %s Stopped" % systemName
	except Exception, e:
		print "*** ERROR: Could not stop %s..." % systemName
		print str(e)
		raise SystemExit, 1

def restart(envPath):
	"""restart(envPath) -> None

	Attempt a restart of the system by first stopping the
	system then starting it again.
	"""

	stop(envPath)
	start(envPath)

def rehash(envPath):
	"""rehash(envPath) -> None

	Rehash the system by sending the SIGUP signal to the
	pid found in the environment given by envPath.
	Check if the given environment is valid before
	attempting this. If an error occurs while trying
	to do this, the error is printed and exitcode 1
	is returned.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)

	try:
		os.kill(int(open(env.config.get(
			systemName, "pidfile") % env.path).read()),
			signal.SIGHUP)
		print "-- %s Rehashed" % systemName
	except Exception, e:
		raise
		print "*** ERROR: Could not rehash %s..." % systemName
		print str(e)
		raise SystemExit, 1

	raise SystemExit, 0

def initEnv(envPath):
	"""initEnv(envPath) -> None

	Initialize (create) a new environment using the path
	specified by envPath. Check that the path doesn't
	already exist, printing an error and returning an
	exitcode of 1 if it does.
	"""

	if os.path.exists(envPath):
		print "ERROR: Path '%s' already exists" % envPath
		raise SystemExit, 1

	env = Environment(envPath, create=True)

	print "%s Environment created at %s" % (systemName, envPath)
	print "You can run %s now:" % systemName
	print "   %s %s start" % (systemName, envPath)

	raise SystemExit, 0

def upgrade(envPath):
	"""upgrade(envPath) -> None

	Upgrade the environment at the path specified by
	envPath. Check if the path exists, printing an
	error fna returning an exitcode of 1 if it doesn't.
	Check that the environment actually needs upgrading,
	printing an error if it doesn't and returning an
	exitcode of 1. Otherwise upgrade the environment.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)
	if not env.needsUpgrade():
		print "ERROR: Upgrade not necessary for " \
				"%s Environment %s" % (systemName, envPath)
		raise SystemExit, 1

	env.upgrade()
	print "%s Environment upgraded." % systemName

	raise SystemExit, 0
<commit_msg>Fixed doc string for update function<commit_after># Filename: main.py
# Module:	main
# Date:		4th August 2004
# Author:	James Mills, prologic at shortcircuit dot net dot au

"""Main

This is the main module. Everything starts from here
after the command-line options have been parsed and
passed to here by the cli module.
"""

import os
import signal
import optparse

from pymills.utils import getProgName, \
		writePID, daemonize

from core import Core
from env import Environment
from kdb import __name__ as systemName
from kdb import __version__ as systemVersion

USAGE = """"%prog [options] <endPath> <command>

commands:
  start    Start %prog
  stop     Stop %prog
  rehash   Rehash (reload environment)
  initenv  Create a new empty environment for %prog.
  upgrade  Upgrade an existing environment."""

VERSION = "%prog v" + systemVersion

def parse_options():
	"""parse_options() -> opts, args

	Parse and command-line options given returning both
	the parsed options and arguments.
	"""

	parser = optparse.OptionParser(usage=USAGE, version=VERSION)

	parser.add_option("-n", "--no-fork",
			action="store_true", default=False, dest="nofork",
			help="Don't fork to background")

	(opts, args) = parser.parse_args()
	if len(args) != 2:
		parser.print_help()
		raise SystemExit, 1
	return opts, args

def run():
	"""run() -> None

	Parse all command-line arguments and options
	determine what command to run. The environment
	path must be the first argument specified.
	If no valid command is given as the second
	argument, an error is printed and the system
	is terminated with an error code of 1.
	"""

	opts, args = parse_options()

	envPath = args[0]
	command = args[1].upper()

	if command == "START":
		start(envPath, not opts.nofork)
	elif command == "STOP":
		stop(envPath)
	elif command == "RESTART":
		restart(envPath)
	elif command == "REHASH":
		rehash(envPath)
	elif command == "INITENV":
		initEnv(envPath)
	elif command == "UPGRADE":
		upgrade(envPath)
	else:
		print "ERROR: Invalid Command %s" % args[0]
		raise SystemExit, 1

def start(envPath, daemon=True):
	"""start(envPath, daemon=True) -> None

	Start the system. Check if the given envPath is
	valid and doesn't need upgrading. Daemonize if the
	daemon option (default is True) is given. Write the
	pid of this process and run the core.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)
	if env.needsUpgrade():
		print "ERROR: %s Environment '%s' needs upgrading" % (
				systemName, envPath)
		print "Run: %s %s upgrade" % (getProgName, envPath)
		raise SystemExit, 1

	print "-- Starting %s...\n" % systemName

	if daemon:
		daemonize()

	writePID(env.config.get(systemName, "pidfile") % env.path)

	core = Core(env.event, env)
	core.run()
	pidfile = env.config.get(systemName, "pidfile") % env.path
	os.remove(pidfile)

def stop(envPath):
	"""stop(envPath) -> None

	Stop the system by sending the KILL signal to the
	pid found in the environment given by envPath.
	Check if the given environment is valid before
	attempting this. If an error occurs while trying
	to do this, the error is printed and exitcode 1
	is returned.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)

	try:
		os.kill(int(open(env.config.get(
			systemName, "pidfile") % env.path).read()),
			signal.SIGTERM)
		print "-- %s Stopped" % systemName
	except Exception, e:
		print "*** ERROR: Could not stop %s..." % systemName
		print str(e)
		raise SystemExit, 1

def restart(envPath):
	"""restart(envPath) -> None

	Attempt a restart of the system by first stopping the
	system then starting it again.
	"""

	stop(envPath)
	start(envPath)

def rehash(envPath):
	"""rehash(envPath) -> None

	Rehash the system by sending the SIGUP signal to the
	pid found in the environment given by envPath.
	Check if the given environment is valid before
	attempting this. If an error occurs while trying
	to do this, the error is printed and exitcode 1
	is returned.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)

	try:
		os.kill(int(open(env.config.get(
			systemName, "pidfile") % env.path).read()),
			signal.SIGHUP)
		print "-- %s Rehashed" % systemName
	except Exception, e:
		raise
		print "*** ERROR: Could not rehash %s..." % systemName
		print str(e)
		raise SystemExit, 1

	raise SystemExit, 0

def initEnv(envPath):
	"""initEnv(envPath) -> None

	Initialize (create) a new environment using the path
	specified by envPath. Check that the path doesn't
	already exist, printing an error and returning an
	exitcode of 1 if it does.
	"""

	if os.path.exists(envPath):
		print "ERROR: Path '%s' already exists" % envPath
		raise SystemExit, 1

	env = Environment(envPath, create=True)

	print "%s Environment created at %s" % (systemName, envPath)
	print "You can run %s now:" % systemName
	print "   %s %s start" % (systemName, envPath)

	raise SystemExit, 0

def upgrade(envPath):
	"""upgrade(envPath) -> None

	Upgrade the environment at the path specified by
	envPath. Check if the path exists, printing an
	error and returning an exitcode of 1 if it doesn't
	exit..
	Check that the environment actually needs upgrading,
	printing an error if it doesn't and returning an
	exitcode of 1. Otherwise upgrade the environment.
	"""

	if not os.path.exists(envPath):
		print "ERROR: Path not found '%s'" % envPath
		raise SystemExit, 1

	env = Environment(envPath)
	if not env.needsUpgrade():
		print "ERROR: Upgrade not necessary for " \
				"%s Environment %s" % (systemName, envPath)
		raise SystemExit, 1

	env.upgrade()
	print "%s Environment upgraded." % systemName

	raise SystemExit, 0
<|endoftext|>