<commit_before>import pandas as pd
import numpy as np
import os
import itertools
import h5py
from multiprocessing import Manager, Queue, Process

class HDF5FileExistsError(Exception):
    pass

class GamFrequencyMatrix(object):
    """A class for abstracting access to the HDF5 store underlying a frequency matrix"""

    def __init__(self, hdf5_store):

        self.data = self.data_from_store(hdf5_store)

    def data_from_store(self, hdf5_store):
        """Get a frequency matrix from the store"""

        return hdf5_store["processed_data"]["frequencies"]

    def any_empty(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        data_array = self.data[loc1_start:loc1_stop, loc2_start:loc2_stop]

        for row in data_array:
            for cell in row:
                if not cell.sum():
                   return True

        return False

    def get_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        if self.any_empty(loc1_start, loc1_stop, loc2_start, loc2_stop):
            return None

        else:
            print 'Using cache'
            return self.data[loc1_start:loc1_stop, loc2_start:loc2_stop]

    def cache_freqs(self, loc1_start, loc1_stop, loc2_start, loc2_stop, freqs):

        self.data[loc1_start:loc1_stop, loc2_start:loc2_stop] = freqs

    @staticmethod
    def create_freq_matrix(hdf5_store, no_windows, compression=None):
        """Create a frequency matrix dataset stored in the HDF5 file"""

        grp = hdf5_store.create_group("processed_data")

        freq_matrix = grp.create_dataset("frequencies", (no_windows,no_windows,2,2), dtype='i', compression=compression)

        return freq_matrix

    @staticmethod
    def from_no_windows(hdf5_store, no_windows, compression=None):

        matrix = GamFrequencyMatrix.create_freq_matrix(hdf5_store, no_windows, compression=compression)

        return GamFrequencyMatrix(hdf5_store)

class GamExperimentalData(object):
    """A class for abstracting access to the original experimental segmentation"""

    def __init__(self, hdf5_store, pseudocount = 0):

        self.store = hdf5_store

        self.data = self.data_from_store()

        self.no_windows = len(self.data.columns)

        self.pseudocount = pseudocount

    def data_from_store(self):
        """Retrieve data from an hdf5 store and use it to recreate the pandas DataFrame"""

        df_data = np.array(self.store["experimental_data"]["segmentation"][:])
        df_columns = pd.MultiIndex.from_tuples(map(lambda i: tuple(i), np.array(self.store["experimental_data"]["columns"][:])))
        df_index = np.array(self.store["experimental_data"]["index"][:])

        return pd.DataFrame(data=df_data, index=df_index, columns=df_columns)

    @staticmethod
    def from_multibam(multibam_file, hdf5_store, compression=None):
        """Read in a multibam file and return a GamExperimentalData object"""

        experimental_data = GamExperimentalData.read_multibam(multibam_file)

        GamExperimentalData.data_to_store(experimental_data, hdf5_store)

        return GamExperimentalData(hdf5_store)

    @staticmethod
    def read_multibam(input_file):
        """Read in a multibam file and return a Pandas DataFrame"""

        return pd.read_csv(input_file, delim_whitespace=True,index_col=[0,1]).transpose()

    @staticmethod
    def data_to_store(data, hdf5_store, compression=None):
        """Save experimental data to an hdf5 store"""

        grp = hdf5_store.create_group("experimental_data")

        #print data
        segmentation = np.array(data)
        #print segmentation
        segmentation_hd = grp.create_dataset("segmentation", segmentation.shape, segmentation.dtype, compression=compression)

        segmentation_hd.write_direct(segmentation)

        index = np.array(list(data.index))

        index_hd = grp.create_dataset("index", index.shape, index.dtype)

        index_hd.write_direct(index)

        columns = np.array(list(data.columns))

        columns_hd = grp.create_dataset("columns", columns.shape, columns.dtype)

        columns_hd.write_direct(columns)

class GamExperiment(object):
    """A class for storing and processing data associated with a GAM Experiment"""

    def __init__(self, hdf5_path, num_processes=1):
        """Open a saved gam_experiment"""

        # Store the number of processes available for matrix computation
        self.num_processes = num_processes

        # Starting the worker pool forks the process, so do this as early as possible to
        # reduce memory usage
        if self.num_processes > 1:
            self.processor = MultithreadedMatrixProcesser(num_processes)
        else:
            self.processor = MatrixProcesser()

        # Open the hdf5 store
        self.store = self.open_store(hdf5_path)

        # Get the experimental data from the store
        self.experimental_data = GamExperimentalData(self.store)

        # Get the frequency matrix
        self.freq_matrix = GamFrequencyMatrix(self.store)

    def get_chrom_processed_matrix(self, chrom, method=None):

        starting_index = ( chrom, self.experimental_data.data[chrom].columns[0] )
        stopping_index = ( chrom, self.experimental_data.data[chrom].columns[-1] )

        return self.get_index_processed_matrix(starting_index, stopping_index, starting_index, stopping_index, method)

    def get_index_processed_matrix(self, index1_start, index1_stop, index2_start, index2_stop, method=None):

        loc1_start = self.experimental_data.data.columns.get_loc(index1_start)
        loc1_stop = self.experimental_data.data.columns.get_loc(index1_stop)
        loc2_start = self.experimental_data.data.columns.get_loc(index2_start)
        loc2_stop = self.experimental_data.data.columns.get_loc(index2_stop)

        return self.get_loc_processed_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop, method)

    def calculate_loc_frequency_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        index1_list = self.experimental_data.data.columns[loc1_start:loc1_stop]
        index2_list = self.experimental_data.data.columns[loc2_start:loc2_stop]

        data_1 = np.array(self.experimental_data.data.iloc[:,loc1_start:loc1_stop])
        len_1 = len(data_1[0])
        data_2 = np.array(self.experimental_data.data.iloc[:,loc2_start:loc2_stop])
        len_2 = len(data_2[0])
        full_data = np.concatenate((data_1, data_2), axis=1)

        return self.processor.process(len_1, len_2, full_data)

    def get_loc_frequency_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        freqs = self.freq_matrix.get_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop)

        if freqs is None:

            freqs = self.calculate_loc_frequency_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop)

            self.freq_matrix.cache_freqs(loc1_start, loc1_stop, loc2_start, loc2_stop, freqs)

        return freqs

    def get_loc_processed_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop, method=None):

        if method is None:
            method = self.odds_ratio

        index1_list = self.experimental_data.data.columns[loc1_start:loc1_stop]
        index2_list = self.experimental_data.data.columns[loc2_start:loc2_stop]

        freqs = self.get_loc_frequency_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop)

        stored_shape = freqs.shape[:2]

        processed = map(method, freqs.reshape((stored_shape[0] * stored_shape[1],2,2)))

        return np.array(processed).reshape(stored_shape)

    def odds_ratio(self, N):
        return np.log(N[0,0]) + np.log(N[1,1]) - np.log(np.log(N[0,1])) - np.log(N[1,0])

    def close(self):

        self.processor.close()
        self.store.close()

    @staticmethod
    def from_multibam(segmentation_multibam, hdf5_path, num_processes=1, compression=None):
        """Create a new experiment from a segmentation multibam file"""

        # Create the hdf5 store
        store = GamExperiment.create_store(hdf5_path)

        # Create the experimental data in the datastore
        experimental_data = GamExperimentalData.from_multibam(segmentation_multibam, store, compression)

        # Create a new frequency matrix
        freq_matrix = GamFrequencyMatrix.from_no_windows(store, experimental_data.no_windows, compression)

        # close the store
        store.close()

        # Return the object
        return GamExperiment(hdf5_path, num_processes)

    @staticmethod
    def create_store(hdf5_path):
        """Create a new hdf5 store without overwriting an old one"""

        if os.path.exists(hdf5_path):
            # Don't overwrite an existing hdf5 file
            raise HDF5FileExistsError('The file {} already exists and would be overwritten by this operation. Please provide the path for a new HDF5 file'.format(hdf5_path))

        # Create the store
        store = h5py.File(hdf5_path,'w')

        return store

    def open_store(self, hdf5_path):
        """Open an hdf5 store and return the store object"""

        if not os.path.exists(hdf5_path):
            raise IOError("[Errno 2] No such file or directory: '{}'".format(hdf5_path))

        store = h5py.File(hdf5_path,'r+')

        return store

    def close(self):

        self.store.close()

def count_frequency(samples):
    """Take a table of two columnds and return [[ no_both_present, no_1_only],[ no_2_only, no_neither_present]]"""

    counts = np.array([[0,0], [0,0]])

    for s in samples:

        counts[s[0]][s[1]] += 1

    return counts

def f(input_queue, result_dict):
    key, data = input_queue.get()
    while key is not None:
        result_dict[key] = count_frequency(data)
        key,data = input_queue.get()

class WorkerPool(object):

    def __init__(self, num_proc):

        self.num_processes = num_proc

        self.queue = Queue(maxsize=self.num_processes)

        self.manager = Manager()
        self.results = self.manager.dict()

        self.processes = []
        for n in range(self.num_processes):
            self.processes.append(Process(target=f, args=(self.queue, self.results)))

        self.start()

    def start(self):

        map(lambda p: p.start(), self.processes)

    def stop(self):

        while not self.queue.empty():
            pass

    def close(self):

        for n in range(self.num_processes):
            self.queue.put((None, None))

        map(lambda p: p.join(), self.processes)

class MatrixProcesser(object):

    def process(self, len_1, len_2, full_data):

        combinations = itertools.product(range(len_1), range(len_1, len_1 + len_2))

        freqs = np.zeros((len_1, len_2, 2, 2))

        for key in combinations:
            loc1, loc2 = key
            data = full_data[:,[loc1, loc2]]
            result = count_frequency(data)
            loc2 = loc2 - len_1
            freqs[loc1][loc2] = result

        return freqs

    def close(self):
        pass

class MultithreadedMatrixProcesser(object):

    def __init__(self, num_processes):

        self.num_processes = num_processes
        self.worker_pool = WorkerPool(num_processes)

    def process(self, len_1, len_2, full_data):

        combinations = itertools.product(range(len_1), range(len_1, len_1 + len_2))

        print 'Using {0} processes'.format(self.num_processes)

        for key in combinations:
            loc1, loc2 = key
            data = full_data[:,[loc1, loc2]]
            self.worker_pool.queue.put((key, data))

        self.worker_pool.stop()

        print 'Done processing'
        while not len(self.worker_pool.results.keys()) == len_1 * len_2:
            pass

        freqs = np.zeros((len_1, len_2, 2, 2))

        for key in self.worker_pool.results.keys():
            loc1, loc2 = key
            loc2 = loc2 - len_1
            freqs[loc1][loc2] = self.worker_pool.results[key]

        self.clean()

        return freqs

    def clean(self):

        self.worker_pool.results.clear()

    def close(self):

        self.worker_pool.close()
<commit_msg>Stopped using pandas to reduce dependecies.<commit_after>import numpy as np
import os
import itertools
import h5py
from multiprocessing import Manager, Queue, Process

class HDF5FileExistsError(Exception):
    pass

class GamFrequencyMatrix(object):
    """A class for abstracting access to the HDF5 store underlying a frequency matrix"""

    def __init__(self, data):

        self.data = data

    @staticmethod
    def from_store(hdf5_store):
        """Get a frequency matrix from the store"""

        data = hdf5_store.store["processed_data"]["frequencies"]
        return GamFrequencyMatrix(data)

    @staticmethod
    def from_no_windows(hdf5_store, no_windows):

        data = hdf5_store.create_freq_matrix(no_windows)

        return GamFrequencyMatrix(data)

    def any_empty(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        data_array = self.data[loc1_start:loc1_stop, loc2_start:loc2_stop]

        for row in data_array:
            for cell in row:
                if not cell.sum():
                   return True

        return False

    def get_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        if self.any_empty(loc1_start, loc1_stop, loc2_start, loc2_stop):
            return None

        else:
            print 'Using cache'
            return self.data[loc1_start:loc1_stop, loc2_start:loc2_stop]

    def cache_freqs(self, loc1_start, loc1_stop, loc2_start, loc2_stop, freqs):

        self.data[loc1_start:loc1_stop, loc2_start:loc2_stop] = freqs

class MultibamProcessor(object):
    """A class for extracting GAM experimental data from the multibam style segmentation file."""
    def __init__(self):
        super(MultibamProcessor, self).__init__()

    def read(self, multibam_pointer):
        """Actually read data from the multibam pointer"""

        windows = []
        data = []
        for lnum, line in enumerate(multibam_pointer):
            if lnum == 0:
                sample_names = self.extract_sample_names(line)
            else:
                window, segmentation = self.parse_line(line)
                windows.append(window)
                data.append(segmentation)

        data = np.array(data).transpose()
        data = np.ascontiguousarray(data, data.dtype)

        return data, sample_names, windows

    def parse_line(self, line):
        """Extract information from one line of the multibam"""

        fields = line.strip().split()
        chrom = fields[0]
        start, end = map(int, fields[1].split('-'))
        window = (chrom, start, end)
        segmentation = map(int,fields[2:])

        return window, segmentation

    def extract_sample_name(self, file_path):
        basename = os.path.basename(file_path)
        return basename.split('.')[0]

    def extract_sample_names(self, header_line):
        return map(self.extract_sample_name, header_line.split()[2:])

class GamHdf5Store(object):

    def __init__(self, store, compression):

        self.store, self.compression = store, compression

    @staticmethod
    def create_store(hdf5_path, compression=None):
        """Create a new hdf5 store without overwriting an old one"""

        if os.path.exists(hdf5_path):
            # Don't overwrite an existing hdf5 file
            raise HDF5FileExistsError('The file {0} already exists and would be overwritten by this operation. Please provide the path for a new HDF5 file'.format(hdf5_path))

        # Create the store
        store = h5py.File(hdf5_path,'w')

        return GamHdf5Store(store, compression)

    @staticmethod
    def open_store(hdf5_path):
        """Open an hdf5 store and return the store object"""

        if not os.path.exists(hdf5_path):
            raise IOError("[Errno 2] No such file or directory: '{0}'".format(hdf5_path))

        store = h5py.File(hdf5_path,'r+')

        # Get the compression
        compression = store['processed_data']['frequencies'].compression

        return GamHdf5Store(store, compression)

    def data_to_store(self, data, columns, windows):
        """Save experimental data to an hdf5 store"""

        grp = self.store.create_group("experimental_data")

        segmentation_hd = grp.create_dataset("segmentation", data.shape, data.dtype, compression=self.compression)

        segmentation_hd.write_direct(data)

        windows = np.array(windows)

        index_hd = grp.create_dataset("windows", windows.shape, windows.dtype)

        index_hd.write_direct(windows)

        columns = np.array(columns)

        columns_hd = grp.create_dataset("columns", columns.shape, columns.dtype)

        columns_hd.write_direct(columns)

    def data_from_store(self):
        """Retrieve data from an hdf5 store and use it to recreate the DataFrame"""

        data = np.array(self.store["experimental_data"]["segmentation"][:])
        columns = map(lambda i: tuple(i), np.array(self.store["experimental_data"]["columns"][:]))
        windows = np.array(self.store["experimental_data"]["windows"][:])

        return data, columns, windows

    def create_freq_matrix(self, no_windows):
        """Create a frequency matrix dataset stored in the HDF5 file"""

        grp = self.store.create_group("processed_data")

        freq_matrix = grp.create_dataset("frequencies", (no_windows,no_windows,2,2), dtype='i', compression=self.compression)

        return freq_matrix

class GamExperimentalData(object):
    """A class for abstracting access to the original experimental segmentation"""

    def __init__(self, data, columns, windows, hdf5_store):

        self.data, self.columns, self.windows, self.store = data, columns, windows, hdf5_store
        self.no_windows = len(self.windows)

    @staticmethod
    def from_multibam(multibam_path, hdf5_store):

        with open(multibam_path, 'r') as multibam_pointer:
            data, columns, windows = MultibamProcessor().read(multibam_pointer)

        hdf5_store.data_to_store(data, columns, windows)

        return GamExperimentalData(data, columns, windows, hdf5_store)

    @staticmethod
    def from_store(hdf5_store):

        data, columns, windows = hdf5_store.data_from_store()

        return GamExperimentalData(data, columns, windows, hdf5_store)

class GamExperiment(object):
    """A class for storing and processing data associated with a GAM Experiment"""

    def __init__(self, hdf5_store, experimental_data, frequency_data, num_processes=1):
        """Open a saved gam_experiment"""

        # Store the number of processes available for matrix computation
        self.num_processes = num_processes

        # Starting the worker pool forks the process, so do this as early as possible to
        # reduce memory usage
        if self.num_processes > 1:
            self.processor = MultithreadedMatrixProcesser(num_processes)
        else:
            self.processor = MatrixProcesser()

        # Open the hdf5 store
        self.store = hdf5_store

        # Get the experimental data from the store
        self.experimental_data = experimental_data

        # Get the frequency matrix
        self.freq_matrix = frequency_data

    @staticmethod
    def from_multibam(segmentation_multibam, hdf5_path, num_processes=1, compression=None):
        """Create a new experiment from a segmentation multibam file"""

        # Create the hdf5 store
        store = GamHdf5Store.create_store(hdf5_path, compression)

        # Create the experimental data in the datastore
        experimental_data = GamExperimentalData.from_multibam(segmentation_multibam, store)

        # Create a new frequency matrix
        freq_matrix = GamFrequencyMatrix.from_no_windows(store, experimental_data.no_windows)

        # Return the object
        return GamExperiment(store, experimental_data, freq_matrix, num_processes)

    @staticmethod
    def load(hdf5_path, num_processes=1):

        # Load the hdf5 store
        store = GamHdf5Store.open_store(hdf5_path)

        experimental_data = GamExperimentalData.from_store(store)

        freq_matrix = GamFrequencyMatrix.from_store(store)

        # Return the object
        return GamExperiment(store, experimental_data, freq_matrix, num_processes)

    def calculate_loc_frequency_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        data_1 = np.array(self.experimental_data.data[:,loc1_start:loc1_stop])
        len_1 = len(data_1[0])
        data_2 = np.array(self.experimental_data.data[:,loc2_start:loc2_stop])
        len_2 = len(data_2[0])
        full_data = np.concatenate((data_1, data_2), axis=1)

        return self.processor.process(len_1, len_2, full_data)

    def get_loc_frequency_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop):

        freqs = self.freq_matrix.get_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop)

        if freqs is None:

            freqs = self.calculate_loc_frequency_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop)

            self.freq_matrix.cache_freqs(loc1_start, loc1_stop, loc2_start, loc2_stop, freqs)

        return freqs

    def get_loc_processed_matrix(self, loc1_start, loc1_stop, loc2_start, loc2_stop, method=None):

        if method is None:
            method = self.odds_ratio

        freqs = self.get_loc_frequency_matrix(loc1_start, loc1_stop, loc2_start, loc2_stop)

        stored_shape = freqs.shape[:2]

        processed = map(method, freqs.reshape((stored_shape[0] * stored_shape[1],2,2)))

        return np.array(processed).reshape(stored_shape)

    def odds_ratio(self, N):
        return np.log(N[0,0]) + np.log(N[1,1]) - np.log(np.log(N[0,1])) - np.log(N[1,0])

    def close(self):

        self.processor.close()
        self.store.close()

def count_frequency(samples):
    """Take a table of two columnds and return [[ no_both_present, no_1_only],[ no_2_only, no_neither_present]]"""

    counts = np.array([[0,0], [0,0]])

    for s in samples:

        counts[s[0]][s[1]] += 1

    return counts

def f(input_queue, result_dict):
    key, data = input_queue.get()
    while key is not None:
        result_dict[key] = count_frequency(data)
        key,data = input_queue.get()

class WorkerPool(object):

    def __init__(self, num_proc):

        self.num_processes = num_proc

        self.queue = Queue(maxsize=self.num_processes)

        self.manager = Manager()
        self.results = self.manager.dict()

        self.processes = []
        for n in range(self.num_processes):
            self.processes.append(Process(target=f, args=(self.queue, self.results)))

        self.start()

    def start(self):

        map(lambda p: p.start(), self.processes)

    def stop(self):

        while not self.queue.empty():
            pass

    def close(self):

        for n in range(self.num_processes):
            self.queue.put((None, None))

        map(lambda p: p.join(), self.processes)

class MatrixProcesser(object):

    def process(self, len_1, len_2, full_data):

        combinations = itertools.product(range(len_1), range(len_1, len_1 + len_2))

        freqs = np.zeros((len_1, len_2, 2, 2))

        for key in combinations:
            loc1, loc2 = key
            data = full_data[:,[loc1, loc2]]
            result = count_frequency(data)
            loc2 = loc2 - len_1
            freqs[loc1][loc2] = result

        return freqs

    def close(self):
        pass

class MultithreadedMatrixProcesser(object):

    def __init__(self, num_processes):

        self.num_processes = num_processes
        self.worker_pool = WorkerPool(num_processes)

    def process(self, len_1, len_2, full_data):

        combinations = itertools.product(range(len_1), range(len_1, len_1 + len_2))

        print 'Using {0} processes'.format(self.num_processes)

        for key in combinations:
            loc1, loc2 = key
            data = full_data[:,[loc1, loc2]]
            self.worker_pool.queue.put((key, data))

        self.worker_pool.stop()

        print 'Done processing'
        while not len(self.worker_pool.results.keys()) == len_1 * len_2:
            pass

        freqs = np.zeros((len_1, len_2, 2, 2))

        for key in self.worker_pool.results.keys():
            loc1, loc2 = key
            loc2 = loc2 - len_1
            freqs[loc1][loc2] = self.worker_pool.results[key]

        self.clean()

        return freqs

    def clean(self):

        self.worker_pool.results.clear()

    def close(self):

        self.worker_pool.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import pickle
import binascii
import json
from datetime import datetime, timedelta

from Crypto.Hash import SHA256

from ..models import Address, Token, File, Contract

from heartbeat import Heartbeat
from RandomIO import RandomIO
from ..startup import db, app

__all__ = ['create_token', 'delete_token', 'get_chunk_contract', 'add_file', 'remove_file']

def create_token(sjcx_address):
    # confirm that sjcx_address is in the list of addresses
    # for now we have a white list
    address = Address.query.filter(Address.address == sjcx_address).first()

    if (address is None):
        # just put it in the db for testing
        address = Address(address=sjcx_address)
        db.session.add(address)
        db.session.commit()
        #raise RuntimeError(
        #    'Invalid address given: address must be in whitelist.')

    beat = Heartbeat()

    token = Token(token=binascii.hexlify(os.urandom(16)).decode('ascii'),
                  address=address.address,
                  heartbeat=pickle.dumps(beat))

    db.session.add(token)
    db.session.commit()

    return token

def delete_token(token):
    db_token = Token.query.filter(Token.token == token).first()

    if (db_token is None):
        raise RuntimeError('Invalid token given. Token does not exist.')

    db.session.delete(db_token)
    db.session.commit()

def get_chunk_contract(token):
    # first, we need to find all the files that are not meeting their
    # redundancy requirements once we have found a candidate list, we sort
    # by when the file was added so that the most recently added file is
    # given out in a contract

    # verify the token
    db_token = Token.query.filter(Token.token == token).first()

    if (db_token is None):
        raise RuntimeError('Invalid token given.')

    # these are the files we are tracking with their current redundancy counts
    # for now comment this since we're just generating a file for each contract
    # candidates = db.session.query(File,func.count(Contracts.file_hash)).\
        # outerjoin(Contracts).group_by(File.hash).all()

    # if (len(candidates) == 0):
        # return None

    # # sort by add date and current redundancy
    # candidates.sort(key = lambda x: x[0].added)
    # candidates.sort(key = lambda x: x[1])

    # # pick the best candidate
    # file = candidates[0]

    # for prototyping, we generate a file for each contract.
    seed = binascii.hexlify(os.urandom(16))

    file = add_file(RandomIO(seed).genfile(100,app.config['FILES_PATH']),1)

    beat = pickle.loads(db_token.heartbeat)

    with open(file.path,'rb') as f:
        (tag,state) = beat.encode(f)

    chal = beat.gen_challenge(state)

    contract = Contract(token = token,
                        file_hash = file.hash,
                        state = pickle.dumps(state),
                        challenge = pickle.dumps(chal),
                        expiration = datetime.utcnow() + timedelta(seconds = file.interval),
                        # for prototyping, include seed
                        seed = seed)

    db.session.add(contract)
    db.session.commit()

    # and write the tag to our temporary files
    path = os.path.join(app.config['TAGS_PATH'],file.hash)
    with open(path,'wb') as f:
        f.write(pickle.dumps(tag))

    return contract

def add_file(chunk_path, redundancy=3, interval=60):
    # first, hash the chunk to determine it's name
    h = SHA256.new()
    bufsz = 65535

    with open(chunk_path,'rb') as f:
        for c in iter(lambda: f.read(bufsz), b''):
            h.update(c)

    hash = h.hexdigest()

    file = File(hash=hash,
                path=chunk_path,
                redundancy=redundancy,
                interval=interval,
                added=datetime.utcnow())

    db.session.add(file)
    db.session.commit()

    return file

def remove_file(hash):
    # remove the file... contracts should also be deleted by cascading
    file = File.query.filter(File.hash==hash).first()

    if (file is None):
        raise RuntimeError(
            'File does not exist.  Cannot remove non existant file')

    db.session.delete(file)
    db.session.commit()

<commit_msg>fixed style<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import pickle
import binascii
from datetime import datetime, timedelta

from Crypto.Hash import SHA256

from ..models import Address, Token, File, Contract

from heartbeat import Heartbeat
from RandomIO import RandomIO
from ..startup import db, app

__all__ = ['create_token',
           'delete_token',
           'get_chunk_contract',
           'add_file',
           'remove_file']

def create_token(sjcx_address):
    # confirm that sjcx_address is in the list of addresses
    # for now we have a white list
    address = Address.query.filter(Address.address == sjcx_address).first()

    if (address is None):
        # just put it in the db for testing
        address = Address(address=sjcx_address)
        db.session.add(address)
        db.session.commit()
        # raise RuntimeError(
        #    'Invalid address given: address must be in whitelist.')

    beat = Heartbeat()

    token = Token(token=binascii.hexlify(os.urandom(16)).decode('ascii'),
                  address=address.address,
                  heartbeat=pickle.dumps(beat))

    db.session.add(token)
    db.session.commit()

    return token

def delete_token(token):
    db_token = Token.query.filter(Token.token == token).first()

    if (db_token is None):
        raise RuntimeError('Invalid token given. Token does not exist.')

    db.session.delete(db_token)
    db.session.commit()

def get_chunk_contract(token):
    # first, we need to find all the files that are not meeting their
    # redundancy requirements once we have found a candidate list, we sort
    # by when the file was added so that the most recently added file is
    # given out in a contract

    # verify the token
    db_token = Token.query.filter(Token.token == token).first()

    if (db_token is None):
        raise RuntimeError('Invalid token given.')

    # these are the files we are tracking with their current redundancy counts
    # for now comment this since we're just generating a file for each contract
    # candidates = db.session.query(File,func.count(Contracts.file_hash)).\
        # outerjoin(Contracts).group_by(File.hash).all()

    # if (len(candidates) == 0):
        # return None

    # # sort by add date and current redundancy
    # candidates.sort(key = lambda x: x[0].added)
    # candidates.sort(key = lambda x: x[1])

    # # pick the best candidate
    # file = candidates[0]

    # for prototyping, we generate a file for each contract.
    seed = binascii.hexlify(os.urandom(16))

    file = add_file(RandomIO(seed).genfile(100, app.config['FILES_PATH']), 1)

    beat = pickle.loads(db_token.heartbeat)

    with open(file.path, 'rb') as f:
        (tag, state) = beat.encode(f)

    chal = beat.gen_challenge(state)

    contract = Contract(token=token,
                        file_hash=file.hash,
                        state=pickle.dumps(state),
                        challenge=pickle.dumps(chal),
                        expiration=(datetime.utcnow() +
                                    timedelta(seconds=file.interval)),
                        # for prototyping, include seed
                        seed = seed)

    db.session.add(contract)
    db.session.commit()

    # and write the tag to our temporary files
    path = os.path.join(app.config['TAGS_PATH'], file.hash)
    with open(path, 'wb') as f:
        f.write(pickle.dumps(tag))

    return contract

def add_file(chunk_path, redundancy=3, interval=60):
    # first, hash the chunk to determine it's name
    h = SHA256.new()
    bufsz = 65535

    with open(chunk_path, 'rb') as f:
        for c in iter(lambda: f.read(bufsz), b''):
            h.update(c)

    hash = h.hexdigest()

    file = File(hash=hash,
                path=chunk_path,
                redundancy=redundancy,
                interval=interval,
                added=datetime.utcnow())

    db.session.add(file)
    db.session.commit()

    return file

def remove_file(hash):
    # remove the file... contracts should also be deleted by cascading
    file = File.query.filter(File.hash == hash).first()

    if (file is None):
        raise RuntimeError(
            'File does not exist.  Cannot remove non existant file')

    db.session.delete(file)
    db.session.commit()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
import cv2
import sys
import numpy as np
from networktables import NetworkTables

import logging
logging.basicConfig(level=logging.DEBUG)

ip = "10.29.28.2"

def ntInit(table):
    NetworkTables.initialize(ip)
    return NetworkTables.getTable(table)

# From http://stackoverflow.com/questions/4961017/clojure-style-function-threading-in-python
def T(*args):
    return reduce(lambda l, r: r(l), args)

def toGrayscale(src):
    return cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)

def brightPass(src):
    return cv2.inRange(src, 230, 255)

def blur(src):
    return cv2.GaussianBlur(src, (13, 13), 40)

def blobbify(src):
    data = src.flat
    dims = src.shape
    print dims
    for i in range(0, len(data)):
        if data[i] > 1:
            cv2.ellipse(src, (i % dims[0], i // dims[0]), (15, 6), 90, 0, 360, 0xFF, -1)
    return src

def outline(src):
    dst = np.zeros(src.shape, np.uint8)
    im2, contours, hierarchy  = cv2.findContours(src, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(dst, contours, -1, (255), 1)
    return dst, contours

def main(camera = 0):
    vc = ntInit('VisionControl')
    ret = True
    cap = cv2.VideoCapture(camera)
    #cv2.namedWindow("Output")
    while ret:
        ret, frame = cap.read()
        frame, contours = outline(T(frame, toGrayscale, brightPass, blur))
        print len(contours)

        vc.putNumber("frameSum", frame.sum()/255)
        print frame.sum()/255
        if(cv2.waitKey(30) & 0xFF == ord('q')):
            break
    cap.release()
if __name__ == "__main__":
    main()
<commit_msg>Quad detection<commit_after>#!/usr/bin/env python2

import cv2
import sys
import numpy as np
#from networktables import NetworkTables

import logging
logging.basicConfig(level=logging.DEBUG)

ip = "10.29.28.2"

'''def ntInit(table):
    NetworkTables.initialize(ip)
    return NetworkTables.getTable(table)'''

# From http://stackoverflow.com/questions/4961017/clojure-style-function-threading-in-python
def T(*args):
    return reduce(lambda l, r: r(l), args)

def toGrayscale(src):
    return cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)

def brightPass(src):
    return cv2.inRange(src, 230, 255)

def blur(src):
    return cv2.GaussianBlur(src, (13, 13), 40)

def blobbify(src):
    data = src.flat
    dims = src.shape
    print dims
    for i in range(0, len(data)):
        if data[i] > 1:
            cv2.ellipse(src, (i % dims[0], i // dims[0]), (15, 6), 90, 0, 360, 0xFF, -1)
    return src

def outline(src):
    dst = np.zeros(src.shape, np.uint8)
    im2, contours, hierarchy  = cv2.findContours(src, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(dst, contours, -1, (255), 1)
    return dst, contours

def quads(contours):
    heightWidthRatio = 3
    tolerance = 1
    boxes = []
    polys = []
    for c in contours:
        polys.append(cv2.approxPolyDP(c, 5, True))
    polys = filter(lambda c: len(c) == 4, polys)
    return polys

def main(camera = 0):
    #vc = ntInit('VisionControl')
    ret = True
    cap = cv2.VideoCapture(camera)
    cv2.namedWindow("Output")
    while ret:
        ret, frame = cap.read()
        frame, contours = outline(T(frame, toGrayscale, brightPass, blur))
        contours = quads(contours)
        cv2.drawContours(frame, contours, -1, (127), 3)
        cv2.imshow("Output", frame)
        #vc.putNumber("frameSum", frame.sum()/255)
        print frame.sum()/255
        if(cv2.waitKey(30) & 0xFF == ord('q')):
            break
    cap.release()
if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import json
import getopt
import operator

from os import listdir
from dateutil.parser import parse
from dateutil.tz import tzutc, tzlocal
from datetime import datetime
from math import floor

from stem.descriptor import parse_file
from stem.exit_policy import AddressType

class Router():
    def __init__(self, router, tminus):
        self.Fingerprint = router.fingerprint
        self.Address = [router.address]
        self.IsAllowedDefault = router.exit_policy._is_allowed_default
        self.IsAllowed = router.exit_policy.is_exiting_allowed()
        self.Rules = []
        self.Tminus = tminus

def get_hours(td):
    try:
        s = td.total_seconds()
    except AttributeError:
        # workaround for py2.6
        s = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 1e6) / 1e6
    return int(floor(s / 3600))

def main(consensuses, exit_lists):
    exits = {}
    now = datetime.now(tzlocal())

    for f in consensuses:

        # strip -consensus
        d = f[:-10]

        # consensus from t hours ago
        p = parse(d).replace(tzinfo=tzutc())
        t = get_hours(now - p)

        # read in consensus and store routes in exits
        for router in parse_file("data/consensuses/" + f,
                                 "network-status-consensus-3 1.0"):
            if router.fingerprint in exits:
                continue
            r = Router(router, t)
            if r.IsAllowed:
                for x in router.exit_policy._get_rules():
                    r.Rules.append({
                        "IsAddressWildcard": True,
                        "Address": "",
                        "Mask": "",
                        "IsAccept": x.is_accept,
                        "MinPort": x.min_port,
                        "MaxPort": x.max_port
                    })
            exits[router.fingerprint] = r

        # get a corresponding exit list
        m = [x for x in exit_lists if x.startswith(d[:-5])]
        if len(m) == 0:
            continue

        # update exit addresses with data from TorDNSEL
        for descriptor in parse_file("data/exit-lists/" + m[0],
                                     "tordnsel 1.0"):
            e = exits.get(descriptor.fingerprint, None)
            if e is not None:
                if e.Tminus == t:
                    e.Address = []
                for a in descriptor.exit_addresses:
                    if a[0] not in e.Address:
                        e.Address.append(a[0])

    # update all with server descriptor info
    for descriptor in parse_file("data/cached-descriptors",
                                 "server-descriptor 1.0"):
        if descriptor.fingerprint in exits:
            r = exits[descriptor.fingerprint]
            r.IsAllowed = descriptor.exit_policy.is_exiting_allowed()
            if r.IsAllowed:
                rules = []
                for x in descriptor.exit_policy._get_rules():
                    is_address_wildcard = x.is_address_wildcard()
                    mask = ""
                    if not is_address_wildcard:
                        address_type = x.get_address_type()
                        if (address_type == AddressType.IPv4 and
                            x._masked_bits != 32) or \
                            (address_type == AddressType.IPv6 and
                                x._masked_bits != 128):
                            mask = x.get_mask()
                    rules.append({
                        "IsAddressWildcard": is_address_wildcard,
                        "Address": "" if x.address is None else x.address,
                        "Mask": "" if mask is None else mask,
                        "IsAccept": x.is_accept,
                        "MinPort": x.min_port,
                        "MaxPort": x.max_port
                    })
                r.Rules = rules

    # output exits to file
    with open("data/exit-policies", "w") as exit_file:
        for e in exits:
            if exits[e].IsAllowed:
                exit_file.write(json.dumps(exits[e].__dict__) + "\n")

def print_help(c):
    print "Usage:", sys.argv[0], "-n <int>"
    sys.exit(c)

if __name__ == "__main__":

    # data
    consensuses = listdir("data/consensuses")
    consensuses.sort(reverse=True)
    exit_lists = listdir("data/exit-lists")

    # args
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hn:", [])
    except getopt.GetoptError:
        print_help(2)
    for opt, arg in opts:
        if opt == "-h":
            print_help(0)
        elif opt in ("-n"):
            consensuses = consensuses[:int(arg)]

    main(consensuses, exit_lists)
<commit_msg>Turn off descriptor validation<commit_after>#!/usr/bin/env python

import sys
import json
import getopt
import operator

from os import listdir
from dateutil.parser import parse
from dateutil.tz import tzutc, tzlocal
from datetime import datetime
from math import floor

from stem.descriptor import parse_file
from stem.exit_policy import AddressType

class Router():
    def __init__(self, router, tminus):
        self.Fingerprint = router.fingerprint
        self.Address = [router.address]
        self.IsAllowedDefault = router.exit_policy._is_allowed_default
        self.IsAllowed = router.exit_policy.is_exiting_allowed()
        self.Rules = []
        self.Tminus = tminus

def get_hours(td):
    try:
        s = td.total_seconds()
    except AttributeError:
        # workaround for py2.6
        s = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 1e6) / 1e6
    return int(floor(s / 3600))

def main(consensuses, exit_lists):
    exits = {}
    now = datetime.now(tzlocal())

    for f in consensuses:

        # strip -consensus
        d = f[:-10]

        # consensus from t hours ago
        p = parse(d).replace(tzinfo=tzutc())
        t = get_hours(now - p)

        # read in consensus and store routes in exits
        for router in parse_file("data/consensuses/" + f,
                                 "network-status-consensus-3 1.0",
                                 validate=False):
            if router.fingerprint in exits:
                continue
            r = Router(router, t)
            if r.IsAllowed:
                for x in router.exit_policy._get_rules():
                    r.Rules.append({
                        "IsAddressWildcard": True,
                        "Address": "",
                        "Mask": "",
                        "IsAccept": x.is_accept,
                        "MinPort": x.min_port,
                        "MaxPort": x.max_port
                    })
            exits[router.fingerprint] = r

        # get a corresponding exit list
        m = [x for x in exit_lists if x.startswith(d[:-5])]
        if len(m) == 0:
            continue

        # update exit addresses with data from TorDNSEL
        for descriptor in parse_file("data/exit-lists/" + m[0],
                                     "tordnsel 1.0", validate=False):
            e = exits.get(descriptor.fingerprint, None)
            if e is not None:
                if e.Tminus == t:
                    e.Address = []
                for a in descriptor.exit_addresses:
                    if a[0] not in e.Address:
                        e.Address.append(a[0])

    # update all with server descriptor info
    for descriptor in parse_file("data/cached-descriptors",
                                 "server-descriptor 1.0", validate=False):
        if descriptor.fingerprint in exits:
            r = exits[descriptor.fingerprint]
            r.IsAllowed = descriptor.exit_policy.is_exiting_allowed()
            if r.IsAllowed:
                rules = []
                for x in descriptor.exit_policy._get_rules():
                    is_address_wildcard = x.is_address_wildcard()
                    mask = ""
                    if not is_address_wildcard:
                        address_type = x.get_address_type()
                        if (address_type == AddressType.IPv4 and
                            x._masked_bits != 32) or \
                            (address_type == AddressType.IPv6 and
                                x._masked_bits != 128):
                            mask = x.get_mask()
                    rules.append({
                        "IsAddressWildcard": is_address_wildcard,
                        "Address": "" if x.address is None else x.address,
                        "Mask": "" if mask is None else mask,
                        "IsAccept": x.is_accept,
                        "MinPort": x.min_port,
                        "MaxPort": x.max_port
                    })
                r.Rules = rules

    # output exits to file
    with open("data/exit-policies", "w") as exit_file:
        for e in exits:
            if exits[e].IsAllowed:
                exit_file.write(json.dumps(exits[e].__dict__) + "\n")

def print_help(c):
    print "Usage:", sys.argv[0], "-n <int>"
    sys.exit(c)

if __name__ == "__main__":

    # data
    consensuses = listdir("data/consensuses")
    consensuses.sort(reverse=True)
    exit_lists = listdir("data/exit-lists")

    # args
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hn:", [])
    except getopt.GetoptError:
        print_help(2)
    for opt, arg in opts:
        if opt == "-h":
            print_help(0)
        elif opt in ("-n"):
            consensuses = consensuses[:int(arg)]

    main(consensuses, exit_lists)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from .s_run import SimRun, SimRunMeta
from .s_exception import SimProcedureError
from .s_helpers import get_and_remove, flagged
import itertools

import logging
l = logging.getLogger(name = "s_absfunc")

symbolic_count = itertools.count()

class SimRunProcedureMeta(SimRunMeta):
	def __call__(mcs, *args, **kwargs):
		addr_from = get_and_remove(kwargs, 'addr_from')
		stmt_from = get_and_remove(kwargs, 'stmt_from')
		convention = get_and_remove(kwargs, 'convention')

		c = super(SimRunProcedureMeta, mcs).make_run(args, kwargs)
		SimProcedure.__init__(c, addr_from=addr_from, stmt_from=stmt_from, convention=convention)
		if not hasattr(c.__init__, 'flagged'):
			c.__init__(*args[1:], **kwargs)
		return c

class SimProcedure(SimRun):
	__metaclass__ = SimRunProcedureMeta

	# The SimProcedure constructor, when receiving a None mode and options, defaults to mode="static"
	#
	#	calling convention is one of: "systemv_x64", "syscall", "microsoft_x64", "cdecl", "arm", "mips"
	@flagged
	def __init__(self, addr_from=-1, stmt_from=-1, convention=None): # pylint: disable=W0231
		self.addr_from = addr_from
		self.stmt_from = stmt_from
		self.convention = None
		self.set_convention(convention)

	def initialize_run(self):
		pass

	def handle_run(self):
		self.handle_procedure()

	def handle_procedure(self):
		raise Exception("SimProcedure.handle_procedure() has been called. This should have been overwritten in class %s.", self.__class__)

	def set_convention(self, convention=None):
		if convention is None:
			if self.state.arch.name == "AMD64":
				convention = "systemv_x64"
			elif self.state.arch.name == "x86":
				convention = "cdecl"
			elif self.state.arch.name == "arm":
				convention = "arm"
			elif self.state.arch.name == "mips":
				convention = "os2_mips"

		self.convention = convention

	# Helper function to get an argument, given a list of register locations it can be and stack information for overflows.
	def arg_reg_stack(self, reg_offsets, stack_skip, stack_step, index):
		if index < len(reg_offsets):
			return self.state.reg_expr(reg_offsets[index])
		else:
			index -= len(reg_offsets)
			return self.state.stack_read(stack_step * (index + stack_skip))

	# Returns a bitvector expression representing the nth argument of a function
	def get_arg_expr(self, index):
		if self.convention == "systemv_x64" and self.state.arch.name == "AMD64":
			reg_offsets = [ 72, 64, 32, 24, 80, 88 ] # rdi, rsi, rdx, rcx, r8, r9
			return self.arg_reg_stack(reg_offsets, 1, -8, index)
		elif self.convention == "syscall" and self.state.arch.name == "AMD64":
			reg_offsets = [ 72, 64, 32, 96, 80, 88 ] # rdi, rsi, rdx, r10, r8, r9
			return self.arg_reg_stack(reg_offsets, 1, -8, index)

		raise SimProcedureError("Unsupported calling convention %s for arguments", self.convention)

	# Sets an expression as the return value. Also updates state.
	def set_return_expr(self, expr):
		if self.state.arch.name == "AMD64":
			self.state.store_reg(16, expr)

		raise SimProcedureError("Unsupported calling convention %s for returns", self.convention)

	# Does a return (pop, etc) and returns an bitvector expression representing the return value. Also updates state.
	def do_return(self):
		if self.state.arch.name == "AMD64":
			return self.state.stack_pop()

		raise SimProcedureError("Unsupported platform %s for return emulation.", self.state.arch.name)
<commit_msg>new interface for returns<commit_after>#!/usr/bin/env python

from .s_run import SimRun, SimRunMeta
from .s_exception import SimProcedureError
from .s_helpers import get_and_remove, flagged
from .s_exit import SimExit
import itertools

import logging
l = logging.getLogger(name = "s_absfunc")

symbolic_count = itertools.count()

class SimRunProcedureMeta(SimRunMeta):
	def __call__(mcs, *args, **kwargs):
		addr_from = get_and_remove(kwargs, 'addr_from')
		stmt_from = get_and_remove(kwargs, 'stmt_from')
		convention = get_and_remove(kwargs, 'convention')

		c = super(SimRunProcedureMeta, mcs).make_run(args, kwargs)
		SimProcedure.__init__(c, addr_from=addr_from, stmt_from=stmt_from, convention=convention)
		if not hasattr(c.__init__, 'flagged'):
			c.__init__(*args[1:], **kwargs)
		return c

class SimProcedure(SimRun):
	__metaclass__ = SimRunProcedureMeta

	# The SimProcedure constructor, when receiving a None mode and options, defaults to mode="static"
	#
	#	calling convention is one of: "systemv_x64", "syscall", "microsoft_x64", "cdecl", "arm", "mips"
	@flagged
	def __init__(self, addr_from=-1, stmt_from=-1, convention=None): # pylint: disable=W0231
		self.addr_from = addr_from
		self.stmt_from = stmt_from
		self.convention = None
		self.set_convention(convention)

	def initialize_run(self):
		pass

	def handle_run(self):
		self.handle_procedure()

	def handle_procedure(self):
		raise Exception("SimProcedure.handle_procedure() has been called. This should have been overwritten in class %s.", self.__class__)

	def set_convention(self, convention=None):
		if convention is None:
			if self.state.arch.name == "AMD64":
				convention = "systemv_x64"
			elif self.state.arch.name == "x86":
				convention = "cdecl"
			elif self.state.arch.name == "arm":
				convention = "arm"
			elif self.state.arch.name == "mips":
				convention = "os2_mips"

		self.convention = convention

	# Helper function to get an argument, given a list of register locations it can be and stack information for overflows.
	def arg_reg_stack(self, reg_offsets, stack_skip, stack_step, index):
		if index < len(reg_offsets):
			return self.state.reg_expr(reg_offsets[index])
		else:
			index -= len(reg_offsets)
			return self.state.stack_read(stack_step * (index + stack_skip))

	# Returns a bitvector expression representing the nth argument of a function
	def get_arg_expr(self, index):
		if self.convention == "systemv_x64" and self.state.arch.name == "AMD64":
			reg_offsets = [ 72, 64, 32, 24, 80, 88 ] # rdi, rsi, rdx, rcx, r8, r9
			return self.arg_reg_stack(reg_offsets, 1, -8, index)
		elif self.convention == "syscall" and self.state.arch.name == "AMD64":
			reg_offsets = [ 72, 64, 32, 96, 80, 88 ] # rdi, rsi, rdx, r10, r8, r9
			return self.arg_reg_stack(reg_offsets, 1, -8, index)

		raise SimProcedureError("Unsupported calling convention %s for arguments", self.convention)

	# Sets an expression as the return value. Also updates state.
	def set_return_expr(self, expr):
		if self.state.arch.name == "AMD64":
			self.state.store_reg(16, expr)

		raise SimProcedureError("Unsupported calling convention %s for returns", self.convention)

	# Does a return (pop, etc) and returns an bitvector expression representing the return value. Also updates state.
	def do_return(self):
		if self.state.arch.name == "AMD64":
			return self.state.stack_pop()

		raise SimProcedureError("Unsupported platform %s for return emulation.", self.state.arch.name)

	# Adds an exit representing the function returning. Modifies the state.
	def exit_return(self, expr=None):
		if expr is not None: self.set_return_expr(expr)
		ret_target = self.do_return()
		self.add_exits(SimExit(expr=ret_target, state=self.state))
<|endoftext|>


<commit_before>"""
Find phase singularities, i.e. spiral tips
"""

import sys, os

import numpy as np

from scipy import ndimage
from skimage.draw import circle_perimeter
from skimage.restoration import unwrap_phase

import matplotlib.pylab as plt
import matplotlib.cm as cm
from mpl_toolkits.axes_grid1 import make_axes_locatable

from utils import timed_run, save_data

def compute_tau(camp):
    """ Compute tau as averaged first zero crossing of autocorrelation
    """
    def get_tau(cell_evo):
        """ Get tau for single run
        """
        acorr = np.correlate(cell_evo, cell_evo, mode='full')[len(cell_evo)-1:]

        sign_changes = (np.diff(np.sign(acorr)) != 0)
        if not any(sign_changes):
            raise RuntimeError('No sign change detected')

        tau = sign_changes.nonzero()[0][0]
        return tau

    width, height, depth = camp.shape
    tau_list = []
    for j in range(width):
        for i in range(height):
            try:
                tau_list.append(get_tau(camp[i, j]))
            except RuntimeError:
                # TODO: what happens in this case?
                pass

    assert len(tau_list) > 0, 'No sign changes found'
    return int(np.mean(tau_list))

def compute_phase_variable(cell_evo, tau):
    """ Generate local phase space
    """
    m = np.mean(cell_evo)
    theta = []
    for t in range(len(cell_evo) - tau):
        cur = np.arctan2(
            cell_evo[t + tau] - m,
            cell_evo[t] - m
        )
        theta.append(cur)

    return theta

@timed_run('Computing gradients')
def compute_discrete_gradient(fields):
    """ Compute discretized gradient on given field
    """
    gradients = []
    for field in fields:
        # fix phase jumps greater than PI
        field = unwrap_phase(field)

        # finite difference operator
        fidi_op_x = np.diff(field, axis=0)
        fidi_op_x = np.vstack((fidi_op_x, fidi_op_x[-1]))
        fidi_op_y = np.diff(field, axis=1)
        fidi_op_y = np.hstack((fidi_op_y, fidi_op_y[:,-1][np.newaxis].T))

        # compute convolution
        nabla_x = np.array([
            [-1/2, 0, 1/2],
            [-1, 0, 1],
            [-1/2, 0, 1/2]
        ])
        nabla_y = np.array([
            [1/2, 1, 1/2],
            [0, 0, 0],
            [-1/2, -1, -1/2]
        ])

        conv_x = ndimage.convolve(
            fidi_op_y, nabla_x,
            mode='constant', cval=0.0
        )
        conv_y = ndimage.convolve(
            fidi_op_x, nabla_y,
            mode='constant', cval=0.0
        )

        # compute discretized gradient
        grad = conv_x + conv_y
        gradients.append(grad)

    return np.array(gradients)

@timed_run('Computing local phase field')
def compute_local_phase_field(camp):
    """ Compute local phase of each cell
    """
    width, height, depth = camp.shape

    # compute thetas
    tau = compute_tau(camp)

    theta = np.empty((width, height)).tolist()
    for j in range(width):
        for i in range(height):
            cur = compute_phase_variable(camp[i, j], tau)
            theta[i][j] = np.array(cur)
    theta = np.rollaxis(np.array(theta), 2, 0)

    return theta

def integrate(cx, cy, radius, grad):
    """ Integrate gradient along circle perimeter of given shape
    """
    rr, cc = circle_perimeter(
        cx, cy, radius,
        method='andres', shape=grad.shape
    )
    res = sum(grad[rr, cc])
    return res

@timed_run('Computing singularity measure')
def compute_singularity_measure(gradients):
    """ Compute singularity measure of data
    """
    singularities = []
    for grad in gradients:
        width, height = grad.shape

        circle_rad = 5
        singularity = np.empty((width, height))
        for j in range(height):
            for i in range(width):
                res = integrate(i, j, circle_rad, grad)
                singularity[i, j] = res
        singularity = np.array(singularity)

        singularities.append(singularity)

    return np.array(singularities)

@timed_run('Preprocessing data')
def preprocess_data(data):
    """ Preprocess data to make singularity detection easier
    """
    # eliminate background
    data = np.diff(data)

    # average elements
    data = np.array([0.5*(data[...,i]+data[...,i+1]) for i in range(0, data.shape[2]-1, 2)])
    data = np.rollaxis(data, 0, 3)

    # apply gaussian filter
    def gf(field):
        sig = 5
        nrows, ncols = field.shape
        return ndimage.gaussian_filter(
            field,
            sigma=(sig * nrows / 100.0, sig * ncols / 100.0),
            order=0
        )
    data = np.array([gf(data[...,i]) for i in range(data.shape[2])])
    data = np.rollaxis(data, 0, 3)

    return data

def singularity_plot(fname):
    """ Plot overview over singularity measure
    """
    cache_dir = 'cache'
    pure_fname = os.path.splitext(os.path.basename(fname))[0]

    if not os.path.isdir(cache_dir):
        # preprocess input
        camp, pacemaker = np.load(sys.argv[1])
        camp = np.rollaxis(camp, 0, 3)
        camp = preprocess_data(camp)

        # compute data
        rolled_camp = np.rollaxis(camp, 2, 0)
        lphase = compute_local_phase_field(camp) # decreases last dim due to tau
        grads = compute_discrete_gradient(lphase)
        singularities = compute_singularity_measure(grads)

        # cache data
        os.path.join(cache_dir, pure_fname, '')
        save_data(os.path.join(cache_dir, pure_fname, 'rolled_camp'), rolled_camp)
        save_data(os.path.join(cache_dir, pure_fname, 'lphase'), lphase)
        save_data(os.path.join(cache_dir, pure_fname, 'grads'), grads)
        save_data(os.path.join(cache_dir, pure_fname, 'singularities'), singularities)
    else:
        print('Using cached data')
        rolled_camp = np.load(os.path.join(cache_dir, pure_fname, 'rolled_camp.npy'))
        lphase = np.load(os.path.join(cache_dir, pure_fname, 'lphase.npy'))
        grads = np.load(os.path.join(cache_dir, pure_fname, 'grads.npy'))
        singularities = np.load(os.path.join(cache_dir, pure_fname, 'singularities.npy'))

    # plot data
    pos_num = 4
    pos_range = range(0, lphase.shape[0], int(lphase.shape[0]/(pos_num)))[1:]

    fig, axarr = plt.subplots(len(pos_range), 4, figsize=(10, 10))
    fig.tight_layout()
    plt.suptitle('pipeline overview')

    def show(data, title, ax):
        ax.set_title(title)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

        im = ax.imshow(
            data, interpolation='nearest', cmap=cm.gray)

        holder = make_axes_locatable(ax)
        cax = holder.append_axes('right', size='20%', pad=0.05)
        plt.colorbar(im, cax=cax, format='%.2f')

    for axrow, pos in enumerate(pos_range):
        show(rolled_camp[pos], 'cell overview', axarr[axrow][0])
        show(lphase[pos], 'local phase', axarr[axrow][1])
        show(grads[pos], 'gradient', axarr[axrow][2])
        show(singularities[pos], 'singularity measure', axarr[axrow][3])

    plt.savefig('images/singularity.png', bbox_inches='tight', dpi=300)
    #plt.show()

    # averaged results
    fig, axarr = plt.subplots(1, 2, figsize=(10, 10))
    avg_singularity = np.mean(singularities, axis=0)

    thres_singularity = avg_singularity.copy()
    thres_singularity[thres_singularity > np.pi] = 2 * np.pi
    thres_singularity[thres_singularity < -np.pi] = -2 * np.pi
    thres_singularity[(thres_singularity > -np.pi) & (thres_singularity < np.pi)] = 0

    show(avg_singularity, 'averaged singularity measure', axarr[0])
    show(thres_singularity, 'thresholded singularity measure', axarr[1])

    plt.savefig('images/averaged_singularity.png', bbox_inches='tight', dpi=300)
    #plt.show()

def main():
    """ Detect phase singularities
    """
    if len(sys.argv) != 2:
        print('Usage: %s <data file>' % sys.argv[0])
        sys.exit(1)

    dname = 'images'
    if not os.path.isdir(dname):
        os.mkdir(dname)

    singularity_plot(sys.argv[1])

if __name__ == '__main__':
    main()
<commit_msg>Fix cache detection<commit_after>"""
Find phase singularities, i.e. spiral tips
"""

import sys, os

import numpy as np

from scipy import ndimage
from skimage.draw import circle_perimeter
from skimage.restoration import unwrap_phase

import matplotlib.pylab as plt
import matplotlib.cm as cm
from mpl_toolkits.axes_grid1 import make_axes_locatable

from utils import timed_run, save_data

def compute_tau(camp):
    """ Compute tau as averaged first zero crossing of autocorrelation
    """
    def get_tau(cell_evo):
        """ Get tau for single run
        """
        acorr = np.correlate(cell_evo, cell_evo, mode='full')[len(cell_evo)-1:]

        sign_changes = (np.diff(np.sign(acorr)) != 0)
        if not any(sign_changes):
            raise RuntimeError('No sign change detected')

        tau = sign_changes.nonzero()[0][0]
        return tau

    width, height, depth = camp.shape
    tau_list = []
    for j in range(width):
        for i in range(height):
            try:
                tau_list.append(get_tau(camp[i, j]))
            except RuntimeError:
                # TODO: what happens in this case?
                pass

    assert len(tau_list) > 0, 'No sign changes found'
    return int(np.mean(tau_list))

def compute_phase_variable(cell_evo, tau):
    """ Generate local phase space
    """
    m = np.mean(cell_evo)
    theta = []
    for t in range(len(cell_evo) - tau):
        cur = np.arctan2(
            cell_evo[t + tau] - m,
            cell_evo[t] - m
        )
        theta.append(cur)

    return theta

@timed_run('Computing gradients')
def compute_discrete_gradient(fields):
    """ Compute discretized gradient on given field
    """
    gradients = []
    for field in fields:
        # fix phase jumps greater than PI
        field = unwrap_phase(field)

        # finite difference operator
        fidi_op_x = np.diff(field, axis=0)
        fidi_op_x = np.vstack((fidi_op_x, fidi_op_x[-1]))
        fidi_op_y = np.diff(field, axis=1)
        fidi_op_y = np.hstack((fidi_op_y, fidi_op_y[:,-1][np.newaxis].T))

        # compute convolution
        nabla_x = np.array([
            [-1/2, 0, 1/2],
            [-1, 0, 1],
            [-1/2, 0, 1/2]
        ])
        nabla_y = np.array([
            [1/2, 1, 1/2],
            [0, 0, 0],
            [-1/2, -1, -1/2]
        ])

        conv_x = ndimage.convolve(
            fidi_op_y, nabla_x,
            mode='constant', cval=0.0
        )
        conv_y = ndimage.convolve(
            fidi_op_x, nabla_y,
            mode='constant', cval=0.0
        )

        # compute discretized gradient
        grad = conv_x + conv_y
        gradients.append(grad)

    return np.array(gradients)

@timed_run('Computing local phase field')
def compute_local_phase_field(camp):
    """ Compute local phase of each cell
    """
    width, height, depth = camp.shape

    # compute thetas
    tau = compute_tau(camp)

    theta = np.empty((width, height)).tolist()
    for j in range(width):
        for i in range(height):
            cur = compute_phase_variable(camp[i, j], tau)
            theta[i][j] = np.array(cur)
    theta = np.rollaxis(np.array(theta), 2, 0)

    return theta

def integrate(cx, cy, radius, grad):
    """ Integrate gradient along circle perimeter of given shape
    """
    rr, cc = circle_perimeter(
        cx, cy, radius,
        method='andres', shape=grad.shape
    )
    res = sum(grad[rr, cc])
    return res

@timed_run('Computing singularity measure')
def compute_singularity_measure(gradients):
    """ Compute singularity measure of data
    """
    singularities = []
    for grad in gradients:
        width, height = grad.shape

        circle_rad = 5
        singularity = np.empty((width, height))
        for j in range(height):
            for i in range(width):
                res = integrate(i, j, circle_rad, grad)
                singularity[i, j] = res
        singularity = np.array(singularity)

        singularities.append(singularity)

    return np.array(singularities)

@timed_run('Preprocessing data')
def preprocess_data(data):
    """ Preprocess data to make singularity detection easier
    """
    # eliminate background
    data = np.diff(data)

    # average elements
    data = np.array([0.5*(data[...,i]+data[...,i+1]) for i in range(0, data.shape[2]-1, 2)])
    data = np.rollaxis(data, 0, 3)

    # apply gaussian filter
    def gf(field):
        sig = 5
        nrows, ncols = field.shape
        return ndimage.gaussian_filter(
            field,
            sigma=(sig * nrows / 100.0, sig * ncols / 100.0),
            order=0
        )
    data = np.array([gf(data[...,i]) for i in range(data.shape[2])])
    data = np.rollaxis(data, 0, 3)

    return data

def singularity_plot(fname):
    """ Plot overview over singularity measure
    """
    cache_dir = 'cache'
    pure_fname = os.path.splitext(os.path.basename(fname))[0]

    if not os.path.isdir(os.path.join(cache_dir, pure_fname)):
        # preprocess input
        camp, pacemaker = np.load(sys.argv[1])
        camp = np.rollaxis(camp, 0, 3)
        camp = preprocess_data(camp)

        # compute data
        rolled_camp = np.rollaxis(camp, 2, 0)
        lphase = compute_local_phase_field(camp) # decreases last dim due to tau
        grads = compute_discrete_gradient(lphase)
        singularities = compute_singularity_measure(grads)

        # cache data
        os.path.join(cache_dir, pure_fname, '')
        save_data(os.path.join(cache_dir, pure_fname, 'rolled_camp'), rolled_camp)
        save_data(os.path.join(cache_dir, pure_fname, 'lphase'), lphase)
        save_data(os.path.join(cache_dir, pure_fname, 'grads'), grads)
        save_data(os.path.join(cache_dir, pure_fname, 'singularities'), singularities)
    else:
        print('Using cached data')
        rolled_camp = np.load(os.path.join(cache_dir, pure_fname, 'rolled_camp.npy'))
        lphase = np.load(os.path.join(cache_dir, pure_fname, 'lphase.npy'))
        grads = np.load(os.path.join(cache_dir, pure_fname, 'grads.npy'))
        singularities = np.load(os.path.join(cache_dir, pure_fname, 'singularities.npy'))

    # plot data
    pos_num = 4
    pos_range = range(0, lphase.shape[0], int(lphase.shape[0]/(pos_num)))[1:]

    fig, axarr = plt.subplots(len(pos_range), 4, figsize=(10, 10))
    fig.tight_layout()
    plt.suptitle('pipeline overview')

    def show(data, title, ax):
        ax.set_title(title)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(False)

        im = ax.imshow(
            data, interpolation='nearest', cmap=cm.gray)

        holder = make_axes_locatable(ax)
        cax = holder.append_axes('right', size='20%', pad=0.05)
        plt.colorbar(im, cax=cax, format='%.2f')

    for axrow, pos in enumerate(pos_range):
        show(rolled_camp[pos], 'cell overview', axarr[axrow][0])
        show(lphase[pos], 'local phase', axarr[axrow][1])
        show(grads[pos], 'gradient', axarr[axrow][2])
        show(singularities[pos], 'singularity measure', axarr[axrow][3])

    plt.savefig('images/singularity.png', bbox_inches='tight', dpi=300)
    #plt.show()

    # averaged results
    fig, axarr = plt.subplots(1, 2, figsize=(10, 10))
    avg_singularity = np.mean(singularities, axis=0)

    thres_singularity = avg_singularity.copy()
    thres_singularity[thres_singularity > np.pi] = 2 * np.pi
    thres_singularity[thres_singularity < -np.pi] = -2 * np.pi
    thres_singularity[(thres_singularity > -np.pi) & (thres_singularity < np.pi)] = 0

    show(avg_singularity, 'averaged singularity measure', axarr[0])
    show(thres_singularity, 'thresholded singularity measure', axarr[1])

    plt.savefig('images/averaged_singularity.png', bbox_inches='tight', dpi=300)
    #plt.show()

def main():
    """ Detect phase singularities
    """
    if len(sys.argv) != 2:
        print('Usage: %s <data file>' % sys.argv[0])
        sys.exit(1)

    dname = 'images'
    if not os.path.isdir(dname):
        os.mkdir(dname)

    singularity_plot(sys.argv[1])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pygame

class Background(pygame.sprite.Sprite):
    def __init__(self, image, location):
        # Call Sprite initializer
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.image.load(image)
        self.rect = self.image.get_rect()
        self.rect.left, self.rect.top = location

class DrawButton:
    def __init__(self, screen, b_color, t_color, text, b_width, b_height, position_x, position_y):
        self.screen = screen
        self.b_color = b_color
        self.t_color = t_color
        self.text = text
        self.b_width = b_width
        self.b_height = b_height
        self.position_x = position_x
        self.position_y = position_y

        self.font = pygame.font.SysFont("Times", 40)

        self.draw()

    def draw(self, image=""):
        pygame.draw.rect(self.screen, self.b_color or image,
                         [self.position_x - self.b_width*0.5, self.position_y - self.b_height*0.5, self.b_width, self.b_height], 0)
        text = self.font.render(str(self.text), 1, self.t_color)
        self.screen.blit(text, (self.position_x - self.b_width*0.5 + self.b_width*0.5 - text.get_width()*0.5,
                                self.position_y - self.b_height * 0.5 + self.b_height*0.5 - text.get_height()*0.5))

    def collision(self, new_color=(0, 0, 0)):
        # Check for collision with mouse and change background color
        mouse = pygame.mouse.get_pos()
        if (mouse[0] in range(int(self.position_x - self.b_width*0.5), int(self.position_x - self.b_width*0.5 + self.b_width))) \
                and (mouse[1] in range(int(self.position_y - self.b_height * 0.5), int( self.position_y - self.b_height * 0.5 + self.b_height))):
            self.b_color = new_color
            self.draw()

            # If pressed on a button change state
            if pygame.mouse.get_pressed()[0]:
                return True

class DrawText:
    def __init__(self, screen, text, color, position_x, position_y, transparent=1):
        self.screen = screen
        self.text = text
        self.color = color
        self.position_x = position_x
        self.position_y = position_y
        self.transparent = transparent

        self.font = pygame.font.SysFont("Times", 40)

        self.draw()

    def draw(self):
        self.text = self.font.render(self.text, self.transparent, self.color)
        self.screen.blit(self.text,
                         (self.position_x - self.text.get_width()*0.5, self.position_y - self.text.get_height()*0.5))

class DrawImage:
    def __init__(self, screen, image, position_x, position_y):
        self.screen = screen
        self.image = pygame.image.load(image)
        self.position_x = position_x
        self.position_y = position_y

        self.draw()

    def draw(self):
        self.screen.blit(self.image, (self.position_x - self.image.get_rect().size[0]*0.5,
                                      self.position_y - self.image.get_rect().size[1]*0.5))

class Player:
    def __init__(self, id, name, score, position):
        self.id = id
        self.name = name
        self.score = score
        self.position = position
<commit_msg>Players are now an object<commit_after>import pygame

class Background(pygame.sprite.Sprite):
    def __init__(self, image, location):
        # Call Sprite initializer
        pygame.sprite.Sprite.__init__(self)
        self.image = pygame.image.load(image)
        self.rect = self.image.get_rect()
        self.rect.left, self.rect.top = location

class DrawButton:
    def __init__(self, screen, b_color, t_color, text, b_width, b_height, position_x, position_y):
        self.screen = screen
        self.b_color = b_color
        self.t_color = t_color
        self.text = text
        self.b_width = b_width
        self.b_height = b_height
        self.position_x = position_x
        self.position_y = position_y

        self.font = pygame.font.SysFont("Times", 40)

        self.draw()

    def draw(self, image=""):
        pygame.draw.rect(self.screen, self.b_color or image,
                         [self.position_x - self.b_width*0.5, self.position_y - self.b_height*0.5, self.b_width, self.b_height], 0)
        text = self.font.render(str(self.text), 1, self.t_color)
        self.screen.blit(text, (self.position_x - self.b_width*0.5 + self.b_width*0.5 - text.get_width()*0.5,
                                self.position_y - self.b_height * 0.5 + self.b_height*0.5 - text.get_height()*0.5))

    def collision(self, new_color=(0, 0, 0)):
        # Check for collision with mouse and change background color
        mouse = pygame.mouse.get_pos()
        if (mouse[0] in range(int(self.position_x - self.b_width*0.5), int(self.position_x - self.b_width*0.5 + self.b_width))) \
                and (mouse[1] in range(int(self.position_y - self.b_height * 0.5), int( self.position_y - self.b_height * 0.5 + self.b_height))):
            self.b_color = new_color
            self.draw()

            # If pressed on a button change state
            if pygame.mouse.get_pressed()[0]:
                return True

class DrawText:
    def __init__(self, screen, text, color, position_x, position_y, transparent=1):
        self.screen = screen
        self.text = text
        self.color = color
        self.position_x = position_x
        self.position_y = position_y
        self.transparent = transparent

        self.font = pygame.font.SysFont("Times", 40)

        self.draw()

    def draw(self):
        self.text = self.font.render(self.text, self.transparent, self.color)
        self.screen.blit(self.text,
                         (self.position_x - self.text.get_width()*0.5, self.position_y - self.text.get_height()*0.5))

class DrawImage:
    def __init__(self, screen, image, position_x, position_y):
        self.screen = screen
        self.image = pygame.image.load(image)
        self.position_x = position_x
        self.position_y = position_y

        self.draw()

    def draw(self):
        self.screen.blit(self.image, (self.position_x - self.image.get_rect().size[0]*0.5,
                                      self.position_y - self.image.get_rect().size[1]*0.5))

class Player:
    def __init__(self, player_id, name, score, position):
        self.id = player_id
        self.name = name
        self.score = score
        self.position = position
<|endoftext|>


<commit_before>"""The base Controller API

Provides the BaseController class for subclassing.
"""
import logging
from time import time

from pylons import config
from pylons.controllers import WSGIController
from pylons import request, response, session, tmpl_context as c, g
from pylons.controllers.util import abort, redirect_to
from pylons.decorators import validate
from pylons.i18n import _, add_fallback, get_lang, set_lang, gettext

import routes

import formencode
import formencode.validators as validators
from formencode import htmlfill

from authorization import has_permission
import authorization as auth
from repoze.what.plugins.pylonshq import ActionProtector

from cache import memoize
from instance import RequireInstance
from xsrf import RequireInternalRequest
from templating import render, NamedPager, StaticPage
from util import get_entity_or_abort
import adhocracy.model as model
import search as libsearch
import helpers as h
import event
import democracy
import tiles
import sorting
import watchlist
#import microblog
import text.i18n as i18n

log = logging.getLogger(__name__)

class BaseController(WSGIController):

    def __call__(self, environ, start_response):
        """Invoke the Controller"""
        # WSGIController.__call__ dispatches to the Controller method
        # the request is routed to. This routing information is
        # available in environ['pylons.routes_dict']
        import adhocracy.lib
        c.lib = adhocracy.lib
        c.model = model
        c.instance = model.filter.get_instance()

        environ['HTTP_HOST'] = environ['HTTP_HOST_ORIGINAL']

        if environ.get('repoze.who.identity'):
            c.user = environ.get('repoze.who.identity').get('user')
            #model.meta.Session.add(c.user)
        else:
            c.user = None

        # have to do this with the user in place
        i18n.handle_request()

        if c.user:
            h.add_rss(_("My Adhocracies"), h.instance_url(None, '/feed.rss'))
        if c.instance:
            h.add_rss("%s News" % c.instance.label,
                      h.instance_url(c.instance, '/instance/%s.rss' % c.instance.key))

        h.add_meta("description", _("A liquid democracy platform for making decisions in "
                   + "distributed, open groups by cooperatively creating proposals and voting "
                   + "on them to establish their support."))
        h.add_meta("keywords", _("adhocracy, direct democracy, liquid democracy, liqd, democracy, wiki, voting,"
                   + "participation, group decisions, decisions, decision-making"))

        try:
            begin_time = time()
            return WSGIController.__call__(self, environ, start_response)
        finally:
            model.meta.Session.remove()
            log.debug("Rendering page %s took %sms" % (environ.get('PATH_INFO'), ((time()-begin_time)*1000)))

def ExpectFormat(formats=['html', 'rss', 'xml', 'json']):
    def _parse(f, *a, **kw):
        #TODO
        return f(*a, **kw)
    return decorator(_parse)
<commit_msg>[svn r583] SHE'LL BE WEARING RED PYJAMAS<commit_after>"""The base Controller API

Provides the BaseController class for subclassing.
"""
import logging
from time import time

from pylons import config
from pylons.controllers import WSGIController
from pylons import request, response, session, tmpl_context as c, g
from pylons.controllers.util import abort, redirect_to
from pylons.decorators import validate
from pylons.i18n import _, add_fallback, get_lang, set_lang, gettext

import routes

import formencode
import formencode.validators as validators
from formencode import htmlfill

from authorization import has_permission
import authorization as auth
from repoze.what.plugins.pylonshq import ActionProtector

from cache import memoize
from instance import RequireInstance
from xsrf import RequireInternalRequest
from templating import render, NamedPager, StaticPage
from util import get_entity_or_abort
import adhocracy.model as model
import search as libsearch
import helpers as h
import event
import democracy
import tiles
import sorting
import watchlist
#import microblog
import text.i18n as i18n

log = logging.getLogger(__name__)

class BaseController(WSGIController):

    def __call__(self, environ, start_response):
        """Invoke the Controller"""
        # WSGIController.__call__ dispatches to the Controller method
        # the request is routed to. This routing information is
        # available in environ['pylons.routes_dict']
        import adhocracy.lib
        c.lib = adhocracy.lib
        c.model = model
        c.instance = model.filter.get_instance()

        environ['HTTP_HOST'] = environ['HTTP_HOST_ORIGINAL']

        if environ.get('repoze.who.identity'):
            c.user = environ.get('repoze.who.identity').get('user')
            model.meta.Session.merge(c.user)
        else:
            c.user = None

        if c.instance is not None:
            model.meta.Session.merge(c.instance)

        # have to do this with the user in place
        i18n.handle_request()

        if c.user:
            h.add_rss(_("My Adhocracies"), h.instance_url(None, '/feed.rss'))
        if c.instance:
            h.add_rss("%s News" % c.instance.label,
                      h.instance_url(c.instance, '/instance/%s.rss' % c.instance.key))

        h.add_meta("description", _("A liquid democracy platform for making decisions in "
                   + "distributed, open groups by cooperatively creating proposals and voting "
                   + "on them to establish their support."))
        h.add_meta("keywords", _("adhocracy, direct democracy, liquid democracy, liqd, democracy, wiki, voting,"
                   + "participation, group decisions, decisions, decision-making"))

        try:
            begin_time = time()
            return WSGIController.__call__(self, environ, start_response)
        finally:
            model.meta.Session.remove()
            log.debug("Rendering page %s took %sms" % (environ.get('PATH_INFO'), ((time()-begin_time)*1000)))

def ExpectFormat(formats=['html', 'rss', 'xml', 'json']):
    def _parse(f, *a, **kw):
        #TODO
        return f(*a, **kw)
    return decorator(_parse)
<|endoftext|>


<commit_before># Copyright (c) 2012. Los Alamos National Security, LLC.

# This material was produced under U.S. Government contract DE-AC52-06NA25396
# for Los Alamos National Laboratory (LANL), which is operated by Los Alamos
# National Security, LLC for the U.S. Department of Energy. The U.S. Government
# has rights to use, reproduce, and distribute this software.

# NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY,
# EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.
# If software is modified to produce derivative works, such modified software should
# be clearly marked, so as not to confuse it with the version available from LANL.

# Additionally, this library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License v 2.1 as published by the
# Free Software Foundation. Accordingly, this library is distributed in the hope that
# it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See LICENSE.txt for more details.

import sys
from greenlet import greenlet

import simx.core as core
#from simx.util import *

class EventScheduler:
    """
    Class for staggered creation of initial events.
    External events will be scheduled upto a time in the future,
    and this process will be repeated till there are no more
    external events

    """

    def __init__(self, event_creator):
        global _evt_scheduler
        _evt_scheduler = self
        core.set_event_scheduler(self)
        self.time_chunk = 1000 * core.get_local_min_delay()
        self.upto_time = core.get_now() + self.time_chunk
        self.event_creator = greenlet(event_creator)
        self.event_creator.switch()

    def schedule_next(self, time):
        """
        Schedules a timer for creating next chunk of events
        """
        if time <= self.upto_time:
            return
        #else
        #schedule a timer just before this event and switch back to main "thread"
        #print "SHOULD NOT"
        core.set_event_scheduler_timer(time)
        greenlet.getcurrent().parent.switch()
    # when switching back to this "thread", execution resumes from here

    def process_scheduler_event(self):
        #print "NEW BATCH OF EVENTS BEING SCHEDULED"
        self.upto_time = core.get_now() + self.time_chunk
        #switch to event_creator thread
        self.event_creator.switch()

def schedule_event(time,entity,address,event):
    """

    Schedules an event to be received by 'entity'

    Keyword Arguments:

    time    --  (When)  time at which event will be processed
    entity  --  (Who )  id of entity that will process the event
    address --  (Where) address of service on entity that will receive
                        event
    event   --  (What)  event payload

    """

    #if is_set_evt_scheduler():
    if not _evt_scheduler is None:
       _evt_scheduler.schedule_next(time)

    info_data = core.InfoData( time, entity,
                                  address, 1000, 1, {}, event)
    core.create_input_info( info_data )

def is_set_evt_scheduler():
    """
    Returns true if the staggered event scheduler
    has been created
    """
    return not _evt_scheduler is None
<commit_msg>Initialised _evt_sceduler to None object; somehow this got dropped during the merge.<commit_after># Copyright (c) 2012. Los Alamos National Security, LLC.

# This material was produced under U.S. Government contract DE-AC52-06NA25396
# for Los Alamos National Laboratory (LANL), which is operated by Los Alamos
# National Security, LLC for the U.S. Department of Energy. The U.S. Government
# has rights to use, reproduce, and distribute this software.

# NEITHER THE GOVERNMENT NOR LOS ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY,
# EXPRESS OR IMPLIED, OR ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE.
# If software is modified to produce derivative works, such modified software should
# be clearly marked, so as not to confuse it with the version available from LANL.

# Additionally, this library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License v 2.1 as published by the
# Free Software Foundation. Accordingly, this library is distributed in the hope that
# it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See LICENSE.txt for more details.

import sys
from greenlet import greenlet

import simx.core as core
#from simx.util import *

_evt_scheduler = None

class EventScheduler:
    """
    Class for staggered creation of initial events.
    External events will be scheduled upto a time in the future,
    and this process will be repeated till there are no more
    external events

    """

    def __init__(self, event_creator):
        global _evt_scheduler
        _evt_scheduler = self
        core.set_event_scheduler(self)
        self.time_chunk = 1000 * core.get_local_min_delay()
        self.upto_time = core.get_now() + self.time_chunk
        self.event_creator = greenlet(event_creator)
        self.event_creator.switch()

    def schedule_next(self, time):
        """
        Schedules a timer for creating next chunk of events
        """
        if time <= self.upto_time:
            return
        #else
        #schedule a timer just before this event and switch back to main "thread"
        #print "SHOULD NOT"
        core.set_event_scheduler_timer(time)
        greenlet.getcurrent().parent.switch()
    # when switching back to this "thread", execution resumes from here

    def process_scheduler_event(self):
        #print "NEW BATCH OF EVENTS BEING SCHEDULED"
        self.upto_time = core.get_now() + self.time_chunk
        #switch to event_creator thread
        self.event_creator.switch()

def schedule_event(time,entity,address,event):
    """

    Schedules an event to be received by 'entity'

    Keyword Arguments:

    time    --  (When)  time at which event will be processed
    entity  --  (Who )  id of entity that will process the event
    address --  (Where) address of service on entity that will receive
                        event
    event   --  (What)  event payload

    """

    #if is_set_evt_scheduler():
    if not _evt_scheduler is None:
       _evt_scheduler.schedule_next(time)

    info_data = core.InfoData( time, entity,
                                  address, 1000, 1, {}, event)
    core.create_input_info( info_data )

def is_set_evt_scheduler():
    """
    Returns true if the staggered event scheduler
    has been created
    """
    return not _evt_scheduler is None
<|endoftext|>