<commit_before># pythonpath modification to make hytra and empryonic available
# for import without requiring it to be installed
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
# standard imports
import h5py
import configargparse as argparse
import logging
import hytra.util.axesconversion

def segmentation_to_hdf5(options):
    """
    The generated segmentation is one HDF5 dataset per timestep,
    and each of these datasets has shape 1(t),x,y,z,1(c).
    """
    in_h5 = h5py.File(options.segmentation_file, 'w')
    out_h5 = h5py.File(options.hdf5Path, 'w')

    data = in_h5[options.segmentation_path].value
    logging.getLogger('segmentation_to_labelimage.py').info("Found image of shape {}".format(data.shape))
    data = hytra.util.axesconversion.adjustOrder(data, options.segmentation_axes, 'txyzc')
    logging.getLogger('segmentation_to_labelimage.py').info("Changed into shape {}".format(data.shape))

    for timeframe in range(data.shape[0]):
        internalPath = options.hdf5ImagePath % (timeframe, timeframe + 1, data.shape[1], data.shape[2], data.shape[3])
        out_h5.create_dataset(internalPath, data=data, dtype='u2', compression='gzip')

    logging.getLogger('segmentation_to_labelimage.py').info("Saved {} timeframes".format(data.shape[0]))

if __name__ == '__main__':
    """
    Convert an h5 segmentation volume to the HDF5 volume format as used in (old) ilastik versions
    """

    parser = argparse.ArgumentParser(
        description='Convert segmentation tif files to hdf5',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-c', '--config', is_config_file=True, help='config file path')

    parser.add_argument('--in-segmentation-file', required=True, type=str, dest='segmentation_file',
                        help='Filename pattern of the all tif files containing the segmentation data')
    parser.add_argument('--in-segmentation-axes', type=str, dest='segmentation_axes', default='txyzc',
                        help='Axes string defining the input volume shape')
    parser.add_argument('--in-segmentation-path', type=str, dest='segmentation_path', default='exported_data',
                        help='Path inside the HDF5 file to the volume')
    parser.add_argument('--label-image-file', required=True, type=str, dest='hdf5Path',
                        help='filename of where the segmentation HDF5 file will be created')
    parser.add_argument('--label-image-path', type=str, dest='hdf5ImagePath',
                        help='Path inside ilastik project file to the label image',
                        default='/TrackingFeatureExtraction/LabelImage/0000/[[%d, 0, 0, 0, 0], [%d, %d, %d, %d, 1]]')
    parser.add_argument("--verbose", dest='verbose', action='store_true', default=False)

    # parse command line
    options, unknown = parser.parse_known_args()

    if options.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    logging.getLogger('segmentation_to_labelimage.py').debug("Ignoring unknown parameters: {}".format(unknown))

    segmentation_to_hdf5(options)
<commit_msg>segmentation-to-labelimage: Open input file in read mode, not write.<commit_after># pythonpath modification to make hytra and empryonic available
# for import without requiring it to be installed
import os
import sys
sys.path.insert(0, os.path.abspath('..'))
# standard imports
import h5py
import configargparse as argparse
import logging
import hytra.util.axesconversion

def segmentation_to_hdf5(options):
    """
    The generated segmentation is one HDF5 dataset per timestep,
    and each of these datasets has shape 1(t),x,y,z,1(c).
    """
    in_h5 = h5py.File(options.segmentation_file, 'r')
    out_h5 = h5py.File(options.hdf5Path, 'w')

    data = in_h5[options.segmentation_path].value
    logging.getLogger('segmentation_to_labelimage.py').info("Found image of shape {}".format(data.shape))
    data = hytra.util.axesconversion.adjustOrder(data, options.segmentation_axes, 'txyzc')
    logging.getLogger('segmentation_to_labelimage.py').info("Changed into shape {}".format(data.shape))

    for timeframe in range(data.shape[0]):
        internalPath = options.hdf5ImagePath % (timeframe, timeframe + 1, data.shape[1], data.shape[2], data.shape[3])
        out_h5.create_dataset(internalPath, data=data, dtype='u2', compression='gzip')

    logging.getLogger('segmentation_to_labelimage.py').info("Saved {} timeframes".format(data.shape[0]))

if __name__ == '__main__':
    """
    Convert an h5 segmentation volume to the HDF5 volume format as used in (old) ilastik versions
    """

    parser = argparse.ArgumentParser(
        description='Convert segmentation tif files to hdf5',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-c', '--config', is_config_file=True, help='config file path')

    parser.add_argument('--in-segmentation-file', required=True, type=str, dest='segmentation_file',
                        help='Filename pattern of the all tif files containing the segmentation data')
    parser.add_argument('--in-segmentation-axes', type=str, dest='segmentation_axes', default='txyzc',
                        help='Axes string defining the input volume shape')
    parser.add_argument('--in-segmentation-path', type=str, dest='segmentation_path', default='exported_data',
                        help='Path inside the HDF5 file to the volume')
    parser.add_argument('--label-image-file', required=True, type=str, dest='hdf5Path',
                        help='filename of where the segmentation HDF5 file will be created')
    parser.add_argument('--label-image-path', type=str, dest='hdf5ImagePath',
                        help='Path inside ilastik project file to the label image',
                        default='/TrackingFeatureExtraction/LabelImage/0000/[[%d, 0, 0, 0, 0], [%d, %d, %d, %d, 1]]')
    parser.add_argument("--verbose", dest='verbose', action='store_true', default=False)

    # parse command line
    options, unknown = parser.parse_known_args()

    if options.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    logging.getLogger('segmentation_to_labelimage.py').debug("Ignoring unknown parameters: {}".format(unknown))

    segmentation_to_hdf5(options)
<|endoftext|>


<commit_before>### Note this is Python 2.7, because Theano ###
from __future__ import division

import pdb
import os
import glob

import numpy as np
from scipy.io import wavfile
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint

import hvc.utils.utils
import hvc.neuralnet.models
from hvc.utils import sequences
from hvc.audio.ev_funcs import load_cbin,load_notmat

#constants for spectrogram
SAMP_FREQ = 32000 # Hz
WINDOW_SIZE= 512
WINDOW_STEP= 32
FREQ_CUTOFFS=[1000,8000]
MAX_SILENT_GAP = 0.08 # s to keep before or after a syllable

# constants used by script
DATA_DIR = os.path.normpath('C:/DATA/gy6or6/032212')
NUM_SONGS_TO_USE = 20
LABELS_TO_USE = list('abcdefghijk')

os.chdir(DATA_DIR)
cbins = glob.glob('*.cbin')

#given that there's only one sampling frequency, use it to figure out the number of time bins in the
#fixed length spectrogram into which the sequences will be padded
timebin_size_in_s = WINDOW_STEP / SAMP_FREQ # for default, 32 / 32000 = 0.001 s, i.e. 1 ms

MAX_SPECT_WIDTH = 300; # time bins of spect, currently ~ 1ms

all_syl_labels = []
all_syl_spects = []
background_noise = []
for cbin_ind,cbin in enumerate(cbins[:NUM_SONGS_TO_USE]):
    print('extracting syllables from song {} of {}\r'.format(cbin_ind,
                                                             NUM_SONGS_TO_USE))
    dat, fs = load_cbin(cbin)
    if fs != SAMP_FREQ:
        raise ValueError(
            'Sampling frequency for {}, {}, does not match expected sampling '
            'frequency of {}'.format(cbin,
                                     fs,
                                     SAMP_FREQ))
    dat,fs = load_cbin(cbin)
    spect_obj = hvc.utils.utils.make_spect(dat,fs,
                                           size=WINDOW_SIZE,
                                           step=WINDOW_STEP,
                                           freq_cutoffs=FREQ_CUTOFFS)
    spect = spect_obj.spect
    time_bins = spect_obj.timeBins

    notmat = load_notmat(cbin)
    labels = notmat['labels']
    onsets = notmat['onsets'] / 1000.0
    offsets = notmat['offsets'] / 1000.0
    onsets_time_bins = [np.argmin(np.abs(time_bins - onset))
                                for onset in onsets]
    offsets_time_bins = [np.argmin(np.abs(time_bins - offset))
                                for offset in offsets]
    #extract each syllable, but include the "silence" around it
    for ind,label in enumerate(labels):
        if label not in LABELS_TO_USE:
            continue
        temp_syl_spect = spect[:,onsets_time_bins[ind]:offsets_time_bins[ind]]
        width_diff = MAX_SPECT_WIDTH - temp_syl_spect.shape[1]
        # take half of difference between spects and make that the start index
        # so one half of 'empty' area will be on one side of spect
        # and the other half will be on other side
        # i.e., center the spectrogram
        left_width = int(round(width_diff / 2))
        right_width = width_diff - left_width
        if left_width > onsets_time_bins[ind]:
            left_width = onsets_time_bins[ind]
            right_width = width_diff - left_width
        elif offsets_time_bins[ind] + right_width > spect.shape[-1]:
            right_width = spect.shape[-1] - offsets_time_bins[ind]
            left_width = width_diff - right_width
        temp_syl_spect = spect[:,onsets_time_bins[ind]-left_width:
                                 offsets_time_bins[ind]+right_width]
        all_syl_labels.append(label)
        all_syl_spects.append(temp_syl_spect)

#scale all spects by mean and std of training set
spect_scaler = StandardScaler()
# concatenate all spects then transpose so Hz bins are 'features'
spect_scaler.fit(np.hstack(all_syl_spects[:]).T)
# now scale each individual training spect
for ind, spect in enumerate(all_syl_spects):
    all_syl_spects[ind] = np.transpose(spect_scaler.transform(spect.T))

#reshape training data for model
all_syl_spects = np.dstack(all_syl_spects[:])
x,y,n = all_syl_spects.shape
all_syl_spects = all_syl_spects.reshape(n,1,x,y)

uniq_syls, syl_counts = np.unique(all_syl_labels,return_counts=True)
print('Training set:')
for syl,count in zip(uniq_syls,syl_counts):
    print('\tSyllable {} -- {} samples.'.format(syl,count))
num_syl_classes = np.size(uniq_syls)
# make a dictionary that maps labels to classes 0 to n-1 where n is number of
# classes of syllables.
# Need this map instead of e.g. converting from char to int because
# keras to_categorical function requires
# input where classes are labeled from 0 to n-1
classes_zero_to_n = range(num_syl_classes)
label_map = dict(zip(uniq_syls,classes_zero_to_n))
all_syl_labels_zero_to_n = np.asarray([label_map[syl]
                                        for syl in all_syl_labels])
#so we can then convert to array of binary / one-hot vectors for training
all_syl_labels_binary = to_categorical(all_syl_labels_zero_to_n,num_syl_classes)

print('Shuffling syllables.')
# shuffle and split into training and test sets
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)
shuffle_ids = np.random.permutation(n)  # n is from bigram_spects_padded.shape
all_syl_spects_shuffled = all_syl_spects[shuffle_ids,:,:,:]
all_syl_labels_shuffled = all_syl_labels_binary[shuffle_ids,:]

#constants for training
NUM_TRAIN_SAMPLES = 512
train_spects = all_syl_spects_shuffled[:NUM_TRAIN_SAMPLES,:,:,:]
train_labels = all_syl_labels_shuffled[:NUM_TRAIN_SAMPLES,:]

test_spects = all_syl_spects_shuffled[-1:-1:-NUM_TRAIN_SAMPLES,:,:,:]
test_labels = all_syl_labels_shuffled[-1:-1:-NUM_TRAIN_SAMPLES,:]

# Also need to know number of rows, i.e. freqbins.
# Will be the same for all spects since we used the same FFT params for all.
# freqBins size is also input shape to LSTM net
# (since at each time point the input is one column of spectrogram)
num_channels, num_freq_bins, num_time_bins = all_syl_spects[0].shape
input_shape = (num_channels,num_freq_bins,num_time_bins)
vgg16 = hvc.neuralnet.models.VGG_16(input_shape=input_shape,
                                   num_syllable_classes=num_syl_classes)

print('Training model.')
vgg16.fit(train_spects,
          train_labels,
          validation_split=0.33,
          batch_size=32,
          nb_epoch=100,
          verbose=1
               )

import pdb;pdb.set_trace()
# save: all_syl_labels, all_syl_labels_shuffled, label_map, etc....
# train lables and train spects of course
<commit_msg>fix matrix reshaping errors<commit_after>### Note this is Python 2.7, because Theano ###
from __future__ import division

import pdb
import os
import glob

import numpy as np
from scipy.io import wavfile
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint

import hvc.utils.utils
import hvc.neuralnet.models
from hvc.utils import sequences
from hvc.audio.ev_funcs import load_cbin,load_notmat

#constants for spectrogram
SAMP_FREQ = 32000 # Hz
WINDOW_SIZE= 512
WINDOW_STEP= 32
FREQ_CUTOFFS=[1000,8000]
MAX_SILENT_GAP = 0.08 # s to keep before or after a syllable

# constants used by script
DATA_DIR = os.path.normpath('C:/DATA/gy6or6/032212')
NUM_SONGS_TO_USE = 20
LABELS_TO_USE = list('abcdefghijk')

os.chdir(DATA_DIR)
cbins = glob.glob('*.cbin')

#given that there's only one sampling frequency, use it to figure out the number of time bins in the
#fixed length spectrogram into which the sequences will be padded
timebin_size_in_s = WINDOW_STEP / SAMP_FREQ # for default, 32 / 32000 = 0.001 s, i.e. 1 ms

MAX_SPECT_WIDTH = 300; # time bins of spect, currently ~ 1ms

all_syl_labels = []
all_syl_spects = []
background_noise = []
for cbin_ind,cbin in enumerate(cbins[:NUM_SONGS_TO_USE]):
    print('extracting syllables from song {} of {}\r'.format(cbin_ind,
                                                             NUM_SONGS_TO_USE))
    dat, fs = load_cbin(cbin)
    if fs != SAMP_FREQ:
        raise ValueError(
            'Sampling frequency for {}, {}, does not match expected sampling '
            'frequency of {}'.format(cbin,
                                     fs,
                                     SAMP_FREQ))
    dat,fs = load_cbin(cbin)
    spect_obj = hvc.utils.utils.make_spect(dat,fs,
                                           size=WINDOW_SIZE,
                                           step=WINDOW_STEP,
                                           freq_cutoffs=FREQ_CUTOFFS)
    spect = spect_obj.spect
    time_bins = spect_obj.timeBins

    notmat = load_notmat(cbin)
    labels = notmat['labels']
    onsets = notmat['onsets'] / 1000.0
    offsets = notmat['offsets'] / 1000.0
    onsets_time_bins = [np.argmin(np.abs(time_bins - onset))
                                for onset in onsets]
    offsets_time_bins = [np.argmin(np.abs(time_bins - offset))
                                for offset in offsets]
    #extract each syllable, but include the "silence" around it
    for ind,label in enumerate(labels):
        if label not in LABELS_TO_USE:
            continue
        temp_syl_spect = spect[:,onsets_time_bins[ind]:offsets_time_bins[ind]]
        width_diff = MAX_SPECT_WIDTH - temp_syl_spect.shape[1]
        # take half of difference between spects and make that the start index
        # so one half of 'empty' area will be on one side of spect
        # and the other half will be on other side
        # i.e., center the spectrogram
        left_width = int(round(width_diff / 2))
        right_width = width_diff - left_width
        if left_width > onsets_time_bins[ind]:
            left_width = onsets_time_bins[ind]
            right_width = width_diff - left_width
        elif offsets_time_bins[ind] + right_width > spect.shape[-1]:
            right_width = spect.shape[-1] - offsets_time_bins[ind]
            left_width = width_diff - right_width
        temp_syl_spect = spect[:,onsets_time_bins[ind]-left_width:
                                 offsets_time_bins[ind]+right_width]
        all_syl_labels.append(label)
        all_syl_spects.append(temp_syl_spect)

#scale all spects by mean and std of training set
spect_scaler = StandardScaler()
# concatenate all spects then rotate  so Hz bins are 'features'
spect_scaler.fit(np.rot90(np.hstack(all_syl_spects[:])))
# now scale each individual training spect
all_syl_spects_scaled = []
for spect in all_syl_spects:
    all_syl_spects_scaled.append(
        np.rot90(spect_scaler.transform(np.rot90(spect),3)))

#reshape training data for model
all_syl_spects = np.stack(all_syl_spects[:],axis=0)
all_syl_spects = np.expand_dims(all_syl_spects,axis=1)

uniq_syls, syl_counts = np.unique(all_syl_labels,return_counts=True)
print('Training set:')
for syl,count in zip(uniq_syls,syl_counts):
    print('\tSyllable {} -- {} samples.'.format(syl,count))
num_syl_classes = np.size(uniq_syls)
# make a dictionary that maps labels to classes 0 to n-1 where n is number of
# classes of syllables.
# Need this map instead of e.g. converting from char to int because
# keras to_categorical function requires
# input where classes are labeled from 0 to n-1
classes_zero_to_n = range(num_syl_classes)
label_map = dict(zip(uniq_syls,classes_zero_to_n))
all_syl_labels_zero_to_n = np.asarray([label_map[syl]
                                        for syl in all_syl_labels])
#so we can then convert to array of binary / one-hot vectors for training
all_syl_labels_binary = to_categorical(all_syl_labels_zero_to_n,num_syl_classes)

print('Shuffling syllables.')
# shuffle and split into training and test sets
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)
shuffle_ids = np.random.permutation(all_syl_spects.shape[0])
all_syl_spects_shuffled = all_syl_spects[shuffle_ids,:,:,:]
all_syl_labels_shuffled = all_syl_labels_binary[shuffle_ids,:]

#constants for training
NUM_TRAIN_SAMPLES = 512
train_spects = all_syl_spects_shuffled[:NUM_TRAIN_SAMPLES,:,:,:]
train_labels = all_syl_labels_shuffled[:NUM_TRAIN_SAMPLES,:]

test_spects = all_syl_spects_shuffled[-1:-1:-NUM_TRAIN_SAMPLES,:,:,:]
test_labels = all_syl_labels_shuffled[-1:-1:-NUM_TRAIN_SAMPLES,:]

# Also need to know number of rows, i.e. freqbins.
# Will be the same for all spects since we used the same FFT params for all.
# freqBins size is also input shape to LSTM net
# (since at each time point the input is one column of spectrogram)
num_channels,num_freqbins, num_timebins = all_syl_spects[0].shape
input_shape = (num_channels,num_freqbins,num_timebins)
vgg16 = hvc.neuralnet.models.VGG_16(input_shape=input_shape,
                                   num_syllable_classes=num_syl_classes)

import pdb;pdb.set_trace()

print('Training model.')
vgg16.fit(train_spects,
          train_labels,
          validation_split=0.33,
          batch_size=32,
          nb_epoch=200,
          verbose=1
               )

import pdb;pdb.set_trace()
# save: all_syl_labels, all_syl_labels_shuffled, label_map, etc....
# train lables and train spects of course
<|endoftext|>


<commit_before><commit_msg>add csv_to_annot_list function to hvc.utils.annotation<commit_after><|endoftext|>


<commit_before>'''
LICENSING
-------------------------------------------------

hypergolix: A python Golix client.
    Copyright (C) 2016 Muterra, Inc.

    Contributors
    ------------
    Nick Badger
        badg@muterra.io | badg@nickbadger.com | nickbadger.com

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the
    Free Software Foundation, Inc.,
    51 Franklin Street,
    Fifth Floor,
    Boston, MA  02110-1301 USA

------------------------------------------------------

Some notes:

'''

# Control * imports. Therefore controls what is available to toplevel
# package through __init__.py
__all__ = [
    'Privateer',
]

# External dependencies
import threading
import collections
import weakref
import traceback

# These are used for secret ratcheting only.
from Crypto.Hash import SHA512
from Crypto.Protocol.KDF import HKDF

# Intra-package dependencies
from .core import _GAO
from .core import _GAODict

from .utils import TraceLogger

from .exceptions import PrivateerError
from .exceptions import RatchetError
from .exceptions import ConflictingSecrets
from .exceptions import SecretUnknown

# ###############################################
# Logging boilerplate
# ###############################################

import logging
logger = logging.getLogger(__name__)

# ###############################################
# Lib
# ###############################################

class Privateer:
    ''' Lookup system to get secret from ghid. Threadsafe?

    Note: a lot of the ratcheting state preservation could POTENTIALLY
    be eliminated (and restored via oracle). But, that should probably
    wait for Golix v2.0, which will change secret ratcheting anyways.
    This will also help us track which secrets use the old ratcheting
    mechanism and which ones use the new one, once that change is
    started.
    '''
    def __init__(self):
        self._modlock = threading.Lock()

        # These must be linked during assemble.
        self._golcore = None
        self._oracle = None

        # These must be bootstrapped.
        self._secrets_persistent = None
        self._secrets_staging = None
        self._secrets = None
        # Keep track of chains (this will need to be distributed)
        # Lookup <proxy ghid>: <container ghid>
        self._chains = None

        # Keep track of chain progress. This does not need to be distributed.
        # Lookup <proxy ghid> -> proxy secret
        self._ratchet_in_progress = {}

        # Just here for diagnosing a testing problem
        self._committment_problems = {}

    def __contains__(self, ghid):
        ''' Check if we think we know a secret for the ghid.
        '''
        return ghid in self._secrets

    def assemble(self, golix_core, oracle):
        # Chicken, meet egg.
        self._golcore = weakref.proxy(golix_core)
        # We need an oracle for ratcheting.
        self._oracle = weakref.proxy(oracle)

    def bootstrap(self, persistent_secrets, staged_secrets):
        ''' Initializes the privateer.
        '''
        # We very obviously need to be able to look up what secrets we have.
        # Lookups: <container ghid>: <container secret>
        self._secrets_persistent = persistent_secrets
        self._secrets_staging = staged_secrets
        self._secrets = collections.ChainMap(
            self._secrets_persistent,
            self._secrets_staging,
        )

        # Keep track of chains (this will need to be distributed)
        # Lookup <proxy ghid>: <container ghid>
        self._chains = {}

    def new_secret(self):
        # Straight pass-through to the golix new_secret bit.
        return self._golcore._identity.new_secret()

    def get(self, ghid):
        ''' Get a secret for a ghid, regardless of status.

        Raises KeyError if secret is not present.
        '''
        try:
            with self._modlock:
                return self._secrets[ghid]
        except KeyError as exc:
            raise SecretUnknown('Secret not found for ' + repr(ghid)) from exc

    def stage(self, ghid, secret):
        ''' Preliminarily set a secret for a ghid.

        If a secret is already staged for that ghid and the ghids are
        not equal, raises ConflictingSecrets.
        '''
        with self._modlock:
            if ghid in self._secrets:
                if self._secrets[ghid] != secret:
                    raise ConflictingSecrets(
                        'Non-matching secret already staged for GHID ' +
                        str(ghid)
                    )
            else:
                self._secrets_staging[ghid] = secret

    def unstage(self, ghid):
        ''' Remove a staged secret, probably due to a SecurityError.
        Returns the secret.
        '''
        with self._modlock:
            try:
                secret = self._secrets_staging.pop(ghid)
            except KeyError as exc:
                raise SecretUnknown(
                    'No currently staged secret for GHID ' + str(ghid)
                ) from exc
        return secret

    def commit(self, ghid):
        ''' Store a secret "permanently". The secret must already be
        staged.

        Raises KeyError if ghid is not currently in staging

        This is indempotent; if a ghid is currently in staging AND
        already committed, will compare the two and raise ValueError if
        they don't match.

        This is transactional and atomic; any errors (ex: ValueError
        above) will return its state to the previous.
        '''
        with self._modlock:
            if ghid in self._secrets_persistent:
                self._compare_staged_to_persistent(ghid)
            else:
                try:
                    secret = self._secrets_staging.pop(ghid)
                except KeyError as exc:
                    raise SecretUnknown(
                        'Secret not currently staged for GHID ' + str(ghid)
                    ) from exc
                else:
                    # It doesn't exist, so commit it directly.
                    self._secrets_persistent[ghid] = secret

                # Just keep track of shit for this fucking error
                self._committment_problems[ghid] = TraceLogger.dump_my_trace()

    def _compare_staged_to_persistent(self, ghid):
        try:
            staged = self._secrets_staging.pop(ghid)
        except KeyError:
            # Nothing is staged. Short-circuit.
            pass
        else:
            if staged != self._secrets_persistent[ghid]:
                # Re-stage, just in case.
                self._secrets_staging[ghid] = staged
                raise ConflictingSecrets(
                    'Non-matching secret already committed for GHID ' +
                    str(ghid)
                )

    def abandon(self, ghid, quiet=True):
        ''' Remove a secret. If quiet=True, silence any KeyErrors.
        '''
        # Short circuit any tests if quiet is enabled
        fail_test = not quiet

        with self._modlock:
            try:
                del self._secrets_staging[ghid]
            except KeyError:
                fail_test &= True
                logger.debug('Secret not staged for GHID ' + str(ghid))
            else:
                fail_test = False

            try:
                del self._secrets_persistent[ghid]
            except KeyError:
                fail_test &= True
                logger.debug('Secret not stored for GHID ' + str(ghid))
            else:
                fail_test = False

        if fail_test:
            raise SecretUnknown('Secret not found for GHID ' + str(ghid))

    def make_chain(self, proxy, container):
        ''' Makes a ratchetable chain. Must be owned by a particular
        dynamic address (proxy). Need to know the container address so
        we can ratchet it properly.
        '''
        with self._modlock:
            if proxy in self._chains:
                raise ValueError('Proxy has already been chained.')

            if container not in self._secrets:
                raise ValueError(
                    'Cannot chain unless the container secret is known.'
                )

            self._chains[proxy] = container

    def update_chain(self, proxy, container):
        ''' Updates a chain container address. Must have been ratcheted
        prior to update, and cannot be ratcheted again until updated.
        '''
        with self._modlock:
            if proxy not in self._chains:
                raise ValueError('No chain for proxy.')

            if proxy not in self._ratchet_in_progress:
                raise ValueError('No ratchet in progress for proxy.')

            secret = self._ratchet_in_progress.pop(proxy)
            self._chains[proxy] = container

            # NOTE: this bypasses the usual stage -> commit process, because it
            # can only be used locally during the creation of a new container.
            self._secrets_persistent[container] = secret

    def reset_chain(self, proxy, container):
        ''' Used to reset a chain back to a pristine state.
        '''
        with self._modlock:
            if proxy not in self._chains:
                raise ValueError('Proxy has no existing chain.')

            if container not in self._secrets:
                raise ValueError('Container secret is unknown; cannot reset.')

            try:
                del self._ratchet_in_progress[proxy]
            except KeyError:
                pass

            self._chains[proxy] = container

    def ratchet_chain(self, proxy):
        ''' Gets a new secret for the proxy. Returns the secret, and
        flags the ratchet as in-progress.
        '''
        with self._modlock:
            if proxy in self._ratchet_in_progress:
                raise ValueError('Must update chain prior to re-ratcheting.')

            if proxy not in self._chains:
                raise ValueError('No chain for that proxy.')

            # TODO: make sure this is not a race condition.
            binding = self._oracle.get_object(gaoclass=_GAO, ghid=proxy)
            last_target = binding._history_targets[0]
            last_frame = binding._history[0]

            try:
                existing_secret = self._secrets[last_target]
            except KeyError as exc:
                raise RatchetError('No secret for existing target?') from exc

            ratcheted = self._ratchet(
                secret = existing_secret,
                proxy = proxy,
                salt_ghid = last_frame
            )

            self._ratchet_in_progress[proxy] = ratcheted

        return ratcheted

    def heal_chain(self, gao, binding):
        ''' Heals the ratchet for a binding using the gao. Call this any
        time an agent RECEIVES a new EXTERNAL ratcheted object. Stages
        the resulting secret for the most recent frame in binding, BUT
        DOES NOT RETURN (or commit) IT.
        '''
        # Get a local copy of _history_targets, since it's not currently tsafe
        # TODO: make gao tsafe; fix this leaky abstraction; make sure not race
        gao_targets = gao._history_targets.copy()
        gao_history = gao._history.copy()

        with self._modlock:
            # This finds the first target for which we have a secret.
            for offset in range(len(gao_targets)):
                if gao_targets[offset] in self._secrets:
                    known_secret = self._secrets[gao_targets[offset]]
                    break
                else:
                    continue

            # If we did not find a target, the ratchet is broken.
            else:
                raise RatchetError(
                    'No available secrets for any of the object\'s known past '
                    'targets. Re-establish the ratchet.'
                )

        # Count backwards in index (and therefore forward in time) from the
        # first new frame to zero (and therefore the current frame).
        # Note that we're using the previous frame's ghid as salt.
        for ii in range(offset, -1, -1):
            known_secret = self._ratchet(
                secret = known_secret,
                proxy = gao.ghid,
                salt_ghid = gao_history[ii]
            )

        # DON'T take _modlock here or we will be reentrant = deadlock
        try:
            self.stage(binding.target, known_secret)
            # Note that opening the container itself will commit the secret.
        except:
            logger.warning(
                'Error while staging new secret for attempted ratchet. The '
                'ratchet is very likely broken.\n' +
                ''.join(traceback.format_exc())
            )

    @staticmethod
    def _ratchet(secret, proxy, salt_ghid):
        ''' Ratchets a key using HKDF-SHA512, using the associated
        address as salt. For dynamic files, this should be the previous
        frame ghid (not the dynamic ghid).

        Note: this ratchet is bound to a particular dynamic address. The
        ratchet algorithm is:

        new_key = HKDF-SHA512(
            IKM = old_secret, (secret IV/nonce | secret key)
            salt = old_frame_ghid, (entire 65 bytes)
            info = dynamic_ghid, (entire 65 bytes)
            new_key_length = len(IV/nonce) + len(key),
            num_keys = 1,
        )
        '''
        cls = type(secret)
        cipher = secret.cipher
        version = secret.version
        len_seed = len(secret.seed)
        len_key = len(secret.key)
        source = bytes(secret.seed + secret.key)
        ratcheted = HKDF(
            master = source,
            salt = bytes(salt_ghid),
            key_len = len_seed + len_key,
            hashmod = SHA512,
            num_keys = 1,
            context = bytes(proxy)
        )
        return cls(
            cipher = cipher,
            version = version,
            key = ratcheted[:len_key],
            seed = ratcheted[len_key:]
        )<commit_msg>Added diff percentage for conflicting secrets.<commit_after>'''
LICENSING
-------------------------------------------------

hypergolix: A python Golix client.
    Copyright (C) 2016 Muterra, Inc.

    Contributors
    ------------
    Nick Badger
        badg@muterra.io | badg@nickbadger.com | nickbadger.com

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the
    Free Software Foundation, Inc.,
    51 Franklin Street,
    Fifth Floor,
    Boston, MA  02110-1301 USA

------------------------------------------------------

Some notes:

'''

# Control * imports. Therefore controls what is available to toplevel
# package through __init__.py
__all__ = [
    'Privateer',
]

# External dependencies
import threading
import collections
import weakref
import traceback

# These are used for secret ratcheting only.
from Crypto.Hash import SHA512
from Crypto.Protocol.KDF import HKDF

# Intra-package dependencies
from .core import _GAO
from .core import _GAODict

from .utils import TraceLogger

from .exceptions import PrivateerError
from .exceptions import RatchetError
from .exceptions import ConflictingSecrets
from .exceptions import SecretUnknown

# ###############################################
# Logging boilerplate
# ###############################################

import logging
logger = logging.getLogger(__name__)

# ###############################################
# Lib
# ###############################################

class Privateer:
    ''' Lookup system to get secret from ghid. Threadsafe?

    Note: a lot of the ratcheting state preservation could POTENTIALLY
    be eliminated (and restored via oracle). But, that should probably
    wait for Golix v2.0, which will change secret ratcheting anyways.
    This will also help us track which secrets use the old ratcheting
    mechanism and which ones use the new one, once that change is
    started.
    '''
    def __init__(self):
        self._modlock = threading.Lock()

        # These must be linked during assemble.
        self._golcore = None
        self._oracle = None

        # These must be bootstrapped.
        self._secrets_persistent = None
        self._secrets_staging = None
        self._secrets = None
        # Keep track of chains (this will need to be distributed)
        # Lookup <proxy ghid>: <container ghid>
        self._chains = None

        # Keep track of chain progress. This does not need to be distributed.
        # Lookup <proxy ghid> -> proxy secret
        self._ratchet_in_progress = {}

        # Just here for diagnosing a testing problem
        self._committment_problems = {}

    def __contains__(self, ghid):
        ''' Check if we think we know a secret for the ghid.
        '''
        return ghid in self._secrets

    def assemble(self, golix_core, oracle):
        # Chicken, meet egg.
        self._golcore = weakref.proxy(golix_core)
        # We need an oracle for ratcheting.
        self._oracle = weakref.proxy(oracle)

    def bootstrap(self, persistent_secrets, staged_secrets):
        ''' Initializes the privateer.
        '''
        # We very obviously need to be able to look up what secrets we have.
        # Lookups: <container ghid>: <container secret>
        self._secrets_persistent = persistent_secrets
        self._secrets_staging = staged_secrets
        self._secrets = collections.ChainMap(
            self._secrets_persistent,
            self._secrets_staging,
        )

        # Keep track of chains (this will need to be distributed)
        # Lookup <proxy ghid>: <container ghid>
        self._chains = {}

    def new_secret(self):
        # Straight pass-through to the golix new_secret bit.
        return self._golcore._identity.new_secret()

    def get(self, ghid):
        ''' Get a secret for a ghid, regardless of status.

        Raises KeyError if secret is not present.
        '''
        try:
            with self._modlock:
                return self._secrets[ghid]
        except KeyError as exc:
            raise SecretUnknown('Secret not found for ' + repr(ghid)) from exc

    def stage(self, ghid, secret):
        ''' Preliminarily set a secret for a ghid.

        If a secret is already staged for that ghid and the ghids are
        not equal, raises ConflictingSecrets.
        '''
        with self._modlock:
            if ghid in self._secrets:
                if self._secrets[ghid] != secret:
                    self._calc_and_log_diff(self._secrets[ghid], secret)
                    raise ConflictingSecrets(
                        'Non-matching secret already staged for GHID ' +
                        str(ghid)
                    )
            else:
                self._secrets_staging[ghid] = secret

    def unstage(self, ghid):
        ''' Remove a staged secret, probably due to a SecurityError.
        Returns the secret.
        '''
        with self._modlock:
            try:
                secret = self._secrets_staging.pop(ghid)
            except KeyError as exc:
                raise SecretUnknown(
                    'No currently staged secret for GHID ' + str(ghid)
                ) from exc
        return secret

    def commit(self, ghid):
        ''' Store a secret "permanently". The secret must already be
        staged.

        Raises KeyError if ghid is not currently in staging

        This is indempotent; if a ghid is currently in staging AND
        already committed, will compare the two and raise ValueError if
        they don't match.

        This is transactional and atomic; any errors (ex: ValueError
        above) will return its state to the previous.
        '''
        with self._modlock:
            if ghid in self._secrets_persistent:
                self._compare_staged_to_persistent(ghid)
            else:
                try:
                    secret = self._secrets_staging.pop(ghid)
                except KeyError as exc:
                    raise SecretUnknown(
                        'Secret not currently staged for GHID ' + str(ghid)
                    ) from exc
                else:
                    # It doesn't exist, so commit it directly.
                    self._secrets_persistent[ghid] = secret

                # Just keep track of shit for this fucking error
                self._committment_problems[ghid] = TraceLogger.dump_my_trace()

    @classmethod
    def _calc_and_log_diff(cls, secret, other):
        ''' Calculate the difference between two secrets.
        '''
        try:
            cipher_match = (secret.cipher == other.cipher)

            if cipher_match:
                key_comp = cls._bitdiff(secret.key, other.key)
                seed_comp = cls._bitdiff(secret.seed, other.seed)
                logger.info('Keys are ' + str(key_comp) + '%% different.')
                logger.info('Seeds are ' + str(seed_comp) + '%% different.')
                print('Keys are ' + str(key_comp) + '%% different.')
                print('Seeds are ' + str(seed_comp) + '%% different.')

            else:
                logger.info('Secret ciphers do not match. Cannot compare.')
                print('Secret ciphers do not match. Cannot compare.')

        except AttributeError:
            logger.error(
                'Attribute error while diffing secrets. Type mismatch? \n'
                '    ' + repr(type(secret)) + '\n'
                '    ' + repr(type(other))
            )
            print(
                'Attribute error while diffing secrets. Type mismatch? \n'
                '    ' + repr(type(secret)) + '\n'
                '    ' + repr(type(other))
            )

    @staticmethod
    def _bitdiff(this_bytes, other_bytes):
        ''' Calculates the percent of different bits between two byte
        strings.
        '''
        if len(this_bytes) == 0 or len(other_bytes) == 0:
            # By returning None, we can explicitly say we couldn't perform the
            # comparison, whilst also preventing math on a comparison.
            return None

        # Mask to extract each bit.
        masks = [
            0b00000001,
            0b00000010,
            0b00000100,
            0b00001000,
            0b00010000,
            0b00100000,
            0b01000000,
            0b10000000,
        ]

        # Counters for bits.
        diffbits = 0
        totalbits = 0

        # First iterate over each byte.
        for this_byte, other_byte in zip(this_bytes, other_bytes):

            # Now, using the masks, iterate over each bit.
            for mask in masks:
                # Extract the bit using bitwise AND.
                this_masked = mask & this_byte
                other_masked = mask & other_byte

                # Do a bool comparison of the bits, and add any != results to
                # diffbits. Note that 7 + False == 7 and 7 + True == 8
                diffbits += (this_masked != other_masked)
                totalbits += 1

        # Finally, calculate a percent difference
        doubdiff = diffbits / totalbits
        return int(doubdiff * 100)

    def _compare_staged_to_persistent(self, ghid):
        try:
            staged = self._secrets_staging.pop(ghid)
        except KeyError:
            # Nothing is staged. Short-circuit.
            pass
        else:
            if staged != self._secrets_persistent[ghid]:
                # Re-stage, just in case.
                self._secrets_staging[ghid] = staged
                self._calc_and_log_diff(self._secrets_persistent[ghid], staged)
                raise ConflictingSecrets(
                    'Non-matching secret already committed for GHID ' +
                    str(ghid)
                )

    def abandon(self, ghid, quiet=True):
        ''' Remove a secret. If quiet=True, silence any KeyErrors.
        '''
        # Short circuit any tests if quiet is enabled
        fail_test = not quiet

        with self._modlock:
            try:
                del self._secrets_staging[ghid]
            except KeyError:
                fail_test &= True
                logger.debug('Secret not staged for GHID ' + str(ghid))
            else:
                fail_test = False

            try:
                del self._secrets_persistent[ghid]
            except KeyError:
                fail_test &= True
                logger.debug('Secret not stored for GHID ' + str(ghid))
            else:
                fail_test = False

        if fail_test:
            raise SecretUnknown('Secret not found for GHID ' + str(ghid))

    def make_chain(self, proxy, container):
        ''' Makes a ratchetable chain. Must be owned by a particular
        dynamic address (proxy). Need to know the container address so
        we can ratchet it properly.
        '''
        with self._modlock:
            if proxy in self._chains:
                raise ValueError('Proxy has already been chained.')

            if container not in self._secrets:
                raise ValueError(
                    'Cannot chain unless the container secret is known.'
                )

            self._chains[proxy] = container

    def update_chain(self, proxy, container):
        ''' Updates a chain container address. Must have been ratcheted
        prior to update, and cannot be ratcheted again until updated.
        '''
        with self._modlock:
            if proxy not in self._chains:
                raise ValueError('No chain for proxy.')

            if proxy not in self._ratchet_in_progress:
                raise ValueError('No ratchet in progress for proxy.')

            secret = self._ratchet_in_progress.pop(proxy)
            self._chains[proxy] = container

            # NOTE: this bypasses the usual stage -> commit process, because it
            # can only be used locally during the creation of a new container.
            self._secrets_persistent[container] = secret

    def reset_chain(self, proxy, container):
        ''' Used to reset a chain back to a pristine state.
        '''
        with self._modlock:
            if proxy not in self._chains:
                raise ValueError('Proxy has no existing chain.')

            if container not in self._secrets:
                raise ValueError('Container secret is unknown; cannot reset.')

            try:
                del self._ratchet_in_progress[proxy]
            except KeyError:
                pass

            self._chains[proxy] = container

    def ratchet_chain(self, proxy):
        ''' Gets a new secret for the proxy. Returns the secret, and
        flags the ratchet as in-progress.
        '''
        with self._modlock:
            if proxy in self._ratchet_in_progress:
                raise ValueError('Must update chain prior to re-ratcheting.')

            if proxy not in self._chains:
                raise ValueError('No chain for that proxy.')

            # TODO: make sure this is not a race condition.
            binding = self._oracle.get_object(gaoclass=_GAO, ghid=proxy)
            last_target = binding._history_targets[0]
            last_frame = binding._history[0]

            try:
                existing_secret = self._secrets[last_target]
            except KeyError as exc:
                raise RatchetError('No secret for existing target?') from exc

            ratcheted = self._ratchet(
                secret = existing_secret,
                proxy = proxy,
                salt_ghid = last_frame
            )

            self._ratchet_in_progress[proxy] = ratcheted

        return ratcheted

    def heal_chain(self, gao, binding):
        ''' Heals the ratchet for a binding using the gao. Call this any
        time an agent RECEIVES a new EXTERNAL ratcheted object. Stages
        the resulting secret for the most recent frame in binding, BUT
        DOES NOT RETURN (or commit) IT.
        '''
        # Get a local copy of _history_targets, since it's not currently tsafe
        # TODO: make gao tsafe; fix this leaky abstraction; make sure not race
        gao_targets = gao._history_targets.copy()
        gao_history = gao._history.copy()

        with self._modlock:
            # This finds the first target for which we have a secret.
            for offset in range(len(gao_targets)):
                if gao_targets[offset] in self._secrets:
                    known_secret = self._secrets[gao_targets[offset]]
                    break
                else:
                    continue

            # If we did not find a target, the ratchet is broken.
            else:
                raise RatchetError(
                    'No available secrets for any of the object\'s known past '
                    'targets. Re-establish the ratchet.'
                )

        # Count backwards in index (and therefore forward in time) from the
        # first new frame to zero (and therefore the current frame).
        # Note that we're using the previous frame's ghid as salt.
        for ii in range(offset, -1, -1):
            known_secret = self._ratchet(
                secret = known_secret,
                proxy = gao.ghid,
                salt_ghid = gao_history[ii]
            )

        # DON'T take _modlock here or we will be reentrant = deadlock
        try:
            self.stage(binding.target, known_secret)
            # Note that opening the container itself will commit the secret.
        except:
            logger.warning(
                'Error while staging new secret for attempted ratchet. The '
                'ratchet is very likely broken.\n' +
                ''.join(traceback.format_exc())
            )

    @staticmethod
    def _ratchet(secret, proxy, salt_ghid):
        ''' Ratchets a key using HKDF-SHA512, using the associated
        address as salt. For dynamic files, this should be the previous
        frame ghid (not the dynamic ghid).

        Note: this ratchet is bound to a particular dynamic address. The
        ratchet algorithm is:

        new_key = HKDF-SHA512(
            IKM = old_secret, (secret IV/nonce | secret key)
            salt = old_frame_ghid, (entire 65 bytes)
            info = dynamic_ghid, (entire 65 bytes)
            new_key_length = len(IV/nonce) + len(key),
            num_keys = 1,
        )
        '''
        cls = type(secret)
        cipher = secret.cipher
        version = secret.version
        len_seed = len(secret.seed)
        len_key = len(secret.key)
        source = bytes(secret.seed + secret.key)
        ratcheted = HKDF(
            master = source,
            salt = bytes(salt_ghid),
            key_len = len_seed + len_key,
            hashmod = SHA512,
            num_keys = 1,
            context = bytes(proxy)
        )
        return cls(
            cipher = cipher,
            version = version,
            key = ratcheted[:len_key],
            seed = ratcheted[len_key:]
        )<|endoftext|>


<commit_before>"""Operations regarding miscellaneous files."""

from __future__ import absolute_import, division, print_function
from contextlib import contextmanager
import logging
import os
import re
import shutil
import tempfile
import zipfile

import numpy as np

try:
    from pathlib import Path, PurePath
except ImportError:
    from pathlib2 import Path, PurePath

import dwi.asciifile
import dwi.dicomfile
import dwi.hdf5
from dwi.patient import Patient, Lesion

log = logging.getLogger(__name__)
COMMENT_PREFIX = '#'

@contextmanager
def temp_dir():
    """A temporary directory context that deletes it afterwards."""
    tmpdir = tempfile.mkdtemp()
    try:
        yield tmpdir
    finally:
        shutil.rmtree(tmpdir)

@contextmanager
def read_archive(archive, max_size=None):
    """Extract a ZIP archive into a temporary directory, use as a context."""
    with zipfile.ZipFile(archive, 'r') as a:
        infos = a.infolist()
        total_size = sum(x.file_size for x in infos)
        if max_size is not None and total_size > max_size:
            raise ValueError('Archive content is too big', archive)
        with temp_dir() as tmpdir:
            # Extract one by one because the docs warn against extractall().
            log.debug('Extracting %s to %s', archive, tmpdir)
            for info in infos:
                a.extract(info, tmpdir)
            yield tmpdir

def toline(iterable):
    """Convert an iterable into a line."""
    return ' '.join(str(x) for x in iterable) + '\n'

def sanitize_line(line):
    """Clean up an input line."""
    return line.split(COMMENT_PREFIX, 1)[0].strip()

def valid_lines(path):
    """Read and yield lines that are neither empty nor comments."""
    with Path(path).open(mode='rU') as fp:
        for line in filter(None, (sanitize_line(x) for x in fp)):
            yield line

def ensure_dir(path):
    """Ensure existence of the file's parent directory."""
    Path(path).parent.mkdir(parents=True, exist_ok=True)

def mapped(shape, dtype, filler=None):
    """Create an array as a memory-mapped temporary file on disk."""
    # import tempfile
    # fileno, path = tempfile.mkstemp(suffix='.texture')
    # a = np.memmap(path, dtype=dtype, mode='w+', shape=shape)
    a = np.memmap(os.tmpfile(), dtype=dtype, mode='w+', shape=shape)
    if filler is not None:
        a.fill(filler)
    return a

def parse_patient(line, include_lines=False):
    """Parse a line in patient list file.
    Format: num name scan1,scan2,... score [location]
    """
    regexp = r"""
        (?P<num>\d+) \s+
        (?P<name>\w+) \s+
        (?P<scans>[\w,]+) \s+
        (?P<score>\d\+\d(\+\d)?) \s* (?P<location>\w+)? \s*
        ((?P<score2>\d\+\d(\+\d)?) \s+ (?P<location2>\w+))? \s*
        ((?P<score3>\d\+\d(\+\d)?) \s+ (?P<location3>\w+))?
        """
    p = re.compile(regexp, flags=re.VERBOSE)
    m = p.match(line)
    if m is None:
        raise Exception('Invalid line in patients file: %s', line)
    num = m.group('num')
    name = m.group('name')
    scans = sorted(m.group('scans').lower().split(','))
    les = [Lesion(0, m.group('score'), 'xx')]
    if m.group('location'):
        # New-style, multi-lesion file.
        les = []
        les.append(Lesion(0, m.group('score'), m.group('location')))
        if m.group('score2'):
            les.append(Lesion(1, m.group('score2'), m.group('location2')))
        if m.group('score3'):
            les.append(Lesion(2, m.group('score3'), m.group('location3')))
    patient = Patient(num, name, scans, les)
    if include_lines:
        patient.line = line
    return patient

def read_patients_file(path, include_lines=False):
    """Read a list of patients."""
    return sorted(parse_patient(x, include_lines=include_lines) for x in
                  valid_lines(path))

def read_sample_list(filename):
    """Read a list of samples from file."""
    entries = []
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)')
    for line in valid_lines(filename):
        m = p.match(line)
        if m:
            case, name, scans = m.groups()
            case = int(case)
            name = name.lower()
            scans = tuple(sorted(scans.lower().split(',')))
            d = dict(case=case, name=name, scans=scans)
            entries.append(d)
    return entries

def read_subregion_file(filename):
    """Read a subregion definition from file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last. The "last" ones need +1
    to get Python-like start:stop indices. They are returned as (z_start,
    z_stop, y_start, y_stop, x_start, x_stop).
    """
    entries = []
    for line in valid_lines(filename):
        entries.append(int(float(line)))
    if len(entries) != 6:
        raise Exception('Invalid subregion file: %s' % filename)
    entries = entries[4:6] + entries[0:4]  # Move z indices to front.
    # Add one to "last" indices get Python-like start:stop indices.
    entries[1] += 1
    entries[3] += 1
    entries[5] += 1
    return tuple(entries)

def write_comment(f, text):
    """Write zero or more lines to file with comment prefix."""
    for line in text.splitlines():
        f.write('{p} {s}\n'.format(p=COMMENT_PREFIX, s=line))

def write_subregion_file(filename, win, comment=''):
    """Write a subregion definition to file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last.
    """
    if len(win) != 6:
        raise Exception('Invalid subregion: %s' % win)
    entries = [win[2], win[3]-1, win[4], win[5]-1, win[0], win[1]-1]
    with open(filename, 'w') as f:
        write_comment(f, comment)
        for entry in entries:
            f.write('%i\n' % entry)

def guess_format(path):
    """Guess file format identifier from it's suffix."""
    return Path(path).suffix[1:]

def write_pmap(filename, pmap, attrs, fmt=None):
    """Write parametric map file either as HDF5 or ASCII."""
    pmap = np.asanyarray(pmap)
    if pmap.ndim < 2:
        raise Exception('Not enough dimensions: {}'.format(pmap.shape))
    if 'parameters' not in attrs:
        attrs['parameters'] = [str(i) for i in range(pmap.shape[-1])]
    if 'shape' not in attrs:
        attrs['shape'] = pmap.shape
    if 'dtype' not in attrs:
        attrs['dtype'] = str(pmap.dtype)
    if pmap.shape[-1] != len(attrs['parameters']):
        raise Exception('Number of values and parameters mismatch')
    assert all(dwi.util.isstring(x) for x in
               attrs['parameters']), attrs['parameters']
    if fmt is None:
        fmt = guess_format(filename)
    if fmt == 'h5':
        dwi.hdf5.write_hdf5(filename, pmap, attrs)
    elif fmt == 'txt':
        pmap = pmap.reshape((-1, pmap.shape[-1]))  # Can't keep shape.
        dwi.asciifile.write_ascii_file(filename, pmap, None, attrs=attrs)
    else:
        raise Exception('Unknown format: {}'.format(fmt))

def asindices(iterable, lst):
    """Replace items in iterable with their corresponding indices in list, or
    convert them to integers. E.g. asindices(['o', '22'], 'foobar') => 1, 22.
    """
    for item in iterable:
        try:
            yield lst.index(item)  # Yield corresponding index.
        except ValueError:
            yield int(item)  # Not found, convert to integer.

def pick_params(pmap, attrs, params):
    """Select a subset of parameters by their indices or names."""
    params = list(asindices(params, attrs['parameters']))
    pmap = pmap[..., params]
    if 'bset' in attrs and len(attrs['bset']) == len(attrs['parameters']):
        attrs['bset'] = [attrs['bset'][x] for x in params]
    if 'echotimes' in attrs and (len(attrs['echotimes']) ==
                                 len(attrs['parameters'])):
        attrs['echotimes'] = [attrs['echotimes'][x] for x in params]
    attrs['parameters'] = [attrs['parameters'][x] for x in params]
    return pmap, attrs

def read_pmap(path, ondisk=False, fmt=None, params=None, dtype=None):
    """Read a parametric map.

    With parameter ondisk it will not be read into memory. Parameter params
    tells which parameter indices should be included.
    """
    if fmt is None:
        fmt = guess_format(path)
    if fmt == 'h5':
        pmap, attrs = dwi.hdf5.read_hdf5(path, ondisk=ondisk)
    elif fmt == 'txt':
        attrs, pmap = dwi.asciifile.read_ascii_file(path)
        if 'parameters' in attrs:
            attrs['parameters'] = attrs['parameters'].split()
    elif fmt == 'zip':
        with read_archive(path) as tempdir:
            return read_pmap(tempdir, ondisk=ondisk, fmt=None, params=params,
                             dtype=dtype)
    else:
        # No extension, assume it's a DICOM directory.
        d = dwi.dicomfile.read_dir(path)
        pmap = d.pop('image')
        attrs = dict(d)
    if 'parameters' not in attrs:
        attrs['parameters'] = range(pmap.shape[-1])
    attrs['parameters'] = [str(x) for x in attrs['parameters']]
    if params is not None:
        pmap, attrs = pick_params(pmap, attrs, params)
    if dtype is not None:
        pmap = pmap.astype(dtype)
    log.debug('Read %s, %s, %s', path, pmap.shape, pmap.dtype)
    return pmap, attrs

def read_mask(path, expected_voxel_spacing=None, n_dec=3, container=None,
              allowed_outside=0.2):
    """Read pmap as a mask.

    Optionally expect voxel spacing to match up to a certain number of
    decimals. The optional parameter allowed_outside indicates how much of the
    smaller mask (lesion) volume may be outside of a larger container mask
    (prostate) without an error being raised.
    """
    mask, attrs = read_pmap(path)
    mask = mask[..., 0].astype(np.bool)
    if expected_voxel_spacing is not None:
        vs = [round(x, n_dec) for x in attrs['voxel_spacing']]
        evs = [round(x, n_dec) for x in expected_voxel_spacing]
        if vs != evs:
            s = '{}: Expected voxel spacing {}, got {}'
            raise ValueError(s.format(path, evs, vs))
    if container is not None:
        portion_outside_container = (np.count_nonzero(mask[~container]) /
                                     np.count_nonzero(mask))
        if portion_outside_container > allowed_outside:
            s = '{}: Portion of selected voxels outside container is {:%}'
            raise ValueError(s.format(path, portion_outside_container))
    return mask
<commit_msg>updated mapped()<commit_after>"""Operations regarding miscellaneous files."""

from __future__ import absolute_import, division, print_function
from contextlib import contextmanager
import logging
import re
import shutil
import tempfile
import zipfile

import numpy as np

try:
    from pathlib import Path, PurePath
except ImportError:
    from pathlib2 import Path, PurePath

import dwi.asciifile
import dwi.dicomfile
import dwi.hdf5
from dwi.patient import Patient, Lesion

log = logging.getLogger(__name__)
COMMENT_PREFIX = '#'

@contextmanager
def temp_dir():
    """A temporary directory context that deletes it afterwards."""
    tmpdir = tempfile.mkdtemp()
    try:
        yield tmpdir
    finally:
        shutil.rmtree(tmpdir)

@contextmanager
def read_archive(archive, max_size=None):
    """Extract a ZIP archive into a temporary directory, use as a context."""
    with zipfile.ZipFile(archive, 'r') as a:
        infos = a.infolist()
        total_size = sum(x.file_size for x in infos)
        if max_size is not None and total_size > max_size:
            raise ValueError('Archive content is too big', archive)
        with temp_dir() as tmpdir:
            # Extract one by one because the docs warn against extractall().
            log.debug('Extracting %s to %s', archive, tmpdir)
            for info in infos:
                a.extract(info, tmpdir)
            yield tmpdir

def toline(iterable):
    """Convert an iterable into a line."""
    return ' '.join(str(x) for x in iterable) + '\n'

def sanitize_line(line):
    """Clean up an input line."""
    return line.split(COMMENT_PREFIX, 1)[0].strip()

def valid_lines(path):
    """Read and yield lines that are neither empty nor comments."""
    with Path(path).open(mode='rU') as fp:
        for line in filter(None, (sanitize_line(x) for x in fp)):
            yield line

def ensure_dir(path):
    """Ensure existence of the file's parent directory."""
    Path(path).parent.mkdir(parents=True, exist_ok=True)

@contextmanager
def mapped(shape, dtype, fill_value=None):
    """Create an array as a memory-mapped temporary file on disk."""
    with tempfile.TemporaryFile() as fp:
        a = np.memmap(fp, dtype=dtype, mode='w+', shape=shape)
        if fill_value is not None:
            a.fill(fill_value)
        yield a

def parse_patient(line, include_lines=False):
    """Parse a line in patient list file.
    Format: num name scan1,scan2,... score [location]
    """
    regexp = r"""
        (?P<num>\d+) \s+
        (?P<name>\w+) \s+
        (?P<scans>[\w,]+) \s+
        (?P<score>\d\+\d(\+\d)?) \s* (?P<location>\w+)? \s*
        ((?P<score2>\d\+\d(\+\d)?) \s+ (?P<location2>\w+))? \s*
        ((?P<score3>\d\+\d(\+\d)?) \s+ (?P<location3>\w+))?
        """
    p = re.compile(regexp, flags=re.VERBOSE)
    m = p.match(line)
    if m is None:
        raise Exception('Invalid line in patients file: %s', line)
    num = m.group('num')
    name = m.group('name')
    scans = sorted(m.group('scans').lower().split(','))
    les = [Lesion(0, m.group('score'), 'xx')]
    if m.group('location'):
        # New-style, multi-lesion file.
        les = []
        les.append(Lesion(0, m.group('score'), m.group('location')))
        if m.group('score2'):
            les.append(Lesion(1, m.group('score2'), m.group('location2')))
        if m.group('score3'):
            les.append(Lesion(2, m.group('score3'), m.group('location3')))
    patient = Patient(num, name, scans, les)
    if include_lines:
        patient.line = line
    return patient

def read_patients_file(path, include_lines=False):
    """Read a list of patients."""
    return sorted(parse_patient(x, include_lines=include_lines) for x in
                  valid_lines(path))

def read_sample_list(filename):
    """Read a list of samples from file."""
    entries = []
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)')
    for line in valid_lines(filename):
        m = p.match(line)
        if m:
            case, name, scans = m.groups()
            case = int(case)
            name = name.lower()
            scans = tuple(sorted(scans.lower().split(',')))
            d = dict(case=case, name=name, scans=scans)
            entries.append(d)
    return entries

def read_subregion_file(filename):
    """Read a subregion definition from file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last. The "last" ones need +1
    to get Python-like start:stop indices. They are returned as (z_start,
    z_stop, y_start, y_stop, x_start, x_stop).
    """
    entries = []
    for line in valid_lines(filename):
        entries.append(int(float(line)))
    if len(entries) != 6:
        raise Exception('Invalid subregion file: %s' % filename)
    entries = entries[4:6] + entries[0:4]  # Move z indices to front.
    # Add one to "last" indices get Python-like start:stop indices.
    entries[1] += 1
    entries[3] += 1
    entries[5] += 1
    return tuple(entries)

def write_comment(f, text):
    """Write zero or more lines to file with comment prefix."""
    for line in text.splitlines():
        f.write('{p} {s}\n'.format(p=COMMENT_PREFIX, s=line))

def write_subregion_file(filename, win, comment=''):
    """Write a subregion definition to file.

    It's formatted as one voxel index per line, zero-based, in order of
    y_first, y_last, x_first, x_last, z_first, z_last.
    """
    if len(win) != 6:
        raise Exception('Invalid subregion: %s' % win)
    entries = [win[2], win[3]-1, win[4], win[5]-1, win[0], win[1]-1]
    with open(filename, 'w') as f:
        write_comment(f, comment)
        for entry in entries:
            f.write('%i\n' % entry)

def guess_format(path):
    """Guess file format identifier from it's suffix."""
    return Path(path).suffix[1:]

def write_pmap(filename, pmap, attrs, fmt=None):
    """Write parametric map file either as HDF5 or ASCII."""
    pmap = np.asanyarray(pmap)
    if pmap.ndim < 2:
        raise Exception('Not enough dimensions: {}'.format(pmap.shape))
    if 'parameters' not in attrs:
        attrs['parameters'] = [str(i) for i in range(pmap.shape[-1])]
    if 'shape' not in attrs:
        attrs['shape'] = pmap.shape
    if 'dtype' not in attrs:
        attrs['dtype'] = str(pmap.dtype)
    if pmap.shape[-1] != len(attrs['parameters']):
        raise Exception('Number of values and parameters mismatch')
    assert all(dwi.util.isstring(x) for x in
               attrs['parameters']), attrs['parameters']
    if fmt is None:
        fmt = guess_format(filename)
    if fmt == 'h5':
        dwi.hdf5.write_hdf5(filename, pmap, attrs)
    elif fmt == 'txt':
        pmap = pmap.reshape((-1, pmap.shape[-1]))  # Can't keep shape.
        dwi.asciifile.write_ascii_file(filename, pmap, None, attrs=attrs)
    else:
        raise Exception('Unknown format: {}'.format(fmt))

def asindices(iterable, lst):
    """Replace items in iterable with their corresponding indices in list, or
    convert them to integers. E.g. asindices(['o', '22'], 'foobar') => 1, 22.
    """
    for item in iterable:
        try:
            yield lst.index(item)  # Yield corresponding index.
        except ValueError:
            yield int(item)  # Not found, convert to integer.

def pick_params(pmap, attrs, params):
    """Select a subset of parameters by their indices or names."""
    params = list(asindices(params, attrs['parameters']))
    pmap = pmap[..., params]
    if 'bset' in attrs and len(attrs['bset']) == len(attrs['parameters']):
        attrs['bset'] = [attrs['bset'][x] for x in params]
    if 'echotimes' in attrs and (len(attrs['echotimes']) ==
                                 len(attrs['parameters'])):
        attrs['echotimes'] = [attrs['echotimes'][x] for x in params]
    attrs['parameters'] = [attrs['parameters'][x] for x in params]
    return pmap, attrs

def read_pmap(path, ondisk=False, fmt=None, params=None, dtype=None):
    """Read a parametric map.

    With parameter ondisk it will not be read into memory. Parameter params
    tells which parameter indices should be included.
    """
    if fmt is None:
        fmt = guess_format(path)
    if fmt == 'h5':
        pmap, attrs = dwi.hdf5.read_hdf5(path, ondisk=ondisk)
    elif fmt == 'txt':
        attrs, pmap = dwi.asciifile.read_ascii_file(path)
        if 'parameters' in attrs:
            attrs['parameters'] = attrs['parameters'].split()
    elif fmt == 'zip':
        with read_archive(path) as tempdir:
            return read_pmap(tempdir, ondisk=ondisk, fmt=None, params=params,
                             dtype=dtype)
    else:
        # No extension, assume it's a DICOM directory.
        d = dwi.dicomfile.read_dir(path)
        pmap = d.pop('image')
        attrs = dict(d)
    if 'parameters' not in attrs:
        attrs['parameters'] = range(pmap.shape[-1])
    attrs['parameters'] = [str(x) for x in attrs['parameters']]
    if params is not None:
        pmap, attrs = pick_params(pmap, attrs, params)
    if dtype is not None:
        pmap = pmap.astype(dtype)
    log.debug('Read %s, %s, %s', path, pmap.shape, pmap.dtype)
    return pmap, attrs

def read_mask(path, expected_voxel_spacing=None, n_dec=3, container=None,
              allowed_outside=0.2):
    """Read pmap as a mask.

    Optionally expect voxel spacing to match up to a certain number of
    decimals. The optional parameter allowed_outside indicates how much of the
    smaller mask (lesion) volume may be outside of a larger container mask
    (prostate) without an error being raised.
    """
    mask, attrs = read_pmap(path)
    mask = mask[..., 0].astype(np.bool)
    if expected_voxel_spacing is not None:
        vs = [round(x, n_dec) for x in attrs['voxel_spacing']]
        evs = [round(x, n_dec) for x in expected_voxel_spacing]
        if vs != evs:
            s = '{}: Expected voxel spacing {}, got {}'
            raise ValueError(s.format(path, evs, vs))
    if container is not None:
        portion_outside_container = (np.count_nonzero(mask[~container]) /
                                     np.count_nonzero(mask))
        if portion_outside_container > allowed_outside:
            s = '{}: Portion of selected voxels outside container is {:%}'
            raise ValueError(s.format(path, portion_outside_container))
    return mask
<|endoftext|>


<commit_before>"""Operations regarding miscellaneous files."""

from __future__ import division, print_function
import collections
import os.path
import re

import numpy as np

import dwi.util

COMMENT_PREFIX = '#'

def toline(seq):
    """Convert sequence to line."""
    return ' '.join(map(str, seq)) + '\n'

def valid_lines(filename):
    """Read and yield lines that are neither empty nor comments."""
    with open(filename, 'rU') as f:
        for line in f:
            line = line.strip()
            if line and not line.startswith(COMMENT_PREFIX):
                yield line

def read_subwindows(filename):
    """Read a list of subwindows from file, return in a dictionary."""
    r = {}
    for line in valid_lines(filename):
        words = line.split()
        if len(words) != 8:
            raise Exception('Cannot parse subwindow: %s' % line)
        case, scan, subwindow = int(words[0]), words[1], map(int, words[2:])
        r[(case, scan)] = subwindow
    return r

def read_patients_file(filename, include_lines=False):
    """Load a list of patients.

    Row format: num name scan1,scan2,... score [location]
    """
    from dwi.patient import GleasonScore, Lesion, Patient
    patients = []
    p = re.compile(r"""
            (?P<num>\d+) \s+
            (?P<name>\w+) \s+
            (?P<scans>[\w,]+) \s+
            (?P<score>\d\+\d(\+\d)?) \s* (?P<location>\w+)? \s*
            ((?P<score2>\d\+\d(\+\d)?) \s+ (?P<location2>\w+))? \s*
            ((?P<score3>\d\+\d(\+\d)?) \s+ (?P<location3>\w+))?
            """,
            flags=re.VERBOSE)
    for line in valid_lines(filename):
        m = p.match(line)
        if m:
            num = int(m.group('num'))
            name = m.group('name').lower()
            scans = sorted(m.group('scans').lower().split(','))
            score = GleasonScore(m.group('score'))
            lesions = [Lesion(0, score, 'xx')]
            if m.group('location'):
                # New-style, multi-lesion file.
                lesions = []
                lesions.append(Lesion(0, GleasonScore(m.group('score')),
                        m.group('location').lower()))
                if m.group('score2'):
                    lesions.append(Lesion(1, GleasonScore(m.group('score2')),
                            m.group('location2').lower()))
                if m.group('score3'):
                    lesions.append(Lesion(2, GleasonScore(m.group('score3')),
                            m.group('location3').lower()))
            patient = Patient(num, name, scans, lesions)
            if include_lines:
                patient.line = line
            patients.append(patient)
        else:
            raise Exception('Invalid line in patients file: %s', line)
    return sorted(patients)

def read_sample_list(filename):
    """Read a list of samples from file."""
    entries = []
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)')
    for line in valid_lines(filename):
        m = p.match(line)
        if m:
            case, name, scans = m.groups()
            case = int(case)
            name = name.lower()
            scans = tuple(sorted(scans.lower().split(',')))
            d = dict(case=case, name=name, scans=scans)
            entries.append(d)
    return entries

def read_mask_file(filename):
    """Read a ROI mask file. XXX: Deprecated, use dwi.mask module instead."""
    arrays = []
    for line in valid_lines(filename):
        if line[0] == '0' or line[0] == '1':
            a = np.array(list(line), dtype=int)
            arrays.append(a)
    mask = np.array(arrays)
    return mask

def read_subregion_file(filename):
    """Read a subregion definition from file.

    It's formatted as one voxel index per line, zero-based, in order of y_first,
    y_last, x_first, x_last, z_first, z_last. The "last" ones need +1 to get
    Python-like start:stop indices. They are returned as (z_start, z_stop,
    y_start, y_stop, x_start, x_stop).
    """
    entries = []
    for line in valid_lines(filename):
        entries.append(int(float(line)))
    if len(entries) != 6:
        raise Exception('Invalid subregion file: %s' % filename)
    entries = entries[4:6] + entries[0:4] # Move z indices to front.
    # Add one to "last" indices get Python-like start:stop indices.
    entries[1] += 1
    entries[3] += 1
    entries[5] += 1
    return tuple(entries)

def write_comment(f, text):
    """Write zero or more lines to file with comment prefix."""
    for line in text.splitlines():
        f.write('{p} {s}\n'.format(p=COMMENT_PREFIX, s=line))

def write_subregion_file(filename, win, comment=''):
    """Write a subregion definition to file.

    It's formatted as one voxel index per line, zero-based, in order of y_first,
    y_last, x_first, x_last, z_first, z_last.
    """
    if len(win) != 6:
        raise Exception('Invalid subregion: %s' % win)
    entries = [win[2], win[3]-1, win[4], win[5]-1, win[0], win[1]-1]
    with open(filename, 'w') as f:
        write_comment(f, comment)
        for entry in entries:
            f.write('%i\n' % entry)

def write_pmap(filename, pmap, params=None, fmt=None):
    """Write parametric map file either as HDF5 or ASCII."""
    pmap = np.asanyarray(pmap)
    if pmap.ndim < 2:
        raise Exception('Not enough dimensions: {}'.format(pmap.shape))
    if params is None:
        params = map(str, range(image.shape[-1]))
    if pmap.shape[-1] != len(params):
        raise Exception('Number of values and parameters mismatch')
    if fmt is None:
        fmt = os.path.splitext(filename)[1][1:]
    if fmt in ['hdf5', 'h5']:
        import dwi.hdf5
        attrs = collections.OrderedDict()
        attrs['parameters'] = params
        dwi.hdf5.write_hdf5(filename, image, attrs)
    elif fmt in ['txt', 'ascii']:
        import dwi.asciifile
        pmap = pmap.reshape((-1, pmap.shape[-1])) # Can't keep shape.
        dwi.asciifile.write_ascii_file(filename, pmap, params)
    else:
        raise Exception('Unknown format: {}'.format(fmt))
<commit_msg>varname typo<commit_after>"""Operations regarding miscellaneous files."""

from __future__ import division, print_function
import collections
import os.path
import re

import numpy as np

import dwi.util

COMMENT_PREFIX = '#'

def toline(seq):
    """Convert sequence to line."""
    return ' '.join(map(str, seq)) + '\n'

def valid_lines(filename):
    """Read and yield lines that are neither empty nor comments."""
    with open(filename, 'rU') as f:
        for line in f:
            line = line.strip()
            if line and not line.startswith(COMMENT_PREFIX):
                yield line

def read_subwindows(filename):
    """Read a list of subwindows from file, return in a dictionary."""
    r = {}
    for line in valid_lines(filename):
        words = line.split()
        if len(words) != 8:
            raise Exception('Cannot parse subwindow: %s' % line)
        case, scan, subwindow = int(words[0]), words[1], map(int, words[2:])
        r[(case, scan)] = subwindow
    return r

def read_patients_file(filename, include_lines=False):
    """Load a list of patients.

    Row format: num name scan1,scan2,... score [location]
    """
    from dwi.patient import GleasonScore, Lesion, Patient
    patients = []
    p = re.compile(r"""
            (?P<num>\d+) \s+
            (?P<name>\w+) \s+
            (?P<scans>[\w,]+) \s+
            (?P<score>\d\+\d(\+\d)?) \s* (?P<location>\w+)? \s*
            ((?P<score2>\d\+\d(\+\d)?) \s+ (?P<location2>\w+))? \s*
            ((?P<score3>\d\+\d(\+\d)?) \s+ (?P<location3>\w+))?
            """,
            flags=re.VERBOSE)
    for line in valid_lines(filename):
        m = p.match(line)
        if m:
            num = int(m.group('num'))
            name = m.group('name').lower()
            scans = sorted(m.group('scans').lower().split(','))
            score = GleasonScore(m.group('score'))
            lesions = [Lesion(0, score, 'xx')]
            if m.group('location'):
                # New-style, multi-lesion file.
                lesions = []
                lesions.append(Lesion(0, GleasonScore(m.group('score')),
                        m.group('location').lower()))
                if m.group('score2'):
                    lesions.append(Lesion(1, GleasonScore(m.group('score2')),
                            m.group('location2').lower()))
                if m.group('score3'):
                    lesions.append(Lesion(2, GleasonScore(m.group('score3')),
                            m.group('location3').lower()))
            patient = Patient(num, name, scans, lesions)
            if include_lines:
                patient.line = line
            patients.append(patient)
        else:
            raise Exception('Invalid line in patients file: %s', line)
    return sorted(patients)

def read_sample_list(filename):
    """Read a list of samples from file."""
    entries = []
    p = re.compile(r'(\d+)\s+(\w+)\s+([\w,]+)')
    for line in valid_lines(filename):
        m = p.match(line)
        if m:
            case, name, scans = m.groups()
            case = int(case)
            name = name.lower()
            scans = tuple(sorted(scans.lower().split(',')))
            d = dict(case=case, name=name, scans=scans)
            entries.append(d)
    return entries

def read_mask_file(filename):
    """Read a ROI mask file. XXX: Deprecated, use dwi.mask module instead."""
    arrays = []
    for line in valid_lines(filename):
        if line[0] == '0' or line[0] == '1':
            a = np.array(list(line), dtype=int)
            arrays.append(a)
    mask = np.array(arrays)
    return mask

def read_subregion_file(filename):
    """Read a subregion definition from file.

    It's formatted as one voxel index per line, zero-based, in order of y_first,
    y_last, x_first, x_last, z_first, z_last. The "last" ones need +1 to get
    Python-like start:stop indices. They are returned as (z_start, z_stop,
    y_start, y_stop, x_start, x_stop).
    """
    entries = []
    for line in valid_lines(filename):
        entries.append(int(float(line)))
    if len(entries) != 6:
        raise Exception('Invalid subregion file: %s' % filename)
    entries = entries[4:6] + entries[0:4] # Move z indices to front.
    # Add one to "last" indices get Python-like start:stop indices.
    entries[1] += 1
    entries[3] += 1
    entries[5] += 1
    return tuple(entries)

def write_comment(f, text):
    """Write zero or more lines to file with comment prefix."""
    for line in text.splitlines():
        f.write('{p} {s}\n'.format(p=COMMENT_PREFIX, s=line))

def write_subregion_file(filename, win, comment=''):
    """Write a subregion definition to file.

    It's formatted as one voxel index per line, zero-based, in order of y_first,
    y_last, x_first, x_last, z_first, z_last.
    """
    if len(win) != 6:
        raise Exception('Invalid subregion: %s' % win)
    entries = [win[2], win[3]-1, win[4], win[5]-1, win[0], win[1]-1]
    with open(filename, 'w') as f:
        write_comment(f, comment)
        for entry in entries:
            f.write('%i\n' % entry)

def write_pmap(filename, pmap, params=None, fmt=None):
    """Write parametric map file either as HDF5 or ASCII."""
    pmap = np.asanyarray(pmap)
    if pmap.ndim < 2:
        raise Exception('Not enough dimensions: {}'.format(pmap.shape))
    if params is None:
        params = map(str, range(pmap.shape[-1]))
    if pmap.shape[-1] != len(params):
        raise Exception('Number of values and parameters mismatch')
    if fmt is None:
        fmt = os.path.splitext(filename)[1][1:]
    if fmt in ['hdf5', 'h5']:
        import dwi.hdf5
        attrs = collections.OrderedDict()
        attrs['parameters'] = params
        dwi.hdf5.write_hdf5(filename, pmap, attrs)
    elif fmt in ['txt', 'ascii']:
        import dwi.asciifile
        pmap = pmap.reshape((-1, pmap.shape[-1])) # Can't keep shape.
        dwi.asciifile.write_ascii_file(filename, pmap, params)
    else:
        raise Exception('Unknown format: {}'.format(fmt))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This script is released under the
# Apache License Version 2.0 http://www.apache.org/licenses/.
#
# It generates vectors with coordinates sampled randomly and uniformly from [0,1]
# If one specifies the flag -b or --binary, the script generates only binary data
#

import random
import argparse
import sys
import numpy
import math

parser = argparse.ArgumentParser(description='vector generator (uniform)')

parser.add_argument('-d','--dim',   required=True, type=int, help='dimensionality (# of vector elements)')
parser.add_argument('-n','--ndata', required=True, type=int, help='# of data points')
parser.add_argument('-o','--outf',  required=True, help='output file')
parser.add_argument('--min_val', default=1e-5, help='the minimum possible vector element')

args = vars(parser.parse_args())

nd   = args['ndata']
outf = args['outf']
dim  = args['dim']
minVal=args['min_val']

f=open(outf, 'w')
for i in xrange(nd):
  # See an explanation from this blog post
  # /home/leo/SourceTreeGit/nmslib.master/data/
  arr = numpy.array([-math.log(random.random()) for _ in xrange(dim)])
  arr /= sum(arr)
  f.write("\t".join([("%g" % max(v,minVal)) for v in arr]) + "\n")
f.close()

<commit_msg>Create genhist_unif.py<commit_after>#!/usr/bin/env python
#
# This script is released under the
# Apache License Version 2.0 http://www.apache.org/licenses/.
#
# It generates vectors with coordinates sampled randomly and uniformly from [0,1]
# If one specifies the flag -b or --binary, the script generates only binary data
#

import random
import argparse
import sys
import numpy
import math

parser = argparse.ArgumentParser(description='vector generator (uniform)')

parser.add_argument('-d','--dim',   required=True, type=int, help='dimensionality (# of vector elements)')
parser.add_argument('-n','--ndata', required=True, type=int, help='# of data points')
parser.add_argument('-o','--outf',  required=True, help='output file')
parser.add_argument('--min_val', default=1e-5, help='the minimum possible vector element')

args = vars(parser.parse_args())

nd   = args['ndata']
outf = args['outf']
dim  = args['dim']
minVal=args['min_val']

f=open(outf, 'w')
for i in xrange(nd):
  # See an explanation from this blog post http://blog.geomblog.org/2005/10/sampling-from-simplex.html
  # There's more on sampling from the simplex:  Luc Devroye's book, 'Non-Uniform Random Variate Generation'.
  arr = numpy.array([-math.log(random.random()) for _ in xrange(dim)])
  arr /= sum(arr)
  f.write("\t".join([("%g" % max(v,minVal)) for v in arr]) + "\n")
f.close()

<|endoftext|>


<commit_before><commit_msg>rom: added missing RomPatch<commit_after><|endoftext|>


<commit_before>from datetime import datetime
import os
from shutil import copy2
import sys

from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import yaml

__engine__ = None
__session__ = None

def get_session():
    return __session__

def get_engine():
    return __engine__

def init_database(connection_string, backup=False):
    global __engine__
    global __session__

    if connection_string[0:10] == "sqlite:///":
        print(
            'WARNING: attempting to use SQLite database! Okay for local debugging\n' +
            'but will not work with multiple workers, due to lack of locking features.',
            file=sys.stderr
        )
        db_path = connection_string[10:]
        if backup and os.path.exists(db_path):
            backup_path = db_path + "." + datetime.now().isoformat() + ".backup"
            copy2(db_path, backup_path)
            print("backing up prexisting database file %s to %s" % (db_path, backup_path))

    __engine__ = create_engine(connection_string)
    __session__ = sessionmaker(bind=__engine__)()

    # Create tables in the engine, if they don't exist already.
    Base.metadata.create_all(__engine__)
    Base.metadata.bind = __engine__

# Import all models
from htsohm.db.base import Base
from htsohm.db.gas_loading import GasLoading
from htsohm.db.surface_area import SurfaceArea
from htsohm.db.void_fraction import VoidFraction
from htsohm.db.material import Material
from htsohm.db.structure import Structure
from htsohm.db.atom_sites import AtomSites
from htsohm.db.lennard_jones import LennardJones
<commit_msg>mehtod to find local .db file and return dbcs<commit_after>from datetime import datetime
from glob import glob
import os
from shutil import copy2
import sys

from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import yaml

__engine__ = None
__session__ = None

def get_session():
    return __session__

def get_engine():
    return __engine__

def get_sqlite_dbcs():
    dbs = glob("*.db")
    if len(dbs) > 1:
        print("WARNING: more than one *.db file found in this directory. Using first one: %s" % dbs[0])
    return "sqlite:///%s" % dbs[0]

def init_database(connection_string, backup=False):
    global __engine__
    global __session__

    if connection_string[0:10] == "sqlite:///":
        print(
            'WARNING: attempting to use SQLite database! Okay for local debugging\n' +
            'but will not work with multiple workers, due to lack of locking features.',
            file=sys.stderr
        )
        db_path = connection_string[10:]
        if backup and os.path.exists(db_path):
            backup_path = db_path + "." + datetime.now().isoformat() + ".backup"
            copy2(db_path, backup_path)
            print("backing up prexisting database file %s to %s" % (db_path, backup_path))

    __engine__ = create_engine(connection_string)
    __session__ = sessionmaker(bind=__engine__)()

    # Create tables in the engine, if they don't exist already.
    Base.metadata.create_all(__engine__)
    Base.metadata.bind = __engine__

# Import all models
from htsohm.db.base import Base
from htsohm.db.gas_loading import GasLoading
from htsohm.db.surface_area import SurfaceArea
from htsohm.db.void_fraction import VoidFraction
from htsohm.db.material import Material
from htsohm.db.structure import Structure
from htsohm.db.atom_sites import AtomSites
from htsohm.db.lennard_jones import LennardJones
<|endoftext|>