<commit_before>from __future__ import print_function, division, absolute_import #, unicode_literals # not casa compatible
from builtins import bytes, dict, object, range, map, input#, str # not casa compatible
from future.utils import itervalues, viewitems, iteritems, listvalues, listitems
from io import open

import numpy as np
import os
import shutil
import subprocess
from astropy import time
from time import sleep
from rfpipe import candidates, fileLock
from realfast import elastic, mcaf_servers

import logging
logger = logging.getLogger(__name__)

_candplot_dir = 'claw@nmpost-master:/lustre/aoc/projects/fasttransients/realfast/plots'
_candplot_url_prefix = 'http://realfast.nrao.edu/plots'

def indexcands_and_plots(cc, scanId, tags, indexprefix, workdir):
    """ Wraps indexcands, makesummaryplot, and moveplots calls.
    """

    if len(cc):
        nc = elastic.indexcands(cc, scanId, tags=tags,
                                url_prefix=_candplot_url_prefix,
                                indexprefix=indexprefix)

        # TODO: makesumaryplot logs cands in all segments
        # this is confusing when only one segment being handled here
        msp = makesummaryplot(workdir, scanId)
        workdir = cc.prefs.workdir + '/'
        moveplots(workdir, scanId, destination='{0}/{1}'.format(_candplot_dir,
                                                                indexprefix))
    else:
        nc = 0
        msp = 0

    if nc or msp:
        logger.info('Indexed {0} cands to {1} and moved plots and '
                    'summarized {2} to {3} for scanId {4}'
                    .format(nc, indexprefix+'cands', msp, _candplot_dir,
                            scanId))
    else:
        logger.info('No candidates or plots found.')

def makesummaryplot(workdir, scanId):
    """ Create summary plot for a given scanId and move it
    """

    candsfile = '{0}/cands_{1}.pkl'.format(workdir, scanId)
    ncands = candidates.makesummaryplot(candsfile)
    return ncands

def moveplots(workdir, scanId, destination=_candplot_dir):
    """ For given fileroot, move candidate plots to public location
    """

    logger.info("Moving scanId {0} plots from {1} to {2}"
                .format(scanId, workdir, destination))

#    nplots = 0
#    candplots = glob.glob('{0}/cands_{1}_seg{2}-*.png'
#                          .format(workdir, scanId, segment))
#    for candplot in candplots:
#        success = rsync(candplot, destination)
#        if not success:  # make robust to a failure
#           success = rsync(candplot, destination)
#
#        nplots += success

    # move summary plot too
#    summaryplot = '{0}/cands_{1}.html'.format(workdir, scanId)
#    summaryplotdest = os.path.join(destination, os.path.basename(summaryplot))
#    if os.path.exists(summaryplot):
#        success = rsync(summaryplot, summaryplotdest)
#        if not success:
#            success = rsync(summaryplot, summaryplotdest)
#    else:
#        logger.warn("No summary plot {0} found".format(summaryplot))

    args = ["rsync", "-av", "--include", "cands_{0}*png".format(scanId),
            "--include", "cands_{0}*.html".format(scanId), "--exclude", "*",
            workdir, destination]

    subprocess.call(args)

def indexcandsfile(candsfile, indexprefix, tags=None):
    """ Use candsfile to index cands, scans, prefs, mocks, and noises.
    Should produce identical index results as real-time operation.
    """

    for cc in candidates.iter_cands(candsfile):
        st = cc.state
        scanId = st.metadata.scanId
        workdir = st.prefs.workdir
        mocks = st.prefs.simulated_transient

        elastic.indexscan(inmeta=st.metadata, preferences=st.prefs,
                          indexprefix=indexprefix)
        indexcands_and_plots(cc, scanId, tags, indexprefix, workdir)
        elastic.indexnoises(scanId, noisefile=cc.state.noisefile,
                            indexprefix=indexprefix)
        if mocks is not None:
            elastic.indexmock(scanId, mocks, indexprefix=indexprefix)

def calc_and_indexnoises(st, segment, data, indexprefix='new'):
    """ Wraps calculation and indexing functions.
    Should get calibrated data as input.
    """

    from rfpipe.util import calc_noise

    noises = calc_noise(st, segment, data)
    elastic.indexnoises(st.metadata.scanId, noises=noises,
                        indexprefix=indexprefix)

def createproducts(candcollection, data, indexprefix=None,
                   savebdfdir='/lustre/evla/wcbe/data/realfast/'):
    """ Create SDMs and BDFs for a given candcollection (time segment).
    Takes data future and calls data only if windows found to cut.
    This uses the mcaf_servers module, which calls the sdm builder server.
    Currently BDFs are moved to no_archive lustre area by default.
    """

    from distributed import Future

    if isinstance(candcollection, Future):
        candcollection = candcollection.result()

    if len(candcollection) == 0:
        logger.info('No candidates to generate products for.')
        return []

    if isinstance(data, Future):
        data = data.result()

    assert isinstance(data, np.ndarray) and data.dtype == 'complex64'

    logger.info("Creating an SDM for {0}, segment {1}, with {2} candidates"
                .format(candcollection.metadata.scanId, candcollection.segment,
                        len(candcollection)))

    wait = candcollection.metadata.endtime_mjd + 10/(24*3600)  # end+10s
    now = time.Time.now().mjd
    if now < wait:
        logger.info("Waiting until {0} for ScanId {1} to complete"
                    .format(wait, candcollection.metadata.scanId))

        while now < wait:
            sleep(1)
            now = time.Time.now().mjd

    metadata = candcollection.metadata
    segment = candcollection.segment
    if not isinstance(segment, int):
        logger.warning("Cannot get unique segment from candcollection")

    st = candcollection.state

    candranges = gencandranges(candcollection)  # finds time windows to save
    calScanTime = candcollection.soltime  # solution saved during search
    logger.info('Getting data for candidate time ranges {0} in segment {1}.'
                .format(candranges, segment))

    ninttot, nbl, nchantot, npol = data.shape
    nchan = metadata.nchan_orig//metadata.nspw_orig
    nspw = metadata.nspw_orig

    sdmlocs = []
    # make sdm for each unique time range (e.g., segment)
    for (startTime, endTime) in set(candranges):
        i = (86400*(startTime-st.segmenttimes[segment][0])/metadata.inttime).astype(int)
        nint = np.round(86400*(endTime-startTime)/metadata.inttime, 1).astype(int)
        logger.info("Cutting {0} ints from int {1} for candidate at {2} in segment {3}"
                    .format(nint, i, startTime, segment))
        data_cut = data[i:i+nint].reshape(nint, nbl, nspw, 1, nchan, npol)

        # TODO: fill in annotation dict as defined in confluence doc on realfast collections
        annotation = {}

# now retrieved from candcollection
#        calScanTime = np.unique(calibration.getsols(st)['mjd'])
#        if len(calScanTime) > 1:
#            logger.warn("Using first of multiple cal times: {0}."
#                        .format(calScanTime))
#        calScanTime = calScanTime[0]

        sdmloc = mcaf_servers.makesdm(startTime, endTime, metadata.datasetId,
                                      data_cut, calScanTime,
                                      annotation=annotation)
        if sdmloc is not None:
            # update index to link to new sdm
            if indexprefix is not None:
                candIds = elastic.candid(cc=candcollection)
                for Id in candIds:
                    elastic.update_field(indexprefix+'cands', 'sdmname',
                                         sdmloc, Id=Id)

            sdmlocs.append(sdmloc)
            logger.info("Created new SDMs at: {0}".format(sdmloc))
            # TODO: migrate bdfdir to newsdmloc once ingest tool is ready
            mcaf_servers.makebdf(startTime, endTime, metadata, data_cut,
                                 bdfdir=savebdfdir)
        else:
            logger.warn("No sdm/bdf made for {0} with start/end time {1}-{2}"
                        .format(metadata.datasetId, startTime, endTime))

    return sdmlocs

def get_sdmname(candcollection):
    """ Use candcollection to get name of SDM output by the sdmbuilder
    """

    datasetId = candcollection.metadata.datasetId
    uid = get_uid(candcollection)
    outputDatasetId = 'realfast_{0}_{1}'.format(datasetId, uid.rsplit('/')[-1])

    return outputDatasetId

def get_uid(candcollection):
    """ Get unix time appropriate for bdf naming
    Expects one segment per candcollection and one range per segment.
    """

    candranges = gencandranges(candcollection)
    assert len(candranges) == 1, "Assuming 1 candrange per candcollection and/or segment"

    startTime, endTime = candranges[0]

    uid = ('uid:///evla/realfastbdf/{0}'
           .format(int(time.Time(startTime, format='mjd').unix*1e3)))

    return uid

def assemble_sdmbdf(candcollection, sdmloc='/home/mctest/evla/mcaf/workspace/',
                    bdfdir='/lustre/evla/wcbe/data/realfast/'):
    """ Use candcollection to assemble cutout SDM and BDF from file system at VLA
    into a viable SDM.
    """

    destination = '.'
    sdm0 = get_sdmname(candcollection)
    bdf0 = get_uid(candcollection).replace(':', '_').replace('/', '_')
    newsdm = os.path.join(destination, sdm0)
    shutil.copytree(os.path.join(sdmloc, sdm0), newsdm)

    bdfdestination = os.path.join(newsdm, 'ASDMBinary')
    os.mkdir(bdfdestination)
    shutil.copy(os.path.join(bdfdir, bdf0), os.path.join(bdfdestination, bdf0))

    logger.info("Assembled SDM/BDF at {0}".format(newsdm))

    return newsdm

def runingest(sdms):
    """ Call archive tool or move data to trigger archiving of sdms.
    This function will ultimately be triggered by candidate portal.
    """

    NotImplementedError
#    /users/vlapipe/workflows/test/bin/ingest -m -p /home/mctest/evla/mcaf/workspace --file

def data_logger(st, segment, data):
    """ Function that inspects read data and writes results to file.
    """

    filename = os.path.join(st.prefs.workdir,
                            "data_" + st.fileroot + ".txt")

    t0 = st.segmenttimes[segment][0]
    timearr = ','.join((t0+st.metadata.inttime*np.arange(st.readints))
                       .astype(str))

    if data.ndim == 4:
        results = ','.join(data.mean(axis=3).mean(axis=2).any(axis=1)
                           .astype(str))
    else:
        results = 'None'

    try:
        with fileLock.FileLock(filename, timeout=60):
            with open(filename, "a") as fp:
                fp.write("{0}: {1} {2} {3}\n".format(segment, t0, timearr,
                                                     results))
    except fileLock.FileLock.FileLockException:
        logger.warn("data_logger on segment {0} failed to write due to file timeout"
                    .format(segment))

def data_logging_report(filename):
    """ Read and summarize data logging file
    """

    with open(filename, 'r') as fp:
        for line in fp.readlines():
            segment, starttime, dts, filled = line.split(' ')
            segment = int(segment.rstrip(':'))
            starttime = float(starttime)
            dts = [float(dt) for dt in dts.split(',')]
            filled = [fill.rstrip() == "True" for fill in filled.split(',')]
            if len(filled) > 1:
                filled = np.array(filled, dtype=int)
                logger.info("Segment {0}: {1}/{2} missing"
                            .format(segment, len(filled)-filled.sum(),
                                    len(filled)))
                logger.info("{0}".format(filled))
            else:
                logger.info("Segment {0}: no data".format(segment))

def gencandranges(candcollection):
    """ Given a candcollection, define a list of candidate time ranges.
    Currently saves the whole segment.
    """

    segment = candcollection.segment
    st = candcollection.state

    # save whole segment
    return [(st.segmenttimes[segment][0], st.segmenttimes[segment][1])]

def initialize_worker():
    from rfpipe import search, state, metadata

    search.set_wisdom(512)
    st = state.State(inmeta=metadata.mock_metadata(0,0.001, 27, 16, 16*32, 4, 5e4, datasource='sim'))

def rsync(original, new):
    """ Uses subprocess.call to rsync from 'filename' to 'new'
    If new is directory, copies original in.
    If new is new file, copies original to that name.
    """

    assert os.path.exists(original), 'Need original file!'
    res = subprocess.call(["rsync", "--timeout", "30", "-a", original.rstrip('/'), new.rstrip('/')])

    return int(res == 0)
<commit_msg>looks for candcollection with signs of excess candidates<commit_after>from __future__ import print_function, division, absolute_import #, unicode_literals # not casa compatible
from builtins import bytes, dict, object, range, map, input#, str # not casa compatible
from future.utils import itervalues, viewitems, iteritems, listvalues, listitems
from io import open

import numpy as np
import os
import shutil
import subprocess
from astropy import time
from time import sleep
from rfpipe import candidates, fileLock
from realfast import elastic, mcaf_servers

import logging
logger = logging.getLogger(__name__)

_candplot_dir = 'claw@nmpost-master:/lustre/aoc/projects/fasttransients/realfast/plots'
_candplot_url_prefix = 'http://realfast.nrao.edu/plots'

def indexcands_and_plots(cc, scanId, tags, indexprefix, workdir):
    """ Wraps indexcands, makesummaryplot, and moveplots calls.
    """

    if len(cc):
        nc = elastic.indexcands(cc, scanId, tags=tags,
                                url_prefix=_candplot_url_prefix,
                                indexprefix=indexprefix)

        # TODO: makesumaryplot logs cands in all segments
        # this is confusing when only one segment being handled here
        msp = makesummaryplot(workdir, scanId)
        workdir = cc.prefs.workdir + '/'
        moveplots(workdir, scanId, destination='{0}/{1}'.format(_candplot_dir,
                                                                indexprefix))
    else:
        nc = 0
        msp = 0
        if len(cc.array) > 0:
            logger.warn('CandCollection in segment {0} is empty. '
                        '{1} realtime detections exceeded maximum'
                        .format(cc.segment, cc.array.ncands))

    if nc or msp:
        logger.info('Indexed {0} cands to {1} and moved plots and '
                    'summarized {2} to {3} for scanId {4}'
                    .format(nc, indexprefix+'cands', msp, _candplot_dir,
                            scanId))
    else:
        logger.info('No candidates or plots found.')

def makesummaryplot(workdir, scanId):
    """ Create summary plot for a given scanId and move it
    """

    candsfile = '{0}/cands_{1}.pkl'.format(workdir, scanId)
    ncands = candidates.makesummaryplot(candsfile)
    return ncands

def moveplots(workdir, scanId, destination=_candplot_dir):
    """ For given fileroot, move candidate plots to public location
    """

    logger.info("Moving scanId {0} plots from {1} to {2}"
                .format(scanId, workdir, destination))

#    nplots = 0
#    candplots = glob.glob('{0}/cands_{1}_seg{2}-*.png'
#                          .format(workdir, scanId, segment))
#    for candplot in candplots:
#        success = rsync(candplot, destination)
#        if not success:  # make robust to a failure
#           success = rsync(candplot, destination)
#
#        nplots += success

    # move summary plot too
#    summaryplot = '{0}/cands_{1}.html'.format(workdir, scanId)
#    summaryplotdest = os.path.join(destination, os.path.basename(summaryplot))
#    if os.path.exists(summaryplot):
#        success = rsync(summaryplot, summaryplotdest)
#        if not success:
#            success = rsync(summaryplot, summaryplotdest)
#    else:
#        logger.warn("No summary plot {0} found".format(summaryplot))

    args = ["rsync", "-av", "--include", "cands_{0}*png".format(scanId),
            "--include", "cands_{0}*.html".format(scanId), "--exclude", "*",
            workdir, destination]

    subprocess.call(args)

def indexcandsfile(candsfile, indexprefix, tags=None):
    """ Use candsfile to index cands, scans, prefs, mocks, and noises.
    Should produce identical index results as real-time operation.
    """

    for cc in candidates.iter_cands(candsfile):
        st = cc.state
        scanId = st.metadata.scanId
        workdir = st.prefs.workdir
        mocks = st.prefs.simulated_transient

        elastic.indexscan(inmeta=st.metadata, preferences=st.prefs,
                          indexprefix=indexprefix)
        indexcands_and_plots(cc, scanId, tags, indexprefix, workdir)
        elastic.indexnoises(scanId, noisefile=cc.state.noisefile,
                            indexprefix=indexprefix)
        if mocks is not None:
            elastic.indexmock(scanId, mocks, indexprefix=indexprefix)

def calc_and_indexnoises(st, segment, data, indexprefix='new'):
    """ Wraps calculation and indexing functions.
    Should get calibrated data as input.
    """

    from rfpipe.util import calc_noise

    noises = calc_noise(st, segment, data)
    elastic.indexnoises(st.metadata.scanId, noises=noises,
                        indexprefix=indexprefix)

def createproducts(candcollection, data, indexprefix=None,
                   savebdfdir='/lustre/evla/wcbe/data/realfast/'):
    """ Create SDMs and BDFs for a given candcollection (time segment).
    Takes data future and calls data only if windows found to cut.
    This uses the mcaf_servers module, which calls the sdm builder server.
    Currently BDFs are moved to no_archive lustre area by default.
    """

    from distributed import Future

    if isinstance(candcollection, Future):
        candcollection = candcollection.result()

    if len(candcollection) == 0:
        logger.info('No candidates to generate products for.')
        return []

    if isinstance(data, Future):
        data = data.result()

    assert isinstance(data, np.ndarray) and data.dtype == 'complex64'

    logger.info("Creating an SDM for {0}, segment {1}, with {2} candidates"
                .format(candcollection.metadata.scanId, candcollection.segment,
                        len(candcollection)))

    wait = candcollection.metadata.endtime_mjd + 10/(24*3600)  # end+10s
    now = time.Time.now().mjd
    if now < wait:
        logger.info("Waiting until {0} for ScanId {1} to complete"
                    .format(wait, candcollection.metadata.scanId))

        while now < wait:
            sleep(1)
            now = time.Time.now().mjd

    metadata = candcollection.metadata
    segment = candcollection.segment
    if not isinstance(segment, int):
        logger.warning("Cannot get unique segment from candcollection")

    st = candcollection.state

    candranges = gencandranges(candcollection)  # finds time windows to save
    calScanTime = candcollection.soltime  # solution saved during search
    logger.info('Getting data for candidate time ranges {0} in segment {1}.'
                .format(candranges, segment))

    ninttot, nbl, nchantot, npol = data.shape
    nchan = metadata.nchan_orig//metadata.nspw_orig
    nspw = metadata.nspw_orig

    sdmlocs = []
    # make sdm for each unique time range (e.g., segment)
    for (startTime, endTime) in set(candranges):
        i = (86400*(startTime-st.segmenttimes[segment][0])/metadata.inttime).astype(int)
        nint = np.round(86400*(endTime-startTime)/metadata.inttime, 1).astype(int)
        logger.info("Cutting {0} ints from int {1} for candidate at {2} in segment {3}"
                    .format(nint, i, startTime, segment))
        data_cut = data[i:i+nint].reshape(nint, nbl, nspw, 1, nchan, npol)

        # TODO: fill in annotation dict as defined in confluence doc on realfast collections
        annotation = {}

# now retrieved from candcollection
#        calScanTime = np.unique(calibration.getsols(st)['mjd'])
#        if len(calScanTime) > 1:
#            logger.warn("Using first of multiple cal times: {0}."
#                        .format(calScanTime))
#        calScanTime = calScanTime[0]

        sdmloc = mcaf_servers.makesdm(startTime, endTime, metadata.datasetId,
                                      data_cut, calScanTime,
                                      annotation=annotation)
        if sdmloc is not None:
            # update index to link to new sdm
            if indexprefix is not None:
                candIds = elastic.candid(cc=candcollection)
                for Id in candIds:
                    elastic.update_field(indexprefix+'cands', 'sdmname',
                                         sdmloc, Id=Id)

            sdmlocs.append(sdmloc)
            logger.info("Created new SDMs at: {0}".format(sdmloc))
            # TODO: migrate bdfdir to newsdmloc once ingest tool is ready
            mcaf_servers.makebdf(startTime, endTime, metadata, data_cut,
                                 bdfdir=savebdfdir)
        else:
            logger.warn("No sdm/bdf made for {0} with start/end time {1}-{2}"
                        .format(metadata.datasetId, startTime, endTime))

    return sdmlocs

def get_sdmname(candcollection):
    """ Use candcollection to get name of SDM output by the sdmbuilder
    """

    datasetId = candcollection.metadata.datasetId
    uid = get_uid(candcollection)
    outputDatasetId = 'realfast_{0}_{1}'.format(datasetId, uid.rsplit('/')[-1])

    return outputDatasetId

def get_uid(candcollection):
    """ Get unix time appropriate for bdf naming
    Expects one segment per candcollection and one range per segment.
    """

    candranges = gencandranges(candcollection)
    assert len(candranges) == 1, "Assuming 1 candrange per candcollection and/or segment"

    startTime, endTime = candranges[0]

    uid = ('uid:///evla/realfastbdf/{0}'
           .format(int(time.Time(startTime, format='mjd').unix*1e3)))

    return uid

def assemble_sdmbdf(candcollection, sdmloc='/home/mctest/evla/mcaf/workspace/',
                    bdfdir='/lustre/evla/wcbe/data/realfast/'):
    """ Use candcollection to assemble cutout SDM and BDF from file system at VLA
    into a viable SDM.
    """

    destination = '.'
    sdm0 = get_sdmname(candcollection)
    bdf0 = get_uid(candcollection).replace(':', '_').replace('/', '_')
    newsdm = os.path.join(destination, sdm0)
    shutil.copytree(os.path.join(sdmloc, sdm0), newsdm)

    bdfdestination = os.path.join(newsdm, 'ASDMBinary')
    os.mkdir(bdfdestination)
    shutil.copy(os.path.join(bdfdir, bdf0), os.path.join(bdfdestination, bdf0))

    logger.info("Assembled SDM/BDF at {0}".format(newsdm))

    return newsdm

def runingest(sdms):
    """ Call archive tool or move data to trigger archiving of sdms.
    This function will ultimately be triggered by candidate portal.
    """

    NotImplementedError
#    /users/vlapipe/workflows/test/bin/ingest -m -p /home/mctest/evla/mcaf/workspace --file

def data_logger(st, segment, data):
    """ Function that inspects read data and writes results to file.
    """

    filename = os.path.join(st.prefs.workdir,
                            "data_" + st.fileroot + ".txt")

    t0 = st.segmenttimes[segment][0]
    timearr = ','.join((t0+st.metadata.inttime*np.arange(st.readints))
                       .astype(str))

    if data.ndim == 4:
        results = ','.join(data.mean(axis=3).mean(axis=2).any(axis=1)
                           .astype(str))
    else:
        results = 'None'

    try:
        with fileLock.FileLock(filename, timeout=60):
            with open(filename, "a") as fp:
                fp.write("{0}: {1} {2} {3}\n".format(segment, t0, timearr,
                                                     results))
    except fileLock.FileLock.FileLockException:
        logger.warn("data_logger on segment {0} failed to write due to file timeout"
                    .format(segment))

def data_logging_report(filename):
    """ Read and summarize data logging file
    """

    with open(filename, 'r') as fp:
        for line in fp.readlines():
            segment, starttime, dts, filled = line.split(' ')
            segment = int(segment.rstrip(':'))
            starttime = float(starttime)
            dts = [float(dt) for dt in dts.split(',')]
            filled = [fill.rstrip() == "True" for fill in filled.split(',')]
            if len(filled) > 1:
                filled = np.array(filled, dtype=int)
                logger.info("Segment {0}: {1}/{2} missing"
                            .format(segment, len(filled)-filled.sum(),
                                    len(filled)))
                logger.info("{0}".format(filled))
            else:
                logger.info("Segment {0}: no data".format(segment))

def gencandranges(candcollection):
    """ Given a candcollection, define a list of candidate time ranges.
    Currently saves the whole segment.
    """

    segment = candcollection.segment
    st = candcollection.state

    # save whole segment
    return [(st.segmenttimes[segment][0], st.segmenttimes[segment][1])]

def initialize_worker():
    from rfpipe import search, state, metadata

    search.set_wisdom(512)
    st = state.State(inmeta=metadata.mock_metadata(0,0.001, 27, 16, 16*32, 4, 5e4, datasource='sim'))

def rsync(original, new):
    """ Uses subprocess.call to rsync from 'filename' to 'new'
    If new is directory, copies original in.
    If new is new file, copies original to that name.
    """

    assert os.path.exists(original), 'Need original file!'
    res = subprocess.call(["rsync", "--timeout", "30", "-a", original.rstrip('/'), new.rstrip('/')])

    return int(res == 0)
<|endoftext|>


<commit_before>import jwt

from marshmallow import fields, Schema, validate, ValidationError

def validate_jwt(value):
    try:
        jwt.decode(value, verify=False, algorithms=['HS256'])
    except jwt.exceptions.DecodeError:
        raise ValidationError('Invalid JWT')

class DeviceDataSchema(Schema):
    jwt = fields.Field(required=True, validate=validate_jwt)
    typology = fields.String(required=True, validate=validate.OneOf(['moto', 'bike']))
    device_id = fields.UUID(required=True)
    recording_id = fields.UUID(required=True)
    user_id = fields.UUID(required=True)
    activity_id = fields.UUID(required=True)

class PointSchema(Schema):
    sequence_id = fields.Integer()
    ele = fields.Float()
    time = fields.DateTime()
    course = fields.Float()
    speed = fields.Float()
    position = fields.Field()
    geoidheight = fields.Float()
    fix = fields.Integer(validate=validate.Range(min=0, max=8))
    sat = fields.Integer()
    hdop = fields.Float()
    vdop = fields.Float()
    pdop = fields.Float()

class PathSchema(Schema):
    coordinates = fields.List(fields.List(fields.Decimal), required=True)

class RecordingSchema(Schema):
    """
    Data structure expected in `recordings` (mallow, sutt) and
    `path_proposal` (sutt, bliss, isaac)
    """
    jwt = fields.Field(required=True, validate=validate_jwt)
    typology = fields.String(required=True, validate=validate.OneOf(['moto', 'bike']))
    id = fields.UUID(required=True)
    user_id = fields.UUID(required=True)
    activity_id = fields.UUID(required=True)
    device_id = fields.UUID(required=True)
    started = fields.DateTime(required=True)
    ended = fields.DateTime(required=True)
    points = fields.Nested(PointSchema, many=True)
    path = fields.Nested(PathSchema)

class MatchRecordingSchema(Schema):
    """
    recording part of the expected message in `recordings_matches`.

    `recording_matches` is a tuple composed by 3 elements: recording,
    matching_segments, matching_ids. This is the first part
    """
    id = fields.UUID(required=True)
    user_id = fields.UUID(required=True)
    activity_id = fields.UUID(required=True)
    jwt = fields.Field(required=True, validate=validate_jwt)
    bbox = fields.List(fields.Decimal())
    path = fields.Nested(PathSchema)
    points = fields.List(fields.Dict())

class SectorSchema(Schema):
    name = fields.String()
    id = fields.UUID()
    start_index = fields.Integer()
    end_index = fields.Integer()

class MatchSchema(Schema):
    indexes = fields.List(fields.Field(), required=True)
    bbox = fields.List(fields.Integer(), required=True)
    path = fields.Nested(PathSchema, required=True)
    sectors = fields.Nested(SectorSchema, many=True)

class MatchingSegmentSchema(Schema):
    """
    matched_segment part of the expected message in `recordings_matches`

    `recording_matches` is a tuple composed by 3 elements: recording,
    matching_segments, matching_ids. This is the second part
    """
    segment_id = fields.Nested(MatchSchema)

class MatchIndexSchema(Schema):
    """
    matching_ids part of the expected message in `recording_matches`

    `recording_matches` is a tuple composed by 3 elements: recording,
    matching_segments, matching_ids. This is the third part
    """
    segment_id = fields.List(fields.Field())
<commit_msg>Created new schema for pathproposal<commit_after>import jwt

from marshmallow import fields, Schema, validate, ValidationError

def validate_jwt(value):
    try:
        jwt.decode(value, verify=False, algorithms=['HS256'])
    except jwt.exceptions.DecodeError:
        raise ValidationError('Invalid JWT')

class DeviceDataSchema(Schema):
    jwt = fields.Field(required=True, validate=validate_jwt)
    typology = fields.String(required=True, validate=validate.OneOf(['moto', 'bike']))
    device_id = fields.UUID(required=True)
    recording_id = fields.UUID(required=True)
    user_id = fields.UUID(required=True)
    activity_id = fields.UUID(required=True)

class PointSchema(Schema):
    sequence_id = fields.Integer()
    ele = fields.Float()
    time = fields.DateTime()
    course = fields.Float()
    speed = fields.Float()
    position = fields.Field()
    geoidheight = fields.Float()
    fix = fields.Integer(validate=validate.Range(min=0, max=8))
    sat = fields.Integer()
    hdop = fields.Float()
    vdop = fields.Float()
    pdop = fields.Float()

class PathSchema(Schema):
    coordinates = fields.List(fields.List(fields.Decimal), required=True)

class RecordingSchema(Schema):
    """
    Data structure expected in `recordings` (mallow, sutt)
    """
    jwt = fields.Field(required=True, validate=validate_jwt)
    typology = fields.String(required=True, validate=validate.OneOf(['moto', 'bike']))
    id = fields.UUID(required=True)
    user_id = fields.UUID(required=True)
    activity_id = fields.UUID(required=True)
    device_id = fields.UUID(required=True)
    started = fields.DateTime(required=True)
    ended = fields.DateTime(required=True)
    points = fields.Nested(PointSchema, many=True)

class PathProposalSchema(RecordingSchema):
    """
    Data structure expected in `path_proposal` (sutt, bliss, isaac)
    Same as RecordingSchema, with 'path' field added
    """
    path = fields.Nested(PathSchema, required=True)

class MatchRecordingSchema(Schema):
    """
    recording part of the expected message in `recordings_matches`.

    `recording_matches` is a tuple composed by 3 elements: recording,
    matching_segments, matching_ids. This is the first part
    """
    id = fields.UUID(required=True)
    user_id = fields.UUID(required=True)
    activity_id = fields.UUID(required=True)
    jwt = fields.Field(required=True, validate=validate_jwt)
    bbox = fields.List(fields.Decimal())
    path = fields.Nested(PathSchema)
    points = fields.List(fields.Dict())

class SectorSchema(Schema):
    name = fields.String()
    id = fields.UUID()
    start_index = fields.Integer()
    end_index = fields.Integer()

class MatchSchema(Schema):
    indexes = fields.List(fields.Field(), required=True)
    bbox = fields.List(fields.Integer(), required=True)
    path = fields.Nested(PathSchema, required=True)
    sectors = fields.Nested(SectorSchema, many=True)

class MatchingSegmentSchema(Schema):
    """
    matched_segment part of the expected message in `recordings_matches`

    `recording_matches` is a tuple composed by 3 elements: recording,
    matching_segments, matching_ids. This is the second part
    """
    segment_id = fields.Nested(MatchSchema)

class MatchIndexSchema(Schema):
    """
    matching_ids part of the expected message in `recording_matches`

    `recording_matches` is a tuple composed by 3 elements: recording,
    matching_segments, matching_ids. This is the third part
    """
    segment_id = fields.List(fields.Field())
<|endoftext|>


<commit_before>#!/usr/bin/python
from select import select
import subprocess
import os
import signal
import itertools
import threading
import shlex
import sys
import time

def start_fio(path, client, storage, exit_code):
    fio_args = list()
    fio_args.append("fio")
    remote_server = ""
    if client:
        fio_args.append("--client=" + client)
    fio_args.append("--minimal")
    fio_args.append("--eta=0")
    fio_args.append("--status-interval=1")
    fio_args.append(path)
    fio_process = subprocess.Popen(fio_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, preexec_fn=os.setpgrp)
    parsing_thread = threading.Thread(target=lambda: parse_fio_output(storage[0]['all'], storage[1]['all'], storage[2]['all'], storage[3]['all'], storage[4]['all'], storage[5]['all'], storage[0]['job_vals'], storage[1]['job_vals'], storage[2]['job_vals'], storage[3]['job_vals'], storage[4]['job_vals'], storage[5]['job_vals'], fio_process, get_jobs(path), exit_code), args=())
    return parsing_thread, fio_process

def parse_fio_output(riops, rbw, rlat, wiops, wbw, wlat, job_riops, job_rbw, job_rlat, job_wiops, job_wbw, job_wlat, fio_process, numjobs, exit_code):
    cur_job = 0
    while fio_process.poll() == None:
        try:
            if cur_job < numjobs:
                split = fio_process.stdout.readline().split(';')
                split[129]
                job_riops[cur_job].append(int(split[7]))
                job_rbw[cur_job].append(int(split[6]))
                job_rlat[cur_job].append(float(split[15]))
                job_wiops[cur_job].append(float(split[48]))
                job_wbw[cur_job].append(int(split[47]))
                job_wlat[cur_job].append(float(split[56]))
                cur_job+=1
            elif cur_job == numjobs:
                cur_job = 0
                riops_tot = 0
                rbw_tot = 0
                wiops_tot = 0
                wbw_tot = 0
                w_lat_tmp = []
                r_lat_tmp = []

                for i in range(0, numjobs):
                    r_lat_tmp.append(job_rlat[i][-1])
                    w_lat_tmp.append(job_wlat[i][-1])
                    riops_tot+=job_riops[i][-1]
                    rbw_tot+=job_rbw[i][-1]
                    wiops_tot+=job_wiops[i][-1]
                    wbw_tot+=job_wbw[i][-1]
                riops.append(riops_tot)
                rbw.append(rbw_tot)
                rlat.append(max(r_lat_tmp))
                wiops.append(wiops_tot)
                wbw.append(wbw_tot)
                wlat.append(max(w_lat_tmp))

        except IndexError:
            poll_res = fio_process.poll()
            if poll_res != None:
                exit_code[0] = poll_res
                return poll_res

def get_jobs(path):
    if path.endswith('ini'):
        cmds = subprocess.Popen(['fio', '--showcmd', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('utf-8').split(' --')
        for cmd in cmds:
            if cmd.startswith('numjobs'):
                return int(cmd.split('=')[1])
<commit_msg>realtime_back.py: fix use of fio eta argument<commit_after>#!/usr/bin/python
from select import select
import subprocess
import os
import signal
import itertools
import threading
import shlex
import sys
import time

def start_fio(path, client, storage, exit_code):
    fio_args = list()
    fio_args.append("fio")
    remote_server = ""
    if client:
        fio_args.append("--client=" + client)
    fio_args.append("--minimal")
    fio_args.append("--eta=never")
    fio_args.append("--status-interval=1")
    fio_args.append(path)
    fio_process = subprocess.Popen(fio_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, preexec_fn=os.setpgrp)
    parsing_thread = threading.Thread(target=lambda: parse_fio_output(storage[0]['all'], storage[1]['all'], storage[2]['all'], storage[3]['all'], storage[4]['all'], storage[5]['all'], storage[0]['job_vals'], storage[1]['job_vals'], storage[2]['job_vals'], storage[3]['job_vals'], storage[4]['job_vals'], storage[5]['job_vals'], fio_process, get_jobs(path), exit_code), args=())
    return parsing_thread, fio_process

def parse_fio_output(riops, rbw, rlat, wiops, wbw, wlat, job_riops, job_rbw, job_rlat, job_wiops, job_wbw, job_wlat, fio_process, numjobs, exit_code):
    cur_job = 0
    while fio_process.poll() == None:
        try:
            if cur_job < numjobs:
                split = fio_process.stdout.readline().split(';')
                split[129]
                job_riops[cur_job].append(int(split[7]))
                job_rbw[cur_job].append(int(split[6]))
                job_rlat[cur_job].append(float(split[15]))
                job_wiops[cur_job].append(float(split[48]))
                job_wbw[cur_job].append(int(split[47]))
                job_wlat[cur_job].append(float(split[56]))
                cur_job+=1
            elif cur_job == numjobs:
                cur_job = 0
                riops_tot = 0
                rbw_tot = 0
                wiops_tot = 0
                wbw_tot = 0
                w_lat_tmp = []
                r_lat_tmp = []

                for i in range(0, numjobs):
                    r_lat_tmp.append(job_rlat[i][-1])
                    w_lat_tmp.append(job_wlat[i][-1])
                    riops_tot+=job_riops[i][-1]
                    rbw_tot+=job_rbw[i][-1]
                    wiops_tot+=job_wiops[i][-1]
                    wbw_tot+=job_wbw[i][-1]
                riops.append(riops_tot)
                rbw.append(rbw_tot)
                rlat.append(max(r_lat_tmp))
                wiops.append(wiops_tot)
                wbw.append(wbw_tot)
                wlat.append(max(w_lat_tmp))

        except IndexError:
            poll_res = fio_process.poll()
            if poll_res != None:
                exit_code[0] = poll_res
                return poll_res

def get_jobs(path):
    if path.endswith('ini'):
        cmds = subprocess.Popen(['fio', '--showcmd', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode('utf-8').split(' --')
        for cmd in cmds:
            if cmd.startswith('numjobs'):
                return int(cmd.split('=')[1])
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

"""
Test fetching source files using the SourceCode() method.
"""

import os
import stat

import TestSCons

test = TestSCons.TestSCons()

test.subdir('sub', 'sub2')

test.write('SConstruct', """\
import os.path

def cat(env, source, target):
    target = str(target[0])
    source = map(str, source)
    f = open(target, "wb")
    for src in source:
        f.write(open(src, "rb").read())
    f.close()

def sc_cat(env, source, target):
    source = []
    for t in target:
        head, tail = os.path.split(str(t))
        source.append(os.path.join(head, 'sc-' + tail))
    cat(env, source, target)

env = Environment(BUILDERS={'Cat':Builder(action=cat)}, SUBDIR='sub')
env.Cat('aaa.out', 'sub/aaa.in')
env.Cat('bbb.out', 'sub/bbb.in')
env.Cat('ccc.out', 'sub/ccc.in')
env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out'])
env.SourceCode('$SUBDIR', Builder(action=sc_cat, env=env))
SConscript('sub/SConscript', "env")

SourceCode('sub2', Builder(action=sc_cat, env=env))
env.Cat('ddd.out', 'sub2/ddd.in')
""")

test.write(['sub', 'sc-aaa.in'], "sub/sc-aaa.in\n")
test.write(['sub', 'sc-bbb.in'], "sub/sc-bbb.in\n")
test.write(['sub', 'sc-ccc.in'], "sub/sc-ccc.in\n")
test.write(['sub2', 'sc-ddd.in'], "sub2/sc-ddd.in\n")

test.write(['sub', 'sc-SConscript'], "'sub/sc-SConscript'\n")

test.run(arguments = '.',
         stdout = test.wrap_stdout(read_str = """\
sc_cat(["%s"], [])
""" % (os.path.join('sub', 'SConscript')),
                                   build_str = """\
sc_cat(["%s"], [])
cat(["aaa.out"], ["%s"])
sc_cat(["%s"], [])
cat(["bbb.out"], ["%s"])
sc_cat(["%s"], [])
cat(["ccc.out"], ["%s"])
cat(["all"], ["aaa.out", "bbb.out", "ccc.out"])
sc_cat(["%s"], [])
cat(["ddd.out"], ["%s"])
""" % (os.path.join('sub', 'aaa.in'),
       os.path.join('sub', 'aaa.in'),
       os.path.join('sub', 'bbb.in'),
       os.path.join('sub', 'bbb.in'),
       os.path.join('sub', 'ccc.in'),
       os.path.join('sub', 'ccc.in'),
       os.path.join('sub2', 'ddd.in'),
       os.path.join('sub2', 'ddd.in'))))

test.must_match(['sub', 'SConscript'], "'sub/sc-SConscript'\n")
test.must_match('all', "sub/sc-aaa.in\nsub/sc-bbb.in\nsub/sc-ccc.in\n")
test.must_match('ddd.out', "sub2/sc-ddd.in\n")

test.pass_test()
<commit_msg>Fix SourceCode test ordering.  (Kevin Quick)<commit_after>#!/usr/bin/env python
#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

"""
Test fetching source files using the SourceCode() method.
"""

import os
import stat

import TestSCons

test = TestSCons.TestSCons()

test.subdir('sub', 'sub2')

test.write('SConstruct', """\
import os.path

def cat(env, source, target):
    target = str(target[0])
    source = map(str, source)
    f = open(target, "wb")
    for src in source:
        f.write(open(src, "rb").read())
    f.close()

def sc_cat(env, source, target):
    source = []
    for t in target:
        head, tail = os.path.split(str(t))
        source.append(os.path.join(head, 'sc-' + tail))
    cat(env, source, target)

env = Environment(BUILDERS={'Cat':Builder(action=cat)}, SUBDIR='sub')
env.SourceCode('$SUBDIR', Builder(action=sc_cat, env=env))
env.Cat('aaa.out', 'sub/aaa.in')
bbb_in = File('sub/bbb.in')
bbb_in.is_pseudo_derived()
env.Cat('bbb.out', bbb_in)
env.Cat('ccc.out', 'sub/ccc.in')
env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out'])
SConscript('sub/SConscript', "env")

SourceCode('sub2', Builder(action=sc_cat, env=env))
env.Cat('ddd.out', 'sub2/ddd.in')
""")

test.write(['sub', 'sc-aaa.in'], "sub/sc-aaa.in\n")
test.write(['sub', 'sc-bbb.in'], "sub/sc-bbb.in\n")
test.write(['sub', 'sc-ccc.in'], "sub/sc-ccc.in\n")
test.write(['sub2', 'sc-ddd.in'], "sub2/sc-ddd.in\n")

test.write(['sub', 'sc-SConscript'], "'sub/sc-SConscript'\n")

test.run(arguments = '.',
         stdout = test.wrap_stdout(read_str = """\
sc_cat(["%s"], [])
""" % (os.path.join('sub', 'SConscript')),
                                   build_str = """\
sc_cat(["%s"], [])
cat(["aaa.out"], ["%s"])
sc_cat(["%s"], [])
cat(["bbb.out"], ["%s"])
sc_cat(["%s"], [])
cat(["ccc.out"], ["%s"])
cat(["all"], ["aaa.out", "bbb.out", "ccc.out"])
sc_cat(["%s"], [])
cat(["ddd.out"], ["%s"])
""" % (os.path.join('sub', 'aaa.in'),
       os.path.join('sub', 'aaa.in'),
       os.path.join('sub', 'bbb.in'),
       os.path.join('sub', 'bbb.in'),
       os.path.join('sub', 'ccc.in'),
       os.path.join('sub', 'ccc.in'),
       os.path.join('sub2', 'ddd.in'),
       os.path.join('sub2', 'ddd.in'))))

test.must_match(['sub', 'SConscript'], "'sub/sc-SConscript'\n")
test.must_match('all', "sub/sc-aaa.in\nsub/sc-bbb.in\nsub/sc-ccc.in\n")
test.must_match('ddd.out', "sub2/sc-ddd.in\n")

test.pass_test()
<|endoftext|>


<commit_before>import re
import os

import numpy as np
from skimage.io import use_plugin, imread

from coords2d import Coords2D

import logging
logger = logging.getLogger('__main__.{}'.format(__name__))
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler())

def sorted_nicely( l ):
    """ Sort the given iterable in the way that humans expect."""
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

class ReconstructedCell(object):
    """Pseudo 3D cell from contiguous z-stacks."""

    def __init__(self, slice_dict):
        logger.debug('Initialising ReconstructedCell')
        self.slice_dict = slice_dict

    def add_slice(self, layer, cellslice):
        """Add ROI from another z-stack."""
        self.slice_dict[layer] = cellslice

    @property
    def pixel_area(self):
        """Return the total cell area across all the z-stacks."""
        return sum(cellslice.pixel_area
                   for cellslice in self.slice_dict.values())

    @property
    def z_extent(self):
        """Return the number of z-stacks in the reconstructed cell."""
        return len(self.slice_dict.keys())

    def measure_total_intensity(self, idata):
        """Return the total intensity across all the z-stacks."""
        total_intensity = 0
        for sID, cellslice in self.slice_dict.items():
            #z_correction = 1 + (0.03 * sID)
            z_correction = 1
            total_intensity += z_correction * sum(idata[sID][cellslice.coord_list])
        return total_intensity

    def measure_mean_intensity(self, idata):
        """Return the mean intensity of the reconstructed cell."""
        return float(self.measure_total_intensity(idata)) / self.pixel_area

    def measure_quartile_intensity(self, idata):
        """Return the quartile intensity of the reconstructed cell."""
        all_measurements = [idata[sID][cellslice.coord_list]
                            for sID, cellslice in self.slice_dict.items()]
        as_single_array = np.concatenate(all_measurements)
        n_frac = len(as_single_array) / 2
        return np.mean(as_single_array[n_frac:])

    def measure_best_slice(self, idata):
        """Return the intensity from the most intense slice."""
        slice_arrays = [idata[sID][cellslice.coord_list]
                        for sID, cellslice in self.slice_dict.items()]
        slice_means = [np.mean(sa) for sa in slice_arrays]
        best_slice = max(slice_means)
        best_z = np.array(slice_means).argmax()
        return best_slice, best_z

    def __repr__(self):
        return "<ReconstructedCell: %s>" % self.slice_dict.__repr__()

    def simple_string_rep(self):
        """Return a simple string representation of the reconstructed cell."""
        return ",".join("%d:%d" % (sID, cellslice.ID)
                        for sID, cellslice
                        in self.slice_dict.items())

    @property
    def centroid(self):
        """Return the centroid of the reconstructed cell."""
        z = float(sum(self.slice_dict.keys())) / len(self.slice_dict)
        csum = sum([c.centroid for c in self.slice_dict.values()],
                   Coords2D(0, 0))
        x, y = map(float, csum / len(self.slice_dict))
        return x, y, z

class CellSlice(object):
    """Slice of a cell in the x, y plane."""

    def __init__(self, ID, coord_list):
        logger.debug('Initialising CellSlice')
        self.ID = ID
        self.coord_list = coord_list
        self.pixel_area = len(coord_list[0])
        self.x_coords = coord_list[0]
        self.y_coords = coord_list[1]

    @property
    def centroid(self):
        """Return the centroid of the cell slice."""
        return Coords2D(sum(self.x_coords),
                        sum(self.y_coords)) / self.pixel_area

    @property
    def summary(self):
        """Return summary string representation of the cell slice."""
        return "<ID: %d, pixel_area: %d, centroid: %s>" % (self.ID,
                                                           self.pixel_area,
                                                           self.centroid)

    def __repr__(self):
        return "<CellSlice, ID %d>" % self.ID

class SegmentationMap(object):
    """Container for the pseudo 3D reconstructed cells."""

    def __init__(self, image_file):
        logger.debug('Initialising SegmentationMap')
        use_plugin('freeimage')
        self.im_array = imread(image_file)
        self.internal_cc = None

    @property
    def cells(self):
        """Return the dictionary of cell slices."""
        if self.internal_cc is not None:
            return self.internal_cc
        else:
            self.internal_cc = cell_dict_from_image_array(self.im_array)
            return self.internal_cc

    def cell_at(self, position):
        """Return the cell at position (x, y)."""
        x, y = position

        ID = self.im_array[x, y]
        if ID == 0:
            return None

        return self.cells[self.im_array[x, y]]

    @property
    def all_ids(self):
        """Return all cell ids."""
        with_zero = list(np.unique(self.im_array))
        with_zero.remove(0)
        return with_zero

    def coord_list(self, cID):
        """Return list of coordinates for cell (ID)."""
        return np.where(self.im_array == cID)

class Reconstruction(object):
    """Pseudo 3D reconstruction."""

    def __init__(self, smaps, start=0):
        logger.debug('Initialising Reconstruction.')
        self.smaps = smaps
        self.rcells = []
        self.lut = {}
        z = start
        for ID in smaps[z].all_ids:
            logger.debug('Loop id: {}'.format(ID))
            rcell = ReconstructedCell({z: self.smaps[z].cells[ID]})
            self.rcells.append(rcell)
            self.lut[(z, ID)] = rcell

    def extend(self, level):
        """Add another z-stack to the reconstruction."""
        z = level
        matches = find_slice_links(self.smaps[z], self.smaps[z+1])

        for f, t in matches.iteritems():
            try:
                rcell = self.lut[(z, f)]
                rcell.add_slice(z+1, self.smaps[z+1].cells[t])
            except KeyError:
                rcell = ReconstructedCell({z+1: self.smaps[z+1].cells[t]})
                self.rcells.append(rcell)

            self.lut[(z+1, t)] = rcell

    def cells_larger_then(self, zext):
        """Return cells which have more than zext z-stacks."""
        return [rcell for rcell in self.rcells
                if rcell.z_extent >= zext]

    def save_to_file(self, filename):
        """Save the reconstructed cells to file using simple cell representation."""
        with open(filename, "w") as f:
            f.write('\n'.join([rcell.simple_string_rep()
                               for rcell in self.rcells]))

def cell_dict_from_image_array(i_array):
    """Return dictionary of cell slices from an array of images."""
    cd = {cid: CellSlice(cid, np.where(i_array == cid))
          for cid in np.unique(i_array)}
    del(cd[0])
    return cd

def load_segmentation_maps(slice_dir):
    """Return list of segmentation maps from a directory of segmentations."""
    image_files = os.listdir(slice_dir)
    image_files = sorted_nicely(image_files)
    smaps = [SegmentationMap(os.path.join(slice_dir, im_file))
             for im_file in image_files]
    return smaps

def find_slice_links(slice_map1, slice_map2):
    """Return dictionary of matched cells between two slice maps."""
    matches = {}
    for cell_slice in slice_map1.cells.values():
        c = cell_slice.centroid
        candidate_slice = slice_map2.cell_at(c)
        if slice_from_same_cell(cell_slice, candidate_slice):
            matches[cell_slice.ID] = candidate_slice.ID
    return matches

def slice_from_same_cell(slice1, slice2):
    """Whether or not two cell slices are from the same cell."""
    if slice1 is None or slice2 is None:
        return False

    area1 = slice1.pixel_area
    area2 = slice2.pixel_area

    dist = slice1.centroid.dist(slice2.centroid)
    area_ratio = float(area1) / area2

    if 0.5 < area_ratio < 1.5 and dist < 20:
        return True
    else:
        return False
<commit_msg>Removed hard coded logger settings from reconstructor.py.<commit_after>import re
import os

import numpy as np
from skimage.io import use_plugin, imread

from coords2d import Coords2D

import logging
logger = logging.getLogger('__main__.{}'.format(__name__))

def sorted_nicely( l ):
    """ Sort the given iterable in the way that humans expect."""
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

class ReconstructedCell(object):
    """Pseudo 3D cell from contiguous z-stacks."""

    def __init__(self, slice_dict):
        logger.debug('Initialising ReconstructedCell')
        self.slice_dict = slice_dict

    def add_slice(self, layer, cellslice):
        """Add ROI from another z-stack."""
        self.slice_dict[layer] = cellslice

    @property
    def pixel_area(self):
        """Return the total cell area across all the z-stacks."""
        return sum(cellslice.pixel_area
                   for cellslice in self.slice_dict.values())

    @property
    def z_extent(self):
        """Return the number of z-stacks in the reconstructed cell."""
        return len(self.slice_dict.keys())

    def measure_total_intensity(self, idata):
        """Return the total intensity across all the z-stacks."""
        total_intensity = 0
        for sID, cellslice in self.slice_dict.items():
            #z_correction = 1 + (0.03 * sID)
            z_correction = 1
            total_intensity += z_correction * sum(idata[sID][cellslice.coord_list])
        return total_intensity

    def measure_mean_intensity(self, idata):
        """Return the mean intensity of the reconstructed cell."""
        return float(self.measure_total_intensity(idata)) / self.pixel_area

    def measure_quartile_intensity(self, idata):
        """Return the quartile intensity of the reconstructed cell."""
        all_measurements = [idata[sID][cellslice.coord_list]
                            for sID, cellslice in self.slice_dict.items()]
        as_single_array = np.concatenate(all_measurements)
        n_frac = len(as_single_array) / 2
        return np.mean(as_single_array[n_frac:])

    def measure_best_slice(self, idata):
        """Return the intensity from the most intense slice."""
        slice_arrays = [idata[sID][cellslice.coord_list]
                        for sID, cellslice in self.slice_dict.items()]
        slice_means = [np.mean(sa) for sa in slice_arrays]
        best_slice = max(slice_means)
        best_z = np.array(slice_means).argmax()
        return best_slice, best_z

    def __repr__(self):
        return "<ReconstructedCell: %s>" % self.slice_dict.__repr__()

    def simple_string_rep(self):
        """Return a simple string representation of the reconstructed cell."""
        return ",".join("%d:%d" % (sID, cellslice.ID)
                        for sID, cellslice
                        in self.slice_dict.items())

    @property
    def centroid(self):
        """Return the centroid of the reconstructed cell."""
        z = float(sum(self.slice_dict.keys())) / len(self.slice_dict)
        csum = sum([c.centroid for c in self.slice_dict.values()],
                   Coords2D(0, 0))
        x, y = map(float, csum / len(self.slice_dict))
        return x, y, z

class CellSlice(object):
    """Slice of a cell in the x, y plane."""

    def __init__(self, ID, coord_list):
        logger.debug('Initialising CellSlice')
        self.ID = ID
        self.coord_list = coord_list
        self.pixel_area = len(coord_list[0])
        self.x_coords = coord_list[0]
        self.y_coords = coord_list[1]

    @property
    def centroid(self):
        """Return the centroid of the cell slice."""
        return Coords2D(sum(self.x_coords),
                        sum(self.y_coords)) / self.pixel_area

    @property
    def summary(self):
        """Return summary string representation of the cell slice."""
        return "<ID: %d, pixel_area: %d, centroid: %s>" % (self.ID,
                                                           self.pixel_area,
                                                           self.centroid)

    def __repr__(self):
        return "<CellSlice, ID %d>" % self.ID

class SegmentationMap(object):
    """Container for the pseudo 3D reconstructed cells."""

    def __init__(self, image_file):
        logger.debug('Initialising SegmentationMap')
        use_plugin('freeimage')
        self.im_array = imread(image_file)
        self.internal_cc = None

    @property
    def cells(self):
        """Return the dictionary of cell slices."""
        if self.internal_cc is not None:
            return self.internal_cc
        else:
            self.internal_cc = cell_dict_from_image_array(self.im_array)
            return self.internal_cc

    def cell_at(self, position):
        """Return the cell at position (x, y)."""
        x, y = position

        ID = self.im_array[x, y]
        if ID == 0:
            return None

        return self.cells[self.im_array[x, y]]

    @property
    def all_ids(self):
        """Return all cell ids."""
        with_zero = list(np.unique(self.im_array))
        with_zero.remove(0)
        return with_zero

    def coord_list(self, cID):
        """Return list of coordinates for cell (ID)."""
        return np.where(self.im_array == cID)

class Reconstruction(object):
    """Pseudo 3D reconstruction."""

    def __init__(self, smaps, start=0):
        logger.debug('Initialising Reconstruction.')
        self.smaps = smaps
        self.rcells = []
        self.lut = {}
        z = start
        for ID in smaps[z].all_ids:
            logger.debug('Loop id: {}'.format(ID))
            rcell = ReconstructedCell({z: self.smaps[z].cells[ID]})
            self.rcells.append(rcell)
            self.lut[(z, ID)] = rcell

    def extend(self, level):
        """Add another z-stack to the reconstruction."""
        z = level
        matches = find_slice_links(self.smaps[z], self.smaps[z+1])

        for f, t in matches.iteritems():
            try:
                rcell = self.lut[(z, f)]
                rcell.add_slice(z+1, self.smaps[z+1].cells[t])
            except KeyError:
                rcell = ReconstructedCell({z+1: self.smaps[z+1].cells[t]})
                self.rcells.append(rcell)

            self.lut[(z+1, t)] = rcell

    def cells_larger_then(self, zext):
        """Return cells which have more than zext z-stacks."""
        return [rcell for rcell in self.rcells
                if rcell.z_extent >= zext]

    def save_to_file(self, filename):
        """Save the reconstructed cells to file using simple cell representation."""
        with open(filename, "w") as f:
            f.write('\n'.join([rcell.simple_string_rep()
                               for rcell in self.rcells]))

def cell_dict_from_image_array(i_array):
    """Return dictionary of cell slices from an array of images."""
    cd = {cid: CellSlice(cid, np.where(i_array == cid))
          for cid in np.unique(i_array)}
    del(cd[0])
    return cd

def load_segmentation_maps(slice_dir):
    """Return list of segmentation maps from a directory of segmentations."""
    image_files = os.listdir(slice_dir)
    image_files = sorted_nicely(image_files)
    smaps = [SegmentationMap(os.path.join(slice_dir, im_file))
             for im_file in image_files]
    return smaps

def find_slice_links(slice_map1, slice_map2):
    """Return dictionary of matched cells between two slice maps."""
    matches = {}
    for cell_slice in slice_map1.cells.values():
        c = cell_slice.centroid
        candidate_slice = slice_map2.cell_at(c)
        if slice_from_same_cell(cell_slice, candidate_slice):
            matches[cell_slice.ID] = candidate_slice.ID
    return matches

def slice_from_same_cell(slice1, slice2):
    """Whether or not two cell slices are from the same cell."""
    if slice1 is None or slice2 is None:
        return False

    area1 = slice1.pixel_area
    area2 = slice2.pixel_area

    dist = slice1.centroid.dist(slice2.centroid)
    area_ratio = float(area1) / area2

    if 0.5 < area_ratio < 1.5 and dist < 20:
        return True
    else:
        return False
<|endoftext|>


<commit_before>import re
import os

import numpy as np
from skimage.io import use_plugin, imread

from coords2d import Coords2D
#import regionmap

def sorted_nicely( l ):
    """ Sort the given iterable in the way that humans expect."""
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

class ReconstructedCell(object):

    def __init__(self, slice_dict):
        self.slice_dict = slice_dict

    def add_slice(self, layer, cellslice):
        self.slice_dict[layer] = cellslice

    @property
    def pixel_area(self):
        return sum(cellslice.pixel_area
                   for cellslice in self.slice_dict.values())

    @property
    def z_extent(self):
        return len(self.slice_dict.keys())

    def measure_total_intensity(self, idata):
        total_intensity = 0

        for sID, cellslice in self.slice_dict.items():
            #z_correction = 1 + (0.03 * sID)
            z_correction = 1
            total_intensity += z_correction * sum(idata[sID][cellslice.coord_list])

        return total_intensity

    def measure_mean_intensity(self, idata):

        return float(self.measure_total_intensity(idata)) / self.pixel_area

    def measure_quartile_intensity(self, idata):

        all_measurements = [idata[sID][cellslice.coord_list]
                            for sID, cellslice in self.slice_dict.items()]

        as_single_array = np.concatenate(all_measurements)

        #dist_frac = 0.25

        n_frac = len(as_single_array) / 2

        return np.mean(as_single_array[n_frac:])

    def measure_best_slice(self, idata):

        slice_arrays = [idata[sID][cellslice.coord_list]
                        for sID, cellslice in self.slice_dict.items()]

        slice_means = [np.mean(sa) for sa in slice_arrays]

        best_slice = max(slice_means)
        best_z = np.array(slice_means).argmax()

        return best_slice, best_z

    def __repr__(self):
        return "<ReconstructedCell: %s>" % self.slice_dict.__repr__()

    def simple_string_rep(self):
        return ",".join("%d:%d" % (sID, cellslice.ID)
                        for sID, cellslice
                        in self.slice_dict.items())

    @property
    def centroid(self):
        z = float(sum(self.slice_dict.keys())) / len(self.slice_dict)
        csum = sum([c.centroid for c in self.slice_dict.values()],
                   Coords2D(0, 0))
        x, y = map(float, csum / len(self.slice_dict))
        return x, y, z

class CellSlice(object):

    def __init__(self, ID, coord_list):
        self.ID = ID
        self.coord_list = coord_list
        self.pixel_area = len(coord_list[0])
        self.x_coords = coord_list[0]
        self.y_coords = coord_list[1]

    @property
    def centroid(self):
        return Coords2D(sum(self.x_coords),
                        sum(self.y_coords)) / self.pixel_area

    @property
    def summary(self):
        return "<ID: %d, pixel_area: %d, centroid: %s>" % (self.ID,
                                                           self.pixel_area,
                                                           self.centroid)

    def __repr__(self):
        return "<CellSlice, ID %d>" % self.ID

class SegmentationMap(object):

    def __init__(self, image_file):
        use_plugin('freeimage')
        self.im_array = imread(image_file)
        # self.im_array = np.transpose(self.im_array)
        # cd = regionmap.load_image(image_file)
        # self.internal_cc = {cid: CellSlice(cid, clist)
        #                     for cid, clist in cd.items()}
        self.internal_cc = None

    @property
    def cells(self):
        #return self.internal_cc
        if self.internal_cc is not None:
            return self.internal_cc
        else:
            self.internal_cc = cell_dict_from_image_array(self.im_array)
            return self.internal_cc

    def cell_at(self, position):
        x, y = position

        ID = self.im_array[x, y]
        if ID == 0:
            return None

        return self.cells[self.im_array[x, y]]

    @property
    def all_ids(self):
        with_zero = list(np.unique(self.im_array))
        with_zero.remove(0)
        return with_zero

    def coord_list(self, cID):
        return np.where(self.im_array == cID)

class Reconstruction(object):

    def __init__(self, smaps, start=0):
        self.smaps = smaps
        self.rcells = []
        self.lut = {}
        z = start
        for ID in smaps[z].all_ids:
            rcell = ReconstructedCell({z: self.smaps[z].cells[ID]})
            self.rcells.append(rcell)
            self.lut[(z, ID)] = rcell

    def extend(self, level):
        z = level
        matches = find_slice_links(self.smaps[z], self.smaps[z+1])

        for f, t in matches.iteritems():
            try:
                rcell = self.lut[(z, f)]
                rcell.add_slice(z+1, self.smaps[z+1].cells[t])
            except KeyError:
                rcell = ReconstructedCell({z+1: self.smaps[z+1].cells[t]})
                self.rcells.append(rcell)

            self.lut[(z+1, t)] = rcell

    def cells_larger_then(self, zext):
        return [rcell for rcell in self.rcells
                if rcell.z_extent >= zext]

    def save_to_file(self, filename):
        with open(filename, "w") as f:
            f.write('\n'.join([rcell.simple_string_rep()
                               for rcell in self.rcells]))

def cell_dict_from_image_array(i_array):

    cd = {cid: CellSlice(cid, np.where(i_array == cid))
          for cid in np.unique(i_array)}

    del(cd[0])

    return cd

def load_segmentation_maps(slice_dir):

    image_files = os.listdir(slice_dir)

    image_files = sorted_nicely(image_files)

    smaps = [SegmentationMap(os.path.join(slice_dir, im_file))
             for im_file in image_files]

    return smaps

def find_slice_links(slice_map1, slice_map2):
    matches = {}

    for cell_slice in slice_map1.cells.values():
        c = cell_slice.centroid
        candidate_slice = slice_map2.cell_at(c)
        if slice_from_same_cell(cell_slice, candidate_slice):
            #matches.append((cell_slice.ID, candidate_slice.ID))
            matches[cell_slice.ID] = candidate_slice.ID

    return matches

def slice_from_same_cell(slice1, slice2):

    if slice1 is None or slice2 is None:
        return False

    area1 = slice1.pixel_area
    area2 = slice2.pixel_area

    dist = slice1.centroid.dist(slice2.centroid)

    area_ratio = float(area1) / area2

    if 0.5 < area_ratio < 1.5 and dist < 20:
        return True
    else:
        return False
<commit_msg>Added docstrings to reconstructor.py.<commit_after>import re
import os

import numpy as np
from skimage.io import use_plugin, imread

from coords2d import Coords2D
#import regionmap

def sorted_nicely( l ):
    """ Sort the given iterable in the way that humans expect."""
    convert = lambda text: int(text) if text.isdigit() else text
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)

class ReconstructedCell(object):
    """Pseudo 3D cell from contiguous z-stacks."""

    def __init__(self, slice_dict):
        self.slice_dict = slice_dict

    def add_slice(self, layer, cellslice):
        """Add ROI from another z-stack."""
        self.slice_dict[layer] = cellslice

    @property
    def pixel_area(self):
        """Return the total cell area across all the z-stacks."""
        return sum(cellslice.pixel_area
                   for cellslice in self.slice_dict.values())

    @property
    def z_extent(self):
        """Return the number of z-stacks in the reconstructed cell."""
        return len(self.slice_dict.keys())

    def measure_total_intensity(self, idata):
        """Return the total intensity across all the z-stacks."""
        total_intensity = 0
        for sID, cellslice in self.slice_dict.items():
            #z_correction = 1 + (0.03 * sID)
            z_correction = 1
            total_intensity += z_correction * sum(idata[sID][cellslice.coord_list])
        return total_intensity

    def measure_mean_intensity(self, idata):
        """Return the mean intensity of the reconstructed cell."""
        return float(self.measure_total_intensity(idata)) / self.pixel_area

    def measure_quartile_intensity(self, idata):
        """Return the quartile intensity of the reconstructed cell."""
        all_measurements = [idata[sID][cellslice.coord_list]
                            for sID, cellslice in self.slice_dict.items()]
        as_single_array = np.concatenate(all_measurements)
        n_frac = len(as_single_array) / 2
        return np.mean(as_single_array[n_frac:])

    def measure_best_slice(self, idata):
        """Return the intensity from the most intense slice."""
        slice_arrays = [idata[sID][cellslice.coord_list]
                        for sID, cellslice in self.slice_dict.items()]
        slice_means = [np.mean(sa) for sa in slice_arrays]
        best_slice = max(slice_means)
        best_z = np.array(slice_means).argmax()
        return best_slice, best_z

    def __repr__(self):
        return "<ReconstructedCell: %s>" % self.slice_dict.__repr__()

    def simple_string_rep(self):
        """Return a simple string representation of the reconstructed cell."""
        return ",".join("%d:%d" % (sID, cellslice.ID)
                        for sID, cellslice
                        in self.slice_dict.items())

    @property
    def centroid(self):
        """Return the centroid of the reconstructed cell."""
        z = float(sum(self.slice_dict.keys())) / len(self.slice_dict)
        csum = sum([c.centroid for c in self.slice_dict.values()],
                   Coords2D(0, 0))
        x, y = map(float, csum / len(self.slice_dict))
        return x, y, z

class CellSlice(object):
    """Slice of a cell in the x, y plane."""

    def __init__(self, ID, coord_list):
        self.ID = ID
        self.coord_list = coord_list
        self.pixel_area = len(coord_list[0])
        self.x_coords = coord_list[0]
        self.y_coords = coord_list[1]

    @property
    def centroid(self):
        """Return the centroid of the cell slice."""
        return Coords2D(sum(self.x_coords),
                        sum(self.y_coords)) / self.pixel_area

    @property
    def summary(self):
        """Return summary string representation of the cell slice."""
        return "<ID: %d, pixel_area: %d, centroid: %s>" % (self.ID,
                                                           self.pixel_area,
                                                           self.centroid)

    def __repr__(self):
        return "<CellSlice, ID %d>" % self.ID

class SegmentationMap(object):
    """Container for the pseudo 3D reconstructed cells."""

    def __init__(self, image_file):
        use_plugin('freeimage')
        self.im_array = imread(image_file)
        # self.im_array = np.transpose(self.im_array)
        # cd = regionmap.load_image(image_file)
        # self.internal_cc = {cid: CellSlice(cid, clist)
        #                     for cid, clist in cd.items()}
        self.internal_cc = None

    @property
    def cells(self):
        """Return the dictionary of cell slices."""
        #return self.internal_cc
        if self.internal_cc is not None:
            return self.internal_cc
        else:
            self.internal_cc = cell_dict_from_image_array(self.im_array)
            return self.internal_cc

    def cell_at(self, position):
        """Return the cell at position (x, y)."""
        x, y = position

        ID = self.im_array[x, y]
        if ID == 0:
            return None

        return self.cells[self.im_array[x, y]]

    @property
    def all_ids(self):
        """Return all cell ids."""
        with_zero = list(np.unique(self.im_array))
        with_zero.remove(0)
        return with_zero

    def coord_list(self, cID):
        """Return list of coordinates for cell (ID)."""
        return np.where(self.im_array == cID)

class Reconstruction(object):
    """Pseudo 3D reconstruction."""

    def __init__(self, smaps, start=0):
        self.smaps = smaps
        self.rcells = []
        self.lut = {}
        z = start
        for ID in smaps[z].all_ids:
            rcell = ReconstructedCell({z: self.smaps[z].cells[ID]})
            self.rcells.append(rcell)
            self.lut[(z, ID)] = rcell

    def extend(self, level):
        """Add another z-stack to the reconstruction."""
        z = level
        matches = find_slice_links(self.smaps[z], self.smaps[z+1])

        for f, t in matches.iteritems():
            try:
                rcell = self.lut[(z, f)]
                rcell.add_slice(z+1, self.smaps[z+1].cells[t])
            except KeyError:
                rcell = ReconstructedCell({z+1: self.smaps[z+1].cells[t]})
                self.rcells.append(rcell)

            self.lut[(z+1, t)] = rcell

    def cells_larger_then(self, zext):
        """Return cells which have more than zext z-stacks."""
        return [rcell for rcell in self.rcells
                if rcell.z_extent >= zext]

    def save_to_file(self, filename):
        """Save the reconstructed cells to file using simple cell representation."""
        with open(filename, "w") as f:
            f.write('\n'.join([rcell.simple_string_rep()
                               for rcell in self.rcells]))

def cell_dict_from_image_array(i_array):
    """Return dictionary of cell slices from an array of images."""
    cd = {cid: CellSlice(cid, np.where(i_array == cid))
          for cid in np.unique(i_array)}
    del(cd[0])
    return cd

def load_segmentation_maps(slice_dir):
    """Return list of segmentation maps from a directory of segmentations."""
    image_files = os.listdir(slice_dir)
    image_files = sorted_nicely(image_files)
    smaps = [SegmentationMap(os.path.join(slice_dir, im_file))
             for im_file in image_files]
    return smaps

def find_slice_links(slice_map1, slice_map2):
    """Return dictionary of matched cells between two slice maps."""
    matches = {}
    for cell_slice in slice_map1.cells.values():
        c = cell_slice.centroid
        candidate_slice = slice_map2.cell_at(c)
        if slice_from_same_cell(cell_slice, candidate_slice):
            matches[cell_slice.ID] = candidate_slice.ID
    return matches

def slice_from_same_cell(slice1, slice2):
    """Whether or not two cell slices are from the same cell."""
    if slice1 is None or slice2 is None:
        return False

    area1 = slice1.pixel_area
    area2 = slice2.pixel_area

    dist = slice1.centroid.dist(slice2.centroid)
    area_ratio = float(area1) / area2

    if 0.5 < area_ratio < 1.5 and dist < 20:
        return True
    else:
        return False
<|endoftext|>


<commit_before>#!/usr/bin/python

import logging
from bridge import Bridge, Service

bridge = Bridge(log_level=logging.DEBUG)

class MsgHandler(Service):
    def msg(self, name, message):
        print(name + ': ' + message)

def start_client():
    bridge.get_service('chatserver', start_chat)

def start_chat(chat):
    print('Got chat service = ', chat)
    chat.join('lobby', MsgHandler(), get_channel)

lobby = None

def get_channel(channel):
    lobby = channel

def send(message):
    lobby.msg('Foo', message)

bridge.ready(start_client)
<commit_msg>updated client chat example<commit_after>#!/usr/bin/python

import logging
from bridge import Bridge, Service

bridge = Bridge(log_level=logging.DEBUG)

class MsgHandler(Service):
    def msg(self, name, message):
        print(name + ': ' + message)

class LobbyHandler(object):
    def __init__(self, name):
        self.name = name
        self.lobby = None

    def __call__(self, channel):
        self.lobby = channel

    def send(self, message):
        self.lobby.msg(self.name, message)

lobby = LobbyHandler('Vedant')

def start_client():
    bridge.get_service('chatserver', start_chat)

def start_chat(chat):
    print('Got chat service.')
    chat.join('lobby', MsgHandler, lobby)

bridge.ready(start_client)
<|endoftext|>


<commit_before>import patrol
import requests
import subprocess

SAFE_LIST = ["GB"]
UNSAFE_IP_LIST = []

ip_list = patrol.main()
process_list = requests.get("https://api.oceanlord.me/ip/info", json={"addresses": ip_list})

if process_list.status_code != 200:
    print(process_list.status_code, process_list.text)
    exit()

for ip_address in list(process_list.json().keys()):
    if ip_address not in UNSAFE_IP_LIST:
        if process_list.json()[ip_address]["country_code"] not in SAFE_LIST and not process_list.json()[ip_address]["private"]:
            UNSAFE_IP_LIST.append(ip_address)

static_info = ["\n", "####### This section was created by a program to block irritating IP's", "\n"]
hosts_file_contents = []

line_counter = 0
info_line = 0
hosts_file_storage = []
BACKED_UP_IP_LIST = []

with open("/etc/hosts.deny") as file:
    for line in file:
        if static_info[1] in line:
            info_line = line_counter
            break
        else:
            hosts_file_storage.append(line)
            line_counter += 1
    line_counter = 0
    for line in file:
        if line_counter > info_line:
            temp_line = line.strip()
            if temp_line != "":
                BACKED_UP_IP_LIST.append(temp_line)
        else:
            line_counter += 1

with open("/etc/hosts.deny", "w") as file:
    for line in hosts_file_storage:
        file.write(line)
    for line in static_info:
        file.write(line)
    for ip in UNSAFE_IP_LIST:
        file.write("sshd: {}\n".format(ip))
    for ip in BACKED_UP_IP_LIST:
        file.write("{}\n".format(ip))

subprocess.call("sudo service sshd restart", shell=True)
<commit_msg>Make hosts.deny list unqiue<commit_after>import patrol
import requests
import subprocess
import re

SAFE_LIST = ["GB"]
UNSAFE_IP_LIST = []
re_luma = re.compile(r'(?P<address>([a-f\d]{1,4}[\.:]){1,7}[a-f\d]{1,4})\sport')

ip_list = patrol.main()
process_list = requests.get("https://api.oceanlord.me/ip/info", json={"addresses": ip_list})

if process_list.status_code != 200:
    print(process_list.status_code, process_list.text)
    exit()

for ip_address in list(process_list.json().keys()):
    if ip_address not in UNSAFE_IP_LIST:
        if process_list.json()[ip_address]["country_code"] not in SAFE_LIST and not process_list.json()[ip_address]["private"]:
            UNSAFE_IP_LIST.append(ip_address)

static_info = ["\n", "####### This section was created by a program to block irritating IP's", "\n"]
hosts_file_contents = []

line_counter = 0
info_line = 0
hosts_file_storage = []
BACKED_UP_IP_LIST = []

with open("/etc/hosts.deny") as file:
    for line in file:
        if static_info[1] in line:
            info_line = line_counter
            break
        else:
            hosts_file_storage.append(line)
            line_counter += 1
    line_counter = 0
    for line in file:
        if line_counter > info_line:
            temp_line = line.strip()
            if temp_line != "":
                ip = re_luma.search(temp_line)
                BACKED_UP_IP_LIST.append(ip.group("addresses"))
        else:
            line_counter += 1

SET_IP_LIST = set(BACKED_UP_IP_LIST + UNSAFE_IP_LIST)

with open("/etc/hosts.deny", "w") as file:
    for line in hosts_file_storage:
        file.write(line)
    for line in static_info:
        file.write(line)
    for ip in SET_IP_LIST:
        file.write("sshd: {}\n".format(ip))

subprocess.call("sudo service sshd restart", shell=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import os

AUTHOR = u'The EuroSciPy team <info@euroscipy.org>'
SITENAME = u'EuroSciPy'
SITEURL = ''

TIMEZONE = 'US/Central'
DEFAULT_LANG = u'en'

# Set the article URL
ARTICLE_URL = 'news/{date:%Y}/{date:%m}/{date:%d}/{slug}/'
ARTICLE_SAVE_AS = 'news/{date:%Y}/{date:%m}/{date:%d}/{slug}/index.html'

# Title menu options
DISPLAY_PAGES_ON_MENU = False
MENUITEMS = [('Euroscipy 2017', 'index.html'),  # i.e. home
             ('About', 'about.html'),  # contact info and who makes up the org and committees
             ('Venue', 'venue.html'),
             ('Program', 'program.html'),
             #('Sponsors', 'sponsors.html'),
             ('Code of Conduct', 'coc.html'),
             ]

NEWEST_FIRST_ARCHIVES = False

DIRECT_TEMPLATES = ('index', )
 # Theme
THEME_DIR = os.path.join(os.getcwd(), "theme")
THEME_NAME = "tuxlite_zf"
THEME = os.path.join(THEME_DIR, THEME_NAME)
RECENT_ARTICLES_COUNT = 3

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

# Blogroll
# LINKS =  (('Pelican', 'http://getpelican.com/'),
#           ('Python.org', 'http://python.org/'),
#           ('Jinja2', 'http://jinja.pocoo.org/'),
#           ('You can modify those links in your config file', '#'),)

# Social widget
#SOCIAL = (('You can add links in your config file', '#'),
#          ('Another social link', '#'),)

DEFAULT_PAGINATION = 5

STATIC_PATHS = ['static2017', 'images', 'pdf', 'CNAME']

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True
<commit_msg>Add sponsor menue link<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import os

AUTHOR = u'The EuroSciPy team <info@euroscipy.org>'
SITENAME = u'EuroSciPy'
SITEURL = ''

TIMEZONE = 'US/Central'
DEFAULT_LANG = u'en'

# Set the article URL
ARTICLE_URL = 'news/{date:%Y}/{date:%m}/{date:%d}/{slug}/'
ARTICLE_SAVE_AS = 'news/{date:%Y}/{date:%m}/{date:%d}/{slug}/index.html'

# Title menu options
DISPLAY_PAGES_ON_MENU = False
MENUITEMS = [('Euroscipy 2017', 'index.html'),  # i.e. home
             ('About', 'about.html'),  # contact info and who makes up the org and committees
             ('Venue', 'venue.html'),
             ('Program', 'program.html'),
             ('Sponsors', 'sponsors.html'),
             ('Code of Conduct', 'coc.html'),
             ]

NEWEST_FIRST_ARCHIVES = False

DIRECT_TEMPLATES = ('index', )
 # Theme
THEME_DIR = os.path.join(os.getcwd(), "theme")
THEME_NAME = "tuxlite_zf"
THEME = os.path.join(THEME_DIR, THEME_NAME)
RECENT_ARTICLES_COUNT = 3

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

# Blogroll
# LINKS =  (('Pelican', 'http://getpelican.com/'),
#           ('Python.org', 'http://python.org/'),
#           ('Jinja2', 'http://jinja.pocoo.org/'),
#           ('You can modify those links in your config file', '#'),)

# Social widget
#SOCIAL = (('You can add links in your config file', '#'),
#          ('Another social link', '#'),)

DEFAULT_PAGINATION = 5

STATIC_PATHS = ['static2017', 'images', 'pdf', 'CNAME']

# Uncomment following line if you want document-relative URLs when developing
RELATIVE_URLS = True
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Edwin Khoo'
SITENAME = 'Edwin Khoo'
SITEURL = ''

PATH = 'content'

TIMEZONE = 'America/New_York'

DEFAULT_LANG = 'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Links
LINKS = (('Bazant Research Group', 'https://bazantgroup.mit.edu'),)

# Social widget
SOCIAL = (('GitHub', 'https://github.com/edwinksl'),
          ('Bitbucket', 'https://bitbucket.org/edwinksl'),
          ('Facebook', 'https://www.facebook.com/edwinksl'),
          ('Twitter', 'https://twitter.com/edwinksl'),
          ('LinkedIn', 'https://www.linkedin.com/in/edwinksl'),
          ('Quora', 'https://www.quora.com/profile/Edwin-Khoo'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True

THEME = '/home/edwinksl/Git/pelican-bootstrap3'
BOOTSTRAP_NAVBAR_INVERSE = True

CUSTOM_CSS = 'static/custom.css'

STATIC_PATHS = ['images', 'extras/CNAME', 'extras/custom.css']
EXTRA_PATH_METADATA = {'extras/CNAME': {'path': 'CNAME'},
                       'extras/custom.css': {'path': 'static/custom.css'}}

ARTICLE_PATHS = ['blog']
ARTICLE_URL = 'blog/{slug}.html'
ARTICLE_SAVE_AS = 'blog/{slug}.html'
INDEX_SAVE_AS = 'blog/index.html'
DISPLAY_PAGES_ON_MENU = False
DISPLAY_CATEGORIES_ON_MENU = False
MENUITEMS = [('Blog', '/blog'),
             ('Research', '/pages/research.html'),
             ('Resources', '/pages/resources.html')]
<commit_msg>Change theme<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Edwin Khoo'
SITENAME = 'Edwin Khoo'
SITEURL = ''

PATH = 'content'

TIMEZONE = 'America/New_York'

DEFAULT_LANG = 'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Links
LINKS = (('Bazant Research Group', 'https://bazantgroup.mit.edu'),)

# Social widget
SOCIAL = (('GitHub', 'https://github.com/edwinksl'),
          ('Bitbucket', 'https://bitbucket.org/edwinksl'),
          ('Facebook', 'https://www.facebook.com/edwinksl'),
          ('Twitter', 'https://twitter.com/edwinksl'),
          ('LinkedIn', 'https://www.linkedin.com/in/edwinksl'),
          ('Quora', 'https://www.quora.com/profile/Edwin-Khoo'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True

THEME = '/home/edwinksl/Git/pelican-bootstrap3'
BOOTSTRAP_THEME = 'flatly'

CUSTOM_CSS = 'static/custom.css'

STATIC_PATHS = ['images', 'extras/CNAME', 'extras/custom.css']
EXTRA_PATH_METADATA = {'extras/CNAME': {'path': 'CNAME'},
                       'extras/custom.css': {'path': 'static/custom.css'}}

ARTICLE_PATHS = ['blog']
ARTICLE_URL = 'blog/{slug}.html'
ARTICLE_SAVE_AS = 'blog/{slug}.html'
INDEX_SAVE_AS = 'blog/index.html'
DISPLAY_PAGES_ON_MENU = False
DISPLAY_CATEGORIES_ON_MENU = False
MENUITEMS = [('Blog', '/blog'),
             ('Research', '/pages/research.html'),
             ('Resources', '/pages/resources.html')]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'nokbar'
SITENAME = u'voltaireMC Blog'
SITEURL = ''

TIMEZONE = 'Europe/Paris'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

# Blogroll
LINKS =  (('Pelican', 'http://getpelican.com/'),
          ('Python.org', 'http://python.org/'),
          ('Jinja2', 'http://jinja.pocoo.org/'),
          ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<commit_msg>update pelican conf<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = u'nokbar'
SITENAME = u'voltaireMC Blog'
SITEURL = 'https://mc.voltaire.sh/blog'

TIMEZONE = 'Etc/UTC'

DEFAULT_LANG = u'en'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None

# Blogroll
LINKS =  (('voltaireMC', 'https://mc.voltaire.sh/'))

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Will Wolf'
SITENAME = 'will wolf'
SITEURL = ''

PATH = 'content'

TIMEZONE = 'America/New_York'

DEFAULT_LANG = 'en'

THEME = "/Users/willwolf/repos/pelican-themes/aboutwilson"
STATIC_PATHS = ['figures', 'images']

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<commit_msg>enable render_math plugin<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

import os
HOME = os.getenv('HOME')

AUTHOR = 'Will Wolf'
SITENAME = 'will wolf'
SITEURL = ''

PATH = 'content'

TIMEZONE = 'America/New_York'

DEFAULT_LANG = 'en'

THEME = os.path.join(HOME, 'repos/pelican-themes/aboutwilson')
STATIC_PATHS = ['figures', 'images']

PLUGIN_PATHS = [os.path.join(HOME, 'repos/pelican-plugins')]
PLUGINS = ['render_math']

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
<|endoftext|>


<commit_before>"""
This module provides JVMs to run programs.
"""

import os
import shlex

from penchy.maven import get_classpath
from penchy.util import extract_classpath

class JVM(object):
    """
    This class represents a JVM.
    """

    def __init__(self, path, options=""):
        """
        :param path: path to jvm executable relative to basepath
        :param options: string of options that will be passed to jvm
        :type options: string
        """

        self.basepath = '/'
        self.path = path
        # keep user_options for user messages around
        self.user_options = options

        self._options = shlex.split(options)
        self._classpath = extract_classpath(self.options)
        self._tool_options = []
        self._payload_options = []

    def configure(self, *args):
        """
        Configure jvm options that allows `args` to run

        :param *args: :class:`Tool` or :class:`Program` instances that should be
                      run.
        """
        #TODO
        pass

    def run(self):
        """
        Run the jvm with the current configuration.
        """
        #TODO
        pass

    @property
    def cmdline(self):
        """
        The command line suitable for `subprocess.Popen` based on the current
        configuration.
        """
        executable = os.path.join(self.basepath, self.path)
        cp = ['-classpath', self._classpath + ":" get_classpath()]
        return executable + self._options + cp

class WrappedJVM(JVM):
    """
    This class is an abstract base class for a JVM that is wrapped by another
    Program.

    Inheriting classes must expose this attributes:

      - ``out``: dictionary that maps logical output names to paths of output
        files
      - ``exports``: set of logical outputs (valid keys for ``out``)
    """
    def __init__(self):
        """
        Inheriting classes must:

          - have compatible arguments with JVM.__init__
          - call JVM.__init__
        """
        raise NotImplementedError("must be implemented")

    def run(self):
        """
        Run with wrapping.
        """
        raise NotImplementedError("must be implemented")

class ValgrindJVM(WrappedJVM):
    """
    This class represents a JVM which is called by valgrind.
    """
    #TODO
    pass
<commit_msg>jvms: Remove `path` and `user` from interface.<commit_after>"""
This module provides JVMs to run programs.
"""

import os
import shlex

from penchy.maven import get_classpath
from penchy.util import extract_classpath

class JVM(object):
    """
    This class represents a JVM.
    """

    def __init__(self, path, options=""):
        """
        :param path: path to jvm executable relative to basepath
        :param options: string of options that will be passed to jvm
        :type options: string
        """

        self.basepath = '/'
        self._path = path
        # keep user_options for user messages around
        self._user_options = options

        self._options = shlex.split(options)
        self._classpath = extract_classpath(self.options)
        self._tool_options = []
        self._payload_options = []

    def configure(self, *args):
        """
        Configure jvm options that allows `args` to run

        :param *args: :class:`Tool` or :class:`Program` instances that should be
                      run.
        """
        #TODO
        pass

    def run(self):
        """
        Run the jvm with the current configuration.
        """
        #TODO
        pass

    @property
    def cmdline(self):
        """
        The command line suitable for `subprocess.Popen` based on the current
        configuration.
        """
        executable = os.path.join(self.basepath, self._path)
        cp = ['-classpath', self._classpath + ":" get_classpath()]
        return executable + self._options + cp

class WrappedJVM(JVM):
    """
    This class is an abstract base class for a JVM that is wrapped by another
    Program.

    Inheriting classes must expose this attributes:

      - ``out``: dictionary that maps logical output names to paths of output
        files
      - ``exports``: set of logical outputs (valid keys for ``out``)
    """
    def __init__(self):
        """
        Inheriting classes must:

          - have compatible arguments with JVM.__init__
          - call JVM.__init__
        """
        raise NotImplementedError("must be implemented")

    def run(self):
        """
        Run with wrapping.
        """
        raise NotImplementedError("must be implemented")

class ValgrindJVM(WrappedJVM):
    """
    This class represents a JVM which is called by valgrind.
    """
    #TODO
    pass
<|endoftext|>