<commit_before>import math
import numpy as np
import random
import struct
from numba import jit
import time

from ibex.utilities.constants import *
from ibex.utilities import dataIO
from ibex.transforms import seg2seg, seg2gold
from ibex.cnns.skeleton.util import SkeletonCandidate
from ibex.data_structures import unionfind
from ibex.evaluation import comparestacks

# save the candidate files for the CNN
def SaveCandidates(output_filename, candidates):
    random.shuffle(candidates)
    ncandidates = len(candidates)

    # write all candidates to the file
    with open(output_filename, 'wb') as fd:
        fd.write(struct.pack('i', len(candidates)))

        # add every candidate to the binary file
        for candidate in candidates:
            # get the labels for this candidate
            label_one = candidate.labels[0]
            label_two = candidate.labels[1]

            # get the location of this candidate
            position = candidate.location
            ground_truth = candidate.ground_truth

            # write this candidate to the evaluation candidate list
            fd.write(struct.pack('qqqqq?', label_one, label_two, position[IB_Z], position[IB_Y], position[IB_X], ground_truth))

@jit(nopython=True)
def FindNeighboringCandidates(segmentation, centroid, candidates, maximum_distance, network_distance, world_res):
    # useful variables
    zres, yres, xres = segmentation.shape
    max_label = np.amax(segmentation) + 1

    # get the radii and label for this centroid
    radii = np.int64((maximum_distance / world_res[IB_Z], maximum_distance / world_res[IB_Y], maximum_distance / world_res[IB_X]))
    network_radii = np.int64((network_distance / world_res[IB_Z], network_distance / world_res[IB_Y], network_distance / world_res[IB_X]))
    label = segmentation[centroid[IB_Z],centroid[IB_Y],centroid[IB_X]]

    # iterate through all the pixels close to the centroid
    for iz in range(centroid[IB_Z]-radii[IB_Z], centroid[IB_Z]+radii[IB_Z]+1):
        if iz < 0 or iz > zres - 1: continue
        for iy in range(centroid[IB_Y]-radii[IB_Y], centroid[IB_Y]+radii[IB_Y]+1):
            if iy < 0 or iy > yres - 1: continue
            for ix in range(centroid[IB_X]-radii[IB_X], centroid[IB_X]+radii[IB_X]+1):
                if ix < 0 or ix > xres - 1: continue
                # skip extracellular and locations with the same label
                if not segmentation[iz,iy,ix]: continue
                if segmentation[iz,iy,ix] == label: continue

                # get the distance from the centroid
                distance = math.sqrt((world_res[IB_Z] * (centroid[IB_Z] - iz)) * (world_res[IB_Z] * (centroid[IB_Z] - iz))  + (world_res[IB_Y] * (centroid[IB_Y] - iy)) * (world_res[IB_Y] * (centroid[IB_Y] - iy)) + (world_res[IB_X] * (centroid[IB_X] - ix)) * (world_res[IB_X] * (centroid[IB_X] - ix)))
                if distance > maximum_distance: continue

                # is there already a closer location
                neighbor_label = segmentation[iz,iy,ix]
                candidates.add(neighbor_label)

# generate features for this prefix
def GenerateFeatures(prefix, threshold, maximum_distance, network_distance, endpoint_distance, topology):
    start_time = time.time()

    # read in the relevant information
    segmentation = dataIO.ReadSegmentationData(prefix)
    gold = dataIO.ReadGoldData(prefix)
    assert (segmentation.shape == gold.shape)
    zres, yres, xres = segmentation.shape

    # remove small connceted components
    thresholded_segmentation = seg2seg.RemoveSmallConnectedComponents(segmentation, threshold=threshold).astype(np.int64)
    max_label = np.amax(segmentation) + 1

    # get the grid size and the world resolution
    grid_size = segmentation.shape
    world_res = dataIO.Resolution(prefix)

    # get the radius in grid coordinates
    radii = np.int64((maximum_distance / world_res[IB_Z], maximum_distance / world_res[IB_Y], maximum_distance / world_res[IB_X]))
    network_radii = np.int64((network_distance / world_res[IB_Z], network_distance / world_res[IB_Y], network_distance / world_res[IB_X]))

    # get all of the skeletons
    if topology: skeletons, endpoints = dataIO.ReadTopologySkeletons(prefix, thresholded_segmentation)
    else: skeletons, _, endpoints = dataIO.ReadSWCSkeletons(prefix, thresholded_segmentation)

    # get the set of all considered pairs
    endpoint_candidates = [set() for _ in range(len(endpoints))]
    for ie, endpoint in enumerate(endpoints):
        # extract the region around this endpoint
        label = endpoint.label
        centroid = endpoint.GridPoint()

        # find the candidates near this endpoint
        candidates = set()
        candidates.add(0)
        FindNeighboringCandidates(thresholded_segmentation, centroid, candidates, maximum_distance, network_distance, world_res)

        for candidate in candidates:
            # skip extracellular
            if not candidate: continue
            endpoint_candidates[ie].add(candidate)

    # get a mapping from the labels to indices in skeletons and endpoints
    label_to_index = [-1 for _ in range(max_label)]
    for ie, skeleton in enumerate(skeletons):
        label_to_index[skeleton.label] = ie

    # begin pruning the candidates based on the endpoints
    endpoint_pairs = {}

    # find the smallest pair between endpoints
    smallest_distances = {}
    midpoints = {}

    #smallest_distances = (endpoint_distance + 1) * np.ones((max_label, max_label), dtype=np.float32)
    #midpoints = np.zeros((max_label, max_label, 3), dtype=np.int32)

    for ie, endpoint in enumerate(endpoints):
        label = endpoint.label
        for neighbor_label in endpoint_candidates[ie]:
            smallest_distances[(label,neighbor_label)] = endpoint_distance + 1
            smallest_distances[(neighbor_label,label)] = endpoint_distance + 1

    for ie, endpoint in enumerate(endpoints):
        # get the endpoint location
        label = endpoint.label

        # go through all currently considered endpoints
        for neighbor_label in endpoint_candidates[ie]:
            for neighbor_endpoint in skeletons[label_to_index[neighbor_label]].endpoints:
                # get the distance
                deltas = endpoint.WorldPoint(world_res) - neighbor_endpoint.WorldPoint(world_res)
                distance = math.sqrt(deltas[IB_Z] * deltas[IB_Z] + deltas[IB_Y] * deltas[IB_Y] + deltas[IB_X] * deltas[IB_X])

                if distance < smallest_distances[(label, neighbor_label)]:
                    midpoint = (endpoint.GridPoint() + neighbor_endpoint.GridPoint()) / 2

                    # make sure the bounding box fits
                    valid_location = True
                    for dim in range(NDIMS):
                        if midpoint[dim]-network_radii[dim] < 0: valid_location = False
                        if midpoint[dim]+network_radii[dim] > grid_size[dim]: valid_location = False
                    if not valid_location: continue

                    smallest_distances[(label,neighbor_label)] = distance
                    smallest_distances[(neighbor_label,label)] = distance

                    # add to the dictionary
                    endpoint_pairs[(label, neighbor_label)] = (endpoint, neighbor_endpoint)
                    endpoint_pairs[(neighbor_label, label)] = (neighbor_endpoint, endpoint)

                    midpoints[(label,neighbor_label)] = midpoint
                    midpoints[(neighbor_label,label)] = midpoint

    # create list of candidates
    positive_candidates = []
    negative_candidates = []
    undetermined_candidates = []

    for ie, match in enumerate(endpoint_pairs):
        if (ie % 100 == 0): print '{}/{}'.format(ie, len(endpoint_pairs))
        endpoint_one = endpoint_pairs[match][0]
        endpoint_two = endpoint_pairs[match][1]

        label_one = endpoint_one.label
        label_two = endpoint_two.label

        if (label_one > label_two): continue

        # extract a bounding box around this midpoint
        midz, midy, midx = midpoints[(label_one,label_two)]

        extracted_segmentation = segmentation[midz-network_radii[IB_Z]:midz+network_radii[IB_Z],midy-network_radii[IB_Y]:midy+network_radii[IB_Y],midx-network_radii[IB_X]:midx+network_radii[IB_X]]
        extracted_gold = gold[midz-network_radii[IB_Z]:midz+network_radii[IB_Z],midy-network_radii[IB_Y]:midy+network_radii[IB_Y],midx-network_radii[IB_X]:midx+network_radii[IB_X]]

        extracted_seg2gold_mapping = seg2gold.Mapping(extracted_segmentation, extracted_gold, match_threshold=0.70, nonzero_threshold=0.40)
        gold_one = extracted_seg2gold_mapping[label_one]
        gold_two = extracted_seg2gold_mapping[label_two]

        ground_truth = (gold_one == gold_two)

        candidate = SkeletonCandidate((label_one, label_two), midpoints[(label_one,label_two)], ground_truth)

        if not extracted_seg2gold_mapping[label_one] or not extracted_seg2gold_mapping[label_two]: undetermined_candidates.append(candidate)
        elif ground_truth: positive_candidates.append(candidate)
        else: negative_candidates.append(candidate)

    # save positive and negative candidates separately
    positive_filename = 'features/skeleton/{}-{}-{}nm-{}nm-{}nm-positive.candidates'.format(prefix, threshold, maximum_distance, endpoint_distance, network_distance)
    negative_filename = 'features/skeleton/{}-{}-{}nm-{}nm-{}nm-negative.candidates'.format(prefix, threshold, maximum_distance, endpoint_distance, network_distance)
    undetermined_filename = 'features/skeleton/{}-{}-{}nm-{}nm-{}nm-undetermined.candidates'.format(prefix, threshold, maximum_distance, endpoint_distance, network_distance)

    SaveCandidates(positive_filename, positive_candidates)
    SaveCandidates(negative_filename, negative_candidates)
    SaveCandidates(undetermined_filename, undetermined_candidates)

    print 'Positive candidates: {}'.format(len(positive_candidates))
    print 'Negative candidates: {}'.format(len(negative_candidates))
    print 'Undetermined candidates: {}'.format(len(undetermined_candidates))

    # perform some tests to see how well this method can do
    max_value = np.amax(segmentation) + 1
    union_find = [unionfind.UnionFindElement(iv) for iv in range(max_value)]

    # iterate over all collapsed edges
    for candidate in positive_candidates:
        label_one, label_two = candidate.labels
        unionfind.Union(union_find[label_one], union_find[label_two])

    # create a mapping for the labels
    mapping = np.zeros(max_value, dtype=np.int64)
    for iv in range(max_value):
        mapping[iv] = unionfind.Find(union_find[iv]).label
    segmentation = seg2seg.MapLabels(segmentation, mapping)

    comparestacks.CremiEvaluate(segmentation, gold, dilate_ground_truth=1, mask_ground_truth=True, filtersize=0)

<commit_msg>generate_features: updates from Odyssey<commit_after>import math
import numpy as np
import random
import struct
from numba import jit
import time

from ibex.utilities.constants import *
from ibex.utilities import dataIO
from ibex.transforms import seg2seg, seg2gold
from ibex.cnns.skeleton.util import SkeletonCandidate
from ibex.data_structures import unionfind
from ibex.evaluation import comparestacks

# save the candidate files for the CNN
def SaveCandidates(output_filename, candidates):
    random.shuffle(candidates)
    ncandidates = len(candidates)

    # write all candidates to the file
    with open(output_filename, 'wb') as fd:
        fd.write(struct.pack('i', len(candidates)))

        # add every candidate to the binary file
        for candidate in candidates:
            # get the labels for this candidate
            label_one = candidate.labels[0]
            label_two = candidate.labels[1]

            # get the location of this candidate
            position = candidate.location
            ground_truth = candidate.ground_truth

            # write this candidate to the evaluation candidate list
            fd.write(struct.pack('qqqqq?', label_one, label_two, position[IB_Z], position[IB_Y], position[IB_X], ground_truth))

@jit(nopython=True)
def FindNeighboringCandidates(segmentation, centroid, candidates, maximum_distance, network_distance, world_res):
    # useful variables
    zres, yres, xres = segmentation.shape
    max_label = np.amax(segmentation) + 1

    # get the radii and label for this centroid
    radii = np.int64((maximum_distance / world_res[IB_Z], maximum_distance / world_res[IB_Y], maximum_distance / world_res[IB_X]))
    network_radii = np.int64((network_distance / world_res[IB_Z], network_distance / world_res[IB_Y], network_distance / world_res[IB_X]))
    label = segmentation[centroid[IB_Z],centroid[IB_Y],centroid[IB_X]]

    # iterate through all the pixels close to the centroid
    for iz in range(centroid[IB_Z]-radii[IB_Z], centroid[IB_Z]+radii[IB_Z]+1):
        if iz < 0 or iz > zres - 1: continue
        for iy in range(centroid[IB_Y]-radii[IB_Y], centroid[IB_Y]+radii[IB_Y]+1):
            if iy < 0 or iy > yres - 1: continue
            for ix in range(centroid[IB_X]-radii[IB_X], centroid[IB_X]+radii[IB_X]+1):
                if ix < 0 or ix > xres - 1: continue
                # skip extracellular and locations with the same label
                if not segmentation[iz,iy,ix]: continue
                if segmentation[iz,iy,ix] == label: continue

                # get the distance from the centroid
                distance = math.sqrt((world_res[IB_Z] * (centroid[IB_Z] - iz)) * (world_res[IB_Z] * (centroid[IB_Z] - iz))  + (world_res[IB_Y] * (centroid[IB_Y] - iy)) * (world_res[IB_Y] * (centroid[IB_Y] - iy)) + (world_res[IB_X] * (centroid[IB_X] - ix)) * (world_res[IB_X] * (centroid[IB_X] - ix)))
                if distance > maximum_distance: continue

                # is there already a closer location
                neighbor_label = segmentation[iz,iy,ix]
                candidates.add(neighbor_label)

# generate features for this prefix
def GenerateFeatures(prefix, threshold, maximum_distance, network_distance, endpoint_distance, topology):
    start_time = time.time()

    # read in the relevant information
    segmentation = dataIO.ReadSegmentationData(prefix)
    gold = dataIO.ReadGoldData(prefix)
    assert (segmentation.shape == gold.shape)
    zres, yres, xres = segmentation.shape

    # remove small connceted components
    thresholded_segmentation = seg2seg.RemoveSmallConnectedComponents(segmentation, threshold=threshold).astype(np.int64)
    max_label = np.amax(segmentation) + 1

    # get the grid size and the world resolution
    grid_size = segmentation.shape
    world_res = dataIO.Resolution(prefix)

    # get the radius in grid coordinates
    radii = np.int64((maximum_distance / world_res[IB_Z], maximum_distance / world_res[IB_Y], maximum_distance / world_res[IB_X]))
    network_radii = np.int64((network_distance / world_res[IB_Z], network_distance / world_res[IB_Y], network_distance / world_res[IB_X]))

    # get all of the skeletons
    if topology: skeletons, endpoints = dataIO.ReadTopologySkeletons(prefix, thresholded_segmentation)
    else: skeletons, _, endpoints = dataIO.ReadSWCSkeletons(prefix, thresholded_segmentation)

    # get the set of all considered pairs
    endpoint_candidates = [set() for _ in range(len(endpoints))]
    for ie, endpoint in enumerate(endpoints):
        # extract the region around this endpoint
        label = endpoint.label
        centroid = endpoint.GridPoint()

        # find the candidates near this endpoint
        candidates = set()
        candidates.add(0)
        FindNeighboringCandidates(thresholded_segmentation, centroid, candidates, maximum_distance, network_distance, world_res)

        for candidate in candidates:
            # skip extracellular
            if not candidate: continue
            endpoint_candidates[ie].add(candidate)

    # get a mapping from the labels to indices in skeletons and endpoints
    label_to_index = [-1 for _ in range(max_label)]
    for ie, skeleton in enumerate(skeletons):
        label_to_index[skeleton.label] = ie

    # begin pruning the candidates based on the endpoints
    endpoint_pairs = {}

    # find the smallest pair between endpoints
    smallest_distances = {}
    midpoints = {}

    #smallest_distances = (endpoint_distance + 1) * np.ones((max_label, max_label), dtype=np.float32)
    #midpoints = np.zeros((max_label, max_label, 3), dtype=np.int32)

    for ie, endpoint in enumerate(endpoints):
        label = endpoint.label
        for neighbor_label in endpoint_candidates[ie]:
            smallest_distances[(label,neighbor_label)] = endpoint_distance + 1
            smallest_distances[(neighbor_label,label)] = endpoint_distance + 1

    for ie, endpoint in enumerate(endpoints):
        # get the endpoint location
        label = endpoint.label

        # go through all currently considered endpoints
        for neighbor_label in endpoint_candidates[ie]:
            for neighbor_endpoint in skeletons[label_to_index[neighbor_label]].endpoints:
                # get the distance
                deltas = endpoint.WorldPoint(world_res) - neighbor_endpoint.WorldPoint(world_res)
                distance = math.sqrt(deltas[IB_Z] * deltas[IB_Z] + deltas[IB_Y] * deltas[IB_Y] + deltas[IB_X] * deltas[IB_X])

                if distance < smallest_distances[(label, neighbor_label)]:
                    midpoint = (endpoint.GridPoint() + neighbor_endpoint.GridPoint()) / 2

                    # make sure the bounding box fits
                    valid_location = True
                    for dim in range(NDIMS):
                        if midpoint[dim]-network_radii[dim] < 0: valid_location = False
                        if midpoint[dim]+network_radii[dim] > grid_size[dim]: valid_location = False
                    if not valid_location: continue

                    smallest_distances[(label,neighbor_label)] = distance
                    smallest_distances[(neighbor_label,label)] = distance

                    # add to the dictionary
                    endpoint_pairs[(label, neighbor_label)] = (endpoint, neighbor_endpoint)
                    endpoint_pairs[(neighbor_label, label)] = (neighbor_endpoint, endpoint)

                    midpoints[(label,neighbor_label)] = midpoint
                    midpoints[(neighbor_label,label)] = midpoint

    # create list of candidates
    positive_candidates = []
    negative_candidates = []
    undetermined_candidates = []

    for ie, match in enumerate(endpoint_pairs):
        if (ie % 1 == 0): print '{}/{}'.format(ie, len(endpoint_pairs))
        endpoint_one = endpoint_pairs[match][0]
        endpoint_two = endpoint_pairs[match][1]

        label_one = endpoint_one.label
        label_two = endpoint_two.label

        if (label_one > label_two): continue

        # extract a bounding box around this midpoint
        midz, midy, midx = midpoints[(label_one,label_two)]

        extracted_segmentation = segmentation[midz-network_radii[IB_Z]:midz+network_radii[IB_Z],midy-network_radii[IB_Y]:midy+network_radii[IB_Y],midx-network_radii[IB_X]:midx+network_radii[IB_X]]
        extracted_gold = gold[midz-network_radii[IB_Z]:midz+network_radii[IB_Z],midy-network_radii[IB_Y]:midy+network_radii[IB_Y],midx-network_radii[IB_X]:midx+network_radii[IB_X]]

        extracted_seg2gold_mapping = seg2gold.Mapping(extracted_segmentation, extracted_gold, match_threshold=0.70, nonzero_threshold=0.40)
        gold_one = extracted_seg2gold_mapping[label_one]
        gold_two = extracted_seg2gold_mapping[label_two]

        ground_truth = (gold_one == gold_two)

        candidate = SkeletonCandidate((label_one, label_two), midpoints[(label_one,label_two)], ground_truth)

        if not extracted_seg2gold_mapping[label_one] or not extracted_seg2gold_mapping[label_two]: undetermined_candidates.append(candidate)
        elif ground_truth: positive_candidates.append(candidate)
        else: negative_candidates.append(candidate)

    # save positive and negative candidates separately
    positive_filename = 'features/skeleton/{}-{}-{}nm-{}nm-{}nm-positive.candidates'.format(prefix, threshold, maximum_distance, endpoint_distance, network_distance)
    negative_filename = 'features/skeleton/{}-{}-{}nm-{}nm-{}nm-negative.candidates'.format(prefix, threshold, maximum_distance, endpoint_distance, network_distance)
    undetermined_filename = 'features/skeleton/{}-{}-{}nm-{}nm-{}nm-undetermined.candidates'.format(prefix, threshold, maximum_distance, endpoint_distance, network_distance)

    SaveCandidates(positive_filename, positive_candidates)
    SaveCandidates(negative_filename, negative_candidates)
    SaveCandidates(undetermined_filename, undetermined_candidates)

    print 'Positive candidates: {}'.format(len(positive_candidates))
    print 'Negative candidates: {}'.format(len(negative_candidates))
    print 'Undetermined candidates: {}'.format(len(undetermined_candidates))

    # perform some tests to see how well this method can do
    max_value = np.amax(segmentation) + 1
    union_find = [unionfind.UnionFindElement(iv) for iv in range(max_value)]
    print 'A'
    # iterate over all collapsed edges
    for candidate in positive_candidates:
        label_one, label_two = candidate.labels
        unionfind.Union(union_find[label_one], union_find[label_two])
    print 'B'
    # create a mapping for the labels
    mapping = np.zeros(max_value, dtype=np.int64)
    for iv in range(max_value):
        mapping[iv] = unionfind.Find(union_find[iv]).label
    print 'C'
    segmentation = seg2seg.MapLabels(segmentation, mapping)
    print 'D'
    comparestacks.CremiEvaluate(segmentation, gold, dilate_ground_truth=1, mask_ground_truth=True, mask_segmentation=True, filtersize=0)
    print 'E'
<|endoftext|>


<commit_before>"""
ncplib
======

:term:`NCP` library for Python 3, developed by `CRFS`_.

.. image:: https://travis-ci.org/CRFS/python3-ncplib.svg?branch=master
    :target: https://travis-ci.org/CRFS/python3-ncplib

Features
--------

-   :doc:`client`.
-   :doc:`server`.
-   Asynchronous connections via :mod:`asyncio`.

Resources
---------

-   `Documentation`_ is on Read the Docs.
-   `Issue tracking`_ and `source code`_ is on GitHub.
-   `Continuous integration`_ is on Travis CI.

Usage
-----

.. toctree::
    :maxdepth: 1

    installation
    client
    server
    connection
    errors
    values

More information
----------------

.. toctree::
    :maxdepth: 1

    contributing
    glossary
    changelog

.. include:: /_include/links.rst
"""

__version__ = (2, 1, 0)

from ncplib.client import *  # noqa
from ncplib.connection import *  # noqa
from ncplib.errors import *  # noqa
from ncplib.helpers import *  # noqa
from ncplib.server import *  # noqa
from ncplib.values import *  # noqa
<commit_msg>Bumping version number<commit_after>"""
ncplib
======

:term:`NCP` library for Python 3, developed by `CRFS`_.

.. image:: https://travis-ci.org/CRFS/python3-ncplib.svg?branch=master
    :target: https://travis-ci.org/CRFS/python3-ncplib

Features
--------

-   :doc:`client`.
-   :doc:`server`.
-   Asynchronous connections via :mod:`asyncio`.

Resources
---------

-   `Documentation`_ is on Read the Docs.
-   `Issue tracking`_ and `source code`_ is on GitHub.
-   `Continuous integration`_ is on Travis CI.

Usage
-----

.. toctree::
    :maxdepth: 1

    installation
    client
    server
    connection
    errors
    values

More information
----------------

.. toctree::
    :maxdepth: 1

    contributing
    glossary
    changelog

.. include:: /_include/links.rst
"""

__version__ = (2, 2, 0)

from ncplib.client import *  # noqa
from ncplib.connection import *  # noqa
from ncplib.errors import *  # noqa
from ncplib.helpers import *  # noqa
from ncplib.server import *  # noqa
from ncplib.values import *  # noqa
<|endoftext|>


<commit_before>import argparse

from coalib.misc.i18n import _

default_arg_parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=__doc__
)

default_arg_parser.add_argument('TARGETS',
                                nargs='*',
                                help=_("Sections to be executed exclusively."))
default_arg_parser.add_argument('-f',
                                '--files',
                                nargs='+',
                                metavar='FILE',
                                help=_('Files that should be checked'))
default_arg_parser.add_argument('-b',
                                '--bears',
                                nargs='+',
                                metavar='NAME',
                                help=_('Names of bears to use'))
BEAR_DIRS_HELP = _('Additional directories where bears may lie')
default_arg_parser.add_argument('-d',
                                '--bear-dirs',
                                nargs='+',
                                metavar='DIR',
                                help=BEAR_DIRS_HELP)
LOG_TYPE_HELP = _("Type of logging (console or any filename)")
default_arg_parser.add_argument('-l',
                                '--log-type',
                                nargs=1,
                                metavar='ENUM',
                                help=LOG_TYPE_HELP)
LOG_LEVEL_HELP = _("Enum('ERR','WARN','DEBUG') to set level of log output")
default_arg_parser.add_argument('-L',
                                '--log-level',
                                nargs=1,
                                choices=['ERR', 'WARN', 'DEBUG'],
                                metavar='ENUM',
                                help=LOG_LEVEL_HELP)
OUTPUT_HELP = _('Type of output (console or any filename)')
default_arg_parser.add_argument('-o',
                                '--output',
                                nargs=1,
                                metavar='FILE',
                                help=OUTPUT_HELP)
CONFIG_HELP = _('Configuration file to be used, defaults to .coafile')
default_arg_parser.add_argument('-c',
                                '--config',
                                nargs=1,
                                metavar='FILE',
                                help=CONFIG_HELP)
SAVE_HELP = _('Filename of file to be saved to, defaults to config file')
default_arg_parser.add_argument('-s',
                                '--save',
                                nargs='?',
                                const=True,
                                metavar='FILE',
                                help=SAVE_HELP)
SETTINGS_HELP = _('Arbitrary settings in the form of section.key=value')
default_arg_parser.add_argument('-S',
                                '--settings',
                                nargs='+',
                                metavar='SETTING',
                                help=SETTINGS_HELP)
JOB_COUNT_HELP = _('Number of processes to be allowed to run at once')
default_arg_parser.add_argument('-j',
                                '--job-count',
                                nargs=1,
                                type=int,
                                metavar='INT',
                                help=JOB_COUNT_HELP)
APPLY_HELP = _("Enum('YES','NO','ASK') to set whether to apply changes")
default_arg_parser.add_argument('-a',
                                '--apply-changes',
                                nargs=1,
                                choices=['YES', 'NO', 'ASK'],
                                metavar='ENUM',
                                help=APPLY_HELP)
<commit_msg>DefaultArgParser: Make -s help more precise<commit_after>import argparse

from coalib.misc.i18n import _

default_arg_parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=__doc__
)

default_arg_parser.add_argument('TARGETS',
                                nargs='*',
                                help=_("Sections to be executed exclusively."))
default_arg_parser.add_argument('-f',
                                '--files',
                                nargs='+',
                                metavar='FILE',
                                help=_('Files that should be checked'))
default_arg_parser.add_argument('-b',
                                '--bears',
                                nargs='+',
                                metavar='NAME',
                                help=_('Names of bears to use'))
BEAR_DIRS_HELP = _('Additional directories where bears may lie')
default_arg_parser.add_argument('-d',
                                '--bear-dirs',
                                nargs='+',
                                metavar='DIR',
                                help=BEAR_DIRS_HELP)
LOG_TYPE_HELP = _("Type of logging (console or any filename)")
default_arg_parser.add_argument('-l',
                                '--log-type',
                                nargs=1,
                                metavar='ENUM',
                                help=LOG_TYPE_HELP)
LOG_LEVEL_HELP = _("Enum('ERR','WARN','DEBUG') to set level of log output")
default_arg_parser.add_argument('-L',
                                '--log-level',
                                nargs=1,
                                choices=['ERR', 'WARN', 'DEBUG'],
                                metavar='ENUM',
                                help=LOG_LEVEL_HELP)
OUTPUT_HELP = _('Type of output (console or any filename)')
default_arg_parser.add_argument('-o',
                                '--output',
                                nargs=1,
                                metavar='FILE',
                                help=OUTPUT_HELP)
CONFIG_HELP = _('Configuration file to be used, defaults to .coafile')
default_arg_parser.add_argument('-c',
                                '--config',
                                nargs=1,
                                metavar='FILE',
                                help=CONFIG_HELP)
SAVE_HELP = _('Filename of file to be saved to, if provided with no '
              'arguments, settings will be stored back to the config file')
default_arg_parser.add_argument('-s',
                                '--save',
                                nargs='?',
                                const=True,
                                metavar='FILE',
                                help=SAVE_HELP)
SETTINGS_HELP = _('Arbitrary settings in the form of section.key=value')
default_arg_parser.add_argument('-S',
                                '--settings',
                                nargs='+',
                                metavar='SETTING',
                                help=SETTINGS_HELP)
JOB_COUNT_HELP = _('Number of processes to be allowed to run at once')
default_arg_parser.add_argument('-j',
                                '--job-count',
                                nargs=1,
                                type=int,
                                metavar='INT',
                                help=JOB_COUNT_HELP)
APPLY_HELP = _("Enum('YES','NO','ASK') to set whether to apply changes")
default_arg_parser.add_argument('-a',
                                '--apply-changes',
                                nargs=1,
                                choices=['YES', 'NO', 'ASK'],
                                metavar='ENUM',
                                help=APPLY_HELP)
<|endoftext|>


<commit_before>import zipfile
import tempfile
import inspect
import urllib2
import threading

from Remote import Remote
from errors import *
import ndio.ramon as ramon

DEFAULT_HOSTNAME = "openconnecto.me"
DEFAULT_PROTOCOL = "http"
DEFAULT_EMAIL = ""

class Invariants:
    SCAN_STATISTIC = ss1 = "ss1"        # the Scan Statistic 1
    TRIANGLE_COUNT = tri = "tri"        # the Triangle Count
    CLUSTERING = cc = "cc"              # the Clustering Coefficient
    MAX_AVG_DEGREE = mad = "mad"        # the Maximum Average Degree
    LOCAL_DEGREE = deg = "deg"          # The Local Degree Count
    EIGENS = eig = "eig"                # top 100 or max avail eigvecs, eigvals

    ALL = [
        self.SCAN_STATISTIC,
        self.TRIANGLE_COUNT,
        self.CLUSTERING,
        self.MAX_AVG_DEGREE,
        self.LOCAL_DEGREE,
        self.EIGENS
    ]

class InputFormats:
    GRAPHML = graphml = "graphml"
    NCOL = ncol = "ncol"
    EDGELIST = edgelist = "edgelist"
    LGL = lgl = "lgl"
    PAJEK = pajek = "pajek"
    GRAPHDB = graphdb = "graphdb"
    NUMPY = numpy = "numpy"
    MAT = mat = "mat"

class m2g(Remote):

    SMALL = S = 's'
    BIG = B = 'b'

    def __init__(self, hostname=DEFAULT_HOSTNAME, protocol=DEFAULT_PROTOCOL,
                 email=DEFAULT_EMAIL):
        super(m2g, self).__init__(hostname, protocol)
        self.email = email

    def __repr__(self):
        """
        Returns a string representation that can be used to reproduce this
        instance. `eval(repr(this))` should return an identical copy.

        Arguments:
            None

        Returns:
            str: Representation of reproducible instance.
        """
        return "ndio.remote.m2g('{}', '{}', '{}')".format(
            self.hostname,
            self.protocol,
            self.email
        )

    def ping(self):
        return super(m2g, self).ping()

    def url(self, suffix=""):
        return super(m2g, self).url('/graph-services/' + suffix)

    def set_default_email(self, email):
        """
        Sets the default email to use for notifications if none is specified.
        Does only basic error-checking.

        Arguments:
            email (str): The email to notify for all server graph services.

        Returns:
            None

        Raises:
            ValueError: If an invalid email is supplied.
        """
        if "@" not in email:
            raise ValueError("Invalid email.")
        self.email = email

    def build_graph(self, project, site, subject, session, scan,
                    size, email=None, invariants=Invariants.ALL,
                    fiber_file=DEFAULT_FIBER_FILE, atlas_file=None,
                    use_threads=False, callback=None):
        """
        Builds a graph using the graph-services endpoint.

        Arguments:
            project (str): The project to use
            site (str): The site in question
            subject (str): The subject's identifier
            session (str): The session (per subject)
            scan (str): The scan identifier
            size (str): Whether to return a big or (m2g.BIG) small (m2g.SMALL)
                graph. For a better explanation of each, see m2g.io.
            email (str)*: An email to which to send the download link
            invariants (str[]: Invariants.ALL)*: An array of invariants to
                compute. You can use the m2g.Invariants class to construct this
                list, or simply pass m2g.Invariants.ALL to compute them all.
            fiber_file (str: DEFAULT_FIBER_FILE)*: A local filename of an
                MRI Studio .dat file
            atlas_file (str: None)*: A local atlas file, in NIFTI .nii format.
                If none is specified, the Desikan atlas is used by default.
            use_threads (bool: False)*: Whether to run the download in a Python
                thread. If set to True, the call to `build_graph` will end
                quickly, and the `callback` will be called with the returned
                status-code of the restful call as its only argument.
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads is False. Otherwise, None

        Raises:
            ValueError: When the supplied values are invalid (contain invalid
                characters, bad email address supplied, etc.)
            RemoteDataNotFoundError: When the data cannot be processed due to
                a server error.
        """

        if use_threads and callback is not None:
            if not hasattr(callback, '__call__'):
                raise ValueError("callback must be a function.")
            if len(inspect.getargspec(callback).args) != 1:
                raise ValueError("callback must take exactly 1 argument.")

        # Once we get here, we know the callback is
        if size not in [self.BIG, self.SMALL]:
            raise ValueError("size must be either m2g.BIG or m2g.SMALL.")

        if email is None:
            email = self.email

        url = "buildgraph/{}/{}/{}/{}/{}/{}/{}/{}/".format(
            project,
            site,
            subject,
            session,
            scan,
            size,
            email,
            "/".join(invariants)
        )

        if " " in url:
            raise ValueError("Arguments must not contain spaces.")

        if use_threads:
            # Run in the background.
            download_thread = threading.Thread(
                target=self._run_build_graph,
                args=[url, fiber_file, atlas_file, callback]
            )
            download_thread.start()

        else:
            # Run in the foreground.
            return self._run_build_graph(url, fiber_file, atlas_file)
        return

    def _run_build_graph(self, url,
                            fiber_file, atlas_file=None, callback=None):

        try:
            # Create a temporary file to store zip contents in memory
            tmpfile = tempfile.NamedTemporaryFile()
            zfile = zipfile.ZipFile(tmpfile.name, "w", allowZip64=True)
            zfile.write(file_file)
            if atlas_file:
                zfile.write(atlas_file)
                zfile.close()
                tmpfile.flush()
                tmpfile.seek(0)
        except:
            raise ValueError("Invalid atlas or fiber file. I don't have " +
                             "any more information than that, sorry!")

        try:
            req = urllib2.Request(self.url(url), tmpfile.read())
            response = urllib2.urlopen(req)

            if callback is not None:
                callback(response)
            else:
                return response.read()
        except:
            raise RemoteDataUploadError("Failed to upload data at " + url)

    def compute_invariants(self, graph_file, input_format,
                           invariants=Invariants.ALL, email=None,
                           use_threads=False, callback=None):
        """
        Compute invariants from an existing GraphML file using the remote
        m2g graph services.

        Arguments:
            graph_file (str): The filename of the graphml file
            input_format (str): One of m2g.GraphFormats
            invariants (str[]: Invariants.ALL)*: An array of m2g.Invariants to
                compute on the graph
            email (str)*: The email to notify upon completion
            use_threads (bool: True)*: Whether to use Python threads to run the
                computation in the background when waiting for the server to
                return the invariants
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads is False. Otherwise, None

        Raises:
            ValueError: If the graph file does not exist, or if there are
                issues with the passed arguments
            RemoteDataUploadError: If there is an issue packing the file
            RemoteError: If the server experiences difficulty computing invs
        """

        if email is None:
            email = self.email

        url = "/graphupload/{}/{}/{}/".format(
            email,
            input_format,
            "/".join(invariants)
        )

        if " " in url:
            raise ValueError("Arguments cannot have spaces in them.")

        if not (os.path.exists(graph_file)):
            raise ValueError("File {} does not exist.".format(graph_file))

        if use_threads:
            # Run in the background.
            upload_thread = threading.Thread(
                target=self._run_compute_invariants,
                args=[url, graph_file, callback]
            )
            upload_thread.start()

        else:
            # Run in the foreground.
            return self._run_compute_invariants(url, fiber_file)
        return

    def _run_compute_invariants(self, url, graph_file, callback=None):
        try:
            tmpfile = tempfile.NamedTemporaryFile()
            zfile = zipfile.ZipFile(tmpfile.name, "w", allowZip64=True)

            zfile.write(graph_file)
            zfile.close()

            tmpfile.flush()
            tmpfile.seek(0)
        except:
            raise ValueError("Unable to zip graph file for upload.")

        try:
            req = urllib2.Request(url, tmpfile.read())
            response = urllib2.urlopen(req)

            if callback is not None:
                callback(response.read())
            else:
                return response.read()
        except:
            raise RemoteDataUploadError("Failed to upload graph file. Try " +
                                        "troubleshooting with a ping()?")
<commit_msg>formatting<commit_after>import zipfile
import tempfile
import inspect
import urllib2
import threading

from Remote import Remote
from errors import *
import ndio.ramon as ramon

DEFAULT_HOSTNAME = "openconnecto.me"
DEFAULT_PROTOCOL = "http"
DEFAULT_EMAIL = ""

class Invariants:
    SCAN_STATISTIC = ss1 = "ss1"        # the Scan Statistic 1
    TRIANGLE_COUNT = tri = "tri"        # the Triangle Count
    CLUSTERING = cc = "cc"              # the Clustering Coefficient
    MAX_AVG_DEGREE = mad = "mad"        # the Maximum Average Degree
    LOCAL_DEGREE = deg = "deg"          # The Local Degree Count
    EIGENS = eig = "eig"                # top 100 or max avail eigvecs, eigvals

    ALL = [
        self.SCAN_STATISTIC,
        self.TRIANGLE_COUNT,
        self.CLUSTERING,
        self.MAX_AVG_DEGREE,
        self.LOCAL_DEGREE,
        self.EIGENS
    ]

class InputFormats:
    GRAPHML = graphml = "graphml"
    NCOL = ncol = "ncol"
    EDGELIST = edgelist = "edgelist"
    LGL = lgl = "lgl"
    PAJEK = pajek = "pajek"
    GRAPHDB = graphdb = "graphdb"
    NUMPY = numpy = "numpy"
    MAT = mat = "mat"

class m2g(Remote):

    SMALL = S = 's'
    BIG = B = 'b'

    def __init__(self, hostname=DEFAULT_HOSTNAME, protocol=DEFAULT_PROTOCOL,
                 email=DEFAULT_EMAIL):
        super(m2g, self).__init__(hostname, protocol)
        self.email = email

    def __repr__(self):
        """
        Returns a string representation that can be used to reproduce this
        instance. `eval(repr(this))` should return an identical copy.

        Arguments:
            None

        Returns:
            str: Representation of reproducible instance.
        """
        return "ndio.remote.m2g('{}', '{}', '{}')".format(
            self.hostname,
            self.protocol,
            self.email
        )

    def ping(self):
        return super(m2g, self).ping()

    def url(self, suffix=""):
        return super(m2g, self).url('/graph-services/' + suffix)

    def set_default_email(self, email):
        """
        Sets the default email to use for notifications if none is specified.
        Does only basic error-checking.

        Arguments:
            email (str): The email to notify for all server graph services.

        Returns:
            None

        Raises:
            ValueError: If an invalid email is supplied.
        """
        if "@" not in email:
            raise ValueError("Invalid email.")
        self.email = email

    def build_graph(self, project, site, subject, session, scan,
                    size, email=None, invariants=Invariants.ALL,
                    fiber_file=DEFAULT_FIBER_FILE, atlas_file=None,
                    use_threads=False, callback=None):
        """
        Builds a graph using the graph-services endpoint.

        Arguments:
            project (str): The project to use
            site (str): The site in question
            subject (str): The subject's identifier
            session (str): The session (per subject)
            scan (str): The scan identifier
            size (str): Whether to return a big or (m2g.BIG) small (m2g.SMALL)
                graph. For a better explanation of each, see m2g.io.
            email (str)*: An email to which to send the download link
            invariants (str[]: Invariants.ALL)*: An array of invariants to
                compute. You can use the m2g.Invariants class to construct this
                list, or simply pass m2g.Invariants.ALL to compute them all.
            fiber_file (str: DEFAULT_FIBER_FILE)*: A local filename of an
                MRI Studio .dat file
            atlas_file (str: None)*: A local atlas file, in NIFTI .nii format.
                If none is specified, the Desikan atlas is used by default.
            use_threads (bool: False)*: Whether to run the download in a Python
                thread. If set to True, the call to `build_graph` will end
                quickly, and the `callback` will be called with the returned
                status-code of the restful call as its only argument.
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads is False. Otherwise, None

        Raises:
            ValueError: When the supplied values are invalid (contain invalid
                characters, bad email address supplied, etc.)
            RemoteDataNotFoundError: When the data cannot be processed due to
                a server error.
        """

        if use_threads and callback is not None:
            if not hasattr(callback, '__call__'):
                raise ValueError("callback must be a function.")
            if len(inspect.getargspec(callback).args) != 1:
                raise ValueError("callback must take exactly 1 argument.")

        # Once we get here, we know the callback is
        if size not in [self.BIG, self.SMALL]:
            raise ValueError("size must be either m2g.BIG or m2g.SMALL.")

        if email is None:
            email = self.email

        url = "buildgraph/{}/{}/{}/{}/{}/{}/{}/{}/".format(
            project,
            site,
            subject,
            session,
            scan,
            size,
            email,
            "/".join(invariants)
        )

        if " " in url:
            raise ValueError("Arguments must not contain spaces.")

        if use_threads:
            # Run in the background.
            download_thread = threading.Thread(
                target=self._run_build_graph,
                args=[url, fiber_file, atlas_file, callback]
            )
            download_thread.start()
        else:
            # Run in the foreground.
            return self._run_build_graph(url, fiber_file, atlas_file)
        return

    def _run_build_graph(self, url,
                            fiber_file, atlas_file=None, callback=None):

        try:
            tmpfile = tempfile.NamedTemporaryFile()
            zfile = zipfile.ZipFile(tmpfile.name, "w", allowZip64=True)
            zfile.write(file_file)
            if atlas_file:
                zfile.write(atlas_file)
                zfile.close()
                tmpfile.flush()
                tmpfile.seek(0)
        except:
            raise ValueError("Invalid atlas or fiber file. I don't have " +
                             "any more information than that, sorry!")

        try:
            req = urllib2.Request(self.url(url), tmpfile.read())
            response = urllib2.urlopen(req)

            if callback is not None:
                callback(response)
            else:
                return response.read()
        except:
            raise RemoteDataUploadError("Failed to upload data at " + url)

    def compute_invariants(self, graph_file, input_format,
                           invariants=Invariants.ALL, email=None,
                           use_threads=False, callback=None):
        """
        Compute invariants from an existing GraphML file using the remote
        m2g graph services.

        Arguments:
            graph_file (str): The filename of the graphml file
            input_format (str): One of m2g.GraphFormats
            invariants (str[]: Invariants.ALL)*: An array of m2g.Invariants to
                compute on the graph
            email (str)*: The email to notify upon completion
            use_threads (bool: True)*: Whether to use Python threads to run the
                computation in the background when waiting for the server to
                return the invariants
            callback (function: None)*: The function to run upon completion of
                the call, if using threads. (Will not be called if use_threads
                is set to False.)

        Returns:
            HTTP Response if use_threads is False. Otherwise, None

        Raises:
            ValueError: If the graph file does not exist, or if there are
                issues with the passed arguments
            RemoteDataUploadError: If there is an issue packing the file
            RemoteError: If the server experiences difficulty computing invs
        """

        if email is None:
            email = self.email

        url = "/graphupload/{}/{}/{}/".format(
            email,
            input_format,
            "/".join(invariants)
        )

        if " " in url:
            raise ValueError("Arguments cannot have spaces in them.")

        if not (os.path.exists(graph_file)):
            raise ValueError("File {} does not exist.".format(graph_file))

        if use_threads:
            # Run in the background.
            upload_thread = threading.Thread(
                target=self._run_compute_invariants,
                args=[url, graph_file, callback]
            )
            upload_thread.start()

        else:
            # Run in the foreground.
            return self._run_compute_invariants(url, fiber_file)
        return

    def _run_compute_invariants(self, url, graph_file, callback=None):
        try:
            tmpfile = tempfile.NamedTemporaryFile()
            zfile = zipfile.ZipFile(tmpfile.name, "w", allowZip64=True)
            zfile.write(graph_file)
            zfile.close()
            tmpfile.flush()
            tmpfile.seek(0)
        except:
            raise ValueError("Unable to zip graph file for upload.")

        try:
            req = urllib2.Request(url, tmpfile.read())
            response = urllib2.urlopen(req)

            if callback is not None:
                callback(response.read())
            else:
                return response.read()
        except:
            raise RemoteDataUploadError("Failed to upload graph file. Try " +
                                        "troubleshooting with a ping()?")
<|endoftext|>


<commit_before>"""
Scoring server for python model format.
The passed int model is expected to have function:
   predict(pandas.Dataframe) -> pandas.DataFrame

Input, expected in text/csv or application/json format,
is parsed into pandas.DataFrame and passed to the model.

Defines four endpoints:
    /ping used for health check
    /health (same as /ping)
    /version used for getting the mlflow version
    /invocations used for scoring
"""
from typing import Tuple, Dict
import flask
import json
import logging
import os
import sys
import traceback

from mlflow.environment_variables import MLFLOW_SCORING_SERVER_REQUEST_TIMEOUT

# NB: We need to be careful what we import form mlflow here. Scoring server is used from within
# model's conda environment. The version of mlflow doing the serving (outside) and the version of
# mlflow in the model's conda environment (inside) can differ. We should therefore keep mlflow
# dependencies to the minimum here.
# ALl of the mlflow dependencies below need to be backwards compatible.
from mlflow.exceptions import MlflowException
from mlflow.types import Schema
from mlflow.utils import reraise
from mlflow.utils.file_utils import path_to_local_file_uri
from mlflow.utils.proto_json_utils import (
    NumpyEncoder,
    dataframe_from_parsed_json,
    _get_jsonable_obj,
    parse_tf_serving_input,
)
from mlflow.version import VERSION

try:
    from mlflow.pyfunc import load_model, PyFuncModel
except ImportError:
    from mlflow.pyfunc import load_pyfunc as load_model
from mlflow.protos.databricks_pb2 import BAD_REQUEST
from mlflow.server.handlers import catch_mlflow_exception
from io import StringIO

_SERVER_MODEL_PATH = "__pyfunc_model_path__"

CONTENT_TYPE_CSV = "text/csv"
CONTENT_TYPE_JSON = "application/json"

CONTENT_TYPES = [
    CONTENT_TYPE_CSV,
    CONTENT_TYPE_JSON,
]

_logger = logging.getLogger(__name__)

DF_RECORDS = "dataframe_records"
DF_SPLIT = "dataframe_split"
INSTANCES = "instances"
INPUTS = "inputs"

SUPPORTED_FORMATS = {DF_RECORDS, DF_SPLIT, INSTANCES, INPUTS}

REQUIRED_INPUT_FORMAT = (
    f"The input must be a JSON dictionary with exactly one of the input fields {SUPPORTED_FORMATS}"
)

def infer_and_parse_json_input(json_input, schema: Schema = None):
    """
    :param json_input: A JSON-formatted string representation of TF serving input or a Pandas
                       DataFrame, or a stream containing such a string representation.
    :param schema: Optional schema specification to be used during parsing.
    """
    if isinstance(json_input, dict):
        decoded_input = json_input
    else:
        try:
            decoded_input = json.loads(json_input)
        except json.decoder.JSONDecodeError as ex:
            raise MlflowException(
                message=(
                    "Failed to parse input from JSON. Ensure that input is a valid JSON"
                    f" formatted string. Error: '{ex}'. Input: \n{json_input}\n"
                ),
                error_code=BAD_REQUEST,
            )
    if isinstance(decoded_input, dict):
        format_keys = set(decoded_input.keys()).intersection(SUPPORTED_FORMATS)
        if len(format_keys) != 1:
            message = f"Received dictionary with input fields: {list(decoded_input.keys())}"
            raise MlflowException(
                message=f"{REQUIRED_INPUT_FORMAT}. {message}.", error_code=BAD_REQUEST
            )
        input_format = format_keys.pop()
        if input_format in (INSTANCES, INPUTS):
            return parse_tf_serving_input(decoded_input, schema=schema)

        elif input_format in (DF_SPLIT, DF_RECORDS):
            # NB: skip the dataframe_ prefix
            pandas_orient = input_format[10:]
            return dataframe_from_parsed_json(
                decoded_input[input_format], pandas_orient=pandas_orient, schema=schema
            )
    elif isinstance(decoded_input, list):
        message = "Received a list"
        raise MlflowException(
            message=f"{REQUIRED_INPUT_FORMAT}. {message}.", error_code=BAD_REQUEST
        )
    else:
        message = f"Received unexpected input type '{type(decoded_input)}'"
        raise MlflowException(
            message=f"{REQUIRED_INPUT_FORMAT}. {message}.", error_code=BAD_REQUEST
        )

def parse_csv_input(csv_input, schema: Schema = None):
    """
    :param csv_input: A CSV-formatted string representation of a Pandas DataFrame, or a stream
                      containing such a string representation.
    :param schema: Optional schema specification to be used during parsing.
    """
    import pandas as pd

    try:
        if schema is None:
            return pd.read_csv(csv_input)
        else:
            dtypes = dict(zip(schema.input_names(), schema.pandas_types()))
            return pd.read_csv(csv_input, dtype=dtypes)
    except Exception:
        _handle_serving_error(
            error_message=(
                "Failed to parse input as a Pandas DataFrame. Ensure that the input is"
                " a valid CSV-formatted Pandas DataFrame produced using the"
                " `pandas.DataFrame.to_csv()` method."
            ),
            error_code=BAD_REQUEST,
        )

def predictions_to_json(raw_predictions, output):
    predictions = _get_jsonable_obj(raw_predictions, pandas_orient="records")
    return json.dump({"predictions": predictions}, output, cls=NumpyEncoder)

def _handle_serving_error(error_message, error_code, include_traceback=True):
    """
    Logs information about an exception thrown by model inference code that is currently being
    handled and reraises it with the specified error message. The exception stack trace
    is also included in the reraised error message.

    :param error_message: A message for the reraised exception.
    :param error_code: An appropriate error code for the reraised exception. This should be one of
                       the codes listed in the `mlflow.protos.databricks_pb2` proto.
    :param include_traceback: Whether to include the current traceback in the returned error.
    """
    if include_traceback:
        traceback_buf = StringIO()
        traceback.print_exc(file=traceback_buf)
        traceback_str = traceback_buf.getvalue()
        e = MlflowException(message=error_message, error_code=error_code, stack_trace=traceback_str)
    else:
        e = MlflowException(message=error_message, error_code=error_code)
    reraise(MlflowException, e)

def init(model: PyFuncModel):

    """
    Initialize the server. Loads pyfunc model from the path.
    """
    app = flask.Flask(__name__)
    input_schema = model.metadata.get_input_schema()

    @app.route("/ping", methods=["GET"])
    @app.route("/health", methods=["GET"])
    def ping():  # pylint: disable=unused-variable
        """
        Determine if the container is working and healthy.
        We declare it healthy if we can load the model successfully.
        """
        health = model is not None
        status = 200 if health else 404
        return flask.Response(response="\n", status=status, mimetype="application/json")

    @app.route("/version", methods=["GET"])
    def version():  # pylint: disable=unused-variable
        """
        Returns the current mlflow version.
        """
        return flask.Response(response=VERSION, status=200, mimetype="application/json")

    @app.route("/invocations", methods=["POST"])
    @catch_mlflow_exception
    def transformation():  # pylint: disable=unused-variable
        """
        Do an inference on a single batch of data. In this sample server,
        we take data as CSV or json, convert it to a Pandas DataFrame or Numpy,
        generate predictions and convert them back to json.
        """

        # Content-Type can include other attributes like CHARSET
        # Content-type RFC: https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
        # TODO: Suport ";" in quoted parameter values
        type_parts = flask.request.content_type.split(";")
        type_parts = list(map(str.strip, type_parts))
        mime_type = type_parts[0]
        parameter_value_pairs = type_parts[1:]
        parameter_values = {}
        for parameter_value_pair in parameter_value_pairs:
            (key, _, value) = parameter_value_pair.partition("=")
            parameter_values[key] = value

        charset = parameter_values.get("charset", "utf-8").lower()
        if charset != "utf-8":
            return flask.Response(
                response="The scoring server only supports UTF-8",
                status=415,
                mimetype="text/plain",
            )

        unexpected_content_parameters = set(parameter_values.keys()).difference({"charset"})
        if unexpected_content_parameters:
            return flask.Response(
                response=(
                    f"Unrecognized content type parameters: "
                    f"{', '.join(unexpected_content_parameters)}."
                ),
                status=415,
                mimetype="text/plain",
            )
        # Convert from CSV to pandas
        if mime_type == CONTENT_TYPE_CSV:
            data = flask.request.data.decode("utf-8")
            csv_input = StringIO(data)
            data = parse_csv_input(csv_input=csv_input, schema=input_schema)
        elif mime_type == CONTENT_TYPE_JSON:
            json_str = flask.request.data.decode("utf-8")
            data = infer_and_parse_json_input(json_str, input_schema)
        else:
            return flask.Response(
                response=(
                    "This predictor only supports the following content types:"
                    f" Types: {CONTENT_TYPES}."
                    f" Got '{flask.request.content_type}'."
                ),
                status=415,
                mimetype="text/plain",
            )

        # Do the prediction
        try:
            raw_predictions = model.predict(data)
        except MlflowException as e:
            raise e
        except Exception:
            raise MlflowException(
                message=(
                    "Encountered an unexpected error while evaluating the model. Verify"
                    " that the serialized input Dataframe is compatible with the model for"
                    " inference."
                ),
                error_code=BAD_REQUEST,
                stack_trace=traceback.format_exc(),
            )
        result = StringIO()
        predictions_to_json(raw_predictions, result)
        return flask.Response(response=result.getvalue(), status=200, mimetype="application/json")

    return app

def _predict(model_uri, input_path, output_path, content_type):
    pyfunc_model = load_model(model_uri)

    if content_type == "json":
        if input_path is None:
            input_str = sys.stdin.read()
        else:
            with open(input_path, "r") as f:
                input_str = f.read()
        df = infer_and_parse_json_input(input_str)
    elif content_type == "csv":
        if input_path is not None:
            df = parse_csv_input(input_path)
        else:
            df = parse_csv_input(sys.stdin)
    else:
        raise Exception("Unknown content type '{}'".format(content_type))

    if output_path is None:
        predictions_to_json(pyfunc_model.predict(df), sys.stdout)
    else:
        with open(output_path, "w") as fout:
            predictions_to_json(pyfunc_model.predict(df), fout)

def _serve(model_uri, port, host):
    pyfunc_model = load_model(model_uri)
    init(pyfunc_model).run(port=port, host=host)

def get_cmd(
    model_uri: str, port: int = None, host: int = None, timeout: int = None, nworkers: int = None
) -> Tuple[str, Dict[str, str]]:
    local_uri = path_to_local_file_uri(model_uri)
    timeout = timeout or MLFLOW_SCORING_SERVER_REQUEST_TIMEOUT.get()
    # NB: Absolute windows paths do not work with mlflow apis, use file uri to ensure
    # platform compatibility.
    if os.name != "nt":
        args = [f"--timeout={timeout}"]
        if port and host:
            args.append(f"-b {host}:{port}")
        elif host:
            args.append(f"-b {host}")

        if nworkers:
            args.append(f"-w {nworkers}")

        command = (
            f"gunicorn {' '.join(args)} ${{GUNICORN_CMD_ARGS}}"
            " -- mlflow.pyfunc.scoring_server.wsgi:app"
        )
    else:
        args = []
        if host:
            args.append(f"--host={host}")

        if port:
            args.append(f"--port={port}")

        command = (
            f"waitress-serve {' '.join(args)} "
            "--ident=mlflow mlflow.pyfunc.scoring_server.wsgi:app"
        )

    command_env = os.environ.copy()
    command_env[_SERVER_MODEL_PATH] = local_uri

    return command, command_env
<commit_msg>Add descriptive MLflow 2.0 migration error message for invalid serving input formats (#7329)<commit_after>"""
Scoring server for python model format.
The passed int model is expected to have function:
   predict(pandas.Dataframe) -> pandas.DataFrame

Input, expected in text/csv or application/json format,
is parsed into pandas.DataFrame and passed to the model.

Defines four endpoints:
    /ping used for health check
    /health (same as /ping)
    /version used for getting the mlflow version
    /invocations used for scoring
"""
from typing import Tuple, Dict
import flask
import json
import logging
import os
import sys
import traceback

from mlflow.environment_variables import MLFLOW_SCORING_SERVER_REQUEST_TIMEOUT

# NB: We need to be careful what we import form mlflow here. Scoring server is used from within
# model's conda environment. The version of mlflow doing the serving (outside) and the version of
# mlflow in the model's conda environment (inside) can differ. We should therefore keep mlflow
# dependencies to the minimum here.
# ALl of the mlflow dependencies below need to be backwards compatible.
from mlflow.exceptions import MlflowException
from mlflow.types import Schema
from mlflow.utils import reraise
from mlflow.utils.file_utils import path_to_local_file_uri
from mlflow.utils.proto_json_utils import (
    NumpyEncoder,
    dataframe_from_parsed_json,
    _get_jsonable_obj,
    parse_tf_serving_input,
)
from mlflow.version import VERSION

try:
    from mlflow.pyfunc import load_model, PyFuncModel
except ImportError:
    from mlflow.pyfunc import load_pyfunc as load_model
from mlflow.protos.databricks_pb2 import BAD_REQUEST
from mlflow.server.handlers import catch_mlflow_exception
from io import StringIO

_SERVER_MODEL_PATH = "__pyfunc_model_path__"

CONTENT_TYPE_CSV = "text/csv"
CONTENT_TYPE_JSON = "application/json"

CONTENT_TYPES = [
    CONTENT_TYPE_CSV,
    CONTENT_TYPE_JSON,
]

_logger = logging.getLogger(__name__)

DF_RECORDS = "dataframe_records"
DF_SPLIT = "dataframe_split"
INSTANCES = "instances"
INPUTS = "inputs"

SUPPORTED_FORMATS = {DF_RECORDS, DF_SPLIT, INSTANCES, INPUTS}

REQUIRED_INPUT_FORMAT = (
    f"The input must be a JSON dictionary with exactly one of the input fields {SUPPORTED_FORMATS}"
)
SCORING_PROTOCOL_CHANGE_INFO = (
    "IMPORTANT: The MLflow Model scoring protocol has changed in MLflow version 2.0. If you are"
    " seeing this error, you are likely using an outdated scoring request format. To resolve the"
    " error, either update your request format or adjust your MLflow Model's requirements file to"
    " specify an older version of MLflow (for example, change the 'mlflow' requirement specifier"
    " to 'mlflow==1.30.0'). If you are making a request using the MLflow client"
    " (e.g. via `mlflow.pyfunc.spark_udf()`), upgrade your MLflow client to a version >= 2.0 in"
    " order to use the new request format. For more information about the updated MLflow"
    " Model scoring protocol in MLflow 2.0, see"
    " https://mlflow.org/docs/latest/models.html#deploy-mlflow-models."
)

def infer_and_parse_json_input(json_input, schema: Schema = None):
    """
    :param json_input: A JSON-formatted string representation of TF serving input or a Pandas
                       DataFrame, or a stream containing such a string representation.
    :param schema: Optional schema specification to be used during parsing.
    """
    if isinstance(json_input, dict):
        decoded_input = json_input
    else:
        try:
            decoded_input = json.loads(json_input)
        except json.decoder.JSONDecodeError as ex:
            raise MlflowException(
                message=(
                    "Failed to parse input from JSON. Ensure that input is a valid JSON"
                    f" formatted string. Error: '{ex}'. Input: \n{json_input}\n"
                ),
                error_code=BAD_REQUEST,
            )
    if isinstance(decoded_input, dict):
        format_keys = set(decoded_input.keys()).intersection(SUPPORTED_FORMATS)
        if len(format_keys) != 1:
            message = f"Received dictionary with input fields: {list(decoded_input.keys())}"
            raise MlflowException(
                message=f"{REQUIRED_INPUT_FORMAT}. {message}. {SCORING_PROTOCOL_CHANGE_INFO}",
                error_code=BAD_REQUEST,
            )
        input_format = format_keys.pop()
        if input_format in (INSTANCES, INPUTS):
            return parse_tf_serving_input(decoded_input, schema=schema)

        elif input_format in (DF_SPLIT, DF_RECORDS):
            # NB: skip the dataframe_ prefix
            pandas_orient = input_format[10:]
            return dataframe_from_parsed_json(
                decoded_input[input_format], pandas_orient=pandas_orient, schema=schema
            )
    elif isinstance(decoded_input, list):
        message = "Received a list"
        raise MlflowException(
            message=f"{REQUIRED_INPUT_FORMAT}. {message}. {SCORING_PROTOCOL_CHANGE_INFO}",
            error_code=BAD_REQUEST,
        )
    else:
        message = f"Received unexpected input type '{type(decoded_input)}'"
        raise MlflowException(
            message=f"{REQUIRED_INPUT_FORMAT}. {message}.", error_code=BAD_REQUEST
        )

def parse_csv_input(csv_input, schema: Schema = None):
    """
    :param csv_input: A CSV-formatted string representation of a Pandas DataFrame, or a stream
                      containing such a string representation.
    :param schema: Optional schema specification to be used during parsing.
    """
    import pandas as pd

    try:
        if schema is None:
            return pd.read_csv(csv_input)
        else:
            dtypes = dict(zip(schema.input_names(), schema.pandas_types()))
            return pd.read_csv(csv_input, dtype=dtypes)
    except Exception:
        _handle_serving_error(
            error_message=(
                "Failed to parse input as a Pandas DataFrame. Ensure that the input is"
                " a valid CSV-formatted Pandas DataFrame produced using the"
                " `pandas.DataFrame.to_csv()` method."
            ),
            error_code=BAD_REQUEST,
        )

def predictions_to_json(raw_predictions, output):
    predictions = _get_jsonable_obj(raw_predictions, pandas_orient="records")
    return json.dump({"predictions": predictions}, output, cls=NumpyEncoder)

def _handle_serving_error(error_message, error_code, include_traceback=True):
    """
    Logs information about an exception thrown by model inference code that is currently being
    handled and reraises it with the specified error message. The exception stack trace
    is also included in the reraised error message.

    :param error_message: A message for the reraised exception.
    :param error_code: An appropriate error code for the reraised exception. This should be one of
                       the codes listed in the `mlflow.protos.databricks_pb2` proto.
    :param include_traceback: Whether to include the current traceback in the returned error.
    """
    if include_traceback:
        traceback_buf = StringIO()
        traceback.print_exc(file=traceback_buf)
        traceback_str = traceback_buf.getvalue()
        e = MlflowException(message=error_message, error_code=error_code, stack_trace=traceback_str)
    else:
        e = MlflowException(message=error_message, error_code=error_code)
    reraise(MlflowException, e)

def init(model: PyFuncModel):

    """
    Initialize the server. Loads pyfunc model from the path.
    """
    app = flask.Flask(__name__)
    input_schema = model.metadata.get_input_schema()

    @app.route("/ping", methods=["GET"])
    @app.route("/health", methods=["GET"])
    def ping():  # pylint: disable=unused-variable
        """
        Determine if the container is working and healthy.
        We declare it healthy if we can load the model successfully.
        """
        health = model is not None
        status = 200 if health else 404
        return flask.Response(response="\n", status=status, mimetype="application/json")

    @app.route("/version", methods=["GET"])
    def version():  # pylint: disable=unused-variable
        """
        Returns the current mlflow version.
        """
        return flask.Response(response=VERSION, status=200, mimetype="application/json")

    @app.route("/invocations", methods=["POST"])
    @catch_mlflow_exception
    def transformation():  # pylint: disable=unused-variable
        """
        Do an inference on a single batch of data. In this sample server,
        we take data as CSV or json, convert it to a Pandas DataFrame or Numpy,
        generate predictions and convert them back to json.
        """

        # Content-Type can include other attributes like CHARSET
        # Content-type RFC: https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
        # TODO: Suport ";" in quoted parameter values
        type_parts = flask.request.content_type.split(";")
        type_parts = list(map(str.strip, type_parts))
        mime_type = type_parts[0]
        parameter_value_pairs = type_parts[1:]
        parameter_values = {}
        for parameter_value_pair in parameter_value_pairs:
            (key, _, value) = parameter_value_pair.partition("=")
            parameter_values[key] = value

        charset = parameter_values.get("charset", "utf-8").lower()
        if charset != "utf-8":
            return flask.Response(
                response="The scoring server only supports UTF-8",
                status=415,
                mimetype="text/plain",
            )

        unexpected_content_parameters = set(parameter_values.keys()).difference({"charset"})
        if unexpected_content_parameters:
            return flask.Response(
                response=(
                    f"Unrecognized content type parameters: "
                    f"{', '.join(unexpected_content_parameters)}. "
                    f"{SCORING_PROTOCOL_CHANGE_INFO}"
                ),
                status=415,
                mimetype="text/plain",
            )
        # Convert from CSV to pandas
        if mime_type == CONTENT_TYPE_CSV:
            data = flask.request.data.decode("utf-8")
            csv_input = StringIO(data)
            data = parse_csv_input(csv_input=csv_input, schema=input_schema)
        elif mime_type == CONTENT_TYPE_JSON:
            json_str = flask.request.data.decode("utf-8")
            data = infer_and_parse_json_input(json_str, input_schema)
        else:
            return flask.Response(
                response=(
                    "This predictor only supports the following content types:"
                    f" Types: {CONTENT_TYPES}."
                    f" Got '{flask.request.content_type}'."
                ),
                status=415,
                mimetype="text/plain",
            )

        # Do the prediction
        try:
            raw_predictions = model.predict(data)
        except MlflowException as e:
            raise e
        except Exception:
            raise MlflowException(
                message=(
                    "Encountered an unexpected error while evaluating the model. Verify"
                    " that the serialized input Dataframe is compatible with the model for"
                    " inference."
                ),
                error_code=BAD_REQUEST,
                stack_trace=traceback.format_exc(),
            )
        result = StringIO()
        predictions_to_json(raw_predictions, result)
        return flask.Response(response=result.getvalue(), status=200, mimetype="application/json")

    return app

def _predict(model_uri, input_path, output_path, content_type):
    pyfunc_model = load_model(model_uri)

    if content_type == "json":
        if input_path is None:
            input_str = sys.stdin.read()
        else:
            with open(input_path, "r") as f:
                input_str = f.read()
        df = infer_and_parse_json_input(input_str)
    elif content_type == "csv":
        if input_path is not None:
            df = parse_csv_input(input_path)
        else:
            df = parse_csv_input(sys.stdin)
    else:
        raise Exception("Unknown content type '{}'".format(content_type))

    if output_path is None:
        predictions_to_json(pyfunc_model.predict(df), sys.stdout)
    else:
        with open(output_path, "w") as fout:
            predictions_to_json(pyfunc_model.predict(df), fout)

def _serve(model_uri, port, host):
    pyfunc_model = load_model(model_uri)
    init(pyfunc_model).run(port=port, host=host)

def get_cmd(
    model_uri: str, port: int = None, host: int = None, timeout: int = None, nworkers: int = None
) -> Tuple[str, Dict[str, str]]:
    local_uri = path_to_local_file_uri(model_uri)
    timeout = timeout or MLFLOW_SCORING_SERVER_REQUEST_TIMEOUT.get()
    # NB: Absolute windows paths do not work with mlflow apis, use file uri to ensure
    # platform compatibility.
    if os.name != "nt":
        args = [f"--timeout={timeout}"]
        if port and host:
            args.append(f"-b {host}:{port}")
        elif host:
            args.append(f"-b {host}")

        if nworkers:
            args.append(f"-w {nworkers}")

        command = (
            f"gunicorn {' '.join(args)} ${{GUNICORN_CMD_ARGS}}"
            " -- mlflow.pyfunc.scoring_server.wsgi:app"
        )
    else:
        args = []
        if host:
            args.append(f"--host={host}")

        if port:
            args.append(f"--port={port}")

        command = (
            f"waitress-serve {' '.join(args)} "
            "--ident=mlflow mlflow.pyfunc.scoring_server.wsgi:app"
        )

    command_env = os.environ.copy()
    command_env[_SERVER_MODEL_PATH] = local_uri

    return command, command_env
<|endoftext|>


<commit_before># nerokid.py

# nerokid.py
#
# one line description
import sys
import os
import time
import subprocess
import shlex
from core import Logger, Socket

INTERVAL = 2.0
LOG_FILES = 'stdout.log', 'stderr.log'

class LogFile(object):

    def __init__(self, path):
        self.path = path
        self.rtime = 0
        self.seek = 0

    def read_changes(self):
        mtime = os.stat(self.path).st_mtime
        if mtime > self.rtime:
            self.rtime = mtime
            with open(self.path, 'r') as stream:
                stream.seek(self.seek)
                changes = stream.read()
                self.seek = stream.tell()
                return changes

class NeroKid(object):

    def __init__(self):
        """"""
        self.sock = None
        self.process = None
        self.logger = Logger('KID')
        self.experiment = ' '.join(sys.argv[3:])
        self.log_files = [LogFile(log_file_path) for log_file_path in LOG_FILES]

    def run(self):
        """The Nerokid main."""
        self.logger.log('Kid launched!')
        self.initialize_socket()
        self.logger.log('Launching the experiment...')
        self.launch_child_process()
        self.monitor_process()
        self.logger.log('Process finished!')

    def initialize_socket(self):
        """initialize socket"""
        host, port = sys.argv[1:3]
        port = int(port)
        # Define a socket
        self.sock = Socket(self.logger, host, port)
        self.logger.log('- Mom address: (%s, %d)' % (host, port))
        self.logger.log('- Experiment: %s' % (self.experiment))

    def send_data_to_neromum(self, text):
        """Send status data to Neromum."""
        sock.send_data(text)

    def launch_child_process(self):
        """Launches received script"""
        #For windows
        #self.process = subprocess.Popen(['python', shlex.split(self.experiment)], universal_newlines=True,
        #stdout=open('stdout.log', 'w'), stderr=open('stderr.log', 'w'),
        #bufsize=1)

        #for linux
        self.process = subprocess.Popen(shlex.split(self.experiment), universal_newlines=True,
        stdout=open('stdout.log', 'w'), stderr=open('stderr.log', 'w'),
        close_fds=True, bufsize=1)

    def monitor_process(self):
        self.logger.log('- Experiment PID: %s' % (self.process.pid))
        while self.process.poll() == None:
            # Sleep to wait for changes
            time.sleep(INTERVAL)
            self.collect_new_file_data()

    def collect_new_file_data(self):
        """Collect data what child process outputs"""
        # Check for any new log output
        log_output = {}
        for log_file in self.log_files:
            changes = log_file.read_changes()
            if changes:
                log_output[log_file.path] = changes
        # Send any new log output to Mum
        if log_output:
            self.sock.send_data({'log_output': log_output})

if __name__ == '__main__':
    NeroKid().run()
<commit_msg>neroki bugfixes<commit_after># nerokid.py

# nerokid.py
#
# one line description
import sys
import os
import time
import subprocess
import shlex
from core import Logger, Socket

INTERVAL = 2.0
LOG_FILES = 'stdout.log', 'stderr.log'

class LogFile(object):

    def __init__(self, path):
        self.path = path
        self.rtime = 0
        self.seek = 0

    def read_changes(self):
        mtime = os.stat(self.path).st_mtime
        if mtime > self.rtime:
            self.rtime = mtime
            with open(self.path, 'r') as stream:
                stream.seek(self.seek)
                changes = stream.read()
                self.seek = stream.tell()
                return changes

class NeroKid(object):

    def __init__(self):
        """"""
        self.sock = None
        self.process = None
        self.logger = Logger('KID')
        self.experiment = ' '.join(sys.argv[3:])
        self.log_files = [LogFile(log_file_path) for log_file_path in LOG_FILES]

    def run(self):
        """The Nerokid main."""
        self.logger.log('Kid launched!')
        self.initialize_socket()
        self.logger.log('Launching the experiment...')
        self.launch_child_process()
        self.monitor_process()
        self.logger.log('Process finished!')

    def initialize_socket(self):
        """initialize socket"""
        host, port = sys.argv[1:3]
        port = int(port)
        # Define a socket
        self.sock = Socket(self.logger, host, port)
        self.logger.log('- Mom address: (%s, %d)' % (host, port))
        self.logger.log('- Experiment: %s' % (self.experiment))

    def send_data_to_neromum(self, text):
        """Send status data to Neromum."""
        self.sock.send_data(text)

    def launch_child_process(self):
        """Launches received script"""
        #For windows
        #self.process = subprocess.Popen(['python', shlex.split(self.experiment)], universal_newlines=True,
        #stdout=open('stdout.log', 'w'), stderr=open('stderr.log', 'w'),
        #bufsize=1)

        #for linux
        self.logger.log(shlex.split(self.experiment))
        self.process = subprocess.Popen(shlex.split(self.experiment), universal_newlines=True,
        stdout=open('stdout.log', 'w'), stderr=open('stderr.log', 'w'),
        close_fds=True, bufsize=1)

    def monitor_process(self):
        self.logger.log('- Experiment PID: %s' % (self.process.pid))
        while self.process.poll() == None:
            # Sleep to wait for changes
            time.sleep(INTERVAL)
            self.collect_new_file_data()

    def collect_new_file_data(self):
        """Collect data what child process outputs"""
        # Check for any new log output
        log_output = {}
        for log_file in self.log_files:
            changes = log_file.read_changes()
            if changes:
                log_output[log_file.path] = changes
        # Send any new log output to Mum
        if log_output:
            self.sock.send_data({'log_output': log_output})

if __name__ == '__main__':
    NeroKid().run()
<|endoftext|>