<commit_before>import pylab
import matplotlib
from matplotlib import pyplot
import sys
import os
import warnings

from Utils import getGoldenRatioDimensions

class PlotLayout(object):
    def __init__(self):
        self.groupedPlots = {}
        self.plots = []
        self.groupOrder = None

        self.width = 1
        self.plotParams = None

        self.dimensions = None
        self.figdimensions = None

        self.figLegendLoc = None
        self.figLegendCols = None

        self.rcParams = None

    def __setRCParam(self, param, value):
        if self.rcParams is None:
            self.rcParams = {}

        self.rcParams[param] = value

    def useLatexLabels(self):
        print >>sys.stderr, "WARNING: Using LaTeX labels requires dvipng and ghostscript"
        self.__setRCParam("font.family", "serif")
        self.__setRCParam("font.serif", "Times, Palatino, New Century Schoolbook, Bookman, Computer Modern Roman")
        self.__setRCParam("font.sans-serif", "Helvetica, Avant Garde, Computer Modern Sans serif")
        self.__setRCParam("font.cursive", "Zapf Chancery")
        self.__setRCParam("font.monospace", "Courier, Computer Modern Typewriter")
        self.__setRCParam("text.usetex", True)

    # def useStandardFont(self):
    #     self.__setRCParam("font.family", "serif")
    #     self.__setRCParam("font.style", "normal")
    #     self.__setRCParam("font.variant", "normal")
    #     self.__setRCParam("font.weight", "normal")
    #     self.__setRCParam("font.stretch", "normal")

    def setAxesLabelSize(self, size):
        self.__setRCParam("axes.labelsize", size)

    def setXTickLabelSize(self, size):
        self.__setRCParam("xtick.labelsize", size)

    def setYTickLabelSize(self, size):
        self.__setRCParam("ytick.labelsize", size)

    def setLegendLabelSize(self, size):
        self.__setRCParam("legend.fontsize", size)

    def setWidth(self, width):
        self.width = int(width)

    def hasFigLegend(self, loc="best", columns=1, numcols=None):
        if numcols is not None:
            # hasLegend uses columns, rather than numcols.
            # trying to make this consistent.
            # this should catch named parameter usage
            columns = numcols
            warnings.warn("numcols deprecated for hasFigLegend", Warning)

        self.figLegendLoc = loc
        self.figLegendCols = columns

    def addPlot(self, plot, grouping=None):
        if grouping == None:
            self.plots.append(plot)
        else:
            if grouping not in self.groupedPlots:
                self.groupedPlots[grouping] = []

            self.groupedPlots[grouping].append(plot)

    def setGroupOrder(self, groupOrder):
        self.groupOrder = groupOrder

    def setPlotDimensions(self, x, y):
        self.dimensions = (x,y)

    def setFigureDimensions(self, x, y):
        self.figdimensions = (x,y)

    def setPlotParameters(self, **kwdict):
        self.plotParams = dict(kwdict)

        if "left" not in self.plotParams:
            self.plotParams["left"] = 0.12

        if "bottom" not in self.plotParams:
            self.plotParams["bottom"] = 0.10

        if "right" not in self.plotParams:
            self.plotParams["right"] = 0.90

        if "top" not in self.plotParams:
            self.plotParams["top"] = 0.90

        if "wspace" not in self.plotParams:
            self.plotParams["wspace"] = 0.20

        if "hspace" not in self.plotParams:
            self.plotParams["hspace"] = 0.20

    def _doPlot(self):
        if len(self.groupedPlots) + len(self.plots) == 0:
            print "PlotLayout.plot(): No data to plot!"
            return

        oldRCParams = {}

        if self.rcParams is not None:

            for (param, value) in self.rcParams.items():
                oldRCParams[param] = pylab.rcParams[param]
                pylab.rcParams[param] = value

        groupedPlotLengths = [len(plots) for plots in self.groupedPlots.values()]

        if len(groupedPlotLengths) == 0:
            maxRowLength = self.width
        else:
            maxRowLength = max(groupedPlotLengths)

        numPlots = len(self.plots)

        if numPlots == 0:
            numExcessRows = 0
        elif numPlots <= maxRowLength:
            numExcessRows = 1
        elif numPlots % maxRowLength == 0:
            numExcessRows = numPlots / maxRowLength
        else:
            numExcessRows = (numPlots / maxRowLength) + 1

        numRows = len(self.groupedPlots.keys()) + numExcessRows

        if self.groupOrder is not None:
            keyList = self.groupOrder
        else:
            keyList = self.groupedPlots.keys()

        currentRow = 0

        if self.figdimensions is not None:
            fig = pyplot.figure(figsize=(self.figdimensions[0],
                                         self.figdimensions[1]))
        elif self.dimensions is not None:
            fig = pyplot.figure(figsize=(self.dimensions[0] * maxRowLength,
                                         self.dimensions[1] * numRows))
        else:
            (figWidth, figHeight) = getGoldenRatioDimensions(8.0)
            figWidth *= maxRowLength
            figHeight *= numRows
            fig = pyplot.figure(figsize=(figWidth, figHeight))
            # figWidth = fig.get_figwidth()
            # print figWidth
            # print fig.get_figheight()
            # (goldenWidth, goldenHeight) = getGoldenRatioDimensions(figWidth)
            # fig.set_figheight(goldenHeight)
            # print fig.get_figheight()

        plotHandles = []
        plotLabels = []

        for grouping in keyList:
            currentColumn = 1
            plots = self.groupedPlots[grouping]

            numPlots = len(plots)

            for plot in plots:
                myRows = numRows
                myCols = numPlots
                myPos = (currentRow * numPlots) + currentColumn

                (currPlotHandles, currPlotLabels) = plot.subplot(
                    myRows, myCols, myPos)

                for i in xrange(len(currPlotHandles)):
                    if currPlotLabels[i] in plotLabels:
                        continue

                    if isinstance(currPlotHandles[i], list):
                        plotHandles.append(currPlotHandles[i][0])
                    else:
                        plotHandles.append(currPlotHandles[i])

                    plotLabels.append(currPlotLabels[i])

#                plotHandles.extend(currPlotHandles)
#                plotLabels.extend(currPlotLabels)

                currentColumn += 1
            currentRow += 1

        ungroupedPlotsRemaining = len(self.plots)
        if ungroupedPlotsRemaining > 0:
            currentColumn = 1

            for plot in self.plots:
                if currentColumn == 1:
                    numColumns = min(maxRowLength, ungroupedPlotsRemaining)

                myRows = numRows
                myCols = numColumns
                myPos = (currentRow * numColumns) + currentColumn

                (currPlotHandles, currPlotLabels) = plot.subplot(
                    myRows, myCols, myPos)
                for i in xrange(len(currPlotHandles)):
                    if currPlotLabels[i] in plotLabels:
                        continue

                    if isinstance(currPlotHandles[i], list):
                        plotHandles.append(currPlotHandles[i][0])
                    else:
                        plotHandles.append(currPlotHandles[i])

                    plotLabels.append(currPlotLabels[i])

                currentColumn += 1

                if currentColumn > numColumns:
                    currentRow += 1
                    currentColumn = 1

                ungroupedPlotsRemaining -= 1

        if self.figLegendLoc is not None:
            figLegendKeywords = {}

            if self.figLegendCols is not None:
                versionPieces = [int(x) for x in matplotlib.__version__.split('.')]

                (superMajor, major, minor) = versionPieces[0:3]

                if superMajor == 0 and major < 98:
                    print >>sys.stderr, "Number of columns support not available in versions of matplotlib prior to 0.98"
                else:
                    figLegendKeywords["ncol"] = self.figLegendCols

            pylab.figlegend(plotHandles, plotLabels,
                            self.figLegendLoc,
                            **figLegendKeywords)

        if self.plotParams is not None:
            pylab.subplots_adjust(left=self.plotParams["left"],
                                  bottom=self.plotParams["bottom"],
                                  right=self.plotParams["right"],
                                  top=self.plotParams["top"],
                                  wspace=self.plotParams["wspace"],
                                  hspace=self.plotParams["hspace"])
        # Restore old RC params
        for (key,value) in oldRCParams.items():
            pylab.rcParams[key] = value

    def plot(self):
        self._doPlot()
        if not pylab.isinteractive():
            pylab.show()
        else:
            pylab.draw()

    def save(self, filename, **kwargs):
        tempDisplayHack = False

        if "DISPLAY" not in os.environ:
            tempDisplayHack = True
            os.environ["DISPLAY"] = ":0.0"
        self._doPlot()
        pylab.savefig(filename,**kwargs)
        pylab.clf()

        if tempDisplayHack == True:
            del os.environ["DISPLAY"]
<commit_msg>Changing print statements and asserts to appropriate warnings and exceptions.<commit_after>import pylab
import matplotlib
from matplotlib import pyplot
import sys
import os
import warnings
from exceptions import BoomslangPlotRenderingException

from Utils import getGoldenRatioDimensions

class PlotLayout(object):
    def __init__(self):
        self.groupedPlots = {}
        self.plots = []
        self.groupOrder = None

        self.width = 1
        self.plotParams = None

        self.dimensions = None
        self.figdimensions = None

        self.figLegendLoc = None
        self.figLegendCols = None

        self.rcParams = None

    def __setRCParam(self, param, value):
        if self.rcParams is None:
            self.rcParams = {}

        self.rcParams[param] = value

    def useLatexLabels(self):
        warnings.warn("WARNING: Using LaTeX labels requires dvipng and "
                      "ghostscript")
        self.__setRCParam("font.family", "serif")
        self.__setRCParam("font.serif", "Times, Palatino, New Century Schoolbook, Bookman, Computer Modern Roman")
        self.__setRCParam("font.sans-serif", "Helvetica, Avant Garde, Computer Modern Sans serif")
        self.__setRCParam("font.cursive", "Zapf Chancery")
        self.__setRCParam("font.monospace", "Courier, Computer Modern Typewriter")
        self.__setRCParam("text.usetex", True)

    # def useStandardFont(self):
    #     self.__setRCParam("font.family", "serif")
    #     self.__setRCParam("font.style", "normal")
    #     self.__setRCParam("font.variant", "normal")
    #     self.__setRCParam("font.weight", "normal")
    #     self.__setRCParam("font.stretch", "normal")

    def setAxesLabelSize(self, size):
        self.__setRCParam("axes.labelsize", size)

    def setXTickLabelSize(self, size):
        self.__setRCParam("xtick.labelsize", size)

    def setYTickLabelSize(self, size):
        self.__setRCParam("ytick.labelsize", size)

    def setLegendLabelSize(self, size):
        self.__setRCParam("legend.fontsize", size)

    def setWidth(self, width):
        self.width = int(width)

    def hasFigLegend(self, loc="best", columns=1, numcols=None):
        if numcols is not None:
            # hasLegend uses columns, rather than numcols.
            # trying to make this consistent.
            # this should catch named parameter usage
            columns = numcols
            warnings.warn("numcols deprecated for hasFigLegend", Warning)

        self.figLegendLoc = loc
        self.figLegendCols = columns

    def addPlot(self, plot, grouping=None):
        if grouping == None:
            self.plots.append(plot)
        else:
            if grouping not in self.groupedPlots:
                self.groupedPlots[grouping] = []

            self.groupedPlots[grouping].append(plot)

    def setGroupOrder(self, groupOrder):
        self.groupOrder = groupOrder

    def setPlotDimensions(self, x, y):
        self.dimensions = (x,y)

    def setFigureDimensions(self, x, y):
        self.figdimensions = (x,y)

    def setPlotParameters(self, **kwdict):
        self.plotParams = dict(kwdict)

        if "left" not in self.plotParams:
            self.plotParams["left"] = 0.12

        if "bottom" not in self.plotParams:
            self.plotParams["bottom"] = 0.10

        if "right" not in self.plotParams:
            self.plotParams["right"] = 0.90

        if "top" not in self.plotParams:
            self.plotParams["top"] = 0.90

        if "wspace" not in self.plotParams:
            self.plotParams["wspace"] = 0.20

        if "hspace" not in self.plotParams:
            self.plotParams["hspace"] = 0.20

    def _doPlot(self):
        if len(self.groupedPlots) + len(self.plots) == 0:
            raise BoomslangPlotRenderingException("No data to plot!")

        oldRCParams = {}

        if self.rcParams is not None:

            for (param, value) in self.rcParams.items():
                oldRCParams[param] = pylab.rcParams[param]
                pylab.rcParams[param] = value

        groupedPlotLengths = [len(plots) for plots in self.groupedPlots.values()]

        if len(groupedPlotLengths) == 0:
            maxRowLength = self.width
        else:
            maxRowLength = max(groupedPlotLengths)

        numPlots = len(self.plots)

        if numPlots == 0:
            numExcessRows = 0
        elif numPlots <= maxRowLength:
            numExcessRows = 1
        elif numPlots % maxRowLength == 0:
            numExcessRows = numPlots / maxRowLength
        else:
            numExcessRows = (numPlots / maxRowLength) + 1

        numRows = len(self.groupedPlots.keys()) + numExcessRows

        if self.groupOrder is not None:
            keyList = self.groupOrder
        else:
            keyList = self.groupedPlots.keys()

        currentRow = 0

        if self.figdimensions is not None:
            fig = pyplot.figure(figsize=(self.figdimensions[0],
                                         self.figdimensions[1]))
        elif self.dimensions is not None:
            fig = pyplot.figure(figsize=(self.dimensions[0] * maxRowLength,
                                         self.dimensions[1] * numRows))
        else:
            (figWidth, figHeight) = getGoldenRatioDimensions(8.0)
            figWidth *= maxRowLength
            figHeight *= numRows
            fig = pyplot.figure(figsize=(figWidth, figHeight))

        plotHandles = []
        plotLabels = []

        for grouping in keyList:
            currentColumn = 1
            plots = self.groupedPlots[grouping]

            numPlots = len(plots)

            for plot in plots:
                myRows = numRows
                myCols = numPlots
                myPos = (currentRow * numPlots) + currentColumn

                (currPlotHandles, currPlotLabels) = plot.subplot(
                    myRows, myCols, myPos)

                for i in xrange(len(currPlotHandles)):
                    if currPlotLabels[i] in plotLabels:
                        continue

                    if isinstance(currPlotHandles[i], list):
                        plotHandles.append(currPlotHandles[i][0])
                    else:
                        plotHandles.append(currPlotHandles[i])

                    plotLabels.append(currPlotLabels[i])

#                plotHandles.extend(currPlotHandles)
#                plotLabels.extend(currPlotLabels)

                currentColumn += 1
            currentRow += 1

        ungroupedPlotsRemaining = len(self.plots)
        if ungroupedPlotsRemaining > 0:
            currentColumn = 1

            for plot in self.plots:
                if currentColumn == 1:
                    numColumns = min(maxRowLength, ungroupedPlotsRemaining)

                myRows = numRows
                myCols = numColumns
                myPos = (currentRow * numColumns) + currentColumn

                (currPlotHandles, currPlotLabels) = plot.subplot(
                    myRows, myCols, myPos)
                for i in xrange(len(currPlotHandles)):
                    if currPlotLabels[i] in plotLabels:
                        continue

                    if isinstance(currPlotHandles[i], list):
                        plotHandles.append(currPlotHandles[i][0])
                    else:
                        plotHandles.append(currPlotHandles[i])

                    plotLabels.append(currPlotLabels[i])

                currentColumn += 1

                if currentColumn > numColumns:
                    currentRow += 1
                    currentColumn = 1

                ungroupedPlotsRemaining -= 1

        if self.figLegendLoc is not None:
            figLegendKeywords = {}

            if self.figLegendCols is not None:
                versionPieces = [int(x) for x in matplotlib.__version__.split('.')]

                (superMajor, major, minor) = versionPieces[0:3]

                if superMajor == 0 and major < 98:
                    warnings.warn("Number of columns support not available in "
                                  "versions of matplotlib prior to 0.98")
                else:
                    figLegendKeywords["ncol"] = self.figLegendCols

            pylab.figlegend(plotHandles, plotLabels,
                            self.figLegendLoc,
                            **figLegendKeywords)

        if self.plotParams is not None:
            pylab.subplots_adjust(left=self.plotParams["left"],
                                  bottom=self.plotParams["bottom"],
                                  right=self.plotParams["right"],
                                  top=self.plotParams["top"],
                                  wspace=self.plotParams["wspace"],
                                  hspace=self.plotParams["hspace"])
        # Restore old RC params
        for (key,value) in oldRCParams.items():
            pylab.rcParams[key] = value

    def plot(self):
        self._doPlot()
        if not pylab.isinteractive():
            pylab.show()
        else:
            pylab.draw()

    def save(self, filename, **kwargs):
        tempDisplayHack = False

        if "DISPLAY" not in os.environ:
            tempDisplayHack = True
            os.environ["DISPLAY"] = ":0.0"
        self._doPlot()
        pylab.savefig(filename,**kwargs)
        pylab.clf()

        if tempDisplayHack == True:
            del os.environ["DISPLAY"]
<|endoftext|>


<commit_before>import cv2
import pandas as pd
import numpy as np
from kobra.imaging import show_images
from os import path
from imutils import translate

class DetectOD(object):
    def __init__(self, im_file, mask_dir):

        assert (path.exists(im_file)), "Image not found"
        assert (path.exists(mask_dir)), "Mask not found"

        self._im_name = path.splitext(path.split(im_file)[1])[0]
        mask_file = path.join(mask_dir, self._im_name + ".png")
        self._img = cv2.imread(im_file)
        self._mask = cv2.imread(mask_file, cv2.IMREAD_GRAYSCALE)

        assert (self._img.size > 0), "Image not found"
        assert (self._mask.size > 0), "Mask not found"

        #self._img = kim.pyr_blurr(self._img)
        self._img = cv2.resize(self._img, (540, 540))
        self._mask = cv2.resize(self._mask, (540, 540))

        self._shifted = cv2.bitwise_and(translate(self._mask, 50, 0), translate(self._mask, -50, 0))

        self._img [self._mask == 0] = 0

        # get green channel
        self._green = self._img[:, :, 1].astype('float32')

        self._kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
        self._processed = self._green.copy()

    @property
    def image(self):
        return self._img

    @property
    def active_channel(self):
        return self._green

    @property
    def processed(self):
        return self._processed

    def remove_light_reflex(self):
        return cv2.morphologyEx(self._processed, cv2.MORPH_OPEN, self._kernel)

    def shade_correct(self):
        gamma = self.remove_light_reflex()

        # mean filter
        self._processed = cv2.blur(self._processed, (3, 3))
        # Gaussian convolution
        self._processed = cv2.GaussianBlur(self._processed, (9, 9), 1.8)

        # further averaging
        # before mean blurring, set black regions of the image to the average gray value
        self._processed[self._mask == 0] = cv2.mean(self._processed, self._mask)[0]

        background = cv2.blur(self._processed, (69, 69))

        # distance matrix between background and the image
        dist = gamma - background
        minDist = np.min(dist)
        interval = np.max(dist) - minDist
        self._processed = np.around((dist - minDist) * 255. / interval)
        self._processed[self._mask == 0] = 0

        return self._processed

    def apply_morphology(self):
        for r in range(2, 12, 2):
            kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * r, 2 * r))
            self._processed = cv2.morphologyEx(self._processed, cv2.MORPH_OPEN, kernel)
            self._processed = cv2.morphologyEx(self._processed, cv2.MORPH_CLOSE, kernel)

        return self._processed

    def locate_disk(self):
        self.shade_correct()
        self.apply_morphology()

        pr = self._processed.copy().astype('uint8')

        # kill possible bright edges of the image
        pr[self._shifted == 0] = 0

        _, maxCol, _, ctr = cv2.minMaxLoc(pr)
        return ctr

    def show_detected(self, ctr):
        pr = self._processed.copy().astype('uint8')
        cv2.circle(pr, ctr, 50, 0, 3)
        cv2.circle(self.image, ctr, 50, (0, 0, 0), 3)
        show_images([self.image, pr])<commit_msg>Adding rescaling to original sizes<commit_after>import cv2
import pandas as pd
import numpy as np
from kobra.imaging import show_images
from os import path
from imutils import translate

class DetectOD(object):
    '''
    Detecting location of the optic disk (OD)
    based on the morphological method proposed in
    Suero, Marin, et. al. -- Locating the Optic Disc in Retinal Images Using
    Morphological Techniques.

    Image masks should be extracted prior to using this class and placed in mask_dir.
    These masks are detected locations of the actual eye in the image being processed
    '''
    def __init__(self, im_file, mask_dir):

        assert (path.exists(im_file)), "Image not found"
        assert (path.exists(mask_dir)), "Mask not found"

        self._im_name = path.splitext(path.split(im_file)[1])[0]
        mask_file = path.join(mask_dir, self._im_name + ".png")
        self._img = cv2.imread(im_file)
        self._mask = cv2.imread(mask_file, cv2.IMREAD_GRAYSCALE)

        assert (self._img.size > 0), "Image not found"
        assert (self._mask.size > 0), "Mask not found"

        self._scale = np.float32(self._img.shape)[:-1] / 540.
        self._scale[0], self._scale[1] = self._scale[1], self._scale[0]

        self._img = cv2.resize(self._img, (540, 540))
        self._mask = cv2.resize(self._mask, (540, 540))

        self._shifted = cv2.bitwise_and(translate(self._mask, 50, 0), translate(self._mask, -50, 0))

        self._img [self._mask == 0] = 0

        # get green channel
        self._green = self._img[:, :, 1].astype('float32')

        self._kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
        self._processed = self._green.copy()

    @property
    def image(self):
        return self._img

    @property
    def active_channel(self):
        return self._green

    @property
    def processed(self):
        return self._processed

    def remove_light_reflex(self):
        return cv2.morphologyEx(self._processed, cv2.MORPH_OPEN, self._kernel)

    def shade_correct(self):
        gamma = self.remove_light_reflex()

        # mean filter
        self._processed = cv2.blur(self._processed, (3, 3))
        # Gaussian convolution
        self._processed = cv2.GaussianBlur(self._processed, (9, 9), 1.8)

        # further averaging
        # before mean blurring, set black regions of the image to the average gray value
        self._processed[self._mask == 0] = cv2.mean(self._processed, self._mask)[0]

        background = cv2.blur(self._processed, (69, 69))

        # distance matrix between background and the image
        dist = gamma - background
        minDist = np.min(dist)
        interval = np.max(dist) - minDist
        self._processed = np.around((dist - minDist) * 255. / interval)
        self._processed[self._mask == 0] = 0

        return self._processed

    def apply_morphology(self):
        for r in range(2, 12, 2):
            kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * r, 2 * r))
            self._processed = cv2.morphologyEx(self._processed, cv2.MORPH_OPEN, kernel)
            self._processed = cv2.morphologyEx(self._processed, cv2.MORPH_CLOSE, kernel)

        return self._processed

    def locate_disk(self):
        self.shade_correct()
        self.apply_morphology()

        pr = self._processed.copy().astype('uint8')

        # kill possible bright edges of the image
        pr[self._shifted == 0] = 0

        _, maxCol, _, ctr = cv2.minMaxLoc(pr)
        return ctr

    def show_detected(self, ctr):
        pr = self._processed.copy().astype('uint8')
        cv2.circle(pr, ctr, 50, 0, 3)
        cv2.circle(self.image, ctr, 50, (0, 0, 0), 3)
        show_images([self.image, pr])

    def rescale_to_original(self, pt):
        ctr_orig = np.float32(pt) * self._scale
        return (ctr_orig[0], ctr_orig[1])<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
oauthlib.oauth2.rfc6749.grant_types
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""
from __future__ import unicode_literals, absolute_import

import json
import logging

from .base import GrantTypeBase
from .. import errors, utils
from ..request_validator import RequestValidator

log = logging.getLogger(__name__)

class RefreshTokenGrant(GrantTypeBase):

    """`Refresh token grant`_

    .. _`Refresh token grant`: http://tools.ietf.org/html/rfc6749#section-6
    """

    @property
    def issue_new_refresh_tokens(self):
        return True

    def __init__(self, request_validator=None, issue_new_refresh_tokens=True):
        self.request_validator = request_validator or RequestValidator()

    def create_token_response(self, request, token_handler):
        """Create a new access token from a refresh_token.

        If valid and authorized, the authorization server issues an access
        token as described in `Section 5.1`_. If the request failed
        verification or is invalid, the authorization server returns an error
        response as described in `Section 5.2`_.

        The authorization server MAY issue a new refresh token, in which case
        the client MUST discard the old refresh token and replace it with the
        new refresh token. The authorization server MAY revoke the old
        refresh token after issuing a new refresh token to the client. If a
        new refresh token is issued, the refresh token scope MUST be
        identical to that of the refresh token included by the client in the
        request.

        .. _`Section 5.1`: http://tools.ietf.org/html/rfc6749#section-5.1
        .. _`Section 5.2`: http://tools.ietf.org/html/rfc6749#section-5.2
        """
        headers = {
            'Content-Type': 'application/json',
            'Cache-Control': 'no-store',
            'Pragma': 'no-cache',
        }
        try:
            log.debug('Validating refresh token request, %r.', request)
            self.validate_token_request(request)
        except errors.OAuth2Error as e:
            return headers, e.json, e.status_code

        token = token_handler.create_token(request,
                                           refresh_token=self.issue_new_refresh_tokens)
        log.debug('Issuing new token to client id %r (%r), %r.',
                  request.client_id, request.client, token)
        return headers, json.dumps(token), 200

    def validate_token_request(self, request):
        # REQUIRED. Value MUST be set to "refresh_token".
        if request.grant_type != 'refresh_token':
            raise errors.UnsupportedGrantTypeError(request=request)

        if request.refresh_token is None:
            raise errors.InvalidRequestError(
                description='Missing refresh token parameter.',
                request=request)

        # Because refresh tokens are typically long-lasting credentials used to
        # request additional access tokens, the refresh token is bound to the
        # client to which it was issued.  If the client type is confidential or
        # the client was issued client credentials (or assigned other
        # authentication requirements), the client MUST authenticate with the
        # authorization server as described in Section 3.2.1.
        # http://tools.ietf.org/html/rfc6749#section-3.2.1
        if self.request_validator.client_authentication_required(request):
            log.debug('Authenticating client, %r.', request)
            if not self.request_validator.authenticate_client(request):
                log.debug('Invalid client (%r), denying access.', request)
                raise errors.InvalidClientError(request=request)
        elif not self.request_validator.authenticate_client_id(request.client_id, request):
            log.debug('Client authentication failed, %r.', request)
            raise errors.InvalidClientError(request=request)

        # Ensure client is authorized use of this grant type
        self.validate_grant_type(request)

        # REQUIRED. The refresh token issued to the client.
        log.debug('Validating refresh token %s for client %r.',
                  request.refresh_token, request.client)
        if not self.request_validator.validate_refresh_token(
                request.refresh_token, request.client, request):
            log.debug('Invalid refresh token, %s, for client %r.',
                      request.refresh_token, request.client)
            raise errors.InvalidGrantError(request=request)

        original_scopes = utils.scope_to_list(
            self.request_validator.get_original_scopes(
                request.refresh_token, request))

        if request.scope:
            request.scopes = utils.scope_to_list(request.scope)
            if (not all((s in original_scopes for s in request.scopes))
                and not self.request_validator.is_within_original_scope(
                    request.scopes, request.refresh_token, request)):
                log.debug('Refresh token %s lack requested scopes, %r.',
                          request.refresh_token, request.scopes)
                raise errors.InvalidScopeError(request=request)
        else:
            request.scopes = original_scopes
<commit_msg>Set issue_new_refresh_tokens in RefreshTokenGrant.__init__<commit_after># -*- coding: utf-8 -*-
"""
oauthlib.oauth2.rfc6749.grant_types
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""
from __future__ import unicode_literals, absolute_import

import json
import logging

from .base import GrantTypeBase
from .. import errors, utils
from ..request_validator import RequestValidator

log = logging.getLogger(__name__)

class RefreshTokenGrant(GrantTypeBase):

    """`Refresh token grant`_

    .. _`Refresh token grant`: http://tools.ietf.org/html/rfc6749#section-6
    """

    def __init__(self, request_validator=None, issue_new_refresh_tokens=True):
        self.request_validator = request_validator or RequestValidator()
        self.issue_new_refresh_tokens = issue_new_refresh_tokens

    def create_token_response(self, request, token_handler):
        """Create a new access token from a refresh_token.

        If valid and authorized, the authorization server issues an access
        token as described in `Section 5.1`_. If the request failed
        verification or is invalid, the authorization server returns an error
        response as described in `Section 5.2`_.

        The authorization server MAY issue a new refresh token, in which case
        the client MUST discard the old refresh token and replace it with the
        new refresh token. The authorization server MAY revoke the old
        refresh token after issuing a new refresh token to the client. If a
        new refresh token is issued, the refresh token scope MUST be
        identical to that of the refresh token included by the client in the
        request.

        .. _`Section 5.1`: http://tools.ietf.org/html/rfc6749#section-5.1
        .. _`Section 5.2`: http://tools.ietf.org/html/rfc6749#section-5.2
        """
        headers = {
            'Content-Type': 'application/json',
            'Cache-Control': 'no-store',
            'Pragma': 'no-cache',
        }
        try:
            log.debug('Validating refresh token request, %r.', request)
            self.validate_token_request(request)
        except errors.OAuth2Error as e:
            return headers, e.json, e.status_code

        token = token_handler.create_token(request,
                                           refresh_token=self.issue_new_refresh_tokens)
        log.debug('Issuing new token to client id %r (%r), %r.',
                  request.client_id, request.client, token)
        return headers, json.dumps(token), 200

    def validate_token_request(self, request):
        # REQUIRED. Value MUST be set to "refresh_token".
        if request.grant_type != 'refresh_token':
            raise errors.UnsupportedGrantTypeError(request=request)

        if request.refresh_token is None:
            raise errors.InvalidRequestError(
                description='Missing refresh token parameter.',
                request=request)

        # Because refresh tokens are typically long-lasting credentials used to
        # request additional access tokens, the refresh token is bound to the
        # client to which it was issued.  If the client type is confidential or
        # the client was issued client credentials (or assigned other
        # authentication requirements), the client MUST authenticate with the
        # authorization server as described in Section 3.2.1.
        # http://tools.ietf.org/html/rfc6749#section-3.2.1
        if self.request_validator.client_authentication_required(request):
            log.debug('Authenticating client, %r.', request)
            if not self.request_validator.authenticate_client(request):
                log.debug('Invalid client (%r), denying access.', request)
                raise errors.InvalidClientError(request=request)
        elif not self.request_validator.authenticate_client_id(request.client_id, request):
            log.debug('Client authentication failed, %r.', request)
            raise errors.InvalidClientError(request=request)

        # Ensure client is authorized use of this grant type
        self.validate_grant_type(request)

        # REQUIRED. The refresh token issued to the client.
        log.debug('Validating refresh token %s for client %r.',
                  request.refresh_token, request.client)
        if not self.request_validator.validate_refresh_token(
                request.refresh_token, request.client, request):
            log.debug('Invalid refresh token, %s, for client %r.',
                      request.refresh_token, request.client)
            raise errors.InvalidGrantError(request=request)

        original_scopes = utils.scope_to_list(
            self.request_validator.get_original_scopes(
                request.refresh_token, request))

        if request.scope:
            request.scopes = utils.scope_to_list(request.scope)
            if (not all((s in original_scopes for s in request.scopes))
                and not self.request_validator.is_within_original_scope(
                    request.scopes, request.refresh_token, request)):
                log.debug('Refresh token %s lack requested scopes, %r.',
                          request.refresh_token, request.scopes)
                raise errors.InvalidScopeError(request=request)
        else:
            request.scopes = original_scopes
<|endoftext|>


<commit_before>from selenium import webdriver
import errno
import os
import requests
import argparse
import getpass
import menu
import logging
from progressbar import ProgressBar
# from selenium.webdriver.support.ui import Select # for <SELECT> HTML form

def make_fs_string(raw_string):
    return "_".join(raw_string.split(" ")).lower()

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def tum_login(driver, user, password=""):
    driver.get("https://www.moodle.tum.de/login/index.php")
    a = driver.find_element_by_link_text('TUM LOGIN')
    a.click()

    # Fill the login form and submit it
    driver.find_element_by_id('j_username').send_keys(user)
    if password:
        driver.find_element_by_id('j_password').send_keys(password)
    else:
        driver.find_element_by_id('j_password').send_keys(getpass.getpass("Please enter your password:"))
    driver.find_element_by_id('Login').submit()
    return driver

def lmu_login():
    pass

def get_session(driver):
    session = requests.Session()
    cookies = driver.get_cookies()
    for cookie in cookies:
        session.cookies.set(cookie['name'], cookie['value'])

def get_driver(driver_type):
    if driver_type == "phantomjs":
        # On Windows, use: webdriver.PhantomJS('C:\phantomjs-1.9.7-windows\phantomjs.exe')
        driver = webdriver.PhantomJS()
    elif driver_type == "chrome":
        driver = webdriver.Chrome()
    elif driver_type == "firefox":
        driver = webdriver.Firefox()

    return driver

# But actually only looking at non-hidden is a pain, so i just filter the list
# def set_semester(semester):
#     if semester == "all":
#         # Switch on to view all courses
#         driver.find_element_by_xpath("//select[@id='coc-filterterm']/option[@value='all']").click()
#         return True
#     else:
#         try:
#             el = driver.find_element_by_xpath("//select[@id='coc-filterterm']/option[@value=\'" + semester + "\']").click()
#         except:
#             print "ERROR: Semester", semester, "does not exist!"

def get_courses(driver, semester):
    courses = driver.find_elements_by_class_name('coc-course')
    course_list = []
    for course in courses:
        term = course.find_element_by_class_name("termdiv") \
                     .get_attribute('class').split(" ")[1].split("-")
        term = '-'.join(term[2:])
        # Easier for now to just filter by semester (if request),
        # than dealing with hidden-classes
        if term == semester or semester == "all":
            a = course.find_element_by_xpath(".//h3/a")
            title = "_".join(a.get_attribute('title').split(" ")).lower()
            course_list.append((term, title, a.get_attribute('href')))

    return course_list

def download_file(session, path, url):
    # NOTE the stream=True parameter
    r = session.get(url, stream=True)
    total_size = int(r.headers['Content-Length'])
    fn = os.path.join(path, r.url.split('/')[-1])
    with open(fn, 'wb') as f:
        pbar = ProgressBar(maxval=total_size).start()
        for idx, chunk in enumerate(r.iter_content(chunk_size=1024)):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                pbar.update(idx * 1024)
        pbar.finish()

def download_files(session, fn_base, files):
    # Download Files
    for file in files:
        print "Processing: ", file[1], file[2]

        # Create Folder if it doesn't exist yet
        path = os.path.join(fn_base, file[0], file[1])
        mkdir_p(path)

        # TODO CHECK IF FILE EXISTS ON FS BEFORE DOWNLOADING

        # Download File
        download_file(session, path, file[3])

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--fn_out', '-o', type=str, default='',
                        help='Output folder',
                        required=True)
    parser.add_argument('--user', '-u', type=str, default='',
                        help='Shibboleth User',
                        required=True)
    parser.add_argument('--pw', '-p', type=str, default='',
                        help='Dev param, password will be visible',
                        required=False)
    parser.add_argument('--semester', '-s', type=str, default='all',
                        required=False)
    parser.add_argument('--driver', '-d', type=str, default='phantomjs',
                        required=False)
    args = parser.parse_args()

# Obtain Driver
print " :: Obtaining Driver"
driver = get_driver(args.driver)

# We are now on the course pages
print " :: Login"
tum_login(driver, args.user, args.pw)

# print " :: Setting Semester to ", args.semester
# set_semester(args.semester)

# Get all courses and filter by semester/all
print " :: Getting Courses of", args.semester
courses = get_courses(driver, args.semester)
# print courses

print " :: Collecting File Information"

# now process all the courses

folder_queue = []
section_queue = []

for course in courses:
    print course[0], course[1]
    driver.get(course[2])

    try:
        # 2. Recurse through folder
        # TODO: Timeout problems?
        folders = driver.find_elements_by_class_name("modtype_folder")
        for folder in folders:
            folder_href = folder.find_element_by_xpath(".//a").get_attribute('href')
            folder_name = make_fs_string(file.find_element_by_class_name("instancename").text)
            folder_queue.append(((course[0], os.path.join(course[1], folder_name), folder_href)))
    except:
        print " :: :: Folder Search Faield!!!"

    # 3. Deal with sections (similar to folder)
    # try:
    #     # 2. Recurse through folder
    #     # TODO: Timeout problems?
    #     folders = driver.find_elements_by_class_name("section-summary")
    #     for folder in folders:
    #         link_el = folder.find_element_by_xpath(".//a")
    #         folder_href = link_el.get_attribute('href')
    #         folder_name = make_fs_string(link_el.text)
    #         section_queue.append(((course[0], os.path.join(course[1], folder_name), folder_href)))
    # except:
    #     print " :: :: Folder Search Faield!!!"

file_store = []
pages = courses + folder_queue + section_queue
for page in pages:
    # 1. Find single files
    driver.get(course[2])
    try:
        files = driver.find_elements_by_class_name("modtype_resource")
        for file in files:
            file_href = file.find_element_by_xpath(".//a").get_attribute('href')
            file_name = make_fs_string(file.find_element_by_class_name("instancename").text)
            file_store.append((course[0], course[1], file_name, file_href))
    except:
        print " :: :: No Files Found!"

print file_store
<commit_msg>refactor, deals with folders/sections and some odd cases now. checks if file exist/filesize identical<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: David Adrian
# david.adrian@tum.de

import errno
import os
import requests
import argparse
import getpass

from selenium import webdriver
from progressbar import ProgressBar
# from selenium.webdriver.support.ui import Select # for <SELECT> HTML form

def make_fs_string(raw_string):
    return "_".join(raw_string.split(" ")).lower()

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def tum_login(driver, user, password=""):
    driver.get("https://www.moodle.tum.de/login/index.php")
    a = driver.find_element_by_link_text('TUM LOGIN')
    a.click()

    # Fill the login form and submit it
    driver.find_element_by_id('j_username').send_keys(user)
    if password:
        driver.find_element_by_id('j_password').send_keys(password)
    else:
        driver.find_element_by_id('j_password').send_keys(getpass.getpass("Please enter your password:"))
    driver.find_element_by_id('Login').submit()
    return driver

def lmu_login():
    pass

def get_session(driver):
    session = requests.Session()
    cookies = driver.get_cookies()
    for cookie in cookies:
        session.cookies.set(cookie['name'], cookie['value'])
    return session

def get_driver(driver_type):
    if driver_type == "phantomjs":
        # On Windows, use: webdriver.PhantomJS('C:\phantomjs-1.9.7-windows\phantomjs.exe')
        driver = webdriver.PhantomJS()
    elif driver_type == "chrome":
        driver = webdriver.Chrome()
    elif driver_type == "firefox":
        driver = webdriver.Firefox()

    return driver

def get_courses(driver, semester):
    courses = driver.find_elements_by_class_name('coc-course')
    course_list = []
    for course in courses:
        term = course.find_element_by_class_name("termdiv") \
                     .get_attribute('class').split(" ")[1].split("-")
        term = '-'.join(term[2:])
        # Easier for now to just filter by semester (if request),
        # than dealing with hidden-classes
        if term == semester or semester == "all":
            a = course.find_element_by_xpath(".//h3/a")
            title = "_".join(a.get_attribute('title').split(" ")).lower()
            course_list.append((term, title, a.get_attribute('href')))

    return course_list

def download_files(session, fn_base, files, force):
    # Download Files
    for file in files:
        # Create Folder if it doesn't exist yet
        path = os.path.join(fn_base, file[0], file[1])
        mkdir_p(path)

        # Download File
        # NOTE the stream=True parameter
        # NOTE add the "&redirect=1", to deal with some odd balls
        r = session.get(file[2] + "&redirect=1", stream=True)
        fs = int(r.headers['Content-Length'])
        name = r.headers["Content-Disposition"].split('\"')[-2].decode('utf-8')
        fn = os.path.join(path, name)
        if not force and os.path.isfile(fn) and os.stat(fn).st_size == fs:
            print "   :: File already exists. Skipping", file[1], name, "!"
        else:
            print "   :: Downloading:", file[1], name
            with open(fn, 'wb') as f:
                pbar = ProgressBar(maxval=fs).start()
                for idx, chunk in enumerate(r.iter_content(chunk_size=1024)):
                    if chunk:  # filter out keep-alive new chunks
                        f.write(chunk)
                        pbar.update(idx * 1024)
                pbar.finish()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--fn_out', '-o', type=str, default='',
                        help='Output folder',
                        required=True)
    parser.add_argument('--user', '-u', type=str, default='',
                        help='Shibboleth User',
                        required=True)
    parser.add_argument('--pw', '-p', type=str, default='',
                        help='(OPTIONAL DEV) Shibboleth Password',
                        required=False)
    parser.add_argument('--semester', '-s', type=str, default='all',
                        help='Filter courses by semester',
                        required=False)
    parser.add_argument('--driver', '-d', type=str, default='phantomjs',
                        help='Phantomjs, Chrome, Firefox supported',
                        required=False)
    parser.add_argument('--force', '-f', dest='force', action='store_true',
                        help='Force re-downloads every file',
                        required=False)
    args = parser.parse_args()

# Obtain Driver
print " :: Obtaining Driver"
driver = get_driver(args.driver.lower())

# We are now on the course pages
print " :: Login User:", args.user
tum_login(driver, args.user, args.pw)

# print " :: Setting Semester to ", args.semester
# set_semester(args.semester)

# Get all courses and filter by semester/all
print "\n :: Getting Courses of", args.semester
courses = get_courses(driver, args.semester)
print "   :: Found:", len(courses), "courses!"

print "\n :: Collecting File Information"
# First get all special cases pre-processed
## TODO: Should be recursive just to make sure!
folder_queue = []
section_queue = []
for course in courses:
    print "   ::", course[0], course[1]
    driver.get(course[2])

    # 2. Get folders
    try:
        # TODO: Timeout problems?
        folders = driver.find_elements_by_class_name("modtype_folder")
        for folder in folders:
            folder_href = folder.find_element_by_xpath(".//a").get_attribute('href')
            folder_name = make_fs_string(folder.find_element_by_class_name("instancename").text)
            folder_queue.append(((course[0], os.path.join(course[1], folder_name), folder_href)))
    except:
        print "   :: Folder Search Failed!!!"

    # 3. Deal with sections (similar to folder)
    try:
        # 2. Recurse through folder
        # TODO: Timeout problems?
        folders = driver.find_elements_by_class_name("section-summary")
        for folder in folders:
            link_el = folder.find_element_by_xpath(".//a")
            folder_href = link_el.get_attribute('href')
            folder_name = make_fs_string(link_el.text)
            section_queue.append(((course[0], os.path.join(course[1], folder_name), folder_href)))
    except:
        print "   :: Folder Search Failed!!!"

# Now treat everything as page and look for files
parsed_files = []
pages = courses + folder_queue + section_queue
# pages = folder_queue
for page in pages:
    # 1. Find single files
    driver.get(page[2])
    try:
        files = driver.find_elements_by_class_name("modtype_resource") + driver.find_elements_by_class_name("fp-filename-icon")
        for file in files:
            try:
                file_href = file.find_element_by_xpath(".//a").get_attribute('href')
                parsed_files.append((page[0], page[1], file_href))
            except:
                # wasnt a file, only icon
                pass
    except:
        print "   :: No Files Found!"

# Download files if they don't exist.
print "\n :: Collected", len(parsed_files), "files"
print " :: Downloading files (please wait)"
download_files(get_session(driver), args.fn_out, parsed_files, args.force)<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os
import git_visitor_base
import git_repo_query
import subprocess

def derive_result(out_str):
    print("mvdebug [%s]" % out_str)
    return "mvtodo"

def visitor_push(repos):

    report = []
    for rp in repos:
        print("\n* Pushing to %s ..." % rp)
        remotes = git_repo_query.get_remotes(rp)
        for rm in remotes:

            try:
                out = subprocess.check_output(["git", "--git-dir=%s" % rp, "--work-tree=%s" % os.path.dirname(rp), "push", rm])
            except OSError as oser:
                out = oser.strerror
            except subprocess.CalledProcessError as cper:
                out = cper.strerror

            res = derive_result(out)
            bn = "mvtodo"
            report.append("%s (remote=%s, branch=%s): %s" % (rp, rm, bn, res))

    for p in report:
        print(p)

if __name__ == "__main__":
    git_visitor_base.do_visit(sys.argv, visitor_push)

<commit_msg>git visitor push now at RC state. it works well.<commit_after>#!/usr/bin/env python

import sys
import os
import subprocess

import git_visitor_base
import git_repo_query

def visitor_push(repos):

    report = []
    for rp in repos:
        print("\n* Pushing to %s ..." % rp)
        remotes = git_repo_query.get_remotes(rp)
        for rm in remotes:

            try:
                out = subprocess.check_output(["git", "--git-dir=%s" % rp, "--work-tree=%s" % os.path.dirname(rp), "push", rm])
                out = "OK."
            except OSError as oser:
                out = "Failed."
            except subprocess.CalledProcessError as cper:
                out = "Failed."

            bn = "mvtodo"
            report.append("%s (remote=%s, branch=%s): %s" % (rp, rm, bn, out))

    print("RESULTS:\n")
    for p in report:
        print(p)

if __name__ == "__main__":
    git_visitor_base.do_visit(sys.argv, visitor_push)

<|endoftext|>


<commit_before>import inspect
from django.conf import settings

if "guardian" in settings.INSTALLED_APPS:
    from guardian.admin import GuardedModelAdmin as ModelAdmin
else:
    from django.contrib.admin import ModelAdmin

from django.contrib import admin
#The following not work with guardian?
#import xadmin as admin

def register_normal_admin(admin_class, class_inst):
    # admin.site.register(class_inst)
    try:
        from normal_admin.admin import user_admin_site
        user_admin_site.register(class_inst, admin_class)
    except:
        pass
        #register(class_inst)

def get_valid_admin_class(admin_class, class_inst):
    if admin_class is None:
        admin_class = type(class_inst.__name__ + "Admin", (ModelAdmin, ), {})
    return admin_class

def register_admin(admin_class, class_inst):
    try:
        admin.site.register(class_inst, admin_class)
    except:
        pass

def register_to_sys(class_inst, admin_class=None):
    admin_class = get_valid_admin_class(admin_class, class_inst)
    register_admin(admin_class, class_inst)
    register_normal_admin(admin_class, class_inst)

def register_all(class_list):
    for i in class_list:
        register_to_sys(i)

def register_all_in_module(module_instance, exclude_name=[]):
    class_list = []
    for name, obj in inspect.getmembers(module_instance):
        if inspect.isclass(obj):
            if obj.__name__ in exclude_name:
                continue
            class_list.append(obj)
    register_all(class_list)<commit_msg>Added admin parent list support.<commit_after>import inspect
from django.conf import settings

if "guardian" in settings.INSTALLED_APPS:
    from guardian.admin import GuardedModelAdmin as SingleModelAdmin
else:
    from django.contrib.admin import ModelAdmin as SingleModelAdmin

from django.contrib.admin import ModelAdmin
from django.contrib import admin
#The following not work with guardian?
#import xadmin as admin

def register_normal_admin(admin_class, class_inst):
    # admin.site.register(class_inst)
    try:
        from normal_admin.admin import user_admin_site
        user_admin_site.register(class_inst, admin_class)
    except:
        pass
        #register(class_inst)

def get_valid_admin_class(admin_class, class_inst):
    if admin_class is None:
        admin_class = type(class_inst.__name__ + "Admin", (SingleModelAdmin, ), {})
    return admin_class

def register_admin(admin_class, class_inst):
    try:
        admin.site.register(class_inst, admin_class)
    except:
        pass

def register_to_sys(class_inst, admin_class=None):
    admin_class = get_valid_admin_class(admin_class, class_inst)
    register_admin(admin_class, class_inst)
    register_normal_admin(admin_class, class_inst)

def get_valid_admin_class_with_list(admin_list, class_inst):
    final_parents = admin_list.append(ModelAdmin)
    admin_class = type(class_inst.__name__ + "Admin", set(final_parents), {})
    return admin_class

def register_to_sys_with_admin_list(class_inst, admin_list=[]):
    admin_class = get_valid_admin_class_with_list(admin_list, class_inst)
    register_admin(admin_class, class_inst)
    register_normal_admin(admin_class, class_inst)

def register_all(class_list, admin_class_list=[]):
    for i in class_list:
        register_to_sys_with_admin_list(i, admin_class_list)

def register_all_in_module(module_instance, exclude_name=[], admin_class_list=[]):
    class_list = []
    for name, obj in inspect.getmembers(module_instance):
        if inspect.isclass(obj):
            if obj.__name__ in exclude_name:
                continue
            class_list.append(obj)
    register_all(class_list, admin_class_list)<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pytest

import numpy as np

from test.zoo.pipeline.utils.test_utils import ZooTestCase
import zoo.pipeline.api.keras.layers as ZLayer
from zoo.pipeline.api.keras.models import Model as ZModel, Sequential as ZSequential
import keras.layers as KLayer
from keras.engine import merge as kmerge, Model as KModel
from keras.models import Sequential as KSequential
import keras.backend as K
from bigdl.keras.converter import WeightsConverter
from zoo.pipeline.api.keras import regularizers

np.random.seed(1337)  # for reproducibility

class TestLayer(ZooTestCase):

    def test_embedding(self):
        input_data = np.random.randint(1000, size=(32, 10))
        zlayer = ZLayer.Embedding(1000, 64, input_shape=(10, ))
        klayer = KLayer.Embedding(1000, 64, input_length=10)
        self.compare_layer(klayer, zlayer, input_data,
                           WeightsConverter.convert_embedding)

    def test_batchnormalization(self):
        print("Running batch normal test")
        K.set_image_dim_ordering("th")
        input_data = np.random.random_sample([2, 5, 32, 32])
        zlayer = ZLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32))
        klayer = KLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32))
        self.compare_layer(klayer, zlayer, input_data,
                           WeightsConverter.convert_batchnormalization)
        K.set_image_dim_ordering("tf")
        input_data2 = np.random.random_sample([2, 32, 32, 4])
        zlayer = ZLayer.BatchNormalization(axis=-1, dim_ordering="tf", input_shape=(32, 32, 4))
        klayer = KLayer.BatchNormalization(axis=-1, input_shape=(32, 32, 4))
        self.compare_layer(klayer, zlayer, input_data2,
                           WeightsConverter.convert_batchnormalization)
        K.set_image_dim_ordering("th")
        input_data = np.random.random_sample([2, 5])
        zlayer = ZLayer.BatchNormalization(axis=1, input_shape=(5,))
        klayer = KLayer.BatchNormalization(axis=1, input_shape=(5,))
        self.compare_layer(klayer, zlayer, input_data,
                           WeightsConverter.convert_batchnormalization)

    def test_merge_sum(self):
        z1 = ZLayer.InputLayer(input_shape=(3, 5))
        z2 = ZLayer.InputLayer(input_shape=(3, 5))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="sum")
        k1 = KLayer.InputLayer(input_shape=(3, 5))
        k2 = KLayer.InputLayer(input_shape=(3, 5))
        klayer = KLayer.Merge(layers=[k1, k2], mode="sum")
        input_data = [np.random.random([2, 3, 5]), np.random.random([2, 3, 5])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_mul(self):
        z1 = ZLayer.InputLayer(input_shape=(3, 5))
        z2 = ZLayer.InputLayer(input_shape=(3, 5))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="mul")
        k1 = KLayer.InputLayer(input_shape=(3, 5))
        k2 = KLayer.InputLayer(input_shape=(3, 5))
        klayer = KLayer.Merge(layers=[k1, k2], mode="mul")
        input_data = [np.random.random([2, 3, 5]), np.random.random([2, 3, 5])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_ave(self):
        z1 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        z2 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="ave")
        k1 = KLayer.InputLayer(input_shape=(2, 5, 8))
        k2 = KLayer.InputLayer(input_shape=(2, 5, 8))
        klayer = KLayer.Merge(layers=[k1, k2], mode="ave")
        input_data = [np.random.random([3, 2, 5, 8]), np.random.random([3, 2, 5, 8])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_max(self):
        z1 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        z2 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="max")
        k1 = KLayer.InputLayer(input_shape=(2, 5, 8))
        k2 = KLayer.InputLayer(input_shape=(2, 5, 8))
        klayer = KLayer.Merge(layers=[k1, k2], mode="max")
        input_data = [np.random.random([3, 2, 5, 8]), np.random.random([3, 2, 5, 8])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_concat(self):
        z1 = ZLayer.InputLayer(input_shape=(2, 5, 11))
        z2 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="concat")
        k1 = KLayer.InputLayer(input_shape=(2, 5, 11))
        k2 = KLayer.InputLayer(input_shape=(2, 5, 8))
        klayer = KLayer.Merge(layers=[k1, k2], mode="concat")
        input_data = [np.random.random([3, 2, 5, 11]), np.random.random([3, 2, 5, 8])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_dot(self):
        z1 = ZLayer.InputLayer(input_shape=(4, ))
        z2 = ZLayer.InputLayer(input_shape=(4, ))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="dot")
        k1 = KLayer.InputLayer(input_shape=(4, ))
        k2 = KLayer.InputLayer(input_shape=(4, ))
        klayer = KLayer.Merge(layers=[k1, k2], mode="dot")
        input_data = [np.random.random([2, 4]), np.random.random([2, 4])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_cos(self):
        z1 = ZLayer.InputLayer(input_shape=(3, ))
        z2 = ZLayer.InputLayer(input_shape=(3, ))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="cos")
        k1 = KLayer.InputLayer(input_shape=(3, ))
        k2 = KLayer.InputLayer(input_shape=(3, ))
        klayer = KLayer.Merge(layers=[k1, k2], mode="cos")
        input_data = [np.random.random([2, 3]), np.random.random([2, 3])]
        self.compare_layer(klayer, zlayer, input_data)

    def convert_two_dense(self, kmodel, weights):
        return [weights[2].T, weights[3], weights[0].T, weights[1]]

    def test_merge_method_sum(self):
        zx1 = ZLayer.Input(shape=(8, ))
        zx2 = ZLayer.Input(shape=(6, ))
        zy1 = ZLayer.Dense(10)(zx1)
        zy2 = ZLayer.Dense(10)(zx2)
        zz = ZLayer.merge([zy1, zy2], mode="sum")
        zmodel = ZModel([zx1, zx2], zz, name="graph1")

        kx1 = KLayer.Input(shape=(8, ))
        kx2 = KLayer.Input(shape=(6, ))
        ky1 = KLayer.Dense(10)(kx1)
        ky2 = KLayer.Dense(10)(kx2)
        kz = kmerge([ky1, ky2], mode="sum")
        kmodel = KModel([kx1, kx2], kz)

        input_data = [np.random.random([2, 8]), np.random.random([2, 6])]
        self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)

    def test_merge_method_model_concat(self):
        zx1 = ZLayer.Input(shape=(4, ))
        zx2 = ZLayer.Input(shape=(5, ))
        zy1 = ZLayer.Dense(6, activation="sigmoid")(zx1)
        zbranch1 = ZModel(zx1, zy1)(zx1)
        zbranch2 = ZLayer.Dense(8)(zx2)
        zz = ZLayer.merge([zbranch1, zbranch2], mode="concat")
        zmodel = ZModel([zx1, zx2], zz)

        kx1 = KLayer.Input(shape=(4, ))
        kx2 = KLayer.Input(shape=(5, ))
        ky1 = KLayer.Dense(6, activation="sigmoid")(kx1)
        kbranch1 = KModel(kx1, ky1)(kx1)
        kbranch2 = KLayer.Dense(8)(kx2)
        kz = KLayer.merge([kbranch1, kbranch2], mode="concat")
        kmodel = KModel([kx1, kx2], kz)

        input_data = [np.random.random([2, 4]), np.random.random([2, 5])]
        self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)

    def test_merge_method_seq_concat(self):
        zx1 = ZLayer.Input(shape=(10, ))
        zx2 = ZLayer.Input(shape=(10, ))
        zy1 = ZLayer.Dense(12, activation="sigmoid")(zx1)
        zbranch1_node = ZModel(zx1, zy1)(zx1)
        zbranch2 = ZSequential()
        zbranch2.add(ZLayer.Dense(12, input_dim=10))
        zbranch2_node = zbranch2(zx2)
        zz = ZLayer.merge([zbranch1_node, zbranch2_node], mode="concat")
        zmodel = ZModel([zx1, zx2], zz)

        kx1 = KLayer.Input(shape=(10, ))
        kx2 = KLayer.Input(shape=(10, ))
        ky1 = KLayer.Dense(12, activation="sigmoid")(kx1)
        kbranch1_node = KModel(kx1, ky1)(kx1)
        kbranch2 = KSequential()
        kbranch2.add(KLayer.Dense(12, input_dim=10))
        kbranch2_node = kbranch2(kx2)
        kz = KLayer.merge([kbranch1_node, kbranch2_node], mode="concat")
        kmodel = KModel([kx1, kx2], kz)

        input_data = [np.random.random([2, 10]), np.random.random([2, 10])]
        self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)

    def test_reshape(self):
        a = np.random.random((2, 2, 3, 4))
        i1 = ZLayer.Input(shape=(2, 3, 4))
        s = ZLayer.Reshape((-1, 2, 12))(i1)
        m = ZModel(i1, s)
        # predict should not generate exception
        y = m.predict(a, distributed=False)

    def test_regularizer(self):
        model = ZSequential()
        model.add(ZLayer.Dense(16, W_regularizer=regularizers.l2(0.001),
                               activation='relu', input_shape=(10000,)))
        model.summary()
        model.compile(optimizer='rmsprop',
                      loss='binary_crossentropy',
                      metrics=['acc'])

if __name__ == "__main__":
    pytest.main([__file__])
<commit_msg>Add Bert python (#1268)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pytest

import numpy as np

from test.zoo.pipeline.utils.test_utils import ZooTestCase
import zoo.pipeline.api.keras.layers as ZLayer
from zoo.pipeline.api.keras.models import Model as ZModel, Sequential as ZSequential
import keras.layers as KLayer
from keras.engine import merge as kmerge, Model as KModel
from keras.models import Sequential as KSequential
import keras.backend as K
from bigdl.keras.converter import WeightsConverter
from zoo.pipeline.api.keras import regularizers

np.random.seed(1337)  # for reproducibility

class TestLayer(ZooTestCase):

    def test_embedding(self):
        input_data = np.random.randint(1000, size=(32, 10))
        zlayer = ZLayer.Embedding(1000, 64, input_shape=(10, ))
        klayer = KLayer.Embedding(1000, 64, input_length=10)
        self.compare_layer(klayer, zlayer, input_data,
                           WeightsConverter.convert_embedding)

    def test_batchnormalization(self):
        print("Running batch normal test")
        K.set_image_dim_ordering("th")
        input_data = np.random.random_sample([2, 5, 32, 32])
        zlayer = ZLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32))
        klayer = KLayer.BatchNormalization(axis=1, input_shape=(5, 32, 32))
        self.compare_layer(klayer, zlayer, input_data,
                           WeightsConverter.convert_batchnormalization)
        K.set_image_dim_ordering("tf")
        input_data2 = np.random.random_sample([2, 32, 32, 4])
        zlayer = ZLayer.BatchNormalization(axis=-1, dim_ordering="tf", input_shape=(32, 32, 4))
        klayer = KLayer.BatchNormalization(axis=-1, input_shape=(32, 32, 4))
        self.compare_layer(klayer, zlayer, input_data2,
                           WeightsConverter.convert_batchnormalization)
        K.set_image_dim_ordering("th")
        input_data = np.random.random_sample([2, 5])
        zlayer = ZLayer.BatchNormalization(axis=1, input_shape=(5,))
        klayer = KLayer.BatchNormalization(axis=1, input_shape=(5,))
        self.compare_layer(klayer, zlayer, input_data,
                           WeightsConverter.convert_batchnormalization)

    def test_merge_sum(self):
        z1 = ZLayer.InputLayer(input_shape=(3, 5))
        z2 = ZLayer.InputLayer(input_shape=(3, 5))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="sum")
        k1 = KLayer.InputLayer(input_shape=(3, 5))
        k2 = KLayer.InputLayer(input_shape=(3, 5))
        klayer = KLayer.Merge(layers=[k1, k2], mode="sum")
        input_data = [np.random.random([2, 3, 5]), np.random.random([2, 3, 5])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_mul(self):
        z1 = ZLayer.InputLayer(input_shape=(3, 5))
        z2 = ZLayer.InputLayer(input_shape=(3, 5))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="mul")
        k1 = KLayer.InputLayer(input_shape=(3, 5))
        k2 = KLayer.InputLayer(input_shape=(3, 5))
        klayer = KLayer.Merge(layers=[k1, k2], mode="mul")
        input_data = [np.random.random([2, 3, 5]), np.random.random([2, 3, 5])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_ave(self):
        z1 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        z2 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="ave")
        k1 = KLayer.InputLayer(input_shape=(2, 5, 8))
        k2 = KLayer.InputLayer(input_shape=(2, 5, 8))
        klayer = KLayer.Merge(layers=[k1, k2], mode="ave")
        input_data = [np.random.random([3, 2, 5, 8]), np.random.random([3, 2, 5, 8])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_max(self):
        z1 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        z2 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="max")
        k1 = KLayer.InputLayer(input_shape=(2, 5, 8))
        k2 = KLayer.InputLayer(input_shape=(2, 5, 8))
        klayer = KLayer.Merge(layers=[k1, k2], mode="max")
        input_data = [np.random.random([3, 2, 5, 8]), np.random.random([3, 2, 5, 8])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_concat(self):
        z1 = ZLayer.InputLayer(input_shape=(2, 5, 11))
        z2 = ZLayer.InputLayer(input_shape=(2, 5, 8))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="concat")
        k1 = KLayer.InputLayer(input_shape=(2, 5, 11))
        k2 = KLayer.InputLayer(input_shape=(2, 5, 8))
        klayer = KLayer.Merge(layers=[k1, k2], mode="concat")
        input_data = [np.random.random([3, 2, 5, 11]), np.random.random([3, 2, 5, 8])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_dot(self):
        z1 = ZLayer.InputLayer(input_shape=(4, ))
        z2 = ZLayer.InputLayer(input_shape=(4, ))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="dot")
        k1 = KLayer.InputLayer(input_shape=(4, ))
        k2 = KLayer.InputLayer(input_shape=(4, ))
        klayer = KLayer.Merge(layers=[k1, k2], mode="dot")
        input_data = [np.random.random([2, 4]), np.random.random([2, 4])]
        self.compare_layer(klayer, zlayer, input_data)

    def test_merge_cos(self):
        z1 = ZLayer.InputLayer(input_shape=(3, ))
        z2 = ZLayer.InputLayer(input_shape=(3, ))
        zlayer = ZLayer.Merge(layers=[z1, z2], mode="cos")
        k1 = KLayer.InputLayer(input_shape=(3, ))
        k2 = KLayer.InputLayer(input_shape=(3, ))
        klayer = KLayer.Merge(layers=[k1, k2], mode="cos")
        input_data = [np.random.random([2, 3]), np.random.random([2, 3])]
        self.compare_layer(klayer, zlayer, input_data)

    def convert_two_dense(self, kmodel, weights):
        return [weights[2].T, weights[3], weights[0].T, weights[1]]

    def test_merge_method_sum(self):
        zx1 = ZLayer.Input(shape=(8, ))
        zx2 = ZLayer.Input(shape=(6, ))
        zy1 = ZLayer.Dense(10)(zx1)
        zy2 = ZLayer.Dense(10)(zx2)
        zz = ZLayer.merge([zy1, zy2], mode="sum")
        zmodel = ZModel([zx1, zx2], zz, name="graph1")

        kx1 = KLayer.Input(shape=(8, ))
        kx2 = KLayer.Input(shape=(6, ))
        ky1 = KLayer.Dense(10)(kx1)
        ky2 = KLayer.Dense(10)(kx2)
        kz = kmerge([ky1, ky2], mode="sum")
        kmodel = KModel([kx1, kx2], kz)

        input_data = [np.random.random([2, 8]), np.random.random([2, 6])]
        self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)

    def test_merge_method_model_concat(self):
        zx1 = ZLayer.Input(shape=(4, ))
        zx2 = ZLayer.Input(shape=(5, ))
        zy1 = ZLayer.Dense(6, activation="sigmoid")(zx1)
        zbranch1 = ZModel(zx1, zy1)(zx1)
        zbranch2 = ZLayer.Dense(8)(zx2)
        zz = ZLayer.merge([zbranch1, zbranch2], mode="concat")
        zmodel = ZModel([zx1, zx2], zz)

        kx1 = KLayer.Input(shape=(4, ))
        kx2 = KLayer.Input(shape=(5, ))
        ky1 = KLayer.Dense(6, activation="sigmoid")(kx1)
        kbranch1 = KModel(kx1, ky1)(kx1)
        kbranch2 = KLayer.Dense(8)(kx2)
        kz = KLayer.merge([kbranch1, kbranch2], mode="concat")
        kmodel = KModel([kx1, kx2], kz)

        input_data = [np.random.random([2, 4]), np.random.random([2, 5])]
        self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)

    def test_merge_method_seq_concat(self):
        zx1 = ZLayer.Input(shape=(10, ))
        zx2 = ZLayer.Input(shape=(10, ))
        zy1 = ZLayer.Dense(12, activation="sigmoid")(zx1)
        zbranch1_node = ZModel(zx1, zy1)(zx1)
        zbranch2 = ZSequential()
        zbranch2.add(ZLayer.Dense(12, input_dim=10))
        zbranch2_node = zbranch2(zx2)
        zz = ZLayer.merge([zbranch1_node, zbranch2_node], mode="concat")
        zmodel = ZModel([zx1, zx2], zz)

        kx1 = KLayer.Input(shape=(10, ))
        kx2 = KLayer.Input(shape=(10, ))
        ky1 = KLayer.Dense(12, activation="sigmoid")(kx1)
        kbranch1_node = KModel(kx1, ky1)(kx1)
        kbranch2 = KSequential()
        kbranch2.add(KLayer.Dense(12, input_dim=10))
        kbranch2_node = kbranch2(kx2)
        kz = KLayer.merge([kbranch1_node, kbranch2_node], mode="concat")
        kmodel = KModel([kx1, kx2], kz)

        input_data = [np.random.random([2, 10]), np.random.random([2, 10])]
        self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)

    def test_reshape(self):
        a = np.random.random((2, 2, 3, 4))
        i1 = ZLayer.Input(shape=(2, 3, 4))
        s = ZLayer.Reshape((-1, 2, 12))(i1)
        m = ZModel(i1, s)
        # predict should not generate exception
        y = m.predict(a, distributed=False)

    def test_regularizer(self):
        model = ZSequential()
        model.add(ZLayer.Dense(16, W_regularizer=regularizers.l2(0.001),
                               activation='relu', input_shape=(10000,)))
        model.summary()
        model.compile(optimizer='rmsprop',
                      loss='binary_crossentropy',
                      metrics=['acc'])

    def test_transformer_forward_backward(self):
        layer = ZLayer.TransformerLayer.init(
            vocab=200, hidden_size=128, n_head=4, seq_len=20)

        train_token = np.random.randint(20, size=(2, 20))
        train_pos = np.zeros((2, 20), dtype=np.int32)
        input = [train_token, train_pos]
        self.assert_forward_backward(layer, input)

    def test_bert_forward_backward(self):
        layer = ZLayer.BERT.init(
            vocab=200, hidden_size=128, n_head=4, seq_len=20, intermediate_size=20)

        train_token = np.random.randint(20, size=(2, 20))
        token_type_id = np.zeros((2, 20), dtype=np.int32)
        train_pos = np.zeros((2, 20), dtype=np.int32)
        mask_attention = np.ones((2, 1, 1, 20), dtype=np.int32)
        input = [train_token, token_type_id, train_pos, mask_attention]
        self.assert_forward_backward(layer, input)

if __name__ == "__main__":
    pytest.main([__file__])
<|endoftext|>


<commit_before>#!/usr/bin/python

import argparse
import sys
import re
import os
import traceback
from operator import itemgetter, attrgetter
from biokbase.probabilistic_annotation.Client import _read_inifile
from biokbase.workspaceService.Client import *
from biokbase.fbaModelServices.Client import *
from biokbase.cdmi.client import CDMI_API, CDMI_EntityAPI

desc = '''
Given a JSON object calculate the false positive, negative rates. Optionally omit everything that isn't
in the specified media.
'''
separator ='///'
rolesToRxnsFileName = 'roles_to_reactions'

if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog='AnalyzePhenotypeSimulationResults.py', description=desc)
    parser.add_argument('genome', help='ID of genome to analyze', action='store', default=None)
    parser.add_argument('model', help='ID of integrated gap filled model to analyze', action='store', default=None)
    parser.add_argument('phenosimset', help='ID of PhenotypeSimulationSet object to analyze', action='store', default=None)
    parser.add_argument('workspace', help='ID of workspace containing objects to analyze', action='store', default=None)
    parser.add_argument('--solution', help='index number of solution in integrated gap filled model', action='store', default='0')
    parser.add_argument('--phenosimsetws', help='workspace containing PhenotypeSimulationSet object (same as workspace if not specified)', action='store', default=None)
    parser.add_argument('--media', help='limit analysis to only this media condition', action='store', dest='media', default=None)
    parser.add_argument('--probanno', help='ID of ProbAnno object for genome', action='store', dest='probanno', default=None)
    parser.add_argument('--probannows', help='workspace containing ProbAnno object (same as workspace if not specified)', action='store', dest='probannows', default=None)
    parser.add_argument('--rxnprobs', help='ID of RxnProbs object for genome', action='store', dest='rxnprobs', default=None)
    parser.add_argument('--rxnprobsws', help='workspace containing RxnProbs object (same as workspace if not specified)', action='store', dest='rxnprobsws', default=None)
    parser.add_argument('--script-dir', help='path to directory with analysis scripts', action='store', dest='scriptDir', default='.')
    parser.add_argument('--fba-url', help='url for fba modeling service', action='store', dest='fbaurl', default='http://bio-data-1.mcs.anl.gov/services/fba')
    parser.add_argument('--ws-url', help='url for workspace service', action='store', dest='wsurl', default='http://www.kbase.us/services/workspace/')
    args = parser.parse_args()

    if args.probanno is None:
        args.probanno = args.genome + '.probanno'
    if args.probannows is None:
        args.probannows = args.workspace
    if args.rxnprobs is None:
        args.rxnprobs = args.genome + '.rxnprobs'
    if args.rxnprobsws is None:
        args.rxnprobsws = args.workspace
    if args.phenosimsetws is None:
        args.phenosimsetws = args.workspace

    # Create the clients for the services we need.
    authdata = _read_inifile()
    token = authdata['token']
    wsClient = workspaceService(args.wsurl)

    # Build roles to reactions dictionary.
    step = 1
    print "+++ Step %d: Build reactions to roles dictionary +++" %(step)
    if not os.path.exists(rolesToRxnsFileName):
        os.system("all_roles_used_in_models | roles_to_complexes | get_relationship_HasStep -to id >%s" %(rolesToRxnsFileName))
    rolesToReactions = dict()
    reactionsToRoles = dict()
    # Each line of the file has four fields: (1) role, (2) hypothetical flag, (3) complex id, (4) reaction id
    rolesToRxnsFile = open(rolesToRxnsFileName, 'r')
    for line in rolesToRxnsFile:
        fields = line.strip('\r\n').split('\t')
        reaction = fields[3]
        if reaction not in reactionsToRoles:
            reactionsToRoles[reaction] = list()
        reactionsToRoles[reaction].append(fields[0])
    rolesToRxnsFile.close()
    print "  %d reactions in reactions to roles dictionary" %(len(reactionsToRoles))

    # Analyze the gap fill results for the specified model.
    step += 1
    print "+++ Step %d: Run AnalyzeGapfillResults.py for model '%s/%s' +++" %(step, args.workspace, args.model)
    rxnprobs = args.rxnprobs
    resultsFileName = args.model + '.results'
    os.system("%s/AnalyzeGapfillResults.py -m %s -w %s --rxnprobs %s --url %s >%s" \
              %(args.scriptDir, args.model, args.workspace, rxnprobs, args.fbaurl, resultsFileName))
    genesToReactions = dict()
    resultsFile = open(resultsFileName, 'r')
    resultsFile.readline() # Throw away header line
    for line in resultsFile:
        fields = line.strip('\r\n').split('\t')
        if fields[0] == args.solution:
            if fields[5] == '0': # Only keep reactions that are not a reversibility change
                if fields[6]: # Only keep reactions that have a GPR
                    rxnid = re.sub(r'rxn0*(\d+)', r'kb|rxn.\1', fields[1])
                    geneList = re.findall('fig\|\d+\.\d+\.peg\.\d+', fields[6])
                    for index in range(len(geneList)):
                        if geneList[index] not in genesToReactions:
                            genesToReactions[geneList[index]] = dict()
                        genesToReactions[geneList[index]][rxnid] = 0.0
    resultsFile.close()
    print "  %d genes in genes to reactions dictionary" %(len(genesToReactions))

    # Get the ProbAnno object from the workspace.
    step += 1
    probanno = args.probanno
    print "+++ Step %d: Get ProbAnno object '%s/%s'" %(step, args.probannows, probanno)
    paObject = wsClient.get_object( { 'id': probanno, 'workspace': args.probannows, 'type': 'ProbAnno', 'auth': token } )
    probAnno = paObject['data']
    print "  %d genes in ProbAnno roleset probabilities dictionary" %(len(probAnno['roleset_probabilities']))

    # Need to go through rolesets dictionary
    # If an entry has more than one role, split into parts
    # Then run the array and combine any duplicate roles by adding probs
    # Parse the roleset probabilities from the ProbAnno object.
    step += 1
    print "+++ Step %d: Parse rolesets into roles and adjust probabilities for duplicates +++" %(step)
    rolesetProbabilities = dict()
    for gene in probAnno['roleset_probabilities']:
        geneRoleList = probAnno['roleset_probabilities'][gene]
        geneRoleDict = dict()
        for index in range(len(geneRoleList)):
            prob = geneRoleList[index][1] # Probability for this roleset
            # Split multiple roles in roleset for this gene
            roleList = geneRoleList[index][0].split(separator)
            # If role occurs more than once, add up the probabilities
            for j in range(len(roleList)):
                if roleList[j] in geneRoleDict:
                    geneRoleDict[roleList[j]] += prob
                else:
                    geneRoleDict[roleList[j]] = prob
        rolesetProbabilities[gene] = geneRoleDict
    print "  %d genes in parsed roleset probabilities dictionary" %(len(rolesetProbabilities))

    # for each reaction in the reactions dictionary, find the roles in the rolesToReactions dictionary
    # then find the roles in the probanno object for the gene
    step += 1
    print "+++ Step %d: Find maximum probability of reaction given gene +++" %(step)
    probsFile = open(args.genome+'.probs', 'w')
    numProbs = 0
    for gene in genesToReactions:
        if gene in rolesetProbabilities:
            geneRoleDict = rolesetProbabilities[gene]
            for reaction in genesToReactions[gene]:
                if reaction not in reactionsToRoles:
                    print 'Why is reaction %s not in the reactionToRoles dictionary?' %(reaction)
                roleList = reactionsToRoles[reaction]
                for index in range(0,len(roleList)):
                    for role in geneRoleDict:
                        if role == roleList[index]:
                            probsFile.write('P(%s | %s) = %f\n' %(reaction, gene, geneRoleDict[role]))
                            genesToReactions[gene][reaction] = max(geneRoleDict[role], genesToReactions[gene][reaction])
                            numProbs += 1
        else:
            print 'Gene %s not found in ProbAnno object' %(gene)
    probsFile.close()
    print "  %d reaction probabilities set in genesToReactions dictionary" %(numProbs)

    # Get the PhenotypeSimulationSet object.
    step += 1
    print "+++ Step %d: Get phenotype simulation set %s/%s +++" %(step, args.phenosimsetws, args.phenosimset)
    pssObject = wsClient.get_object( { 'id': args.phenosimset, 'workspace': args.phenosimsetws, 'type': 'PhenotypeSimulationSet', 'auth': token } )
    phenoSimSet = pssObject['data']

    # Make sure the model matches in the phenotype simulation set.
    if phenoSimSet['model'] != args.model or phenoSimSet['model_workspace'] != args.workspace:
        print 'Specified model %s/%s does not match model %s/%s in phenotype simulation set' \
            %(args.workspace, args.model, phenoSimSet['model_workspace'], phenoSimSet['model'])
    print "  %d simulations in phenotype simulation set" %(len(phenoSimSet['phenotypeSimulations']))

    # Go through the list of simulations, for each gene and see if the gene is in the genesToReactions
    # dictionary.  If so, see if the gene was a lethal or non-lethal knockout and add it to the
    # corresponding dictionary along with reactions and
    lethalList = list()
    nonlethalList = list()
    for sim in phenoSimSet['phenotypeSimulations']:
        if sim[3] == 'CP' or sim[3] == 'CN':
            right = 1
        else:
            right = 0
        geneList = sim[0][0]
        if sim[1] > 0:
            for gene in geneList:
                if gene in genesToReactions:
                    for reaction in genesToReactions[gene]:
                        nonlethalList.append( (gene, reaction, genesToReactions[gene][reaction], right ) )
        else:
            for gene in geneList:
                if gene in genesToReactions:
                    for reaction in genesToReactions[gene]:
                        lethalList.append( (gene, reaction, genesToReactions[gene][reaction], right ) )

    # Sort the lists by reaction probability.
    lethalList.sort(key=itemgetter(2))
    nonlethalList.sort(key=itemgetter(2))

    # Walk through each list, generating points for a plot.
    right = 0
    wrong = 0
    lethalFile = open(args.phenosimset+'.lethal.csv', 'w')
    lethalFile.write('wrong,right\n')
    for index in range(len(lethalList)):
        if lethalList[index][3]:
            right += 1
        else:
            wrong += 1
        lethalFile.write('%d,%d\n' %(wrong,right))
    lethalFile.close()

    right = 0
    wrong = 0
    nonlethalFile = open(args.phenosimset+'.nonlethal.csv', 'w')
    nonlethalFile.write('wrong,right\n')
    for index in range(len(nonlethalList)):
        if nonlethalList[index][3]:
            right += 1
        else:
            wrong += 1
        nonlethalFile.write('%d,%d\n' %(wrong,right))
    nonlethalFile.close()
    exit(0)
    <commit_msg>minor cleanup to script<commit_after>#!/usr/bin/python -u

import argparse
import sys
import re
import os
import traceback
from operator import itemgetter, attrgetter
from biokbase.probabilistic_annotation.Client import _read_inifile
from biokbase.workspaceService.Client import *

desc = '''
Given a JSON object calculate the false positive, negative rates. Optionally omit everything that isn't
in the specified media.
'''
separator ='///'
rolesToRxnsFileName = 'roles_to_reactions'

if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog='GeneratePhenotypeROCCurve.py', description=desc)
    parser.add_argument('genome', help='ID of genome to analyze', action='store', default=None)
    parser.add_argument('model', help='ID of integrated gap filled model to analyze', action='store', default=None)
    parser.add_argument('phenosimset', help='ID of PhenotypeSimulationSet object to analyze', action='store', default=None)
    parser.add_argument('workspace', help='ID of workspace containing objects to analyze', action='store', default=None)
    parser.add_argument('--solution', help='index number of solution in integrated gap filled model', action='store', default='0')
    parser.add_argument('--phenosimsetws', help='workspace containing PhenotypeSimulationSet object (same as workspace if not specified)', action='store', default=None)
    parser.add_argument('--media', help='limit analysis to only this media condition', action='store', dest='media', default=None)
    parser.add_argument('--probanno', help='ID of ProbAnno object for genome', action='store', dest='probanno', default=None)
    parser.add_argument('--probannows', help='workspace containing ProbAnno object (same as workspace if not specified)', action='store', dest='probannows', default=None)
    parser.add_argument('--rxnprobs', help='ID of RxnProbs object for genome', action='store', dest='rxnprobs', default=None)
    parser.add_argument('--rxnprobsws', help='workspace containing RxnProbs object (same as workspace if not specified)', action='store', dest='rxnprobsws', default=None)
    parser.add_argument('--script-dir', help='path to directory with analysis scripts', action='store', dest='scriptDir', default='.')
    parser.add_argument('--fba-url', help='url for fba modeling service', action='store', dest='fbaurl', default='http://bio-data-1.mcs.anl.gov/services/fba')
    parser.add_argument('--ws-url', help='url for workspace service', action='store', dest='wsurl', default='http://www.kbase.us/services/workspace/')
    args = parser.parse_args()

    if args.probanno is None:
        args.probanno = args.genome + '.probanno'
    if args.probannows is None:
        args.probannows = args.workspace
    if args.rxnprobs is None:
        args.rxnprobs = args.genome + '.rxnprobs'
    if args.rxnprobsws is None:
        args.rxnprobsws = args.workspace
    if args.phenosimsetws is None:
        args.phenosimsetws = args.workspace

    # Create the clients for the services we need.
    authdata = _read_inifile()
    token = authdata['token']
    wsClient = workspaceService(args.wsurl)

    # Build roles to reactions dictionary.
    step = 1
    print "+++ Step %d: Build reactions to roles dictionary +++" %(step)
    if not os.path.exists(rolesToRxnsFileName):
        os.system("all_roles_used_in_models | roles_to_complexes | get_relationship_HasStep -to id >%s" %(rolesToRxnsFileName))
    rolesToReactions = dict()
    reactionsToRoles = dict()
    # Each line of the file has four fields: (1) role, (2) hypothetical flag, (3) complex id, (4) reaction id
    rolesToRxnsFile = open(rolesToRxnsFileName, 'r')
    for line in rolesToRxnsFile:
        fields = line.strip('\r\n').split('\t')
        reaction = fields[3]
        if reaction not in reactionsToRoles:
            reactionsToRoles[reaction] = list()
        reactionsToRoles[reaction].append(fields[0])
    rolesToRxnsFile.close()
    print "  %d reactions in reactions to roles dictionary" %(len(reactionsToRoles))

    # Analyze the gap fill results for the specified model.
    step += 1
    print "+++ Step %d: Run AnalyzeGapfillResults.py for model '%s/%s' +++" %(step, args.workspace, args.model)
    rxnprobs = args.rxnprobs
    resultsFileName = args.model + '.results'
    os.system("%s/AnalyzeGapfillResults.py -m %s -w %s --rxnprobs %s --url %s >%s" \
              %(args.scriptDir, args.model, args.workspace, rxnprobs, args.fbaurl, resultsFileName))
    genesToReactions = dict()
    resultsFile = open(resultsFileName, 'r')
    resultsFile.readline() # Throw away header line
    for line in resultsFile:
        fields = line.strip('\r\n').split('\t')
        if fields[0] == args.solution:
            if fields[5] == '0': # Only keep reactions that are not a reversibility change
                if fields[6]: # Only keep reactions that have a GPR
                    rxnid = re.sub(r'rxn0*(\d+)', r'kb|rxn.\1', fields[1])
                    geneList = re.findall('fig\|\d+\.\d+\.peg\.\d+', fields[6])
                    for index in range(len(geneList)):
                        if geneList[index] not in genesToReactions:
                            genesToReactions[geneList[index]] = dict()
                        genesToReactions[geneList[index]][rxnid] = 0.0
    resultsFile.close()
    print "  %d genes in genes to reactions dictionary" %(len(genesToReactions))

    # Get the ProbAnno object from the workspace.
    step += 1
    probanno = args.probanno
    print "+++ Step %d: Get ProbAnno object '%s/%s'" %(step, args.probannows, probanno)
    paObject = wsClient.get_object( { 'id': probanno, 'workspace': args.probannows, 'type': 'ProbAnno', 'auth': token } )
    probAnno = paObject['data']
    print "  %d genes in ProbAnno roleset probabilities dictionary" %(len(probAnno['roleset_probabilities']))

    # Need to go through rolesets dictionary
    # If an entry has more than one role, split into parts
    # Then run the array and combine any duplicate roles by adding probs
    # Parse the roleset probabilities from the ProbAnno object.
    step += 1
    print "+++ Step %d: Parse rolesets into roles and adjust probabilities for duplicates +++" %(step)
    rolesetProbabilities = dict()
    for gene in probAnno['roleset_probabilities']:
        geneRoleList = probAnno['roleset_probabilities'][gene]
        geneRoleDict = dict()
        for index in range(len(geneRoleList)):
            prob = geneRoleList[index][1] # Probability for this roleset
            # Split multiple roles in roleset for this gene
            roleList = geneRoleList[index][0].split(separator)
            # If role occurs more than once, add up the probabilities
            for j in range(len(roleList)):
                if roleList[j] in geneRoleDict:
                    geneRoleDict[roleList[j]] += prob
                else:
                    geneRoleDict[roleList[j]] = prob
        rolesetProbabilities[gene] = geneRoleDict
    print "  %d genes in parsed roleset probabilities dictionary" %(len(rolesetProbabilities))

    # for each reaction in the reactions dictionary, find the roles in the rolesToReactions dictionary
    # then find the roles in the probanno object for the gene
    step += 1
    print "+++ Step %d: Find maximum probability of reaction given gene +++" %(step)
    probsFile = open(args.genome+'.probs', 'w')
    numProbs = 0
    for gene in genesToReactions:
        if gene in rolesetProbabilities:
            geneRoleDict = rolesetProbabilities[gene]
            for reaction in genesToReactions[gene]:
                if reaction not in reactionsToRoles:
                    print 'Why is reaction %s not in the reactionToRoles dictionary?' %(reaction)
                roleList = reactionsToRoles[reaction]
                for index in range(0,len(roleList)):
                    for role in geneRoleDict:
                        if role == roleList[index]:
                            probsFile.write('P(%s | %s) = %f\n' %(reaction, gene, geneRoleDict[role]))
                            genesToReactions[gene][reaction] = max(geneRoleDict[role], genesToReactions[gene][reaction])
                            numProbs += 1
        else:
            print 'Gene %s not found in ProbAnno object' %(gene)
    probsFile.close()
    print "  %d reaction probabilities set in genesToReactions dictionary" %(numProbs)

    # Get the PhenotypeSimulationSet object.
    step += 1
    print "+++ Step %d: Get phenotype simulation set %s/%s +++" %(step, args.phenosimsetws, args.phenosimset)
    pssObject = wsClient.get_object( { 'id': args.phenosimset, 'workspace': args.phenosimsetws, 'type': 'PhenotypeSimulationSet', 'auth': token } )
    phenoSimSet = pssObject['data']

    # Make sure the model matches in the phenotype simulation set.
    if phenoSimSet['model'] != args.model or phenoSimSet['model_workspace'] != args.workspace:
        print 'Specified model %s/%s does not match model %s/%s in phenotype simulation set' \
            %(args.workspace, args.model, phenoSimSet['model_workspace'], phenoSimSet['model'])
    print "  %d simulations in phenotype simulation set" %(len(phenoSimSet['phenotypeSimulations']))

    # Go through the list of simulations, for each gene and see if the gene is in the genesToReactions
    # dictionary.  If so, see if the gene was a lethal or non-lethal knockout and add it to the
    # corresponding dictionary along with reactions and
    lethalList = list()
    nonlethalList = list()
    for sim in phenoSimSet['phenotypeSimulations']:
        if sim[3] == 'CP' or sim[3] == 'CN':
            right = 1
        else:
            right = 0
        geneList = sim[0][0]
        if sim[1] > 0:
            for gene in geneList:
                if gene in genesToReactions:
                    for reaction in genesToReactions[gene]:
                        nonlethalList.append( (gene, reaction, genesToReactions[gene][reaction], right ) )
        else:
            for gene in geneList:
                if gene in genesToReactions:
                    for reaction in genesToReactions[gene]:
                        lethalList.append( (gene, reaction, genesToReactions[gene][reaction], right ) )

    # Sort the lists by reaction probability.
    lethalList.sort(key=itemgetter(2))
    nonlethalList.sort(key=itemgetter(2))

    # Walk through each list, generating points for a plot.
    print "  %d knockouts in lethal list" %(len(lethalList))
    right = 0
    wrong = 0
    lethalFile = open(args.phenosimset+'.lethal.csv', 'w')
    lethalFile.write('wrong,right\n')
    for index in range(len(lethalList)):
        if lethalList[index][3]:
            right += 1
        else:
            wrong += 1
        lethalFile.write('%d,%d\n' %(wrong,right))
    lethalFile.close()

    print "  %d knockouts in non-lethal list" %(len(nonlethalList))
    right = 0
    wrong = 0
    nonlethalFile = open(args.phenosimset+'.nonlethal.csv', 'w')
    nonlethalFile.write('wrong,right\n')
    for index in range(len(nonlethalList)):
        if nonlethalList[index][3]:
            right += 1
        else:
            wrong += 1
        nonlethalFile.write('%d,%d\n' %(wrong,right))
    nonlethalFile.close()
    exit(0)
    <|endoftext|>