<commit_before># -*- coding: iso-8859-1 -*-#

#!/usr/bin/env python

import wx
import random
import  cStringIO
import thread
import numpy as np

import _init_paths
from web_server import WebService
from detect_engine import Detector, CLASSES

# This has been set up to optionally use the wx.BufferedDC if
# USE_BUFFERED_DC is True, it will be used. Otherwise, it uses the raw
# wx.Memory DC , etc.

#USE_BUFFERED_DC = False
USE_BUFFERED_DC = True

class BufferedWindow(wx.Window):

    """

    A Buffered window class.

    To use it, subclass it and define a Draw(DC) method that takes a DC
    to draw to. In that method, put the code needed to draw the picture
    you want. The window will automatically be double buffered, and the
    screen will be automatically updated when a Paint event is received.

    When the drawing needs to change, you app needs to call the
    UpdateDrawing() method. Since the drawing is stored in a bitmap, you
    can also save the drawing to file by calling the
    SaveToFile(self, file_name, file_type) method.

    """
    def __init__(self, *args, **kwargs):
        # make sure the NO_FULL_REPAINT_ON_RESIZE style flag is set.
        kwargs['style'] = kwargs.setdefault('style', wx.NO_FULL_REPAINT_ON_RESIZE) | wx.NO_FULL_REPAINT_ON_RESIZE
        wx.Window.__init__(self, *args, **kwargs)

        wx.EVT_PAINT(self, self.OnPaint)
        wx.EVT_SIZE(self, self.OnSize)

        # OnSize called to make sure the buffer is initialized.
        # This might result in OnSize getting called twice on some
        # platforms at initialization, but little harm done.
        self.OnSize(None)
        self.paint_count = 0

    def Draw(self, dc):
        ## just here as a place holder.
        ## This method should be over-ridden when subclassed
        pass

    def OnPaint(self, event):
        # All that is needed here is to draw the buffer to screen
        if USE_BUFFERED_DC:
            dc = wx.BufferedPaintDC(self, self._Buffer)
        else:
            dc = wx.PaintDC(self)
            dc.DrawBitmap(self._Buffer, 0, 0)

    def OnSize(self,event):
        # The Buffer init is done here, to make sure the buffer is always
        # the same size as the Window
        #Size  = self.GetClientSizeTuple()
        Size  = self.ClientSize

        # Make new offscreen bitmap: this bitmap will always have the
        # current drawing in it, so it can be used to save the image to
        # a file, or whatever.
        self._Buffer = wx.EmptyBitmap(*Size)
        self.UpdateDrawing()

    def SaveToFile(self, FileName, FileType=wx.BITMAP_TYPE_PNG):
        ## This will save the contents of the buffer
        ## to the specified file. See the wxWindows docs for
        ## wx.Bitmap::SaveFile for the details
        self._Buffer.SaveFile(FileName, FileType)

    def UpdateDrawing(self):
        """
        This would get called if the drawing needed to change, for whatever reason.

        The idea here is that the drawing is based on some data generated
        elsewhere in the system. If that data changes, the drawing needs to
        be updated.

        This code re-draws the buffer, then calls Update, which forces a paint event.
        """
        dc = wx.MemoryDC()
        dc.SelectObject(self._Buffer)
        self.Draw(dc)
        del dc # need to get rid of the MemoryDC before Update() is called.
        self.Refresh()
        self.Update()

class DrawWindow(BufferedWindow):
    def __init__(self, *args, **kwargs):
        ## Any data the Draw() function needs must be initialized before
        ## calling BufferedWindow.__init__, as it will call the Draw
        ## function.
        self.file_name = None
        self.DrawData = {}
        BufferedWindow.__init__(self, *args, **kwargs)

    def GetColorText(self, key):
        text = CLASSES[key]
        if key == 1:
            color = 'BLUE'
        elif key == 2:
            color = 'RED'
        elif key == 3:
            color = 'GREY'
        elif key == 4:
            color = 'BLACK'
        elif key == 5:
            color = 'BROWN'
        elif key == 6:
            color = 'CYAN'
        elif key == 7:
            color = 'GREEN'
        elif key == 8:
            color = 'MAGENTA'
        elif key == 9:
            color = 'ORANGE'
        elif key == 10:
            color = 'YELLOW'
        elif key == 11:
            color = 'PINK'
        elif key == 12:
            color = 'PURPLE'
        elif key == 13:
            color = 'VIOLET'
        elif key == 14:
            color = 'PLUM'
        elif key == 15:
            color = 'AQUAMARINE'
        elif key == 16:
            color = 'GOLD'
        elif key == 17:
            color = 'KHAKI'
        elif key == 18:
            color = 'NAVY'
        elif key == 19:
            color = 'ORCHID'
        elif key == 20:
            color = 'CORAL'

        return color, text

    def Draw(self, dc):
        #dc.SetBackground( wx.Brush("White") )
        dc.Clear() # make sure you clear the bitmap!

        #self.file_name = '/home/dj/big/haha.jpeg'

        #print 'DrawWindow.Draw() self.file_name : %s' % self.file_name

        if self.file_name != None:
            data = open(self.file_name, "rb").read()
            # convert to a data stream
            stream = cStringIO.StringIO(data)
            # convert to a bitmap
            bmp = wx.BitmapFromImage( wx.ImageFromStream( stream ))
            # show the bitmap, (5, 5) are upper left corner coordinates
            #wx.StaticBitmap(self, -1, bmp, (5, 5))

            dc.DrawBitmap(bmp, 0, 0, True)

        for key, data in self.DrawData.items():
            color, text = self.GetColorText(key)

            for r in data:
                dc.SetBrush(wx.Brush('WHITE'))
                dc.SetPen(wx.Pen('WHITE', 1))
                dc.DrawRectangle(r[0], r[1] - 20, 100, 20)
                dc.SetTextForeground('BLACK')
                dc.DrawText(text, r[0], r[1] - 20)

                dc.SetBrush(wx.TRANSPARENT_BRUSH)
                dc.SetPen(wx.Pen(color, 3))
                #dc.DrawRectangle(*r)
                dc.DrawRectangle(r[0], r[1], r[2]-r[0], r[3]-r[1])

            """
            elif key == "Ellipses":
                dc.SetBrush(wx.Brush("GREEN YELLOW"))
                dc.SetPen(wx.Pen('CADET BLUE', 2))
                for r in data:
                    dc.DrawEllipse(*r)
            elif key == "Polygons":
                dc.SetBrush(wx.Brush("SALMON"))
                dc.SetPen(wx.Pen('VIOLET RED', 4))
                for r in data:
                    dc.DrawPolygon(r)
            """

class TestFrame(wx.Frame):
    def __init__(self, parent=None):
        wx.Frame.__init__(self, parent,
                          size = (1280, 960),
                          title="Double Buffered Test",
                          style=wx.DEFAULT_FRAME_STYLE)

        ## Set up the MenuBar
        MenuBar = wx.MenuBar()

        file_menu = wx.Menu()

        item = file_menu.Append(wx.ID_EXIT, text="&Exit")
        self.Bind(wx.EVT_MENU, self.OnQuit, item)
        MenuBar.Append(file_menu, "&File")

        draw_menu = wx.Menu()
        item = draw_menu.Append(wx.ID_ANY, "&New Drawing","Update the Drawing Data")
        self.Bind(wx.EVT_MENU, self.NewDrawing, item)
        item = draw_menu.Append(wx.ID_ANY,'&Save Drawing\tAlt-I','')
        self.Bind(wx.EVT_MENU, self.SaveToFile, item)
        MenuBar.Append(draw_menu, "&Draw")

        self.SetMenuBar(MenuBar)
        self.Window = DrawWindow(self)
        self.Show()
        # Initialize a drawing -- it has to be done after Show() is called
        #   so that the Windows has teh right size.
        #self.NewDrawing()

    def OnQuit(self,event):
        self.Close(True)

    def NewDrawing(self, event=None):
        self.Window.UpdateDrawing()

    def SaveToFile(self,event):
        dlg = wx.FileDialog(self, "Choose a file name to save the image as a PNG to",
                           defaultDir = "",
                           defaultFile = "",
                           wildcard = "*.png",
                           style = wx.SAVE)
        if dlg.ShowModal() == wx.ID_OK:
            self.Window.SaveToFile(dlg.GetPath(), wx.BITMAP_TYPE_PNG)
        dlg.Destroy()

class DemoApp(wx.App):
    def OnInit(self):
        self.in_progress = False
        self.frame = TestFrame()
        self.SetTopWindow(self.frame)

        return True

    def process_data(self, file_name):
        if self.in_progress == True:
            return None

        self.in_progress = True

        detect_result = self.detector.detect(file_name)

        #print 'detect_result : %s' % detect_result

        rects = {}
        for key in detect_result.keys():
            if key == 0:        # background
                continue

            l = []
            values = detect_result[key]

            inds = np.where(values[:, -1]> self.conf_thresh)[0]

            if len(inds) == 0:
                continue

            values = values.astype(np.int)

            for ind in inds:
                one_rect = values[ind]
                #l.append((one_rect[0], one_rect[1], one_rect[2]-one_rect[0], one_rect[3]-one_rect[1]))
                l.append((one_rect[0], one_rect[1], one_rect[2], one_rect[3]))

            rects[key] = l

        self.frame.Window.DrawData = rects
        self.frame.Window.file_name = file_name
        wx.CallAfter(self.frame.Window.UpdateDrawing)

        self.in_progress = False

        #ret = rects.copy()
        #ret['file_name'] = file_name.split('/')[-1]

        return rects

    def gogo(self, ip, port, conf_thresh):
        self.detector = Detector()
        self.detector.initialize(app)
        self.conf_thresh = conf_thresh

        webService = WebService()
        thread.start_new_thread(webService.initialize, (ip, port, app))

        app.MainLoop()

if __name__ == "__main__":
    IP = "192.168.0.18"
    PORT = 8080
    CONF_THRESH = 0.95

    app = DemoApp(0)
    app.gogo(IP, PORT, CONF_THRESH)
<commit_msg>Increase frame rates by using queue.<commit_after># -*- coding: iso-8859-1 -*-#

#!/usr/bin/env python

import wx
import random
import  cStringIO
import thread
import numpy as np

import _init_paths
from web_server import WebService
from detect_engine import Detector, CLASSES
from threading import Condition

# This has been set up to optionally use the wx.BufferedDC if
# USE_BUFFERED_DC is True, it will be used. Otherwise, it uses the raw
# wx.Memory DC , etc.

#USE_BUFFERED_DC = False
USE_BUFFERED_DC = True

class BufferedWindow(wx.Window):

    """

    A Buffered window class.

    To use it, subclass it and define a Draw(DC) method that takes a DC
    to draw to. In that method, put the code needed to draw the picture
    you want. The window will automatically be double buffered, and the
    screen will be automatically updated when a Paint event is received.

    When the drawing needs to change, you app needs to call the
    UpdateDrawing() method. Since the drawing is stored in a bitmap, you
    can also save the drawing to file by calling the
    SaveToFile(self, file_name, file_type) method.

    """
    def __init__(self, *args, **kwargs):
        # make sure the NO_FULL_REPAINT_ON_RESIZE style flag is set.
        kwargs['style'] = kwargs.setdefault('style', wx.NO_FULL_REPAINT_ON_RESIZE) | wx.NO_FULL_REPAINT_ON_RESIZE
        wx.Window.__init__(self, *args, **kwargs)

        wx.EVT_PAINT(self, self.OnPaint)
        wx.EVT_SIZE(self, self.OnSize)

        # OnSize called to make sure the buffer is initialized.
        # This might result in OnSize getting called twice on some
        # platforms at initialization, but little harm done.
        self.OnSize(None)
        self.paint_count = 0

    def Draw(self, dc):
        ## just here as a place holder.
        ## This method should be over-ridden when subclassed
        pass

    def OnPaint(self, event):
        # All that is needed here is to draw the buffer to screen
        if USE_BUFFERED_DC:
            dc = wx.BufferedPaintDC(self, self._Buffer)
        else:
            dc = wx.PaintDC(self)
            dc.DrawBitmap(self._Buffer, 0, 0)

    def OnSize(self,event):
        # The Buffer init is done here, to make sure the buffer is always
        # the same size as the Window
        #Size  = self.GetClientSizeTuple()
        Size  = self.ClientSize

        # Make new offscreen bitmap: this bitmap will always have the
        # current drawing in it, so it can be used to save the image to
        # a file, or whatever.
        self._Buffer = wx.EmptyBitmap(*Size)
        self.UpdateDrawing()

    def SaveToFile(self, FileName, FileType=wx.BITMAP_TYPE_PNG):
        ## This will save the contents of the buffer
        ## to the specified file. See the wxWindows docs for
        ## wx.Bitmap::SaveFile for the details
        self._Buffer.SaveFile(FileName, FileType)

    def UpdateDrawing(self):
        """
        This would get called if the drawing needed to change, for whatever reason.

        The idea here is that the drawing is based on some data generated
        elsewhere in the system. If that data changes, the drawing needs to
        be updated.

        This code re-draws the buffer, then calls Update, which forces a paint event.
        """
        dc = wx.MemoryDC()
        dc.SelectObject(self._Buffer)
        self.Draw(dc)
        del dc # need to get rid of the MemoryDC before Update() is called.
        self.Refresh()
        self.Update()

class DrawWindow(BufferedWindow):
    def __init__(self, *args, **kwargs):
        ## Any data the Draw() function needs must be initialized before
        ## calling BufferedWindow.__init__, as it will call the Draw
        ## function.
        self.file_name = None
        self.DrawData = {}
        BufferedWindow.__init__(self, *args, **kwargs)

    def GetColorText(self, key):
        text = CLASSES[key]
        if key == 1:
            color = 'BLUE'
        elif key == 2:
            color = 'RED'
        elif key == 3:
            color = 'GREY'
        elif key == 4:
            color = 'BLACK'
        elif key == 5:
            color = 'BROWN'
        elif key == 6:
            color = 'CYAN'
        elif key == 7:
            color = 'GREEN'
        elif key == 8:
            color = 'MAGENTA'
        elif key == 9:
            color = 'ORANGE'
        elif key == 10:
            color = 'YELLOW'
        elif key == 11:
            color = 'PINK'
        elif key == 12:
            color = 'PURPLE'
        elif key == 13:
            color = 'VIOLET'
        elif key == 14:
            color = 'PLUM'
        elif key == 15:
            color = 'AQUAMARINE'
        elif key == 16:
            color = 'GOLD'
        elif key == 17:
            color = 'KHAKI'
        elif key == 18:
            color = 'NAVY'
        elif key == 19:
            color = 'ORCHID'
        elif key == 20:
            color = 'CORAL'

        return color, text

    def Draw(self, dc):
        #dc.SetBackground( wx.Brush("White") )
        dc.Clear() # make sure you clear the bitmap!

        #self.file_name = '/home/dj/big/haha.jpeg'

        #print 'DrawWindow.Draw() self.file_name : %s' % self.file_name

        if self.file_name != None:
            data = open(self.file_name, "rb").read()
            # convert to a data stream
            stream = cStringIO.StringIO(data)
            # convert to a bitmap
            bmp = wx.BitmapFromImage( wx.ImageFromStream( stream ))
            # show the bitmap, (5, 5) are upper left corner coordinates
            #wx.StaticBitmap(self, -1, bmp, (5, 5))

            dc.DrawBitmap(bmp, 0, 0, True)

        for key, data in self.DrawData.items():
            color, text = self.GetColorText(key)

            for r in data:
                dc.SetBrush(wx.Brush('WHITE'))
                dc.SetPen(wx.Pen('WHITE', 1))
                dc.DrawRectangle(r[0], r[1] - 20, 100, 20)
                dc.SetTextForeground('BLACK')
                dc.DrawText(text, r[0], r[1] - 20)

                dc.SetBrush(wx.TRANSPARENT_BRUSH)
                dc.SetPen(wx.Pen(color, 3))
                #dc.DrawRectangle(*r)
                dc.DrawRectangle(r[0], r[1], r[2]-r[0], r[3]-r[1])

            """
            elif key == "Ellipses":
                dc.SetBrush(wx.Brush("GREEN YELLOW"))
                dc.SetPen(wx.Pen('CADET BLUE', 2))
                for r in data:
                    dc.DrawEllipse(*r)
            elif key == "Polygons":
                dc.SetBrush(wx.Brush("SALMON"))
                dc.SetPen(wx.Pen('VIOLET RED', 4))
                for r in data:
                    dc.DrawPolygon(r)
            """

class TestFrame(wx.Frame):
    def __init__(self, parent=None):
        wx.Frame.__init__(self, parent,
                          size = (1280, 960),
                          title="Double Buffered Test",
                          style=wx.DEFAULT_FRAME_STYLE)

        ## Set up the MenuBar
        MenuBar = wx.MenuBar()

        file_menu = wx.Menu()

        item = file_menu.Append(wx.ID_EXIT, text="&Exit")
        self.Bind(wx.EVT_MENU, self.OnQuit, item)
        MenuBar.Append(file_menu, "&File")

        draw_menu = wx.Menu()
        item = draw_menu.Append(wx.ID_ANY, "&New Drawing","Update the Drawing Data")
        self.Bind(wx.EVT_MENU, self.NewDrawing, item)
        item = draw_menu.Append(wx.ID_ANY,'&Save Drawing\tAlt-I','')
        self.Bind(wx.EVT_MENU, self.SaveToFile, item)
        MenuBar.Append(draw_menu, "&Draw")

        self.SetMenuBar(MenuBar)
        self.Window = DrawWindow(self)
        self.Show()
        # Initialize a drawing -- it has to be done after Show() is called
        #   so that the Windows has teh right size.
        #self.NewDrawing()

    def OnQuit(self,event):
        self.Close(True)

    def NewDrawing(self, event=None):
        self.Window.UpdateDrawing()

    def SaveToFile(self,event):
        dlg = wx.FileDialog(self, "Choose a file name to save the image as a PNG to",
                           defaultDir = "",
                           defaultFile = "",
                           wildcard = "*.png",
                           style = wx.SAVE)
        if dlg.ShowModal() == wx.ID_OK:
            self.Window.SaveToFile(dlg.GetPath(), wx.BITMAP_TYPE_PNG)
        dlg.Destroy()

class DemoApp(wx.App):
    def OnInit(self):
        self.working_process_no = 0
        self.cv = Condition()
        self.frame = TestFrame()
        self.SetTopWindow(self.frame)

        return True

    def process_data(self, file_name):
        if self.working_process_no == 2:
            return None

        self.working_process_no += 1

        self.cv.acquire()

        if self.working_process_no == 2:
            self.cv.wait()

        detect_result = self.detector.detect(file_name)

        #print 'detect_result : %s' % detect_result

        rects = {}
        for key in detect_result.keys():
            if key == 0:        # background
                continue

            l = []
            values = detect_result[key]

            inds = np.where(values[:, -1]> self.conf_thresh)[0]

            if len(inds) == 0:
                continue

            values = values.astype(np.int)

            for ind in inds:
                one_rect = values[ind]
                l.append((one_rect[0], one_rect[1], one_rect[2], one_rect[3]))

            rects[key] = l

        self.frame.Window.DrawData = rects
        self.frame.Window.file_name = file_name
        wx.CallAfter(self.frame.Window.UpdateDrawing)

        self.working_process_no -= 1

        self.cv.notify()
        self.cv.release()

        return rects

    def gogo(self, ip, port, conf_thresh):
        self.detector = Detector()
        self.detector.initialize(app)
        self.conf_thresh = conf_thresh

        webService = WebService()
        thread.start_new_thread(webService.initialize, (ip, port, app))

        app.MainLoop()

if __name__ == "__main__":
    IP = "192.168.0.18"
    PORT = 8080
    CONF_THRESH = 0.95

    app = DemoApp(0)
    app.gogo(IP, PORT, CONF_THRESH)
<|endoftext|>


<commit_before>__version__ = "0.1.1a5"
<commit_msg>increased version number<commit_after>__version__ = "0.1.1a6"
<|endoftext|>


<commit_before>import unittest

import numpy as np

import nipy.neurospin.group.spatial_relaxation_onesample as os

verbose = False

def make_data(n=10, dim=20, r=5, mdim=15, maskdim=20, amplitude=10,
                            noise=1, jitter=None, activation=False):
    XYZvol = np.zeros((dim,dim,dim),int)
    XYZ = np.array(np.where(XYZvol==0))
    p = XYZ.shape[1]
    #mask = np.arange(p)
    XYZvol[XYZ[0],XYZ[1],XYZ[2]] = np.arange(p)
    o = np.array([dim/2,dim/2,dim/2])
    I = XYZvol[(dim-mdim)/2:(dim+mdim)/2,
               (dim-mdim)/2:(dim+mdim)/2,
               (dim-mdim)/2:(dim+mdim)/2 ].ravel()
    mask = XYZvol[(dim-maskdim)/2:(dim+maskdim)/2,
                  (dim-maskdim)/2:(dim+maskdim)/2,
                  (dim-maskdim)/2:(dim+maskdim)/2 ].ravel()
    q = len(mask)
    maskvol = np.zeros((dim, dim, dim), int)
    maskvol[XYZ[0, mask], XYZ[1,mask], XYZ[2, mask]] = np.arange(q)
    Isignal = maskvol[dim/2-r:dim/2+r, dim/2-r:dim/2+r, dim/2-r:dim/2+r].ravel()
    signal = np.zeros(q, float)
    signal[Isignal] += amplitude
    X = np.zeros((n, p), float) + np.nan
    data = np.zeros((n, p), float) + np.nan
    vardata = np.zeros((n, p), float) + np.nan
    for i in xrange(n):
        X[i, I] = np.random.randn(len(I))
        if activation:
            o = np.array([dim/2, dim/2, dim/2])
            if jitter!=None:
                o += np.round(np.random.randn(3)*jitter).clip(r-mdim/2,mdim/2-r)
        Ii = XYZvol[o[0]-r:o[0]+r, o[1]-r:o[1]+r, o[2]-r:o[2]+r].ravel()
        X[i,Ii] += amplitude
        vardata[i, I] = np.square(np.random.randn(len(I)))*noise**2
        data[i, I] = X[i, I] + np.random.randn(len(I))*np.sqrt(vardata[i, I])
    return data, XYZ, mask, XYZvol, vardata, signal

class TestMultivariateStatSaem(unittest.TestCase):

    def test_evaluate_exact(self):
        # without mfx nor spatial relaxation
        data, XYZ, mask, XYZvol, vardata, signal = \
                make_data(n=20, dim=20, r=3, mdim=15, maskdim=15,
                          amplitude=5, noise=0, jitter=0, activation=False)
        XYZvol *= 0
        XYZvol[XYZ[0,mask],XYZ[1,mask],XYZ[2,mask]] = np.arange(len(mask))
        P = os.multivariate_stat(data[:, mask])
        P.init_hidden_variables()
        P.evaluate(nsimu=100, burnin=100, J=[XYZvol[5, 5, 5]],
                   compute_post_mean=True, verbose=verbose)
        P.log_likelihood_values = P.compute_log_region_likelihood()
        # Verify code consistency
        Q = os.multivariate_stat(data[:, mask], vardata[:, mask]*0,
                                 XYZ[:, mask], std=0, sigma=5)
        Q.init_hidden_variables()
        Q.evaluate(nsimu=100, burnin=100, J = [XYZvol[5,5,5]],
                   compute_post_mean=True, update_spatial=False,
                   verbose=verbose)
        Q.log_likelihood_values = Q.compute_log_region_likelihood()
        self.assertAlmostEqual(P.mean_m.mean(),
                               Q.mean_m.mean(),
                               int(np.log10(P.nsimu))-1)
        self.assertAlmostEqual(Q.log_likelihood_values.sum(),
                               P.log_likelihood_values.sum(), 1)

    def test_evaluate_mfx_spatial_fixed(self):
        data, XYZ, mask, XYZvol, vardata, signal = \
                make_data(n=20, dim=20, r=3, mdim=15, maskdim=15,
                          amplitude=5, noise=1, jitter=1, activation=True)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask],
                                 XYZ[:, mask], std=1, sigma=3)
        P.init_hidden_variables()
        P.evaluate(nsimu=10, burnin=10, J=[P.D.XYZ_vol[10, 10, 10]],
                   verbose=verbose, compute_post_mean=True)
        proposal_mean = P.mean_U
        proposal_std = np.sqrt(np.clip(P.var_U, 1e-4, 4))
        P.evaluate(10, 0, [P.D.XYZ_vol[10, 10, 10]], False,
                   'fixed', proposal_std, proposal_mean)
        self.assertEqual(max(abs(P.labels - np.zeros(len(mask), int))), 0)

    def test_model_selection_exact(self):
        data, XYZ, mask, XYZvol, vardata, signal = make_data(n=30, dim=30, r=3, mdim=10, maskdim=10, amplitude=1, noise=1, jitter=1, activation=False)
        labels = (signal > 0).astype(int)
        P1 = os.multivariate_stat(data[:, mask], labels=labels)
        P1.init_hidden_variables()
        P1.evaluate(nsimu=100, burnin=10, verbose=verbose)
        L1 = P1.compute_log_region_likelihood()
        Prior1 = P1.compute_log_prior()
        Post1 = P1.compute_log_posterior(nsimu=1e2, burnin=1e2,
                                         verbose=verbose)
        M1 = L1 + Prior1[:-1] - Post1
        self.assertAlmostEqual(M1.mean(),
                               P1.compute_marginal_likelihood().mean(), 0)
        P0 = os.multivariate_stat(data[:, mask], labels=labels)
        P0.network *= 0
        P0.init_hidden_variables()
        P0.evaluate(nsimu=100, burnin=100, verbose=verbose)
        L0 = P0.compute_log_region_likelihood()
        Prior0 = P0.compute_log_prior()
        Post0 = P0.compute_log_posterior(nsimu=1e2, burnin=1e2,
                                         verbose=verbose)
        M0 = L0 + Prior0[:-1] - Post0
        self.assertAlmostEqual(M0.mean(),
                               P0.compute_marginal_likelihood().mean(), 0)
        self.assertTrue(M1[1] > M0[1])
        self.assertTrue(M1[0] < M0[0])

    def test_model_selection_mfx_spatial_rand_walk(self):
        data, XYZ, mask, XYZvol, vardata, signal = \
                make_data(n=20, dim=20, r=3, mdim=15, maskdim=15,
                          amplitude=3, noise=1, jitter=1, activation=True)
        labels = (signal > 0).astype(int)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask],
                    XYZ[:, mask], std=2.0, sigma=5, labels=labels)
        P.network[:] = 0
        P.init_hidden_variables()
        P.evaluate(nsimu=100, burnin=100, verbose=verbose,
                    proposal='rand_walk', proposal_std=1.0)
        L00 = P.compute_log_region_likelihood()
        # Test simulated annealing procedure
        P.estimate_displacements_SA(nsimu=10, c=0.99, proposal_std=0.5,
                    verbose=verbose)
        L0 = P.compute_log_region_likelihood()[0]
        self.assertTrue(np.all(L0 > L00))
        Prior0 = P.compute_log_prior()[0]
        Post0 = P.compute_log_posterior(nsimu=1e2, burnin=1e2,
                    verbose=verbose)[0]
        M0 = L0 + Prior0 - Post0
        self.assertAlmostEqual(0.1*M0,
                    0.1*P.compute_marginal_likelihood(verbose=verbose)[0],
                    0)
        P.network[:] = 1
        P.init_hidden_variables(init_spatial=False)
        P.evaluate(nsimu=100, burnin=100, verbose=verbose,
                    update_spatial=False)
        L10 = P.compute_log_region_likelihood()[0]
        P.estimate_displacements_SA(nsimu=10, c=0.99,
                                    proposal_std=0.5, verbose=verbose)
        L1 = P.compute_log_region_likelihood()[0]
        self.assertTrue(L1 > L10)
        Prior1 = P.compute_log_prior()[0]
        Post1 = P.compute_log_posterior(nsimu=1e2, burnin=1e2,
                                        verbose=verbose)[0]
        M1 = L1 + Prior1 - Post1
        self.assertAlmostEqual(0.1*M1,
                    0.1*P.compute_marginal_likelihood(verbose=verbose)[0], 0)
        self.assertTrue(np.all(M1 > M0))

class TestMultivariateStatMcmc(unittest.TestCase):

    def test_evaluate_mfx_spatial(self):
        data, XYZ, mask, XYZvol, vardata, signal = make_data(n=20,
                    dim=20, r=3, mdim=15, maskdim=15, amplitude=5,
                    noise=1, jitter=1, activation=False)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask],
                                 XYZ[:, mask], std=1, sigma=3)
        P.init_hidden_variables()
        P.evaluate(nsimu=10, burnin=10,
                   J=[P.D.XYZ_vol[10, 10, 10]],
                   verbose=verbose, mode='mcmc')
        # Test log_likelihood computation
        v = P.v.copy()
        m_var = P.m_var.copy()
        m_mean = P.m_mean.copy()
        L1 = P.compute_log_region_likelihood_slow(v, m_mean, m_var)
        L2 = P.compute_log_region_likelihood(v, m_mean, m_var)
        self.assertAlmostEqual(-L1.sum(), L2.sum()*2, 2)
        # Test posterior density computation
        Prior = P.compute_log_prior(v, m_mean, m_var)
        Post = P.compute_log_posterior(v, m_mean, m_var, nsimu=10,
                                       burnin=10, verbose=verbose)

class TestLabelsPrior(unittest.TestCase):

    def test_update_labels(self):
        data, XYZ, mask, XYZvol, vardata, signal = make_data(n=20,
                        dim=20, r=3, mdim=15, maskdim=15, amplitude=5,
                        noise=1, jitter=1, activation=True)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask], XYZ[:, mask])
        P.init_hidden_variables()
        p = P.data.shape[1]
        P.labels_prior = np.ones((1, p), float)
        P.label_values = np.zeros((1, p), int)
        P.labels_prior_mask = np.arange(p)
        P.update_labels()
        self.assertEqual(max(abs(P.labels - np.zeros(p, int))), 0)

if __name__ == "__main__":
    unittest.main()
<commit_msg>spatial relaxation test corrected<commit_after>import unittest

import numpy as np

import nipy.neurospin.group.spatial_relaxation_onesample as os

verbose = False

def make_data(n=10, dim=20, r=5, mdim=15, maskdim=20, amplitude=10,
                            noise=1, jitter=None, activation=False):
    XYZvol = np.zeros((dim,dim,dim),int)
    XYZ = np.array(np.where(XYZvol==0))
    p = XYZ.shape[1]
    #mask = np.arange(p)
    XYZvol[XYZ[0],XYZ[1],XYZ[2]] = np.arange(p)
    o = np.array([dim/2,dim/2,dim/2])
    I = XYZvol[(dim-mdim)/2:(dim+mdim)/2,
               (dim-mdim)/2:(dim+mdim)/2,
               (dim-mdim)/2:(dim+mdim)/2 ].ravel()
    mask = XYZvol[(dim-maskdim)/2:(dim+maskdim)/2,
                  (dim-maskdim)/2:(dim+maskdim)/2,
                  (dim-maskdim)/2:(dim+maskdim)/2 ].ravel()
    q = len(mask)
    maskvol = np.zeros((dim, dim, dim), int)
    maskvol[XYZ[0, mask], XYZ[1,mask], XYZ[2, mask]] = np.arange(q)
    Isignal = maskvol[dim/2-r:dim/2+r, dim/2-r:dim/2+r, dim/2-r:dim/2+r].ravel()
    signal = np.zeros(q, float)
    signal[Isignal] += amplitude
    X = np.zeros((n, p), float) + np.nan
    data = np.zeros((n, p), float) + np.nan
    vardata = np.zeros((n, p), float) + np.nan
    for i in xrange(n):
        X[i, I] = np.random.randn(len(I))
        if activation:
            o = np.array([dim/2, dim/2, dim/2])
            if jitter!=None:
                o += np.round(np.random.randn(3)*jitter).clip(r-mdim/2,mdim/2-r)
        Ii = XYZvol[o[0]-r:o[0]+r, o[1]-r:o[1]+r, o[2]-r:o[2]+r].ravel()
        X[i,Ii] += amplitude
        vardata[i, I] = np.square(np.random.randn(len(I)))*noise**2
        data[i, I] = X[i, I] + np.random.randn(len(I))*np.sqrt(vardata[i, I])
    return data, XYZ, mask, XYZvol, vardata, signal

class TestMultivariateStatSaem(unittest.TestCase):

    def test_evaluate_exact(self):
        # without mfx nor spatial relaxation
        data, XYZ, mask, XYZvol, vardata, signal = \
                make_data(n=20, dim=20, r=3, mdim=15, maskdim=15,
                          amplitude=5, noise=0, jitter=0, activation=False)
        XYZvol *= 0
        XYZvol[XYZ[0,mask],XYZ[1,mask],XYZ[2,mask]] = np.arange(len(mask))
        P = os.multivariate_stat(data[:, mask])
        P.init_hidden_variables()
        P.evaluate(nsimu=100, burnin=100, J=[XYZvol[5, 5, 5]],
                   compute_post_mean=True, verbose=verbose)
        P.log_likelihood_values = P.compute_log_region_likelihood()
        # Verify code consistency
        Q = os.multivariate_stat(data[:, mask], vardata[:, mask]*0,
                                 XYZ[:, mask], std=0, sigma=5)
        Q.init_hidden_variables()
        Q.evaluate(nsimu=100, burnin=100, J = [XYZvol[5,5,5]],
                   compute_post_mean=True, update_spatial=False,
                   verbose=verbose)
        Q.log_likelihood_values = Q.compute_log_region_likelihood()
        self.assertAlmostEqual(P.mean_m.mean(),
                               Q.mean_m.mean(),
                               int(np.log10(P.nsimu))-1)
        self.assertAlmostEqual(Q.log_likelihood_values.sum(),
                               P.log_likelihood_values.sum(), 1)

    def test_evaluate_mfx_spatial_fixed(self):
        data, XYZ, mask, XYZvol, vardata, signal = \
                make_data(n=20, dim=20, r=3, mdim=15, maskdim=15,
                          amplitude=5, noise=1, jitter=1, activation=True)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask],
                                 XYZ[:, mask], std=1, sigma=3)
        P.init_hidden_variables()
        P.evaluate(nsimu=10, burnin=10, J=[P.D.XYZ_vol[10, 10, 10]],
                   verbose=verbose, compute_post_mean=True)
        proposal_mean = P.mean_U
        proposal_std = np.sqrt(np.clip(P.var_U, 1e-4, 4))
        P.evaluate(10, 0, [P.D.XYZ_vol[10, 10, 10]], False,
                   'fixed', proposal_std, proposal_mean)
        self.assertEqual(max(abs(P.labels - np.zeros(len(mask), int))), 0)

    def test_model_selection_exact(self):
        data, XYZ, mask, XYZvol, vardata, signal = make_data(n=30, dim=30, r=3, mdim=10, maskdim=10, amplitude=1, noise=1, jitter=1, activation=False)
        labels = (signal > 0).astype(int)
        P1 = os.multivariate_stat(data[:, mask], labels=labels)
        P1.init_hidden_variables()
        P1.evaluate(nsimu=100, burnin=10, verbose=verbose)
        L1 = P1.compute_log_region_likelihood()
        Prior1 = P1.compute_log_prior()
        Post1 = P1.compute_log_posterior(nsimu=1e2, burnin=1e2,
                                         verbose=verbose)
        M1 = L1 + Prior1[:-1] - Post1
        self.assertAlmostEqual(M1.mean(),
                               P1.compute_marginal_likelihood().mean(), 0)
        P0 = os.multivariate_stat(data[:, mask], labels=labels)
        P0.network *= 0
        P0.init_hidden_variables()
        P0.evaluate(nsimu=100, burnin=100, verbose=verbose)
        L0 = P0.compute_log_region_likelihood()
        Prior0 = P0.compute_log_prior()
        Post0 = P0.compute_log_posterior(nsimu=1e2, burnin=1e2,
                                         verbose=verbose)
        M0 = L0 + Prior0[:-1] - Post0
        self.assertAlmostEqual(M0.mean(),
                               P0.compute_marginal_likelihood().mean(), 0)
        self.assertTrue(M1[1] > M0[1])
        self.assertTrue(M1[0] < M0[0])

    def test_model_selection_mfx_spatial_rand_walk(self):
        data, XYZ, mask, XYZvol, vardata, signal = \
                make_data(n=20, dim=20, r=3, mdim=15, maskdim=15,
                          amplitude=3, noise=1, jitter=0.5, activation=True)
        labels = (signal > 0).astype(int)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask],
                    XYZ[:, mask], std=0.5, sigma=5, labels=labels)
        P.network[:] = 0
        P.init_hidden_variables()
        P.evaluate(nsimu=100, burnin=100, verbose=verbose,
                    proposal='rand_walk', proposal_std=0.5)
        L00 = P.compute_log_region_likelihood()
        # Test simulated annealing procedure
        P.estimate_displacements_SA(nsimu=100, c=0.99, proposal_std=0.5,
                    verbose=verbose)
        L0 = P.compute_log_region_likelihood()
        self.assertTrue(L0.sum() > L00.sum())
        Prior0 = P.compute_log_prior()
        Post0 = P.compute_log_posterior(nsimu=1e2, burnin=1e2, verbose=verbose)
        M0 = L0 + Prior0[:-1] - Post0
        self.assertAlmostEqual(0.1*M0.sum(), 0.1*P.compute_marginal_likelihood(verbose=verbose).sum(), 0)
        P.network[:] = 1
        P.init_hidden_variables(init_spatial=False)
        P.evaluate(nsimu=100, burnin=100, verbose=verbose,
                    update_spatial=False)
        L1 = P.compute_log_region_likelihood()
        Prior1 = P.compute_log_prior()
        Post1 = P.compute_log_posterior(nsimu=1e2, burnin=1e2, verbose=verbose)
        M1 = L1 + Prior1[:-1] - Post1
        self.assertAlmostEqual(0.1*M1.sum(), 0.1*P.compute_marginal_likelihood(verbose=verbose).sum(), 0)
        self.assertTrue(M1[1] > M0[1])
        self.assertTrue(M1[0] < M0[0])

class TestMultivariateStatMcmc(unittest.TestCase):

    def test_evaluate_mfx_spatial(self):
        data, XYZ, mask, XYZvol, vardata, signal = make_data(n=20,
                    dim=20, r=3, mdim=15, maskdim=15, amplitude=5,
                    noise=1, jitter=1, activation=False)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask],
                                 XYZ[:, mask], std=1, sigma=3)
        P.init_hidden_variables()
        P.evaluate(nsimu=10, burnin=10,
                   J=[P.D.XYZ_vol[10, 10, 10]],
                   verbose=verbose, mode='mcmc')
        # Test log_likelihood computation
        v = P.v.copy()
        m_var = P.m_var.copy()
        m_mean = P.m_mean.copy()
        L1 = P.compute_log_region_likelihood_slow(v, m_mean, m_var)
        L2 = P.compute_log_region_likelihood(v, m_mean, m_var)
        self.assertAlmostEqual(-L1.sum(), L2.sum()*2, 2)
        # Test posterior density computation
        Prior = P.compute_log_prior(v, m_mean, m_var)
        Post = P.compute_log_posterior(v, m_mean, m_var, nsimu=10,
                                       burnin=10, verbose=verbose)

class TestLabelsPrior(unittest.TestCase):

    def test_update_labels(self):
        data, XYZ, mask, XYZvol, vardata, signal = make_data(n=20,
                        dim=20, r=3, mdim=15, maskdim=15, amplitude=5,
                        noise=1, jitter=1, activation=True)
        P = os.multivariate_stat(data[:, mask], vardata[:, mask], XYZ[:, mask])
        P.init_hidden_variables()
        p = P.data.shape[1]
        P.labels_prior = np.ones((1, p), float)
        P.label_values = np.zeros((1, p), int)
        P.labels_prior_mask = np.arange(p)
        P.update_labels()
        self.assertEqual(max(abs(P.labels - np.zeros(p, int))), 0)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># flake8: NOQA

"""Script to run Cloudify's Agent Packager via command line

Usage:
    cfy-ap [--config=<path> --force --dryrun --no-validation -v]
    cfy-ap --version

Options:
    -h --help                   Show this screen
    -c --config=<path>          Path to config yaml (defaults to config.yaml)
    -f --force                  Forces deletion and creation of venv and tar file.
    -d --dryrun                 Prints out the modules to be installed without actually installing them.
    -n --no-validation          Does not validate that all modules were installed correctly.
    -v --verbose                verbose level logging
    --version                   Display current version
"""

from docopt import docopt
import pkg_resources

from . import logger, packager

lgr = logger.init()

def ver_check():
    version = None
    version = pkg_resources.get_distribution('agent_packager').version
    return version

def _run(test_options=None):
    version = ver_check()
    options = test_options or docopt(__doc__, version=version)
    packager.set_global_verbosity_level(options.get('--verbose'))
    lgr.debug(options)

    packager.create(
        config_file=options.get('--config'),
        force=options.get('--force'),
        dryrun=options.get('--dryrun'),
        no_validate=options.get('--no-validation'),
        verbose=options.get('--verbose')
        )

def main():
    _run()

if __name__ == '__main__':
    main()
<commit_msg>ver_check: use the right package name (#27)<commit_after># flake8: NOQA

"""Script to run Cloudify's Agent Packager via command line

Usage:
    cfy-ap [--config=<path> --force --dryrun --no-validation -v]
    cfy-ap --version

Options:
    -h --help                   Show this screen
    -c --config=<path>          Path to config yaml (defaults to config.yaml)
    -f --force                  Forces deletion and creation of venv and tar file.
    -d --dryrun                 Prints out the modules to be installed without actually installing them.
    -n --no-validation          Does not validate that all modules were installed correctly.
    -v --verbose                verbose level logging
    --version                   Display current version
"""

from docopt import docopt
import pkg_resources

from . import logger, packager

lgr = logger.init()

def ver_check():
    version = None
    version = pkg_resources.get_distribution('cloudify-agent-packager').version
    return version

def _run(test_options=None):
    version = ver_check()
    options = test_options or docopt(__doc__, version=version)
    packager.set_global_verbosity_level(options.get('--verbose'))
    lgr.debug(options)

    packager.create(
        config_file=options.get('--config'),
        force=options.get('--force'),
        dryrun=options.get('--dryrun'),
        no_validate=options.get('--no-validation'),
        verbose=options.get('--verbose')
        )

def main():
    _run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># This Source Code Form is licensed MPL-2.0: http://mozilla.org/MPL/2.0
"""AidaJsonipcStub - Aida Code Generator for Jsonipc

More details at https://beast.testbit.org/
"""
import Decls, re, sys, os

# == Utilities ==
def hasancestor (child, parent):
  for p in child.prerequisites:
    if p == parent or hasancestor (p, parent):
      return True
def inherit_reduce (type_list):
  # find the type(s) we *directly* derive from
  reduced = []
  while type_list:
    p = type_list.pop()
    skip = 0
    for c in type_list + reduced:
      if c == p or hasancestor (c, p):
        skip = 1
        break
    if not skip:
      reduced = [ p ] + reduced
  return reduced
def bases (tp):
  ancestors = [pr for pr in tp.prerequisites]
  return inherit_reduce (ancestors)
def type_namespace_names (tp):
  namespaces = tp.list_namespaces() # [ Namespace... ]
  return [d.name for d in namespaces if d.name]
def identifier_name (joiner, type_name, member = None):
  parts = type_namespace_names (type_name) + [ type_name.name ]
  if member: parts += [ member ]
  return joiner.join (parts)
def ident (name):
  return re.sub (r'[^a-z0-9_A-Z]+', '_', name)
def enumstring (name):
  return re.sub (r'[^a-z0-9A-Z]+', '-', name.lower())
def get_cxxclass (tp):
  return identifier_name ('::', tp)
def get_cxxiface (tp):
  return get_cxxclass (tp) + 'Iface'
def get_cxximpl (tp):
  return get_cxxclass (tp) + 'Impl'
def get_jsclass (tp):
  return tp.name

class Generator:
  def __init__ (self, idl_file, module_name):
    assert isinstance (module_name, str)
    self.idl_file = idl_file
    self.module_name = module_name
    self.strip_path = ""
    self.idcounter = 1001
    self.nblocks = 0
  def generate_types_jsonipc (self, implementation_types):
    s  = '// === Generated by JsonipcStub.py ===\n'
    # includes
    s += '#include "jsonipc/jsonipc.hh"\n'
    s += '\n'
    # collect impl types
    namespaces = []
    base_objects = []
    types = []
    for tp in implementation_types:
      if tp.isimpl:
        types += [ tp ]
        namespaces.append (tp.list_namespaces())
    if not types:
      return s
    # check unique namespace
    while len (namespaces) >= 2 and namespaces[0] == namespaces[1]:
      namespaces = namespaces[1:]
    namespaces = [[n for n in names if n.name] for names in namespaces] # strip (initial) empty names
    max_namespaces = max (len (namespaces), len (namespaces[0]))
    if max_namespaces != 1:
      raise NotImplementedError ('Jsonipc code generation requires exactly 1 namespace (%d given)' % max_namespaces)
    self.namespace = namespaces[0][0].name
    del namespaces, max_namespaces
    # Collect Jsonipc::Class types
    jip_enum_types, jip_class_types = [], []
    for tp in types:
      if tp.is_forward:
        continue
      if tp.storage in (Decls.SEQUENCE, Decls.RECORD, Decls.INTERFACE):
        jip_class_types += [ tp ]
      elif tp.storage == Decls.ENUM:
        jip_enum_types += [ tp ]
    # split for parallel compilation
    if self.nblocks:
      total = len (jip_enum_types) + len (jip_class_types) * 2 # *2 for Impl types
      nblocktypes = max (1, (total + self.nblocks - 1) / self.nblocks)
    else:
      nblocktypes = 9e99
    # Main binding registration
    s += 'static void\n'
    s += '%s_jsonipc_stub ()\n{\n' % self.namespace
    nthblock = 1
    if nblocktypes < 9e99:
      s += '#if BLOCK_TYPES == %d\n' % nthblock
      nthblock += 1
    typecount = 0
    for tp in jip_enum_types:
      if typecount >= nblocktypes:
        s += '#endif\n#if BLOCK_TYPES == %d\n' % nthblock
        nthblock += 1
        typecount = 0
      typecount += 1
      # tp.storage == Decls.ENUM:
      cxxclass, jsclass = get_cxxclass (tp), get_jsclass (tp)
      s += '  Jsonipc::Enum<%s> jsonipc__%s ("%s");\n' % (cxxclass, ident (cxxclass), jsclass)
      s += '  jsonipc__%s\n' % ident (cxxclass)
      for opt in tp.options:
        (evident, label, blurb, number) = opt
        # s += '  %s.set_const ("%s", %s);\n' % (v8ppenum (tp), ident, identifier_name ('::', tp, ident))
        s += '    .set (%s, "%s") // %s\n' % (identifier_name ('::', tp, evident), evident, enumstring (get_cxxclass (tp) + '-' + evident))
      s += '  ;\n'
    if jip_class_types:
      s += '  Jsonipc::Class<Aida::ImplicitBase> jsonipc__Aida_ImplicitBase ("AidaImplicitBase");\n'
    for tp in jip_class_types:
      if typecount >= nblocktypes:
        s += '#endif\n#if BLOCK_TYPES == %d\n' % nthblock
        nthblock += 1
        typecount = 0
      typecount += 1
      b = ''
      # Records
      if tp.storage == Decls.RECORD:
        cxxclass, jsclass = get_cxxclass (tp), get_jsclass (tp)
        s += '  Jsonipc::Serializable<%s> jsonipc__%s ("%s");\n' % (cxxclass, ident (cxxclass), jsclass)
        # fields
        for fname, ftp in tp.fields:
          b += '    .set ("%s", &%s::%s)\n' % (fname, cxxclass, fname)
        # output non-empty binding
        if b:
          s += '  jsonipc__%s\n' % ident (cxxclass) + b + '  ;\n'
      # Interfaces
      if tp.storage == Decls.INTERFACE:
        cxxclass, jsclass = get_cxxiface (tp), get_jsclass (tp)
        s += '  Jsonipc::Class<%s> jsonipc__%s ("%sIface");\n' % (cxxclass, ident (cxxclass), jsclass)
        # inherit
        for tb in bases (tp):
          b += '    .inherit<%s>()\n' % get_cxxiface (tb)
        if not tp.prerequisites: # no bases
          b += '    .inherit<Aida::ImplicitBase>()\n'
        # properties
        for fname, ftp in tp.fields:
          b += '    .set ("%s", ' % fname
          b += '&%s::%s, &%s::%s)\n' % (cxxclass, fname, cxxclass, fname)
        # methods
        for mtp in tp.methods:
          mname = mtp.name
          b += '    .set ("%s", &%s::%s)\n' % (mname, cxxclass, mname)
        # output non-empty binding
        if b:
          s += '  jsonipc__%s\n' % ident (cxxclass) + b + '  ;\n'
    # Registration of conventional Impl types
    for tp in []: # for tp in jip_class_types:
      if typecount >= nblocktypes:
        s += '#endif\n#if BLOCK_TYPES == %d\n' % nthblock
        nthblock += 1
        typecount = 0
      typecount += 1
      if tp.storage == Decls.INTERFACE:
        cxxiface, cxximpl, jsclass = get_cxxiface (tp), get_cxximpl (tp), get_jsclass (tp)
        jsclass_ = 'Object_' if jsclass == 'Object' else jsclass # work around Javascript nameclash
        s += '  Jsonipc::Class<%s> jsonipc__%s ("%s");\n' % (cxximpl, ident (cxximpl), jsclass_)
        s += '  jsonipc__%s\n' % ident (cxximpl)
        s += '    .inherit<%s>()\n' % cxxiface
        s += '  ;\n'
    if nblocktypes < 9e99:
      s += '#endif\n'
    s += '}\n'
    return s

def generate (namespace_list, **args):
  import sys, tempfile, os
  config = {}
  config.update (args)
  if '--print-include-path' in config.get ('backend-options', []):
    includestem = os.path.dirname (os.path.dirname (os.path.abspath (__file__)))
    includepath = os.path.join (includestem, 'jsonipc')
    print includepath
  else:
    outname = config.get ('output', 'testmodule')
    if outname == '-':
      raise RuntimeError ("-: stdout is not support for generation of multiple files")
    idlfiles = config['files']
    if len (idlfiles) != 1:
      raise RuntimeError ("JsonipcStub: exactly one IDL input file is required")
    gg = Generator (idlfiles[0], outname)
    for opt in config['backend-options']:
      if opt.startswith ('strip-path='):
        gg.strip_path += opt[11:]
      if opt.startswith ('nblocks='):
        gg.nblocks = int (opt[8:])
    fname = outname
    fout = open (fname, 'w')
    textstring = gg.generate_types_jsonipc (config['implementation_types'])
    fout.write (textstring)
    fout.close()

# register extension hooks
__Aida__.add_backend (__file__, generate, __doc__)
<commit_msg>AIDACC: JsonipcStub.py: fix count of BLOCK_TYPES<commit_after># This Source Code Form is licensed MPL-2.0: http://mozilla.org/MPL/2.0
"""AidaJsonipcStub - Aida Code Generator for Jsonipc

More details at https://beast.testbit.org/
"""
import Decls, re, sys, os

# == Utilities ==
def hasancestor (child, parent):
  for p in child.prerequisites:
    if p == parent or hasancestor (p, parent):
      return True
def inherit_reduce (type_list):
  # find the type(s) we *directly* derive from
  reduced = []
  while type_list:
    p = type_list.pop()
    skip = 0
    for c in type_list + reduced:
      if c == p or hasancestor (c, p):
        skip = 1
        break
    if not skip:
      reduced = [ p ] + reduced
  return reduced
def bases (tp):
  ancestors = [pr for pr in tp.prerequisites]
  return inherit_reduce (ancestors)
def type_namespace_names (tp):
  namespaces = tp.list_namespaces() # [ Namespace... ]
  return [d.name for d in namespaces if d.name]
def identifier_name (joiner, type_name, member = None):
  parts = type_namespace_names (type_name) + [ type_name.name ]
  if member: parts += [ member ]
  return joiner.join (parts)
def ident (name):
  return re.sub (r'[^a-z0-9_A-Z]+', '_', name)
def enumstring (name):
  return re.sub (r'[^a-z0-9A-Z]+', '-', name.lower())
def get_cxxclass (tp):
  return identifier_name ('::', tp)
def get_cxxiface (tp):
  return get_cxxclass (tp) + 'Iface'
def get_cxximpl (tp):
  return get_cxxclass (tp) + 'Impl'
def get_jsclass (tp):
  return tp.name

class Generator:
  def __init__ (self, idl_file, module_name):
    assert isinstance (module_name, str)
    self.idl_file = idl_file
    self.module_name = module_name
    self.strip_path = ""
    self.idcounter = 1001
    self.nblocks = 0
  def generate_types_jsonipc (self, implementation_types):
    s  = '// === Generated by JsonipcStub.py ===\n'
    # includes
    s += '#include "jsonipc/jsonipc.hh"\n'
    s += '\n'
    # collect impl types
    namespaces = []
    base_objects = []
    types = []
    for tp in implementation_types:
      if tp.isimpl:
        types += [ tp ]
        namespaces.append (tp.list_namespaces())
    if not types:
      return s
    # check unique namespace
    while len (namespaces) >= 2 and namespaces[0] == namespaces[1]:
      namespaces = namespaces[1:]
    namespaces = [[n for n in names if n.name] for names in namespaces] # strip (initial) empty names
    max_namespaces = max (len (namespaces), len (namespaces[0]))
    if max_namespaces != 1:
      raise NotImplementedError ('Jsonipc code generation requires exactly 1 namespace (%d given)' % max_namespaces)
    self.namespace = namespaces[0][0].name
    del namespaces, max_namespaces
    # Collect Jsonipc::Class types
    jip_enum_types, jip_class_types = [], []
    for tp in types:
      if tp.is_forward:
        continue
      if tp.storage in (Decls.SEQUENCE, Decls.RECORD, Decls.INTERFACE):
        jip_class_types += [ tp ]
      elif tp.storage == Decls.ENUM:
        jip_enum_types += [ tp ]
    # split for parallel compilation
    if self.nblocks:
      total = len (jip_enum_types) + len (jip_class_types)
      nblocktypes = max (1, (total + self.nblocks - 1) / self.nblocks)
    else:
      nblocktypes = 9e99
    # Main binding registration
    s += 'static void\n'
    s += '%s_jsonipc_stub ()\n{\n' % self.namespace
    nthblock = 1
    if nblocktypes < 9e99:
      s += '#if BLOCK_TYPES == %d\n' % nthblock
      nthblock += 1
    typecount = 0
    for tp in jip_enum_types:
      if typecount >= nblocktypes:
        s += '#endif\n#if BLOCK_TYPES == %d\n' % nthblock
        nthblock += 1
        typecount = 0
      typecount += 1
      # tp.storage == Decls.ENUM:
      cxxclass, jsclass = get_cxxclass (tp), get_jsclass (tp)
      s += '  Jsonipc::Enum<%s> jsonipc__%s ("%s");\n' % (cxxclass, ident (cxxclass), jsclass)
      s += '  jsonipc__%s\n' % ident (cxxclass)
      for opt in tp.options:
        (evident, label, blurb, number) = opt
        # s += '  %s.set_const ("%s", %s);\n' % (v8ppenum (tp), ident, identifier_name ('::', tp, ident))
        s += '    .set (%s, "%s") // %s\n' % (identifier_name ('::', tp, evident), evident, enumstring (get_cxxclass (tp) + '-' + evident))
      s += '  ;\n'
    if jip_class_types:
      s += '  Jsonipc::Class<Aida::ImplicitBase> jsonipc__Aida_ImplicitBase ("AidaImplicitBase");\n'
    for tp in jip_class_types:
      if typecount >= nblocktypes:
        s += '#endif\n#if BLOCK_TYPES == %d\n' % nthblock
        nthblock += 1
        typecount = 0
      typecount += 1
      b = ''
      # Records
      if tp.storage == Decls.RECORD:
        cxxclass, jsclass = get_cxxclass (tp), get_jsclass (tp)
        s += '  Jsonipc::Serializable<%s> jsonipc__%s ("%s");\n' % (cxxclass, ident (cxxclass), jsclass)
        # fields
        for fname, ftp in tp.fields:
          b += '    .set ("%s", &%s::%s)\n' % (fname, cxxclass, fname)
        # output non-empty binding
        if b:
          s += '  jsonipc__%s\n' % ident (cxxclass) + b + '  ;\n'
      # Interfaces
      if tp.storage == Decls.INTERFACE:
        cxxclass, jsclass = get_cxxiface (tp), get_jsclass (tp)
        s += '  Jsonipc::Class<%s> jsonipc__%s ("%sIface");\n' % (cxxclass, ident (cxxclass), jsclass)
        # inherit
        for tb in bases (tp):
          b += '    .inherit<%s>()\n' % get_cxxiface (tb)
        if not tp.prerequisites: # no bases
          b += '    .inherit<Aida::ImplicitBase>()\n'
        # properties
        for fname, ftp in tp.fields:
          b += '    .set ("%s", ' % fname
          b += '&%s::%s, &%s::%s)\n' % (cxxclass, fname, cxxclass, fname)
        # methods
        for mtp in tp.methods:
          mname = mtp.name
          b += '    .set ("%s", &%s::%s)\n' % (mname, cxxclass, mname)
        # output non-empty binding
        if b:
          s += '  jsonipc__%s\n' % ident (cxxclass) + b + '  ;\n'
    # close BLOCK_TYPES blocks
    if nblocktypes < 9e99:
      s += '#endif\n'
    # Registration of conventional Impl types
    for tp in []: # for tp in jip_class_types:
      if tp.storage == Decls.INTERFACE:
        cxxiface, cxximpl, jsclass = get_cxxiface (tp), get_cxximpl (tp), get_jsclass (tp)
        jsclass_ = 'Object_' if jsclass == 'Object' else jsclass # work around Javascript nameclash
        s += '  Jsonipc::Class<%s> jsonipc__%s ("%s");\n' % (cxximpl, ident (cxximpl), jsclass_)
        s += '  jsonipc__%s\n' % ident (cxximpl)
        s += '    .inherit<%s>()\n' % cxxiface
        s += '  ;\n'
    s += '}\n'
    return s

def generate (namespace_list, **args):
  import sys, tempfile, os
  config = {}
  config.update (args)
  if '--print-include-path' in config.get ('backend-options', []):
    includestem = os.path.dirname (os.path.dirname (os.path.abspath (__file__)))
    includepath = os.path.join (includestem, 'jsonipc')
    print includepath
  else:
    outname = config.get ('output', 'testmodule')
    if outname == '-':
      raise RuntimeError ("-: stdout is not support for generation of multiple files")
    idlfiles = config['files']
    if len (idlfiles) != 1:
      raise RuntimeError ("JsonipcStub: exactly one IDL input file is required")
    gg = Generator (idlfiles[0], outname)
    for opt in config['backend-options']:
      if opt.startswith ('strip-path='):
        gg.strip_path += opt[11:]
      if opt.startswith ('nblocks='):
        gg.nblocks = int (opt[8:])
    fname = outname
    fout = open (fname, 'w')
    textstring = gg.generate_types_jsonipc (config['implementation_types'])
    fout.write (textstring)
    fout.close()

# register extension hooks
__Aida__.add_backend (__file__, generate, __doc__)
<|endoftext|>


<commit_before>import asyncio
import struct
from riak_pb import messages

MAX_CHUNK_SIZE = 65536

async def create_connection(host='localhost', port=8087, loop=None):
    reader, writer = await asyncio.open_connection(
        host, port, loop=loop)
    conn = RiakConnection(reader, writer, loop=loop)
    return conn

class RPBParser:
    def __init__(self, *args):
        self._data = bytearray(*args)
        self._writer = self._feed_data()
        self._header_parsed = False
        self._exception = None
        self._eof = False
        self._msglen = 0
        self._msg = bytearray()
        next(self._writer)

    def at_eof(self):
        return self._eof

    def _feed_data(self):
        while True:
            chunk = yield
            if chunk:
                self._data.extend(chunk)
                if not self._header_parsed and len(self._data) >= 4:
                    self._msglen, = struct.unpack('!i', self._data[:4])
                    self._header_parsed = True
                if self._header_parsed and \
                        len(self._data) >= self._msglen + 4:
                    self._eof = True
                    self._msg = self._data[4:self._msglen + 4]
                    msg_code, = struct.unpack("B", self._msg[:1])
                    if msg_code is messages.MSG_CODE_ERROR_RESP:
                        raise Exception('Raik error')
                    elif msg_code in messages.MESSAGE_CLASSES:
                        print('Normal message')
                    else:
                        raise Exception('Unknown message code')
            if self._exception:
                raise self._exception

    def feed_data(self, data):
        if not self._exception:
            self._writer.send(data)

class RiakConnection:

    def __init__(self, reader, writer, loop=None):
        self._loop = loop or asyncio.get_event_loop()
        self._writer = writer
        self._reader = reader
        self._parser = RPBParser()

    def _encode_message(self, msg_code, msg=None):
        if msg is None:
            return struct.pack("!iB", 1, msg_code)
        # msgstr = msg.SerializeToString()
        # slen = len(msgstr)
        # hdr = struct.pack("!iB", 1 + slen, msg_code)
        # return hdr + msgstr

    def _request(self, msg_code, msg=None):
        self._writer.write(self._encode_message(msg_code, msg))
        print('+')
        response = self._read_response()
        print('self._request', response)
        return response

    async def _read_response(self):
        while not self._reader.at_eof():
            print('at eof:', self._reader.at_eof())
            try:
                data = await self._reader.read(MAX_CHUNK_SIZE)
            except asyncio.CancelledError:
                break
            except Exception:
                # XXX: for QUIT command connection error can be received
                #       before response
                # logger.error("Exception on data read %r", exc, exc_info=True)
                break
            self._parser.feed_data(data)
            if self._parser.at_eof():
                break
            print('data:', data)
        return self._parser._msg

    async def ping(self):
        response = await self._request(messages.MSG_CODE_PING_REQ)
        return response

if __name__ == '__main__':
    loop = asyncio.get_event_loop()

    async def ping():
        conn = await create_connection(loop=loop)
        val = await conn.ping()
        print(val)

    loop.run_until_complete(ping())
<commit_msg>Reinitialize RPBParser on new response mesage<commit_after>import asyncio
import struct
from riak_pb import messages

MAX_CHUNK_SIZE = 65536

async def create_connection(host='localhost', port=8087, loop=None):
    reader, writer = await asyncio.open_connection(
        host, port, loop=loop)
    conn = RiakConnection(reader, writer, loop=loop)
    return conn

class RPBParser:
    def __init__(self, *args):
        self._data = bytearray(*args)
        self._writer = self._feed_data()
        self._header_parsed = False
        self._exception = None
        self._eof = False
        self._msglen = 0
        self._msg = bytearray()
        next(self._writer)

    def at_eof(self):
        return self._eof

    def _feed_data(self):
        while True:
            chunk = yield
            if chunk:
                self._data.extend(chunk)
                if not self._header_parsed and len(self._data) >= 4:
                    self._msglen, = struct.unpack('!i', self._data[:4])
                    self._header_parsed = True
                if self._header_parsed and \
                        len(self._data) >= self._msglen + 4:
                    self._eof = True
                    self._msg = self._data[4:self._msglen + 4]
                    self.msg_code, = struct.unpack("B", self._msg[:1])
                    if self.msg_code is messages.MSG_CODE_ERROR_RESP:
                        raise Exception('Raik error', self._msg)
                    elif self.msg_code in messages.MESSAGE_CLASSES:
                        print('Normal message')
                    else:
                        raise Exception('Unknown message code')
            if self._exception:
                raise self._exception

    def feed_data(self, data):
        if not self._exception:
            self._writer.send(data)

class RiakConnection:

    def __init__(self, reader, writer, loop=None):
        self._loop = loop or asyncio.get_event_loop()
        self._writer = writer
        self._reader = reader
        self._parser = RPBParser()

    def _encode_message(self, msg_code, msg=None):
        if msg is None:
            return struct.pack("!iB", 1, msg_code)
        # msgstr = msg.SerializeToString()
        # slen = len(msgstr)
        # hdr = struct.pack("!iB", 1 + slen, msg_code)
        # return hdr + msgstr

    def _request(self, msg_code, msg=None):
        self._writer.write(self._encode_message(msg_code, msg))
        print('+')
        response = self._read_response()
        print('self._request', response)
        self._parser = RPBParser()
        return response

    async def _read_response(self):
        while not self._reader.at_eof():
            print('at eof:', self._reader.at_eof())
            try:
                data = await self._reader.read(MAX_CHUNK_SIZE)
            except asyncio.CancelledError:
                break
            except Exception:
                # XXX: for QUIT command connection error can be received
                #       before response
                # logger.error("Exception on data read %r", exc, exc_info=True)
                break
            self._parser.feed_data(data)
            if self._parser.at_eof():
                break
            print('data:', data)
        return self._parser.msg_code

    async def ping(self, error=False):
        if error:
            response = await self._request(messages.MSG_CODE_PING_RESP)
        else:
            response = await self._request(messages.MSG_CODE_PING_REQ)
        return response

if __name__ == '__main__':
    loop = asyncio.get_event_loop()

    async def ping():
        conn = await create_connection(loop=loop)
        print('=' * 80)
        val = await conn.ping()
        print(val)
        print('=' * 80)
        val = await conn.ping(error=True)
        print(val)
        print('=' * 80)

    loop.run_until_complete(ping())
<|endoftext|>


<commit_before>from graph_tool.all import *
from words import words
from valuable import words as dictionary
from math import log10

def get_vertex(name, vertices):
    # If we already have this vertex, just use it
    if name in vertices:
        return vertices[name]
    # Otherwise we have to create new one
    a = g.add_vertex()
    word[a] = name
    vertices[name] = a
    return a

if __name__ == '__main__':

    # Create directed graph
    g = Graph(directed=True)

    # Create 'weight' property for edge: will contain tf-idf
    weight = g.new_edge_property('float')
    # Create 'word' property for vertex: will contain string with current word
    word = g.new_vertex_property('string')
    # Dictionary with existent vertices (cache)
    vertices = {}

    for w in words:
        # Check, whether at least one word is in dictionary of valuable words
        if w[0] not in dictionary and w[1] not in dictionary:
            continue
        # Check, whether at least one word is valuable
        if dictionary[w[0]] < 0.1 and dictionary[w[1]] < 0.1:
            continue
        a = get_vertex(w[0], vertices)
        b = get_vertex(w[1], vertices)
        e = g.add_edge(a, b)
        # Set weight for new edge (tf-idf)
        weight[e] = log10(w[2])*10 + 23

    graph_draw(g, vertex_font_size=12, edge_pen_width=weight, vertex_text=word,
               output='words.png', output_size=(2000, 2000))

<commit_msg>Added comments to lab4<commit_after>from graph_tool.all import *
from words import words
from valuable import words as dictionary
from math import log10

def get_vertex(name, vertices):
    # If we already have this vertex, just use it from cache
    if name in vertices:
        return vertices[name]
    # Otherwise we have to create new one
    a = g.add_vertex()
    word[a] = name
    # Add the vertex to cache
    vertices[name] = a
    return a

if __name__ == '__main__':

    # Create directed graph
    g = Graph(directed=True)

    # Create 'weight' property for edge: will contain tf-idf
    weight = g.new_edge_property('float')
    # Create 'word' property for vertex: will contain string with current word
    word = g.new_vertex_property('string')
    # Dictionary with existent vertices (cache)
    vertices = {}

    for w in words:
        # Check, whether at least one word is in dictionary of valuable words
        if w[0] not in dictionary and w[1] not in dictionary:
            continue
        # Check, whether at least one word is valuable
        if dictionary[w[0]] < 0.1 and dictionary[w[1]] < 0.1:
            continue
        a = get_vertex(w[0], vertices)
        b = get_vertex(w[1], vertices)
        e = g.add_edge(a, b)
        # Set weight for new edge (tf-idf)
        # Just empirical formula, which gives edges with 1 <= width < 30
        weight[e] = log10(w[2])*10 + 23

    # Draw the graph;
    # Weight is responsible for edges widths
    # Word contains labels for vertices
    graph_draw(g, vertex_font_size=12, edge_pen_width=weight, vertex_text=word,
               output='words.png', output_size=(2000, 2000))

<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os
import re
import requests
import logging
from subprocess import Popen, PIPE

from .utils import conf_to_dict
from .env import CondaCreator
from .compatibility import FileNotFoundError, urlparse
from .exceptions import HDFSConfigException

logger = logging.getLogger(__name__)

JAR_FILE = "knit-1.0-SNAPSHOT.jar"
JAVA_APP = "io.continuum.knit.Client"

class Knit(object):
    """
    Connection to HDFS/YARN

    Parameters
    ----------
    nn: str
        Namenode hostname/ip
    nn: str
        Namenode hostname/ip
    nn_port: int
        Namenode Port (default: 9000)
    rm: str
        Resource Manager hostname
    rm_port: int
        Resource Manager port (default: 8088)
    autodetect: bool
        Autodetect NN/RM IP/Ports

    Examples
    --------

    >>> k = Knit()
    >>> app_id = k.start('sleep 100', num_containers=5, memory=1024)
    """
    def __init__(self, nn="localhost", nn_port=9000,
                 rm="localhost", rm_port=8088, autodetect=False):

        if autodetect:
             self.nn,  self.nn_port = self._yarn_conf(autodetect)
             self.rm,  self.rm_port = self._hdfs_conf(autodetect)

        self.nn = nn
        self.nn_port = str(nn_port)

        self.rm = rm
        self.rm_port = str(rm_port)

        self.java_lib_dir = os.path.join(os.path.dirname(__file__), "java_libs")
        self.KNIT_HOME = os.environ.get('KNIT_HOME') or self.java_lib_dir

        # must set KNIT_HOME ENV for YARN App
        os.environ['KNIT_HOME'] = self.KNIT_HOME
        self._hdfs_conf()
        self._yarn_conf()

    @property
    def JAR_FILE_PATH(self):
        return os.path.join(self.KNIT_HOME, JAR_FILE)

    def _yarn_conf(self, autodetect=False):
        """
        Load YARN config from default locations.
        Parameters
        ----------
        autodetect: bool

        Returns
        -------
            tuple (ip, port)

        """
        confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                               '') + '/hadoop/conf')
        afile = 'yarn-site.xml'
        conf = {}

        try:
            conf.update(conf_to_dict(os.sep.join([confd, afile])))
        except FileNotFoundError:
            pass
        if 'yarn.resourcemanager.webapp.address' in conf:
            url = conf['yarn.resourcemanager.webapp.address']
            u = urlparse(url)

            # handle host:port with no :// preabmle
            if u.path == url:
                conf['host'], conf['port'] = url.split(':')
            else:
                conf['host'] = u.hostname
                conf['port'] = u.port
        else:
            conf['host'] = "localhost"
            conf['port'] = "8088"

        if autodetect:
            return (conf['host'], conf['port'])

        if self.rm != conf['host']:
            msg = "Possible Resource Manager hostname mismatch.  Detected {}".format(conf['host'])
            raise HDFSConfigException(msg)
        if self.rm_port != conf['port']:
            msg = "Possible Resource Manager port mismatch.  Detected {}".format(conf['port'])
            raise HDFSConfigException(msg)

        return conf

    def _hdfs_conf(self, autodetect=False):
        """"
        Parameters
        ----------
        autodetect: bool

        Returns
        -------
            tuple (ip, port)

        """

        confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                               '') + '/hadoop/conf')
        files = 'core-site.xml', 'hdfs-site.xml'
        conf = {}
        for afile in files:
            try:
                conf.update(conf_to_dict(os.sep.join([confd, afile])))
            except FileNotFoundError:
                pass
        if 'fs.defaultFS' in conf:
            u = urlparse(conf['fs.defaultFS'])
            conf['host'] = u.hostname
            conf['port'] = u.port
        else:
            conf['host'] = "localhost"
            conf['port'] = "9000"

        if autodetect:
            return (conf['host'], conf['port'])

        if self.nn != conf['host']:
            msg = "Possible Namenode hostname mismatch.  Detected {}".format(conf['host'])
            raise HDFSConfigException(msg)
        if self.nn_port != str(conf['port']):
            msg = "Possible Namenode port mismatch.  Detected {}".format(conf['port'])
            raise HDFSConfigException(msg)

        return conf

    def start(self, cmd, num_containers=1, virtual_cores=1, memory=128, env=""):
        """
        Method to start a yarn app with a distributed shell

        Parameters
        ----------
        cmd: str
            command to run in each yarn container
        num_containers: int
            number of containers to start (default 1)
        num_containers: int
            Number of containers YARN should request (default: 1)
            * A container should be requested with the number of cores it can saturate, i.e.
            * the average number of threads it expects to have runnable at a time.
        virtual_cores: int
            Number of virtual cores per container (default: 1)
            * A node's capacity should be configured with virtual cores equal to
            * its number of physical cores.
        memory: int
            Memory per container (default: 128)
            * The unit for memory is megabytes.
        env: string
            Full Path to zipped Python environment

        Returns
        -------
        applicationId: str
            A yarn application ID string
        """

        args = ["hadoop", "jar", self.JAR_FILE_PATH, JAVA_APP, "--numContainers", str(num_containers),
                "--command", cmd, "--virtualCores", str(virtual_cores), "--memory", str(memory)]

        if env:
            args = args + ["--pythonEnv", str(env)]

        logger.debug("Running Command: {}".format(' '.join(args)))
        proc = Popen(args, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()

        logger.debug(out)
        logger.debug(err)

        # last string in out is applicationId
        # TODO Better JAVA Python communcation: appId, Resources, Yarn, etc.

        appId = out.split()[-1].decode("utf-8")
        appId = re.sub('id', '', appId)
        return appId

    def logs(self, app_id, shell=False):
        """
        Collect logs from each container

        Parameters
        ----------
        app_id: str
             A yarn application ID string
        shell: bool
             Shell out to yarn CLI (default False)

        Returns
        -------
        log: dictionary
            logs from each container
        """

        if shell:

            args = ["yarn", "logs", "-applicationId", app_id]

            proc = Popen(args, stdout=PIPE, stderr=PIPE)
            out, err = proc.communicate()

            logger.debug(out)
            logger.debug(err)
            return str(out)

        host_port = "{}:{}".format(self.rm, self.rm_port)
        url = "http://{}/ws/v1/cluster/apps/{}".format(host_port, app_id)
        logger.debug("Getting Resource Manager Info: {}".format(url))
        r = requests.get(url)
        data = r.json()
        logger.debug(data)

        try:
            amHostHttpAddress = data['app']['amHostHttpAddress']
        except KeyError:
            msg = "Local logs unavailable. State: {} finalStatus: {} Possibly check logs " \
                  "with `yarn logs -applicationId`".format(data['app']['state'],
                                                           data['app']['finalStatus'])
            raise Exception(msg)

        url = "http://{}/ws/v1/node/containers".format(amHostHttpAddress)
        r = requests.get(url)
        data = r.json()['containers']['container']
        logger.debug(data)

        # container_1452274436693_0001_01_000001
        def get_app_id_num(x):
            return "_".join(x.split("_")[1:3])

        app_id_num = get_app_id_num(app_id)
        containers = [d for d in data if get_app_id_num(d['id']) == app_id_num]

        logs = {}
        for c in containers:
            log = {}
            log['nodeId'] = c['nodeId']

            # grab stdout
            url = "{}/stdout/?start=0".format(c['containerLogsLink'])
            logger.debug("Gather stdout/stderr data from {}: {}".format(c['nodeId'], url))
            r = requests.get(url)
            log['stdout'] = r.text

            # grab stderr
            url = "{}/stderr/?start=0".format(c['containerLogsLink'])
            r = requests.get(url)
            log['stderr'] = r.text

            logs[c['id']] = log

        return logs

    def status(self, app_id):
        """ Get status of an application

        Parameters
        ----------
        app_id: str
             A yarn application ID string

        Returns
        -------
        log: dictionary
            status of application
        """
        host_port = "{}:{}".format(self.rm, self.rm_port)
        url = "http://{}/ws/v1/cluster/apps/{}".format(host_port, app_id)
        logger.debug("Getting Application Info: {}".format(url))
        r = requests.get(url)
        data = r.json()

        return data

    def kill(self, application_id):
        """
        Method to kill a yarn application

        Parameters
        ----------
        application_id: str
            YARN applicaiton id

        Returns
        -------
        bool:
            True if successful, False otherwise.
        """

        args = ["yarn", "application", "-kill", application_id]

        proc = Popen(args, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()

        logger.debug(out)
        logger.debug(err)

        return any("Killed application" in s for s in [str(out), str(err)])

    @staticmethod
    def create_env(env_name, packages=None, conda_root=None, remove=False):
        """
        Create zipped directory of a conda environment

        Parameters
        ----------
        env_name : str
        packages : list
        conda_root : str, optional
        remove : bool
            remove possible conda environment before creating

        Returns
        -------
        path: str
            path to zipped conda environment

        Examples
        --------

        >>> k = Knit()
        >>> pkg_path = k.create_env(env_name='dev', packages=[python=3, numpy, distributed])
        """

        c = CondaCreator(conda_root=conda_root)
        path = c.create_env(env_name, packages=packages, remove=remove)

        return path

<commit_msg>small fix for autodetecting ip/ports<commit_after>from __future__ import absolute_import, division, print_function

import os
import re
import requests
import logging
from subprocess import Popen, PIPE

from .utils import conf_to_dict
from .env import CondaCreator
from .compatibility import FileNotFoundError, urlparse
from .exceptions import HDFSConfigException

logger = logging.getLogger(__name__)

JAR_FILE = "knit-1.0-SNAPSHOT.jar"
JAVA_APP = "io.continuum.knit.Client"

class Knit(object):
    """
    Connection to HDFS/YARN

    Parameters
    ----------
    nn: str
        Namenode hostname/ip
    nn: str
        Namenode hostname/ip
    nn_port: str
        Namenode Port (default: 9000)
    rm: str
        Resource Manager hostname
    rm_port: str
        Resource Manager port (default: 8088)
    autodetect: bool
        Autodetect NN/RM IP/Ports

    Examples
    --------

    >>> k = Knit()
    >>> app_id = k.start('sleep 100', num_containers=5, memory=1024)
    """
    def __init__(self, nn="localhost", nn_port=9000,
                 rm="localhost", rm_port=8088, autodetect=False):

        self.nn = nn
        self.nn_port = str(nn_port)

        self.rm = rm
        self.rm_port = str(rm_port)

        if autodetect:
            self.nn,  self.nn_port = self._hdfs_conf(autodetect)
            self.rm,  self.rm_port = self._yarn_conf(autodetect)
        else:
            # validates IP/Port is correct
            self._hdfs_conf()
            self._yarn_conf()

        self.java_lib_dir = os.path.join(os.path.dirname(__file__), "java_libs")
        self.KNIT_HOME = os.environ.get('KNIT_HOME') or self.java_lib_dir

        # must set KNIT_HOME ENV for YARN App
        os.environ['KNIT_HOME'] = self.KNIT_HOME

    @property
    def JAR_FILE_PATH(self):
        return os.path.join(self.KNIT_HOME, JAR_FILE)

    def _yarn_conf(self, autodetect=False):
        """
        Load YARN config from default locations.
        Parameters
        ----------
        autodetect: bool

        Returns
        -------
            tuple (ip, port)

        """
        confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                               '') + '/hadoop/conf')
        afile = 'yarn-site.xml'
        conf = {}

        try:
            conf.update(conf_to_dict(os.sep.join([confd, afile])))
        except FileNotFoundError:
            pass
        if 'yarn.resourcemanager.webapp.address' in conf:
            url = conf['yarn.resourcemanager.webapp.address']
            u = urlparse(url)

            # handle host:port with no :// preabmle
            if u.path == url:
                conf['host'], conf['port'] = url.split(':')
            else:
                conf['host'] = u.hostname
                conf['port'] = u.port
        else:
            conf['host'] = "localhost"
            conf['port'] = "8088"

        if autodetect:
            return (conf['host'], conf['port'])

        if self.rm != conf['host']:
            msg = "Possible Resource Manager hostname mismatch.  Detected {}".format(conf['host'])
            raise HDFSConfigException(msg)
        if str(self.rm_port) != str(conf['port']):
            msg = "Possible Resource Manager port mismatch.  Detected {}".format(conf['port'])
            raise HDFSConfigException(msg)

        return conf

    def _hdfs_conf(self, autodetect=False):
        """"
        Parameters
        ----------
        autodetect: bool

        Returns
        -------
            tuple (ip, port)

        """

        confd = os.environ.get('HADOOP_CONF_DIR', os.environ.get('HADOOP_INSTALL',
                               '') + '/hadoop/conf')
        afile = 'core-site.xml'
        conf = {}
        try:
            conf.update(conf_to_dict(os.sep.join([confd, afile])))
        except FileNotFoundError:
            pass

        if 'fs.defaultFS' in conf:
            u = urlparse(conf['fs.defaultFS'])
            conf['host'] = u.hostname
            conf['port'] = u.port
        else:
            conf['host'] = "localhost"
            conf['port'] = "9000"

        if autodetect:
            return (conf['host'], conf['port'])

        if self.nn != conf['host']:
            msg = "Possible Namenode hostname mismatch.  Detected {}".format(conf['host'])
            raise HDFSConfigException(msg)

        if str(self.nn_port) != str(conf['port']):
            msg = "Possible Namenode port mismatch.  Detected {}".format(conf['port'])
            raise HDFSConfigException(msg)

        return conf

    def start(self, cmd, num_containers=1, virtual_cores=1, memory=128, env=""):
        """
        Method to start a yarn app with a distributed shell

        Parameters
        ----------
        cmd: str
            command to run in each yarn container
        num_containers: int
            number of containers to start (default 1)
        num_containers: int
            Number of containers YARN should request (default: 1)
            * A container should be requested with the number of cores it can saturate, i.e.
            * the average number of threads it expects to have runnable at a time.
        virtual_cores: int
            Number of virtual cores per container (default: 1)
            * A node's capacity should be configured with virtual cores equal to
            * its number of physical cores.
        memory: int
            Memory per container (default: 128)
            * The unit for memory is megabytes.
        env: string
            Full Path to zipped Python environment

        Returns
        -------
        applicationId: str
            A yarn application ID string
        """

        args = ["hadoop", "jar", self.JAR_FILE_PATH, JAVA_APP, "--numContainers", str(num_containers),
                "--command", cmd, "--virtualCores", str(virtual_cores), "--memory", str(memory)]

        if env:
            args = args + ["--pythonEnv", str(env)]

        logger.debug("Running Command: {}".format(' '.join(args)))
        proc = Popen(args, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()

        logger.debug(out)
        logger.debug(err)

        # last string in out is applicationId
        # TODO Better JAVA Python communcation: appId, Resources, Yarn, etc.

        appId = out.split()[-1].decode("utf-8")
        appId = re.sub('id', '', appId)
        return appId

    def logs(self, app_id, shell=False):
        """
        Collect logs from each container

        Parameters
        ----------
        app_id: str
             A yarn application ID string
        shell: bool
             Shell out to yarn CLI (default False)

        Returns
        -------
        log: dictionary
            logs from each container
        """

        if shell:

            args = ["yarn", "logs", "-applicationId", app_id]

            proc = Popen(args, stdout=PIPE, stderr=PIPE)
            out, err = proc.communicate()

            logger.debug(out)
            logger.debug(err)
            return str(out)

        host_port = "{}:{}".format(self.rm, self.rm_port)
        url = "http://{}/ws/v1/cluster/apps/{}".format(host_port, app_id)
        logger.debug("Getting Resource Manager Info: {}".format(url))
        r = requests.get(url)
        data = r.json()
        logger.debug(data)

        try:
            amHostHttpAddress = data['app']['amHostHttpAddress']
        except KeyError:
            msg = "Local logs unavailable. State: {} finalStatus: {} Possibly check logs " \
                  "with `yarn logs -applicationId`".format(data['app']['state'],
                                                           data['app']['finalStatus'])
            raise Exception(msg)

        url = "http://{}/ws/v1/node/containers".format(amHostHttpAddress)
        r = requests.get(url)
        data = r.json()['containers']['container']
        logger.debug(data)

        # container_1452274436693_0001_01_000001
        def get_app_id_num(x):
            return "_".join(x.split("_")[1:3])

        app_id_num = get_app_id_num(app_id)
        containers = [d for d in data if get_app_id_num(d['id']) == app_id_num]

        logs = {}
        for c in containers:
            log = {}
            log['nodeId'] = c['nodeId']

            # grab stdout
            url = "{}/stdout/?start=0".format(c['containerLogsLink'])
            logger.debug("Gather stdout/stderr data from {}: {}".format(c['nodeId'], url))
            r = requests.get(url)
            log['stdout'] = r.text

            # grab stderr
            url = "{}/stderr/?start=0".format(c['containerLogsLink'])
            r = requests.get(url)
            log['stderr'] = r.text

            logs[c['id']] = log

        return logs

    def status(self, app_id):
        """ Get status of an application

        Parameters
        ----------
        app_id: str
             A yarn application ID string

        Returns
        -------
        log: dictionary
            status of application
        """
        host_port = "{}:{}".format(self.rm, self.rm_port)
        url = "http://{}/ws/v1/cluster/apps/{}".format(host_port, app_id)
        logger.debug("Getting Application Info: {}".format(url))
        r = requests.get(url)
        data = r.json()

        return data

    def kill(self, application_id):
        """
        Method to kill a yarn application

        Parameters
        ----------
        application_id: str
            YARN applicaiton id

        Returns
        -------
        bool:
            True if successful, False otherwise.
        """

        args = ["yarn", "application", "-kill", application_id]

        proc = Popen(args, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()

        logger.debug(out)
        logger.debug(err)

        return any("Killed application" in s for s in [str(out), str(err)])

    @staticmethod
    def create_env(env_name, packages=None, conda_root=None, remove=False):
        """
        Create zipped directory of a conda environment

        Parameters
        ----------
        env_name : str
        packages : list
        conda_root : str, optional
        remove : bool
            remove possible conda environment before creating

        Returns
        -------
        path: str
            path to zipped conda environment

        Examples
        --------

        >>> k = Knit()
        >>> pkg_path = k.create_env(env_name='dev', packages=[python=3, numpy, distributed])
        """

        c = CondaCreator(conda_root=conda_root)
        path = c.create_env(env_name, packages=packages, remove=remove)

        return path

<|endoftext|>