<commit_before>'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
This is the Mypanel class for GUI module.
It creates the buttons and packs them into box.
It also implements the binding of buttons to various events using functions to
filter, browse location, and download content.

It imports:
    -wx
    -re
    -get_urls
    -downloader_script
It defines:
    -__init__
    -download
    -filter
    -browse
    -cancel
    -close
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
"""Required modules"""
import wx
import get_urls,downloader_script
import re

class Mypanel(object):
    def __init__(self,bkg,win):
        self.win = win
        self.bkg = bkg

        #creating buttons
        download_btn = wx.Button(bkg,label = "download",size=(80,25))
        download_btn.Bind(wx.EVT_BUTTON,self.download)

        browse_btn = wx.Button(bkg,label = "browse",size=(80,25))
        browse_btn.Bind(wx.EVT_BUTTON,self.browse)

        cancel_btn = wx.Button(bkg,label = "cancel",size=(80,25))
        cancel_btn.Bind(wx.EVT_BUTTON,self.cancel)

        clsbtn = wx.Button(bkg,label = "CLOSE",size=(80,25))
        clsbtn.Bind(wx.EVT_BUTTON,self.close)

        fbtn = wx.Button(bkg,label="Filter",size=(80,25))
        fbtn.Bind(wx.EVT_BUTTON,self.filter)

        #defining text areas; to input text

        self.url = wx.TextCtrl(bkg,size=(390,25),pos=(5,5),
                               style=wx.TE_PROCESS_ENTER)
        self.url.Bind(wx.EVT_TEXT_ENTER, self.enter)
        self.dir = wx.TextCtrl(bkg,size=(390,25),pos=(5,30))
        self.contents = wx.TextCtrl(bkg,style = wx.TE_MULTILINE|wx.HSCROLL,
                                   size=(100,245)
                                    )

        self.count = wx.TextCtrl(bkg,size=(255,25),pos=(420,295))
        self.regex = wx.TextCtrl(bkg,size=(255,25),pos=(420,310))
        self.progress = wx.TextCtrl(bkg,size=(390,25),pos=(5,335))

        #Adding CheckBoxes

        cb1 = wx.CheckBox(bkg, -1, "jpeg",
                          (5, 295), (75, 25))
        cb2 = wx.CheckBox(bkg, -1, "png",
                          (5, 310), (75, 25))
        cb3 = wx.CheckBox(bkg, -1, "gif",
                          (5, 335), (75, 25))
        cb4 = wx.CheckBox(bkg, -1, "mp4",
                          (35, 295), (75, 25))
        cb5 = wx.CheckBox(bkg, -1, "3gp",
                          (35,310), (75, 25))
        cb6 = wx.CheckBox(bkg, -1, "avi",
                          (35, 335), (75, 25))
        cb7 = wx.CheckBox(bkg, -1, "flv",
                          (70, 295), (75, 25))
        cb8 = wx.CheckBox(bkg, -1, "mp3",
                          (70, 310), (75, 25))

        #--------------------------------------------------------------------------

        #WRAPPING UP THE BOXES
        #The text controls
        vbox11 = wx.BoxSizer(wx.VERTICAL)
        vbox11.Add(self.url,proportion=0,flag=wx.ALL|wx.EXPAND,border=5)
        vbox11.Add(self.dir,proportion=0,flag=wx.ALL|wx.EXPAND,border=5)
        vbox11.Add(self.contents,proportion=1,flag=wx.TE_MULTILINE
                  |wx.EXPAND
                  |wx.ALL,border=5
                  )
        self.vbox11 = vbox11

        #The checkBoxes
        vbox12 = wx.BoxSizer(wx.VERTICAL)
        vbox12.Add(cb1, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox12.Add(cb2, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox12.Add(cb3, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)

        vbox13 = wx.BoxSizer(wx.VERTICAL)
        vbox13.Add(cb4, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox13.Add(cb5, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox13.Add(cb6, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)

        vbox14 = wx.BoxSizer(wx.VERTICAL)
        vbox14.Add(cb7, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox14.Add(cb8, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)

        #--------------------------------------------------------------------------
        #Extra feature boxes
        vbox15 = wx.BoxSizer(wx.VERTICAL)
        vbox15.Add(self.count,proportion=1,flag=wx.ALL|wx.EXPAND,border=3)
        vbox15.Add(self.regex,proportion=1,flag=wx.ALL|wx.EXPAND,border=3)
        vbox15.Add(fbtn,proportion=0,flag=wx.LEFT,border=178)

        #--------------------------------------------------------------------------
        #All vertical boxes in horizontal box
        hbox1 = wx.BoxSizer()
        hbox1.Add(vbox12,proportion=0,flag=wx.EXPAND)
        hbox1.Add(vbox13,proportion=0,flag=wx.EXPAND)
        hbox1.Add(vbox14,proportion=0,flag=wx.EXPAND)
        hbox1.Add(vbox15,proportion=0)

        #The two vertical boxes in one
        vbox1 = wx.BoxSizer(wx.VERTICAL)
        vbox1.Add(vbox11,proportion=1,flag=wx.EXPAND)
        vbox1.Add(hbox1,proportion=0,flag=wx.EXPAND)
        vbox1.Add(self.progress,proportion=0,flag=wx.ALL|wx.EXPAND,border=5)

        #--------------------------------------------------------------------------
        #Buttons
        vbox21 = wx.BoxSizer(wx.VERTICAL)
        vbox21.Add(download_btn,proportion=0,border=5,flag=wx.TOP)
        vbox21.Add(browse_btn,proportion=0,border=10,flag=wx.TOP)
        vbox21.Add(cancel_btn,proportion=0,border=10,flag=wx.TOP)

        #Close button
        vbox22 = wx.BoxSizer(wx.VERTICAL)
        vbox22.Add(clsbtn,proportion=0,border=5,flag=wx.RIGHT|wx.TOP|wx.BOTTOM)

        #The two vertical boxes in single one.
        vbox2 = wx.BoxSizer(wx.VERTICAL)
        vbox2.Add(vbox21,proportion=1,flag=wx.EXPAND)
        vbox2.Add(vbox22,proportion=0)

        #--------------------------------------------------------------------------

        #Vertical boxes into horizontal hbox.
        hbox = wx.BoxSizer()
        hbox.Add(vbox1,proportion = 1,flag = wx.EXPAND|wx.BOTTOM)
        hbox.Add(vbox2,proportion = 0,flag = wx.EXPAND,border = 5)

        bkg.SetSizer(hbox)

    #--------------------------------------------------------------------------
    def enter(self, event):
        '''
        The function to prepare a list of all urls found on home page,
        It works on text_enter_event of textctrl box called 'url'.
        '''
        #print "hello"
        self.urls = get_urls.main(self.url.GetValue())
        if self.urls:

            #destroying the contents box, so as to put the checklist inplace.
            if self.contents:
                self.contents.Destroy()

            self.check_list = wx.CheckListBox(self.bkg, -1, (5,75),
                                                  (488,245),self.urls,
                                                  style = wx.HSCROLL)

            self.vbox11.Add(self.check_list,proportion=1,flag=wx.EXPAND
                      |wx.ALL,border=5
                      )

            self.check_list.Bind(wx.EVT_CHECKLISTBOX, self.EvtCheckListBox)

            #setting the count of links
            self.count.SetValue("No of links found: "+str(len(self.urls)))

    #--------------------------------------------------------------------------
    def download(self,event):
        '''
        The function is bind with download button and download the content.
        It fetches urls from the 'contents' textctrl and sends urls to
        downloader_script.
        Uses 'browse' filedialogs current path to save files.
        '''
        urls_to_download = self.toDownload
        try:
            downloader_script.main(urls_to_download,self.path)
        except AttributeError:
            print "Please select a location"

    #--------------------------------------------------------------------------
    def browse(self,event):
        '''
        The function is bind with the browse button.
        It opens a directory location.
        Treats the dir input in dir_ box as the default.
        '''

        dir_= self.dir.GetValue()
        dlg = wx.DirDialog(self.win, "Choose a directory:",
                          style=wx.DD_DEFAULT_STYLE
                           | wx.DD_DIR_MUST_EXIST
                           | wx.DD_CHANGE_DIR
                           )

        dlg.SetPath(dir_)
        if dlg.ShowModal() == wx.ID_OK:
            print('You selected: %s\n' % dlg.GetPath())
            self.path = dlg.GetPath()

        dlg.Destroy()
        self.dir.SetValue(self.path)

    #--------------------------------------------------------------------------
    def cancel(self,event):
        '''
        This is bound to cancel button, when pressed, it clears all text areas.
        '''
        self.url.SetValue(" ")
        self.check_list.Destroy()
        self.regex.SetValue(" ")
        self.count.SetValue(" ")
        self.contents = wx.TextCtrl(self.bkg,style = wx.TE_MULTILINE|wx.HSCROLL,
                                   size=(100,245)
                                    )
        self.vbox11.Add(self.contents,proportion=1,flag=wx.TE_MULTILINE
                       |wx.EXPAND
                       |wx.ALL,border=5
                       )
        self.contents.SetValue(" ")

    #--------------------------------------------------------------------------
    def filter(self,event):
        '''
        The function filters links found on the page.
        It uses regex to filter and display a list of urls matching the pattern.
        The pattern is specified in the box called regex.
        '''

        pattern = re.compile(self.regex.GetValue())
        if self.urls:

            filtered = re.findall(pattern,'\n'.join(self.urls))

            self.check_list.Destroy()                                       #Destriying old list
            self.check_list = wx.CheckListBox(self.bkg, -1, (5,75),         #creating new filtered list
                                              (388,245),filtered,
                                              style = wx.HSCROLL)

            self.vbox11.Add(self.check_list,proportion=1,flag=wx.EXPAND      #Adding the list box to container
                  |wx.ALL,border=5
                  )

            #list of selected links
            self.toDownload = []

            self.check_list.Bind(wx.EVT_CHECKLISTBOX, self.EvtCheckListBox)
            self.check_list.SetSelection(0)

    def EvtCheckListBox(self, event):
        '''
        Function to implement checking and unchecking of list items.
        Also, according to checking and unchecking, adds and removes
        items to/from download list.
        '''
        index = event.GetSelection()
        label = self.check_list.GetString(index)
        status = 'un'

        if self.check_list.IsChecked(index):
            self.toDownload.append(self.check_list.GetString(index))
            status = ''
            self.check_list.SetSelection(index)                             #so that (un)checking also selects (moves the highlight)
        if not self.check_list.IsChecked(index):
            self.toDownload.pop(index)
            self.check_list.SetSelection(0)

    #--------------------------------------------------------------------------
    def close(self,event):
        '''
        Destroys the window object.
        '''
        self.win.Destroy()
<commit_msg>adds checkboxes for some common filter types for direct selection, unchecking now removes the item from list, also resolves errors related to url extraction and some minor updation<commit_after>'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
This is the Mypanel class for GUI module.
It creates the buttons and packs them into box.
It also implements the binding of buttons to various events using functions to
filter, browse location, and download content.

It imports:
    -wx
    -re
    -get_urls
    -downloader_script
It defines:
    -__init__
    -download
    -filter
    -browse
    -cancel
    -close
'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
"""Required modules"""
import wx
import get_urls,downloader_script
import re

class Mypanel(object):
    def __init__(self,bkg,win):
        self.win = win
        self.bkg = bkg

        #creating buttons
        download_btn = wx.Button(bkg,label = "download",size=(80,25))
        download_btn.Bind(wx.EVT_BUTTON,self.download)

        browse_btn = wx.Button(bkg,label = "browse",size=(80,25))
        browse_btn.Bind(wx.EVT_BUTTON,self.browse)

        cancel_btn = wx.Button(bkg,label = "cancel",size=(80,25))
        cancel_btn.Bind(wx.EVT_BUTTON,self.cancel)

        clsbtn = wx.Button(bkg,label = "CLOSE",size=(80,25))
        clsbtn.Bind(wx.EVT_BUTTON,self.close)

        fbtn = wx.Button(bkg,label="Filter",size=(80,25))
        fbtn.Bind(wx.EVT_BUTTON,self.filter)

        #defining text areas; to input text

        self.url = wx.TextCtrl(bkg,size=(390,25),pos=(5,5),
                               style=wx.TE_PROCESS_ENTER)
        self.url.Bind(wx.EVT_TEXT_ENTER, self.enter)
        self.dir = wx.TextCtrl(bkg,size=(390,25),pos=(5,30))
        self.contents = wx.TextCtrl(bkg,style = wx.TE_MULTILINE|wx.HSCROLL,
                                   size=(100,245)
                                    )

        self.count = wx.TextCtrl(bkg,size=(255,25),pos=(420,295))
        self.regex = wx.TextCtrl(bkg,size=(255,25),pos=(420,310))
        self.progress = wx.TextCtrl(bkg,size=(390,25),pos=(5,335))

        #Adding CheckBoxes

        cb1 = wx.CheckBox(bkg, -1, "jpeg",
                          (5, 295), (75, 25))
        cb2 = wx.CheckBox(bkg, -1, "png",
                          (5, 310), (75, 25))
        cb3 = wx.CheckBox(bkg, -1, "gif",
                          (5, 335), (75, 25))
        cb4 = wx.CheckBox(bkg, -1, "mp4",
                          (35, 295), (75, 25))
        cb5 = wx.CheckBox(bkg, -1, "3gp",
                          (35,310), (75, 25))
        cb6 = wx.CheckBox(bkg, -1, "avi",
                          (35, 335), (75, 25))
        cb7 = wx.CheckBox(bkg, -1, "flv",
                          (70, 295), (75, 25))
        cb8 = wx.CheckBox(bkg, -1, "mp3",
                          (70, 310), (75, 25))

        #Binding events with checkboxes

        cb1.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb2.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb3.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb4.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb5.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb6.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb7.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)
        cb8.Bind(wx.EVT_CHECKBOX, self.EvtCheckBox)

        #--------------------------------------------------------------------------

        #WRAPPING UP THE BOXES
        #The text controls
        vbox11 = wx.BoxSizer(wx.VERTICAL)
        vbox11.Add(self.url,proportion=0,flag=wx.ALL|wx.EXPAND,border=5)
        vbox11.Add(self.dir,proportion=0,flag=wx.ALL|wx.EXPAND,border=5)
        vbox11.Add(self.contents,proportion=1,flag=wx.TE_MULTILINE
                  |wx.EXPAND
                  |wx.ALL,border=5
                  )
        self.vbox11 = vbox11

        #The checkBoxes
        vbox12 = wx.BoxSizer(wx.VERTICAL)

        vbox12.Add(cb1, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox12.Add(cb2, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox12.Add(cb3, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)

        vbox13 = wx.BoxSizer(wx.VERTICAL)
        vbox13.Add(cb4, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox13.Add(cb5, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox13.Add(cb6, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)

        vbox14 = wx.BoxSizer(wx.VERTICAL)
        vbox14.Add(cb7, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)
        vbox14.Add(cb8, proportion=0,flag=wx.EXPAND
                      |wx.ALL,border=2)

        #--------------------------------------------------------------------------
        #Extra feature boxes
        vbox15 = wx.BoxSizer(wx.VERTICAL)
        vbox15.Add(self.count,proportion=1,flag=wx.ALL|wx.EXPAND,border=3)
        vbox15.Add(self.regex,proportion=1,flag=wx.ALL|wx.EXPAND,border=3)
        vbox15.Add(fbtn,proportion=0,flag=wx.LEFT,border=178)

        #--------------------------------------------------------------------------
        #All vertical boxes in horizontal box
        hbox1 = wx.BoxSizer()
        hbox1.Add(vbox12,proportion=0,flag=wx.EXPAND)
        hbox1.Add(vbox13,proportion=0,flag=wx.EXPAND)
        hbox1.Add(vbox14,proportion=0,flag=wx.EXPAND)
        hbox1.Add(vbox15,proportion=0)

        #The two vertical boxes in one
        vbox1 = wx.BoxSizer(wx.VERTICAL)
        vbox1.Add(vbox11,proportion=1,flag=wx.EXPAND)
        vbox1.Add(hbox1,proportion=0,flag=wx.EXPAND)
        vbox1.Add(self.progress,proportion=0,flag=wx.ALL|wx.EXPAND,border=5)

        #--------------------------------------------------------------------------
        #Buttons
        vbox21 = wx.BoxSizer(wx.VERTICAL)
        vbox21.Add(download_btn,proportion=0,border=5,flag=wx.TOP)
        vbox21.Add(browse_btn,proportion=0,border=10,flag=wx.TOP)
        vbox21.Add(cancel_btn,proportion=0,border=10,flag=wx.TOP)

        #Close button
        vbox22 = wx.BoxSizer(wx.VERTICAL)
        vbox22.Add(clsbtn,proportion=0,border=5,flag=wx.RIGHT|wx.TOP|wx.BOTTOM)

        #The two vertical boxes in single one.
        vbox2 = wx.BoxSizer(wx.VERTICAL)
        vbox2.Add(vbox21,proportion=1,flag=wx.EXPAND)
        vbox2.Add(vbox22,proportion=0)

        #--------------------------------------------------------------------------

        #Vertical boxes into horizontal hbox.
        hbox = wx.BoxSizer()
        hbox.Add(vbox1,proportion = 1,flag = wx.EXPAND|wx.BOTTOM)
        hbox.Add(vbox2,proportion = 0,flag = wx.EXPAND,border = 5)

        bkg.SetSizer(hbox)

    #--------------------------------------------------------------------------
        self.toDownload = []
    def EvtCheckBox(self, event):
        if event.IsChecked():
            self.toDownload.append(event.GetString())

    #--------------------------------------------------------------------------
    def enter(self, event):
        '''
        The function to prepare a list of all urls found on home page,
        It works on text_enter_event of textctrl box called 'url'.
        '''
        self.urls = get_urls.main(self.url.GetValue())
        if self.urls:

            #destroying the contents box, so as to put the checklist inplace.
            if self.contents:
                self.contents.Destroy()

            self.check_list = wx.CheckListBox(self.bkg, -1, (5,75),
                                              (488,245),self.urls,
                                              style = wx.HSCROLL)

            self.vbox11.Add(self.check_list,proportion=1,flag=wx.EXPAND
                      |wx.ALL,border=5
                      )

            self.toDownload = []
            self.check_list.Bind(wx.EVT_CHECKLISTBOX, self.EvtCheckListBox)

            #setting the count of links
            self.count.SetValue("No. of links found: "+str(len(self.urls)))

    #--------------------------------------------------------------------------
    def download(self,event):
        '''
        The function is bind with download button and download the content.
        It fetches urls from the 'contents' textctrl and sends urls to
        downloader_script.
        Uses 'browse' filedialogs current path to save files.
        '''
        urls_to_download = self.toDownload
        try:
            downloader_script.main(urls_to_download,self.path)
        except AttributeError:
            print "Please select a location"

    #--------------------------------------------------------------------------
    def browse(self,event):
        '''
        The function is bind with the browse button.
        It opens a directory location.
        Treats the dir input in dir_ box as the default.
        '''

        dir_= self.dir.GetValue()
        dlg = wx.DirDialog(self.win, "Choose a directory:",
                          style=wx.DD_DEFAULT_STYLE
                           | wx.DD_DIR_MUST_EXIST
                           | wx.DD_CHANGE_DIR
                           )

        dlg.SetPath(dir_)
        if dlg.ShowModal() == wx.ID_OK:
            print('You selected: %s\n' % dlg.GetPath())
            self.path = dlg.GetPath()

        dlg.Destroy()
        self.dir.SetValue(self.path)

    #--------------------------------------------------------------------------
    def cancel(self,event):
        '''
        This is bound to cancel button, when pressed, it clears all text areas.
        '''
        self.url.SetValue(" ")
        self.check_list.Destroy()
        self.regex.SetValue(" ")
        self.count.SetValue(" ")
        self.contents = wx.TextCtrl(self.bkg,style = wx.TE_MULTILINE|wx.HSCROLL,
                                   size=(100,245)
                                    )
        self.vbox11.Add(self.contents,proportion=1,flag=wx.TE_MULTILINE
                       |wx.EXPAND
                       |wx.ALL,border=5
                       )
        self.contents.SetValue(" ")

    #--------------------------------------------------------------------------
    def filter(self,event):
        '''
        The function filters links found on the page.
        It uses regex to filter and display a list of urls matching the pattern.
        The pattern is specified in the box called regex.
        '''

        pattern = re.compile(self.regex.GetValue())
        if self.urls:

            filtered = re.findall(pattern,'\n'.join(self.urls))

            self.check_list.Destroy()                                       #Destriying old list
            self.check_list = wx.CheckListBox(self.bkg, -1, (5,75),         #creating new filtered list
                                              (488,245),filtered,
                                              style = wx.HSCROLL)

            self.vbox11.Add(self.check_list,proportion=1,flag=wx.EXPAND      #Adding the list box to container
                  |wx.ALL,border=5
                  )

            #list of selected links
            self.toDownload = []

            self.check_list.Bind(wx.EVT_CHECKLISTBOX, self.EvtCheckListBox)
            self.check_list.SetSelection(0)

            self.count.SetValue("No. of links found: "+str(len(filtered)))

    def EvtCheckListBox(self, event):
        '''
        Function to implement checking and unchecking of list items.
        Also, according to checking and unchecking, adds and removes
        items to/from download list.
        '''
        index = event.GetSelection()
        label = self.check_list.GetString(index)
        status = 'un'
        string_at_index = self.check_list.GetString(index)

        if self.check_list.IsChecked(index):
            self.toDownload.append(string_at_index)
            status = ''
            self.check_list.SetSelection(index)                             #so that (un)checking also selects (moves the highlight)

        if not self.check_list.IsChecked(index):
            self.toDownload.remove(string_at_index)
            self.check_list.SetSelection(0)

        print self.toDownload

    #--------------------------------------------------------------------------
    def close(self,event):
        '''
        Destroys the window object.
        '''
        self.win.Destroy()
<|endoftext|>


<commit_before>import unittest
from hwt.hdlObjects.constants import INTF_DIRECTION, DIRECTION
from hwt.pyUtils.arrayQuery import single, NoValueExc

class BaseSynthesizerTC(unittest.TestCase):
    def assertIsM(self, intf):
        self.assertEqual(intf._direction, INTF_DIRECTION.MASTER)

    def assertIsS(self, intf):
        self.assertEqual(intf._direction, INTF_DIRECTION.SLAVE)

    def assertDir(self, u, portName, direction):
        try:
            p = self.getPort(u._entity, portName)
        except NoValueExc:
            self.assertTrue(False, "port %s exists" % portName)
        self.assertEqual(p.direction, direction, "port %s should have direction %s" % (portName, direction))

    def assertDirIn(self, u, portName):
        self.assertDir(u, portName, DIRECTION.IN)

    def assertDirOut(self, u, portName):
        self.assertDir(u, portName, DIRECTION.OUT)

    @staticmethod
    def getPort(entity, portName):
        return single(entity.ports, lambda x: x.name == portName)
        <commit_msg>pragma: no cover for debug code<commit_after>import unittest
from hwt.hdlObjects.constants import INTF_DIRECTION, DIRECTION
from hwt.pyUtils.arrayQuery import single, NoValueExc

class BaseSynthesizerTC(unittest.TestCase):
    def assertIsM(self, intf):
        self.assertEqual(intf._direction, INTF_DIRECTION.MASTER)

    def assertIsS(self, intf):
        self.assertEqual(intf._direction, INTF_DIRECTION.SLAVE)

    def assertDir(self, u, portName, direction):
        try:
            p = self.getPort(u._entity, portName)
        except NoValueExc: # pragma: no cover
            self.assertTrue(False, "port %s exists" % portName)
        self.assertEqual(p.direction, direction, "port %s should have direction %s" % (portName, direction))

    def assertDirIn(self, u, portName):
        self.assertDir(u, portName, DIRECTION.IN)

    def assertDirOut(self, u, portName):
        self.assertDir(u, portName, DIRECTION.OUT)

    @staticmethod
    def getPort(entity, portName):
        return single(entity.ports, lambda x: x.name == portName)
        <|endoftext|>


<commit_before># ERPNext - web based ERP (http://erpnext.com)
# Copyright (C) 2012 Web Notes Technologies Pvt Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d, dl

	def validate(self):
		"""make custom css"""
		from jinja2 import Template

		with open('erpnext/website/doctype/style_settings/custom_template.css', 'r') as f:
			temp = Template(f.read())

		if not self.doc.font_size:
			self.doc.font_size = '13px'

		self.doc.small_font_size = str(int(self.doc.font_size[:-2])-2) + 'px'

		self.doc.custom_css = temp.render(doc = self.doc)
		if self.doc.add_css:
			self.doc.custom_css += '\n\n/* User CSS */\n\n' + self.doc.add_css

		from webnotes.cms.make import make_web_core
		make_web_core()

		from webnotes.session_cache import clear_cache
		clear_cache('Guest')

		del self.doc.fields['small_font_size']

		<commit_msg>cms fixes<commit_after># ERPNext - web based ERP (http://erpnext.com)
# Copyright (C) 2012 Web Notes Technologies Pvt Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d, dl

	def validate(self):
		"""make custom css"""
		from jinja2 import Template
		import os

		with open(os.path.dirname(__file__), 'custom_template.css'), 'r') as f:
			temp = Template(f.read())

		if not self.doc.font_size:
			self.doc.font_size = '13px'

		self.doc.small_font_size = str(int(self.doc.font_size[:-2])-2) + 'px'

		self.doc.custom_css = temp.render(doc = self.doc)
		if self.doc.add_css:
			self.doc.custom_css += '\n\n/* User CSS */\n\n' + self.doc.add_css

		from webnotes.cms.make import make_web_core
		make_web_core()

		from webnotes.session_cache import clear_cache
		clear_cache('Guest')

		del self.doc.fields['small_font_size']

		<|endoftext|>


<commit_before>from bottle_utils.common import basestring, unicode

try:
    from bottle_utils.i18n import lazy_gettext as _
except ImportError:
    _ = lambda x: x

from .exceptions import ValidationError
from .validators import DateValidator

class ErrorMixin(object):
    """
    Mixin class used to provide error rendering functionality.
    """

    @property
    def error(self):
        """
        Human readable error message. This property evaluates to empty string
        if there are no errors.
        """
        if not self._error:
            return ''
        message = self.messages.get(self._error.message)
        if not message:
            return self.generic_error
        if self._error.params:
            return message.format(**self._error.params)
        return message

class Ordered(object):
    #: Counter attribute holding the number of fields found on a form, used
    #: to assign indexes to individual fields, so their order can be kept
    _counter = 0

    def __init__(self):
        #: Assign index to the instantiated field and increment the shared
        #: counter, so the next field will get a higher index
        self._order = self._counter
        Ordered._counter += 1

class DormantField(Ordered):
    """
    Proxy for unbound fields. This class holds the the field constructor
    arguments until the data can be bound to it.

    You never need to use this class directly.
    """

    def __init__(self, field_cls, args, kwargs):
        super(DormantField, self).__init__()  # needed to apply ordering
        self.field_cls = field_cls
        self.args = args
        self.kwargs = kwargs

    def bind(self, name):
        return self.field_cls(name=name, *self.args, **self.kwargs)

class Field(Ordered, ErrorMixin):
    """
    Form field base class. This class provides the base functionality for all
    form fields.

    The ``label`` argument is used to specify the field's label.

    The ``validators`` argument is used to specify the validators that will be
    used on the field data.

    If any data should be bound to a field, the ``value`` argument can be used
    to specify it. Value can be a callable, in which case it is called and its
    return value used as ``value``.

    The ``name`` argument is used to specify the field name.

    The ``messages`` argument is used to customize the validation error
    messages. These override any messages found in the :py:attr:`~messages`
    attribute.

    Any extra keyword attributes passed to the constructor are stored as
    :py:attr:`~options` property on the instance.
    """

    ValidationError = ValidationError

    #: Field ID attribute prefix (this is prepended to the field name)
    _id_prefix = ''

    #: Generic error message to be used when no messages match the validation
    #: error.
    # Translators, used as generic error message in form fields, 'value' should
    # not be translated.
    generic_error = _('Invalid value for this field')
    #: Validation error messages.
    messages = {}
    #: Field markup type. This is arbitrary and normally used in the templates
    #: to differentiate between field types. It is up to the template author to
    #: decide how this should be treated.
    type = 'text'

    def __new__(cls, *args, **kwargs):
        if 'name' in kwargs:
            return super(Field, cls).__new__(cls)
        return DormantField(cls, args, kwargs)

    def __init__(self, label=None, validators=None, value=None, name=None,
                 messages={}, **options):
        super(Field, self).__init__()  # needed to apply ordering

        #: Field name
        self.name = name

        #: Field label
        self.label = label

        #: Field validators
        self.validators = validators or []

        #: Raw value of the field
        self.value = value() if callable(value) else value

        #: Processed value of the field
        self.processed_value = None

        #: Whether value is bound to this field
        self.is_value_bound = False

        self._error = None

        #: Extra keyword argument passed to the constructor
        self.options = options

        # Copy the messages so that modifying them does not result in class'
        # ``messages`` attribute to be altered.
        self.messages = self.messages.copy()

        # Collect default messages from all validators into the messages dict
        for validator in self.validators:
            self.messages.update(validator.messages)

        # Update the messages dict with any user-supplied messages
        self.messages.update(messages)

    def bind_value(self, value):
        """
        Binds a value. This method also sets the :py:attr:`~is_value_bound`
        property to ``True``.
        """
        self.value = value
        self.is_value_bound = True

    def is_valid(self):
        """
        Validate form field and return ``True`` is data is valid. If there is
        an error during Validation, the error object is stored in the
        :py:attr:`~_error` property. Before validation, the raw value is
        processed using the :py:meth:`~parse` method, and stored in
        :py:attr:`~processed_value` attribute.

        When parsing fails with ``ValueError`` exception, a 'generic' error is
        stored. The default message for this error is stored in the
        :py:attr:`~generic_error` property, and can be customized by passing
        a ``'generic'`` message as part of the ``messages`` constructor
        argument.
        """
        try:
            self.processed_value = self.parse(self.processed_value)
        except ValueError as exc:
            self._error = ValidationError('generic',
                                          {'value': self.processed_value})
            return False

        for validate in self.validators:
            try:
                validate(self.processed_value)
            except ValidationError as exc:
                self._error = exc
                return False

        return True

    def parse(self, value):
        """
        Parse the raw value and convert to Python object. Subclasses should
        return the value in it's correct type. In case the passed in value
        cannot be cast into it's correct type, the method should raise a
        ``ValueError`` exception with an appropriate error message.
        """
        raise NotImplementedError()

class StringField(Field):
    """
    Field for working with string values.

    :Python type: str (unicode in Python 2.x)
    :type: text
    """

    def parse(self, value):
        if value is None:
            return ''
        return unicode(value)

class PasswordField(StringField):
    """
    Field for working with passwords.

    :Python type: str (unicode in Python 2.x)
    :type: password
    """

    type = 'password'

class HiddenField(StringField):
    """
    Field for working with hidden inputs.

    :Python type: str (unicode in Python 2.x)
    :type: hidden
    """

    type = 'hidden'

class EmailField(StringField):
    """
    Field for working with emails.

    :Python type: str (unicode in Python 2.x)
    :type: text
    """
    pass

class TextAreaField(StringField):
    """
    Field for working with textareas.

    :Python type: str (unicode in Python 2.x)
    :type: textarea
    """

    type = 'textarea'

class DateField(StringField):
    """
    Field for working with dates. This field overloads the base class'
    constructor to add a :py:class:`~bottle_utils.validators.DateValidator`.

    :Python type: str (unicode in Python 2.x)
    :type: text
    """

    def __init__(self, label=None, validators=None, value=None, **options):
        validators = [DateValidator()] + list(validators or [])
        super(DateField, self).__init__(label,
                                        validators=validators,
                                        value=value,
                                        **options)

class FileField(Field):
    """
    Field for working with file uploads.

    :Python type: raw value
    :type: file
    """

    type = 'file'

    def parse(self, value):
        return value

class IntegerField(Field):
    """
    Field for working with integers.

    :Python type: int
    :type: text
    """

    def parse(self, value):
        if value is None:
            return value

        try:
            return int(value)
        except Exception:
            raise ValueError(_("Invalid value for an integer."))

class FloatField(Field):
    """
    Field for working with floating-point numbers.

    :Python type: float
    :type: text
    """

    def parse(self, value):
        if value is None:
            return value

        try:
            return float(value)
        except Exception:
            raise ValueError(_("Invalid value for a float."))

class BooleanField(Field):
    """
    Field for working with boolean values.

    Two additional constructor arguments are added. The ``default`` argument is
    used to specify the default state of the field, which can be used in the
    template to, for instance, check or uncheck a checkbox or radio button. The
    ``expected_value`` is the base value of the field against which the bound
    value is checked: if they match, the Python value of the field is ``True``.

    :Python type: bool
    :type: checkbox
    """

    type = 'checkbox'

    def __init__(self, label=None, validators=None, value=None, default=False,
                 **options):
        #: Default state of the field
        self.default = default
        #: Base value of the field against which bound value is checked
        self.expected_value = value
        super(BooleanField, self).__init__(label,
                                           validators=validators,
                                           value=value,
                                           **options)

    @property
    def checked(self):
        if not self.is_value_bound:
            return self.default
        # when value is bound to the field, it must match the expected value
        return self.parse(self.value)

    def parse(self, value):
        if not value or isinstance(value, basestring):
            return self.expected_value == value
        # Check if value is present in collection
        return self.expected_value in value

class SelectField(Field):
    """
    Field for dealing with select lists.

    The ``choices`` argument is used to specify the list of value-label pairs
    that are used to present the valid choices in the interface. The field
    value is then checked against the list of value and the parser validates
    whether the supplied value is among the valid ones.

    :Python type: str (unicode in Python 2.x)
    :type: select
    """

    type = 'select'

    def __init__(self, label=None, validators=None, value=None, choices=None,
                 **options):
        #: Iterable of value-label pairs of valid choices
        self.choices = choices or tuple()
        super(SelectField, self).__init__(label,
                                          validators=validators,
                                          value=value,
                                          **options)

    def parse(self, value):
        chosen = unicode(value)
        for (candidate, label) in self.choices:
            if unicode(candidate) == chosen:
                return chosen if value is not None else value

        raise ValueError(_("This is not a valid choice."))
<commit_msg>Handle case where initial value is given in bool form already<commit_after>from bottle_utils.common import basestring, unicode

try:
    from bottle_utils.i18n import lazy_gettext as _
except ImportError:
    _ = lambda x: x

from .exceptions import ValidationError
from .validators import DateValidator

class ErrorMixin(object):
    """
    Mixin class used to provide error rendering functionality.
    """

    @property
    def error(self):
        """
        Human readable error message. This property evaluates to empty string
        if there are no errors.
        """
        if not self._error:
            return ''
        message = self.messages.get(self._error.message)
        if not message:
            return self.generic_error
        if self._error.params:
            return message.format(**self._error.params)
        return message

class Ordered(object):
    #: Counter attribute holding the number of fields found on a form, used
    #: to assign indexes to individual fields, so their order can be kept
    _counter = 0

    def __init__(self):
        #: Assign index to the instantiated field and increment the shared
        #: counter, so the next field will get a higher index
        self._order = self._counter
        Ordered._counter += 1

class DormantField(Ordered):
    """
    Proxy for unbound fields. This class holds the the field constructor
    arguments until the data can be bound to it.

    You never need to use this class directly.
    """

    def __init__(self, field_cls, args, kwargs):
        super(DormantField, self).__init__()  # needed to apply ordering
        self.field_cls = field_cls
        self.args = args
        self.kwargs = kwargs

    def bind(self, name):
        return self.field_cls(name=name, *self.args, **self.kwargs)

class Field(Ordered, ErrorMixin):
    """
    Form field base class. This class provides the base functionality for all
    form fields.

    The ``label`` argument is used to specify the field's label.

    The ``validators`` argument is used to specify the validators that will be
    used on the field data.

    If any data should be bound to a field, the ``value`` argument can be used
    to specify it. Value can be a callable, in which case it is called and its
    return value used as ``value``.

    The ``name`` argument is used to specify the field name.

    The ``messages`` argument is used to customize the validation error
    messages. These override any messages found in the :py:attr:`~messages`
    attribute.

    Any extra keyword attributes passed to the constructor are stored as
    :py:attr:`~options` property on the instance.
    """

    ValidationError = ValidationError

    #: Field ID attribute prefix (this is prepended to the field name)
    _id_prefix = ''

    #: Generic error message to be used when no messages match the validation
    #: error.
    # Translators, used as generic error message in form fields, 'value' should
    # not be translated.
    generic_error = _('Invalid value for this field')
    #: Validation error messages.
    messages = {}
    #: Field markup type. This is arbitrary and normally used in the templates
    #: to differentiate between field types. It is up to the template author to
    #: decide how this should be treated.
    type = 'text'

    def __new__(cls, *args, **kwargs):
        if 'name' in kwargs:
            return super(Field, cls).__new__(cls)
        return DormantField(cls, args, kwargs)

    def __init__(self, label=None, validators=None, value=None, name=None,
                 messages={}, **options):
        super(Field, self).__init__()  # needed to apply ordering

        #: Field name
        self.name = name

        #: Field label
        self.label = label

        #: Field validators
        self.validators = validators or []

        #: Raw value of the field
        self.value = value() if callable(value) else value

        #: Processed value of the field
        self.processed_value = None

        #: Whether value is bound to this field
        self.is_value_bound = False

        self._error = None

        #: Extra keyword argument passed to the constructor
        self.options = options

        # Copy the messages so that modifying them does not result in class'
        # ``messages`` attribute to be altered.
        self.messages = self.messages.copy()

        # Collect default messages from all validators into the messages dict
        for validator in self.validators:
            self.messages.update(validator.messages)

        # Update the messages dict with any user-supplied messages
        self.messages.update(messages)

    def bind_value(self, value):
        """
        Binds a value. This method also sets the :py:attr:`~is_value_bound`
        property to ``True``.
        """
        self.value = value
        self.is_value_bound = True

    def is_valid(self):
        """
        Validate form field and return ``True`` is data is valid. If there is
        an error during Validation, the error object is stored in the
        :py:attr:`~_error` property. Before validation, the raw value is
        processed using the :py:meth:`~parse` method, and stored in
        :py:attr:`~processed_value` attribute.

        When parsing fails with ``ValueError`` exception, a 'generic' error is
        stored. The default message for this error is stored in the
        :py:attr:`~generic_error` property, and can be customized by passing
        a ``'generic'`` message as part of the ``messages`` constructor
        argument.
        """
        try:
            self.processed_value = self.parse(self.processed_value)
        except ValueError as exc:
            self._error = ValidationError('generic',
                                          {'value': self.processed_value})
            return False

        for validate in self.validators:
            try:
                validate(self.processed_value)
            except ValidationError as exc:
                self._error = exc
                return False

        return True

    def parse(self, value):
        """
        Parse the raw value and convert to Python object. Subclasses should
        return the value in it's correct type. In case the passed in value
        cannot be cast into it's correct type, the method should raise a
        ``ValueError`` exception with an appropriate error message.
        """
        raise NotImplementedError()

class StringField(Field):
    """
    Field for working with string values.

    :Python type: str (unicode in Python 2.x)
    :type: text
    """

    def parse(self, value):
        if value is None:
            return ''
        return unicode(value)

class PasswordField(StringField):
    """
    Field for working with passwords.

    :Python type: str (unicode in Python 2.x)
    :type: password
    """

    type = 'password'

class HiddenField(StringField):
    """
    Field for working with hidden inputs.

    :Python type: str (unicode in Python 2.x)
    :type: hidden
    """

    type = 'hidden'

class EmailField(StringField):
    """
    Field for working with emails.

    :Python type: str (unicode in Python 2.x)
    :type: text
    """
    pass

class TextAreaField(StringField):
    """
    Field for working with textareas.

    :Python type: str (unicode in Python 2.x)
    :type: textarea
    """

    type = 'textarea'

class DateField(StringField):
    """
    Field for working with dates. This field overloads the base class'
    constructor to add a :py:class:`~bottle_utils.validators.DateValidator`.

    :Python type: str (unicode in Python 2.x)
    :type: text
    """

    def __init__(self, label=None, validators=None, value=None, **options):
        validators = [DateValidator()] + list(validators or [])
        super(DateField, self).__init__(label,
                                        validators=validators,
                                        value=value,
                                        **options)

class FileField(Field):
    """
    Field for working with file uploads.

    :Python type: raw value
    :type: file
    """

    type = 'file'

    def parse(self, value):
        return value

class IntegerField(Field):
    """
    Field for working with integers.

    :Python type: int
    :type: text
    """

    def parse(self, value):
        if value is None:
            return value

        try:
            return int(value)
        except Exception:
            raise ValueError(_("Invalid value for an integer."))

class FloatField(Field):
    """
    Field for working with floating-point numbers.

    :Python type: float
    :type: text
    """

    def parse(self, value):
        if value is None:
            return value

        try:
            return float(value)
        except Exception:
            raise ValueError(_("Invalid value for a float."))

class BooleanField(Field):
    """
    Field for working with boolean values.

    Two additional constructor arguments are added. The ``default`` argument is
    used to specify the default state of the field, which can be used in the
    template to, for instance, check or uncheck a checkbox or radio button. The
    ``expected_value`` is the base value of the field against which the bound
    value is checked: if they match, the Python value of the field is ``True``.

    :Python type: bool
    :type: checkbox
    """

    type = 'checkbox'

    def __init__(self, label=None, validators=None, value=None, default=False,
                 **options):
        #: Default state of the field
        self.default = default
        #: Base value of the field against which bound value is checked
        self.expected_value = value
        super(BooleanField, self).__init__(label,
                                           validators=validators,
                                           value=value,
                                           **options)

    @property
    def checked(self):
        if not self.is_value_bound:
            return self.default
        # when value is bound to the field, it must match the expected value
        return self.parse(self.value)

    def parse(self, value):
        if not value or isinstance(value, basestring):
            return self.expected_value == value
        # Bool values may be passed to it as initial values
        if isinstance(value, bool):
            return value
        # Check if value is present in collection
        return self.expected_value in value

class SelectField(Field):
    """
    Field for dealing with select lists.

    The ``choices`` argument is used to specify the list of value-label pairs
    that are used to present the valid choices in the interface. The field
    value is then checked against the list of value and the parser validates
    whether the supplied value is among the valid ones.

    :Python type: str (unicode in Python 2.x)
    :type: select
    """

    type = 'select'

    def __init__(self, label=None, validators=None, value=None, choices=None,
                 **options):
        #: Iterable of value-label pairs of valid choices
        self.choices = choices or tuple()
        super(SelectField, self).__init__(label,
                                          validators=validators,
                                          value=value,
                                          **options)

    def parse(self, value):
        chosen = unicode(value)
        for (candidate, label) in self.choices:
            if unicode(candidate) == chosen:
                return chosen if value is not None else value

        raise ValueError(_("This is not a valid choice."))
<|endoftext|>


<commit_before># The MIT License
#
# Copyright (c) 2009 Andreas Stuehrk
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import imp
import os
import sys

# The cached list of all known modules
modules = set()
fully_loaded = False

def complete(line, cw):
    """Construct a full list of possibly completions for imports."""
    if not cw:
        return None

    tokens = line.split()
    if tokens[0] not in ['from', 'import']:
        return None

    completing_from = False
    if tokens[0] == 'from':
        if len(tokens) > 3:
            if '.' in cw:
                # This will result in a SyntaxError, so do not return
                # any matches
                return None
            completing_from = True
            cw = '%s.%s' % (tokens[1], cw)
        elif len(tokens) == 3:
            return ['import']

    matches = list()
    for name in modules:
        if not (name.startswith(cw) and name.find('.', len(cw)) == -1):
            continue
        if completing_from:
            name = name[len(tokens[1]) + 1:]
        matches.append(name)
    if not matches:
        return []
    return matches

def find_modules(path):
    """Find all modules (and packages) for a given directory."""
    if not os.path.isdir(path):
        # Perhaps a zip file
        return

    for name in os.listdir(path):
        if not any(name.endswith(suffix[0]) for suffix in imp.get_suffixes()):
            # Possibly a package
            if '.' in name:
                continue
        name = os.path.splitext(name)[0]
        try:
            fo, pathname, _ = imp.find_module(name, [path])
        except ImportError:
            continue
        else:
            if fo is not None:
                fo.close()
            else:
                # Yay, package
                for subname in find_modules(pathname):
                    if subname != '__init__':
                        yield '%s.%s' % (name, subname)
            yield name

def find_all_modules(path=None):
    """Return a list with all modules in `path`, which should be a list of
    directory names. If path is not given, sys.path will be used."""
    if path is None:
        modules.update(sys.builtin_module_names)
        path = sys.path

    for p in path:
        for module in find_modules(p):
            modules.add(module)
            yield

def find_coroutine():
    global fully_loaded

    if fully_loaded:
        return None

    try:
        find_iterator.next()
    except StopIteration:
        fully_loaded = True

    return True

def reload():
    """Refresh the list of known modules."""
    modules.clear()
    for _ in find_all_modules():
        pass

find_iterator = find_all_modules()
<commit_msg>Catch SyntaxError in import completion.<commit_after># The MIT License
#
# Copyright (c) 2009 Andreas Stuehrk
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import imp
import os
import sys

# The cached list of all known modules
modules = set()
fully_loaded = False

def complete(line, cw):
    """Construct a full list of possibly completions for imports."""
    if not cw:
        return None

    tokens = line.split()
    if tokens[0] not in ['from', 'import']:
        return None

    completing_from = False
    if tokens[0] == 'from':
        if len(tokens) > 3:
            if '.' in cw:
                # This will result in a SyntaxError, so do not return
                # any matches
                return None
            completing_from = True
            cw = '%s.%s' % (tokens[1], cw)
        elif len(tokens) == 3:
            return ['import']

    matches = list()
    for name in modules:
        if not (name.startswith(cw) and name.find('.', len(cw)) == -1):
            continue
        if completing_from:
            name = name[len(tokens[1]) + 1:]
        matches.append(name)
    if not matches:
        return []
    return matches

def find_modules(path):
    """Find all modules (and packages) for a given directory."""
    if not os.path.isdir(path):
        # Perhaps a zip file
        return

    for name in os.listdir(path):
        if not any(name.endswith(suffix[0]) for suffix in imp.get_suffixes()):
            # Possibly a package
            if '.' in name:
                continue
        name = os.path.splitext(name)[0]
        try:
            fo, pathname, _ = imp.find_module(name, [path])
        except (ImportError, SyntaxError):
            continue
        else:
            if fo is not None:
                fo.close()
            else:
                # Yay, package
                for subname in find_modules(pathname):
                    if subname != '__init__':
                        yield '%s.%s' % (name, subname)
            yield name

def find_all_modules(path=None):
    """Return a list with all modules in `path`, which should be a list of
    directory names. If path is not given, sys.path will be used."""
    if path is None:
        modules.update(sys.builtin_module_names)
        path = sys.path

    for p in path:
        for module in find_modules(p):
            modules.add(module)
            yield

def find_coroutine():
    global fully_loaded

    if fully_loaded:
        return None

    try:
        find_iterator.next()
    except StopIteration:
        fully_loaded = True

    return True

def reload():
    """Refresh the list of known modules."""
    modules.clear()
    for _ in find_all_modules():
        pass

find_iterator = find_all_modules()
<|endoftext|>


<commit_before>import abc
import os
import subprocess as sp
import uuid
import batch.client

from .resource import ResourceFile, ResourceGroup, InputResourceFile, TaskResourceFile
from .utils import escape_string, flatten

class Backend:
    @abc.abstractmethod
    def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit):
        return

class LocalBackend(Backend):
    """
    Backend that executes pipelines on a local computer.

    Examples
    --------

    >>> local_backend = LocalBackend(tmp_dir='/tmp/user/')
    >>> p = Pipeline(backend=local_backend)

    Parameters
    ----------
    tmp_dir: :obj:`str`, optional
        Temporary directory to use.
    """

    def __init__(self, tmp_dir='/tmp/'):
        self._tmp_dir = tmp_dir

    def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit):  # pylint: disable=R0915
        tmpdir = self._get_scratch_dir()

        script = ['#!/bin/bash',
                  'set -e' + 'x' if verbose else '',
                  '\n',
                  '# change cd to tmp directory',
                  f"cd {tmpdir}",
                  '\n']

        copied_input_resource_files = set()
        os.makedirs(tmpdir + 'inputs/', exist_ok=True)

        for task in pipeline._tasks:
            os.makedirs(tmpdir + task._uid + '/', exist_ok=True)

            script.append(f"# {task._uid} {task._label if task._label else ''}")

            def copy_input(r):
                if isinstance(r, InputResourceFile):
                    if r not in copied_input_resource_files:
                        copied_input_resource_files.add(r)

                        if r._input_path.startswith('gs://'):
                            return [f'gsutil cp {r._input_path} {r._get_path(tmpdir)}']
                        else:
                            absolute_input_path = escape_string(os.path.realpath(r._input_path))
                            if task._image is not None:  # pylint: disable-msg=W0640
                                return [f'cp {absolute_input_path} {r._get_path(tmpdir)}']
                            else:
                                return [f'ln -sf {absolute_input_path} {r._get_path(tmpdir)}']
                    else:
                        return []
                elif isinstance(r, ResourceGroup):
                    return [x for _, rf in r._resources.items() for x in copy_input(rf)]
                else:
                    assert isinstance(r, TaskResourceFile)
                    return []

            script += [x for r in task._inputs for x in copy_input(r)]

            resource_defs = [r._declare(tmpdir) for r in task._inputs.union(task._mentioned)]

            if task._image:
                defs = '; '.join(resource_defs) + '; ' if resource_defs else ''
                cmd = " && ".join(task._command)
                memory = f'-m {task._memory}' if task._memory else ''
                cpu = f'--cpus={task._cpu}' if task._cpu else ''

                script += [f"docker run "
                           f"-v {tmpdir}:{tmpdir} "
                           f"-w {tmpdir} "
                           f"{memory} "
                           f"{cpu} "
                           f"{task._image} /bin/bash "
                           f"-c {escape_string(defs + cmd)}",
                           '\n']
            else:
                script += resource_defs
                script += task._command + ['\n']

        def write_pipeline_outputs(r, dest):
            if not dest.startswith('gs://'):
                dest = os.path.abspath(dest)
                directory = os.path.dirname(dest)
                os.makedirs(directory, exist_ok=True)
                cp = 'cp'
            else:
                cp = 'gsutil cp'

            if isinstance(r, InputResourceFile):
                return [f'{cp} {escape_string(r._input_path)} {escape_string(dest)}']
            elif isinstance(r, TaskResourceFile):
                return [f'{cp} {r._get_path(tmpdir)} {escape_string(dest)}']
            else:
                assert isinstance(r, ResourceGroup)
                return [x for ext, rf in r._resources.items()
                        for x in write_pipeline_outputs(rf, dest + '.' + ext)]

        outputs = [x for _, r in pipeline._resource_map.items()
                   for dest in r._output_paths
                   for x in write_pipeline_outputs(r, dest)]

        script += ["# Write resources to output destinations"]
        script += outputs

        script = "\n".join(script)
        if dry_run:
            print(script)
        else:
            try:
                sp.check_output(script, shell=True)
            except sp.CalledProcessError as e:
                print(e.output)
                raise e
            finally:
                if delete_scratch_on_exit:
                    sp.run(f'rm -rf {tmpdir}', shell=True)

    def _get_scratch_dir(self):
        def _get_random_name():
            directory = self._tmp_dir + '/pipeline-{}/'.format(uuid.uuid4().hex[:12])

            if os.path.isdir(directory):
                return _get_random_name()
            else:
                os.makedirs(directory, exist_ok=True)
                return directory

        return _get_random_name()

class BatchBackend(Backend):
    """
    Backend that executes pipelines on a Kubernetes cluster using `batch`.

    Examples
    --------

    >>> batch_backend = BatchBackend(tmp_dir='http://localhost:5000')
    >>> p = Pipeline(backend=batch_backend)

    Parameters
    ----------
    url: :obj:`str`
        URL to batch server.
    """

    def __init__(self, url):
        self._batch_client = batch.client.BatchClient(url)

    def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit):  # pylint: disable-msg=R0915
        if dry_run:
            raise NotImplementedError

        hail_scratch_bucket = 'hail-pipeline-scratch'
        subdir_name = 'pipeline-{}'.format(uuid.uuid4().hex[:12])

        remote_tmpdir = f'gs://{hail_scratch_bucket}/{subdir_name}'
        local_tmpdir = f'/tmp/pipeline/{subdir_name}'

        default_image = 'ubuntu'

        batch = self._batch_client.create_batch()
        n_jobs_submitted = 0

        task_to_job_mapping = {}
        job_id_to_command = {}

        volumes = [{'volume': {'name': 'pipeline-test-0-1--hail-is-service-account-key',
                               'secret': {'optional': False,
                                          'secretName': 'pipeline-test-0-1--hail-is-service-account-key'}},
                    'volume_mount': {'mountPath': '/secrets',
                                     'name': 'pipeline-test-0-1--hail-is-service-account-key',
                                     'readOnly': True}}]

        activate_service_account = 'gcloud auth activate-service-account ' \
                                   'pipeline-test-0-1--hail-is@hail-vdc.iam.gserviceaccount.com ' \
                                   '--key-file /secrets/pipeline-test-0-1--hail-is.key'

        for task in pipeline._tasks:
            def copy_input(r):
                if isinstance(r, ResourceFile):
                    if isinstance(r, InputResourceFile):
                        return [f'gsutil cp {escape_string(r._input_path)} {r._get_path(local_tmpdir)}']
                    else:
                        assert isinstance(r, TaskResourceFile)
                        return [f'gsutil cp {r._get_path(remote_tmpdir)} {r._get_path(local_tmpdir)}']
                else:
                    assert isinstance(r, ResourceGroup)
                    return [x for _, rf in r._resources.items() for x in copy_input(rf)]

            def copy_output(r):
                assert r._source == task  # pylint: disable-msg=W0640
                if isinstance(r, TaskResourceFile):
                    return [f'gsutil cp {r._get_path(local_tmpdir)} {r._get_path(remote_tmpdir)}']
                else:
                    assert isinstance(r, ResourceGroup)
                    return [x for _, rf in r._resources.items() for x in copy_output(rf)]

            copy_task_inputs = [x for r in task._inputs for x in copy_input(r)]
            copy_task_outputs = [x for r in task._outputs for x in copy_output(r)]

            local_dirs_needed = [local_tmpdir,
                                 local_tmpdir + '/inputs/',
                                 local_tmpdir + f'/{task._uid}/']
            local_dirs_needed += [local_tmpdir + f'/{dep._uid}/' for dep in task._dependencies]
            make_local_tmpdir = [f'mkdir -p {dir}' for dir in local_dirs_needed]

            task_inputs = make_local_tmpdir + [activate_service_account] + copy_task_inputs
            task_outputs = [activate_service_account] + copy_task_outputs
            resource_defs = [r._declare(directory=local_tmpdir) for r in task._inputs.union(task._mentioned)]

            if task._image is None:
                if verbose:
                    print(f"Using image '{default_image}' since no image was specified.")

            defs = '; '.join(resource_defs) + '; ' if resource_defs else ''
            task_command = [cmd.strip() for cmd in task._command]

            cmd = " && ".join(task_inputs + task_command + task_outputs)
            parent_ids = [task_to_job_mapping[t].id for t in task._dependencies]

            attributes = {'task_uid': task._uid}
            if task._label:
                attributes['label'] = task._label

            resources = {'requests': {}}
            if task._cpu:
                resources['requests']['cpu'] = task._cpu
            if task._memory:
                resources['requests']['memory'] = task._memory

            j = batch.create_job(image=task._image if task._image else default_image,
                                 command=['/bin/bash', '-c', defs + cmd],
                                 parent_ids=parent_ids,
                                 attributes=attributes,
                                 volumes=volumes,
                                 resources=resources)
            n_jobs_submitted += 1

            task_to_job_mapping[task] = j
            job_id_to_command[j.id] = defs + cmd
            if verbose:
                print(f"Submitted Job {j.id} with command: {defs + cmd}")

        def write_pipeline_outputs(r, dest):
            if isinstance(r, InputResourceFile):
                copy_output = activate_service_account + ' && ' + \
                              f'gsutil cp {escape_string(r._input_path)} {escape_string(dest)}'
                return [(task_to_job_mapping[r._source].id, copy_output)]
            elif isinstance(r, TaskResourceFile):
                copy_output = activate_service_account + ' && ' + \
                              f'gsutil cp {r._get_path(remote_tmpdir)} {escape_string(dest)}'
                return [(task_to_job_mapping[r._source].id, copy_output)]
            else:
                assert isinstance(r, ResourceGroup)
                return [write_pipeline_outputs(rf, dest + '.' + ext) for ext, rf in r._resources.items()]

        pipeline_outputs = list(flatten([write_pipeline_outputs(r, dest)
                                         for _, r in pipeline._resource_map.items()
                                         for dest in r._output_paths]))

        for parent_id, write_cmd in pipeline_outputs:
            j = batch.create_job(image='google/cloud-sdk:alpine',
                                 command=['/bin/bash', '-c', write_cmd],
                                 parent_ids=[parent_id],
                                 attributes={'label': 'write_output'},
                                 volumes=volumes)
            job_id_to_command[j.id] = write_cmd
            n_jobs_submitted += 1
            if verbose:
                print(f"Submitted Job {j.id} with command: {write_cmd}")

        status = batch.wait()

        if delete_scratch_on_exit:
            rm_cmd = f'gsutil rm -r {remote_tmpdir}'
            j = self._batch_client.create_job(image='google/cloud-sdk:alpine',
                                              command=['/bin/bash', '-c', activate_service_account + '&&' + rm_cmd],
                                              volumes=volumes,
                                              attributes={'label': 'remove_tmpdir'})

        failed_jobs = [(j['id'], j['exit_code']) for j in status['jobs'] if 'exit_code' in j and j['exit_code'] > 0]

        fail_msg = ''
        for jid, ec in failed_jobs:
            jstatus = self._batch_client.get_job(jid).status()
            log = jstatus['log']
            label = jstatus['attributes'].get('label', None)
            fail_msg += (
                f"Job {jid} failed with exit code {ec}:\n"
                f"  Task label:\t{label}\n"
                f"  Command:\t{job_id_to_command[jid]}\n"
                f"  Log:\t{log}\n")

        n_complete = sum([j['state'] == 'Complete' for j in status['jobs']])
        if failed_jobs or n_complete != n_jobs_submitted:
            raise Exception(fail_msg)

        print("Pipeline completed successfully!")
<commit_msg>[pipeline] don't get job log from status (#5971)<commit_after>import abc
import os
import subprocess as sp
import uuid
import batch.client

from .resource import ResourceFile, ResourceGroup, InputResourceFile, TaskResourceFile
from .utils import escape_string, flatten

class Backend:
    @abc.abstractmethod
    def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit):
        return

class LocalBackend(Backend):
    """
    Backend that executes pipelines on a local computer.

    Examples
    --------

    >>> local_backend = LocalBackend(tmp_dir='/tmp/user/')
    >>> p = Pipeline(backend=local_backend)

    Parameters
    ----------
    tmp_dir: :obj:`str`, optional
        Temporary directory to use.
    """

    def __init__(self, tmp_dir='/tmp/'):
        self._tmp_dir = tmp_dir

    def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit):  # pylint: disable=R0915
        tmpdir = self._get_scratch_dir()

        script = ['#!/bin/bash',
                  'set -e' + 'x' if verbose else '',
                  '\n',
                  '# change cd to tmp directory',
                  f"cd {tmpdir}",
                  '\n']

        copied_input_resource_files = set()
        os.makedirs(tmpdir + 'inputs/', exist_ok=True)

        for task in pipeline._tasks:
            os.makedirs(tmpdir + task._uid + '/', exist_ok=True)

            script.append(f"# {task._uid} {task._label if task._label else ''}")

            def copy_input(r):
                if isinstance(r, InputResourceFile):
                    if r not in copied_input_resource_files:
                        copied_input_resource_files.add(r)

                        if r._input_path.startswith('gs://'):
                            return [f'gsutil cp {r._input_path} {r._get_path(tmpdir)}']
                        else:
                            absolute_input_path = escape_string(os.path.realpath(r._input_path))
                            if task._image is not None:  # pylint: disable-msg=W0640
                                return [f'cp {absolute_input_path} {r._get_path(tmpdir)}']
                            else:
                                return [f'ln -sf {absolute_input_path} {r._get_path(tmpdir)}']
                    else:
                        return []
                elif isinstance(r, ResourceGroup):
                    return [x for _, rf in r._resources.items() for x in copy_input(rf)]
                else:
                    assert isinstance(r, TaskResourceFile)
                    return []

            script += [x for r in task._inputs for x in copy_input(r)]

            resource_defs = [r._declare(tmpdir) for r in task._inputs.union(task._mentioned)]

            if task._image:
                defs = '; '.join(resource_defs) + '; ' if resource_defs else ''
                cmd = " && ".join(task._command)
                memory = f'-m {task._memory}' if task._memory else ''
                cpu = f'--cpus={task._cpu}' if task._cpu else ''

                script += [f"docker run "
                           f"-v {tmpdir}:{tmpdir} "
                           f"-w {tmpdir} "
                           f"{memory} "
                           f"{cpu} "
                           f"{task._image} /bin/bash "
                           f"-c {escape_string(defs + cmd)}",
                           '\n']
            else:
                script += resource_defs
                script += task._command + ['\n']

        def write_pipeline_outputs(r, dest):
            if not dest.startswith('gs://'):
                dest = os.path.abspath(dest)
                directory = os.path.dirname(dest)
                os.makedirs(directory, exist_ok=True)
                cp = 'cp'
            else:
                cp = 'gsutil cp'

            if isinstance(r, InputResourceFile):
                return [f'{cp} {escape_string(r._input_path)} {escape_string(dest)}']
            elif isinstance(r, TaskResourceFile):
                return [f'{cp} {r._get_path(tmpdir)} {escape_string(dest)}']
            else:
                assert isinstance(r, ResourceGroup)
                return [x for ext, rf in r._resources.items()
                        for x in write_pipeline_outputs(rf, dest + '.' + ext)]

        outputs = [x for _, r in pipeline._resource_map.items()
                   for dest in r._output_paths
                   for x in write_pipeline_outputs(r, dest)]

        script += ["# Write resources to output destinations"]
        script += outputs

        script = "\n".join(script)
        if dry_run:
            print(script)
        else:
            try:
                sp.check_output(script, shell=True)
            except sp.CalledProcessError as e:
                print(e.output)
                raise e
            finally:
                if delete_scratch_on_exit:
                    sp.run(f'rm -rf {tmpdir}', shell=True)

    def _get_scratch_dir(self):
        def _get_random_name():
            directory = self._tmp_dir + '/pipeline-{}/'.format(uuid.uuid4().hex[:12])

            if os.path.isdir(directory):
                return _get_random_name()
            else:
                os.makedirs(directory, exist_ok=True)
                return directory

        return _get_random_name()

class BatchBackend(Backend):
    """
    Backend that executes pipelines on a Kubernetes cluster using `batch`.

    Examples
    --------

    >>> batch_backend = BatchBackend(tmp_dir='http://localhost:5000')
    >>> p = Pipeline(backend=batch_backend)

    Parameters
    ----------
    url: :obj:`str`
        URL to batch server.
    """

    def __init__(self, url):
        self._batch_client = batch.client.BatchClient(url)

    def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit):  # pylint: disable-msg=R0915
        if dry_run:
            raise NotImplementedError

        hail_scratch_bucket = 'hail-pipeline-scratch'
        subdir_name = 'pipeline-{}'.format(uuid.uuid4().hex[:12])

        remote_tmpdir = f'gs://{hail_scratch_bucket}/{subdir_name}'
        local_tmpdir = f'/tmp/pipeline/{subdir_name}'

        default_image = 'ubuntu'

        batch = self._batch_client.create_batch()
        n_jobs_submitted = 0

        task_to_job_mapping = {}
        job_id_to_command = {}

        volumes = [{'volume': {'name': 'pipeline-test-0-1--hail-is-service-account-key',
                               'secret': {'optional': False,
                                          'secretName': 'pipeline-test-0-1--hail-is-service-account-key'}},
                    'volume_mount': {'mountPath': '/secrets',
                                     'name': 'pipeline-test-0-1--hail-is-service-account-key',
                                     'readOnly': True}}]

        activate_service_account = 'gcloud auth activate-service-account ' \
                                   'pipeline-test-0-1--hail-is@hail-vdc.iam.gserviceaccount.com ' \
                                   '--key-file /secrets/pipeline-test-0-1--hail-is.key'

        for task in pipeline._tasks:
            def copy_input(r):
                if isinstance(r, ResourceFile):
                    if isinstance(r, InputResourceFile):
                        return [f'gsutil cp {escape_string(r._input_path)} {r._get_path(local_tmpdir)}']
                    else:
                        assert isinstance(r, TaskResourceFile)
                        return [f'gsutil cp {r._get_path(remote_tmpdir)} {r._get_path(local_tmpdir)}']
                else:
                    assert isinstance(r, ResourceGroup)
                    return [x for _, rf in r._resources.items() for x in copy_input(rf)]

            def copy_output(r):
                assert r._source == task  # pylint: disable-msg=W0640
                if isinstance(r, TaskResourceFile):
                    return [f'gsutil cp {r._get_path(local_tmpdir)} {r._get_path(remote_tmpdir)}']
                else:
                    assert isinstance(r, ResourceGroup)
                    return [x for _, rf in r._resources.items() for x in copy_output(rf)]

            copy_task_inputs = [x for r in task._inputs for x in copy_input(r)]
            copy_task_outputs = [x for r in task._outputs for x in copy_output(r)]

            local_dirs_needed = [local_tmpdir,
                                 local_tmpdir + '/inputs/',
                                 local_tmpdir + f'/{task._uid}/']
            local_dirs_needed += [local_tmpdir + f'/{dep._uid}/' for dep in task._dependencies]
            make_local_tmpdir = [f'mkdir -p {dir}' for dir in local_dirs_needed]

            task_inputs = make_local_tmpdir + [activate_service_account] + copy_task_inputs
            task_outputs = [activate_service_account] + copy_task_outputs
            resource_defs = [r._declare(directory=local_tmpdir) for r in task._inputs.union(task._mentioned)]

            if task._image is None:
                if verbose:
                    print(f"Using image '{default_image}' since no image was specified.")

            defs = '; '.join(resource_defs) + '; ' if resource_defs else ''
            task_command = [cmd.strip() for cmd in task._command]

            cmd = " && ".join(task_inputs + task_command + task_outputs)
            parent_ids = [task_to_job_mapping[t].id for t in task._dependencies]

            attributes = {'task_uid': task._uid}
            if task._label:
                attributes['label'] = task._label

            resources = {'requests': {}}
            if task._cpu:
                resources['requests']['cpu'] = task._cpu
            if task._memory:
                resources['requests']['memory'] = task._memory

            j = batch.create_job(image=task._image if task._image else default_image,
                                 command=['/bin/bash', '-c', defs + cmd],
                                 parent_ids=parent_ids,
                                 attributes=attributes,
                                 volumes=volumes,
                                 resources=resources)
            n_jobs_submitted += 1

            task_to_job_mapping[task] = j
            job_id_to_command[j.id] = defs + cmd
            if verbose:
                print(f"Submitted Job {j.id} with command: {defs + cmd}")

        def write_pipeline_outputs(r, dest):
            if isinstance(r, InputResourceFile):
                copy_output = activate_service_account + ' && ' + \
                              f'gsutil cp {escape_string(r._input_path)} {escape_string(dest)}'
                return [(task_to_job_mapping[r._source].id, copy_output)]
            elif isinstance(r, TaskResourceFile):
                copy_output = activate_service_account + ' && ' + \
                              f'gsutil cp {r._get_path(remote_tmpdir)} {escape_string(dest)}'
                return [(task_to_job_mapping[r._source].id, copy_output)]
            else:
                assert isinstance(r, ResourceGroup)
                return [write_pipeline_outputs(rf, dest + '.' + ext) for ext, rf in r._resources.items()]

        pipeline_outputs = list(flatten([write_pipeline_outputs(r, dest)
                                         for _, r in pipeline._resource_map.items()
                                         for dest in r._output_paths]))

        for parent_id, write_cmd in pipeline_outputs:
            j = batch.create_job(image='google/cloud-sdk:alpine',
                                 command=['/bin/bash', '-c', write_cmd],
                                 parent_ids=[parent_id],
                                 attributes={'label': 'write_output'},
                                 volumes=volumes)
            job_id_to_command[j.id] = write_cmd
            n_jobs_submitted += 1
            if verbose:
                print(f"Submitted Job {j.id} with command: {write_cmd}")

        status = batch.wait()

        if delete_scratch_on_exit:
            rm_cmd = f'gsutil rm -r {remote_tmpdir}'
            j = self._batch_client.create_job(image='google/cloud-sdk:alpine',
                                              command=['/bin/bash', '-c', activate_service_account + '&&' + rm_cmd],
                                              volumes=volumes,
                                              attributes={'label': 'remove_tmpdir'})

        failed_jobs = [(j['id'], j['exit_code']) for j in status['jobs'] if 'exit_code' in j and j['exit_code'] > 0]

        fail_msg = ''
        for jid, ec in failed_jobs:
            job = self._batch_client.get_job(jid)
            log = job.log()
            label = job.status()['attributes'].get('label', None)
            fail_msg += (
                f"Job {jid} failed with exit code {ec}:\n"
                f"  Task label:\t{label}\n"
                f"  Command:\t{job_id_to_command[jid]}\n"
                f"  Log:\t{log}\n")

        n_complete = sum([j['state'] == 'Complete' for j in status['jobs']])
        if failed_jobs or n_complete != n_jobs_submitted:
            raise Exception(fail_msg)

        print("Pipeline completed successfully!")
<|endoftext|>


<commit_before>import sys
import json
import logging
import urllib2
import urllib

from utils import settings

"""
* PERSON:  People, including fictional.
* NORP:    Nationalities or religious or political groups.
* FAC: Facilities, such as buildings, airports, highways, bridges, etc.
* ORG: Companies, agencies, institutions, etc.
* GPE: Countries, cities, states.
* LOC: Non-GPE locations, mountain ranges, bodies of water.
* PRODUCT: Vehicles, weapons, foods, etc. (Not services)
* EVENT:   Named hurricanes, battles, wars, sports events, etc.
* WORK_OF_ART: Titles of books, songs, etc.
* LAW: Named documents made into laws
* LANGUAGE:    Any named language
"""

NLP_FIELDS = ['description', 'text']
NLP_GEO_KEYS = ['GPE', 'LOC', 'FAC']

logging.basicConfig(filename="{0}/nlp_worker.log".format(settings.LOG_FILE_PATH), level=logging.DEBUG)

def post_to_nlp_service(text):
    text = unicode(text).encode('utf-8')
    text = urllib.quote(text)
    try:
        req = urllib2.Request("http://{0}:{1}/nlp".format(settings.SERVICE_ADDRESS, settings.SERVICE_PORT), text.encode('utf-8'))
        response = urllib2.urlopen(req)
        result = response.read()
        # logging.debug("\n\n--> sent to NLP %s \n\nGOT BACK: %s" % (text, result))
        return result
    except Exception as e:
        logging.error("NLP error. Sent {0}, error: \n {1}".format(text, e))

def run(entry, *args):
    if args is not None:
        NLP_FIELDS = list(args)

    new_entry = json.load(open(entry, "rb"))

    text = ''
    for field in NLP_FIELDS:
        if field in new_entry['entry']['fields'].keys():
            v = new_entry['entry']['fields'][field]
            if v is not None:
                if isinstance(v, list):
                    text = u'{0}, {1}'.format(text, ', '.join(v))
                else:
                    text = u'{0}, {1}'.format(text, v)

    if len(text) > 0:
        nlp_items = json.loads(post_to_nlp_service(text))
    else:
        nlp_items = dict()

    try:
        for nlp_item_key in nlp_items.keys():
            nlp_text_field_name = "ftt_NLP_{0}".format(nlp_item_key)
            new_entry['entry']['fields'][nlp_text_field_name] = nlp_items[nlp_item_key]

        geo_text = ""
        for field in NLP_GEO_KEYS:
            if field in nlp_items.keys():
                v = nlp_items[field]
                geo_text = "{0} {1}".format(geo_text, ' '.join(v))

        new_entry['entry']['fields']['ft_NLP_Geo'] = geo_text

    except Exception as e:
        logging.error("could not get response from NLP parser. ")
        logging.error(e)

    sys.stdout.write(json.dumps(new_entry))
    sys.stdout.flush()
<commit_msg>changed nlp field type from multivalued text to multivalued string.<commit_after>import sys
import json
import logging
import urllib2
import urllib

from utils import settings

"""
* PERSON:  People, including fictional.
* NORP:    Nationalities or religious or political groups.
* FAC: Facilities, such as buildings, airports, highways, bridges, etc.
* ORG: Companies, agencies, institutions, etc.
* GPE: Countries, cities, states.
* LOC: Non-GPE locations, mountain ranges, bodies of water.
* PRODUCT: Vehicles, weapons, foods, etc. (Not services)
* EVENT:   Named hurricanes, battles, wars, sports events, etc.
* WORK_OF_ART: Titles of books, songs, etc.
* LAW: Named documents made into laws
* LANGUAGE:    Any named language
"""

NLP_FIELDS = ['description', 'text']
NLP_GEO_KEYS = ['GPE', 'LOC', 'FAC']

logging.basicConfig(filename="{0}/nlp_worker.log".format(settings.LOG_FILE_PATH), level=logging.DEBUG)

def post_to_nlp_service(text):
    text = unicode(text).encode('utf-8')
    text = urllib.quote(text)
    try:
        req = urllib2.Request("http://{0}:{1}/nlp".format(settings.SERVICE_ADDRESS, settings.SERVICE_PORT), text.encode('utf-8'))
        response = urllib2.urlopen(req)
        result = response.read()
        # logging.debug("\n\n--> sent to NLP %s \n\nGOT BACK: %s" % (text, result))
        return result
    except Exception as e:
        logging.error("NLP error. Sent {0}, error: \n {1}".format(text, e))

def run(entry, *args):
    if args is not None:
        NLP_FIELDS = list(args)

    new_entry = json.load(open(entry, "rb"))

    text = ''
    for field in NLP_FIELDS:
        if field in new_entry['entry']['fields'].keys():
            v = new_entry['entry']['fields'][field]
            if v is not None:
                if isinstance(v, list):
                    text = u'{0}, {1}'.format(text, ', '.join(v))
                else:
                    text = u'{0}, {1}'.format(text, v)

    if len(text) > 0:
        nlp_items = json.loads(post_to_nlp_service(text))
    else:
        nlp_items = dict()

    try:
        for nlp_item_key in nlp_items.keys():
            nlp_text_field_name = "fss_NLP_{0}".format(nlp_item_key)
            new_entry['entry']['fields'][nlp_text_field_name] = nlp_items[nlp_item_key]

        geo_text = ""
        for field in NLP_GEO_KEYS:
            if field in nlp_items.keys():
                v = nlp_items[field]
                geo_text = "{0} {1}".format(geo_text, ' '.join(v))

        new_entry['entry']['fields']['ft_NLP_Geo'] = geo_text

    except Exception as e:
        logging.error("could not get response from NLP parser. ")
        logging.error(e)

    sys.stdout.write(json.dumps(new_entry))
    sys.stdout.flush()
<|endoftext|>


<commit_before>from datetime import date

from . import base
from . import mixins

class TransformedRecord(mixins.GenericCompensationMixin,
        mixins.GenericIdentifierMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):
    MAP = {
        'last_name': 'FAMILY NAME',
        'first_name': 'GIVEN NAME',
        'department': 'DEPARTMENT',
        'job_title': 'JOBTITLE',
        'gender': 'GENDER',
        'race': 'RACE/ETHNICITY',
        'hire_date': 'LAST HIRE DATE',
        'compensation': 'ANNUAL PAY',
        'longevity': 'ANNUALIZED LONGEVITY',
        'employee_type': 'FULL/PART TIME',
    }

    NAME_FIELDS = ('first_name', 'last_name', )

    gender_map = {'Female': 'F', 'Male': 'M'}

    ORGANIZATION_NAME = 'The University of Texas Medical Branch at Galveston'

    ORGANIZATION_CLASSIFICATION = 'University Hospital'

    DATE_PROVIDED = date(2015, 6, 24)

    URL = 'http://raw.texastribune.org.s3.amazonaws.com/ut_medical_branch/salaries/2015-06/ut_medical_galveston.xlsx'

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''

    @property
    def person(self):
        name = self.get_name()
        data = {
            'family_name': name.last,
            'given_name': name.first,
            'name': unicode(name),
            'gender': self.gender_map[self.gender.strip()]
        }

        return data

    @property
    def compensation_type(self):
        if self.employee_type == 'Part-time':
            return 'PT'
        else:
            return 'FT'

    @property
    def description(self):
        if self.employee_type == 'Part-time':
            return "Part-time annual compensation"
        else:
            return "Annual compensation"

    def process_compensation(self):
        #longevity is in addition to base annual_pay, add if applicable
        if self.longevity.strip() == '0':
            return self.compensation
        else:
            longevity = self.longevity.strip().replace(',', '')
            salary = self.compensation.strip().replace(',', '')
            return float(salary) + float(longevity)

    @property
    def compensation(self):
        return self.process_compensation()

transform = base.transform_factory(TransformedRecord)
<commit_msg>attempting to fix weird gender issue<commit_after>from . import base
from . import mixins

from datetime import date

class TransformedRecord(
        mixins.GenericCompensationMixin,
        mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
        mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
        mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
        mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'last_name': 'FAMILY NAME',
        'first_name': 'GIVEN NAME',
        'department': 'DEPARTMENT',
        'job_title': 'JOBTITLE',
        'gender': 'GENDER',
        'race': 'RACE/ETHNICITY',
        'hire_date': 'LAST HIRE DATE',
        'compensation': 'ANNUAL PAY',
        'longevity': 'ANNUALIZED LONGEVITY',
        'employee_type': 'FULL/PART TIME',
    }

    NAME_FIELDS = ('first_name', 'last_name', )

    gender_map = {'Female': 'F', 'Male': 'M'}

    ORGANIZATION_NAME = 'The University of Texas Medical Branch at Galveston'

    ORGANIZATION_CLASSIFICATION = 'University Hospital'

    DATE_PROVIDED = date(2015, 6, 24)

    URL = ('http://raw.texastribune.org.s3.amazonaws.com/ut_medical_branch/'
            'salaries/2015-06/ut_medical_galveston.xlsx')

    @property
    def compensation_type(self):

        if self.employee_type == 'Part-time':
            return 'PT'
        else:
            return 'FT'

    @property
    def description(self):

        if self.employee_type == 'Part-time':
            return "Part-time annual compensation"
        else:
            return "Annual compensation"

    @property
    def compensation(self):
        if self.get_mapped_value('longevity') == '0':
            return self.get_mapped_value('compensation')
        else:
            longevity = self.get_mapped_value('longevity')
            salary = self.get_mapped_value('compensation')
            return float(salary) + float(longevity)

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''

    @property
    def person(self):
        data = {
            'family_name': self.last_name,
            'given_name': self.first_name,
            'name': self.get_raw_name(),
            'gender': self.gender_map[self.gender.strip()]
        }

        return data

transform = base.transform_factory(TransformedRecord)
<|endoftext|>