<commit_before># -----
# panda3dIOClass
# -----
# by Reto Spoerri
# rspoerri AT nouser.org
# http://www.nouser.org
# -----
# Creates a frame in a panda3d window, with a directEntry
# uses the clipboard class to copy, cut & paste
# output can be scrolled by page_up/down
# input can be scrolled by arrow_up/down
# -----

from shared import *

import textwrap, re, string, sys, os

from direct.gui.DirectGui import DirectFrame, DirectEntry, DirectLabel, DGG
from direct.showbase import DirectObject
from pandac.PandaModules import TextNode, Vec3, VBase4
from direct.gui.OnscreenText import OnscreenText

from clipboard import clipboard

class panda3dIOClass( DirectObject.DirectObject ):
  # set gui key to None if you want to call toggleConsole from outside this class
  gui_key = PANDA3D_CONSOLE_TOGGLE_KEY
  autocomplete_key = PANDA3D_CONSOLE_AUTOCOMPLETE_KEY
  autohelp_key = PANDA3d_CONSOLE_AUTOHELP_KEY
  # change size of text and number of characters on one line
  # scale of frame ( must be small (0.0x)
  scale = PANDA3D_CONSOLE_SCALE
  # to define a special font, if loading fails the default font is used (without warning)
  font = PANDA3D_CONSOLE_FONT
  fontWidth = PANDA3D_CONSOLE_FONT_WIDTH
  # frame position and size (vertical & horizontal)
  h_pos   = PANDA3D_CONSOLE_HORIZONTAL_POS
  h_size  = PANDA3D_CONSOLE_HORIZONTAL_SIZE
  # v_size + v_pos should not exceed 2.0, else parts of the interface will not be visible
  # space above the frame ( must be below 2.0, best between 0.0 and 1.0 )
  v_pos   = PANDA3D_CONSOLE_VERTICAL_POS
  # vertical size of the frame ( must be at max 2.0, best between 0.5 and 2.0 )
  v_size  = PANDA3D_CONSOLE_VERTICAL_SIZE
  linelength = int((h_size/scale - 5) / fontWidth)
  print "max number of characters on a length:", linelength
  numlines = int(v_size/scale - 5)
  defaultTextColor  = (0.0,0.0,0.0,1.0)
  autoCompleteColor = (0.9,0.4,0.2,1.0)
  def __init__( self, parent ):
    self.parent = parent

    # line wrapper
    self.linewrap = textwrap.TextWrapper()
    self.linewrap.width = self.linelength

    # calculate window size
    left   = (self.h_pos) / self.scale
    right  = (self.h_pos + self.h_size) / self.scale
    bottom = (self.v_pos) / self.scale
    top    = (self.v_pos + self.v_size) /self.scale

    # panda3d interface
    self.consoleFrame = DirectFrame ( relief = DGG.GROOVE
                                    , frameColor = (200, 200, 200, 0.5)
                                    , scale=self.scale
                                    , frameSize = (0, self.h_size / self.scale, 0, self.v_size / self.scale) )
    self.windowEvent( base.win )

    # try to load the defined font
    fixedWidthFont = loader.loadFont(self.font)
    # if font is not valid use default font
    if not fixedWidthFont.isValid():
      if self.font is None:
        print "pandaInteractiveConsole.py :: could not load the defined font %s" % str(self.font)
      fixedWidthFont = DGG.getDefaultFont()

    # text entry line
    self.consoleEntry = DirectEntry ( self.consoleFrame
                                    , text        = ""
                                    , command     = self.onEnterPress
                                    , width       = self.h_size/self.scale - 2
                                    , pos         = (1, 0, 1.5)
                                    , initialText = ""
                                    , numLines    = 1
                                    , focus       = 1
                                    , entryFont   = fixedWidthFont)

    # output lines
    self.consoleOutputList = list()
    for i in xrange( self.numlines ):
      label = OnscreenText( parent = self.consoleFrame
                          , text = ""
                          , pos = (1, -i+3+self.numlines)
                          , align=TextNode.ALeft
                          , mayChange=1
                          , scale=1.0
                          , fg = self.defaultTextColor )
      label.setFont( fixedWidthFont )
      self.consoleOutputList.append( label )

    # list of the last commands of the user
    self.userCommandList = list()
    self.userCommandListLength = 100
    for i in xrange(self.userCommandListLength):
      self.userCommandList.append('')
    self.userCommandPos = 0

    # buffer for data
    self.textBuffer = list()
    self.textBufferLength = 1000
    for i in xrange(self.textBufferLength):
      self.textBuffer.append(['', DEFAULT_COLOR])
    self.textBufferPos = self.textBufferLength-self.numlines

    # toggle the window at least once to activate the events
    self.toggleConsole()
    self.toggleConsole()

    self.help()

  def help( self ):
    # output some info text about this module
    infoTxt = """ ------ Panda3dConsole ------
- press F1 to toggle it on/off
- use the usual copy, cut & paste keys
- page_up    : scrolls up
- page_down  : scrolls down
- arrow_up   : previous command
- arrow_down : next command
- BUGS       : if you paste a to long text, the entry blocks
         FIX : use cut to remove the whole line"""
    #for line in infoTxt.split('\n'):
    #  self.write( line, color=DEFAULT_COLOR )
    return infoTxt

  # write a string to the panda3d console
  def write( self, printString, color=defaultTextColor ):
    # remove not printable characters (which can be input by console input)
    printString = re.sub( r'[^%s]' % re.escape(string.printable[:95]), "", printString)

    splitLines = self.linewrap.wrap(printString)
    for line in splitLines:
      self.textBuffer.append( [line, color] )
      self.textBuffer.pop(0)

    self.updateOutput()

  def updateOutput( self ):
    for lineNumber in xrange(self.numlines):
      lineText, color = self.textBuffer[lineNumber + self.textBufferPos]
      self.consoleOutputList[lineNumber].setText( lineText )
      self.consoleOutputList[lineNumber]['fg'] = color

  # toggle the gui console
  def toggleConsole( self ):
    self.consoleFrame.toggleVis()
    hidden = self.consoleFrame.isHidden()
    self.consoleEntry['focus'] != hidden
    if hidden:
      self.ignoreAll()
#      self.accept( self.gui_key, self.toggleConsole )
    else:
      self.ignoreAll()
      self.accept( 'page_up', self.scroll, [-5] )
      self.accept( 'page_up-repeat', self.scroll, [-5] )
      self.accept( 'page_down', self.scroll, [5] )
      self.accept( 'page_down-repeat', self.scroll, [5] )
      self.accept( 'window-event', self.windowEvent)

      self.accept( 'arrow_up'  , self.scrollCmd, [ 1] )
      self.accept( 'arrow_down', self.scrollCmd, [-1] )

#      self.accept( self.gui_key, self.toggleConsole )
      self.accept( self.autocomplete_key, self.autocomplete )
      self.accept( self.autohelp_key, self.autohelp )

      # accept v, c and x, where c & x copy's the whole console text
      #messenger.toggleVerbose()
      #for osx use ('meta')
      if sys.platform == 'darwin':
        self.accept( 'meta', self.unfocus )
        self.accept( 'meta-up', self.focus )
        self.accept( 'meta-c', self.copy )
        self.accept( 'meta-x', self.cut )
        self.accept( 'meta-v', self.paste )
      #for windows use ('control')
      if sys.platform == 'win32' or sys.platform == 'linux2':
        self.accept( 'control', self.unfocus )
        self.accept( 'control-up', self.focus )
        self.accept( 'control-c', self.copy )
        self.accept( 'control-x', self.cut )
        self.accept( 'control-v', self.paste )

  def autocomplete( self ):
    currentText = self.consoleEntry.get()
    currentPos = self.consoleEntry.guiItem.getCursorPosition()
    newText = self.parent.autocomplete( currentText, currentPos )
    if newText != currentText:
      self.consoleEntry.set( newText )
      self.consoleEntry.setCursorPosition( len(newText) )

  def autohelp( self ):
    currentText = self.consoleEntry.get()
    currentPos = self.consoleEntry.guiItem.getCursorPosition()
    self.parent.autohelp( currentText, currentPos )

  def focus( self ):
    self.consoleEntry['focus'] = 1

  def unfocus( self ):
    self.consoleEntry['focus'] = 0

  def copy( self ):
    copy = self.consoleEntry.get()
    clipboard.setText( copy )

  def paste( self ):
    oldCursorPos = self.consoleEntry.guiItem.getCursorPosition()
    clipboardText = clipboard.getText()

    # compose new text line
    oldText = self.consoleEntry.get()
    newText = oldText[0:oldCursorPos] + clipboardText + oldText[oldCursorPos:]

    clipboardTextLines = newText.split(os.linesep)

    for i in xrange( len(clipboardTextLines)-1 ):
      currentLine = clipboardTextLines[i]
      # we only want printable characters
      currentLine = re.sub( r'[^' + re.escape(string.printable[:95]) + ']', "", currentLine)

      # set new text and position
      self.consoleEntry.set( currentLine )
      self.onEnterPress( currentLine )
    currentLine = clipboardTextLines[-1]
    currentLine = re.sub( r'[^' + re.escape(string.printable[:95]) + ']', "", currentLine)
    self.consoleEntry.set( currentLine )
    self.consoleEntry.setCursorPosition( len(self.consoleEntry.get()) )
    self.focus()

  def cut( self ):
    clipboard.setText( self.consoleEntry.get() )
    self.consoleEntry.enterText('')
    self.focus()

  def scroll( self, step ):
    self.textBufferPos += step
    self.textBufferPos = min( self.textBufferLength-self.numlines, max( 0, self.textBufferPos ) )
    self.updateOutput()

  def scrollCmd( self, step ):
    oldCmdPos = self.userCommandPos
    self.userCommandPos += step
    self.userCommandPos = min( self.userCommandListLength-1, max( 0, self.userCommandPos ) )
    self.userCommandList[oldCmdPos] = self.consoleEntry.get()
    newCmd = self.userCommandList[self.userCommandPos]
    self.consoleEntry.set( newCmd )
    self.consoleEntry.setCursorPosition( len(newCmd) )

  def onEnterPress( self, textEntered ):
    # set to last message
    self.textBufferPos = self.textBufferLength-self.numlines
    # clear line
    self.consoleEntry.enterText('')
    self.focus()
    # add text entered to user command list & remove oldest entry
    self.userCommandList.insert( 1, textEntered )
    self.userCommandList[0] = ''
    self.userCommandList.pop( -1 )
    self.userCommandPos = 0
    # call our parent
    self.parent.push( textEntered )

  def windowEvent( self, window ):
    """
    This is a special callback.
    It is called when the panda window is modified.
    """
    wp = window.getProperties()
    width = wp.getXSize() / float(wp.getYSize())
    height = wp.getYSize() / float(wp.getXSize())
    if width > height:
      height = 1.0
    else:
      width = 1.0
    # aligned to center
    consolePos = Vec3(-self.h_size/2, 0, -self.v_size/2)
    # aligned to left bottom
    #consolePos = Vec3(-width+self.h_pos, 0, -height+self.v_pos)
    # aligned to right top
    #consolePos = Vec3(width-self.h_size, 0, height-self.v_size)
    # set position
    self.consoleFrame.setPos( consolePos )
<commit_msg>switched console to a different cull bin in order to render on top of everything else<commit_after># -----
# panda3dIOClass
# -----
# by Reto Spoerri
# rspoerri AT nouser.org
# http://www.nouser.org
# -----
# Creates a frame in a panda3d window, with a directEntry
# uses the clipboard class to copy, cut & paste
# output can be scrolled by page_up/down
# input can be scrolled by arrow_up/down
# -----

from shared import *

import textwrap, re, string, sys, os

from direct.gui.DirectGui import DirectFrame, DirectEntry, DirectLabel, DGG
from direct.showbase import DirectObject
from pandac.PandaModules import TextNode, Vec3, VBase4
from direct.gui.OnscreenText import OnscreenText

# vardis 27-02-2009: Added special cull bin in order to render on top of everything
from constants import PanoConstants
from pandac.PandaModules import CullBinManager

from clipboard import clipboard

class panda3dIOClass( DirectObject.DirectObject ):
  # set gui key to None if you want to call toggleConsole from outside this class
  gui_key = PANDA3D_CONSOLE_TOGGLE_KEY
  autocomplete_key = PANDA3D_CONSOLE_AUTOCOMPLETE_KEY
  autohelp_key = PANDA3d_CONSOLE_AUTOHELP_KEY
  # change size of text and number of characters on one line
  # scale of frame ( must be small (0.0x)
  scale = PANDA3D_CONSOLE_SCALE
  # to define a special font, if loading fails the default font is used (without warning)
  font = PANDA3D_CONSOLE_FONT
  fontWidth = PANDA3D_CONSOLE_FONT_WIDTH
  # frame position and size (vertical & horizontal)
  h_pos   = PANDA3D_CONSOLE_HORIZONTAL_POS
  h_size  = PANDA3D_CONSOLE_HORIZONTAL_SIZE
  # v_size + v_pos should not exceed 2.0, else parts of the interface will not be visible
  # space above the frame ( must be below 2.0, best between 0.0 and 1.0 )
  v_pos   = PANDA3D_CONSOLE_VERTICAL_POS
  # vertical size of the frame ( must be at max 2.0, best between 0.5 and 2.0 )
  v_size  = PANDA3D_CONSOLE_VERTICAL_SIZE
  linelength = int((h_size/scale - 5) / fontWidth)
  print "max number of characters on a length:", linelength
  numlines = int(v_size/scale - 5)
  defaultTextColor  = (0.0,0.0,0.0,1.0)
  autoCompleteColor = (0.9,0.4,0.2,1.0)
  def __init__( self, parent ):
    self.parent = parent

    # line wrapper
    self.linewrap = textwrap.TextWrapper()
    self.linewrap.width = self.linelength

    # calculate window size
    left   = (self.h_pos) / self.scale
    right  = (self.h_pos + self.h_size) / self.scale
    bottom = (self.v_pos) / self.scale
    top    = (self.v_pos + self.v_size) /self.scale

    # panda3d interface
    self.consoleFrame = DirectFrame ( relief = DGG.GROOVE
                                    , frameColor = (200, 200, 200, 0.5)
                                    , scale=self.scale
                                    , frameSize = (0, self.h_size / self.scale, 0, self.v_size / self.scale) )

    # vardis 27-02-2009: Added special cull bin in order to render on top of everything
    CullBinManager.getGlobalPtr().addBin(PanoConstants.CONSOLE_CULL_BIN_NAME, CullBinManager.BTUnsorted, PanoConstants.CONSOLE_CULL_BIN_VAL)
    self.consoleFrame.setBin(PanoConstants.CONSOLE_CULL_BIN_NAME, 0)

    self.windowEvent( base.win )

    # try to load the defined font
    fixedWidthFont = loader.loadFont(self.font)
    # if font is not valid use default font
    if not fixedWidthFont.isValid():
      if self.font is None:
        print "pandaInteractiveConsole.py :: could not load the defined font %s" % str(self.font)
      fixedWidthFont = DGG.getDefaultFont()

    # text entry line
    self.consoleEntry = DirectEntry ( self.consoleFrame
                                    , text        = ""
                                    , command     = self.onEnterPress
                                    , width       = self.h_size/self.scale - 2
                                    , pos         = (1, 0, 1.5)
                                    , initialText = ""
                                    , numLines    = 1
                                    , focus       = 1
                                    , entryFont   = fixedWidthFont)

    # output lines
    self.consoleOutputList = list()
    for i in xrange( self.numlines ):
      label = OnscreenText( parent = self.consoleFrame
                          , text = ""
                          , pos = (1, -i+3+self.numlines)
                          , align=TextNode.ALeft
                          , mayChange=1
                          , scale=1.0
                          , fg = self.defaultTextColor )
      label.setFont( fixedWidthFont )
      self.consoleOutputList.append( label )

    # list of the last commands of the user
    self.userCommandList = list()
    self.userCommandListLength = 100
    for i in xrange(self.userCommandListLength):
      self.userCommandList.append('')
    self.userCommandPos = 0

    # buffer for data
    self.textBuffer = list()
    self.textBufferLength = 1000
    for i in xrange(self.textBufferLength):
      self.textBuffer.append(['', DEFAULT_COLOR])
    self.textBufferPos = self.textBufferLength-self.numlines

    # toggle the window at least once to activate the events
    self.toggleConsole()
    self.toggleConsole()

    self.help()

  def help( self ):
    # output some info text about this module
    infoTxt = """ ------ Panda3dConsole ------
- press F1 to toggle it on/off
- use the usual copy, cut & paste keys
- page_up    : scrolls up
- page_down  : scrolls down
- arrow_up   : previous command
- arrow_down : next command
- BUGS       : if you paste a to long text, the entry blocks
         FIX : use cut to remove the whole line"""
    #for line in infoTxt.split('\n'):
    #  self.write( line, color=DEFAULT_COLOR )
    return infoTxt

  # write a string to the panda3d console
  def write( self, printString, color=defaultTextColor ):
    # remove not printable characters (which can be input by console input)
    printString = re.sub( r'[^%s]' % re.escape(string.printable[:95]), "", printString)

    splitLines = self.linewrap.wrap(printString)
    for line in splitLines:
      self.textBuffer.append( [line, color] )
      self.textBuffer.pop(0)

    self.updateOutput()

  def updateOutput( self ):
    for lineNumber in xrange(self.numlines):
      lineText, color = self.textBuffer[lineNumber + self.textBufferPos]
      self.consoleOutputList[lineNumber].setText( lineText )
      self.consoleOutputList[lineNumber]['fg'] = color

  # toggle the gui console
  def toggleConsole( self ):
    self.consoleFrame.toggleVis()
    hidden = self.consoleFrame.isHidden()
    self.consoleEntry['focus'] != hidden
    if hidden:
      self.ignoreAll()
#      self.accept( self.gui_key, self.toggleConsole )
    else:
      self.ignoreAll()
      self.accept( 'page_up', self.scroll, [-5] )
      self.accept( 'page_up-repeat', self.scroll, [-5] )
      self.accept( 'page_down', self.scroll, [5] )
      self.accept( 'page_down-repeat', self.scroll, [5] )
      self.accept( 'window-event', self.windowEvent)

      self.accept( 'arrow_up'  , self.scrollCmd, [ 1] )
      self.accept( 'arrow_down', self.scrollCmd, [-1] )

#      self.accept( self.gui_key, self.toggleConsole )
      self.accept( self.autocomplete_key, self.autocomplete )
      self.accept( self.autohelp_key, self.autohelp )

      # accept v, c and x, where c & x copy's the whole console text
      #messenger.toggleVerbose()
      #for osx use ('meta')
      if sys.platform == 'darwin':
        self.accept( 'meta', self.unfocus )
        self.accept( 'meta-up', self.focus )
        self.accept( 'meta-c', self.copy )
        self.accept( 'meta-x', self.cut )
        self.accept( 'meta-v', self.paste )
      #for windows use ('control')
      if sys.platform == 'win32' or sys.platform == 'linux2':
        self.accept( 'control', self.unfocus )
        self.accept( 'control-up', self.focus )
        self.accept( 'control-c', self.copy )
        self.accept( 'control-x', self.cut )
        self.accept( 'control-v', self.paste )

  def autocomplete( self ):
    currentText = self.consoleEntry.get()
    currentPos = self.consoleEntry.guiItem.getCursorPosition()
    newText = self.parent.autocomplete( currentText, currentPos )
    if newText != currentText:
      self.consoleEntry.set( newText )
      self.consoleEntry.setCursorPosition( len(newText) )

  def autohelp( self ):
    currentText = self.consoleEntry.get()
    currentPos = self.consoleEntry.guiItem.getCursorPosition()
    self.parent.autohelp( currentText, currentPos )

  def focus( self ):
    self.consoleEntry['focus'] = 1

  def unfocus( self ):
    self.consoleEntry['focus'] = 0

  def copy( self ):
    copy = self.consoleEntry.get()
    clipboard.setText( copy )

  def paste( self ):
    oldCursorPos = self.consoleEntry.guiItem.getCursorPosition()
    clipboardText = clipboard.getText()

    # compose new text line
    oldText = self.consoleEntry.get()
    newText = oldText[0:oldCursorPos] + clipboardText + oldText[oldCursorPos:]

    clipboardTextLines = newText.split(os.linesep)

    for i in xrange( len(clipboardTextLines)-1 ):
      currentLine = clipboardTextLines[i]
      # we only want printable characters
      currentLine = re.sub( r'[^' + re.escape(string.printable[:95]) + ']', "", currentLine)

      # set new text and position
      self.consoleEntry.set( currentLine )
      self.onEnterPress( currentLine )
    currentLine = clipboardTextLines[-1]
    currentLine = re.sub( r'[^' + re.escape(string.printable[:95]) + ']', "", currentLine)
    self.consoleEntry.set( currentLine )
    self.consoleEntry.setCursorPosition( len(self.consoleEntry.get()) )
    self.focus()

  def cut( self ):
    clipboard.setText( self.consoleEntry.get() )
    self.consoleEntry.enterText('')
    self.focus()

  def scroll( self, step ):
    self.textBufferPos += step
    self.textBufferPos = min( self.textBufferLength-self.numlines, max( 0, self.textBufferPos ) )
    self.updateOutput()

  def scrollCmd( self, step ):
    oldCmdPos = self.userCommandPos
    self.userCommandPos += step
    self.userCommandPos = min( self.userCommandListLength-1, max( 0, self.userCommandPos ) )
    self.userCommandList[oldCmdPos] = self.consoleEntry.get()
    newCmd = self.userCommandList[self.userCommandPos]
    self.consoleEntry.set( newCmd )
    self.consoleEntry.setCursorPosition( len(newCmd) )

  def onEnterPress( self, textEntered ):
    # set to last message
    self.textBufferPos = self.textBufferLength-self.numlines
    # clear line
    self.consoleEntry.enterText('')
    self.focus()
    # add text entered to user command list & remove oldest entry
    self.userCommandList.insert( 1, textEntered )
    self.userCommandList[0] = ''
    self.userCommandList.pop( -1 )
    self.userCommandPos = 0
    # call our parent
    self.parent.push( textEntered )

  def windowEvent( self, window ):
    """
    This is a special callback.
    It is called when the panda window is modified.
    """
    wp = window.getProperties()
    width = wp.getXSize() / float(wp.getYSize())
    height = wp.getYSize() / float(wp.getXSize())
    if width > height:
      height = 1.0
    else:
      width = 1.0
    # aligned to center
    consolePos = Vec3(-self.h_size/2, 0, -self.v_size/2)
    # aligned to left bottom
    #consolePos = Vec3(-width+self.h_pos, 0, -height+self.v_pos)
    # aligned to right top
    #consolePos = Vec3(width-self.h_size, 0, height-self.v_size)
    # set position
    self.consoleFrame.setPos( consolePos )
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand
from xbrowse_server.base.models import Project as BaseProject, Family as BaseFamily, VariantNote as BaseVariantNote, \
    VariantTag as BaseVariantTag, ProjectTag, VCFFile
from xbrowse_server.base.model_utils import update_xbrowse_model, find_matching_seqr_model, get_or_create_xbrowse_model
from seqr.views.apis.phenotips_api import _get_phenotips_uname_and_pwd_for_project, _add_user_to_patient, \
    phenotips_patient_exists, _get_patient_data

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('--from-project', required=True)
        parser.add_argument('--to-project', required=True)
        parser.add_argument('family_ids', nargs='+')

    def handle(self, *args, **options):
        from_project = BaseProject.objects.get(project_id=options['from_project'])
        to_project = BaseProject.objects.get(project_id=options['to_project'])
        to_seqr_project = find_matching_seqr_model(to_project)
        family_ids = options['family_ids']
        families = BaseFamily.objects.filter(project=from_project, family_id__in=family_ids)
        print('Found {} out of {} families. No match for: {}.'.format(len(families), len(set(family_ids)), set(family_ids) - set([f.family_id for f in families])))

        for f in families:
            print("==> Moving {}".format(f))
            for individual in f.individual_set.all():
                individual.project = to_project
                individual.save()
                # Update individuals in phenotips
                if phenotips_patient_exists(individual.seqr_individual):
                    # make sure phenotips_patient_id is up to date
                    seqr_individual = individual.seqr_individual
                    data_json = _get_patient_data(
                        to_project.seqr_project,
                        seqr_individual,
                    )

                    seqr_individual.phenotips_patient_id = data_json["id"]
                    seqr_individual.save()

                    # update permissions
                    phenotips_readonly_username, _ = _get_phenotips_uname_and_pwd_for_project(to_project.seqr_project.phenotips_user_id, read_only=True)
                    _add_user_to_patient(phenotips_readonly_username, seqr_individual.phenotips_patient_id, allow_edit=False)

                    phenotips_readwrite_username, _ = _get_phenotips_uname_and_pwd_for_project(to_project.seqr_project.phenotips_user_id, read_only=False)
                    _add_user_to_patient(phenotips_readwrite_username, seqr_individual.phenotips_patient_id, allow_edit=True)

                # Update individuals samples/ VCFs
                for from_vcf_file in individual.vcf_files.all():
                    to_vcf_file, _ = VCFFile.objects.get_or_create(
                        project=to_project,
                        elasticsearch_index=from_vcf_file.elasticsearch_index,
                        file_path=from_vcf_file.file_path,
                        dataset_type=from_vcf_file.dataset_type,
                        sample_type=from_vcf_file.sample_type,
                        loaded_date=from_vcf_file.loaded_date,
                    )
                    individual.vcf_files.add(to_vcf_file)
                    individual.vcf_files.remove(from_vcf_file)

            # Update variant tags/ notes
            saved_variants = set()
            for note in BaseVariantNote.objects.filter(family=f):
                seqr_note = find_matching_seqr_model(note)
                if seqr_note:
                    saved_variants.add(seqr_note.saved_variant)
                note.project = to_project
                note.save()

            for variant_tag in BaseVariantTag.objects.filter(family=f):
                to_project_tag, created = get_or_create_xbrowse_model(
                    ProjectTag, project=to_project, tag=variant_tag.project_tag.tag
                )
                if created:
                    update_xbrowse_model(
                        to_project_tag,
                        category=variant_tag.project_tag.category,
                        title=variant_tag.project_tag.title,
                        color=variant_tag.project_tag.color,
                        order=variant_tag.project_tag.order,
                    )
                update_xbrowse_model(variant_tag, project_tag=to_project_tag)
                seqr_variant_tag = find_matching_seqr_model(variant_tag)
                if seqr_variant_tag:
                    saved_variants.add(seqr_variant_tag.saved_variant)

            for saved_variant in saved_variants:
                saved_variant.project = to_seqr_project
                saved_variant.save()

            # Update families
            update_xbrowse_model(f, project=to_project)

        print("Done.")
<commit_msg>fix transfer families manage commmand<commit_after>from django.core.management.base import BaseCommand
from xbrowse_server.base.models import Project as BaseProject, Family as BaseFamily, VariantNote as BaseVariantNote, \
    VariantTag as BaseVariantTag, ProjectTag, VCFFile
from xbrowse_server.base.model_utils import update_xbrowse_model, find_matching_seqr_model, get_or_create_xbrowse_model
from seqr.views.apis.phenotips_api import _get_phenotips_uname_and_pwd_for_project, _add_user_to_patient, \
    _phenotips_patient_exists, _get_patient_data

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('--from-project', required=True)
        parser.add_argument('--to-project', required=True)
        parser.add_argument('family_ids', nargs='+')

    def handle(self, *args, **options):
        from_project = BaseProject.objects.get(project_id=options['from_project'])
        to_project = BaseProject.objects.get(project_id=options['to_project'])
        to_seqr_project = find_matching_seqr_model(to_project)
        family_ids = options['family_ids']
        families = BaseFamily.objects.filter(project=from_project, family_id__in=family_ids)
        print('Found {} out of {} families. No match for: {}.'.format(len(families), len(set(family_ids)), set(family_ids) - set([f.family_id for f in families])))

        for f in families:
            print("==> Moving {}".format(f))
            for individual in f.individual_set.all():
                individual.project = to_project
                individual.save()
                # Update individuals in phenotips
                if _phenotips_patient_exists(individual.seqr_individual):
                    # make sure phenotips_patient_id is up to date
                    seqr_individual = individual.seqr_individual
                    data_json = _get_patient_data(
                        to_project.seqr_project,
                        seqr_individual,
                    )

                    seqr_individual.phenotips_patient_id = data_json["id"]
                    seqr_individual.save()

                    # update permissions
                    phenotips_readonly_username, _ = _get_phenotips_uname_and_pwd_for_project(to_project.seqr_project.phenotips_user_id, read_only=True)
                    _add_user_to_patient(phenotips_readonly_username, seqr_individual.phenotips_patient_id, allow_edit=False)

                    phenotips_readwrite_username, _ = _get_phenotips_uname_and_pwd_for_project(to_project.seqr_project.phenotips_user_id, read_only=False)
                    _add_user_to_patient(phenotips_readwrite_username, seqr_individual.phenotips_patient_id, allow_edit=True)

                # Update individuals samples/ VCFs
                for from_vcf_file in individual.vcf_files.all():
                    to_vcf_file, _ = VCFFile.objects.get_or_create(
                        project=to_project,
                        elasticsearch_index=from_vcf_file.elasticsearch_index,
                        file_path=from_vcf_file.file_path,
                        dataset_type=from_vcf_file.dataset_type,
                        sample_type=from_vcf_file.sample_type,
                        loaded_date=from_vcf_file.loaded_date,
                    )
                    individual.vcf_files.add(to_vcf_file)
                    individual.vcf_files.remove(from_vcf_file)

            # Update variant tags/ notes
            saved_variants = set()
            for note in BaseVariantNote.objects.filter(family=f):
                seqr_note = find_matching_seqr_model(note)
                if seqr_note:
                    saved_variants.add(seqr_note.saved_variant)
                note.project = to_project
                note.save()

            for variant_tag in BaseVariantTag.objects.filter(family=f):
                to_project_tag, created = get_or_create_xbrowse_model(
                    ProjectTag, project=to_project, tag=variant_tag.project_tag.tag
                )
                if created:
                    update_xbrowse_model(
                        to_project_tag,
                        category=variant_tag.project_tag.category,
                        title=variant_tag.project_tag.title,
                        color=variant_tag.project_tag.color,
                        order=variant_tag.project_tag.order,
                    )
                update_xbrowse_model(variant_tag, project_tag=to_project_tag)
                seqr_variant_tag = find_matching_seqr_model(variant_tag)
                if seqr_variant_tag:
                    saved_variants.add(seqr_variant_tag.saved_variant)

            for saved_variant in saved_variants:
                saved_variant.project = to_seqr_project
                saved_variant.save()

            # Update families
            update_xbrowse_model(f, project=to_project)

        print("Done.")
<|endoftext|>


<commit_before>#
# ovirt-host-deploy -- ovirt host deployer
# Copyright (C) 2012-2014 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""kdump packages plugin."""

import gettext
import platform
import re
import socket
import rpm

from otopi import constants as otopicons
from otopi import filetransaction
from otopi import plugin
from otopi import util

from ovirt_host_deploy import constants as odeploycons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-host-deploy')

@util.export
class Plugin(plugin.PluginBase):
    """Required packages installation.

    Environment:
        KdumpEnv.ENABLE              -- perform kdump detection configuration
        KdumpEnv.SUPPORTED           -- kexec-tools package supports
                                        fence_kdump configuration
        KdumpEnv.DESTINATION_ADDRESS -- server to send fence_kdump messages to
        KdumpEnv.DESTINATION_PORT    -- port to send fence_kdump messages to
        KdumpEnv.MESSAGE_INTERVAL    -- interval between fence_kdump messages

    """

    _KEXEC_TOOLS_PKG = 'kexec-tools'

    # pattern to match lines with fence_kdump configuration
    _FK_OPTS_REGEX = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            fence_kdump_(nodes|args)
            \s
            .*
            $
        """
    )

    # pattern to match lines with ovirt-host-deploy configuration backup
    _FK_BACKUP_REGEX = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            \#ovirt-host-deploy:backup-begin
            .*
            $
        """
    )

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)
        self._enabled = True

    def _get_min_kexec_tools_version(self):
        (name, version, desc) = platform.linux_distribution(
            full_distribution_name=0
        )
        min_version = None
        if name in ('redhat', 'centos'):
            major = version.split('.', 1)[0]
            if major == '7':
                min_version = 0, '2.0.4', '32.1'
        elif name in ('fedora', 'ibm_powerkvm'):
            min_version = 0, '2.0.4', '27'

        return min_version

    def _crashkernel_param_present(self):
        crashkernel = False
        with open('/proc/cmdline') as f:
            for line in f.read().splitlines():
                if 'crashkernel=' in line:
                    crashkernel = True
                    break
        return crashkernel

    def _resolve_destination_address(self, host):
        try:
            result = socket.getaddrinfo(
                host,
                None,
            )
            self.logger.debug(
                "Kdump destination '%s' addresses: '%s'",
                host,
                result,
            )
        except Exception:
            self.logger.debug(
                "Cannot resolve kdump destination address '%s'",
                host,
                exc_info=True,
            )
            raise RuntimeError(
                _("Cannot resolve kdump destination address '{host}'").format(
                    host=host,
                )
            )

    def _create_hdr(self, epoch, version, release):
        hdr = rpm.hdr()
        hdr[rpm.RPMTAG_EPOCH] = epoch
        hdr[rpm.RPMTAG_VERSION] = version
        hdr[rpm.RPMTAG_RELEASE] = release
        return hdr

    def _kexec_tools_version_supported(self):
        result = False

        if self.environment[odeploycons.VdsmEnv.OVIRT_VINTAGE_NODE]:
            # on node check ovirt-node-plugin-vdsm features
            result = 'kdump' in self.environment[
                odeploycons.VdsmEnv.NODE_PLUGIN_VDSM_FEATURES
            ]
        else:
            # on standard host use packager
            min_version = self._get_min_kexec_tools_version()
            min_hdr = self._create_hdr(*min_version)
            if min_version is not None:
                pkgs = self.packager.queryPackages(
                    patterns=(self._KEXEC_TOOLS_PKG,),
                )
                for package in pkgs:
                    cur_hdr = self._create_hdr(
                        0,
                        package['version'],
                        package['release']
                    )
                    if rpm.versionCompare(cur_hdr, min_hdr) >= 0:
                        result = True
                        break

        return result

    def _update_kdump_conf(
            self,
            content,
            engine_node,
            port,
            interval,
    ):
        new_content = []
        backup = []
        backup_exists = False

        for line in content:
            if self._FK_OPTS_REGEX.match(line) is not None:
                # line with fence_kdump config
                backup.append(line)
            else:
                if self._FK_BACKUP_REGEX.match(line) is not None:
                    # line with fence_kdump config backup
                    backup_exists = True

                new_content.append(line)

        if not backup_exists and backup:
            new_content.append('#ovirt-host-deploy:backup-begin')
            new_content.extend(['#' + l for l in backup])
            new_content.append('#ovirt-host-deploy:backup-end')

        new_content.extend(
            (
                'fence_kdump_nodes %s' % (
                    engine_node,
                ),
                'fence_kdump_args -p %s -i %s' % (
                    port,
                    interval,
                ),
            )
        )
        return new_content

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
    )
    def _init(self):
        self.environment.setdefault(odeploycons.KdumpEnv.ENABLE, False)
        self.environment.setdefault(odeploycons.KdumpEnv.SUPPORTED, False)

    @plugin.event(
        stage=plugin.Stages.STAGE_CUSTOMIZATION,
        priority=plugin.Stages.PRIORITY_HIGH,
    )
    def _customization(self):
        self.environment[odeploycons.KdumpEnv.SUPPORTED] = (
            self._crashkernel_param_present() and
            self._kexec_tools_version_supported()
        )

        self.logger.info(
            _('Kdump {result}').format(
                result=(
                    'supported'
                    if self.environment[odeploycons.KdumpEnv.SUPPORTED]
                    else 'unsupported'
                ),
            )
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_PACKAGES,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _packages(self):
        self.packager.installUpdate(
            packages=(self._KEXEC_TOOLS_PKG,),
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_VALIDATION,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _validate(self):
        self._resolve_destination_address(
            self.environment[odeploycons.KdumpEnv.DESTINATION_ADDRESS],
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_MISC,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _misc(self):
        with open(odeploycons.FileLocations.KDUMP_CONFIG_FILE, 'r') as f:
            content = f.read().splitlines()

        new_content = self._update_kdump_conf(
            content=content,
            engine_node=self.environment[
                odeploycons.KdumpEnv.DESTINATION_ADDRESS
            ],
            port=self.environment[
                odeploycons.KdumpEnv.DESTINATION_PORT
            ],
            interval=self.environment[
                odeploycons.KdumpEnv.MESSAGE_INTERVAL
            ],
        )

        self.environment[
            otopicons.CoreEnv.MAIN_TRANSACTION
        ].append(
            filetransaction.FileTransaction(
                name=odeploycons.FileLocations.KDUMP_CONFIG_FILE,
                content=new_content,
                modifiedList=self.environment[
                    otopicons.CoreEnv.MODIFIED_FILES
                ],
            )
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_CLOSEUP,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _closeup(self):
        self.logger.info(_('Restarting kdump'))
        self.services.startup('kdump', True)
        try:
            for state in (False, True):
                self.services.state('kdump', state)
        except RuntimeError:
            self.logger.debug('kdump service failed', exc_info=True)
            raise RuntimeError(
                _(
                    'kdump service restart failed. Please either redeploy '
                    'with Kdump Integration disabled or fix kdump '
                    'configuration manually and redeploy the host'
                )
            )

# vim: expandtab tabstop=4 shiftwidth=4
<commit_msg>Add el8 to _get_min_kexec_tools_version<commit_after>#
# ovirt-host-deploy -- ovirt host deployer
# Copyright (C) 2012-2014 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""kdump packages plugin."""

import gettext
import platform
import re
import socket
import rpm

from otopi import constants as otopicons
from otopi import filetransaction
from otopi import plugin
from otopi import util

from ovirt_host_deploy import constants as odeploycons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-host-deploy')

@util.export
class Plugin(plugin.PluginBase):
    """Required packages installation.

    Environment:
        KdumpEnv.ENABLE              -- perform kdump detection configuration
        KdumpEnv.SUPPORTED           -- kexec-tools package supports
                                        fence_kdump configuration
        KdumpEnv.DESTINATION_ADDRESS -- server to send fence_kdump messages to
        KdumpEnv.DESTINATION_PORT    -- port to send fence_kdump messages to
        KdumpEnv.MESSAGE_INTERVAL    -- interval between fence_kdump messages

    """

    _KEXEC_TOOLS_PKG = 'kexec-tools'

    # pattern to match lines with fence_kdump configuration
    _FK_OPTS_REGEX = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            fence_kdump_(nodes|args)
            \s
            .*
            $
        """
    )

    # pattern to match lines with ovirt-host-deploy configuration backup
    _FK_BACKUP_REGEX = re.compile(
        flags=re.VERBOSE,
        pattern=r"""
            ^
            \#ovirt-host-deploy:backup-begin
            .*
            $
        """
    )

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)
        self._enabled = True

    def _get_min_kexec_tools_version(self):
        (name, version, desc) = platform.linux_distribution(
            full_distribution_name=0
        )
        min_version = None
        if name in ('redhat', 'centos'):
            major = version.split('.', 1)[0]
            if major == '7':
                min_version = 0, '2.0.4', '32.1'
            if major == '8':
                min_version = 0, '2.0.17', '28'
        elif name in ('fedora', 'ibm_powerkvm'):
            min_version = 0, '2.0.4', '27'

        return min_version

    def _crashkernel_param_present(self):
        crashkernel = False
        with open('/proc/cmdline') as f:
            for line in f.read().splitlines():
                if 'crashkernel=' in line:
                    crashkernel = True
                    break
        return crashkernel

    def _resolve_destination_address(self, host):
        try:
            result = socket.getaddrinfo(
                host,
                None,
            )
            self.logger.debug(
                "Kdump destination '%s' addresses: '%s'",
                host,
                result,
            )
        except Exception:
            self.logger.debug(
                "Cannot resolve kdump destination address '%s'",
                host,
                exc_info=True,
            )
            raise RuntimeError(
                _("Cannot resolve kdump destination address '{host}'").format(
                    host=host,
                )
            )

    def _create_hdr(self, epoch, version, release):
        hdr = rpm.hdr()
        hdr[rpm.RPMTAG_EPOCH] = epoch
        hdr[rpm.RPMTAG_VERSION] = version
        hdr[rpm.RPMTAG_RELEASE] = release
        return hdr

    def _kexec_tools_version_supported(self):
        result = False

        if self.environment[odeploycons.VdsmEnv.OVIRT_VINTAGE_NODE]:
            # on node check ovirt-node-plugin-vdsm features
            result = 'kdump' in self.environment[
                odeploycons.VdsmEnv.NODE_PLUGIN_VDSM_FEATURES
            ]
        else:
            # on standard host use packager
            min_version = self._get_min_kexec_tools_version()
            min_hdr = self._create_hdr(*min_version)
            if min_version is not None:
                pkgs = self.packager.queryPackages(
                    patterns=(self._KEXEC_TOOLS_PKG,),
                )
                for package in pkgs:
                    cur_hdr = self._create_hdr(
                        0,
                        package['version'],
                        package['release']
                    )
                    if rpm.versionCompare(cur_hdr, min_hdr) >= 0:
                        result = True
                        break

        return result

    def _update_kdump_conf(
            self,
            content,
            engine_node,
            port,
            interval,
    ):
        new_content = []
        backup = []
        backup_exists = False

        for line in content:
            if self._FK_OPTS_REGEX.match(line) is not None:
                # line with fence_kdump config
                backup.append(line)
            else:
                if self._FK_BACKUP_REGEX.match(line) is not None:
                    # line with fence_kdump config backup
                    backup_exists = True

                new_content.append(line)

        if not backup_exists and backup:
            new_content.append('#ovirt-host-deploy:backup-begin')
            new_content.extend(['#' + l for l in backup])
            new_content.append('#ovirt-host-deploy:backup-end')

        new_content.extend(
            (
                'fence_kdump_nodes %s' % (
                    engine_node,
                ),
                'fence_kdump_args -p %s -i %s' % (
                    port,
                    interval,
                ),
            )
        )
        return new_content

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
    )
    def _init(self):
        self.environment.setdefault(odeploycons.KdumpEnv.ENABLE, False)
        self.environment.setdefault(odeploycons.KdumpEnv.SUPPORTED, False)

    @plugin.event(
        stage=plugin.Stages.STAGE_CUSTOMIZATION,
        priority=plugin.Stages.PRIORITY_HIGH,
    )
    def _customization(self):
        self.environment[odeploycons.KdumpEnv.SUPPORTED] = (
            self._crashkernel_param_present() and
            self._kexec_tools_version_supported()
        )

        self.logger.info(
            _('Kdump {result}').format(
                result=(
                    'supported'
                    if self.environment[odeploycons.KdumpEnv.SUPPORTED]
                    else 'unsupported'
                ),
            )
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_PACKAGES,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _packages(self):
        self.packager.installUpdate(
            packages=(self._KEXEC_TOOLS_PKG,),
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_VALIDATION,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _validate(self):
        self._resolve_destination_address(
            self.environment[odeploycons.KdumpEnv.DESTINATION_ADDRESS],
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_MISC,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _misc(self):
        with open(odeploycons.FileLocations.KDUMP_CONFIG_FILE, 'r') as f:
            content = f.read().splitlines()

        new_content = self._update_kdump_conf(
            content=content,
            engine_node=self.environment[
                odeploycons.KdumpEnv.DESTINATION_ADDRESS
            ],
            port=self.environment[
                odeploycons.KdumpEnv.DESTINATION_PORT
            ],
            interval=self.environment[
                odeploycons.KdumpEnv.MESSAGE_INTERVAL
            ],
        )

        self.environment[
            otopicons.CoreEnv.MAIN_TRANSACTION
        ].append(
            filetransaction.FileTransaction(
                name=odeploycons.FileLocations.KDUMP_CONFIG_FILE,
                content=new_content,
                modifiedList=self.environment[
                    otopicons.CoreEnv.MODIFIED_FILES
                ],
            )
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_CLOSEUP,
        condition=lambda self: self.environment[odeploycons.KdumpEnv.ENABLE],
    )
    def _closeup(self):
        self.logger.info(_('Restarting kdump'))
        self.services.startup('kdump', True)
        try:
            for state in (False, True):
                self.services.state('kdump', state)
        except RuntimeError:
            self.logger.debug('kdump service failed', exc_info=True)
            raise RuntimeError(
                _(
                    'kdump service restart failed. Please either redeploy '
                    'with Kdump Integration disabled or fix kdump '
                    'configuration manually and redeploy the host'
                )
            )

# vim: expandtab tabstop=4 shiftwidth=4
<|endoftext|>


<commit_before>import textwrap
import collections

def indent(text, amount, ch=' '):
    return textwrap.indent(text, amount * ch)

def flatten(l):
    for el in l:
        if isinstance(
                el, collections.Iterable) and not isinstance(el, (str, bytes)):
            yield from flatten(el)
        else:
            yield el

# def makeReferenceList(originList):
#     referenceList = []
#     for i in originList:
#         referenceList.append(i)
#     return referenceList

class SingleBool():
    '''
    A type of Single that only have a single tag.
    Usually indicate true of false.
    '''
    scalar = ''

    def __init__(self, scalar):
        self.scalar = scalar

    def printMe(self, scalar):
        text = '</{scalar}>\n'.format(scalar=scalar)
        return text

class Single():
    '''
    A type of structure that only have a tag and it's values.
    At init, tag and value will be set to empty.
    '''
    tag = ''
    value = []

    def __init__(self, tag, *value):
        self.tag = tag
        self.value = list(flatten(value))

    def printMe(self, tag, value):
        if len(value) == 0:
            return ''
        elif len(value) == 1:
            text = '<{tag}>{value}</{tag}>\n'.format(tag=tag, value=value[0])
            return text
        else:
            valueText = ''
            for single in value:
                valueText += single.printMe(single.tag, single.value)
            valueText = indent(valueText, 4)
            text = '<{tag}>\n'.format(
                tag=tag) + valueText + '</{tag}>\n'.format(tag=tag)
            return text

class TypedSingle(Single):
    '''
    A little sugar so that you don't need to
    type the tag every time creating a single
    '''

    def __init__(self, *value):
        tag = self.__class__.__name__.replace('Single', '').lower()
        super().__init__(tag, value)

class StringSingle(TypedSingle):
    pass

class ArraySingle(TypedSingle):
    pass

class DictSingle(TypedSingle):
    pass

class IntegerSingle(TypedSingle):
    pass

class Pair():
    '''
    A type of structure that have <key>Key</key> and it's value.
    By default, value is set to empty.
    Its value should contain only one single.
    If not provide in init parameter, key is set to subclass name in init.
    Init also accept value in combination of strings
    and lists of strings.

    '''
    key = ''
    value = []

    def __init__(self, key='', *value):
        if key == '':
            self.key = self.__class__.__name__
        else:
            self.key = key
        if len(value) != 0:
            self.value = list(flatten(value))

    def printMe(self, key, value):
        text = '<key>{keyName}</key>\n'.format(keyName=key)

        if len(value) == 0:
            return ''
        else:
            valueText = ''
            for single in value:
                # may be remove the loop
                # single = value[0]
                if isinstance(single, Single):
                    valueText += single.printMe(single.tag, single.value)
                elif isinstance(single, SingleBool):
                    valueText += single.printMe(single.scalar)

        text += valueText
        return text

    def clear(self):
        self.value = []

class BoolPairTemplate(Pair):
    '''
    A special type of pair that contains it's key and
    only one tag, usually </true> or </false>.
    At init, value will be set as true.
    '''

    def __init__(self):
        super().__init__()
        self.setToTrue()

    def setToTrue(self):
        '''
        This function sets the value of key true.
        '''
        trueBool = SingleBool('true')
        self.value = [trueBool]

class SingleStringPairTemplate(Pair):
    '''
    Pair that only contain a key and a string single.
    '''

    def __init__(self, label):
        super().__init__()
        self.changeTo(label)

    def update(self, single):
        self.clear()
        self.value.append(single)

    def changeTo(self, label):
        stringSingle = StringSingle(label)
        self.update(stringSingle)

class ArrayPairTemplate(Pair):
    def __init__(self, value):
        super().__init__()
        self.add(value)

    def update(self, *value):
        self.value.append(list(flatten(value)))

    def add(self, *value):
        arraySingle = ArraySingle(value)

class Label(SingleStringPairTemplate):
    pass

class Program(SingleStringPairTemplate):
    pass

class ProgramArguments():
    def update(self):
        pass

class EnvironmentVariables():
    def update(self):
        pass

class StandardInPath():
    def update(self):
        pass

class StandardOutPath():
    def update(self):
        pass

class StandardErrorPath():
    def update(self):
        pass

class WorkingDirectory():
    def update(self):
        pass

class SoftResourceLimit():
    def __init__(self):
        pass

    def update(self):
        pass

class HardResourceLimit():
    def __init__(self):
        pass

    def update(self):
        pass

class RunAtLoad(BoolPairTemplate):
    pass

class StartInterval(Pair):
    baseNumber = 1
    magnification = 1
    key = 'StartInterval'

    def __init__(self):
        pass

    def every(self, baseNumber):
        self.baseNumber = baseNumber
        return self

    def update(self, baseNumber, magnification):
        schedule = baseNumber * magnification
        integerSingle = Single('integer', schedule)
        if len(self.value) != 0:
            self.value = []
        self.value.append(integerSingle)

    @property
    def second(self):
        self.magnification = 1
        self.update(self.baseNumber, self.magnification)

    @property
    def minute(self):
        self.magnification = 60
        self.update(self.baseNumber, self.magnification)

    @property
    def hour(self):
        self.magnification = 3600
        self.update(self.baseNumber, self.magnification)

    @property
    def day(self):
        self.magnification = 86400
        self.update(self.baseNumber, self.magnification)

    @property
    def week(self):
        self.magnification = 345600
        self.update(self.baseNumber, self.magnification)

class StartCalendarInterval():
    def __init__(self):
        pass

    def update(self):
        pass

class StartOnMount():
    pass

class WatchPath():
    pass

class QueueDirecotries():
    pass

class KeepAlive():
    pass

class UserName():
    pass

class GroupName():
    pass

class InitGroups():
    pass

class Umask():
    def __init__(self):
        pass

    def update(self):
        pass

class RootDirecotry():
    def __init__(self):
        pass

    def update(self):
        pass

class AbandonProcessGroup():
    def __init__(self):
        pass

    def update(self):
        pass

class ExitTimeOut():
    def __init__(self):
        pass

    def update(self):
        pass

class Timeout():
    def __init__(self):
        pass

    def update(self):
        pass

class ThrottleInverval():
    def __init__(self):
        pass

    def update(self):
        pass

class LegacyTimers():
    def __init__(self):
        pass

    def update(self):
        pass

class Nice():
    def __init__(self):
        pass

    def update(self):
        pass

if __name__ == '__main__':
    # innerSingle1 = Single('inner1', 'aaaaaaaaaaaaa')
    # innerSingle2 = Single('inner2', 'bbbbbbbbbbbbb')
    # single1 = Single('single tag1', innerSingle1, innerSingle2)
    # single2 = Single('single tag2', innerSingle1, innerSingle2)
    # singleSingle = Single('singleSingle', 'something')
    # pair1 = Pair('my key', single1, single2)
    # pair2 = Pair('mey key', singleSingle)
    # print(pair2.printMe(pair2.key, pair2.value))

    # schedule = StartInterval()
    # schedule.every(5).day
    # schedule.every(10).minute
    schedule = RunAtLoad()
    print(schedule.key)
    print(schedule.printMe(schedule.key, schedule.value))
<commit_msg>add() and make() for ArrayPairTemplate<commit_after>import textwrap
import collections

def indent(text, amount, ch=' '):
    return textwrap.indent(text, amount * ch)

def flatten(l):
    for el in l:
        if isinstance(
                el, collections.Iterable) and not isinstance(el, (str, bytes)):
            yield from flatten(el)
        else:
            yield el

# def makeReferenceList(originList):
#     referenceList = []
#     for i in originList:
#         referenceList.append(i)
#     return referenceList

class SingleBool():
    '''
    A type of Single that only have a single tag.
    Usually indicate true of false.
    '''
    scalar = ''

    def __init__(self, scalar):
        self.scalar = scalar

    def printMe(self, scalar):
        text = '</{scalar}>\n'.format(scalar=scalar)
        return text

class Single():
    '''
    A type of structure that only have a tag and it's values.
    At init, tag and value will be set to empty.
    '''
    tag = ''
    value = []

    def __init__(self, tag, *value):
        self.tag = tag
        self.value = list(flatten(value))

    def printMe(self, tag, value):
        if len(value) == 0:
            return ''
        elif len(value) == 1:
            text = '<{tag}>{value}</{tag}>\n'.format(tag=tag, value=value[0])
            return text
        else:
            valueText = ''
            for single in value:
                valueText += single.printMe(single.tag, single.value)
            valueText = indent(valueText, 4)
            text = '<{tag}>\n'.format(
                tag=tag) + valueText + '</{tag}>\n'.format(tag=tag)
            return text

class TypedSingle(Single):
    '''
    A little sugar so that you don't need to
    type the tag every time creating a single
    '''

    def __init__(self, *value):
        tag = self.__class__.__name__.replace('Single', '').lower()
        super().__init__(tag, value)

class StringSingle(TypedSingle):
    pass

class ArraySingle(TypedSingle):
    pass

class DictSingle(TypedSingle):
    pass

class IntegerSingle(TypedSingle):
    pass

class Pair():
    '''
    A type of structure that have <key>Key</key> and it's value.
    By default, value is set to empty.
    Its value should contain only one single.
    If not provide in init parameter, key is set to subclass name in init.
    Init also accept value in combination of strings
    and lists of strings.

    '''
    key = ''
    value = []

    def __init__(self, key='', *value):
        if key == '':
            self.key = self.__class__.__name__
        else:
            self.key = key
        if len(value) != 0:
            self.value = list(flatten(value))

    def printMe(self, key, value):
        text = '<key>{keyName}</key>\n'.format(keyName=key)

        if len(value) == 0:
            return ''
        else:
            valueText = ''
            for single in value:
                # may be remove the loop
                # single = value[0]
                if isinstance(single, Single):
                    valueText += single.printMe(single.tag, single.value)
                elif isinstance(single, SingleBool):
                    valueText += single.printMe(single.scalar)

        text += valueText
        return text

    def clear(self):
        self.value = []

class BoolPairTemplate(Pair):
    '''
    A special type of pair that contains it's key and
    only one tag, usually </true> or </false>.
    At init, value will be set as true.
    '''

    def __init__(self):
        super().__init__()
        self.setToTrue()

    def setToTrue(self):
        '''
        This function sets the value of key true.
        '''
        trueBool = SingleBool('true')
        self.value = [trueBool]

class SingleStringPairTemplate(Pair):
    '''
    Pair that only contain a key and a string single.
    '''

    def __init__(self, label):
        super().__init__()
        self.changeTo(label)

    def update(self, single):
        self.clear()
        self.value.append(single)

    def changeTo(self, label):
        stringSingle = StringSingle(label)
        self.update(stringSingle)

class ArrayPairTemplate(Pair):
    '''
    The type of Pair that contains a array single
    '''

    def __init__(self, value):
        super().__init__()
        self.add(value)

    def update(self, *value):
        self.value.append(list(flatten(value)))

    def make(self, *value):
        arraySingle = ArraySingle(value)
        return arraySingle

    def add(self, single):
        for valueElement in self.value:
            valueElement.add(single)
        self.value.append(ArraySingle)

class Label(SingleStringPairTemplate):
    pass

class Program(SingleStringPairTemplate):
    pass

class ProgramArguments():
    def update(self):
        pass

class EnvironmentVariables():
    def update(self):
        pass

class StandardInPath():
    def update(self):
        pass

class StandardOutPath():
    def update(self):
        pass

class StandardErrorPath():
    def update(self):
        pass

class WorkingDirectory():
    def update(self):
        pass

class SoftResourceLimit():
    def __init__(self):
        pass

    def update(self):
        pass

class HardResourceLimit():
    def __init__(self):
        pass

    def update(self):
        pass

class RunAtLoad(BoolPairTemplate):
    pass

class StartInterval(Pair):
    baseNumber = 1
    magnification = 1
    key = 'StartInterval'

    def __init__(self):
        pass

    def every(self, baseNumber):
        self.baseNumber = baseNumber
        return self

    def update(self, baseNumber, magnification):
        schedule = baseNumber * magnification
        integerSingle = Single('integer', schedule)
        if len(self.value) != 0:
            self.value = []
        self.value.append(integerSingle)

    @property
    def second(self):
        self.magnification = 1
        self.update(self.baseNumber, self.magnification)

    @property
    def minute(self):
        self.magnification = 60
        self.update(self.baseNumber, self.magnification)

    @property
    def hour(self):
        self.magnification = 3600
        self.update(self.baseNumber, self.magnification)

    @property
    def day(self):
        self.magnification = 86400
        self.update(self.baseNumber, self.magnification)

    @property
    def week(self):
        self.magnification = 345600
        self.update(self.baseNumber, self.magnification)

class StartCalendarInterval():
    def __init__(self):
        pass

    def update(self):
        pass

class StartOnMount():
    pass

class WatchPath():
    pass

class QueueDirecotries():
    pass

class KeepAlive():
    pass

class UserName():
    pass

class GroupName():
    pass

class InitGroups():
    pass

class Umask():
    def __init__(self):
        pass

    def update(self):
        pass

class RootDirecotry():
    def __init__(self):
        pass

    def update(self):
        pass

class AbandonProcessGroup():
    def __init__(self):
        pass

    def update(self):
        pass

class ExitTimeOut():
    def __init__(self):
        pass

    def update(self):
        pass

class Timeout():
    def __init__(self):
        pass

    def update(self):
        pass

class ThrottleInverval():
    def __init__(self):
        pass

    def update(self):
        pass

class LegacyTimers():
    def __init__(self):
        pass

    def update(self):
        pass

class Nice():
    def __init__(self):
        pass

    def update(self):
        pass

if __name__ == '__main__':
    # innerSingle1 = Single('inner1', 'aaaaaaaaaaaaa')
    # innerSingle2 = Single('inner2', 'bbbbbbbbbbbbb')
    # single1 = Single('single tag1', innerSingle1, innerSingle2)
    # single2 = Single('single tag2', innerSingle1, innerSingle2)
    # singleSingle = Single('singleSingle', 'something')
    # pair1 = Pair('my key', single1, single2)
    # pair2 = Pair('mey key', singleSingle)
    # print(pair2.printMe(pair2.key, pair2.value))

    # schedule = StartInterval()
    # schedule.every(5).day
    # schedule.every(10).minute
    schedule = RunAtLoad()
    print(schedule.key)
    print(schedule.printMe(schedule.key, schedule.value))
<|endoftext|>


<commit_before># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

__author__ = 'John Sirois'

import os
import textwrap

from collections import defaultdict

from twitter.common.collections import OrderedDict
from twitter.common.dirutil import safe_mkdir, safe_open, touch

from twitter.pants import is_scala, is_scalac_plugin
from twitter.pants.base.target import Target
from twitter.pants.targets.scala_library import ScalaLibrary
from twitter.pants.targets.scala_tests import ScalaTests
from twitter.pants.targets import resolve_target_sources
from twitter.pants.tasks import TaskError
from twitter.pants.tasks.binary_utils import nailgun_profile_classpath
from twitter.pants.tasks.jvm_compiler_dependencies import Dependencies
from twitter.pants.tasks.nailgun_task import NailgunTask

# Well known metadata file required to register scalac plugins with nsc.
_PLUGIN_INFO_FILE = 'scalac-plugin.xml'

class ScalaCompile(NailgunTask):
  @staticmethod
  def _has_scala_sources(target):
    return isinstance(target, ScalaLibrary) or isinstance(target, ScalaTests)

  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    NailgunTask.setup_parser(option_group, args, mkflag)

    option_group.add_option(mkflag("warnings"), mkflag("warnings", negate=True),
                            dest="scala_compile_warnings", default=True,
                            action="callback", callback=mkflag.set_bool,
                            help="[%default] Compile scala code with all configured warnings "
                                 "enabled.")

    option_group.add_option(mkflag("flatten"), mkflag("flatten", negate=True),
                            dest="scala_compile_flatten",
                            action="callback", callback=mkflag.set_bool,
                            help="[%default] Compile scala code for all dependencies in a "
                                 "single compilation.")

    option_group.add_option(mkflag("color"), mkflag("color", negate=True),
                            dest="scala_compile_color",
                            action="callback", callback=mkflag.set_bool,
                            help="[True] Enable color in logging.")

  def __init__(self, context, workdir=None):
    NailgunTask.__init__(self, context, workdir=context.config.get('scala-compile', 'nailgun_dir'))

    self._flatten = \
      context.options.scala_compile_flatten if context.options.scala_compile_flatten is not None else \
      context.config.getbool('scala-compile', 'default_to_flatten')

    # We use the scala_compile_color flag if it is explicitly set on the command line.
    self._color = \
      context.options.scala_compile_color if context.options.scala_compile_color is not None else \
      context.config.getbool('scala-compile', 'color', default=True)

    self._compile_profile = context.config.get('scala-compile', 'compile-profile')  # The target scala version.
    self._zinc_profile = context.config.get('scala-compile', 'zinc-profile')
    self._depemitter_profile = context.config.get('scala-compile', 'dependencies-plugin-profile')

    # All scala targets implicitly depend on the selected scala runtime.
    scaladeps = []
    for spec in context.config.getlist('scala-compile', 'scaladeps'):
      scaladeps.extend(context.resolve(spec))
    for target in context.targets(is_scala):
      target.update_dependencies(scaladeps)

    self._workdir = context.config.get('scala-compile', 'workdir') if workdir is None else workdir
    self._incremental_classes_dir = os.path.join(self._workdir, 'incremental.classes')
    self._flat_classes_dir = os.path.join(self._workdir, 'classes')
    self._analysis_cache_dir = os.path.join(self._workdir, 'analysis_cache')
    self._resources_dir = os.path.join(self._workdir, 'resources')

    self._main = context.config.get('scala-compile', 'main')

    self._args = context.config.getlist('scala-compile', 'args')
    self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')
    if context.options.scala_compile_warnings:
      self._args.extend(context.config.getlist('scala-compile', 'warning_args'))
    else:
      self._args.extend(context.config.getlist('scala-compile', 'no_warning_args'))

    self._confs = context.config.getlist('scala-compile', 'confs')
    self._depfile_dir = os.path.join(self._workdir, 'depfiles')

  def product_type(self):
    return 'classes'

  def invalidate_for(self):
    return self._flatten

  def execute(self, targets):
    scala_targets = filter(ScalaCompile._has_scala_sources, targets)
    if scala_targets:
      safe_mkdir(self._depfile_dir)
      safe_mkdir(self._analysis_cache_dir)

      # Map from output directory to { analysis_cache_dir, [ analysis_cache_file ]}
      upstream_analysis_caches = self.context.products.get('upstream')

      with self.context.state('classpath', []) as cp:
        for conf in self._confs:
          cp.insert(0, (conf, self._resources_dir))

      with self.invalidated(scala_targets, invalidate_dependants=True) as invalidated:
        if self._flatten:
          # We must defer invalidation to zinc. If we exclude files from a repeat build, zinc will assume
          # the files were deleted and will nuke the corresponding class files. So we build all_targets
          # in one pass and let zinc figure it out.
          self.execute_single_compilation(invalidated.combined_all_versioned_targets(), cp, upstream_analysis_caches)
        else:
          # We must pass all targets,even valid ones, to execute_single_compilation(), so it can
          # track the deps and the upstream analysis map correctly.
          for vt in invalidated.all_versioned_targets():
            self.execute_single_compilation(vt, cp, upstream_analysis_caches)

  def execute_single_compilation(self, versioned_target_set, cp, upstream_analysis_caches):
    """Execute a single compilation, updating upstream_analysis_caches if needed."""
    if self._flatten:
      compilation_id = 'flat'
      output_dir = self._flat_classes_dir
    else:
      compilation_id = Target.maybe_readable_identify(versioned_target_set.targets)
      # Each compilation must output to its own directory, so zinc can then associate those with the appropriate
      # analysis caches of previous compilations. We then copy the results out to the real output dir.
      output_dir = os.path.join(self._incremental_classes_dir, compilation_id)

    depfile = os.path.join(self._depfile_dir, compilation_id) + '.dependencies'
    analysis_cache = os.path.join(self._analysis_cache_dir, compilation_id) + '.analysis_cache'

    safe_mkdir(output_dir)

    if not versioned_target_set.valid:
      with self.check_artifact_cache(versioned_target_set,
                                     build_artifacts=[output_dir, depfile, analysis_cache],
                                     artifact_root=self._workdir) as needs_building:
        if needs_building:
          self.context.log.info('Compiling targets %s' % versioned_target_set.targets)
          sources_by_target = self.calculate_sources(versioned_target_set.targets)
          if sources_by_target:
            sources = reduce(lambda all, sources: all.union(sources), sources_by_target.values())
            if not sources:
              touch(depfile)  # Create an empty depfile, since downstream code may assume that one exists.
              self.context.log.warn('Skipping scala compile for targets with no sources:\n  %s' %
                                    '\n  '.join(str(t) for t in sources_by_target.keys()))
            else:
              classpath = [jar for conf, jar in cp if conf in self._confs]
              result = self.compile(classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile)
              if result != 0:
                raise TaskError('%s returned %d' % (self._main, result))

    # Note that the following post-processing steps must happen even for valid targets.

    # Read in the deps created either just now or by a previous compiler run on these targets.
    if self.context.products.isrequired('classes'):
      self.context.log.debug('Reading dependencies from ' + depfile)
      deps = Dependencies(output_dir)
      deps.load(depfile)

      genmap = self.context.products.get('classes')

      for target, classes_by_source in deps.findclasses(versioned_target_set.targets).items():
        for source, classes in classes_by_source.items():
          genmap.add(source, output_dir, classes)
          genmap.add(target, output_dir, classes)

      # TODO(John Sirois): Map target.resources in the same way
      # Create and Map scala plugin info files to the owning targets.
      for target in versioned_target_set.targets:
        if is_scalac_plugin(target) and target.classname:
          basedir = self.write_plugin_info(target)
          genmap.add(target, basedir, [_PLUGIN_INFO_FILE])

    # Update the upstream analysis map.
    analysis_cache_parts = os.path.split(analysis_cache)
    if not upstream_analysis_caches.has(output_dir):
      # A previous chunk might have already updated this. It is certainly possible for a later chunk to
      # independently depend on some target that a previous chunk already built.
      upstream_analysis_caches.add(output_dir, analysis_cache_parts[0], [ analysis_cache_parts[1] ])

    # Update the classpath.
    with self.context.state('classpath', []) as cp:
      for conf in self._confs:
        cp.insert(0, (conf, output_dir))

  def calculate_sources(self, targets):
    sources = defaultdict(set)
    def collect_sources(target):
      src = (os.path.join(target.target_base, source)
             for source in target.sources if source.endswith('.scala'))
      if src:
        sources[target].update(src)

        if (isinstance(target, ScalaLibrary) or isinstance(target, ScalaTests)) and (
            target.java_sources):
          sources[target].update(resolve_target_sources(target.java_sources, '.java'))

    for target in targets:
      collect_sources(target)
    return sources

  def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile):
    compiler_classpath = nailgun_profile_classpath(self, self._compile_profile)
    compiler_args = []

    # TODO(John Sirois): separate compiler profile from runtime profile
    compiler_args.extend([
      # Support for outputting a dependencies file of source -> class
      '-Xplugin:%s' % self.get_depemitter_plugin(),
      '-P:depemitter:file:%s' % depfile
    ])
    compiler_args.extend(self._args)

    # To pass options to scalac simply prefix with -S.
    args = ['-S' + x for x in compiler_args]

    def analysis_cache_full_path(analysis_cache_product):
      # We expect the argument to be { analysis_cache_dir, [ analysis_cache_file ]}.
      if len(analysis_cache_product) != 1:
        raise TaskError('There can only be one analysis cache file per output directory')
      analysis_cache_dir, analysis_cache_files = analysis_cache_product.iteritems().next()
      if len(analysis_cache_files) != 1:
        raise TaskError('There can only be one analysis cache file per output directory')
      return os.path.join(analysis_cache_dir, analysis_cache_files[0])

    # Strings of <output dir>:<full path to analysis cache file for the classes in that dir>.
    analysis_map = \
      OrderedDict([ (k, analysis_cache_full_path(v)) for k, v in upstream_analysis_caches.itermappings() ])

    if len(analysis_map) > 0:
      args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in analysis_map.items()]) ])

    zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile)
    zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath)
    for (name, jarpath) in zinc_jars.items():  # The zinc jar names are also the flag names.
      args.extend(['-%s' % name, jarpath])

    args.extend([
      '-analysis-cache', analysis_cache,
      '-log-level', self.context.options.log_level or 'info',
      '-classpath', ':'.join(zinc_classpath + classpath),
      '-d', output_dir
    ])

    if not self._color:
      args.append('-no-color')

    args.extend(sources)

    self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args)))
    return self.runjava(self._main, classpath=zinc_classpath, args=args, jvmargs=self._jvm_args)

  def get_depemitter_plugin(self):
    depemitter_classpath = nailgun_profile_classpath(self, self._depemitter_profile)
    depemitter_jar = depemitter_classpath.pop()
    if depemitter_classpath:
      raise TaskError('Expected only 1 jar for the depemitter plugin, '
                      'found these extra: ' % depemitter_classpath)
    return depemitter_jar

  def write_plugin_info(self, target):
    basedir = os.path.join(self._resources_dir, target.id)
    with safe_open(os.path.join(basedir, _PLUGIN_INFO_FILE), 'w') as f:
      f.write(textwrap.dedent('''
        <plugin>
          <name>%s</name>
          <classname>%s</classname>
        </plugin>
      ''' % (target.plugin, target.classname)).strip())
    return basedir

  # These are the names of the various jars zinc needs. They are, conveniently and non-coincidentally,
  # the names of the flags used to pass the jar locations to zinc.
  compiler_jar_names = [ 'scala-library', 'scala-compiler' ]  # Compiler version.
  zinc_jar_names = [ 'compiler-interface', 'sbt-interface' ]  # Other jars zinc needs to be pointed to.

  @staticmethod
  def identify_zinc_jars(compiler_classpath, zinc_classpath):
    """Find the named jars in the compiler and zinc classpaths.

    TODO: When profiles migrate to regular pants jar() deps instead of ivy.xml files we can make these
          mappings explicit instead of deriving them by jar name heuristics.
    """
    ret = OrderedDict()
    ret.update(ScalaCompile.identify_jars(ScalaCompile.compiler_jar_names, compiler_classpath))
    ret.update(ScalaCompile.identify_jars(ScalaCompile.zinc_jar_names, zinc_classpath))
    return ret

  @staticmethod
  def identify_jars(names, jars):
    jars_by_name = {}
    jars_and_filenames = [(x, os.path.basename(x)) for x in jars]

    for name in names:
      jar_for_name = None
      for jar, filename in jars_and_filenames:
        if filename.startswith(name):
          jar_for_name = jar
          break
      if jar_for_name is None:
        raise TaskError('Couldn\'t find jar named %s' % name)
      else:
        jars_by_name[name] = jar_for_name
    return jars_by_name
<commit_msg>Let zinc emit the scalac depfile instead of the plugin.<commit_after># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

__author__ = 'John Sirois'

import os
import textwrap

from collections import defaultdict

from twitter.common.collections import OrderedDict
from twitter.common.dirutil import safe_mkdir, safe_open, touch

from twitter.pants import is_scala, is_scalac_plugin
from twitter.pants.base.target import Target
from twitter.pants.targets.scala_library import ScalaLibrary
from twitter.pants.targets.scala_tests import ScalaTests
from twitter.pants.targets import resolve_target_sources
from twitter.pants.tasks import TaskError
from twitter.pants.tasks.binary_utils import nailgun_profile_classpath
from twitter.pants.tasks.jvm_compiler_dependencies import Dependencies
from twitter.pants.tasks.nailgun_task import NailgunTask

# Well known metadata file required to register scalac plugins with nsc.
_PLUGIN_INFO_FILE = 'scalac-plugin.xml'

class ScalaCompile(NailgunTask):
  @staticmethod
  def _has_scala_sources(target):
    return isinstance(target, ScalaLibrary) or isinstance(target, ScalaTests)

  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    NailgunTask.setup_parser(option_group, args, mkflag)

    option_group.add_option(mkflag("warnings"), mkflag("warnings", negate=True),
                            dest="scala_compile_warnings", default=True,
                            action="callback", callback=mkflag.set_bool,
                            help="[%default] Compile scala code with all configured warnings "
                                 "enabled.")

    option_group.add_option(mkflag("flatten"), mkflag("flatten", negate=True),
                            dest="scala_compile_flatten",
                            action="callback", callback=mkflag.set_bool,
                            help="[%default] Compile scala code for all dependencies in a "
                                 "single compilation.")

    option_group.add_option(mkflag("color"), mkflag("color", negate=True),
                            dest="scala_compile_color",
                            action="callback", callback=mkflag.set_bool,
                            help="[True] Enable color in logging.")

  def __init__(self, context, workdir=None):
    NailgunTask.__init__(self, context, workdir=context.config.get('scala-compile', 'nailgun_dir'))

    self._flatten = \
      context.options.scala_compile_flatten if context.options.scala_compile_flatten is not None else \
      context.config.getbool('scala-compile', 'default_to_flatten')

    # We use the scala_compile_color flag if it is explicitly set on the command line.
    self._color = \
      context.options.scala_compile_color if context.options.scala_compile_color is not None else \
      context.config.getbool('scala-compile', 'color', default=True)

    self._compile_profile = context.config.get('scala-compile', 'compile-profile')  # The target scala version.
    self._zinc_profile = context.config.get('scala-compile', 'zinc-profile')

    # All scala targets implicitly depend on the selected scala runtime.
    scaladeps = []
    for spec in context.config.getlist('scala-compile', 'scaladeps'):
      scaladeps.extend(context.resolve(spec))
    for target in context.targets(is_scala):
      target.update_dependencies(scaladeps)

    self._workdir = context.config.get('scala-compile', 'workdir') if workdir is None else workdir
    self._incremental_classes_dir = os.path.join(self._workdir, 'incremental.classes')
    self._flat_classes_dir = os.path.join(self._workdir, 'classes')
    self._analysis_cache_dir = os.path.join(self._workdir, 'analysis_cache')
    self._resources_dir = os.path.join(self._workdir, 'resources')

    self._main = context.config.get('scala-compile', 'main')

    self._args = context.config.getlist('scala-compile', 'args')
    self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')
    if context.options.scala_compile_warnings:
      self._args.extend(context.config.getlist('scala-compile', 'warning_args'))
    else:
      self._args.extend(context.config.getlist('scala-compile', 'no_warning_args'))

    self._confs = context.config.getlist('scala-compile', 'confs')
    self._depfile_dir = os.path.join(self._workdir, 'depfiles')

  def product_type(self):
    return 'classes'

  def invalidate_for(self):
    return self._flatten

  def execute(self, targets):
    scala_targets = filter(ScalaCompile._has_scala_sources, targets)
    if scala_targets:
      safe_mkdir(self._depfile_dir)
      safe_mkdir(self._analysis_cache_dir)

      # Map from output directory to { analysis_cache_dir, [ analysis_cache_file ]}
      upstream_analysis_caches = self.context.products.get('upstream')

      with self.context.state('classpath', []) as cp:
        for conf in self._confs:
          cp.insert(0, (conf, self._resources_dir))

      with self.invalidated(scala_targets, invalidate_dependants=True) as invalidated:
        if self._flatten:
          # We must defer invalidation to zinc. If we exclude files from a repeat build, zinc will assume
          # the files were deleted and will nuke the corresponding class files. So we build all_targets
          # in one pass and let zinc figure it out.
          self.execute_single_compilation(invalidated.combined_all_versioned_targets(), cp, upstream_analysis_caches)
        else:
          # We must pass all targets,even valid ones, to execute_single_compilation(), so it can
          # track the deps and the upstream analysis map correctly.
          for vt in invalidated.all_versioned_targets():
            self.execute_single_compilation(vt, cp, upstream_analysis_caches)

  def execute_single_compilation(self, versioned_target_set, cp, upstream_analysis_caches):
    """Execute a single compilation, updating upstream_analysis_caches if needed."""
    if self._flatten:
      compilation_id = 'flat'
      output_dir = self._flat_classes_dir
    else:
      compilation_id = Target.maybe_readable_identify(versioned_target_set.targets)
      # Each compilation must output to its own directory, so zinc can then associate those with the appropriate
      # analysis caches of previous compilations. We then copy the results out to the real output dir.
      output_dir = os.path.join(self._incremental_classes_dir, compilation_id)

    depfile = os.path.join(self._depfile_dir, compilation_id) + '.dependencies'
    analysis_cache = os.path.join(self._analysis_cache_dir, compilation_id) + '.analysis_cache'

    safe_mkdir(output_dir)

    if not versioned_target_set.valid:
      with self.check_artifact_cache(versioned_target_set,
                                     build_artifacts=[output_dir, depfile, analysis_cache],
                                     artifact_root=self._workdir) as needs_building:
        if needs_building:
          self.context.log.info('Compiling targets %s' % versioned_target_set.targets)
          sources_by_target = self.calculate_sources(versioned_target_set.targets)
          if sources_by_target:
            sources = reduce(lambda all, sources: all.union(sources), sources_by_target.values())
            if not sources:
              touch(depfile)  # Create an empty depfile, since downstream code may assume that one exists.
              self.context.log.warn('Skipping scala compile for targets with no sources:\n  %s' %
                                    '\n  '.join(str(t) for t in sources_by_target.keys()))
            else:
              classpath = [jar for conf, jar in cp if conf in self._confs]
              result = self.compile(classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile)
              if result != 0:
                raise TaskError('%s returned %d' % (self._main, result))

    # Note that the following post-processing steps must happen even for valid targets.

    # Read in the deps created either just now or by a previous compiler run on these targets.
    if self.context.products.isrequired('classes'):
      self.context.log.debug('Reading dependencies from ' + depfile)
      deps = Dependencies(output_dir)
      deps.load(depfile)

      genmap = self.context.products.get('classes')

      for target, classes_by_source in deps.findclasses(versioned_target_set.targets).items():
        for source, classes in classes_by_source.items():
          genmap.add(source, output_dir, classes)
          genmap.add(target, output_dir, classes)

      # TODO(John Sirois): Map target.resources in the same way
      # Create and Map scala plugin info files to the owning targets.
      for target in versioned_target_set.targets:
        if is_scalac_plugin(target) and target.classname:
          basedir = self.write_plugin_info(target)
          genmap.add(target, basedir, [_PLUGIN_INFO_FILE])

    # Update the upstream analysis map.
    analysis_cache_parts = os.path.split(analysis_cache)
    if not upstream_analysis_caches.has(output_dir):
      # A previous chunk might have already updated this. It is certainly possible for a later chunk to
      # independently depend on some target that a previous chunk already built.
      upstream_analysis_caches.add(output_dir, analysis_cache_parts[0], [ analysis_cache_parts[1] ])

    # Update the classpath.
    with self.context.state('classpath', []) as cp:
      for conf in self._confs:
        cp.insert(0, (conf, output_dir))

  def calculate_sources(self, targets):
    sources = defaultdict(set)
    def collect_sources(target):
      src = (os.path.join(target.target_base, source)
             for source in target.sources if source.endswith('.scala'))
      if src:
        sources[target].update(src)

        if (isinstance(target, ScalaLibrary) or isinstance(target, ScalaTests)) and (
            target.java_sources):
          sources[target].update(resolve_target_sources(target.java_sources, '.java'))

    for target in targets:
      collect_sources(target)
    return sources

  def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile):
    compiler_classpath = nailgun_profile_classpath(self, self._compile_profile)

    # To pass options to scalac simply prefix with -S.
    args = ['-S' + x for x in self._args]

    def analysis_cache_full_path(analysis_cache_product):
      # We expect the argument to be { analysis_cache_dir, [ analysis_cache_file ]}.
      if len(analysis_cache_product) != 1:
        raise TaskError('There can only be one analysis cache file per output directory')
      analysis_cache_dir, analysis_cache_files = analysis_cache_product.iteritems().next()
      if len(analysis_cache_files) != 1:
        raise TaskError('There can only be one analysis cache file per output directory')
      return os.path.join(analysis_cache_dir, analysis_cache_files[0])

    # Strings of <output dir>:<full path to analysis cache file for the classes in that dir>.
    analysis_map = \
      OrderedDict([ (k, analysis_cache_full_path(v)) for k, v in upstream_analysis_caches.itermappings() ])

    if len(analysis_map) > 0:
      args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in analysis_map.items()]) ])

    zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile)
    zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath)
    for (name, jarpath) in zinc_jars.items():  # The zinc jar names are also the flag names.
      args.extend(['-%s' % name, jarpath])

    args.extend([
      '-analysis-cache', analysis_cache,
      '-log-level', self.context.options.log_level or 'info',
      '-classpath', ':'.join(zinc_classpath + classpath),
      '-output-products', depfile,
      '-d', output_dir
    ])

    if not self._color:
      args.append('-no-color')

    args.extend(sources)

    self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args)))
    return self.runjava(self._main, classpath=zinc_classpath, args=args, jvmargs=self._jvm_args)

  def write_plugin_info(self, target):
    basedir = os.path.join(self._resources_dir, target.id)
    with safe_open(os.path.join(basedir, _PLUGIN_INFO_FILE), 'w') as f:
      f.write(textwrap.dedent('''
        <plugin>
          <name>%s</name>
          <classname>%s</classname>
        </plugin>
      ''' % (target.plugin, target.classname)).strip())
    return basedir

  # These are the names of the various jars zinc needs. They are, conveniently and non-coincidentally,
  # the names of the flags used to pass the jar locations to zinc.
  compiler_jar_names = [ 'scala-library', 'scala-compiler' ]  # Compiler version.
  zinc_jar_names = [ 'compiler-interface', 'sbt-interface' ]  # Other jars zinc needs to be pointed to.

  @staticmethod
  def identify_zinc_jars(compiler_classpath, zinc_classpath):
    """Find the named jars in the compiler and zinc classpaths.

    TODO: When profiles migrate to regular pants jar() deps instead of ivy.xml files we can make these
          mappings explicit instead of deriving them by jar name heuristics.
    """
    ret = OrderedDict()
    ret.update(ScalaCompile.identify_jars(ScalaCompile.compiler_jar_names, compiler_classpath))
    ret.update(ScalaCompile.identify_jars(ScalaCompile.zinc_jar_names, zinc_classpath))
    return ret

  @staticmethod
  def identify_jars(names, jars):
    jars_by_name = {}
    jars_and_filenames = [(x, os.path.basename(x)) for x in jars]

    for name in names:
      jar_for_name = None
      for jar, filename in jars_and_filenames:
        if filename.startswith(name):
          jar_for_name = jar
          break
      if jar_for_name is None:
        raise TaskError('Couldn\'t find jar named %s' % name)
      else:
        jars_by_name[name] = jar_for_name
    return jars_by_name
<|endoftext|>