<commit_before>"""Extend introspect.py for Java based Jython classes."""

from org.python.core import PyReflectedFunction
from java.lang import Class, Object
from java.lang.reflect import Modifier
from introspect import *
from sets import Set
import string
import re
import types

__author__ = "Don Coleman <dcoleman@chariotsolutions.com>"

_re_import_package = re.compile('import\s+(.+)\.') # import package
# TODO need to check for a trailing '.'  example: "from java import lang." don't autocomplete on trailing '.'
_re_from_package_import = re.compile('from\s+(\w+(?:\.\w+)*)\.?(?:\s*import\s*)?') # from package import class

def completePackageName(target):
    """ Get a package object given the full name."""
    targetComponents = target.split('.')
    base = targetComponents[0]
    baseModule = __import__(base, globals(), locals())
    module = baseModule

    for component in targetComponents[1:]:
        module = getattr(module, component)

    list = dir(module)
    list.remove('__name__')
    list.append('*')
    return list

def getPackageName(command):

    match = _re_import_package.match(command)
    if not match:
        #try the other re
        match = _re_from_package_import.match(command)

    return match.groups()[0]

def getAutoCompleteList(command='', locals=None, includeMagic=1, includeSingle=1, includeDouble=1):
    """
    Return list of auto-completion options for command.
    The list of options will be based on the locals namespace.
    """

    # Temp KLUDGE here rather than in console.py
    command += "."

    attributes = []
    # Get the proper chunk of code from the command.
    root = getRoot(command, terminator='.')

    # check to see if the user is attempting to import a package
    # this may need to adjust this so that it doesn't pollute the namespace
    if command.startswith('import ') or command.startswith('from '):
        target = getPackageName(command)
        return completePackageName(target)

    try:
        if locals is not None:
            object = eval(root, locals)
        else:
            object = eval(root)
    except:
        return attributes

    if ispython(object):  # use existing code
        attributes = getAttributeNames(object, includeMagic, includeSingle, includeDouble)
    else:
        if inspect.isclass(object):
            attributes = staticMethodNames(object)
            attributes.extend(staticFieldNames(object))
        else:
            attributes = list(instanceMethodNames(object.__class__))

    return attributes

def instanceMethodNames(clazz):
    """return a Set of instance method name for a Class"""

    method_names = Set()
    declared_methods = Class.getDeclaredMethods(clazz)
    for method in declared_methods:
        modifiers = method.getModifiers()
        if not Modifier.isStatic(modifiers) and Modifier.isPublic(modifiers):
            name = method.name
            method_names.add(name)
            if name.startswith("get") and len(name) > 3 and len(method.getParameterTypes()) == 0:
                property_name = name[3].lower() + name[4:]
                method_names.add(property_name)

    for eachBase in clazz.__bases__:
        method_names = method_names | instanceMethodNames(eachBase)

    return method_names

def staticMethodNames(clazz):
    """return a list of static method name for a class"""

    static_methods = {}
    declared_methods = Class.getDeclaredMethods(clazz)
    for method in declared_methods:
        if Modifier.isStatic(method.getModifiers()) and Modifier.isPublic(method.getModifiers()):
            static_methods[method.name] = method
    methods = static_methods.keys()

    for eachBase in clazz.__bases__:
        # with Jython 2.5 type is a base of Object, which puts asName in the list
        # will be a problem for real Java objects that extend Python objects
        if not ispython(eachBase):
            methods.extend(staticMethodNames(eachBase))

    return methods

def staticFieldNames(clazz):
    """return a list of static field names for class"""

    static_fields = {}
    declared_fields = Class.getDeclaredFields(clazz)
    for field in declared_fields:
        if Modifier.isStatic(field.getModifiers()) and Modifier.isPublic(field.getModifiers()):
            static_fields[field.name] = field
    fields = static_fields.keys()

    for eachBase in clazz.__bases__:
        if not ispython(eachBase):
            fields.extend(staticFieldNames(eachBase))

    return fields

def getCallTipJava(command='', locals=None):
    """For a command, return a tuple of object name, argspec, tip text.

    The call tip information will be based on the locals namespace."""

    calltip = ('', '', '')  # object name, argspec, tip text.

    # Get the proper chunk of code from the command.
    root = getRoot(command, terminator='(')

    try:
        if locals is not None:
            object = eval(root, locals)
        else:
            object = eval(root)
    except:
        return calltip

    if ispython(object):
        # Patrick's code handles python code
        # TODO fix in future because getCallTip runs eval() again
        return getCallTip(command, locals)

    name = ''
    try:
        name = object.__name__
    except AttributeError:
        pass

    tipList = []
    argspec = '' # not using argspec for Java

    if inspect.isclass(object):
        # get the constructor(s)
        # TODO consider getting modifiers since jython can access private methods
        constructors = object.getConstructors()
        for constructor in constructors:
            paramList = []
            paramTypes = constructor.getParameterTypes()
            # paramTypes is an array of classes, we need Strings
            # TODO consider list comprehension
            for param in paramTypes:
                paramList.append(param.__name__)
            paramString = string.join(paramList,', ')
            tip = "%s(%s)" % (constructor.name, paramString)
            tipList.append(tip)

    elif inspect.ismethod(object) or isinstance(object, PyReflectedFunction):
        method = object
        try:
            object = method.im_class
        except: # PyReflectedFunction
            object = method.argslist[0].declaringClass

        # java allows overloading so we may have more than one method
        methodArray = object.getMethods()

        for eachMethod in methodArray:
            if eachMethod.name == method.__name__:
                paramList = []
                for eachParam in eachMethod.parameterTypes:
                    paramList.append(eachParam.__name__)

                paramString = string.join(paramList,', ')

                # create a python style string a la PyCrust
                # we're showing the parameter type rather than the parameter name, since that's all I can get
                # we need to show multiple methods for overloading
                # do we want to show the method visibility?  how about exceptions?
                # note: name, return type and exceptions same for EVERY overload method

                tip = "%s(%s) -> %s" % (eachMethod.name, paramString, eachMethod.returnType.name)
                tipList.append(tip)

    tip_text = beautify(string.join(tipList,"\n"))
    calltip = (name, argspec, tip_text)
    return calltip

def beautify(tip_text):
    "Make the call tip text prettier"
    tip_text = tip_text.replace("java.lang.", "")
    if "[" in tip_text:
        tip_text = tip_text.replace("[B", "byte[]")
        tip_text = tip_text.replace("[S", "short[]")
        tip_text = tip_text.replace("[I", "int[]")
        tip_text = tip_text.replace("[J", "long[]")
        tip_text = tip_text.replace("[F", "float[]")
        tip_text = tip_text.replace("[D", "double[]")
        tip_text = tip_text.replace("[Z", "boolean[]")
        tip_text = tip_text.replace("[C", "char[]")
    return tip_text

def ispython21(object):
    """
    Figure out if this is Python code or Java Code

    """
    pyclass = 0
    pycode = 0
    pyinstance = 0

    if inspect.isclass(object):
        try:
            object.__doc__
            pyclass = 1
        except AttributeError:
            pyclass = 0

    elif inspect.ismethod(object):
        try:
            object.__dict__
            pycode = 1
        except AttributeError:
            pycode = 0
    else: # I guess an instance of an object falls here
        try:
            object.__dict__
            pyinstance = 1
        except AttributeError:
            pyinstance = 0

    #    print "object", object, "pyclass", pyclass, "pycode", pycode, "returning", pyclass | pycode

    return pyclass | pycode | pyinstance

def ispython22(object):
    """
    Return true if object is Python code.
    """

    object_type = type(object)

    if object_type.__name__.startswith("java") or isinstance(object, PyReflectedFunction):
        python = False

    elif object_type is types.MethodType:
        # both Java and Python methods return MethodType
        try:
            object.__dict__
            python = True
        except AttributeError:
            python = False
    else:
        # assume everything else is python
        python = True

    return python

def ispython25(object):
    """
    Return true if object is Python code.
    """

    if isinstance(object, Class):
        python = False
    elif isinstance(object, Object):
        python = False
    elif isinstance(object, PyReflectedFunction):
        python = False
    elif type(object) == types.MethodType and not ispython(object.im_class):
        python = False
    else:
        # assume everything else is python
        python = True

    return python

# Dynamically assign the version of ispython
# To deal with differences between Jython 2.1, 2.2 and 2.5
if sys.version == '2.1':
    ispython = ispython21
elif sys.version.startswith('2.5'):
    ispython = ispython25
else:
    ispython = ispython22

def debug(name, value=None):
    if value == None:
        print >> sys.stderr, name
    else:
        print >> sys.stderr, "%s = %s" % (name, value)
<commit_msg>remove python classes from java base for jython 2.5 support<commit_after>"""Extend introspect.py for Java based Jython classes."""

from org.python.core import PyReflectedFunction
from java.lang import Class, Object
from java.lang.reflect import Modifier
from introspect import *
from sets import Set
import string
import re
import types

__author__ = "Don Coleman <dcoleman@chariotsolutions.com>"

_re_import_package = re.compile('import\s+(.+)\.') # import package
# TODO need to check for a trailing '.'  example: "from java import lang." don't autocomplete on trailing '.'
_re_from_package_import = re.compile('from\s+(\w+(?:\.\w+)*)\.?(?:\s*import\s*)?') # from package import class

def completePackageName(target):
    """ Get a package object given the full name."""
    targetComponents = target.split('.')
    base = targetComponents[0]
    baseModule = __import__(base, globals(), locals())
    module = baseModule

    for component in targetComponents[1:]:
        module = getattr(module, component)

    list = dir(module)
    list.remove('__name__')
    list.append('*')
    return list

def getPackageName(command):

    match = _re_import_package.match(command)
    if not match:
        #try the other re
        match = _re_from_package_import.match(command)

    return match.groups()[0]

def getAutoCompleteList(command='', locals=None, includeMagic=1, includeSingle=1, includeDouble=1):
    """
    Return list of auto-completion options for command.
    The list of options will be based on the locals namespace.
    """

    # Temp KLUDGE here rather than in console.py
    command += "."

    attributes = []
    # Get the proper chunk of code from the command.
    root = getRoot(command, terminator='.')

    # check to see if the user is attempting to import a package
    # this may need to adjust this so that it doesn't pollute the namespace
    if command.startswith('import ') or command.startswith('from '):
        target = getPackageName(command)
        return completePackageName(target)

    try:
        if locals is not None:
            object = eval(root, locals)
        else:
            object = eval(root)
    except:
        return attributes

    if ispython(object):  # use existing code
        attributes = getAttributeNames(object, includeMagic, includeSingle, includeDouble)
    else:
        if inspect.isclass(object):
            attributes = staticMethodNames(object)
            attributes.extend(staticFieldNames(object))
        else:
            attributes = list(instanceMethodNames(object.__class__))

    return attributes

def instanceMethodNames(clazz):
    """return a Set of instance method name for a Class"""

    method_names = Set()
    declared_methods = Class.getDeclaredMethods(clazz)
    for method in declared_methods:
        modifiers = method.getModifiers()
        if not Modifier.isStatic(modifiers) and Modifier.isPublic(modifiers):
            name = method.name
            method_names.add(name)
            if name.startswith("get") and len(name) > 3 and len(method.getParameterTypes()) == 0:
                property_name = name[3].lower() + name[4:]
                method_names.add(property_name)

    for eachBase in clazz.__bases__:
        if not ispython(eachBase):
            method_names = method_names | instanceMethodNames(eachBase)

    return method_names

def staticMethodNames(clazz):
    """return a list of static method name for a class"""

    static_methods = {}
    declared_methods = Class.getDeclaredMethods(clazz)
    for method in declared_methods:
        if Modifier.isStatic(method.getModifiers()) and Modifier.isPublic(method.getModifiers()):
            static_methods[method.name] = method
    methods = static_methods.keys()

    for eachBase in clazz.__bases__:
        # with Jython 2.5 type is a base of Object, which puts asName in the list
        # will be a problem for real Java objects that extend Python objects
        # see similar "fixes" in instanceMethodNames and staticFieldNames
        if not ispython(eachBase):
            methods.extend(staticMethodNames(eachBase))

    return methods

def staticFieldNames(clazz):
    """return a list of static field names for class"""

    static_fields = {}
    declared_fields = Class.getDeclaredFields(clazz)
    for field in declared_fields:
        if Modifier.isStatic(field.getModifiers()) and Modifier.isPublic(field.getModifiers()):
            static_fields[field.name] = field
    fields = static_fields.keys()

    for eachBase in clazz.__bases__:
        if not ispython(eachBase):
            fields.extend(staticFieldNames(eachBase))

    return fields

def getCallTipJava(command='', locals=None):
    """For a command, return a tuple of object name, argspec, tip text.

    The call tip information will be based on the locals namespace."""

    calltip = ('', '', '')  # object name, argspec, tip text.

    # Get the proper chunk of code from the command.
    root = getRoot(command, terminator='(')

    try:
        if locals is not None:
            object = eval(root, locals)
        else:
            object = eval(root)
    except:
        return calltip

    if ispython(object):
        # Patrick's code handles python code
        # TODO fix in future because getCallTip runs eval() again
        return getCallTip(command, locals)

    name = ''
    try:
        name = object.__name__
    except AttributeError:
        pass

    tipList = []
    argspec = '' # not using argspec for Java

    if inspect.isclass(object):
        # get the constructor(s)
        # TODO consider getting modifiers since jython can access private methods
        constructors = object.getConstructors()
        for constructor in constructors:
            paramList = []
            paramTypes = constructor.getParameterTypes()
            # paramTypes is an array of classes, we need Strings
            # TODO consider list comprehension
            for param in paramTypes:
                paramList.append(param.__name__)
            paramString = string.join(paramList,', ')
            tip = "%s(%s)" % (constructor.name, paramString)
            tipList.append(tip)

    elif inspect.ismethod(object) or isinstance(object, PyReflectedFunction):
        method = object
        try:
            object = method.im_class
        except: # PyReflectedFunction
            object = method.argslist[0].declaringClass

        # java allows overloading so we may have more than one method
        methodArray = object.getMethods()

        for eachMethod in methodArray:
            if eachMethod.name == method.__name__:
                paramList = []
                for eachParam in eachMethod.parameterTypes:
                    paramList.append(eachParam.__name__)

                paramString = string.join(paramList,', ')

                # create a python style string a la PyCrust
                # we're showing the parameter type rather than the parameter name, since that's all I can get
                # we need to show multiple methods for overloading
                # do we want to show the method visibility?  how about exceptions?
                # note: name, return type and exceptions same for EVERY overload method

                tip = "%s(%s) -> %s" % (eachMethod.name, paramString, eachMethod.returnType.name)
                tipList.append(tip)

    tip_text = beautify(string.join(tipList,"\n"))
    calltip = (name, argspec, tip_text)
    return calltip

def beautify(tip_text):
    "Make the call tip text prettier"
    tip_text = tip_text.replace("java.lang.", "")
    if "[" in tip_text:
        tip_text = tip_text.replace("[B", "byte[]")
        tip_text = tip_text.replace("[S", "short[]")
        tip_text = tip_text.replace("[I", "int[]")
        tip_text = tip_text.replace("[J", "long[]")
        tip_text = tip_text.replace("[F", "float[]")
        tip_text = tip_text.replace("[D", "double[]")
        tip_text = tip_text.replace("[Z", "boolean[]")
        tip_text = tip_text.replace("[C", "char[]")
    return tip_text

def ispython21(object):
    """
    Figure out if this is Python code or Java Code

    """
    pyclass = 0
    pycode = 0
    pyinstance = 0

    if inspect.isclass(object):
        try:
            object.__doc__
            pyclass = 1
        except AttributeError:
            pyclass = 0

    elif inspect.ismethod(object):
        try:
            object.__dict__
            pycode = 1
        except AttributeError:
            pycode = 0
    else: # I guess an instance of an object falls here
        try:
            object.__dict__
            pyinstance = 1
        except AttributeError:
            pyinstance = 0

    #    print "object", object, "pyclass", pyclass, "pycode", pycode, "returning", pyclass | pycode

    return pyclass | pycode | pyinstance

def ispython22(object):
    """
    Return true if object is Python code.
    """

    object_type = type(object)

    if object_type.__name__.startswith("java") or isinstance(object, PyReflectedFunction):
        python = False

    elif object_type is types.MethodType:
        # both Java and Python methods return MethodType
        try:
            object.__dict__
            python = True
        except AttributeError:
            python = False
    else:
        # assume everything else is python
        python = True

    return python

def ispython25(object):
    """
    Return true if object is Python code.
    """

    if isinstance(object, Class):
        python = False
    elif isinstance(object, Object):
        python = False
    elif isinstance(object, PyReflectedFunction):
        python = False
    elif type(object) == types.MethodType and not ispython(object.im_class):
        python = False
    else:
        # assume everything else is python
        python = True

    return python

# Dynamically assign the version of ispython
# To deal with differences between Jython 2.1, 2.2 and 2.5
if sys.version == '2.1':
    ispython = ispython21
elif sys.version.startswith('2.5'):
    ispython = ispython25
else:
    ispython = ispython22

def debug(name, value=None):
    if value == None:
        print >> sys.stderr, name
    else:
        print >> sys.stderr, "%s = %s" % (name, value)
<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Qt.ListModel import ListModel
from UM.Application import Application

from PyQt5.QtCore import Qt, pyqtSlot

class ProfilesModel(ListModel):
    IdRole = Qt.UserRole + 1
    NameRole = Qt.UserRole + 2
    ActiveRole = Qt.UserRole + 3

    def __init__(self):
        super().__init__()

        self.addRoleName(self.IdRole, "id")
        self.addRoleName(self.NameRole, "name")
        self.addRoleName(self.ActiveRole, "active")

        self._manager = Application.getInstance().getMachineManager()

        self._manager.profilesChanged.connect(self._onProfilesChanged)
        self._manager.activeProfileChanged.connect(self._onActiveProfileChanged)
        self._onProfilesChanged()

    def _onProfilesChanged(self):
        self.clear()
        profiles = self._manager.getProfiles()
        profiles.sort(key = lambda k: k.getName())
        for profile in profiles:
            self.appendItem({ "id": id(profile), "name": profile.getName(), "active": self._manager.getActiveProfile() == profile })

    def _onActiveMachineChanged(self):
        active_profile = self._manager.getActiveProfile()
        for index in range(len(self.items)):
            self.setProperty(index, "active", id(active_profile) == self.items[index]["id"])

<commit_msg>Add a property that adds a <Use Global> entry to ProfilesModel<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from UM.Qt.ListModel import ListModel
from UM.Application import Application

from PyQt5.QtCore import Qt, pyqtSlot, pyqtSignal, pyqtProperty

from UM.i18n import i18nCatalog
catalog = i18nCatalog("uranium")

class ProfilesModel(ListModel):
    IdRole = Qt.UserRole + 1
    NameRole = Qt.UserRole + 2
    ActiveRole = Qt.UserRole + 3

    def __init__(self, parent = None):
        super().__init__(parent)

        self._select_global = False

        self.addRoleName(self.IdRole, "id")
        self.addRoleName(self.NameRole, "name")
        self.addRoleName(self.ActiveRole, "active")

        self._manager = Application.getInstance().getMachineManager()

        self._manager.profilesChanged.connect(self._onProfilesChanged)
        self._manager.activeProfileChanged.connect(self._onActiveProfileChanged)
        self._onProfilesChanged()

    selectGlobalChanged = pyqtSignal()

    def setSelectGlobal(self, select):
        if select != self._select_global:
            self._select_global = select
            self._onProfilesChanged()
            self.selectGlobalChanged.emit()

    @pyqtProperty(bool, fset = setSelectGlobal, notify = selectGlobalChanged)
    def selectGlobal(self):
        return self._select_global

    def _onProfilesChanged(self):
        self.clear()

        if self._select_global:
            self.appendItem({ "id": 1, "name": catalog.i18nc("", "<Use Global Profile>"), "active": False })

        profiles = self._manager.getProfiles()
        profiles.sort(key = lambda k: k.getName())
        for profile in profiles:
            self.appendItem({ "id": id(profile), "name": profile.getName(), "active": self._manager.getActiveProfile() == profile })

    def _onActiveProfileChanged(self):
        active_profile = self._manager.getActiveProfile()
        for index in range(len(self.items)):
            self.setProperty(index, "active", id(active_profile) == self.items[index]["id"])

<|endoftext|>


<commit_before>""" Main module for the ConfigObj version of the EPAR task editor

$Id$
"""

import configobj, glob, os, tkMessageBox
import cfgpars, editpar, filedlg
from cfgpars import APP_NAME

# Starts a GUI session
def epar(theTask, parent=None, isChild=0, loadOnly=False):
    if loadOnly:
        return cfgpars.getObjectFromTaskArg(theTask)
    else:
        dlg = ConfigObjEparDialog(theTask, parent, isChild)
        if dlg.canceled():
            return None
        else:
            return dlg.getTaskParsObj()

# Main class
class ConfigObjEparDialog(editpar.EditParDialog):

    def __init__(self, theTask, parent=None, isChild=0,
                 title=APP_NAME, childList=None):

        # Init base - calls _setTaskParsObj(), sets self.taskName, etc
        editpar.EditParDialog.__init__(self, theTask, parent, isChild,
                                       title, childList,
                                       resourceDir=cfgpars.getAppDir())
        # We don't return from this until the GUI is closed

    def _overrideMasterSettings(self):
        """ Override so that we can run in a different mode. """
        cod = self._getGuiSettings()
        self._appName             = APP_NAME
        self._useSimpleAutoClose  = False # is a fundamental issue here
        self._showExtraHelpButton = False
        self._saveAndCloseOnExec  = cod.get('saveAndCloseOnExec', False)
        self._showHelpInBrowser   = cod.get('showHelpInBrowser', False)
        self._optFile             = APP_NAME.lower()+".optionDB"

    def _preMainLoop(self):
        """ Override so that we can do some things right before activating. """
        # Put the fname in the title. EditParDialog doesn't do this by default
        self.updateTitle(self._taskParsObj.filename)

    def _doActualSave(self, fname, comment):
        """ Override this so we can handle case of file not writable. """
        try:
            rv=self._taskParsObj.saveParList(filename=fname,comment=comment)
            return rv
        except IOError:
            # User does not have privs to write to this file. Get name of local
            # choice and try to use that.
            if not fname:
                fname = self._taskParsObj.filename
            mine = self._rcDir+os.sep+os.path.basename(fname)
            # Tell them the context is changing, and where we are saving
            msg = 'Installed config file for task "'+ \
                  self._taskParsObj.getName()+'" is not to be overwritten.'+ \
                  '  Values will be saved to: \n\n\t"'+mine+'".'
            tkMessageBox.showwarning(message=msg, title="Will not overwrite!")
            # Try saving their copy
            rv=self._taskParsObj.saveParList(filename=mine, comment=comment)
            # Treat like a save-as
            self._saveAsPostSave_Hook(mine)
            return rv

    def _saveAsPostSave_Hook(self, fnameToBeUsed_UNUSED):
        """ Override this so we can update the title bar. """
        self.updateTitle(self._taskParsObj.filename) # _taskParsObj is correct

    # Employ an edited callback for a given item?
    def _defineEditedCallbackObjectFor(self, parScope, parName):
        """ Override to allow us to use an edited callback. """

        # We know that the _taskParsObj is a ConfigObjPars
        triggerStr = self._taskParsObj.getTriggerStr(parScope, parName)

        # Some items will have a trigger, but likely most won't
        if triggerStr:
            return self
        else:
            return None

    def edited(self, scope, name, lastSavedVal, newVal):
        """ This is the callback function invoked when an item is edited.
            This is only called for those items which were previously
            specified to use this mechanism.  We do not turn this on for
            all items because the performance might be prohibitive. """
        # the print line is a stand-in
        triggerStr = self._taskParsObj.getTriggerStr(scope, name)
        # call triggers in a general way, not directly here # !!!
        if triggerStr.find('_section_switch_')>=0:
            state = str(newVal).lower() in ('on','yes','true')
            self._toggleSectionActiveState(scope, state, (name,))
        else:
            print "val: "+newVal+", trigger: "+triggerStr

    def _setTaskParsObj(self, theTask):
        """ Overridden version for ConfigObj. theTask can be either
            a .cfg file name or a ConfigObjPars object. """
        self._taskParsObj = cfgpars.getObjectFromTaskArg(theTask)

    def _getSaveAsFilter(self):
        """ Return a string to be used as the filter arg to the save file
            dialog during Save-As. """
        filt = '*.cfg'
        envVarName = APP_NAME.upper()+'_CFG'
        if envVarName in os.environ:
            upx = os.environ[envVarName]
            if len(upx) > 0:  filt = upx+"/*.cfg"
        return filt

    def _getOpenChoices(self):
        """ Go through all possible sites to find applicable .cfg files.
            Return as an iterable. """
        tsk = self._taskParsObj.getName()
        taskFiles = set()
        dirsSoFar = [] # this helps speed this up (skip unneeded globs)

        # last dir
        aDir = os.path.dirname(self._taskParsObj.filename)
        if len(aDir) < 1: aDir = os.curdir
        dirsSoFar.append(aDir)
        taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # current dir
        aDir = os.getcwd()
        if aDir not in dirsSoFar:
            dirsSoFar.append(aDir)
            taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # task's python pkg dir (if tsk == python pkg name)
        try:
            pkgf = cfgpars.findCfgFileForPkg(tsk, '.cfg', taskName=tsk)[1]
            taskFiles.update( (pkgf,) )
        except:
            pass # no big deal - maybe there is no python package

        # user's own resourceDir
        aDir = self._rcDir
        if aDir not in dirsSoFar:
            dirsSoFar.append(aDir)
            taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # extra loc - see if they used the app's env. var
        aDir = dirsSoFar[0] # flag to skip this if no env var found
        envVarName = APP_NAME.upper()+'_CFG'
        if envVarName in os.environ: aDir = os.environ[envVarName]
        if aDir not in dirsSoFar:
            dirsSoFar.append(aDir)
            taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # At the very end, add an option which we will later interpret to mean
        # to open the file dialog.
        taskFiles = list(taskFiles) # so as to keep next item at end of seq
        taskFiles.sort()
        taskFiles.append("Other ...")

        return taskFiles

    # OPEN: load parameter settings from a user-specified file
    def pfopen(self, event=None):
        """ Load the parameter settings from a user-specified file. """

        # Get the selected file name
        fname = self._openMenuChoice.get()

        # Also allow them to simply find any file - do not check _task_name_...
        # (could use Tkinter's FileDialog, but this one is prettier)
        if fname[-3:] == '...':
            fd = filedlg.PersistLoadFileDialog(self.top, "Load Config File",
                                               self._getSaveAsFilter())
            if fd.Show() != 1:
                fd.DialogCleanup()
                return
            fname = fd.GetFileName()
            fd.DialogCleanup()
            if fname == None: return # canceled

        # load it into a tmp object
        tmpObj = cfgpars.ConfigObjPars(fname)

        # check it to make sure it is a match
        if not self._taskParsObj.isSameTaskAs(tmpObj):
            msg = 'The current task is "'+self._taskParsObj.getName()+ \
                  '", but the selected file is for task "'+tmpObj.getName()+ \
                  '".  This file was not loaded.'
            tkMessageBox.showerror(message=msg,
                title="Error in "+os.path.basename(fname))
            return

        # Set the GUI entries to these values (let the user Save after)
        newParList = tmpObj.getParList()
        try:
            self.setAllEntriesFromParList(newParList)
        except editpar.UnfoundParamError, pe:
            tkMessageBox.showwarning(message=pe.message, title="Error in "+\
                                     os.path.basename(fname))

        # This new fname is our current context
        self.updateTitle(fname)
        self._taskParsObj.filename = fname # !! maybe try setCurrentContext() ?

    def unlearn(self, event=None):
        """ Override this so that we can set to default values our way. """
        self._setToDefaults()

    def _setToDefaults(self):
        """ Load the default parameter settings into the GUI. """

        # Create an empty object, where every item is set to it's default value
        try:
            tmpObj = cfgpars.ConfigObjPars(self._taskParsObj.filename,
                                           setAllToDefaults=True)
            print "Loading default "+self.taskName+" values via: "+ \
                  os.path.basename(tmpObj._original_configspec)
        except Exception, ex:
            msg = "Error Determining Defaults"
            tkMessageBox.showerror(message=msg+'\n\n'+ex.message,
                                   title="Error Determining Defaults")
            return

        # Set the GUI entries to these values (let the user Save after)
        newParList = tmpObj.getParList()
        try:
            self.setAllEntriesFromParList(newParList)
        except editpar.UnfoundParamError, pe:
            tkMessageBox.showerror(message=pe.message,
                                   title="Error Setting to Default Values")

    def _getGuiSettings(self):
        """ Return a dict (ConfigObj) of all user settings found in rcFile. """
        # Put the settings into a ConfigObj dict (don't use a config-spec)
        rcFile = self._rcDir+os.sep+APP_NAME.lower()+'.cfg'
        if os.path.exists(rcFile):
            return configobj.ConfigObj(rcFile, unrepr=True)
            # unrepr: for simple types, eliminates need for .cfgspc
        else:
            return {}

    def _saveGuiSettings(self):
        """ The base class doesn't implement this, so we will - save settings
        (only GUI stuff, not task related) to a file. """
        # Put the settings into a ConfigObj dict (don't use a config-spec)
        rcFile = self._rcDir+os.sep+APP_NAME.lower()+'.cfg'
        #
        if os.path.exists(rcFile): os.remove(rcFile)
        co = configobj.ConfigObj(rcFile)
        self._showHelpInBrowser = self.helpChoice.get() != "WINDOW"
        co['showHelpInBrowser']  = self._showHelpInBrowser
        co['saveAndCloseOnExec'] = self._saveAndCloseOnExec
        co.initial_comment = ['This file is automatically generated by '+\
                              APP_NAME+'.  Do not edit.']
        co.final_comment = [''] # ensure \n at EOF
        co.unrepr = True # for simple types, eliminates need for .cfgspc
        co.write()

<commit_msg>for #374 - removed unused line<commit_after>""" Main module for the ConfigObj version of the EPAR task editor

$Id$
"""

import configobj, glob, os, tkMessageBox
import cfgpars, editpar, filedlg
from cfgpars import APP_NAME

# Starts a GUI session
def epar(theTask, parent=None, isChild=0, loadOnly=False):
    if loadOnly:
        return cfgpars.getObjectFromTaskArg(theTask)
    else:
        dlg = ConfigObjEparDialog(theTask, parent, isChild)
        if dlg.canceled():
            return None
        else:
            return dlg.getTaskParsObj()

# Main class
class ConfigObjEparDialog(editpar.EditParDialog):

    def __init__(self, theTask, parent=None, isChild=0,
                 title=APP_NAME, childList=None):

        # Init base - calls _setTaskParsObj(), sets self.taskName, etc
        editpar.EditParDialog.__init__(self, theTask, parent, isChild,
                                       title, childList,
                                       resourceDir=cfgpars.getAppDir())
        # We don't return from this until the GUI is closed

    def _overrideMasterSettings(self):
        """ Override so that we can run in a different mode. """
        cod = self._getGuiSettings()
        self._appName             = APP_NAME
        self._useSimpleAutoClose  = False # is a fundamental issue here
        self._showExtraHelpButton = False
        self._saveAndCloseOnExec  = cod.get('saveAndCloseOnExec', False)
        self._showHelpInBrowser   = cod.get('showHelpInBrowser', False)
        self._optFile             = APP_NAME.lower()+".optionDB"

    def _preMainLoop(self):
        """ Override so that we can do some things right before activating. """
        # Put the fname in the title. EditParDialog doesn't do this by default
        self.updateTitle(self._taskParsObj.filename)

    def _doActualSave(self, fname, comment):
        """ Override this so we can handle case of file not writable. """
        try:
            rv=self._taskParsObj.saveParList(filename=fname,comment=comment)
            return rv
        except IOError:
            # User does not have privs to write to this file. Get name of local
            # choice and try to use that.
            if not fname:
                fname = self._taskParsObj.filename
            mine = self._rcDir+os.sep+os.path.basename(fname)
            # Tell them the context is changing, and where we are saving
            msg = 'Installed config file for task "'+ \
                  self._taskParsObj.getName()+'" is not to be overwritten.'+ \
                  '  Values will be saved to: \n\n\t"'+mine+'".'
            tkMessageBox.showwarning(message=msg, title="Will not overwrite!")
            # Try saving their copy
            rv=self._taskParsObj.saveParList(filename=mine, comment=comment)
            # Treat like a save-as
            self._saveAsPostSave_Hook(mine)
            return rv

    def _saveAsPostSave_Hook(self, fnameToBeUsed_UNUSED):
        """ Override this so we can update the title bar. """
        self.updateTitle(self._taskParsObj.filename) # _taskParsObj is correct

    # Employ an edited callback for a given item?
    def _defineEditedCallbackObjectFor(self, parScope, parName):
        """ Override to allow us to use an edited callback. """

        # We know that the _taskParsObj is a ConfigObjPars
        triggerStr = self._taskParsObj.getTriggerStr(parScope, parName)

        # Some items will have a trigger, but likely most won't
        if triggerStr:
            return self
        else:
            return None

    def edited(self, scope, name, lastSavedVal, newVal):
        """ This is the callback function invoked when an item is edited.
            This is only called for those items which were previously
            specified to use this mechanism.  We do not turn this on for
            all items because the performance might be prohibitive. """
        # the print line is a stand-in
        triggerStr = self._taskParsObj.getTriggerStr(scope, name)
        # call triggers in a general way, not directly here # !!!
        if triggerStr.find('_section_switch_')>=0:
            state = str(newVal).lower() in ('on','yes','true')
            self._toggleSectionActiveState(scope, state, (name,))
        else:
            print "val: "+newVal+", trigger: "+triggerStr

    def _setTaskParsObj(self, theTask):
        """ Overridden version for ConfigObj. theTask can be either
            a .cfg file name or a ConfigObjPars object. """
        self._taskParsObj = cfgpars.getObjectFromTaskArg(theTask)

    def _getSaveAsFilter(self):
        """ Return a string to be used as the filter arg to the save file
            dialog during Save-As. """
        filt = '*.cfg'
        envVarName = APP_NAME.upper()+'_CFG'
        if envVarName in os.environ:
            upx = os.environ[envVarName]
            if len(upx) > 0:  filt = upx+"/*.cfg"
        return filt

    def _getOpenChoices(self):
        """ Go through all possible sites to find applicable .cfg files.
            Return as an iterable. """
        tsk = self._taskParsObj.getName()
        taskFiles = set()
        dirsSoFar = [] # this helps speed this up (skip unneeded globs)

        # last dir
        aDir = os.path.dirname(self._taskParsObj.filename)
        if len(aDir) < 1: aDir = os.curdir
        dirsSoFar.append(aDir)
        taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # current dir
        aDir = os.getcwd()
        if aDir not in dirsSoFar:
            dirsSoFar.append(aDir)
            taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # task's python pkg dir (if tsk == python pkg name)
        try:
            pkgf = cfgpars.findCfgFileForPkg(tsk, '.cfg', taskName=tsk)[1]
            taskFiles.update( (pkgf,) )
        except:
            pass # no big deal - maybe there is no python package

        # user's own resourceDir
        aDir = self._rcDir
        if aDir not in dirsSoFar:
            dirsSoFar.append(aDir)
            taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # extra loc - see if they used the app's env. var
        aDir = dirsSoFar[0] # flag to skip this if no env var found
        envVarName = APP_NAME.upper()+'_CFG'
        if envVarName in os.environ: aDir = os.environ[envVarName]
        if aDir not in dirsSoFar:
            dirsSoFar.append(aDir)
            taskFiles.update(cfgpars.getCfgFilesInDirForTask(aDir, tsk))

        # At the very end, add an option which we will later interpret to mean
        # to open the file dialog.
        taskFiles = list(taskFiles) # so as to keep next item at end of seq
        taskFiles.sort()
        taskFiles.append("Other ...")

        return taskFiles

    # OPEN: load parameter settings from a user-specified file
    def pfopen(self, event=None):
        """ Load the parameter settings from a user-specified file. """

        # Get the selected file name
        fname = self._openMenuChoice.get()

        # Also allow them to simply find any file - do not check _task_name_...
        # (could use Tkinter's FileDialog, but this one is prettier)
        if fname[-3:] == '...':
            fd = filedlg.PersistLoadFileDialog(self.top, "Load Config File",
                                               self._getSaveAsFilter())
            if fd.Show() != 1:
                fd.DialogCleanup()
                return
            fname = fd.GetFileName()
            fd.DialogCleanup()
            if fname == None: return # canceled

        # load it into a tmp object
        tmpObj = cfgpars.ConfigObjPars(fname)

        # check it to make sure it is a match
        if not self._taskParsObj.isSameTaskAs(tmpObj):
            msg = 'The current task is "'+self._taskParsObj.getName()+ \
                  '", but the selected file is for task "'+tmpObj.getName()+ \
                  '".  This file was not loaded.'
            tkMessageBox.showerror(message=msg,
                title="Error in "+os.path.basename(fname))
            return

        # Set the GUI entries to these values (let the user Save after)
        newParList = tmpObj.getParList()
        try:
            self.setAllEntriesFromParList(newParList)
        except editpar.UnfoundParamError, pe:
            tkMessageBox.showwarning(message=pe.message, title="Error in "+\
                                     os.path.basename(fname))

        # This new fname is our current context
        self.updateTitle(fname)
        self._taskParsObj.filename = fname # !! maybe try setCurrentContext() ?

    def unlearn(self, event=None):
        """ Override this so that we can set to default values our way. """
        self._setToDefaults()

    def _setToDefaults(self):
        """ Load the default parameter settings into the GUI. """

        # Create an empty object, where every item is set to it's default value
        try:
            tmpObj = cfgpars.ConfigObjPars(self._taskParsObj.filename,
                                           setAllToDefaults=True)
            print "Loading default "+self.taskName+" values via: "+ \
                  os.path.basename(tmpObj._original_configspec)
        except Exception, ex:
            msg = "Error Determining Defaults"
            tkMessageBox.showerror(message=msg+'\n\n'+ex.message,
                                   title="Error Determining Defaults")
            return

        # Set the GUI entries to these values (let the user Save after)
        newParList = tmpObj.getParList()
        try:
            self.setAllEntriesFromParList(newParList)
        except editpar.UnfoundParamError, pe:
            tkMessageBox.showerror(message=pe.message,
                                   title="Error Setting to Default Values")

    def _getGuiSettings(self):
        """ Return a dict (ConfigObj) of all user settings found in rcFile. """
        # Put the settings into a ConfigObj dict (don't use a config-spec)
        rcFile = self._rcDir+os.sep+APP_NAME.lower()+'.cfg'
        if os.path.exists(rcFile):
            return configobj.ConfigObj(rcFile, unrepr=True)
            # unrepr: for simple types, eliminates need for .cfgspc
        else:
            return {}

    def _saveGuiSettings(self):
        """ The base class doesn't implement this, so we will - save settings
        (only GUI stuff, not task related) to a file. """
        # Put the settings into a ConfigObj dict (don't use a config-spec)
        rcFile = self._rcDir+os.sep+APP_NAME.lower()+'.cfg'
        #
        if os.path.exists(rcFile): os.remove(rcFile)
        co = configobj.ConfigObj(rcFile)
        co['showHelpInBrowser']  = self._showHelpInBrowser
        co['saveAndCloseOnExec'] = self._saveAndCloseOnExec
        co.initial_comment = ['This file is automatically generated by '+\
                              APP_NAME+'.  Do not edit.']
        co.final_comment = [''] # ensure \n at EOF
        co.unrepr = True # for simple types, eliminates need for .cfgspc
        co.write()

<|endoftext|>


<commit_before># Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import sys
import csv
import logging
import datetime
import inspect

from lib.cuckoo.common.abstracts import Processing
from lib.cuckoo.common.utils import convert_to_printable, logtime
from lib.cuckoo.common.netlog import NetlogParser

log = logging.getLogger(__name__)

class ParseProcessLog(list):
    """Parses process log file."""

    def __init__(self, log_path):
        """@param log_path: log file path."""
        self._log_path = log_path
        self.fd = None
        self.parser = None

        self.process_id = None
        self.process_name = None
        self.parent_id = None
        self.first_seen = None
        self.calls = self
        self.lastcall = None

        if os.path.exists(log_path) and os.stat(log_path).st_size > 0:
            self.parse_first_and_reset()

    def parse_first_and_reset(self):
        self.fd = open(self._log_path, "rb")
        self.parser = NetlogParser(self)
        self.parser.read_next_message()
        self.fd.seek(0)

    def read(self, length):
        if length == 0: return b''
        buf = self.fd.read(length)
        if not buf or len(buf) != length: raise EOFError()
        return buf

    def __iter__(self):
        #log.debug('iter called by this guy: {0}'.format(inspect.stack()[1]))
        return self

    def __getitem__(self, key):
        return getattr(self, key)

    def __repr__(self):
        return 'ParseProcessLog {0}'.format(self._log_path)

    def __nonzero__(self):
        return True

    def next(self):
        while not self.lastcall:
            r = None
            try: r = self.parser.read_next_message()
            except EOFError:
                self.fd.seek(0)
                raise StopIteration()

            if not r: raise StopIteration()

        tmp, self.lastcall = self.lastcall, None
        return tmp

    def log_process(self, context, timestring, pid, ppid, modulepath, procname):
        self.process_id, self.parent_id, self.process_name = pid, ppid, procname
        self.first_seen = timestring

    def log_thread(self, context, pid):
        pass

    def log_call(self, context, apiname, modulename, arguments):
        apiindex, status, returnval, tid, timediff = context

        current_time = self.first_seen + datetime.timedelta(0,0, timediff*1000)
        timestring = logtime(current_time)

        self.lastcall = self._parse([timestring, tid, modulename, apiname,
            status, returnval] + arguments)

    def _parse(self, row):
        """Parse log row.
        @param row: row data.
        @return: parsed information dict.
        """
        call = {}
        arguments = []

        try:
            timestamp = row[0]    # Timestamp of current API call invocation.
            thread_id = row[1]    # Thread ID.
            category = row[2]     # Win32 function category.
            api_name = row[3]     # Name of the Windows API.
            status_value = row[4] # Success or Failure?
            return_value = row[5] # Value returned by the function.
        except IndexError as e:
            log.debug("Unable to parse process log row: %s" % e)
            return False

        # Now walk through the remaining columns, which will contain API
        # arguments.
        for index in range(6, len(row)):
            argument = {}

            # Split the argument name with its value based on the separator.
            try:
                (arg_name, arg_value) = row[index]
            except ValueError as e:
                log.debug("Unable to parse analysis row argument (row=%s): %s" % (row[index], e))
                continue

            argument["name"] = arg_name
            argument["value"] = convert_to_printable(str(arg_value)).lstrip("\\??\\")
            arguments.append(argument)

        call["timestamp"] = timestamp
        call["thread_id"] = str(thread_id)
        call["category"] = category
        call["api"] = api_name
        call["status"] = str(status_value)
        call["return"] = convert_to_printable(str(return_value))
        call["arguments"] = arguments
        call["repeated"] = 0

        return call

class Processes:
    """Processes analyzer."""

    def __init__(self, logs_path):
        """@param  logs_path: logs path."""
        self._logs_path = logs_path

    def run(self):
        """Run analysis.
        @return: processes infomartion list.
        """
        results = []

        if not os.path.exists(self._logs_path):
            log.error("Analysis results folder does not exist at path \"%s\"."
                      % self._logs_path)
            return results

        if len(os.listdir(self._logs_path)) == 0:
            log.error("Analysis results folder does not contain any file.")
            return results

        for file_name in os.listdir(self._logs_path):
            file_path = os.path.join(self._logs_path, file_name)

            if os.path.isdir(file_path):
                continue

            if not file_path.endswith(".raw"):
                continue

            # Invoke parsing of current log file.
            current_log = ParseProcessLog(file_path)
            if current_log.process_id == None: continue

            # If the current log actually contains any data, add its data to
            # the global results list.
            results.append({
                "process_id": current_log.process_id,
                "process_name": current_log.process_name,
                "parent_id": current_log.parent_id,
                "first_seen": logtime(current_log.first_seen),
                "calls": current_log
            })

        # Sort the items in the results list chronologically. In this way we
        # can have a sequential order of spawned processes.
        results.sort(key=lambda process: process["first_seen"])

        return results

class Summary:
    """Generates summary information."""

    def __init__(self, proc_results):
        """@param oroc_results: enumerated processes results."""
        self.proc_results = proc_results

    def run(self):
        """Get registry keys, mutexes and files.
        @return: Summary of keys, mutexes and files.
        """
        keys = []
        mutexes = []
        files = []

        def _check_registry(handles, registry, subkey, handle):
            for known_handle in handles:
                if handle != 0 and handle == known_handle["handle"]:
                    return None

            name = ""
            if registry == 0x80000000:
                name = "HKEY_CLASSES_ROOT\\"
            elif registry == 0x80000001:
                name = "HKEY_CURRENT_USER\\"
            elif registry == 0x80000002:
                name = "HKEY_LOCAL_MACHINE\\"
            elif registry == 0x80000003:
                name = "HKEY_USERS\\"
            elif registry == 0x80000004:
                name = "HKEY_PERFORMANCE_DATA\\"
            elif registry == 0x80000005:
                name = "HKEY_CURRENT_CONFIG\\"
            elif registry == 0x80000006:
                name = "HKEY_DYN_DATA\\"
            else:
                for known_handle in handles:
                    if registry == known_handle["handle"]:
                        name = known_handle["name"] + "\\"

            handles.append({"handle" : handle, "name" : name + subkey})
            return name + subkey

        def _remove_handle(handles, handle):
            for known_handle in handles:
                if handle != 0 and handle == known_handle["handle"]:
                    handles.remove(known_handle)

        for process in self.proc_results:
            handles = []

            for call in process["calls"]:
                if call["api"].startswith("RegOpenKeyEx") or call["api"].startswith("RegCreateKeyEx"):
                    registry = 0
                    subkey = ""
                    handle = 0

                    for argument in call["arguments"]:
                        if argument["name"] == "Registry":
                            registry = int(argument["value"], 16)
                        elif argument["name"] == "SubKey":
                            subkey = argument["value"]
                        elif argument["name"] == "Handle":
                            handle = int(argument["value"], 16)

                    name = _check_registry(handles, registry, subkey, handle)
                    if name and name not in keys:
                        keys.append(name)
                elif call["api"].startswith("RegCloseKey"):
                    handle = 0

                    for argument in call["arguments"]:
                        if argument["name"] == "Handle":
                            handle = int(argument["value"], 16)
                    _remove_handle(handles, handle)

                elif call["category"] == "filesystem":
                    for argument in call["arguments"]:
                        if argument["name"] == "FileName":
                            value = argument["value"].strip()
                            if not value:
                                continue

                            if value not in files:
                                files.append(value)

                elif call["category"] == "synchronization":
                    for argument in call["arguments"]:
                        if argument["name"] == "MutexName":
                            value = argument["value"].strip()
                            if not value:
                                continue

                            if value not in mutexes:
                                mutexes.append(value)

        return {"files": files, "keys": keys, "mutexes": mutexes}

class ProcessTree:
    """Creates process tree."""

    def __init__(self, proc_results):
        """@param proc_results: enumerated processes information."""
        self.proc_results = proc_results
        self.processes = []
        self.proctree = []

    def gen_proclist(self):
        """Generate processes list.
        @return: True.
        """
        for entry in self.proc_results:
            process = {}
            process["name"] = entry["process_name"]
            process["pid"] = int(entry["process_id"])
            process["children"] = []

            for call in entry["calls"]:
                if call["api"] == "CreateProcessInternalW":
                    for argument in call["arguments"]:
                        if argument["name"] == "ProcessId":
                            process["children"].append(int(argument["value"]))

            self.processes.append(process)

        return True

    def add_node(self, node, parent_id, tree):
        """Add a node to a tree.
        @param node: node to add.
        @param parent_id: parent node.
        @param tree: processes tree.
        @return: boolean with operation success status.
        """
        for process in tree:
            if process["pid"] == parent_id:
                new = {}
                new["name"] = node["name"]
                new["pid"] = node["pid"]
                new["children"] = []
                process["children"].append(new)
                return True
            self.add_node(node, parent_id, process["children"])

        return False

    def populate(self, node):
        """Populate tree.
        @param node: node to add.
        @return: True.
        """
        for children in node["children"]:
            for proc in self.processes:
                if int(proc["pid"]) == int(children):
                    self.add_node(proc, node["pid"], self.proctree)
                    self.populate(proc)

        return True

    def run(self):
        """Run analysis.
        @return: results dict or None.
        """
        if not self.proc_results or len(self.proc_results) == 0:
            return None

        self.gen_proclist()
        root = {}
        root["name"] = self.processes[0]["name"]
        root["pid"] = self.processes[0]["pid"]
        root["children"] = []
        self.proctree.append(root)
        self.populate(self.processes[0])

        return self.proctree

class BehaviorAnalysis(Processing):
    """Behavior Analyzer."""

    def run(self):
        """Run analysis.
        @return: results dict.
        """
        self.key = "behavior"

        behavior = {}
        behavior["processes"]   = Processes(self.logs_path).run()
        behavior["processtree"] = ProcessTree(behavior["processes"]).run()
        behavior["summary"]     = Summary(behavior["processes"]).run()

        return behavior
<commit_msg>Changed values of behavioral logs<commit_after># Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import sys
import csv
import logging
import datetime
import inspect

from lib.cuckoo.common.abstracts import Processing
from lib.cuckoo.common.utils import convert_to_printable, logtime
from lib.cuckoo.common.netlog import NetlogParser

log = logging.getLogger(__name__)

class ParseProcessLog(list):
    """Parses process log file."""

    def __init__(self, log_path):
        """@param log_path: log file path."""
        self._log_path = log_path
        self.fd = None
        self.parser = None

        self.process_id = None
        self.process_name = None
        self.parent_id = None
        self.first_seen = None
        self.calls = self
        self.lastcall = None

        if os.path.exists(log_path) and os.stat(log_path).st_size > 0:
            self.parse_first_and_reset()

    def parse_first_and_reset(self):
        self.fd = open(self._log_path, "rb")
        self.parser = NetlogParser(self)
        self.parser.read_next_message()
        self.fd.seek(0)

    def read(self, length):
        if length == 0: return b''
        buf = self.fd.read(length)
        if not buf or len(buf) != length: raise EOFError()
        return buf

    def __iter__(self):
        #log.debug('iter called by this guy: {0}'.format(inspect.stack()[1]))
        return self

    def __getitem__(self, key):
        return getattr(self, key)

    def __repr__(self):
        return 'ParseProcessLog {0}'.format(self._log_path)

    def __nonzero__(self):
        return True

    def next(self):
        while not self.lastcall:
            r = None
            try: r = self.parser.read_next_message()
            except EOFError:
                self.fd.seek(0)
                raise StopIteration()

            if not r: raise StopIteration()

        tmp, self.lastcall = self.lastcall, None
        return tmp

    def log_process(self, context, timestring, pid, ppid, modulepath, procname):
        self.process_id, self.parent_id, self.process_name = pid, ppid, procname
        self.first_seen = timestring

    def log_thread(self, context, pid):
        pass

    def log_call(self, context, apiname, modulename, arguments):
        apiindex, status, returnval, tid, timediff = context

        current_time = self.first_seen + datetime.timedelta(0,0, timediff*1000)
        timestring = logtime(current_time)

        self.lastcall = self._parse([timestring, tid, modulename, apiname,
            status, returnval] + arguments)

    def _parse(self, row):
        """Parse log row.
        @param row: row data.
        @return: parsed information dict.
        """
        call = {}
        arguments = []

        try:
            timestamp = row[0]    # Timestamp of current API call invocation.
            thread_id = row[1]    # Thread ID.
            category = row[2]     # Win32 function category.
            api_name = row[3]     # Name of the Windows API.
            status_value = row[4] # Success or Failure?
            return_value = row[5] # Value returned by the function.
        except IndexError as e:
            log.debug("Unable to parse process log row: %s" % e)
            return False

        # Now walk through the remaining columns, which will contain API
        # arguments.
        for index in range(6, len(row)):
            argument = {}

            # Split the argument name with its value based on the separator.
            try:
                (arg_name, arg_value) = row[index]
            except ValueError as e:
                log.debug("Unable to parse analysis row argument (row=%s): %s" % (row[index], e))
                continue

            argument["name"] = arg_name
            argument["value"] = convert_to_printable(str(arg_value)).lstrip("\\??\\")
            arguments.append(argument)

        call["timestamp"] = timestamp
        call["thread_id"] = str(thread_id)
        call["category"] = category
        call["api"] = api_name
        call["status"] = bool(int(status_value))

        if isinstance(return_value, int):
            call["return"] = "0x%.08x" % return_value
        else:
            call["return"] = convert_to_printable(str(return_value))

        call["arguments"] = arguments
        call["repeated"] = 0

        return call

class Processes:
    """Processes analyzer."""

    def __init__(self, logs_path):
        """@param  logs_path: logs path."""
        self._logs_path = logs_path

    def run(self):
        """Run analysis.
        @return: processes infomartion list.
        """
        results = []

        if not os.path.exists(self._logs_path):
            log.error("Analysis results folder does not exist at path \"%s\"."
                      % self._logs_path)
            return results

        if len(os.listdir(self._logs_path)) == 0:
            log.error("Analysis results folder does not contain any file.")
            return results

        for file_name in os.listdir(self._logs_path):
            file_path = os.path.join(self._logs_path, file_name)

            if os.path.isdir(file_path):
                continue

            if not file_path.endswith(".raw"):
                continue

            # Invoke parsing of current log file.
            current_log = ParseProcessLog(file_path)
            if current_log.process_id == None: continue

            # If the current log actually contains any data, add its data to
            # the global results list.
            results.append({
                "process_id": current_log.process_id,
                "process_name": current_log.process_name,
                "parent_id": current_log.parent_id,
                "first_seen": logtime(current_log.first_seen),
                "calls": current_log
            })

        # Sort the items in the results list chronologically. In this way we
        # can have a sequential order of spawned processes.
        results.sort(key=lambda process: process["first_seen"])

        return results

class Summary:
    """Generates summary information."""

    def __init__(self, proc_results):
        """@param oroc_results: enumerated processes results."""
        self.proc_results = proc_results

    def run(self):
        """Get registry keys, mutexes and files.
        @return: Summary of keys, mutexes and files.
        """
        keys = []
        mutexes = []
        files = []

        def _check_registry(handles, registry, subkey, handle):
            for known_handle in handles:
                if handle != 0 and handle == known_handle["handle"]:
                    return None

            name = ""
            if registry == 0x80000000:
                name = "HKEY_CLASSES_ROOT\\"
            elif registry == 0x80000001:
                name = "HKEY_CURRENT_USER\\"
            elif registry == 0x80000002:
                name = "HKEY_LOCAL_MACHINE\\"
            elif registry == 0x80000003:
                name = "HKEY_USERS\\"
            elif registry == 0x80000004:
                name = "HKEY_PERFORMANCE_DATA\\"
            elif registry == 0x80000005:
                name = "HKEY_CURRENT_CONFIG\\"
            elif registry == 0x80000006:
                name = "HKEY_DYN_DATA\\"
            else:
                for known_handle in handles:
                    if registry == known_handle["handle"]:
                        name = known_handle["name"] + "\\"

            handles.append({"handle" : handle, "name" : name + subkey})
            return name + subkey

        def _remove_handle(handles, handle):
            for known_handle in handles:
                if handle != 0 and handle == known_handle["handle"]:
                    handles.remove(known_handle)

        for process in self.proc_results:
            handles = []

            for call in process["calls"]:
                if call["api"].startswith("RegOpenKeyEx") or call["api"].startswith("RegCreateKeyEx"):
                    registry = 0
                    subkey = ""
                    handle = 0

                    for argument in call["arguments"]:
                        if argument["name"] == "Registry":
                            registry = int(argument["value"], 16)
                        elif argument["name"] == "SubKey":
                            subkey = argument["value"]
                        elif argument["name"] == "Handle":
                            handle = int(argument["value"], 16)

                    name = _check_registry(handles, registry, subkey, handle)
                    if name and name not in keys:
                        keys.append(name)
                elif call["api"].startswith("RegCloseKey"):
                    handle = 0

                    for argument in call["arguments"]:
                        if argument["name"] == "Handle":
                            handle = int(argument["value"], 16)
                    _remove_handle(handles, handle)

                elif call["category"] == "filesystem":
                    for argument in call["arguments"]:
                        if argument["name"] == "FileName":
                            value = argument["value"].strip()
                            if not value:
                                continue

                            if value not in files:
                                files.append(value)

                elif call["category"] == "synchronization":
                    for argument in call["arguments"]:
                        if argument["name"] == "MutexName":
                            value = argument["value"].strip()
                            if not value:
                                continue

                            if value not in mutexes:
                                mutexes.append(value)

        return {"files": files, "keys": keys, "mutexes": mutexes}

class ProcessTree:
    """Creates process tree."""

    def __init__(self, proc_results):
        """@param proc_results: enumerated processes information."""
        self.proc_results = proc_results
        self.processes = []
        self.proctree = []

    def gen_proclist(self):
        """Generate processes list.
        @return: True.
        """
        for entry in self.proc_results:
            process = {}
            process["name"] = entry["process_name"]
            process["pid"] = int(entry["process_id"])
            process["children"] = []

            for call in entry["calls"]:
                if call["api"] == "CreateProcessInternalW":
                    for argument in call["arguments"]:
                        if argument["name"] == "ProcessId":
                            process["children"].append(int(argument["value"]))

            self.processes.append(process)

        return True

    def add_node(self, node, parent_id, tree):
        """Add a node to a tree.
        @param node: node to add.
        @param parent_id: parent node.
        @param tree: processes tree.
        @return: boolean with operation success status.
        """
        for process in tree:
            if process["pid"] == parent_id:
                new = {}
                new["name"] = node["name"]
                new["pid"] = node["pid"]
                new["children"] = []
                process["children"].append(new)
                return True
            self.add_node(node, parent_id, process["children"])

        return False

    def populate(self, node):
        """Populate tree.
        @param node: node to add.
        @return: True.
        """
        for children in node["children"]:
            for proc in self.processes:
                if int(proc["pid"]) == int(children):
                    self.add_node(proc, node["pid"], self.proctree)
                    self.populate(proc)

        return True

    def run(self):
        """Run analysis.
        @return: results dict or None.
        """
        if not self.proc_results or len(self.proc_results) == 0:
            return None

        self.gen_proclist()
        root = {}
        root["name"] = self.processes[0]["name"]
        root["pid"] = self.processes[0]["pid"]
        root["children"] = []
        self.proctree.append(root)
        self.populate(self.processes[0])

        return self.proctree

class BehaviorAnalysis(Processing):
    """Behavior Analyzer."""

    def run(self):
        """Run analysis.
        @return: results dict.
        """
        self.key = "behavior"

        behavior = {}
        behavior["processes"]   = Processes(self.logs_path).run()
        behavior["processtree"] = ProcessTree(behavior["processes"]).run()
        behavior["summary"]     = Summary(behavior["processes"]).run()

        return behavior
<|endoftext|>


<commit_before># * -*- coding: utf-8 -*-

# Copyright(C) 2011-2012  Johann Broudin, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.browser import PagesBrowser, URL
from .pages import RadioPage, JsonPage, PodcastPage

__all__ = ['RadioFranceBrowser']

class RadioFranceBrowser(PagesBrowser):
    json_page = URL('sites/default/files/(?P<json_url>.*).json',
                    'player-json/reecoute/(?P<json_url_fip>.*)',
                    'station/(?P<fbplayer>.*)', JsonPage)
    podcast_page = URL('podcast09/rss_(?P<podcast_id>.*)\.xml', PodcastPage)
    radio_page = URL('(?P<page>.*)', RadioPage)

    def get_radio_url(self, radio, player):
        self.BASEURL = 'http://www.%s.fr/' % radio
        if radio == 'francebleu':
            return self.json_page.go(fbplayer=player).get_fburl()
        else:
            return self.radio_page.go(page=player).get_url()

    def get_current(self, radio, url):
        self.BASEURL = 'http://www.%s.fr/' % radio
        if radio == 'francebleu':
            return self.radio_page.go(page=url).get_current()
        else:
            return self.json_page.go(json_url=url).get_current()

    def get_selection(self, radio_url, json_url, radio_id):
        self.BASEURL = 'http://www.%s.fr/' % radio_url
        if radio_id == 'fipradio':
            return self.json_page.go(json_url_fip=json_url).get_selection(radio_id=radio_id)

        return self.json_page.go(json_url=json_url).get_selection(radio_id=radio_id)

    def get_audio(self, _id, radio_url, json_url, radio_id):
        for item in self.get_selection(radio_url, json_url, radio_id):
            if item.id == _id:
                return item
        return []

    def search_audio(self, pattern, radio_url, json_url, radio_id):
        for item in self.get_selection(radio_url, json_url, radio_id):
            if pattern.upper() in item.title.upper():
                yield item

    def get_podcast_emissions(self, radio_url, podcast_url, split_path):
        self.BASEURL = 'http://www.%s.fr/' % radio_url
        if split_path[0] == 'franceinter':
            return self.radio_page.go(page=podcast_url).get_france_inter_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'franceculture':
            return self.radio_page.go(page=podcast_url).get_france_culture_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'franceinfo':
            return self.radio_page.go(page=podcast_url).get_france_info_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'francemusique':
            return self.radio_page.go(page=podcast_url).get_france_musique_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'mouv':
            return self.radio_page.go(page=podcast_url).get_mouv_podcast_emissions(split_path=split_path)

    def get_podcasts(self, podcast_id):
        self.BASEURL = 'http://radiofrance-podcast.net/'
        return self.podcast_page.go(podcast_id=podcast_id).iter_podcasts()

    def get_france_culture_podcasts_url(self, url):
        self.BASEURL = 'http://www.franceculture.fr/podcast/'
        return self.radio_page.go(page=url).get_france_culture_podcasts_url()
<commit_msg>radiofrance: remove useless else<commit_after># * -*- coding: utf-8 -*-

# Copyright(C) 2011-2012  Johann Broudin, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from weboob.browser import PagesBrowser, URL
from .pages import RadioPage, JsonPage, PodcastPage

__all__ = ['RadioFranceBrowser']

class RadioFranceBrowser(PagesBrowser):
    json_page = URL('sites/default/files/(?P<json_url>.*).json',
                    'player-json/reecoute/(?P<json_url_fip>.*)',
                    'station/(?P<fbplayer>.*)', JsonPage)
    podcast_page = URL('podcast09/rss_(?P<podcast_id>.*)\.xml', PodcastPage)
    radio_page = URL('(?P<page>.*)', RadioPage)

    def get_radio_url(self, radio, player):
        self.BASEURL = 'http://www.%s.fr/' % radio
        if radio == 'francebleu':
            return self.json_page.go(fbplayer=player).get_fburl()

        return self.radio_page.go(page=player).get_url()

    def get_current(self, radio, url):
        self.BASEURL = 'http://www.%s.fr/' % radio
        if radio == 'francebleu':
            return self.radio_page.go(page=url).get_current()

        return self.json_page.go(json_url=url).get_current()

    def get_selection(self, radio_url, json_url, radio_id):
        self.BASEURL = 'http://www.%s.fr/' % radio_url
        if radio_id == 'fipradio':
            return self.json_page.go(json_url_fip=json_url).get_selection(radio_id=radio_id)

        return self.json_page.go(json_url=json_url).get_selection(radio_id=radio_id)

    def get_audio(self, _id, radio_url, json_url, radio_id):
        for item in self.get_selection(radio_url, json_url, radio_id):
            if item.id == _id:
                return item
        return []

    def search_audio(self, pattern, radio_url, json_url, radio_id):
        for item in self.get_selection(radio_url, json_url, radio_id):
            if pattern.upper() in item.title.upper():
                yield item

    def get_podcast_emissions(self, radio_url, podcast_url, split_path):
        self.BASEURL = 'http://www.%s.fr/' % radio_url
        if split_path[0] == 'franceinter':
            return self.radio_page.go(page=podcast_url).get_france_inter_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'franceculture':
            return self.radio_page.go(page=podcast_url).get_france_culture_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'franceinfo':
            return self.radio_page.go(page=podcast_url).get_france_info_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'francemusique':
            return self.radio_page.go(page=podcast_url).get_france_musique_podcast_emissions(split_path=split_path)
        elif split_path[0] == 'mouv':
            return self.radio_page.go(page=podcast_url).get_mouv_podcast_emissions(split_path=split_path)

    def get_podcasts(self, podcast_id):
        self.BASEURL = 'http://radiofrance-podcast.net/'
        return self.podcast_page.go(podcast_id=podcast_id).iter_podcasts()

    def get_france_culture_podcasts_url(self, url):
        self.BASEURL = 'http://www.franceculture.fr/podcast/'
        return self.radio_page.go(page=url).get_france_culture_podcasts_url()
<|endoftext|>


<commit_before>"""
MISEQ_MONITOR.py
1) For runs flagged 'needsprocessing' that have not yet been processed, copy fastqs to local disk
2) Process the run by calling MISEQ_PIPELINE.py
3) Upload results to the network drive
"""

pipeline_version = '5.3g-single-sample'

import logging, miseq_logging, miseqUtils, os, subprocess, sys, time
from settings import *
from glob import glob

if sys.version_info[:2] != (2, 7):
    raise Exception("Python 2.7 not detected")

def mark_run_as_disabled(curr_run):
    open(curr_run.replace(NEEDS_PROCESSING, ERROR_PROCESSING), 'w').close()

def execute_command(command):
    logger.info(" ".join(command))
    stdout, stderr = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()
    if stderr != "": logging.warn(stderr)
    return stdout

def post_files(files, destination):
    for file in files:
        execute_command(['rsync', '-a', file, '{}/{}'.format(destination, os.path.basename(file))])

# Process runs flagged for processing not already processed by this version of the pipeline
while True:

    #runs = glob(rawdata_mount + 'MiSeq/runs/*/{}'.format(NEEDS_PROCESSING))
    #runs = glob(rawdata_mount + 'MiSeq/runs/140127_M01841_0051_000000000-A64E4/{}'.format(NEEDS_PROCESSING))
    #runs += glob(rawdata_mount + 'MiSeq/runs/131119_M01841_0041_000000000-A5EPY/{}'.format(NEEDS_PROCESSING))
    #runs += glob(rawdata_mount + 'MiSeq/runs/131112_M01841_0040_000000000-A5F9H/{}'.format(NEEDS_PROCESSING))
    #runs += glob(rawdata_mount + 'MiSeq/runs/131101_M01841_0037_000000000-A5F9E/{}'.format(NEEDS_PROCESSING))
    runs = glob(rawdata_mount + 'MiSeq/runs/131119_M01841_0041_000000000-A5EPY/{}'.format(NEEDS_PROCESSING))

    runs_needing_processing = []
    for run in runs:
        result_path = '{}/version_{}'.format(run.replace(NEEDS_PROCESSING, 'Results'), pipeline_version)

        if os.path.exists(run.replace(NEEDS_PROCESSING, ERROR_PROCESSING)):
            continue

        if not os.path.exists(result_path):
            runs_needing_processing.append(run)

    if not runs_needing_processing:
        logging.info('No runs need processing')
        time.sleep(delay)
        continue

    # Process most recently generated run and work backwards
    runs_needing_processing.sort(reverse=True)
    curr_run = runs_needing_processing[0]
    root = curr_run.replace(NEEDS_PROCESSING, '')
    run_name = root.split('/')[-2]

    # Make folder on the cluster for intermediary files
    if not os.path.exists(home+run_name):
        os.mkdir(home+run_name)

    # Record standard input / output of monitor
    log_file = home + run_name + '/MISEQ_MONITOR_OUTPUT.log'
    try:
        logger = miseq_logging.init_logging(log_file, file_log_level=logging.DEBUG, console_log_level=logging.INFO)
        logger.info('===== Processing {} with pipeline version {} ====='.format(root, pipeline_version))
    except Exception as e:
        raise Exception("Couldn't setup logging (init_logging() threw exception '{}') - HALTING NOW!".format(str(e)))

    # SampleSheet.csv needed to determine the mode and sample T-primer
    remote_file = curr_run.replace(NEEDS_PROCESSING, 'SampleSheet.csv')
    local_file = home + run_name + '/SampleSheet.csv'
    execute_command(['rsync', '-a', remote_file, local_file])

    # Delete raw tiffs from this run
    # logger.info("Deleting tiffs from {}Images/Focus/L001/*".format(root))
    #for tiff in glob("{}Images/Focus/L001/*/*.tif".format(root)):
    #    log.info("os.remove({})".format(tiff))
    #    os.remove(tiff)

    try:
        with open(local_file, 'rU') as sample_sheet:
            run_info = miseqUtils.sampleSheetParser(sample_sheet)
            mode = run_info['Description']
    except Exception as e:
        logger.error("Exception thrown while parsing sample sheet: '{}' - skipping run and marking with flag '{}'".format(str(e),ERROR_PROCESSING))
        mark_run_as_disabled(curr_run)
        continue

    if mode not in ['Nextera', 'Amplicon']:
        logger.error("{} not a valid mode: skipping run and marking with {}".format(mode, ERROR_PROCESSING))
        mark_run_as_disabled(curr_run)
        continue

    # Copy fastq.gz files to the cluster and unzip them
    for gz_file in glob(root+'Data/Intensities/BaseCalls/*.fastq.gz'):
        filename = os.path.basename(gz_file)

        # Report number of reads failing to demultiplex to the log
        if filename.startswith('Undetermined'):

            # Second file has the same number of lines - don't bother
            if filename.endswith('_L001_R2_001.fastq.gz'):
                continue

            # Do word count directly on stream redirected from gunzip
            p1 = subprocess.Popen(['gunzip', '-c', gz_file], stdout=subprocess.PIPE)
            p2 = subprocess.Popen(['wc', '-l'], stdin=p1.stdout, stdout=subprocess.PIPE)
            output = p2.communicate()[0]
            failed_demultiplexing = output.strip(' \n')
            logger.info("{} reads failed to demultiplex in {} (removing file)".format(failed_demultiplexing, filename))
            continue

		# If a local copy of the unzipped fastq exists, skip this step
        local_file = home + run_name + '/' + filename
        if os.path.exists(local_file.replace('.gz', '')):
            continue

        execute_command(['rsync', '-a', gz_file, local_file])
        execute_command(['gunzip', '-f', local_file])

    # Store output of MISEQ_PIPELINE.py in a log + poll continuously to display output to console
    pipeline_log_path = home + run_name + '/MISEQ_PIPELINE_OUTPUT.log'
    with open(pipeline_log_path, "wb") as PIPELINE_log:

        # Standard out/error concatenates to the log
        command = ['python2.7', '-u', 'MISEQ_PIPELINE.py', home+run_name]
        p = subprocess.Popen(command, stdout = PIPELINE_log, stderr = PIPELINE_log)
        logger.info(" ".join(command))

        # Poll the log of MISEQ_PIPELINE.py and display output to console as it appears
        with open(pipeline_log_path, 'rb') as cursor:
            while True:
                if p.poll() == 0:
                    PIPELINE_log.flush()
                    sys.stdout.write(cursor.read())
                    break
                time.sleep(1)
                PIPELINE_log.flush()
                sys.stdout.write(cursor.read())

    # Determine output paths
    result_path = curr_run.replace(NEEDS_PROCESSING, 'Results')
    result_path_final = '{}/version_{}'.format(result_path, pipeline_version)
    log_path = '{}/logs'.format(result_path_final)
    coverage_maps_path = '{}/coverage_maps'.format(result_path_final)

    # Create sub-folders if needed
    for path in [result_path, result_path_final, log_path, coverage_maps_path]:
        if not os.path.exists(path): os.mkdir(path)

    # Post files to appropriate sub-folders
    logging.info("Posting results to {}".format(result_path))
    if mode == 'Amplicon':
        v3_path = '{}/v3_tropism'.format(result_path_final)
        if not os.path.exists(v3_path): os.mkdir(v3_path)
        post_files(glob(home + run_name + '/*.v3prot'), v3_path)

    post_files(glob(home + run_name + '/*.log'), log_path)
    post_files([x for x in glob(home + run_name + '/*.csv') if 'indel' not in x], result_path_final)
    post_files(glob(home + run_name + '/coverage_maps/*.png'), coverage_maps_path)

    # Close the log and copy it to rawdata
    logger.info("===== {} successfully completed! =====".format(run_name))
    logging.shutdown()
    execute_command(['rsync', '-a', log_file, '{}/{}'.format(log_path, os.path.basename(log_file))])
<commit_msg>re-enabled monitor on all runs<commit_after>"""
MISEQ_MONITOR.py
1) For runs flagged 'needsprocessing' that have not yet been processed, copy fastqs to local disk
2) Process the run by calling MISEQ_PIPELINE.py
3) Upload results to the network drive
"""

#pipeline_version = '5.3g-single-sample'
pipeline_version = '5.4'
import logging, miseq_logging, miseqUtils, os, subprocess, sys, time
from settings import *
from glob import glob

if sys.version_info[:2] != (2, 7):
    raise Exception("Python 2.7 not detected")

def mark_run_as_disabled(curr_run):
    open(curr_run.replace(NEEDS_PROCESSING, ERROR_PROCESSING), 'w').close()

def execute_command(command):
    logger.info(" ".join(command))
    stdout, stderr = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()
    if stderr != "": logging.warn(stderr)
    return stdout

def post_files(files, destination):
    for file in files:
        execute_command(['rsync', '-a', file, '{}/{}'.format(destination, os.path.basename(file))])

# Process runs flagged for processing not already processed by this version of the pipeline
while True:

    runs = glob(rawdata_mount + 'MiSeq/runs/*/{}'.format(NEEDS_PROCESSING))
    #runs = glob(rawdata_mount + 'MiSeq/runs/140127_M01841_0051_000000000-A64E4/{}'.format(NEEDS_PROCESSING))
    #runs += glob(rawdata_mount + 'MiSeq/runs/131119_M01841_0041_000000000-A5EPY/{}'.format(NEEDS_PROCESSING))
    #runs += glob(rawdata_mount + 'MiSeq/runs/131112_M01841_0040_000000000-A5F9H/{}'.format(NEEDS_PROCESSING))
    #runs += glob(rawdata_mount + 'MiSeq/runs/131101_M01841_0037_000000000-A5F9E/{}'.format(NEEDS_PROCESSING))
    #runs = glob(rawdata_mount + 'MiSeq/runs/131119_M01841_0041_000000000-A5EPY/{}'.format(NEEDS_PROCESSING))

    runs_needing_processing = []
    for run in runs:
        result_path = '{}/version_{}'.format(run.replace(NEEDS_PROCESSING, 'Results'), pipeline_version)

        if os.path.exists(run.replace(NEEDS_PROCESSING, ERROR_PROCESSING)):
            continue

        if not os.path.exists(result_path):
            runs_needing_processing.append(run)

    if not runs_needing_processing:
        logging.info('No runs need processing')
        time.sleep(delay)
        continue

    # Process most recently generated run and work backwards
    runs_needing_processing.sort(reverse=True)
    curr_run = runs_needing_processing[0]
    root = curr_run.replace(NEEDS_PROCESSING, '')
    run_name = root.split('/')[-2]

    # Make folder on the cluster for intermediary files
    if not os.path.exists(home+run_name):
        os.mkdir(home+run_name)

    # Record standard input / output of monitor
    log_file = home + run_name + '/MISEQ_MONITOR_OUTPUT.log'
    try:
        logger = miseq_logging.init_logging(log_file, file_log_level=logging.DEBUG, console_log_level=logging.INFO)
        logger.info('===== Processing {} with pipeline version {} ====='.format(root, pipeline_version))
    except Exception as e:
        raise Exception("Couldn't setup logging (init_logging() threw exception '{}') - HALTING NOW!".format(str(e)))

    # SampleSheet.csv needed to determine the mode and sample T-primer
    remote_file = curr_run.replace(NEEDS_PROCESSING, 'SampleSheet.csv')
    local_file = home + run_name + '/SampleSheet.csv'
    execute_command(['rsync', '-a', remote_file, local_file])

    # Delete raw tiffs from this run
    # logger.info("Deleting tiffs from {}Images/Focus/L001/*".format(root))
    #for tiff in glob("{}Images/Focus/L001/*/*.tif".format(root)):
    #    log.info("os.remove({})".format(tiff))
    #    os.remove(tiff)

    try:
        with open(local_file, 'rU') as sample_sheet:
            run_info = miseqUtils.sampleSheetParser(sample_sheet)
            mode = run_info['Description']
    except Exception as e:
        logger.error("Exception thrown while parsing sample sheet: '{}' - skipping run and marking with flag '{}'".format(str(e),ERROR_PROCESSING))
        mark_run_as_disabled(curr_run)
        continue

    if mode not in ['Nextera', 'Amplicon']:
        logger.error("{} not a valid mode: skipping run and marking with {}".format(mode, ERROR_PROCESSING))
        mark_run_as_disabled(curr_run)
        continue

    # Copy fastq.gz files to the cluster and unzip them
    for gz_file in glob(root+'Data/Intensities/BaseCalls/*.fastq.gz'):
        filename = os.path.basename(gz_file)

        # Report number of reads failing to demultiplex to the log
        if filename.startswith('Undetermined'):

            # Second file has the same number of lines - don't bother
            if filename.endswith('_L001_R2_001.fastq.gz'):
                continue

            # Do word count directly on stream redirected from gunzip
            p1 = subprocess.Popen(['gunzip', '-c', gz_file], stdout=subprocess.PIPE)
            p2 = subprocess.Popen(['wc', '-l'], stdin=p1.stdout, stdout=subprocess.PIPE)
            output = p2.communicate()[0]
            failed_demultiplexing = output.strip(' \n')
            logger.info("{} reads failed to demultiplex in {} (removing file)".format(failed_demultiplexing, filename))
            continue

        # If a local copy of the unzipped fastq exists, skip this step
        local_file = home + run_name + '/' + filename
        if os.path.exists(local_file.replace('.gz', '')):
            continue

        execute_command(['rsync', '-a', gz_file, local_file])
        execute_command(['gunzip', '-f', local_file])

    # Store output of MISEQ_PIPELINE.py in a log + poll continuously to display output to console
    pipeline_log_path = home + run_name + '/MISEQ_PIPELINE_OUTPUT.log'
    with open(pipeline_log_path, "wb") as PIPELINE_log:

        # Standard out/error concatenates to the log
        command = ['python2.7', '-u', 'MISEQ_PIPELINE.py', home+run_name]
        p = subprocess.Popen(command, stdout = PIPELINE_log, stderr = PIPELINE_log)
        logger.info(" ".join(command))

        # Poll the log of MISEQ_PIPELINE.py and display output to console as it appears
        with open(pipeline_log_path, 'rb') as cursor:
            while True:
                if p.poll() == 0:
                    PIPELINE_log.flush()
                    sys.stdout.write(cursor.read())
                    break
                time.sleep(1)
                PIPELINE_log.flush()
                sys.stdout.write(cursor.read())

    # Determine output paths
    result_path = curr_run.replace(NEEDS_PROCESSING, 'Results')
    result_path_final = '{}/version_{}'.format(result_path, pipeline_version)
    log_path = '{}/logs'.format(result_path_final)
    coverage_maps_path = '{}/coverage_maps'.format(result_path_final)

    # Create sub-folders if needed
    for path in [result_path, result_path_final, log_path, coverage_maps_path]:
        if not os.path.exists(path): os.mkdir(path)

    # Post files to appropriate sub-folders
    logging.info("Posting results to {}".format(result_path))
    if mode == 'Amplicon':
        v3_path = '{}/v3_tropism'.format(result_path_final)
        if not os.path.exists(v3_path): os.mkdir(v3_path)
        post_files(glob(home + run_name + '/*.v3prot'), v3_path)

    post_files(glob(home + run_name + '/*.log'), log_path)
    post_files([x for x in glob(home + run_name + '/*.csv') if 'indel' not in x], result_path_final)
    post_files(glob(home + run_name + '/coverage_maps/*.png'), coverage_maps_path)

    # Close the log and copy it to rawdata
    logger.info("===== {} successfully completed! =====".format(run_name))
    logging.shutdown()
    execute_command(['rsync', '-a', log_file, '{}/{}'.format(log_path, os.path.basename(log_file))])
<|endoftext|>