<commit_before>#Memory Management Simulation System
import sys

#PROCESS CLASS
class Process(object):
    """docstring for Process"""
    def __init__(self, processArgs):
        if len(processArgs) < 4:                        # Process wasn't provided at least a name, req frames
            print "Error Initializing Process!"         #   and one set of arrival and exit times
            print "Not Enough Arguments..."
            return

        self.name = processArgs[0]                      # Process Name
        self.reqMemFrames = int(processArgs[1])         # Process Size
        self.memLoc = (0,0)                             # location if allocated, else (0,0)
        self.arrivalTimes = []                          # list of arrival times, in order
        self.exitTimes = []                             # list of exit times, in order
        self.done = False                               # True if last arrival time has occurred (note still needs to exit and be removed)

        i = 0
        for x in processArgs[2:]:                   # List of arrival and exit times
            if i%2 == 0:
                self.arrivalTimes.append(int(x))
            else:
                self.exitTimes.append(int(x))
            i += 1

#MEMORY SIMULATION CLASS
class MainMemorySimulator(object):
    """docstring for MainMemory"""
    def __init__(self, processList):
        self.simTime = 0                    # Master Time Handler
        self.lastTime = 0                   # Time that last process exits
        self.processes = processList        # Master List of all Processes
        self.freeSpace = [(80,1600)]        # List of tuples of start and end of free space
        self.runningProcesses = []          # List of Processes currently running
        self.memFrames = ""                 # String Representation of Memory
        self.numMemFrames = 1600            # Total amount of space (memFrames size)
        self.numOpSysProc = 80              # Number of spaces in Memory Representation dedicated to op sys processes

        # Finding all processes that start at 0
        for p in processList:
            if p.arrivalTimes[0] == 0:
                (free_check, begin, end) = self.find_first_free_space( p.reqMemFrames )
                if free_check:
                    tSize = p.reqMemFrames + begin
                    p.memLoc = ( begin, tSize )
                    if tSize > 0:
                        self.freeSpace.append( (tSize,end) )
                    self.runningProcesses.append(p) # and adding them to runningProcesses list
                    self.free_sort()
                    self.remove_entry_time( p )
                else:
                    print "ERROR: OUT-OF-MEMORY"
                    sys.exit()

            for et in p.exitTimes:
                if et > self.lastTime:
                    self.lastTime = et

        self.printMemory()                          # Print the structure

    def getkey(self, tuple1):
        return tuple1[0]

    #resort freeSpace List to be in order after appending
    def free_sort(self):
        sorted( self.freeSpace, key=self.getkey )

    #finds first available free space
    def find_first_free_space( self, reqMemFrames ):
        for space in self.freeSpace:
            if ( space[1] - space[0] ) >= reqMemFrames:
                self.freeSpace.remove(space)
                return ( True, space[0], space[1] )
        return ( False, 0 , 0 )

    #scans through runnning processes and prints
    def printMemory(self):
        self.memFrames = ""                 #output string
        memFrameItr = self.numOpSysProc     # Master index handler for memory representation
        temp_processes = self.runningProcesses
        temp_space = self.freeSpace

        for i in range(0, self.numOpSysProc):   # Adding Operating System Frames to beginning
            self.memFrames += "#"

        while memFrameItr < self.numMemFrames:      # Fill in each spot of memory representation
            for p in temp_processes:
                if memFrameItr == p.memLoc[0]:
                    while memFrameItr < p.memLoc[1]:
                        self.memFrames += p.name
                        memFrameItr += 1
                    temp_processes.remove( p )
            for space in temp_space:
                if space[0] == memFrameItr:
                    while memFrameItr < space[1]:
                        self.memFrames += "."
                        memFrameItr += 1
                    temp_space.remove( space )

        #printing set string of memory from above in correct format
        print "Memory at time %d" %self.simTime
        i = 0
        printList = []
        while i < self.numMemFrames:
            if i + 79 < self.numMemFrames:
                printList.append(self.memFrames[i:i + 80])
            else:
                printList.append(self.memFrames[i:])
            i += 80

        for m in printList:
            print m

    def incrementTime(self):
        self.simTime += 1

    #run iterations
    def run(self, quiet_mode, alloc_method):
        while self.simTime <= self.lastTime:
            change = False
            t = 0
            if not quiet_mode:
                #get time, t, input from user
                pass

            #increment time
            self.incrementTime()

            #check for exiting processes + deallocate if found
            for p in self.processes:
                if p.exitTimes[0] == self.simTime:
                    self.deallocate(p)

            #check for entering processes + allocate if found
            for p in self.processes:
                if not p.done:
                    if p.arrivalTimes[0] == self.simTime:
                        self.select_n_cal( alloc_method, p )

            #prints at time requested or on every change for quiet_mode
            if quiet_mode and change:
                self.printMemory()
            elif self.simTime == t:
                self.printMemory()

    #defragment memory if free blocks don't allow allocation
    def defragment(self):
        pass

    #deallocates a proces from memory
    def deallocate(self, aProcess):
        self.remove_exit_time( aProcess )

    #selects the proper algorithm based on command line argument
    def select_n_cal(self, alloc_method, aProcess):
        if alloc_method == "first":
            self.exec_first( aProcess )
        elif alloc_method == "best":
            self.exec_best( aProcess )
        elif alloc_method == "next":
            self.exec_next( aProcess )
        elif alloc_method == "worst":
            self.exec_worst( aProcess )
        elif alloc_method == "noncontig":
            self.exec_noncontig( aProcess )
        fTime = aProcess.arrivalTimes[0]
        if fTime != self.remove_entry_time( aProcess ):
            print "ERROR: Removed Wrong arrival time from process " + aProcess.name
            sys.exit()

    #removes entry time from process list after process has been added to memory
    def remove_entry_time(self, aProcess):
        if len(aProcess.arrivalTimes) == 1:
            aProcess.done = True
        index = self.processes.index( aProcess )
        return self.processes[index].arrivalTimes.pop(0)

    #removes a exit time from process list after process leaves memory
    #if last exit time, it removes process from processes
    def remove_exit_time(self, aProcess):
        if aProcess.done == True:
            i = self.processes.index( aProcess )
            self.processes.pop( i )
        else:
            index = self.processes.index( aProcess )
            self.processes[index].exitTimes.pop(0)
            aProcess.memLoc = ( 0, 0 )

    #implements First-Fit Memory Allocation Algorithm
    def exec_first(self, aProcess):
        print "gets here at time " + str(self.simTime)
        pass

    #implements Best-Fit Memory Allocation Algorithm
    def exec_best(self, aProcess):
        pass

    #implements Next-Fit Memory Allocation Algorithm
    def exec_next(self, aProcess):
        pass

    #implements Worst-Fit Memory Allocation Algorithm
    def exec_worst(self, aProcess):
        pass

    #implements Non-Contiguous Memory Allocation Algorithm
    def exec_noncontig(self, aProcess):
        pass

########################    END CLASS   ###########################################################

def check_filename(file_name):
    if file_name[len(file_name) - 4:] == ".txt":
        return True
    return False

def check_allocmethod(alloc_method):
    possible = ["first", "best", "next", "worst", "noncontig"]
    for a in possible:
        if alloc_method == a:
            return True
    return False

def parse(argv):
    #memsim -q <input-file> { first | best | next | worst }
    quiet_mode = False
    file_name = ""
    alloc_method = ""
    if len(argv) == 4:
        if argv[1] == "-q":
            quiet_mode = True
        else:
            print "USAGE: memsim [-q] <input-file> { first | best | next | worst }"
            sys.exit()
        file_name = argv[2]
        alloc_method = argv[3]
    elif len(argv) == 3:
        file_name = argv[1]
        alloc_method = argv[2]
    else:
        print "USAGE: memsim [-q] <input-file> { first | best | next | worst }"
        sys.exit()
    if check_allocmethod(alloc_method) and check_filename(file_name):
        return (quiet_mode, file_name, alloc_method)
    else:
        print "USAGE: memsim [-q] <input-file> { first | best | next | worst }"
        sys.exit()

#############################     MAIN FUNCTION     #############################################

if __name__ == '__main__':
    #Some Pseudocode

    #parse cli arguments
    (quiet_mode, file_name, alloc_method) = parse(sys.argv)

    #Creating Process and populating list
    processList = []
    with open(file_name, "r") as in_file:
        for line in in_file:
            line = line.split()
            if len(line) > 1:
                processList.append(Process(line))

    #initializing MainMemorySimulator
    M = MainMemorySimulator(processList)

    #Run Iterations
    M.run( quiet_mode, alloc_method )<commit_msg>implemented exec_first, but need to fix problem with freeSpace list allocations disappearing<commit_after>#Memory Management Simulation System
import sys

#PROCESS CLASS
class Process(object):
    """docstring for Process"""
    def __init__(self, processArgs):
        if len(processArgs) < 4:                        # Process wasn't provided at least a name, req frames
            print "Error Initializing Process!"         #   and one set of arrival and exit times
            print "Not Enough Arguments..."
            return

        self.name = processArgs[0]                      # Process Name
        self.reqMemFrames = int(processArgs[1])         # Process Size
        self.memLoc = (0,0)                             # location if allocated, else (0,0)
        self.arrivalTimes = []                          # list of arrival times, in order
        self.exitTimes = []                             # list of exit times, in order
        self.done = False                               # True if last arrival time has occurred (note still needs to exit and be removed)

        i = 0
        for x in processArgs[2:]:                   # List of arrival and exit times
            if i%2 == 0:
                self.arrivalTimes.append(int(x))
            else:
                self.exitTimes.append(int(x))
            i += 1

#MEMORY SIMULATION CLASS
class MainMemorySimulator(object):
    """docstring for MainMemory"""
    def __init__(self, processList):
        self.simTime = 0                    # Master Time Handler
        self.lastTime = 0                   # Time that last process exits
        self.processes = processList        # Master List of all Processes
        self.freeSpace = [(80,1600)]        # List of tuples of start and end of free space
        self.runningProcesses = []          # List of Processes currently running
        self.memFrames = ""                 # String Representation of Memory
        self.numMemFrames = 1600            # Total amount of space (memFrames size)
        self.numOpSysProc = 80              # Number of spaces in Memory Representation dedicated to op sys processes
        self.change = False
        self.t = 0

        # Finding all processes that start at 0
        for p in processList:
            if p.arrivalTimes[0] == 0:
                (free_check, begin, end) = self.find_first_free_space( p.reqMemFrames )
                if free_check:
                    tSize = p.reqMemFrames + begin
                    p.memLoc = ( begin, tSize )
                    print self.freeSpace
                    if ( end - tSize ) > 0:
                        self.freeSpace.append( (tSize,end) )
                    print self.freeSpace
                    self.runningProcesses.append(p) # and adding them to runningProcesses list
                    self.free_sort()
                    self.remove_entry_time( p )
                else:
                    print "ERROR: OUT-OF-MEMORY"
                    sys.exit()

            for et in p.exitTimes:
                if et > self.lastTime:
                    self.lastTime = et

        self.printMemory()                          # Print the structure

    def getkey(self, tuple1):
        return tuple1[0]

    #resort freeSpace List to be in order after appending
    def free_sort(self):
        sorted( self.freeSpace, key=self.getkey )

    #finds first available free space
    def find_first_free_space( self, reqMemFrames ):
        print str(reqMemFrames) + " " + str(self.freeSpace)
        for space in self.freeSpace:
            if ( space[1] - space[0] ) >= reqMemFrames:
                self.freeSpace.remove(space)
                return ( True, space[0], space[1] )
        return ( False, 0 , 0 )

    #scans through runnning processes and prints
    def printMemory(self):
        self.memFrames = ""                 #output string
        memFrameItr = self.numOpSysProc     # Master index handler for memory representation
        temp_processes = self.runningProcesses
        temp_space = self.freeSpace

        for i in range(0, self.numOpSysProc):   # Adding Operating System Frames to beginning
            self.memFrames += "#"

        while memFrameItr < self.numMemFrames:      # Fill in each spot of memory representation
            for p in temp_processes:
                if memFrameItr == p.memLoc[0]:
                    while memFrameItr < p.memLoc[1]:
                        self.memFrames += p.name
                        memFrameItr += 1
                    temp_processes.remove( p )
            for space in temp_space:
                if space[0] == memFrameItr:
                    while memFrameItr < space[1]:
                        self.memFrames += "."
                        memFrameItr += 1
                    temp_space.remove( space )

        #printing set string of memory from above in correct format
        print "Memory at time %d" %self.simTime
        i = 0
        printList = []
        while i < self.numMemFrames:
            if i + 79 < self.numMemFrames:
                printList.append(self.memFrames[i:i + 80])
            else:
                printList.append(self.memFrames[i:])
            i += 80

        for m in printList:
            print m

    def incrementTime(self):
        self.simTime += 1

    #run iterations
    def run(self, quiet_mode, alloc_method):
        while self.simTime <= self.lastTime:
            if not quiet_mode:
                #get time, t, input from user
                pass

            #increment time
            self.incrementTime()

            #check for exiting processes + deallocate if found
            for p in self.processes:
                if p.exitTimes[0] == self.simTime:
                    self.deallocate(p)

            #check for entering processes + allocate if found
            for p in self.processes:
                if not p.done:
                    if p.arrivalTimes[0] == self.simTime:
                        self.select_n_cal( alloc_method, p )

            #prints at time requested or on every change for quiet_mode
            if quiet_mode and self.change:
                self.change = False
                self.printMemory()
            elif self.simTime == self.t:
                self.printMemory()

    #defragment memory if free blocks don't allow allocation
    def defragment(self):
        return False

    #deallocates a proces from memory
    def deallocate(self, aProcess):
        # self.remove_exit_time( aProcess )
        pass

    #selects the proper algorithm based on command line argument
    def select_n_cal(self, alloc_method, aProcess):
        if alloc_method == "first":
            self.exec_first( aProcess )
        elif alloc_method == "best":
            self.exec_best( aProcess )
        elif alloc_method == "next":
            self.exec_next( aProcess )
        elif alloc_method == "worst":
            self.exec_worst( aProcess )
        elif alloc_method == "noncontig":
            self.exec_noncontig( aProcess )
        fTime = aProcess.arrivalTimes[0]
        if fTime != self.remove_entry_time( aProcess ):
            print "ERROR: Removed Wrong arrival time from process " + aProcess.name
            sys.exit()
        else:
            self.change = True

    #removes entry time from process list after process has been added to memory
    def remove_entry_time(self, aProcess):
        if len(aProcess.arrivalTimes) == 1:
            aProcess.done = True
        index = self.processes.index( aProcess )
        return self.processes[index].arrivalTimes.pop(0)

    #removes a exit time from process list after process leaves memory
    #if last exit time, it removes process from processes
    def remove_exit_time(self, aProcess):
        if aProcess.done == True:
            i = self.processes.index( aProcess )
            self.processes.pop( i )
        else:
            index = self.processes.index( aProcess )
            self.processes[index].exitTimes.pop(0)
            aProcess.memLoc = ( 0, 0 )

    #implements First-Fit Memory Allocation Algorithm
    def exec_first(self, aProcess):
        print "gets here at time " + str(self.simTime)

        n = 0
        while n < 2:
            print "n is "+ str(n)
            (free_check, begin, end) = self.find_first_free_space( aProcess.reqMemFrames )
            if free_check:
                tSize = aProcess.reqMemFrames + begin
                aProcess.memLoc = ( begin, tSize )
                if ( end - tSize ) > 0:
                    self.freeSpace.append( (tSize,end) )
                self.runningProcesses.append( aProcess ) # and adding them to runningProcesses list
                self.free_sort()
                self.remove_entry_time( aProcess )
                n = 2
            elif n == 0:
                self.defragment()
                n += 1
            else:
                print "ERROR: OUT-OF-MEMORY"
                sys.exit()

    #implements Best-Fit Memory Allocation Algorithm
    def exec_best(self, aProcess):
        pass

    #implements Next-Fit Memory Allocation Algorithm
    def exec_next(self, aProcess):
        pass

    #implements Worst-Fit Memory Allocation Algorithm
    def exec_worst(self, aProcess):
        pass

    #implements Non-Contiguous Memory Allocation Algorithm
    def exec_noncontig(self, aProcess):
        pass

########################    END CLASS   ###########################################################

def check_filename(file_name):
    if file_name[len(file_name) - 4:] == ".txt":
        return True
    return False

def check_allocmethod(alloc_method):
    possible = ["first", "best", "next", "worst", "noncontig"]
    for a in possible:
        if alloc_method == a:
            return True
    return False

def parse(argv):
    #memsim -q <input-file> { first | best | next | worst }
    quiet_mode = False
    file_name = ""
    alloc_method = ""
    if len(argv) == 4:
        if argv[1] == "-q":
            quiet_mode = True
        else:
            print "USAGE: memsim [-q] <input-file> { first | best | next | worst }"
            sys.exit()
        file_name = argv[2]
        alloc_method = argv[3]
    elif len(argv) == 3:
        file_name = argv[1]
        alloc_method = argv[2]
    else:
        print "USAGE: memsim [-q] <input-file> { first | best | next | worst }"
        sys.exit()
    if check_allocmethod(alloc_method) and check_filename(file_name):
        return (quiet_mode, file_name, alloc_method)
    else:
        print "USAGE: memsim [-q] <input-file> { first | best | next | worst }"
        sys.exit()

#############################     MAIN FUNCTION     #############################################

if __name__ == '__main__':
    #Some Pseudocode

    #parse cli arguments
    (quiet_mode, file_name, alloc_method) = parse(sys.argv)

    #Creating Process and populating list
    processList = []
    with open(file_name, "r") as in_file:
        for line in in_file:
            line = line.split()
            if len(line) > 1:
                processList.append(Process(line))

    #initializing MainMemorySimulator
    M = MainMemorySimulator(processList)

    #Run Iterations
    M.run( quiet_mode, alloc_method )<|endoftext|>


<commit_before>"""
fs.zipfs
========

A FS object that represents the contents of a Zip file

"""

from fs.base import *
from fs.path import *
from fs.errors import *

from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED, BadZipfile, LargeZipFile
from memoryfs import MemoryFS

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

import tempfs

class ZipOpenError(CreateFailedError):
    """Thrown when the zip file could not be opened"""
    pass

class ZipNotFoundError(CreateFailedError):
    """Thrown when the requested zip file does not exist"""
    pass

class _TempWriteFile(object):

    """Proxies a file object and calls a callback when the file is closed."""

    def __init__(self, fs, filename, close_callback):
        self.fs = fs
        self.filename = filename
        self._file = self.fs.open(filename, 'w+')
        self.close_callback = close_callback

    def write(self, data):
        return self._file.write(data)

    def tell(self):
        return self._file.tell()

    def close(self):
        self._file.close()
        self.close_callback(self.filename)

class _ExceptionProxy(object):

    """A placeholder for an object that may no longer be used."""

    def __getattr__(self, name):
        raise ValueError("Zip file has been closed")

    def __setattr__(self, name, value):
        raise ValueError("Zip file has been closed")

    def __nonzero__(self):
        return False

class ZipFS(FS):

    """A FileSystem that represents a zip file."""

    def __init__(self, zip_file, mode="r", compression="deflated", allow_zip_64=False, encoding="CP437", thread_synchronize=True):
        """Create a FS that maps on to a zip file.

        :param zip_file: A (system) path, or a file-like object
        :param mode: Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending
        :param compression: Can be 'deflated' (default) to compress data or 'stored' to just store date
        :param allow_zip_64: -- Set to True to use zip files greater than 2 GB, default is False
        :param encoding: --  The encoding to use for unicode filenames
        :param thread_synchronize: -- Set to True (default) to enable thread-safety
        :raises ZipOpenError: Thrown when the zip file could not be opened
        :raises ZipNotFoundError: Thrown when the zip file does not exist (derived from ZipOpenError)

        """
        super(ZipFS, self).__init__(thread_synchronize=thread_synchronize)
        if compression == "deflated":
            compression_type = ZIP_DEFLATED
        elif compression == "stored":
            compression_type = ZIP_STORED
        else:
            raise ValueError("Compression should be 'deflated' (default) or 'stored'")

        if len(mode) > 1 or mode not in "rwa":
            raise ValueError("mode must be 'r', 'w' or 'a'")

        self.zip_mode = mode
        self.encoding = encoding
        try:
            self.zf = ZipFile(zip_file, mode, compression_type, allow_zip_64)
        except BadZipfile, bzf:
            raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                               details=bzf)
        except IOError, ioe:
            if str(ioe).startswith('[Errno 22] Invalid argument'):
                raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                                   details=bzf)
            raise ZipNotFoundError("Zip file not found (%s)" % str(zip_file),
                                  details=ioe)

        self.zip_path = str(zip_file)

        self.temp_fs = None
        if mode in 'wa':
            self.temp_fs = tempfs.TempFS()

        self._path_fs = MemoryFS()
        if mode in 'ra':
            self._parse_resource_list()

    def __str__(self):
        return "<ZipFS: %s>" % self.zip_path

    def __unicode__(self):
        return unicode(self.__str__())

    def _parse_resource_list(self):
        for path in self.zf.namelist():
            self._add_resource(path.decode(self.encoding))

    def _add_resource(self, path):
        if path.endswith('/'):
            path = path[:-1]
            if path:
                self._path_fs.makedir(path, recursive=True, allow_recreate=True)
        else:
            dirpath, filename = pathsplit(path)
            if dirpath:
                self._path_fs.makedir(dirpath, recursive=True, allow_recreate=True)
            f = self._path_fs.open(path, 'w')
            f.close()

    def close(self):
        """Finalizes the zip file so that it can be read.
        No further operations will work after this method is called."""

        if hasattr(self, 'zf') and self.zf:
            self.zf.close()
            self.zf = _ExceptionProxy()

    @synchronize
    def open(self, path, mode="r", **kwargs):
        path = normpath(relpath(path))

        if 'r' in mode:
            if self.zip_mode not in 'ra':
                raise OperationFailedError("open file", path=path, msg="Zip file must be opened for reading ('r') or appending ('a')")
            try:
                contents = self.zf.read(path.encode(self.encoding))
            except KeyError:
                raise ResourceNotFoundError(path)
            return StringIO(contents)

        if 'w' in mode:
            dirname, filename = pathsplit(path)
            if dirname:
                self.temp_fs.makedir(dirname, recursive=True, allow_recreate=True)

            self._add_resource(path)
            f = _TempWriteFile(self.temp_fs, path, self._on_write_close)

            return f

        raise ValueError("Mode must contain be 'r' or 'w'")

    @synchronize
    def getcontents(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(relpath(path))
        try:
            contents = self.zf.read(path.encode(self.encoding))
        except KeyError:
            raise ResourceNotFoundError(path)
        except RuntimeError:
            raise OperationFailedError("read file", path=path, msg="Zip file must be oppened with 'r' or 'a' to read")
        return contents

    @synchronize
    def _on_write_close(self, filename):
        sys_path = self.temp_fs.getsyspath(filename)
        self.zf.write(sys_path, filename.encode(self.encoding))

    def desc(self, path):
        if self.isdir(path):
            return "Dir in zip file: %s" % self.zip_path
        else:
            return "File in zip file: %s" % self.zip_path

    def isdir(self, path):
        return self._path_fs.isdir(path)

    def isfile(self, path):
        return self._path_fs.isfile(path)

    def exists(self, path):
        return self._path_fs.exists(path)

    @synchronize
    def makedir(self, dirname, recursive=False, allow_recreate=False):
        dirname = normpath(dirname)
        if self.zip_mode not in "wa":
            raise OperationFailedError("create directory", path=dirname, msg="Zip file must be opened for writing ('w') or appending ('a')")
        if not dirname.endswith('/'):
            dirname += '/'
        self._add_resource(dirname)

    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
        return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)

    @synchronize
    def getinfo(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(path).lstrip('/')
        try:
            zi = self.zf.getinfo(path.encode(self.encoding))
            zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
            for k, v in zinfo.iteritems():
                if callable(v):
                    zinfo[k] = v()
        except KeyError:
            zinfo = {'file_size':0}
        info = {'size' : zinfo['file_size'] }
        if 'date_time' in zinfo:
            info['created_time'] = datetime.datetime(*zinfo['date_time'])
        info.update(zinfo)
        return info
<commit_msg>Fixed stupid error<commit_after>"""
fs.zipfs
========

A FS object that represents the contents of a Zip file

"""

from fs.base import *
from fs.path import *
from fs.errors import *

from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED, BadZipfile, LargeZipFile
from memoryfs import MemoryFS

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

import tempfs

class ZipOpenError(CreateFailedError):
    """Thrown when the zip file could not be opened"""
    pass

class ZipNotFoundError(CreateFailedError):
    """Thrown when the requested zip file does not exist"""
    pass

class _TempWriteFile(object):

    """Proxies a file object and calls a callback when the file is closed."""

    def __init__(self, fs, filename, close_callback):
        self.fs = fs
        self.filename = filename
        self._file = self.fs.open(filename, 'w+')
        self.close_callback = close_callback

    def write(self, data):
        return self._file.write(data)

    def tell(self):
        return self._file.tell()

    def close(self):
        self._file.close()
        self.close_callback(self.filename)

class _ExceptionProxy(object):

    """A placeholder for an object that may no longer be used."""

    def __getattr__(self, name):
        raise ValueError("Zip file has been closed")

    def __setattr__(self, name, value):
        raise ValueError("Zip file has been closed")

    def __nonzero__(self):
        return False

class ZipFS(FS):

    """A FileSystem that represents a zip file."""

    def __init__(self, zip_file, mode="r", compression="deflated", allow_zip_64=False, encoding="CP437", thread_synchronize=True):
        """Create a FS that maps on to a zip file.

        :param zip_file: A (system) path, or a file-like object
        :param mode: Mode to open zip file: 'r' for reading, 'w' for writing or 'a' for appending
        :param compression: Can be 'deflated' (default) to compress data or 'stored' to just store date
        :param allow_zip_64: -- Set to True to use zip files greater than 2 GB, default is False
        :param encoding: --  The encoding to use for unicode filenames
        :param thread_synchronize: -- Set to True (default) to enable thread-safety
        :raises ZipOpenError: Thrown when the zip file could not be opened
        :raises ZipNotFoundError: Thrown when the zip file does not exist (derived from ZipOpenError)

        """
        super(ZipFS, self).__init__(thread_synchronize=thread_synchronize)
        if compression == "deflated":
            compression_type = ZIP_DEFLATED
        elif compression == "stored":
            compression_type = ZIP_STORED
        else:
            raise ValueError("Compression should be 'deflated' (default) or 'stored'")

        if len(mode) > 1 or mode not in "rwa":
            raise ValueError("mode must be 'r', 'w' or 'a'")

        self.zip_mode = mode
        self.encoding = encoding
        try:
            self.zf = ZipFile(zip_file, mode, compression_type, allow_zip_64)
        except BadZipfile, bzf:
            raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                               details=bzf)
        except IOError, ioe:
            if str(ioe).startswith('[Errno 22] Invalid argument'):
                raise ZipOpenError("Not a zip file or corrupt (%s)" % str(zip_file),
                                   details=ioe)
            raise ZipNotFoundError("Zip file not found (%s)" % str(zip_file),
                                  details=ioe)

        self.zip_path = str(zip_file)

        self.temp_fs = None
        if mode in 'wa':
            self.temp_fs = tempfs.TempFS()

        self._path_fs = MemoryFS()
        if mode in 'ra':
            self._parse_resource_list()

    def __str__(self):
        return "<ZipFS: %s>" % self.zip_path

    def __unicode__(self):
        return unicode(self.__str__())

    def _parse_resource_list(self):
        for path in self.zf.namelist():
            self._add_resource(path.decode(self.encoding))

    def _add_resource(self, path):
        if path.endswith('/'):
            path = path[:-1]
            if path:
                self._path_fs.makedir(path, recursive=True, allow_recreate=True)
        else:
            dirpath, filename = pathsplit(path)
            if dirpath:
                self._path_fs.makedir(dirpath, recursive=True, allow_recreate=True)
            f = self._path_fs.open(path, 'w')
            f.close()

    def close(self):
        """Finalizes the zip file so that it can be read.
        No further operations will work after this method is called."""

        if hasattr(self, 'zf') and self.zf:
            self.zf.close()
            self.zf = _ExceptionProxy()

    @synchronize
    def open(self, path, mode="r", **kwargs):
        path = normpath(relpath(path))

        if 'r' in mode:
            if self.zip_mode not in 'ra':
                raise OperationFailedError("open file", path=path, msg="Zip file must be opened for reading ('r') or appending ('a')")
            try:
                contents = self.zf.read(path.encode(self.encoding))
            except KeyError:
                raise ResourceNotFoundError(path)
            return StringIO(contents)

        if 'w' in mode:
            dirname, filename = pathsplit(path)
            if dirname:
                self.temp_fs.makedir(dirname, recursive=True, allow_recreate=True)

            self._add_resource(path)
            f = _TempWriteFile(self.temp_fs, path, self._on_write_close)

            return f

        raise ValueError("Mode must contain be 'r' or 'w'")

    @synchronize
    def getcontents(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(relpath(path))
        try:
            contents = self.zf.read(path.encode(self.encoding))
        except KeyError:
            raise ResourceNotFoundError(path)
        except RuntimeError:
            raise OperationFailedError("read file", path=path, msg="Zip file must be oppened with 'r' or 'a' to read")
        return contents

    @synchronize
    def _on_write_close(self, filename):
        sys_path = self.temp_fs.getsyspath(filename)
        self.zf.write(sys_path, filename.encode(self.encoding))

    def desc(self, path):
        if self.isdir(path):
            return "Dir in zip file: %s" % self.zip_path
        else:
            return "File in zip file: %s" % self.zip_path

    def isdir(self, path):
        return self._path_fs.isdir(path)

    def isfile(self, path):
        return self._path_fs.isfile(path)

    def exists(self, path):
        return self._path_fs.exists(path)

    @synchronize
    def makedir(self, dirname, recursive=False, allow_recreate=False):
        dirname = normpath(dirname)
        if self.zip_mode not in "wa":
            raise OperationFailedError("create directory", path=dirname, msg="Zip file must be opened for writing ('w') or appending ('a')")
        if not dirname.endswith('/'):
            dirname += '/'
        self._add_resource(dirname)

    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
        return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)

    @synchronize
    def getinfo(self, path):
        if not self.exists(path):
            raise ResourceNotFoundError(path)
        path = normpath(path).lstrip('/')
        try:
            zi = self.zf.getinfo(path.encode(self.encoding))
            zinfo = dict((attrib, getattr(zi, attrib)) for attrib in dir(zi) if not attrib.startswith('_'))
            for k, v in zinfo.iteritems():
                if callable(v):
                    zinfo[k] = v()
        except KeyError:
            zinfo = {'file_size':0}
        info = {'size' : zinfo['file_size'] }
        if 'date_time' in zinfo:
            info['created_time'] = datetime.datetime(*zinfo['date_time'])
        info.update(zinfo)
        return info
<|endoftext|>


<commit_before>import sublime
import subprocess, re, os, threading, tempfile
from subprocess import Popen, PIPE

try:
	STARTUP_INFO = subprocess.STARTUPINFO()
	STARTUP_INFO.dwFlags |= subprocess.STARTF_USESHOWWINDOW
	STARTUP_INFO.wShowWindow = subprocess.SW_HIDE
except (AttributeError):
	STARTUP_INFO = None

_sem = threading.Semaphore()
_settings = {
	"env": {},
	"gscomplete_enabled": False,
	"gocode_cmd": "",
	"fmt_enabled": False,
	"fmt_tab_indent": True,
	"fmt_tab_width": 8,
	"gslint_enabled": False,
	"gslint_timeout": 0,
	"autocomplete_snippets": False,
	"autocomplete_tests": False,
	"margo_cmd": [],
	"margo_addr": ""
}

GLOBAL_SNIPPET_PACKAGE = [
	(u'package\tpackage [name] \u0282', 'package ${1:NAME}'),
	(u'package main\tpackage main \u0282', 'package main\n\nfunc main() {\n\t$0\n}\n')
]
GLOBAL_SNIPPET_IMPORT = (u'import\timport (...) \u0282', 'import (\n\t"$1"\n)')
GLOBAL_SNIPPETS = [
	GLOBAL_SNIPPET_IMPORT,
	(u'func\tfunc {...} \u0282', 'func ${1:name}($2)$3 {\n\t$0\n}'),
	(u'func\tfunc ([receiver]) {...} \u0282', 'func (${1:receiver} ${2:type}) ${3:name}($4)$5 {\n\t$0\n}'),
	(u'func main\tfunc main {...} \u0282', 'func main() {\n\t$0\n}\n'),
	(u'var\tvar (...) \u0282', 'var (\n\t$1\n)'),
	(u'const\tconst (...) \u0282', 'const (\n\t$1\n)'),
]

LOCAL_SNIPPETS = [
	(u'func\tfunc{...}() \u0282', 'func($1) {\n\t$0\n}($2)'),
	(u'var\tvar [name] [type] \u0282', 'var ${1:name} ${2:type}'),
]

CLASS_PREFIXES = {
	'const': u'\u0196',
	'func': u'\u0192',
	'type': u'\u0288',
	'var':  u'\u03BD',
	'package': u'package \u03C1',
}

NAME_PREFIXES = {
	'interface': u'\u00A1',
}

GOARCHES = [
	'386',
	'amd64',
	'arm',
]

GOOSES = [
	'darwin',
	'freebsd',
	'linux',
	'netbsd',
	'openbsd',
	'plan9',
	'windows',
]

GOOSARCHES = []
for s in GOOSES:
	for arch in GOARCHES:
		GOOSARCHES.append('%s_%s' % (s, arch))

GOOSARCHES_PAT = re.compile(r'^(.+?)(?:_(%s))?(?:_(%s))?\.go$' % ('|'.join(GOOSES), '|'.join(GOARCHES)))

IGNORED_SCOPES = frozenset([
	'string.quoted.double.go',
	'string.quoted.single.go',
	'string.quoted.raw.go',
	'comment.line.double-slash.go',
	'comment.block.go'
])

def temp_file(suffix='', prefix='', delete=True):
	tmpdir = os.path.join(tempfile.gettempdir(), 'GoSublime')
	try:
		os.mkdir(tmpdir)
	except Exception as ex:
		pass
	try:
		f = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=tmpdir, delete=delete)
	except Exception as ex:
		return (None, 'Error: %s' % ex)
	return (f, '')

def basedir_or_cwd(fn):
	if fn:
		return os.path.dirname(fn)
	return os.getcwd()

def runcmd(args, input=None, stdout=PIPE, stderr=PIPE, shell=False):
	out = ""
	err = ""
	exc = None

	old_env = os.environ.copy()
	os.environ.update(env())
	try:
		p = Popen(args, stdout=stdout, stderr=stderr, stdin=PIPE,
			startupinfo=STARTUP_INFO, shell=shell)
		if isinstance(input, unicode):
			input = input.encode('utf-8')
		out, err = p.communicate(input=input)
		out = out.decode('utf-8') if out else ''
		err = err.decode('utf-8') if err else ''
	except (Exception) as e:
		err = u'Error while running %s: %s' % (args[0], e)
		exc = e
	os.environ.update(old_env)
	return (out, err, exc)

def settings_obj():
	return sublime.load_settings("GoSublime.sublime-settings")

def setting(key, default=None):
	with _sem:
		return _settings.get(key, default)

def notice(domain, txt):
	txt = "** %s: %s **" % (domain, txt)
	print(txt)
	sublime.set_timeout(lambda: sublime.status_message(txt), 0)

def notice_undo(domain, txt, view, should_undo):
	def cb():
		if should_undo:
			view.run_command('undo')
		notice(domain, txt)
	sublime.set_timeout(cb, 0)

def show_output(panel_name, s, print_output=True, syntax_file=''):
	def cb(panel_name, s, print_output, win):
		if print_output:
			print('%s output: %s' % (panel_name, s))

		win = sublime.active_window()
		if win:
			panel = win.get_output_panel(panel_name)
			edit = panel.begin_edit()
			try:
				panel.set_read_only(False)
				panel.sel().clear()
				panel.settings().set("rulers", [])
				panel.replace(edit, sublime.Region(0, panel.size()), s)
				if syntax_file:
					panel.set_syntax_file(syntax_file)
				panel.set_read_only(True)
			finally:
				panel.end_edit(edit)
			win.run_command("show_panel", {"panel": "output.%s" % panel_name})
	sublime.set_timeout(lambda: cb(panel_name, s, print_output, syntax_file), 0)

def is_go_source_view(view=None, strict=True):
	if not view:
		return False

	if strict:
		return view.score_selector(view.sel()[0].begin(), 'source.go') > 0

	# todo: check the directory tree as well
	fn = view.file_name() or ''
	return fn.lower().endswith('.go')

def active_valid_go_view(win=None, strict=True):
	if not win:
		win = sublime.active_window()
	if win:
		view = win.active_view()
		if view and is_go_source_view(view, strict):
			return view
	return None

def rowcol(view):
	return view.rowcol(view.sel()[0].begin())

def env():
	e = os.environ.copy()
	e.update(setting('env', {}))
	roots = e.get('GOPATH', '').split(os.pathsep)
	roots.append(e.get('GOROOT', ''))
	add_path = e.get('PATH', '').split(os.pathsep)
	for s in roots:
		if s:
			s = os.path.join(s, 'bin')
			if s not in add_path:
				add_path.append(s)
	e['PATH'] = os.pathsep.join(add_path)
	return e

def sync_settings():
	global _settings
	so = settings_obj()
	with _sem:
		for k in _settings:
			v = so.get(k, None)
			if v is not None:
				# todo: check the type of `v`
				_settings[k] = v

		e = _settings.get('env', {})
		vfn = ''
		win = sublime.active_window()
		if win:
			view = win.active_view()
			if view:
				vfn = view.file_name()
				psettings = view.settings().get('GoSublime')
				if psettings:
					for k in _settings:
						v = psettings.get(k, None)
						if v is not None and k != "env":
							_settings[k] = v
					penv = psettings.get('env')
					if penv:
						e.update(penv)

		vfn = basedir_or_cwd(vfn)
		comps = vfn.split(os.sep)
		gs_gopath = []
		for i, s in enumerate(comps):
			if s.lower() == "src":
				gs_gopath.append(os.sep.join(comps[:i]))
		gs_gopath.reverse()
		gs_gopath = str(os.pathsep.join(gs_gopath))

		for k in e:
			e[k] = e[k].replace('$GS_GOPATH', gs_gopath)
		for k in e:
			e[k] = str(os.path.expandvars(os.path.expanduser(e[k])))

		_settings['env'] = e

settings_obj().clear_on_change("GoSublime.settings")
settings_obj().add_on_change("GoSublime.settings", sync_settings)
sync_settings()
<commit_msg>- add special-case for go code folding in go files<commit_after>import sublime
import subprocess, re, os, threading, tempfile
from subprocess import Popen, PIPE

try:
	STARTUP_INFO = subprocess.STARTUPINFO()
	STARTUP_INFO.dwFlags |= subprocess.STARTF_USESHOWWINDOW
	STARTUP_INFO.wShowWindow = subprocess.SW_HIDE
except (AttributeError):
	STARTUP_INFO = None

_sem = threading.Semaphore()
_settings = {
	"env": {},
	"gscomplete_enabled": False,
	"gocode_cmd": "",
	"fmt_enabled": False,
	"fmt_tab_indent": True,
	"fmt_tab_width": 8,
	"gslint_enabled": False,
	"gslint_timeout": 0,
	"autocomplete_snippets": False,
	"autocomplete_tests": False,
	"margo_cmd": [],
	"margo_addr": ""
}

GLOBAL_SNIPPET_PACKAGE = [
	(u'package\tpackage [name] \u0282', 'package ${1:NAME}'),
	(u'package main\tpackage main \u0282', 'package main\n\nfunc main() {\n\t$0\n}\n')
]
GLOBAL_SNIPPET_IMPORT = (u'import\timport (...) \u0282', 'import (\n\t"$1"\n)')
GLOBAL_SNIPPETS = [
	GLOBAL_SNIPPET_IMPORT,
	(u'func\tfunc {...} \u0282', 'func ${1:name}($2)$3 {\n\t$0\n}'),
	(u'func\tfunc ([receiver]) {...} \u0282', 'func (${1:receiver} ${2:type}) ${3:name}($4)$5 {\n\t$0\n}'),
	(u'func main\tfunc main {...} \u0282', 'func main() {\n\t$0\n}\n'),
	(u'var\tvar (...) \u0282', 'var (\n\t$1\n)'),
	(u'const\tconst (...) \u0282', 'const (\n\t$1\n)'),
]

LOCAL_SNIPPETS = [
	(u'func\tfunc{...}() \u0282', 'func($1) {\n\t$0\n}($2)'),
	(u'var\tvar [name] [type] \u0282', 'var ${1:name} ${2:type}'),
]

CLASS_PREFIXES = {
	'const': u'\u0196',
	'func': u'\u0192',
	'type': u'\u0288',
	'var':  u'\u03BD',
	'package': u'package \u03C1',
}

NAME_PREFIXES = {
	'interface': u'\u00A1',
}

GOARCHES = [
	'386',
	'amd64',
	'arm',
]

GOOSES = [
	'darwin',
	'freebsd',
	'linux',
	'netbsd',
	'openbsd',
	'plan9',
	'windows',
]

GOOSARCHES = []
for s in GOOSES:
	for arch in GOARCHES:
		GOOSARCHES.append('%s_%s' % (s, arch))

GOOSARCHES_PAT = re.compile(r'^(.+?)(?:_(%s))?(?:_(%s))?\.go$' % ('|'.join(GOOSES), '|'.join(GOARCHES)))

IGNORED_SCOPES = frozenset([
	'string.quoted.double.go',
	'string.quoted.single.go',
	'string.quoted.raw.go',
	'comment.line.double-slash.go',
	'comment.block.go'
])

def temp_file(suffix='', prefix='', delete=True):
	tmpdir = os.path.join(tempfile.gettempdir(), 'GoSublime')
	try:
		os.mkdir(tmpdir)
	except Exception as ex:
		pass
	try:
		f = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=tmpdir, delete=delete)
	except Exception as ex:
		return (None, 'Error: %s' % ex)
	return (f, '')

def basedir_or_cwd(fn):
	if fn:
		return os.path.dirname(fn)
	return os.getcwd()

def runcmd(args, input=None, stdout=PIPE, stderr=PIPE, shell=False):
	out = ""
	err = ""
	exc = None

	old_env = os.environ.copy()
	os.environ.update(env())
	try:
		p = Popen(args, stdout=stdout, stderr=stderr, stdin=PIPE,
			startupinfo=STARTUP_INFO, shell=shell)
		if isinstance(input, unicode):
			input = input.encode('utf-8')
		out, err = p.communicate(input=input)
		out = out.decode('utf-8') if out else ''
		err = err.decode('utf-8') if err else ''
	except (Exception) as e:
		err = u'Error while running %s: %s' % (args[0], e)
		exc = e
	os.environ.update(old_env)
	return (out, err, exc)

def settings_obj():
	return sublime.load_settings("GoSublime.sublime-settings")

def setting(key, default=None):
	with _sem:
		return _settings.get(key, default)

def notice(domain, txt):
	txt = "** %s: %s **" % (domain, txt)
	print(txt)
	sublime.set_timeout(lambda: sublime.status_message(txt), 0)

def notice_undo(domain, txt, view, should_undo):
	def cb():
		if should_undo:
			view.run_command('undo')
		notice(domain, txt)
	sublime.set_timeout(cb, 0)

def show_output(panel_name, s, print_output=True, syntax_file=''):
	def cb(panel_name, s, print_output, win):
		if print_output:
			print('%s output: %s' % (panel_name, s))

		win = sublime.active_window()
		if win:
			panel = win.get_output_panel(panel_name)
			edit = panel.begin_edit()
			try:
				panel.set_read_only(False)
				panel.sel().clear()
				panel.settings().set("rulers", [])
				panel.replace(edit, sublime.Region(0, panel.size()), s)
				if syntax_file:
					if syntax_file == 'go':
						panel.set_syntax_file('Packages/Go/Go.tmLanguage')
						func_regions = panel.find_by_selector('source.go meta.function.plain.go meta.block.go')
						if func_regions:
							panel.fold(func_regions)
					else:
						panel.set_syntax_file(syntax_file)
				panel.set_read_only(True)
			finally:
				panel.end_edit(edit)
			win.run_command("show_panel", {"panel": "output.%s" % panel_name})
	sublime.set_timeout(lambda: cb(panel_name, s, print_output, syntax_file), 0)

def is_go_source_view(view=None, strict=True):
	if not view:
		return False

	if strict:
		return view.score_selector(view.sel()[0].begin(), 'source.go') > 0

	# todo: check the directory tree as well
	fn = view.file_name() or ''
	return fn.lower().endswith('.go')

def active_valid_go_view(win=None, strict=True):
	if not win:
		win = sublime.active_window()
	if win:
		view = win.active_view()
		if view and is_go_source_view(view, strict):
			return view
	return None

def rowcol(view):
	return view.rowcol(view.sel()[0].begin())

def env():
	e = os.environ.copy()
	e.update(setting('env', {}))
	roots = e.get('GOPATH', '').split(os.pathsep)
	roots.append(e.get('GOROOT', ''))
	add_path = e.get('PATH', '').split(os.pathsep)
	for s in roots:
		if s:
			s = os.path.join(s, 'bin')
			if s not in add_path:
				add_path.append(s)
	e['PATH'] = os.pathsep.join(add_path)
	return e

def sync_settings():
	global _settings
	so = settings_obj()
	with _sem:
		for k in _settings:
			v = so.get(k, None)
			if v is not None:
				# todo: check the type of `v`
				_settings[k] = v

		e = _settings.get('env', {})
		vfn = ''
		win = sublime.active_window()
		if win:
			view = win.active_view()
			if view:
				vfn = view.file_name()
				psettings = view.settings().get('GoSublime')
				if psettings:
					for k in _settings:
						v = psettings.get(k, None)
						if v is not None and k != "env":
							_settings[k] = v
					penv = psettings.get('env')
					if penv:
						e.update(penv)

		vfn = basedir_or_cwd(vfn)
		comps = vfn.split(os.sep)
		gs_gopath = []
		for i, s in enumerate(comps):
			if s.lower() == "src":
				gs_gopath.append(os.sep.join(comps[:i]))
		gs_gopath.reverse()
		gs_gopath = str(os.pathsep.join(gs_gopath))

		for k in e:
			e[k] = e[k].replace('$GS_GOPATH', gs_gopath)
		for k in e:
			e[k] = str(os.path.expandvars(os.path.expanduser(e[k])))

		_settings['env'] = e

settings_obj().clear_on_change("GoSublime.settings")
settings_obj().add_on_change("GoSublime.settings", sync_settings)
sync_settings()
<|endoftext|>


<commit_before>""" This module contains class definitions for:

    * VistrailModuleType
    * PipelineElementType
    * VisPort
    * ModuleParam
    * ModuleFunction
"""

from core.utils import enum
import copy
import __builtin__
import core.modules.module_registry
import core.modules.vistrails_module
from core.utils import VistrailsInternalError, all, eprint

if __name__ == '__main__':
    import qt
    global app
    app = qt.createBogusQtApp()

VistrailModuleType = enum('VistrailModuleType',
                          ['Invalid', 'Abstract', 'Filter', 'Object', 'Plugin', 'Module'])
PipelineElementType = enum('PipelineElementType',
                           ['Module', 'Connection', 'Function', 'Parameter'])
VisPortEndPoint = enum('VisportEndPoint',
                       ['Invalid', 'Source', 'Destination'])

class VisPort(object):
    """ A port denotes one endpoint of a VisConnection.

    self.spec: list of list of (module, str)"""

    def makeSpec(self, specStr, localRegistry=None, loose=True):
        """Parses a string representation of a port spec and returns the spec. Uses
own type to decide between source and destination ports."""
        if specStr[0] != '(' or specStr[-1] != ')':
            raise VistrailsInternalError("invalid port spec")
        specStr = specStr[1:-1]
        descriptor = core.modules.module_registry.registry.getDescriptorByName(self.moduleName)
        if localRegistry:
            localDescriptor = localRegistry.getDescriptorByName(self.moduleName)
        else:
            localDescriptor = None
        if self.endPoint == VisPortEndPoint.Source:
            ports = copy.copy(descriptor.outputPorts)
            if localDescriptor:
                ports.update(localDescriptor.outputPorts)
        elif self.endPoint == VisPortEndPoint.Destination:
            ports = copy.copy(descriptor.inputPorts)
            if localDescriptor:
                ports.update(localDescriptor.inputPorts)
        else:
            raise VistrailsInternalError("Invalid port endpoint")
        values = specStr.split(", ")
        if not ports.has_key(self.name):
            if loose:
                return [[(core.modules.module_registry.registry.getDescriptorByName(v).module,
                         '<no description>')
                        for v in values]]
            else:
                raise VistrailsInternalError("Port name is inexistent in ModuleDescriptor")
        specs = ports[self.name]
        for spec in specs:
            if all(zip(spec, values),
                   lambda ((klass, descr), name): issubclass(core.modules.module_registry.registry.getDescriptorByName(name).module, klass)):
                return [copy.copy(spec)]
        print self.moduleName
        print self.name
        print specStr
        print specs
        raise VistrailsInternalError("No port spec matches the given string")

    def getSignatures(self):
        """Returns a list of all accepted signatures of this port, by generating
a string representation of each port spec."""
        def getSig(spec):
            if type(spec) == __builtin__.list:
                return "(" + ", ".join([getSig(s) for s in spec]) + ")"
            assert type(spec == __builtin__.tuple)
            spec = spec[0]
            if issubclass(spec, core.modules.vistrails_module.Module):
                return spec.__name__
            raise VistrailsInternalError("getSig Can't handle type %s" % type(spec))
        return [getSig(spec) for spec in self.spec]

    def toolTip(self):
        """Generates an appropriate tooltip for the port."""
        def endPointType():
            d = {VisPortEndPoint.Invalid: "Invalid",
                 VisPortEndPoint.Source: "Output",
                 VisPortEndPoint.Destination: "Input"}
            return d[self.endPoint]
        return "%s port %s\n%s" % (endPointType(), self.name, "; ".join(self.getSignatures()))

    def __init__(self):
        self.endPoint = VisPortEndPoint.Invalid
        self.moduleId = 0
        self.connectionId = 0
        self.moduleName = ""
        self.name = ""
        self.type = VistrailModuleType.Module
        self.spec = None
        self.optional = False

    def __str__(self):
        """ Returns a string representation of a VisPort object \

        Returns
        -------
        - 'str'

        """
        return '<VisPort endPoint="%s" moduleId=%s connectionId=%s name=%s type=Module %s/>' % \
               (self.endPoint,
                self.moduleId,
                self.connectionId,
                self.name,
                self.spec)

class ModuleParam(object):
    __fields__ = ['type', 'strValue', 'name', 'minValue', 'maxValue', 'alias']
    """ Stores a parameter setting for a vistrail function """
    def __init__(self):
        self.type = ""
        self.strValue = ""
        self.name = ""
        self.alias = ""
        self.minValue = ""
        self.maxValue = ""
        self.evaluatedStrValue = ""

    def serialize(self, dom, element):
        """ Writes itself in XML """
        child = dom.createElement('param')
        child.setAttribute('name',self.name)
        ctype = dom.createElement('type')
        cval = dom.createElement('val')
        calias = dom.createElement('alias')
        ttype = dom.createTextNode(self.type)
        tval = dom.createTextNode(self.strValue)
        talias = dom.createTextNode(self.alias)
        child.appendchild(ctype)
        child.appendChild(cval)
        ctype.appendChild(ttype)
        cval.appendChild(tval)
        calias.appendChild(talias)
        element.appendChild(child)

    def value(self):
        """  """
        if self.strValue == "":
            self.strValue = ModuleParam.defaultValue[self.type][0]
            return ModuleParam.defaultValue[self.type][1]
        return ModuleParam.dispatchValue[self.type](self.strValue)

    dispatchValue = {'Float': float,
                     'Integer': int,
                     'String': str,
                     'Boolean': bool}

    defaultValue = {'Float': ("0", 0.0),
                    'Integer': ("0", 0),
                    'String': ("", ""),
                    'Boolean': ("False", "False")}

#     dispatchValue = {'float': float,
#                      'double': float,
#                      'int': int,
#                      'vtkIdType': int,
#                      'string': str,
#                      'str': str,
#                      'const char *': str,
#                      'const char*': str,
#                      'char *': str,
#                      'char*': str}

    def quoteValue(self):
        """ """
        return ModuleParam.dispatchQuoteValue[self.type](self.strValue)

    dispatchQuoteValue = {'float': str,
                          'double': str,
                          'int': str,
                          'vtkIdType': str,
                          'str': lambda x: "'" + str(x) + "'",
                          'string': lambda x: "'" + str(x) + "'",
                          'const char *': lambda x: "'" + str(x) + "'",
                          'const char*': lambda x: "'" + str(x) + "'",
                          'char *': lambda x: "'" + str(x) + "'",
                          'char*': lambda x: "'" + str(x) + "'"}

    def __str__(self):
        if self.minValue != "":
            f = (self.name, self.type, self.strValue, self.minValue, self.maxValue, self.alias)
            return "<<name='%s' type='%s' strValue='%s' minValue='%s' maxValue='%s' alias='%s'>>" % f
        else:
            return "<<name='%s' type='%s' strValue='%s' alias='%s'>>" % (self.name,
                                                                         self.type,
                                                                         self.strValue,
                                                                         self.alias)

class ModuleFunction(object):
    __fields__ = ['name', 'returnType', 'params']
    """ Stores a function from a vistrail module """

    @staticmethod
    def fromSpec(port, spec):

        def fromSourcePort():
            f = ModuleFunction()
            f.name = port.name
            f.returnType = spec[0].__name__
            return f

        def fromDestinationPort():
            f = ModuleFunction()
            f.name = port.name
            f.returnType = 'void'
            for specitem in spec:
                p = ModuleParam()
                p.type = specitem[0].__name__
                p.name = specitem[1]
                f.params.append(p)
            return f

        if port.endPoint == VisPortEndPoint.Source:
            return fromSourcePort()
        elif port.endPoint == VisPortEndPoint.Destination:
            return fromDestinationPort()
        else:
            raise VistrailsInternalError("Wasn't expecting an invalid endpoint")

#     @staticmethod
#     def fromSetterSignature(name,sig):
#         """fromSetterSignature(name,sig) -> ModuleFunction
# Creates a ModuleFunction from a vtk_rtti signature."""
#         params = sig[1]
#         f = ModuleFunction()
#         f.name = name
#         f.returnType = 'void'
#         if params == None:
#             return f
#         for param in params:
#             p = ModuleParam()
#             p.name = '(unnamed)'
#             if type(param) == type(()):
#                 p.type = param[0]
#                 p.minValue = param[1]
#                 p.maxValue = param[2]
#             else:
#                 p.type = param
#             f.params.append(p)
#         return f

#     @staticmethod
#     def fromStateSignature(name, state):
#         f = ModuleFunction()
#         f.name = 'Set' + name + 'To' + state
#         f.returnType = 'void'
#         return f

#     @staticmethod
#     def fromToggleSignatures(name):
#         f1 = ModuleFunction()
#         f1.name = 'Set' + name
#         f1.returnType = 'void'
#         p = ModuleParam()
#         p.type = 'int'
#         p.name = '(unnamed)'
#         f1.params.append(p)
#         f2 = ModuleFunction()
#         f2.name = name + 'On'
#         f2.returnType = 'void'
#         f3 = ModuleFunction()
#         f3.name = name + 'Off'
#         f3.returnType = 'void'
#         return [f1,f2,f3]

    def __init__(self):
        self.name = ""
	self.returnType = ""
	self.params = []

    def getNumParams(self):
        """ Returns the number of params
        Returns
        -------

        - 'int'
         the number of params

        """
        return len(self.params)

    def serialize(self, doc, element):
	""" Writes itself in XML """
	child = doc.createElement('function')
	child.setAttribute('name',self.name)
	child.setAttribute('returnType',self.type)
	for p in self.params:
		p.serialize(doc,child)
	element.appendChild(child)

    def getSignature(self):
        """ Returns its signature

        Returns
        -------

        - 'str'

        """
        result = self.returnType + "("
        for p in self.params:
            result = result + p.type + ", "
        if result.rfind(",") != -1:
            result = result[0:result.rfind(",")]
        else:
            result = result + " "
        result = result + ")"
        return result

    def __str__(self):
        return "<<name='%s' returnType='%s' params=%s>>" % (self.name,
                                                            self.returnType,
                                                            self.params)

    def stringAsUserSetter(self):
        s = "<<name='%s' params=" % self.name
        for p in self.params:
            s += ' ' + str(p)
        s += " >>"
        return s

    def __copy__(self):
        cp = ModuleFunction()
        cp.name = self.name
        cp.returnType = self.returnType
        cp.params = [copy.copy(p) for p in self.params]
        return cp

################################################################################

import unittest

if __name__ == '__main__':
    import core.modules.basic_modules
    import core.modules.module_registry

class TestVisTypes(unittest.TestCase):

    def testPort(self):
        x = VisPort()
        a = str(x)
        x.type = VistrailModuleType.Filter
        a = str(x)
        x.type = VistrailModuleType.Object
        a = str(x)

    def testPortSpec(self):
        descriptor = core.modules.module_registry.registry.getDescriptorByName('String')
        ports = core.modules.module_registry.registry.sourcePortsFromDescriptor(descriptor)
        assert all(ports, lambda x: x.moduleName == 'String')
        portRepr = 'value(String)'
        p = core.modules.module_registry.registry.portFromRepresentation('String', portRepr, VisPortEndPoint.Source)
        assert p.name == 'value'
        assert p.moduleName == 'String'

    def testPortSpec2(self):
        descriptor = core.modules.module_registry.registry.getDescriptorByName('String')
        ports = core.modules.module_registry.registry.sourcePortsFromDescriptor(descriptor)
        assert all(ports, lambda x: x.moduleName == 'String')
        portRepr = 'value(Float)'
        try:
            p = core.modules.module_registry.registry.portFromRepresentation('String', portRepr, VisPortEndPoint.Source)
            self.fail("Expected to fail - passed an incompatible spec representation")
        except VistrailsInternalError:
            pass

if __name__ == '__main__':
    unittest.main()
<commit_msg>split vis_types in port.py, module_param.py and module_function.py and moved them to core.vistrail/ <commit_after><|endoftext|>


<commit_before>from .jwt_manager import JWTManager
from .utils import create_access_token
from .utils import create_refresh_token
from .utils import current_user
from .utils import decode_token
from .utils import get_csrf_token
from .utils import get_current_user
from .utils import get_jti
from .utils import get_jwt
from .utils import get_jwt_header
from .utils import get_jwt_identity
from .utils import get_jwt_request_location
from .utils import get_unverified_jwt_headers
from .utils import set_access_cookies
from .utils import set_refresh_cookies
from .utils import unset_access_cookies
from .utils import unset_jwt_cookies
from .utils import unset_refresh_cookies
from .view_decorators import jwt_required
from .view_decorators import verify_jwt_in_request

__version__ = "4.2.0"
<commit_msg>Bump to 4.2.1<commit_after>from .jwt_manager import JWTManager
from .utils import create_access_token
from .utils import create_refresh_token
from .utils import current_user
from .utils import decode_token
from .utils import get_csrf_token
from .utils import get_current_user
from .utils import get_jti
from .utils import get_jwt
from .utils import get_jwt_header
from .utils import get_jwt_identity
from .utils import get_jwt_request_location
from .utils import get_unverified_jwt_headers
from .utils import set_access_cookies
from .utils import set_refresh_cookies
from .utils import unset_access_cookies
from .utils import unset_jwt_cookies
from .utils import unset_refresh_cookies
from .view_decorators import jwt_required
from .view_decorators import verify_jwt_in_request

__version__ = "4.2.1"
<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, absolute_import
import os
import shutil
import logging
import time

from flexget import plugin
from flexget.event import event
from flexget.utils.template import RenderError
from flexget.utils.pathscrub import pathscrub

def get_directory_size(directory):
    """
    :param directory: Path
    :return: Size in bytes (recursively)
    """
    dir_size = 0
    for (path, dirs, files) in os.walk(directory):
        for file in files:
            filename = os.path.join(path, file)
            dir_size += os.path.getsize(filename)
    return dir_size

class BaseFileOps(object):

    # Defined by subclasses
    log = None

    def on_task_output(self, task, config):
        if config is True:
            config = {}
        elif config is False:
            return

        sexts = []
        if 'along' in config:
            sexts = [('.' + s).replace('..', '.').lower() for s in config['along']]

        for entry in task.accepted:
            if not 'location' in entry:
                self.log.verbose('Cannot handle %s because it does not have the field location.' % entry['title'])
                continue

            # check location
            src = entry['location']
            src_isdir = os.path.isdir(src)
            try:
                if not os.path.exists(src):
                    raise Exception('does not exists (anymore).')
                if src_isdir:
                    if not config.get('allow_dir'):
                        raise Exception('is a directory.')
                elif not os.path.isfile(src):
                    raise Exception('is not a file.')
            except Exception as err:
                self.log.warning('Cannot handle %s because location `%s` %s' % (entry['title'], src, err))
                continue

            # search for namesakes
            siblings = []
            if not src_isdir and 'along' in config:
                src_file, src_ext = os.path.splitext(src)
                for ext in sexts:
                    if ext != src_ext.lower() and os.path.exists(src_file + ext):
                        siblings.append(src_file + ext)

            # execute action in subclasses
            self.handle_entry(task, config, entry, siblings)

    def clean_source(self, task, config, entry):
        min_size = entry.get('clean_source', config.get('clean_source', -1))
        if min_size < 0:
            return
        base_path = os.path.split(entry['location'])[0]
        if not os.path.isdir(base_path):
            self.log.warning('Cannot delete path `%s` because it does not exists (anymore).' % base_path)
            return
        dir_size = get_directory_size(base_path) / 1024 / 1024
        if dir_size >= min_size:
            self.log.info('Path `%s` left because it exceeds safety value set in clean_source option.' % base_path)
            return
        if task.options.test:
            self.log.info('Would delete `%s` and everything under it.' % base_path)
            return
        try:
            shutil.rmtree(base_path)
            self.log.info('Path `%s` has been deleted because was less than clean_source safe value.' % base_path)
        except Exception as err:
            self.log.warning('Unable to delete path `%s`: %s' % (base_path, err))

class DeleteFiles(BaseFileOps):
    """Delete all accepted files."""

    schema = {
        'oneOf': [
            {'type': 'boolean'},
            {
                'type': 'object',
                'properties': {
                    'allow_dir': {'type': 'boolean'},
                    'along': {'type': 'array', 'items': {'type': 'string'}},
                    'clean_source': {'type': 'number'}
                },
                'additionalProperties': False
            }
        ]
    }

    log = logging.getLogger('delete')

    def handle_entry(self, task, config, entry, siblings):
        src = entry['location']
        src_isdir = os.path.isdir(src)

        if task.options.test:
            if src_isdir:
                self.log.info('Would delete `%s` and all its content.' % src)
            else:
                self.log.info('Would delete `%s`' % src)
                for s in siblings:
                    self.log.info('Would also delete `%s`' % s)
            return

        try:
            if src_isdir:
                shutil.rmtree(src)
                self.log.info('`%s` and all its content has been deleted.' % src)
            else:
                os.remove(src)
                self.log.info('`%s` has been deleted.' % src)
        except Exception as err:
            entry.fail('delete error: %s' % err)
            return

        for s in siblings:
            try:
                os.remove(s)
                self.log.info('`%s` has been deleted as well.' % s)
            except Exception as err:
                # the target file has been successfully deleted, we cannot mark the entry as failed anymore.
                self.log.warning('Unable to delete `%s`: %s' % (s, err))

        if not src_isdir:
            self.clean_source(task, config, entry)

class TransformingOps(BaseFileOps):

    # Defined by subclasses
    move = None

    def handle_entry(self, task, config, entry, siblings):
        src = entry['location']
        src_isdir = os.path.isdir(src)
        src_path, src_name = os.path.split(src)

        # get proper value in order of: entry, config, above split
        dst_path = entry.get('path', config.get('to', src_path))
        dst_name = entry.get('filename', config.get('filename', src_name))

        try:
            dst_path = entry.render(dst_path)
        except RenderError as err:
            self.log.warning('Path value replacement `%s` failed for %s: %s' % (dst_path, entry['title'], err))
            return
        try:
            dst_name = entry.render(dst_name)
        except RenderError as err:
            self.log.warning('Filename value replacement `%s` failed for %s: %s' % (dst_name, entry['title'], err))
            return

        # Clean invalid characters with pathscrub plugin
        dst_path = os.path.expanduser(dst_path)
        dst_name = pathscrub(dst_name, filename=True)

        # Join path and filename
        dst = os.path.join(dst_path, dst_name)
        if dst == entry['location']:
            self.log.warning('Cannot handle %s because source and destination are the same.' % entry['title'])
            return

        if not os.path.exists(dst_path):
            if task.options.test:
                self.log.info('Would create `%s`' % dst_path)
            else:
                self.log.info('Creating destination directory `%s`' % dst_path)
                os.makedirs(dst_path)
        if not os.path.isdir(dst_path) and not task.options.test:
            self.log.warning('Cannot handle %s because destination `%s` is not a directory' % (entry['title'], dst_path))
            return

        # unpack_safety
        if config.get('unpack_safety', entry.get('unpack_safety', True)):
            count = 0
            while True:
                if count > 60 * 30:
                    entry.fail('The task has been waiting unpacking for 30 minutes')
                    return
                size = os.path.getsize(src)
                time.sleep(1)
                new_size = os.path.getsize(src)
                if size != new_size:
                    if not count % 10:
                        self.log.verbose('File `%s` is possibly being unpacked, waiting ...' % src_name)
                else:
                    break
                count += 1

        src_file, src_ext = os.path.splitext(src)
        dst_file, dst_ext = os.path.splitext(dst)

        # Check dst contains src_ext
        if dst_ext != src_ext:
            self.log.verbose('Adding extension `%s` to dst `%s`' % (src_ext, dst))
            dst += src_ext

        funct_name = 'move' if self.move else 'copy'
        funct_done = 'moved' if self.move else 'copied'

        if task.options.test:
            self.log.info('Would %s `%s` to `%s`' % (funct_name, src, dst))
            for s in siblings:
                # we cannot rely on splitext for extensions here (subtitles may have the language code)
                d = dst_file + s[len(src_file):]
                self.log.info('Would also %s `%s` to `%s`' % (funct_name, s, d))
        else:
            try:
                if self.move:
                    shutil.move(src, dst)
                elif src_isdir:
                    shutil.copytree(src, dst)
                else:
                    shutil.copy(src, dst)
                self.log.info('`%s` has been %s to `%s`' % (src, funct_done, dst))
            except Exception as err:
                entry.fail('%s error: %s' % (funct_name, err))
                return
            for s in siblings:
                # we cannot rely on splitext for extensions here (subtitles may have the language code)
                d = dst_file + s[len(src_file):]
                try:
                    if self.move:
                        shutil.move(s, d)
                    else:
                        shutil.copy(s, d)
                    self.log.info('`%s` has been %s to `%s` as well.' % (s, funct_done, d))
                except Exception as err:
                    # the target file has been successfully handled, we cannot mark the entry as failed anymore.
                    self.log.warning('Unable to %s `%s` to `%s`: %s' % (funct_name, s, d, err))

        entry['output'] = dst

        if self.move and not src_isdir:
            self.clean_source(task, config, entry)

class CopyFiles(TransformingOps):
    """Copy all accepted files."""

    schema = {
        'oneOf': [
            {'type': 'boolean'},
            {
                'type': 'object',
                'properties': {
                    'to': {'type': 'string', 'format': 'path'},
                    'filename': {'type': 'string'},
                    'allow_dir': {'type': 'boolean'},
                    'unpack_safety': {'type': 'boolean'},
                    'along': {'type': 'array', 'items': {'type': 'string'}}
                },
                'additionalProperties': False
            }
        ]
    }

    move = False
    log = logging.getLogger('copy')

class MoveFiles(TransformingOps):
    """Move all accepted files."""

    schema = {
        'oneOf': [
            {'type': 'boolean'},
            {
                'type': 'object',
                'properties': {
                    'to': {'type': 'string', 'format': 'path'},
                    'filename': {'type': 'string'},
                    'allow_dir': {'type': 'boolean'},
                    'unpack_safety': {'type': 'boolean'},
                    'along': {'type': 'array', 'items': {'type': 'string'}},
                    'clean_source': {'type': 'number'}
                },
                'additionalProperties': False
            }
        ]
    }

    move = True
    log = logging.getLogger('move')

@event('plugin.register')
def register_plugin():
    plugin.register(DeleteFiles, 'delete', api_ver=2)
    plugin.register(CopyFiles, 'copy', api_ver=2)
    plugin.register(MoveFiles, 'move', api_ver=2)
<commit_msg>Just a missing pathscrub<commit_after>from __future__ import unicode_literals, division, absolute_import
import os
import shutil
import logging
import time

from flexget import plugin
from flexget.event import event
from flexget.utils.template import RenderError
from flexget.utils.pathscrub import pathscrub

def get_directory_size(directory):
    """
    :param directory: Path
    :return: Size in bytes (recursively)
    """
    dir_size = 0
    for (path, dirs, files) in os.walk(directory):
        for file in files:
            filename = os.path.join(path, file)
            dir_size += os.path.getsize(filename)
    return dir_size

class BaseFileOps(object):

    # Defined by subclasses
    log = None

    def on_task_output(self, task, config):
        if config is True:
            config = {}
        elif config is False:
            return

        sexts = []
        if 'along' in config:
            sexts = [('.' + s).replace('..', '.').lower() for s in config['along']]

        for entry in task.accepted:
            if not 'location' in entry:
                self.log.verbose('Cannot handle %s because it does not have the field location.' % entry['title'])
                continue

            # check location
            src = entry['location']
            src_isdir = os.path.isdir(src)
            try:
                if not os.path.exists(src):
                    raise Exception('does not exists (anymore).')
                if src_isdir:
                    if not config.get('allow_dir'):
                        raise Exception('is a directory.')
                elif not os.path.isfile(src):
                    raise Exception('is not a file.')
            except Exception as err:
                self.log.warning('Cannot handle %s because location `%s` %s' % (entry['title'], src, err))
                continue

            # search for namesakes
            siblings = []
            if not src_isdir and 'along' in config:
                src_file, src_ext = os.path.splitext(src)
                for ext in sexts:
                    if ext != src_ext.lower() and os.path.exists(src_file + ext):
                        siblings.append(src_file + ext)

            # execute action in subclasses
            self.handle_entry(task, config, entry, siblings)

    def clean_source(self, task, config, entry):
        min_size = entry.get('clean_source', config.get('clean_source', -1))
        if min_size < 0:
            return
        base_path = os.path.split(entry['location'])[0]
        if not os.path.isdir(base_path):
            self.log.warning('Cannot delete path `%s` because it does not exists (anymore).' % base_path)
            return
        dir_size = get_directory_size(base_path) / 1024 / 1024
        if dir_size >= min_size:
            self.log.info('Path `%s` left because it exceeds safety value set in clean_source option.' % base_path)
            return
        if task.options.test:
            self.log.info('Would delete `%s` and everything under it.' % base_path)
            return
        try:
            shutil.rmtree(base_path)
            self.log.info('Path `%s` has been deleted because was less than clean_source safe value.' % base_path)
        except Exception as err:
            self.log.warning('Unable to delete path `%s`: %s' % (base_path, err))

class DeleteFiles(BaseFileOps):
    """Delete all accepted files."""

    schema = {
        'oneOf': [
            {'type': 'boolean'},
            {
                'type': 'object',
                'properties': {
                    'allow_dir': {'type': 'boolean'},
                    'along': {'type': 'array', 'items': {'type': 'string'}},
                    'clean_source': {'type': 'number'}
                },
                'additionalProperties': False
            }
        ]
    }

    log = logging.getLogger('delete')

    def handle_entry(self, task, config, entry, siblings):
        src = entry['location']
        src_isdir = os.path.isdir(src)

        if task.options.test:
            if src_isdir:
                self.log.info('Would delete `%s` and all its content.' % src)
            else:
                self.log.info('Would delete `%s`' % src)
                for s in siblings:
                    self.log.info('Would also delete `%s`' % s)
            return

        try:
            if src_isdir:
                shutil.rmtree(src)
                self.log.info('`%s` and all its content has been deleted.' % src)
            else:
                os.remove(src)
                self.log.info('`%s` has been deleted.' % src)
        except Exception as err:
            entry.fail('delete error: %s' % err)
            return

        for s in siblings:
            try:
                os.remove(s)
                self.log.info('`%s` has been deleted as well.' % s)
            except Exception as err:
                # the target file has been successfully deleted, we cannot mark the entry as failed anymore.
                self.log.warning('Unable to delete `%s`: %s' % (s, err))

        if not src_isdir:
            self.clean_source(task, config, entry)

class TransformingOps(BaseFileOps):

    # Defined by subclasses
    move = None

    def handle_entry(self, task, config, entry, siblings):
        src = entry['location']
        src_isdir = os.path.isdir(src)
        src_path, src_name = os.path.split(src)

        # get proper value in order of: entry, config, above split
        dst_path = entry.get('path', config.get('to', src_path))
        dst_name = entry.get('filename', config.get('filename', src_name))

        try:
            dst_path = entry.render(dst_path)
        except RenderError as err:
            self.log.warning('Path value replacement `%s` failed for %s: %s' % (dst_path, entry['title'], err))
            return
        try:
            dst_name = entry.render(dst_name)
        except RenderError as err:
            self.log.warning('Filename value replacement `%s` failed for %s: %s' % (dst_name, entry['title'], err))
            return

        # Clean invalid characters with pathscrub plugin
        dst_path = pathscrub(os.path.expanduser(dst_path))
        dst_name = pathscrub(dst_name, filename=True)

        # Join path and filename
        dst = os.path.join(dst_path, dst_name)
        if dst == entry['location']:
            self.log.warning('Cannot handle %s because source and destination are the same.' % entry['title'])
            return

        if not os.path.exists(dst_path):
            if task.options.test:
                self.log.info('Would create `%s`' % dst_path)
            else:
                self.log.info('Creating destination directory `%s`' % dst_path)
                os.makedirs(dst_path)
        if not os.path.isdir(dst_path) and not task.options.test:
            self.log.warning('Cannot handle %s because destination `%s` is not a directory' % (entry['title'], dst_path))
            return

        # unpack_safety
        if config.get('unpack_safety', entry.get('unpack_safety', True)):
            count = 0
            while True:
                if count > 60 * 30:
                    entry.fail('The task has been waiting unpacking for 30 minutes')
                    return
                size = os.path.getsize(src)
                time.sleep(1)
                new_size = os.path.getsize(src)
                if size != new_size:
                    if not count % 10:
                        self.log.verbose('File `%s` is possibly being unpacked, waiting ...' % src_name)
                else:
                    break
                count += 1

        src_file, src_ext = os.path.splitext(src)
        dst_file, dst_ext = os.path.splitext(dst)

        # Check dst contains src_ext
        if dst_ext != src_ext:
            self.log.verbose('Adding extension `%s` to dst `%s`' % (src_ext, dst))
            dst += src_ext

        funct_name = 'move' if self.move else 'copy'
        funct_done = 'moved' if self.move else 'copied'

        if task.options.test:
            self.log.info('Would %s `%s` to `%s`' % (funct_name, src, dst))
            for s in siblings:
                # we cannot rely on splitext for extensions here (subtitles may have the language code)
                d = dst_file + s[len(src_file):]
                self.log.info('Would also %s `%s` to `%s`' % (funct_name, s, d))
        else:
            try:
                if self.move:
                    shutil.move(src, dst)
                elif src_isdir:
                    shutil.copytree(src, dst)
                else:
                    shutil.copy(src, dst)
                self.log.info('`%s` has been %s to `%s`' % (src, funct_done, dst))
            except Exception as err:
                entry.fail('%s error: %s' % (funct_name, err))
                return
            for s in siblings:
                # we cannot rely on splitext for extensions here (subtitles may have the language code)
                d = dst_file + s[len(src_file):]
                try:
                    if self.move:
                        shutil.move(s, d)
                    else:
                        shutil.copy(s, d)
                    self.log.info('`%s` has been %s to `%s` as well.' % (s, funct_done, d))
                except Exception as err:
                    # the target file has been successfully handled, we cannot mark the entry as failed anymore.
                    self.log.warning('Unable to %s `%s` to `%s`: %s' % (funct_name, s, d, err))

        entry['output'] = dst

        if self.move and not src_isdir:
            self.clean_source(task, config, entry)

class CopyFiles(TransformingOps):
    """Copy all accepted files."""

    schema = {
        'oneOf': [
            {'type': 'boolean'},
            {
                'type': 'object',
                'properties': {
                    'to': {'type': 'string', 'format': 'path'},
                    'filename': {'type': 'string'},
                    'allow_dir': {'type': 'boolean'},
                    'unpack_safety': {'type': 'boolean'},
                    'along': {'type': 'array', 'items': {'type': 'string'}}
                },
                'additionalProperties': False
            }
        ]
    }

    move = False
    log = logging.getLogger('copy')

class MoveFiles(TransformingOps):
    """Move all accepted files."""

    schema = {
        'oneOf': [
            {'type': 'boolean'},
            {
                'type': 'object',
                'properties': {
                    'to': {'type': 'string', 'format': 'path'},
                    'filename': {'type': 'string'},
                    'allow_dir': {'type': 'boolean'},
                    'unpack_safety': {'type': 'boolean'},
                    'along': {'type': 'array', 'items': {'type': 'string'}},
                    'clean_source': {'type': 'number'}
                },
                'additionalProperties': False
            }
        ]
    }

    move = True
    log = logging.getLogger('move')

@event('plugin.register')
def register_plugin():
    plugin.register(DeleteFiles, 'delete', api_ver=2)
    plugin.register(CopyFiles, 'copy', api_ver=2)
    plugin.register(MoveFiles, 'move', api_ver=2)
<|endoftext|>