<commit_before>import sys, os
import netlib.utils
import flow, filt, utils

class DumpError(Exception): pass

class Options(object):
    attributes = [
        "app",
        "app_external",
        "app_host",
        "app_port",
        "anticache",
        "anticomp",
        "client_replay",
        "eventlog",
        "keepserving",
        "kill",
        "no_server",
        "nopop",
        "refresh_server_playback",
        "replacements",
        "rfile",
        "rheaders",
        "setheaders",
        "server_replay",
        "scripts",
        "showhost",
        "stickycookie",
        "stickyauth",
        "verbosity",
        "wfile",
    ]
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)
        for i in self.attributes:
            if not hasattr(self, i):
                setattr(self, i, None)

def str_response(resp):
    r = "%s %s"%(resp.code, resp.msg)
    if resp.is_replay():
        r = "[replay] " + r
    return r

def str_request(req, showhost):
    if req.client_conn:
        c = req.client_conn.address[0]
    else:
        c = "[replay]"
    r = "%s %s %s"%(c, req.method, req.get_url(showhost))
    if req.stickycookie:
        r = "[stickycookie] " + r
    return r

class DumpMaster(flow.FlowMaster):
    def __init__(self, server, options, filtstr, outfile=sys.stdout):
        flow.FlowMaster.__init__(self, server, flow.State())
        self.outfile = outfile
        self.o = options
        self.anticache = options.anticache
        self.anticomp = options.anticomp
        self.eventlog = options.eventlog
        self.showhost = options.showhost
        self.refresh_server_playback = options.refresh_server_playback

        if filtstr:
            self.filt = filt.parse(filtstr)
        else:
            self.filt = None

        if options.stickycookie:
            self.set_stickycookie(options.stickycookie)

        if options.stickyauth:
            self.set_stickyauth(options.stickyauth)

        if options.wfile:
            path = os.path.expanduser(options.wfile)
            try:
                f = file(path, "wb")
                self.start_stream(f, self.filt)
            except IOError, v:
                raise DumpError(v.strerror)

        if options.replacements:
            for i in options.replacements:
                self.replacehooks.add(*i)

        if options.setheaders:
            for i in options.setheaders:
                self.setheaders.add(*i)

        if options.server_replay:
            self.start_server_playback(
                self._readflow(options.server_replay),
                options.kill, options.rheaders,
                not options.keepserving,
                options.nopop
            )

        if options.client_replay:
            self.start_client_playback(
                self._readflow(options.client_replay),
                not options.keepserving
            )

        scripts = options.scripts or []
        for command in scripts:
            err = self.load_script(command)
            if err:
                raise DumpError(err)

        if options.rfile:
            path = os.path.expanduser(options.rfile)
            try:
                f = file(path, "rb")
                freader = flow.FlowReader(f)
            except IOError, v:
                raise DumpError(v.strerror)
            try:
                self.load_flows(freader)
            except flow.FlowReadError, v:
                self.add_event("Flow file corrupted. Stopped loading.")

        if self.o.app:
            self.start_app(self.o.app_host, self.o.app_port, self.o.app_external)

    def _readflow(self, path):
        path = os.path.expanduser(path)
        try:
            f = file(path, "rb")
            flows = list(flow.FlowReader(f).stream())
        except (IOError, flow.FlowReadError), v:
            raise DumpError(v.strerror)
        return flows

    def add_event(self, e, level="info"):
        if self.eventlog:
            print >> self.outfile, e
            self.outfile.flush()

    def indent(self, n, t):
        l = str(t).strip().split("\n")
        return "\n".join(" "*n + i for i in l)

    def _process_flow(self, f):
        self.state.delete_flow(f)
        if self.filt and not f.match(self.filt):
            return

        if f.response:
            sz = utils.pretty_size(len(f.response.content))
            if self.o.verbosity > 0:
                result = " << %s %s"%(str_response(f.response), sz)
            if self.o.verbosity > 1:
                result = result + "\n\n" + self.indent(4, f.response.headers)
            if self.o.verbosity > 2:
                if utils.isBin(f.response.content):
                    d = netlib.utils.hexdump(f.response.content)
                    d = "\n".join("%s\t%s %s"%i for i in d)
                    cont = self.indent(4, d)
                elif f.response.content:
                    cont = self.indent(4, f.response.content)
                else:
                    cont = ""
                result = result + "\n\n" + cont
        elif f.error:
            result = " << %s"%f.error.msg

        if self.o.verbosity == 1:
            print >> self.outfile, str_request(f.request, self.showhost)
            print >> self.outfile, result
        elif self.o.verbosity == 2:
            print >> self.outfile, str_request(f.request, self.showhost)
            print >> self.outfile, self.indent(4, f.request.headers)
            print >> self.outfile
            print >> self.outfile, result
            print >> self.outfile, "\n"
        elif self.o.verbosity >= 3:
            print >> self.outfile, str_request(f.request, self.showhost)
            print >> self.outfile, self.indent(4, f.request.headers)
            if utils.isBin(f.request.content):
                print >> self.outfile, self.indent(4, netlib.utils.hexdump(f.request.content))
            elif f.request.content:
                print >> self.outfile, self.indent(4, f.request.content)
            print >> self.outfile
            print >> self.outfile, result
            print >> self.outfile, "\n"
        if self.o.verbosity:
            self.outfile.flush()

    def handle_log(self, l):
        self.add_event(l.msg)
        l.reply()

    def handle_request(self, r):
        f = flow.FlowMaster.handle_request(self, r)
        if f:
            r.reply()
        return f

    def handle_response(self, msg):
        f = flow.FlowMaster.handle_response(self, msg)
        if f:
            msg.reply()
            self._process_flow(f)
        return f

    def handle_error(self, msg):
        f = flow.FlowMaster.handle_error(self, msg)
        if f:
            self._process_flow(f)
        return f

    def shutdown(self):  # pragma: no cover
        return flow.FlowMaster.shutdown(self)

    def run(self):  # pragma: no cover
        if self.o.rfile and not self.o.keepserving:
            for script in self.scripts:
                self.unload_script(script)
            return
        try:
            return flow.FlowMaster.run(self)
        except BaseException:
            self.shutdown()
            raise
<commit_msg>fixing bug: AttributeError: DumpMaster instance has no attribute 'unload_script'<commit_after>import sys, os
import netlib.utils
import flow, filt, utils

class DumpError(Exception): pass

class Options(object):
    attributes = [
        "app",
        "app_external",
        "app_host",
        "app_port",
        "anticache",
        "anticomp",
        "client_replay",
        "eventlog",
        "keepserving",
        "kill",
        "no_server",
        "nopop",
        "refresh_server_playback",
        "replacements",
        "rfile",
        "rheaders",
        "setheaders",
        "server_replay",
        "scripts",
        "showhost",
        "stickycookie",
        "stickyauth",
        "verbosity",
        "wfile",
    ]
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)
        for i in self.attributes:
            if not hasattr(self, i):
                setattr(self, i, None)

def str_response(resp):
    r = "%s %s"%(resp.code, resp.msg)
    if resp.is_replay():
        r = "[replay] " + r
    return r

def str_request(req, showhost):
    if req.client_conn:
        c = req.client_conn.address[0]
    else:
        c = "[replay]"
    r = "%s %s %s"%(c, req.method, req.get_url(showhost))
    if req.stickycookie:
        r = "[stickycookie] " + r
    return r

class DumpMaster(flow.FlowMaster):
    def __init__(self, server, options, filtstr, outfile=sys.stdout):
        flow.FlowMaster.__init__(self, server, flow.State())
        self.outfile = outfile
        self.o = options
        self.anticache = options.anticache
        self.anticomp = options.anticomp
        self.eventlog = options.eventlog
        self.showhost = options.showhost
        self.refresh_server_playback = options.refresh_server_playback

        if filtstr:
            self.filt = filt.parse(filtstr)
        else:
            self.filt = None

        if options.stickycookie:
            self.set_stickycookie(options.stickycookie)

        if options.stickyauth:
            self.set_stickyauth(options.stickyauth)

        if options.wfile:
            path = os.path.expanduser(options.wfile)
            try:
                f = file(path, "wb")
                self.start_stream(f, self.filt)
            except IOError, v:
                raise DumpError(v.strerror)

        if options.replacements:
            for i in options.replacements:
                self.replacehooks.add(*i)

        if options.setheaders:
            for i in options.setheaders:
                self.setheaders.add(*i)

        if options.server_replay:
            self.start_server_playback(
                self._readflow(options.server_replay),
                options.kill, options.rheaders,
                not options.keepserving,
                options.nopop
            )

        if options.client_replay:
            self.start_client_playback(
                self._readflow(options.client_replay),
                not options.keepserving
            )

        scripts = options.scripts or []
        for command in scripts:
            err = self.load_script(command)
            if err:
                raise DumpError(err)

        if options.rfile:
            path = os.path.expanduser(options.rfile)
            try:
                f = file(path, "rb")
                freader = flow.FlowReader(f)
            except IOError, v:
                raise DumpError(v.strerror)
            try:
                self.load_flows(freader)
            except flow.FlowReadError, v:
                self.add_event("Flow file corrupted. Stopped loading.")

        if self.o.app:
            self.start_app(self.o.app_host, self.o.app_port, self.o.app_external)

    def _readflow(self, path):
        path = os.path.expanduser(path)
        try:
            f = file(path, "rb")
            flows = list(flow.FlowReader(f).stream())
        except (IOError, flow.FlowReadError), v:
            raise DumpError(v.strerror)
        return flows

    def add_event(self, e, level="info"):
        if self.eventlog:
            print >> self.outfile, e
            self.outfile.flush()

    def indent(self, n, t):
        l = str(t).strip().split("\n")
        return "\n".join(" "*n + i for i in l)

    def _process_flow(self, f):
        self.state.delete_flow(f)
        if self.filt and not f.match(self.filt):
            return

        if f.response:
            sz = utils.pretty_size(len(f.response.content))
            if self.o.verbosity > 0:
                result = " << %s %s"%(str_response(f.response), sz)
            if self.o.verbosity > 1:
                result = result + "\n\n" + self.indent(4, f.response.headers)
            if self.o.verbosity > 2:
                if utils.isBin(f.response.content):
                    d = netlib.utils.hexdump(f.response.content)
                    d = "\n".join("%s\t%s %s"%i for i in d)
                    cont = self.indent(4, d)
                elif f.response.content:
                    cont = self.indent(4, f.response.content)
                else:
                    cont = ""
                result = result + "\n\n" + cont
        elif f.error:
            result = " << %s"%f.error.msg

        if self.o.verbosity == 1:
            print >> self.outfile, str_request(f.request, self.showhost)
            print >> self.outfile, result
        elif self.o.verbosity == 2:
            print >> self.outfile, str_request(f.request, self.showhost)
            print >> self.outfile, self.indent(4, f.request.headers)
            print >> self.outfile
            print >> self.outfile, result
            print >> self.outfile, "\n"
        elif self.o.verbosity >= 3:
            print >> self.outfile, str_request(f.request, self.showhost)
            print >> self.outfile, self.indent(4, f.request.headers)
            if utils.isBin(f.request.content):
                print >> self.outfile, self.indent(4, netlib.utils.hexdump(f.request.content))
            elif f.request.content:
                print >> self.outfile, self.indent(4, f.request.content)
            print >> self.outfile
            print >> self.outfile, result
            print >> self.outfile, "\n"
        if self.o.verbosity:
            self.outfile.flush()

    def handle_log(self, l):
        self.add_event(l.msg)
        l.reply()

    def handle_request(self, r):
        f = flow.FlowMaster.handle_request(self, r)
        if f:
            r.reply()
        return f

    def handle_response(self, msg):
        f = flow.FlowMaster.handle_response(self, msg)
        if f:
            msg.reply()
            self._process_flow(f)
        return f

    def handle_error(self, msg):
        f = flow.FlowMaster.handle_error(self, msg)
        if f:
            self._process_flow(f)
        return f

    def shutdown(self):  # pragma: no cover
        return flow.FlowMaster.shutdown(self)

    def run(self):  # pragma: no cover
        if self.o.rfile and not self.o.keepserving:
            self.unload_scripts()
            return
        try:
            return flow.FlowMaster.run(self)
        except BaseException:
            self.shutdown()
            raise
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
fs.memoryfs
===========

A Filesystem that exists in memory only.

File objects returned by MemoryFS.objects use StringIO objects for storage.

"""

import datetime
from fs.path import iteratepath
from fs.base import *
from fs import _thread_synchronize_default

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

def _check_mode(mode, mode_chars):
    for c in mode_chars:
        if c not in mode:
            return False
    return True

class MemoryFile(object):

    def __init__(self, path, memory_fs, value, mode):
        self.closed = False
        self.path = path
        self.memory_fs = memory_fs
        self.mode = mode

        self.mem_file = None

        if _check_mode(mode, 'wa'):
            self.mem_file = StringIO()
            self.mem_file.write(value)

        elif _check_mode(mode, 'w'):
            self.mem_file = StringIO()

        elif _check_mode(mode, 'ra'):
            self.mem_file = StringIO()
            self.mem_file.write(value)

        elif _check_mode(mode, 'r'):
            self.mem_file = StringIO(value)

        elif _check_mode(mode, "a"):
            self.mem_file = StringIO()
            self.mem_file.write(value)

        else:
            if value is not None:
                self.mem_file = StringIO(value)
            else:
                self.mem_file = StringIO()

        assert self.mem_file is not None, "self.mem_file should have a value"

    def __str__(self):
        return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)

    __repr__ = __str__

    def __unicode__(self):
        return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)

    def __del__(self):
        if not self.closed:
            self.close()

    def flush(self):
        value = self.mem_file.getvalue()
        self.memory_fs._on_flush_memory_file(self.path, value)

    def __iter__(self):
        return iter(self.mem_file)

    def next(self):
        return self.mem_file.next()

    def readline(self, *args, **kwargs):
        return self.mem_file.readline(*args, **kwargs)

    def close(self):
        if not self.closed and self.mem_file is not None:
            value = self.mem_file.getvalue()
            self.memory_fs._on_close_memory_file(self, self.path, value)
            self.mem_file.close()
            self.closed = True

    def read(self, size=None):
        if size is None:
            size = -1
        return self.mem_file.read(size)

    def seek(self, *args, **kwargs):
        return self.mem_file.seek(*args, **kwargs)

    def tell(self):
        return self.mem_file.tell()

    def truncate(self, *args, **kwargs):
        return self.mem_file.truncate(*args, **kwargs)

    def write(self, data):
        return self.mem_file.write(data)

    def writelines(self, *args, **kwargs):
        return self.mem_file.writelines(*args, **kwargs)

    def __enter__(self):
        return self

    def __exit__(self,exc_type,exc_value,traceback):
        self.close()
        return False

class DirEntry(object):

    def __init__(self, type, name, contents=None):

        assert type in ("dir", "file"), "Type must be dir or file!"

        self.type = type
        self.name = name

        if contents is None and type == "dir":
            contents = {}

        self.open_files = []
        self.contents = contents
        self.data = None
        self.locks = 0
        self.created_time = datetime.datetime.now()

    def lock(self):
        self.locks += 1

    def unlock(self):
        self.locks -=1
        assert self.locks >=0, "Lock / Unlock mismatch!"

    def desc_contents(self):
        if self.isfile():
            return "<file %s>"%self.name
        elif self.isdir():
            return "<dir %s>"%"".join( "%s: %s"% (k, v.desc_contents()) for k, v in self.contents.iteritems())

    def isdir(self):
        return self.type == "dir"

    def isfile(self):
        return self.type == "file"

    def islocked(self):
        return self.locks > 0

    def __str__(self):
        return "%s: %s" % (self.name, self.desc_contents())

class MemoryFS(FS):

    """ An in-memory filesystem.

    MemoryFS objects are very fast, but non-permantent. They are useful for creating a directory structure prior to writing it somewhere permanent.

    """

    def _make_dir_entry(self, *args, **kwargs):
        return self.dir_entry_factory(*args, **kwargs)

    def __init__(self, file_factory=None):
        super(MemoryFS, self).__init__(thread_synchronize=_thread_synchronize_default)

        self.dir_entry_factory = DirEntry
        self.file_factory = file_factory or MemoryFile

        self.root = self._make_dir_entry('dir', 'root')

    def __str__(self):
        return "<MemoryFS>"

    __repr__ = __str__

    def __unicode__(self):
        return unicode(self.__str__())

    @synchronize
    def _get_dir_entry(self, dirpath):
        current_dir = self.root
        for path_component in iteratepath(dirpath):
            if current_dir.contents is None:
                return None
            dir_entry = current_dir.contents.get(path_component, None)
            if dir_entry is None:
                return None
            current_dir = dir_entry
        return current_dir

    @synchronize
    def desc(self, path):
        if self.isdir(path):
            return "Memory dir"
        elif self.isfile(path):
            return "Memory file object"
        else:
            return "No description available"

    @synchronize
    def isdir(self, path):
        dir_item = self._get_dir_entry(normpath(path))
        if dir_item is None:
            return False
        return dir_item.isdir()

    @synchronize
    def isfile(self, path):
        dir_item = self._get_dir_entry(normpath(path))
        if dir_item is None:
            return False
        return dir_item.isfile()

    @synchronize
    def exists(self, path):
        return self._get_dir_entry(path) is not None

    @synchronize
    def makedir(self, dirname, recursive=False, allow_recreate=False):
        if not dirname:
            raise PathError(dirname)
        fullpath = dirname
        dirpath, dirname = pathsplit(dirname)

        if recursive:
            parent_dir = self._get_dir_entry(dirpath)
            if parent_dir is not None:
                if parent_dir.isfile():
                    raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
                else:
                    if not allow_recreate:
                        if dirname in parent_dir.contents:
                            raise DestinationExistsError(dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")

            current_dir = self.root
            for path_component in iteratepath(dirpath)[:-1]:
                dir_item = current_dir.contents.get(path_component, None)
                if dir_item is None:
                    break
                if not dir_item.isdir():
                    raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
                current_dir = dir_item

            current_dir = self.root
            for path_component in iteratepath(dirpath):
                dir_item = current_dir.contents.get(path_component, None)
                if dir_item is None:
                    new_dir = self._make_dir_entry("dir", path_component)
                    current_dir.contents[path_component] = new_dir
                    current_dir = new_dir
                else:
                    current_dir = dir_item

            parent_dir = current_dir

        else:
            parent_dir = self._get_dir_entry(dirpath)
            if parent_dir is None:
                raise ParentDirectoryMissingError(dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")

        dir_item = parent_dir.contents.get(dirname, None)
        if dir_item is not None:
            if dir_item.isdir():
                if not allow_recreate:
                    raise DestinationExistsError(dirname)
            else:
                raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")

        if dir_item is None:
            parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)

        return self

    def _orphan_files(self, file_dir_entry):
        for f in file_dir_entry.open_files:
            f.close()

    @synchronize
    def _lock_dir_entry(self, path):
        dir_entry = self._get_dir_entry(path)
        dir_entry.lock()

    @synchronize
    def _unlock_dir_entry(self, path):
        dir_entry = self._get_dir_entry(path)
        dir_entry.unlock()

    @synchronize
    def _is_dir_locked(self, path):
        dir_entry = self._get_dir_entry(path)
        return dir_entry.islocked()

    @synchronize
    def open(self, path, mode="r", **kwargs):
        filepath, filename = pathsplit(path)
        parent_dir_entry = self._get_dir_entry(filepath)

        if parent_dir_entry is None or not parent_dir_entry.isdir():
            raise ResourceNotFoundError(path)

        if 'r' in mode or 'a' in mode:
            if filename not in parent_dir_entry.contents:
                raise ResourceNotFoundError(path)

            file_dir_entry = parent_dir_entry.contents[filename]

            if 'a' in mode and  file_dir_entry.islocked():
                raise ResourceLockedError(path)

            self._lock_dir_entry(path)
            mem_file = self.file_factory(path, self, file_dir_entry.data, mode)
            file_dir_entry.open_files.append(mem_file)
            return mem_file

        elif 'w' in mode:
            if filename not in parent_dir_entry.contents:
                file_dir_entry = self._make_dir_entry("file", filename)
                parent_dir_entry.contents[filename] = file_dir_entry
            else:
                file_dir_entry = parent_dir_entry.contents[filename]

            if file_dir_entry.islocked():
                raise ResourceLockedError(path)

            self._lock_dir_entry(path)

            mem_file = self.file_factory(path, self, None, mode)
            file_dir_entry.open_files.append(mem_file)
            return mem_file

        if parent_dir_entry is None:
            raise ResourceNotFoundError(path)

    @synchronize
    def remove(self, path):
        dir_entry = self._get_dir_entry(path)

        if dir_entry is None:
            raise ResourceNotFoundError(path)

        if dir_entry.islocked():
            self._orphan_files(dir_entry)
            #raise ResourceLockedError(path)

        if dir_entry.isdir():
            raise ResourceInvalidError(path,msg="That's a directory, not a file: %(path)s")

        pathname, dirname = pathsplit(path)

        parent_dir = self._get_dir_entry(pathname)

        del parent_dir.contents[dirname]

    @synchronize
    def removedir(self, path, recursive=False, force=False):
        dir_entry = self._get_dir_entry(path)

        if dir_entry is None:
            raise ResourceNotFoundError(path)
        if dir_entry.islocked():
            raise ResourceLockedError(path)
        if not dir_entry.isdir():
            raise ResourceInvalidError(path, msg="Can't remove resource, its not a directory: %(path)s" )

        if dir_entry.contents and not force:
            raise DirectoryNotEmptyError(path)

        if recursive:
            rpathname = path
            while rpathname:
                rpathname, dirname = pathsplit(rpathname)
                parent_dir = self._get_dir_entry(rpathname)
                del parent_dir.contents[dirname]
        else:
            pathname, dirname = pathsplit(path)
            parent_dir = self._get_dir_entry(pathname)
            del parent_dir.contents[dirname]

    @synchronize
    def rename(self, src, dst):
        src_dir,src_name = pathsplit(src)
        src_entry = self._get_dir_entry(src)
        if src_entry is None:
            raise ResourceNotFoundError(src)
        open_files = src_entry.open_files[:]
        for f in open_files:
            f.flush()
            f.path = dst

        dst_dir,dst_name = pathsplit(dst)
        dst_entry = self._get_dir_entry(dst)
        if dst_entry is not None:
            raise DestinationExistsError(path)

        src_dir_entry = self._get_dir_entry(src_dir)
        dst_dir_entry = self._get_dir_entry(dst_dir)
        if dst_dir_entry is None:
            raise ParentDirectoryMissingError(dst)
        dst_dir_entry.contents[dst_name] = src_dir_entry.contents[src_name]
        dst_dir_entry.contents[dst_name].name = dst_name
        del src_dir_entry.contents[src_name]

    @synchronize
    def _on_close_memory_file(self, open_file, path, value):
        filepath, filename = pathsplit(path)
        dir_entry = self._get_dir_entry(path)
        if dir_entry is not None and value is not None:
            dir_entry.data = value
            dir_entry.open_files.remove(open_file)
            self._unlock_dir_entry(path)

    @synchronize
    def _on_flush_memory_file(self, path, value):
        filepath, filename = pathsplit(path)
        dir_entry = self._get_dir_entry(path)
        dir_entry.data = value

    @synchronize
    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
        dir_entry = self._get_dir_entry(path)
        if dir_entry is None:
            raise ResourceNotFoundError(path)
        if dir_entry.isfile():
            raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
        paths = dir_entry.contents.keys()
        for (i,p) in enumerate(paths):
            if not isinstance(p,unicode):
                paths[i] = unicode(p)
        return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)

    @synchronize
    def getinfo(self, path):
        dir_entry = self._get_dir_entry(path)

        if dir_entry is None:
            raise ResourceNotFoundError(path)

        info = {}
        info['created_time'] = dir_entry.created_time

        if dir_entry.isfile():
            info['size'] = len(dir_entry.data or '')

        return info
<commit_msg>add modified_time and accessed_time to MemoryFS file/dir info<commit_after>#!/usr/bin/env python
"""
fs.memoryfs
===========

A Filesystem that exists in memory only.

File objects returned by MemoryFS.objects use StringIO objects for storage.

"""

import datetime
from fs.path import iteratepath
from fs.base import *
from fs import _thread_synchronize_default

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

def _check_mode(mode, mode_chars):
    for c in mode_chars:
        if c not in mode:
            return False
    return True

class MemoryFile(object):

    def __init__(self, path, memory_fs, value, mode):
        self.closed = False
        self.path = path
        self.memory_fs = memory_fs
        self.mode = mode

        self.mem_file = None

        if _check_mode(mode, 'wa'):
            self.mem_file = StringIO()
            self.mem_file.write(value)

        elif _check_mode(mode, 'w'):
            self.mem_file = StringIO()

        elif _check_mode(mode, 'ra'):
            self.mem_file = StringIO()
            self.mem_file.write(value)

        elif _check_mode(mode, 'r'):
            self.mem_file = StringIO(value)

        elif _check_mode(mode, "a"):
            self.mem_file = StringIO()
            self.mem_file.write(value)

        else:
            if value is not None:
                self.mem_file = StringIO(value)
            else:
                self.mem_file = StringIO()

        assert self.mem_file is not None, "self.mem_file should have a value"

    def __str__(self):
        return "<MemoryFile in %s %s>" % (self.memory_fs, self.path)

    __repr__ = __str__

    def __unicode__(self):
        return u"<MemoryFile in %s %s>" % (self.memory_fs, self.path)

    def __del__(self):
        if not self.closed:
            self.close()

    def flush(self):
        value = self.mem_file.getvalue()
        self.memory_fs._on_flush_memory_file(self.path, value)

    def __iter__(self):
        return iter(self.mem_file)

    def next(self):
        return self.mem_file.next()

    def readline(self, *args, **kwargs):
        return self.mem_file.readline(*args, **kwargs)

    def close(self):
        if not self.closed and self.mem_file is not None:
            value = self.mem_file.getvalue()
            self.memory_fs._on_close_memory_file(self, self.path, value)
            self.mem_file.close()
            self.closed = True

    def read(self, size=None):
        if size is None:
            size = -1
        return self.mem_file.read(size)

    def seek(self, *args, **kwargs):
        return self.mem_file.seek(*args, **kwargs)

    def tell(self):
        return self.mem_file.tell()

    def truncate(self, *args, **kwargs):
        return self.mem_file.truncate(*args, **kwargs)

    def write(self, data):
        return self.mem_file.write(data)

    def writelines(self, *args, **kwargs):
        return self.mem_file.writelines(*args, **kwargs)

    def __enter__(self):
        return self

    def __exit__(self,exc_type,exc_value,traceback):
        self.close()
        return False

class DirEntry(object):

    def __init__(self, type, name, contents=None):

        assert type in ("dir", "file"), "Type must be dir or file!"

        self.type = type
        self.name = name

        if contents is None and type == "dir":
            contents = {}

        self.open_files = []
        self.contents = contents
        self.data = None
        self.locks = 0
        self.created_time = datetime.datetime.now()
        self.modified_time = self.created_time
        self.accessed_time = self.created_time

    def lock(self):
        self.locks += 1

    def unlock(self):
        self.locks -=1
        assert self.locks >=0, "Lock / Unlock mismatch!"

    def desc_contents(self):
        if self.isfile():
            return "<file %s>"%self.name
        elif self.isdir():
            return "<dir %s>"%"".join( "%s: %s"% (k, v.desc_contents()) for k, v in self.contents.iteritems())

    def isdir(self):
        return self.type == "dir"

    def isfile(self):
        return self.type == "file"

    def islocked(self):
        return self.locks > 0

    def __str__(self):
        return "%s: %s" % (self.name, self.desc_contents())

class MemoryFS(FS):

    """ An in-memory filesystem.

    MemoryFS objects are very fast, but non-permantent. They are useful for creating a directory structure prior to writing it somewhere permanent.

    """

    def _make_dir_entry(self, *args, **kwargs):
        return self.dir_entry_factory(*args, **kwargs)

    def __init__(self, file_factory=None):
        super(MemoryFS, self).__init__(thread_synchronize=_thread_synchronize_default)

        self.dir_entry_factory = DirEntry
        self.file_factory = file_factory or MemoryFile

        self.root = self._make_dir_entry('dir', 'root')

    def __str__(self):
        return "<MemoryFS>"

    __repr__ = __str__

    def __unicode__(self):
        return unicode(self.__str__())

    @synchronize
    def _get_dir_entry(self, dirpath):
        current_dir = self.root
        for path_component in iteratepath(dirpath):
            if current_dir.contents is None:
                return None
            dir_entry = current_dir.contents.get(path_component, None)
            if dir_entry is None:
                return None
            current_dir = dir_entry
        return current_dir

    @synchronize
    def desc(self, path):
        if self.isdir(path):
            return "Memory dir"
        elif self.isfile(path):
            return "Memory file object"
        else:
            return "No description available"

    @synchronize
    def isdir(self, path):
        dir_item = self._get_dir_entry(normpath(path))
        if dir_item is None:
            return False
        return dir_item.isdir()

    @synchronize
    def isfile(self, path):
        dir_item = self._get_dir_entry(normpath(path))
        if dir_item is None:
            return False
        return dir_item.isfile()

    @synchronize
    def exists(self, path):
        return self._get_dir_entry(path) is not None

    @synchronize
    def makedir(self, dirname, recursive=False, allow_recreate=False):
        if not dirname:
            raise PathError(dirname)
        fullpath = dirname
        dirpath, dirname = pathsplit(dirname)

        if recursive:
            parent_dir = self._get_dir_entry(dirpath)
            if parent_dir is not None:
                if parent_dir.isfile():
                    raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
                else:
                    if not allow_recreate:
                        if dirname in parent_dir.contents:
                            raise DestinationExistsError(dirname, msg="Can not create a directory that already exists (try allow_recreate=True): %(path)s")

            current_dir = self.root
            for path_component in iteratepath(dirpath)[:-1]:
                dir_item = current_dir.contents.get(path_component, None)
                if dir_item is None:
                    break
                if not dir_item.isdir():
                    raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")
                current_dir = dir_item

            current_dir = self.root
            for path_component in iteratepath(dirpath):
                dir_item = current_dir.contents.get(path_component, None)
                if dir_item is None:
                    new_dir = self._make_dir_entry("dir", path_component)
                    current_dir.contents[path_component] = new_dir
                    current_dir = new_dir
                else:
                    current_dir = dir_item

            parent_dir = current_dir

        else:
            parent_dir = self._get_dir_entry(dirpath)
            if parent_dir is None:
                raise ParentDirectoryMissingError(dirname, msg="Could not make dir, as parent dir does not exist: %(path)s")

        dir_item = parent_dir.contents.get(dirname, None)
        if dir_item is not None:
            if dir_item.isdir():
                if not allow_recreate:
                    raise DestinationExistsError(dirname)
            else:
                raise ResourceInvalidError(dirname, msg="Can not create a directory, because path references a file: %(path)s")

        if dir_item is None:
            parent_dir.contents[dirname] = self._make_dir_entry("dir", dirname)

        return self

    def _orphan_files(self, file_dir_entry):
        for f in file_dir_entry.open_files:
            f.close()

    @synchronize
    def _lock_dir_entry(self, path):
        dir_entry = self._get_dir_entry(path)
        dir_entry.lock()

    @synchronize
    def _unlock_dir_entry(self, path):
        dir_entry = self._get_dir_entry(path)
        dir_entry.unlock()

    @synchronize
    def _is_dir_locked(self, path):
        dir_entry = self._get_dir_entry(path)
        return dir_entry.islocked()

    @synchronize
    def open(self, path, mode="r", **kwargs):
        filepath, filename = pathsplit(path)
        parent_dir_entry = self._get_dir_entry(filepath)

        if parent_dir_entry is None or not parent_dir_entry.isdir():
            raise ResourceNotFoundError(path)

        if 'r' in mode or 'a' in mode:
            if filename not in parent_dir_entry.contents:
                raise ResourceNotFoundError(path)

            file_dir_entry = parent_dir_entry.contents[filename]

            if 'a' in mode:
                if file_dir_entry.islocked():
                    raise ResourceLockedError(path)
                file_dir_entry.modified_time = datetime.datetime.now()
            file_dir_entry.accessed_time = datetime.datetime.now()

            self._lock_dir_entry(path)
            mem_file = self.file_factory(path, self, file_dir_entry.data, mode)
            file_dir_entry.open_files.append(mem_file)
            return mem_file

        elif 'w' in mode:
            if filename not in parent_dir_entry.contents:
                file_dir_entry = self._make_dir_entry("file", filename)
                parent_dir_entry.contents[filename] = file_dir_entry
            else:
                file_dir_entry = parent_dir_entry.contents[filename]

            if file_dir_entry.islocked():
                raise ResourceLockedError(path)
            file_dir_entry.accessed_time = datetime.datetime.now()
            file_dir_entry.modified_time = datetime.datetime.now()

            self._lock_dir_entry(path)

            mem_file = self.file_factory(path, self, None, mode)
            file_dir_entry.open_files.append(mem_file)
            return mem_file

        if parent_dir_entry is None:
            raise ResourceNotFoundError(path)

    @synchronize
    def remove(self, path):
        dir_entry = self._get_dir_entry(path)

        if dir_entry is None:
            raise ResourceNotFoundError(path)

        if dir_entry.islocked():
            self._orphan_files(dir_entry)
            #raise ResourceLockedError(path)

        if dir_entry.isdir():
            raise ResourceInvalidError(path,msg="That's a directory, not a file: %(path)s")

        pathname, dirname = pathsplit(path)

        parent_dir = self._get_dir_entry(pathname)

        del parent_dir.contents[dirname]

    @synchronize
    def removedir(self, path, recursive=False, force=False):
        dir_entry = self._get_dir_entry(path)

        if dir_entry is None:
            raise ResourceNotFoundError(path)
        if dir_entry.islocked():
            raise ResourceLockedError(path)
        if not dir_entry.isdir():
            raise ResourceInvalidError(path, msg="Can't remove resource, its not a directory: %(path)s" )

        if dir_entry.contents and not force:
            raise DirectoryNotEmptyError(path)

        if recursive:
            rpathname = path
            while rpathname:
                rpathname, dirname = pathsplit(rpathname)
                parent_dir = self._get_dir_entry(rpathname)
                del parent_dir.contents[dirname]
        else:
            pathname, dirname = pathsplit(path)
            parent_dir = self._get_dir_entry(pathname)
            del parent_dir.contents[dirname]

    @synchronize
    def rename(self, src, dst):
        src_dir,src_name = pathsplit(src)
        src_entry = self._get_dir_entry(src)
        if src_entry is None:
            raise ResourceNotFoundError(src)
        open_files = src_entry.open_files[:]
        for f in open_files:
            f.flush()
            f.path = dst

        dst_dir,dst_name = pathsplit(dst)
        dst_entry = self._get_dir_entry(dst)
        if dst_entry is not None:
            raise DestinationExistsError(path)

        src_dir_entry = self._get_dir_entry(src_dir)
        dst_dir_entry = self._get_dir_entry(dst_dir)
        if dst_dir_entry is None:
            raise ParentDirectoryMissingError(dst)
        dst_dir_entry.contents[dst_name] = src_dir_entry.contents[src_name]
        dst_dir_entry.contents[dst_name].name = dst_name
        del src_dir_entry.contents[src_name]

    @synchronize
    def _on_close_memory_file(self, open_file, path, value):
        filepath, filename = pathsplit(path)
        dir_entry = self._get_dir_entry(path)
        if dir_entry is not None and value is not None:
            dir_entry.data = value
            dir_entry.open_files.remove(open_file)
            self._unlock_dir_entry(path)

    @synchronize
    def _on_flush_memory_file(self, path, value):
        filepath, filename = pathsplit(path)
        dir_entry = self._get_dir_entry(path)
        dir_entry.data = value

    @synchronize
    def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
        dir_entry = self._get_dir_entry(path)
        if dir_entry is None:
            raise ResourceNotFoundError(path)
        if dir_entry.isfile():
            raise ResourceInvalidError(path,msg="that's a file, not a directory: %(path)s")
        paths = dir_entry.contents.keys()
        for (i,p) in enumerate(paths):
            if not isinstance(p,unicode):
                paths[i] = unicode(p)
        return self._listdir_helper(path, paths, wildcard, full, absolute, dirs_only, files_only)

    @synchronize
    def getinfo(self, path):
        dir_entry = self._get_dir_entry(path)

        if dir_entry is None:
            raise ResourceNotFoundError(path)

        info = {}
        info['created_time'] = dir_entry.created_time
        info['modified_time'] = dir_entry.modified_time
        info['accessed_time'] = dir_entry.accessed_time

        if dir_entry.isfile():
            info['size'] = len(dir_entry.data or '')

        return info
<|endoftext|>


<commit_before>__all__ = ['Node', 'RoutingTable', 'Contact']

import math
import uuid
import struct
import random
import socket
import asyncio
import marshal

class Contact(object):
    def __init__(self, id=None, local_host=None, local_port=None, remote_host=None, remote_port=None, version=None):
        self.id = id
        self.local_host = local_host
        self.local_port = local_port
        self.remote_host = remote_host
        self.remote_port = remote_port
        self.version = version

    def __repr__(self):
        return '<{} id={} local={}:{} remote={}:{} ver={}>'.format(
            self.__class__.__name__,
            self.id,
            self.local_host,
            self.local_port,
            self.remote_host,
            self.remote_port,
            self.version,
        )

    def __getstate__(self):
        return {
            'id': self.id,
            'local_host': self.local_host,
            'local_port': self.local_port,
            'remote_host': self.remote_host,
            'remote_port': self.remote_port,
            'version': self.version,
        }

class RoutingTable(object):
    def __init__(self):
        self.version = 0
        self.contacts = [] # [Contact(), ...]

    def add(self, c):
        c.version = self.version
        self.version += 1
        self.contacts.append(c)

    def get(self, id):
        for c in self.contacts:
            if c.id == id:
                return c

    def remove(self, c_or_id):
        if isinstance(c_or_id, Contact):
            self.contacts.remove(c)
        else:
            for i, c in enumerate(self.contacts[:]):
                if c.id == c_or_id:
                    del self.contacts[i]
                    break

    def update_or_add(self, c):
        for n in self.contacts:
            if n.id == c.id:
                n.local_host = c.local_host
                n.local_port = c.local_port
                n.remote_host = c.remote_host
                n.remote_port = c.remote_port
                break
            elif n.remote_host == c.remote_host and n.remote_port == c.remote_port:
                n.id = c.id
                n.local_host = c.local_host
                n.local_port = c.local_port
                break
        else:
            self.contacts.append(c)

    def update_by_address(self, remote_host, remote_port, id, local_host, local_port):
        for c in self.contacts:
            if c.remote_host == remote_host and c.remote_port == remote_port:
                c.id = id
                c.local_host = local_host
                c.local_port = local_port
                break

    def random(self, without_id=None):
        if len(self.contacts):
            c = random.choice(self.contacts)

            if c.id == without_id:
                c = None
        else:
            c = None

        return c

    def all(self, version=0):
        contacts = []

        for c in self.contacts:
            # if c.version is None:
            #     continue

            # if c.version < version:
            #     continue

            contacts.append(c)

        return contacts

class Node(object):
    # protocol version 1.0
    NODE_PROTOCOL_VERSION_MAJOR = 1
    NODE_PROTOCOL_VERSION_MINOR = 0

    # protocol types
    NODE_PROTOCOL_REQ = 0
    NODE_PROTOCOL_RES = 1

    # protocol commands
    NODE_PROTOCOL_PING = 0
    NODE_PROTOCOL_DISCOVER_NODES = 1

    def __init__(self, loop, id=None, listen_host='127.0.0.1', listen_port=6633):
        self.loop = loop

        if id == None:
            id = str(uuid.uuid4())

        self.id = id

        self.listen_host = listen_host
        self.listen_port = listen_port

        self.recv_buffer = {} # {(remote_host: remote_port): [socket_data, ...]}
        self.recv_packs = {} # {msg_id: {pack_index: pack_data}}
        self.rt = RoutingTable()

        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.bind((self.listen_host, self.listen_port))
        self.loop.add_reader(self.sock, self.rect_sock_data)

        self.loop.call_soon(self.discover_nodes)
        self.loop.call_soon(self.check_recv_buffer)

    def check_recv_buffer(self):
        for remote_address, data in self.recv_buffer.items():
            self.process_sock_data(data, remote_address)

        self.loop.call_later(1.0, self.check_recv_buffer)

    def rect_sock_data(self):
        data, remote_address = self.sock.recvfrom(1500)
        remote_host, remote_port = remote_address
        # print('read_sock [DATA]:', remote_address, len(data), data)

        self.process_sock_data(data, remote_address)

    def process_sock_data(self, data, remote_address):
        if remote_address not in self.recv_buffer:
            self.recv_buffer[remote_address] = []

        self.recv_buffer[remote_address].append(data)
        recv_buffer = b''.join(self.recv_buffer[remote_address])
        pack_header_size = struct.calcsize('!QIIII')

        if len(recv_buffer) < pack_header_size:
            return

        del self.recv_buffer[remote_address][:]
        pack_header = recv_buffer[:pack_header_size]
        recv_buffer_rest = recv_buffer[pack_header_size:]
        msg_id, msg_size, msg_n_packs, pack_size, pack_index = struct.unpack('!QIIII', pack_header)

        if pack_size > len(recv_buffer_rest):
            self.recv_buffer[remote_address].append(pack_header)
            self.recv_buffer[remote_address].append(recv_buffer_rest)
            return

        pack_data = recv_buffer_rest[:pack_size]
        rest_data = recv_buffer_rest[pack_size:]
        self.recv_buffer[remote_address].append(rest_data)
        # print('read_sock [PACK]:', msg_size, msg_n_packs, pack_size, pack_index, pack_data)

        if msg_id not in self.recv_packs:
            self.recv_packs[msg_id] = {}

        self.recv_packs[msg_id][pack_index] = pack_data

        if len(self.recv_packs[msg_id]) < msg_n_packs:
            return

        msg = b''.join([self.recv_packs[msg_id][i] for i in range(msg_n_packs)])
        # print('read_sock [MSG]:', msg)

        self.parse_message(msg, remote_host, remote_port)

    def build_message(self, message_data):
        message_id = random.randint(0, 2 ** 64)
        step = 1400 - 3 * 4
        pack_index = 0

        message_n_packs = int(math.ceil(len(message_data) / step))

        for s in range(0, len(message_data), step):
            e = s + step
            pack_data = message_data[s:e]
            pack = self.build_pack(message_id, len(message_data), message_n_packs, len(pack_data), pack_index, pack_data)
            pack_index += 1
            yield pack

    def build_pack(self, message_id, message_size, message_n_packs, pack_size, pack_index, pack_data):
        pack = struct.pack('!QIIII', message_id, message_size, message_n_packs, pack_size, pack_index)
        pack += pack_data
        return pack

    def parse_message(self, message, remote_host, remote_port):
        message_header_size = struct.calcsize('!BBBB')
        message_header = message[:message_header_size]
        message_data = message[message_header_size:]
        message_version_major, message_version_minor, message_type, message_command = struct.unpack('!BBBB', message_header)

        if message_command == self.NODE_PROTOCOL_PING:
            pass
        elif message_command == self.NODE_PROTOCOL_DISCOVER_NODES:
            if message_type == self.NODE_PROTOCOL_REQ:
                if message_data:
                    args, kwargs = marshal.loads(message_data)
                else:
                    args, kwargs = (), {}

                self.on_req_discover_nodes(remote_host, remote_port, *args, **kwargs)
            elif message_type == self.NODE_PROTOCOL_RES:
                res = marshal.loads(message_data)
                self.on_res_discover_nodes(remote_host, remote_port, res)

    #
    # discover_nodes
    #
    def discover_nodes(self):
        # request
        c = self.rt.random(without_id=self.id)
        print('discover_nodes:', c)

        if not c:
            self.loop.call_later(2.0, self.discover_nodes)
            return

        node_id = self.id
        node_local_host = self.listen_host
        node_local_port = self.listen_port

        args = ()
        kwargs = {
            'id': node_id,
            'local_host': node_local_host,
            'local_port': node_local_port,
        }

        res = (args, kwargs)
        req_data = marshal.dumps(res)

        # message
        message_data = struct.pack(
            '!BBBB',
            self.NODE_PROTOCOL_VERSION_MAJOR,
            self.NODE_PROTOCOL_VERSION_MINOR,
            self.NODE_PROTOCOL_REQ,
            self.NODE_PROTOCOL_DISCOVER_NODES,
        )

        message_data += req_data

        # send message
        for pack in self.build_message(message_data):
            self.sock.sendto(pack, (c.remote_host, c.remote_port))

    def on_req_discover_nodes(self, remote_host, remote_port, *args, **kwargs):
        print('on_req_discover_nodes:', remote_host, remote_port, args, kwargs)

        # update/add contact which is requesting response
        c = Contact(
            id = kwargs['id'],
            local_host = kwargs['local_host'],
            local_port = kwargs['local_port'],
            remote_host = remote_host,
            remote_port = remote_port,
        )

        if c.id != self.id:
            self.rt.update_or_add(c)

        # forward to res_discover_nodes
        self.res_discover_nodes(remote_host, remote_port, *args, **kwargs)

    def res_discover_nodes(self, remote_host, remote_port, *args, **kwargs):
        # response
        node_id = self.id
        node_local_host = self.listen_host
        node_local_port = self.listen_port
        node_contacts = self.rt.all(version=kwargs.get('version', 0))
        node_contacts = [c.__getstate__() for c in node_contacts]

        res = {
            'id': node_id,
            'local_host': node_local_host,
            'local_port': node_local_port,
            'contacts': node_contacts,
        }

        res_data = marshal.dumps(res)

        # message data
        message_data = struct.pack(
            '!BBBB',
            self.NODE_PROTOCOL_VERSION_MAJOR,
            self.NODE_PROTOCOL_VERSION_MINOR,
            self.NODE_PROTOCOL_RES,
            self.NODE_PROTOCOL_DISCOVER_NODES,
        )

        message_data += res_data

        # send message
        for pack in self.build_message(message_data):
            self.sock.sendto(pack, (remote_host, remote_port))

    def on_res_discover_nodes(self, remote_host, remote_port, res):
        print('on_res_discover_nodes:', remote_host, remote_port, res)

        self.rt.update_by_address(
            remote_host,
            remote_port,
            id = res['id'],
            local_host = res['local_host'],
            local_port = res['local_port'],
        )

        for cd in res['contacts']:
            c = Contact(
                id = cd['id'],
                local_host = cd['local_host'],
                local_port = cd['local_port'],
                remote_host = cd['remote_host'],
                remote_port = cd['remote_port'],
            )

            if c.id != self.id:
                self.rt.update_or_add(c)

        self.loop.call_later(2.0, self.discover_nodes)

if __name__ == '__main__':
    # event loop
    loop = asyncio.get_event_loop()

    node = Node(loop)
    node.rt.add(Contact(str(uuid.uuid4()), '127.0.0.1', 6633, '127.0.0.1', 6633))
    node.rt.add(Contact(str(uuid.uuid4()), '127.0.0.1', 6634, '127.0.0.1', 6633))
    node.rt.add(Contact(str(uuid.uuid4()), '127.0.0.1', 6635, '127.0.0.1', 6633))

    # run loop
    loop.run_forever()
    loop.close()
<commit_msg>req-res<commit_after>__all__ = ['Node', 'RoutingTable', 'Contact']

import math
import uuid
import struct
import random
import socket
import asyncio
import marshal

class Contact(object):
    def __init__(self, id=None, local_host=None, local_port=None, remote_host=None, remote_port=None, version=None):
        self.id = id
        self.local_host = local_host
        self.local_port = local_port
        self.remote_host = remote_host
        self.remote_port = remote_port
        self.version = version

    def __repr__(self):
        return '<{} id={} local={}:{} remote={}:{} ver={}>'.format(
            self.__class__.__name__,
            self.id,
            self.local_host,
            self.local_port,
            self.remote_host,
            self.remote_port,
            self.version,
        )

    def __getstate__(self):
        return {
            'id': self.id,
            'local_host': self.local_host,
            'local_port': self.local_port,
            'remote_host': self.remote_host,
            'remote_port': self.remote_port,
            'version': self.version,
        }

class RoutingTable(object):
    def __init__(self):
        self.version = 0
        self.contacts = [] # [Contact(), ...]

    def add(self, c):
        c.version = self.version
        self.version += 1
        self.contacts.append(c)

    def get(self, id):
        for c in self.contacts:
            if c.id == id:
                return c

    def remove(self, c_or_id):
        if isinstance(c_or_id, Contact):
            self.contacts.remove(c)
        else:
            for i, c in enumerate(self.contacts[:]):
                if c.id == c_or_id:
                    del self.contacts[i]
                    break

    def update_or_add(self, c):
        for n in self.contacts:
            if n.id == c.id:
                n.local_host = c.local_host
                n.local_port = c.local_port
                n.remote_host = c.remote_host
                n.remote_port = c.remote_port
                break
            elif n.remote_host == c.remote_host and n.remote_port == c.remote_port:
                n.id = c.id
                n.local_host = c.local_host
                n.local_port = c.local_port
                break
        else:
            self.contacts.append(c)

    def update_by_address(self, remote_host, remote_port, id, local_host, local_port):
        for c in self.contacts:
            if c.remote_host == remote_host and c.remote_port == remote_port:
                c.id = id
                c.local_host = local_host
                c.local_port = local_port
                break

    def random(self, without_id=None):
        if len(self.contacts):
            c = random.choice(self.contacts)

            if c.id == without_id:
                c = None
        else:
            c = None

        return c

    def all(self, version=0):
        contacts = []

        for c in self.contacts:
            # if c.version is None:
            #     continue

            # if c.version < version:
            #     continue

            contacts.append(c)

        return contacts

class Node(object):
    # protocol version 1.0
    NODE_PROTOCOL_VERSION_MAJOR = 1
    NODE_PROTOCOL_VERSION_MINOR = 0

    # protocol types
    NODE_PROTOCOL_REQ = 0
    NODE_PROTOCOL_RES = 1

    # protocol commands
    NODE_PROTOCOL_PING = 0
    NODE_PROTOCOL_DISCOVER_NODES = 1

    def __init__(self, loop, id=None, listen_host='127.0.0.1', listen_port=6633):
        self.loop = loop

        if id == None:
            id = str(uuid.uuid4())

        self.id = id

        self.listen_host = listen_host
        self.listen_port = listen_port

        self.recv_buffer = {} # {(remote_host: remote_port): [socket_data, ...]}
        self.recv_packs = {} # {msg_id: {pack_index: pack_data}}
        self.rt = RoutingTable()

        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.bind((self.listen_host, self.listen_port))
        self.loop.add_reader(self.sock, self.rect_sock_data)

        self.loop.call_soon(self.discover_nodes)
        self.loop.call_soon(self.check_recv_buffer)

    def check_recv_buffer(self):
        for remote_address in self.recv_buffer:
            self.process_sock_data(b'', remote_address)

        self.loop.call_later(1.0, self.check_recv_buffer)

    def rect_sock_data(self):
        data, remote_address = self.sock.recvfrom(1500)
        remote_host, remote_port = remote_address
        # print('read_sock [DATA]:', remote_address, len(data), data)

        self.process_sock_data(data, remote_address)

    def process_sock_data(self, data, remote_address):
        if remote_address not in self.recv_buffer:
            self.recv_buffer[remote_address] = []

        self.recv_buffer[remote_address].append(data)
        recv_buffer = b''.join(self.recv_buffer[remote_address])
        pack_header_size = struct.calcsize('!QIIII')

        if len(recv_buffer) < pack_header_size:
            return

        del self.recv_buffer[remote_address][:]
        pack_header = recv_buffer[:pack_header_size]
        recv_buffer_rest = recv_buffer[pack_header_size:]
        msg_id, msg_size, msg_n_packs, pack_size, pack_index = struct.unpack('!QIIII', pack_header)

        if pack_size > len(recv_buffer_rest):
            self.recv_buffer[remote_address].append(pack_header)
            self.recv_buffer[remote_address].append(recv_buffer_rest)
            return

        pack_data = recv_buffer_rest[:pack_size]
        rest_data = recv_buffer_rest[pack_size:]
        self.recv_buffer[remote_address].append(rest_data)
        # print('read_sock [PACK]:', msg_size, msg_n_packs, pack_size, pack_index, pack_data)

        if msg_id not in self.recv_packs:
            self.recv_packs[msg_id] = {}

        self.recv_packs[msg_id][pack_index] = pack_data

        if len(self.recv_packs[msg_id]) < msg_n_packs:
            return

        msg = b''.join([self.recv_packs[msg_id][i] for i in range(msg_n_packs)])
        # print('read_sock [MSG]:', msg)

        self.parse_message(msg, remote_host, remote_port)

    def build_message(self, message_data):
        message_id = random.randint(0, 2 ** 64)
        step = 1400 - 3 * 4
        pack_index = 0

        message_n_packs = int(math.ceil(len(message_data) / step))

        for s in range(0, len(message_data), step):
            e = s + step
            pack_data = message_data[s:e]
            pack = self.build_pack(message_id, len(message_data), message_n_packs, len(pack_data), pack_index, pack_data)
            pack_index += 1
            yield pack

    def build_pack(self, message_id, message_size, message_n_packs, pack_size, pack_index, pack_data):
        pack = struct.pack('!QIIII', message_id, message_size, message_n_packs, pack_size, pack_index)
        pack += pack_data
        return pack

    def parse_message(self, message, remote_host, remote_port):
        message_header_size = struct.calcsize('!BBBB')
        message_header = message[:message_header_size]
        message_data = message[message_header_size:]
        message_version_major, message_version_minor, message_type, message_command = struct.unpack('!BBBB', message_header)

        if message_command == self.NODE_PROTOCOL_PING:
            pass
        elif message_command == self.NODE_PROTOCOL_DISCOVER_NODES:
            if message_type == self.NODE_PROTOCOL_REQ:
                if message_data:
                    args, kwargs = marshal.loads(message_data)
                else:
                    args, kwargs = (), {}

                self.on_req_discover_nodes(remote_host, remote_port, *args, **kwargs)
            elif message_type == self.NODE_PROTOCOL_RES:
                res = marshal.loads(message_data)
                self.on_res_discover_nodes(remote_host, remote_port, res)

    #
    # discover_nodes
    #
    def discover_nodes(self):
        # request
        c = self.rt.random(without_id=self.id)
        print('discover_nodes:', c)

        if not c:
            self.loop.call_later(2.0, self.discover_nodes)
            return

        node_id = self.id
        node_local_host = self.listen_host
        node_local_port = self.listen_port

        args = ()
        kwargs = {
            'id': node_id,
            'local_host': node_local_host,
            'local_port': node_local_port,
        }

        res = (args, kwargs)
        req_data = marshal.dumps(res)

        # message
        message_data = struct.pack(
            '!BBBB',
            self.NODE_PROTOCOL_VERSION_MAJOR,
            self.NODE_PROTOCOL_VERSION_MINOR,
            self.NODE_PROTOCOL_REQ,
            self.NODE_PROTOCOL_DISCOVER_NODES,
        )

        message_data += req_data

        # send message
        for pack in self.build_message(message_data):
            self.sock.sendto(pack, (c.remote_host, c.remote_port))

    def on_req_discover_nodes(self, remote_host, remote_port, *args, **kwargs):
        print('on_req_discover_nodes:', remote_host, remote_port, args, kwargs)

        # update/add contact which is requesting response
        c = Contact(
            id = kwargs['id'],
            local_host = kwargs['local_host'],
            local_port = kwargs['local_port'],
            remote_host = remote_host,
            remote_port = remote_port,
        )

        if c.id != self.id:
            self.rt.update_or_add(c)

        # forward to res_discover_nodes
        self.res_discover_nodes(remote_host, remote_port, *args, **kwargs)

    def res_discover_nodes(self, remote_host, remote_port, *args, **kwargs):
        # response
        node_id = self.id
        node_local_host = self.listen_host
        node_local_port = self.listen_port
        node_contacts = self.rt.all(version=kwargs.get('version', 0))
        node_contacts = [c.__getstate__() for c in node_contacts]

        res = {
            'id': node_id,
            'local_host': node_local_host,
            'local_port': node_local_port,
            'contacts': node_contacts,
        }

        res_data = marshal.dumps(res)

        # message data
        message_data = struct.pack(
            '!BBBB',
            self.NODE_PROTOCOL_VERSION_MAJOR,
            self.NODE_PROTOCOL_VERSION_MINOR,
            self.NODE_PROTOCOL_RES,
            self.NODE_PROTOCOL_DISCOVER_NODES,
        )

        message_data += res_data

        # send message
        for pack in self.build_message(message_data):
            self.sock.sendto(pack, (remote_host, remote_port))

    def on_res_discover_nodes(self, remote_host, remote_port, res):
        print('on_res_discover_nodes:', remote_host, remote_port, res)

        self.rt.update_by_address(
            remote_host,
            remote_port,
            id = res['id'],
            local_host = res['local_host'],
            local_port = res['local_port'],
        )

        for cd in res['contacts']:
            c = Contact(
                id = cd['id'],
                local_host = cd['local_host'],
                local_port = cd['local_port'],
                remote_host = cd['remote_host'],
                remote_port = cd['remote_port'],
            )

            if c.id != self.id:
                self.rt.update_or_add(c)

        self.loop.call_later(2.0, self.discover_nodes)

if __name__ == '__main__':
    # event loop
    loop = asyncio.get_event_loop()

    node = Node(loop)
    node.rt.add(Contact(str(uuid.uuid4()), '127.0.0.1', 6633, '127.0.0.1', 6633))
    node.rt.add(Contact(str(uuid.uuid4()), '127.0.0.1', 6634, '127.0.0.1', 6633))
    node.rt.add(Contact(str(uuid.uuid4()), '127.0.0.1', 6635, '127.0.0.1', 6633))

    # run loop
    loop.run_forever()
    loop.close()
<|endoftext|>


<commit_before>##################################################
# Extract a dbtable for every UID in a geo-file  #
# Written by Thomas Fish                         #
# Released under the MIT License                 #
##################################################

#Start dependency tree
import arcpy
from arcpy import env

#declare map
mxd = arcpy.mapping.MapDocument("CURRENT")

#Import data from ArcMap
fileToConvert = arcpy.GetParameterAsText(0)
listFieldValues = arcpy.GetParameterAsText(1)
outLocation = arcpy.GetParameterAsText(2)
fieldName = arcpy.GetParameterAsText(3)
#future place for field on what type of table to end up as
spreadsheetType = arcpy.GetParameterAsText(4)

if spreadsheetType == "dbf":
	fileType = ".dbf"
else if spreadsheetType == "csv":
	fileType = ".txt"
else if spreadsheetType == "txt":
	fileType = ".txt"
else
	arcpy.AddMessage("ERROR: Not a valid file type")

#translate file into layer
lyr = arcpy.mapping.Layer(fileToConvert)
lyr.name = "MyFile"

#declare array for UIDs
fieldValues = []

#declare the file to open
fin = open (listFieldValues)

#tell end user if there is an error
if fin.closed:
	arcpy.AddMessage("File failed to open")

for line in fin:
	#Turn each line in the file into an entry to the array
	lineClear = str.strip ( line )
	fieldValues.append(lineClear)

#declare array for file names

fileName = []

#make file names from UIDs
for i in range (len(fieldValues)):
        fileName.append (fieldValues[i] + "fileType")

filePath = []

for y in range (len(fileName)):
        filePath.append(outLocation + "\\" + fileName[y])

for z in range (len(fieldValues)):
	#Extract out only the entries that are wanted
    arcpy.AddMessage("Currently extracting "+ fieldValues[z] + " in " + fieldName)
    if lyr.name == "MyFile":
        lyr.definitionQuery = fieldName + " =" + "'" + fieldValues[z] + "'"
        arcpy.AddMessage("narrow data")
	#Convert the table into a dbf
	arcpy.AddMessage("Converting: " + fieldValues[z])
	#Error is here
	arcpy.CopyRows_management(lyr, filePath[z])

<commit_msg>clarify notice at the top<commit_after>##################################################################
# Extract a dbtable for every instance of a value in a geo-file  #
# Written by Thomas Fish                                         #
# Released under the MIT License                                 #
#                                                                #
#                 FOR ARCMAP 10.X ONLY                           #
##################################################################

#Start dependency tree
import arcpy
from arcpy import env

#declare map
mxd = arcpy.mapping.MapDocument("CURRENT")

#Import data from ArcMap
fileToConvert = arcpy.GetParameterAsText(0)
listFieldValues = arcpy.GetParameterAsText(1)
outLocation = arcpy.GetParameterAsText(2)
fieldName = arcpy.GetParameterAsText(3)
#future place for field on what type of table to end up as
spreadsheetType = arcpy.GetParameterAsText(4)

if spreadsheetType == "dbf":
	fileType = ".dbf"
else if spreadsheetType == "csv":
	fileType = ".txt"
else if spreadsheetType == "txt":
	fileType = ".txt"
else
	arcpy.AddMessage("ERROR: Not a valid file type")

#translate file into layer
lyr = arcpy.mapping.Layer(fileToConvert)
lyr.name = "MyFile"

#declare array for UIDs
fieldValues = []

#declare the file to open
fin = open (listFieldValues)

#tell end user if there is an error
if fin.closed:
	arcpy.AddMessage("File failed to open")

for line in fin:
	#Turn each line in the file into an entry to the array
	lineClear = str.strip ( line )
	fieldValues.append(lineClear)

#declare array for file names

fileName = []

#make file names from UIDs
for i in range (len(fieldValues)):
        fileName.append (fieldValues[i] + "fileType")

filePath = []

for y in range (len(fileName)):
        filePath.append(outLocation + "\\" + fileName[y])

for z in range (len(fieldValues)):
	#Extract out only the entries that are wanted
    arcpy.AddMessage("Currently extracting "+ fieldValues[z] + " in " + fieldName)
    if lyr.name == "MyFile":
        lyr.definitionQuery = fieldName + " =" + "'" + fieldValues[z] + "'"
        arcpy.AddMessage("narrow data")
	#Convert the table into a dbf
	arcpy.AddMessage("Converting: " + fieldValues[z])
	#Error is here
	arcpy.CopyRows_management(lyr, filePath[z])

<|endoftext|>


<commit_before>#!/usr/bin/env python3

## Fishbowl startup script

## As a distributed program, the startup phase of
## Fishbowl/IPython is non-trivial, and requires access to
## the filesystem.   Since Python (3.x) is a requirement
## for Fishbowl, the startup script is also written in
## Python

import subprocess
import sys
import shutil
import os

def halt(msg):
    print(msg, file=sys.stderr)
    print("Abort.", file=sys.stderr)
    sys.exit(1)

FISHBOWL_HEADER = """
Fishbowl -- an enhanced interactive Common Lisp shell
(C) 2014-2015 Frederic Peschanski (cf. LICENSE)
----"""

print(FISHBOWL_HEADER)

# check that we run as a script
if __name__ != "__main__":
    halt("Error: Fishbowl startup must be run as a script")

# check the python version, needs at least 3.2
if sys.version_info.major < 3 \
   or sys.version_info.minor < 3:
    halt("Error: Fishbowl requires Python v3.3 or above")

# check if ipython is available
try:
    import IPython
except ImportError:
    halt("Error: IPython not available (check your Python Path)")

# check Ipython version

ipython_version_major, ipython_version_minor, ipython_version_patch, ipython_version_tag = IPython.version_info
if ipython_version_major != 2:
    halt("Error: IPython v2.x required (found v{}.{})".format(ipython_version_major, ipython_version_minor))

print("... Frontend: using IPython v{}.{}".format(ipython_version_major, ipython_version_minor))

###################################
## (Ad-hoc) command-line parsing ##
###################################

class Config:
    def __init__(self):
        self.ipython_dir = IPython.utils.path.get_ipython_dir()
        self.ipython_profile_dir = self.ipython_dir + "/profile_fishbowl"
        self.lisp_implementation = "sbcl" # TODO: ccl support (others ? requires threading)
        self.ipython_executable = shutil.which("ipython3")
        self.ipython_command = "console"

def process_command_line(argv):
    config = Config()

    import inspect
    import os.path
    config.fishbowl_startup_def_dir = os.path.dirname(os.path.realpath(inspect.getsourcefile(Config)))
    #print("Fishbowl startup def dir = {}".format(config.fishbowl_startup_def_dir))

    config.fishbowl_startup_run_dir = os.path.realpath(os.getcwd())
    #print("Fishbowl startup run dir = {}".format(config.fishbowl_startup_run_dir))

    config.fishbowl_startup_script = os.path.realpath(argv[0])
    #print("Fishbowl startup script = {}".format(config.fishbowl_startup_script))

    i = 1
    if len(argv) > 1 and not (argv[i].startswith('-')):  # first argument should be the ipython command
        config.ipython_command = argv[i]
        i += 1

    # print("IPython command = {}".format(config.ipython_command))
    # default is "console"

    if config.ipython_command not in { "console", "notebook" }:
        halt("Error: command '{}' not available\n  ==> choose 'console' (default) or 'notebook'".format(config.ipython_command))

    profile_dir_set = False
    profile_set = False
    lisp_set = False
    ipython_exec_set = False

    while i < len(argv):
        #print("cmd line option #{}: {}".format(i, argv[i]))

        if argv[i].startswith("--profile-dir="):
            if profile_dir_set or profile_set:
                halt("Error: unexpected '--profile-dir' option, profile already set")
            config.ipython_profile_dir = argv[i][14:]
            profile_dir_set = True
        elif argv[i].startswith("--profile="):
            if profile_set or profile_dir_set:
                halt("Error: unexpected '--profile' option, profile already set")
            config.ipython_profile_dir = config.ipython_dir + "/profile_" + argv[i][10:]
            profile_set = True
        elif argv[i].startswith("--lisp="):
            if lisp_set:
                halt("Error: --lisp option set twice")
            config.lisp_implementation = argv[i][7:]
            lisp_set = True
        elif argv[i].startswith("--ipython-exec"):
            if ipython_exec_set:
                halt("Error: --ipython-exec option set twice")
            config.ipython_executable = shutil.which(argv[i][15:])
            ipython_exec_set = True
        else:
            halt("Error: unexpected option '{}'".format(argv[i]))

        i += 1

    #print("IPython profile directory = {}".format(config.ipython_profile_dir))
    #print("Lisp implementation = {}".format(config.lisp_implementation))
    #print("IPython executable = {}".format(config.ipython_executable))

    return config

config = process_command_line(sys.argv)

###################################
## Check Ipython executable      ##
###################################

if not config.ipython_executable:
    halt("Error: Ipython executable not found")
else:
    try:
        ipython_version_string = subprocess.check_output([config.ipython_executable, "--version"]).decode()
    except FileNotFoundError:
        halt("Error: cannot find ipython executable")
    except subprocess.CalledProcessError as e:
        halt("Error: {}".format(e))

    #print("ipython version string = {}".format(ipython_version_string))
    ipython_version = tuple([int(d) for d in ipython_version_string.split(".")])
    #print("ipython version = {}".format(ipython_version))
    if (ipython_version[0] != ipython_version_major) \
       or (ipython_version[1] != ipython_version_minor):
        halt("Error: mismatch ipython version ({}.{} vs {}.{})".format(ipython_version[0], ipython_version[1],
                                                                       ipython_version_major, ipython_version_minor))

###################################
## Check the lisp implementation ##
###################################

if config.lisp_implementation == "sbcl":
    try:
        sbcl_version_string = subprocess.check_output(["sbcl", "--version"]).decode()
    except FileNotFoundError:
        halt("Error: 'sbcl' executable not in PATH")
    except subprocess.CalledProcessError as e:
        halt("Error: {} from SBCL".format(e))

    #print("sbcl version string = {}".format(sbcl_version_string))

    import re
    m = re.match(r".*([0-9]+\.[0-9]+\.[0-9]+)", sbcl_version_string)
    if not m:
        halt("Error: issue with sbcl version string (please report)")

    config.sbcl_version = tuple([int(d) for d in m.group(1).split(".")])
    #print("sbcl version = {}".format(config.sbcl_version))
    if config.sbcl_version[0] < 1 or config.sbcl_version[1] < 2:
        halt("Error: require SBCL v1.2.x or above")

    print("... Kernel: using {}".format(sbcl_version_string))

elif config.lisp_implementation == "ccl":
    try:
        ccl_version_string = subprocess.check_output(["ccl", "-V"]).decode()
    except FileNotFoundError:
        halt("Error: 'ccl' executable not in PATH")
    except subprocess.CalledProcessError as e:
        halt("Error: {} from CCL".format(e))

    #print("ccl version string = {}".format(ccl_version_string))

    import re
    m = re.match(r".*([0-9]+\.[0-9]+)", ccl_version_string)
    if not m:
        halt("Error: issue with ccl version string (please report)")

    config.ccl_version = tuple([int(d) for d in m.group(1).split(".")])
    #print("ccl version = {}".format(config.ccl_version))
    if config.ccl_version[0] < 1 or config.ccl_version[1] < 10:
        halt("Error: require CCL v1.10 or above")

    print("... Kernel: using {}".format(ccl_version_string))

elif config.lisp_implementation == "ecl":
    halt("Error: ECL not (yet) supported")
elif config.lisp_implementation == "cmucl":
    halt("Error: CMUCL not (yet) supported")
elif config.lisp_implementation == "clisp":
    halt("Error: CLisp not (yet) supported")
else:
    halt("Error: Common Lisp implementation '{}' not supported".format(config.lisp_implementation))

##############################
## Installation of profile  ##
##############################

print("... profile customization")

custom_js_file = None

nb_try = 0
while not custom_js_file:
    try:
        custom_js_file = open(config.ipython_profile_dir + "/static/custom/custom.js", "r")
    except FileNotFoundError:
        # profile creation
        print("... create profile '{}'".format(config.ipython_profile_dir))
        ### XXX: Issue when running ipython for different commands multiple times
        ### (MultipleInstanceError) ... So run in a subprocess
        #IPython.start_ipython([config.ipython_executable,
        #                       "profile", "create",
        #                       "--profile-dir={}".format(config.ipython_profile_dir)])

        try:
            subprocess.check_call([config.ipython_executable,
                                  'profile', 'create',
                                  "--profile-dir={}".format(config.ipython_profile_dir)])
        except FileNotFoundError:
            halt("Error: '{}' executable not found".format(config.ipython_executable))
        except subprocess.CalledProcessError as e:
            halt("Error: {} from IPython".format(e))

    nb_try += 1
    if nb_try > 2:
        halt("Error: could not create profile (please report)")

## copy the custom js file

shutil.copy(config.fishbowl_startup_def_dir + "/profile/custom.js",
            config.ipython_profile_dir + "/static/custom/custom.js")

os.makedirs(config.ipython_profile_dir + "/static/components/codemirror/mode/commonlisp/", exist_ok=True)
shutil.copy(config.fishbowl_startup_def_dir + "/profile/commonlisp.js",
            config.ipython_profile_dir + "/static/components/codemirror/mode/commonlisp/commonlisp.js")

##############################
## Run the IPython command  ##
##############################

print("... launch frontend")

### XXX: strange MultipleInstanceError error raise, use subprocess
# IPython.start_ipython([config.ipython_executable, config.ipython_command,
#                        "--profile-dir={}".format(config.ipython_profile_dir),
#                        "--Session.key=b''",
#                        "--KernelManager.kernel_cmd=['sbcl', '--non-interactive', '--load', '{}/fishbowl.lisp', '{{connection_file}}']".format(config.fishbowl_startup_def_dir)])

if config.lisp_implementation == "sbcl":
    KERNEL_CMD = "--KernelManager.kernel_cmd=['sbcl', '--non-interactive', '--load', '{0}/fishbowl.lisp', '{0}/src', '{1}', '{{connection_file}}']".format(config.fishbowl_startup_def_dir, config.fishbowl_startup_run_dir)

elif config.lisp_implementation == "ccl":
    KERNEL_CMD = "--KernelManager.kernel_cmd=['ccl', '--batch', '--load', '{0}/fishbowl.lisp', '--', '{0}/src', '{1}', '{{connection_file}}']".format(config.fishbowl_startup_def_dir, config.fishbowl_startup_run_dir)

else:
    halt("Error: unsupported lisp implementation '{}'".format(lisp_implementation))

try:
    import signal
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    subprocess.check_call([config.ipython_executable,
                           config.ipython_command,
                           "--profile-dir={}".format(config.ipython_profile_dir),
                           "--Session.key=b''", KERNEL_CMD],
                          stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr, shell=False)
except FileNotFoundError:
    halt("Error: '{}' executable not found".format(config.ipython_executable))
except subprocess.CalledProcessError as e:
    halt("Error: {} from IPython".format(e))

<commit_msg>lisp preload option<commit_after>#!/usr/bin/env python3

## Fishbowl startup script

## As a distributed program, the startup phase of
## Fishbowl/IPython is non-trivial, and requires access to
## the filesystem.   Since Python (3.x) is a requirement
## for Fishbowl, the startup script is also written in
## Python

import subprocess
import sys
import shutil
import os

def halt(msg):
    print(msg, file=sys.stderr)
    print("Abort.", file=sys.stderr)
    sys.exit(1)

FISHBOWL_HEADER = """
Fishbowl -- an enhanced interactive Common Lisp shell
(C) 2014-2015 Frederic Peschanski (cf. LICENSE)
----"""

print(FISHBOWL_HEADER)

# check that we run as a script
if __name__ != "__main__":
    halt("Error: Fishbowl startup must be run as a script")

# check the python version, needs at least 3.2
if sys.version_info.major < 3 \
   or sys.version_info.minor < 3:
    halt("Error: Fishbowl requires Python v3.3 or above")

# check if ipython is available
try:
    import IPython
except ImportError:
    halt("Error: IPython not available (check your Python Path)")

# check Ipython version

ipython_version_major, ipython_version_minor, ipython_version_patch, ipython_version_tag = IPython.version_info
if ipython_version_major != 2:
    halt("Error: IPython v2.x required (found v{}.{})".format(ipython_version_major, ipython_version_minor))

print("... Frontend: using IPython v{}.{}".format(ipython_version_major, ipython_version_minor))

###################################
## (Ad-hoc) command-line parsing ##
###################################

class Config:
    def __init__(self):
        self.ipython_dir = IPython.utils.path.get_ipython_dir()
        self.ipython_profile_dir = self.ipython_dir + "/profile_fishbowl"
        self.lisp_implementation = "sbcl" # TODO: ccl support (others ? requires threading)
        self.ipython_executable = shutil.which("ipython3")
        self.ipython_command = "console"
        self.lisp_preload = None

def process_command_line(argv):
    config = Config()

    import inspect
    import os.path
    config.fishbowl_startup_def_dir = os.path.dirname(os.path.realpath(inspect.getsourcefile(Config)))
    #print("Fishbowl startup def dir = {}".format(config.fishbowl_startup_def_dir))

    config.fishbowl_startup_run_dir = os.path.realpath(os.getcwd())
    #print("Fishbowl startup run dir = {}".format(config.fishbowl_startup_run_dir))

    config.fishbowl_startup_script = os.path.realpath(argv[0])
    #print("Fishbowl startup script = {}".format(config.fishbowl_startup_script))

    i = 1
    if len(argv) > 1 and not (argv[i].startswith('-')):  # first argument should be the ipython command
        config.ipython_command = argv[i]
        i += 1

    # print("IPython command = {}".format(config.ipython_command))
    # default is "console"

    if config.ipython_command not in { "console", "notebook" }:
        halt("Error: command '{}' not available\n  ==> choose 'console' (default) or 'notebook'".format(config.ipython_command))

    profile_dir_set = False
    profile_set = False
    lisp_set = False
    ipython_exec_set = False

    while i < len(argv):
        #print("cmd line option #{}: {}".format(i, argv[i]))

        if argv[i].startswith("--profile-dir="):
            if profile_dir_set or profile_set:
                halt("Error: unexpected '--profile-dir' option, profile already set")
            config.ipython_profile_dir = argv[i][14:]
            profile_dir_set = True
        elif argv[i].startswith("--profile="):
            if profile_set or profile_dir_set:
                halt("Error: unexpected '--profile' option, profile already set")
            config.ipython_profile_dir = config.ipython_dir + "/profile_" + argv[i][10:]
            profile_set = True
        elif argv[i].startswith("--lisp="):
            if lisp_set:
                halt("Error: --lisp option set twice")
            config.lisp_implementation = argv[i][7:]
            lisp_set = True
        elif argv[i].startswith("--ipython-exec="):
            if ipython_exec_set:
                halt("Error: --ipython-exec option set twice")
            config.ipython_executable = shutil.which(argv[i][15:])
            ipython_exec_set = True
        elif argv[i].startswith("--lisp-preload="):
            config.lisp_preload = argv[i][15:]
        else:
            halt("Error: unexpected option '{}'".format(argv[i]))

        i += 1

    #print("IPython profile directory = {}".format(config.ipython_profile_dir))
    #print("Lisp implementation = {}".format(config.lisp_implementation))
    #print("IPython executable = {}".format(config.ipython_executable))

    return config

config = process_command_line(sys.argv)

###################################
## Check Ipython executable      ##
###################################

if not config.ipython_executable:
    halt("Error: Ipython executable not found")
else:
    try:
        ipython_version_string = subprocess.check_output([config.ipython_executable, "--version"]).decode()
    except FileNotFoundError:
        halt("Error: cannot find ipython executable")
    except subprocess.CalledProcessError as e:
        halt("Error: {}".format(e))

    #print("ipython version string = {}".format(ipython_version_string))
    ipython_version = tuple([int(d) for d in ipython_version_string.split(".")])
    #print("ipython version = {}".format(ipython_version))
    if (ipython_version[0] != ipython_version_major) \
       or (ipython_version[1] != ipython_version_minor):
        halt("Error: mismatch ipython version ({}.{} vs {}.{})".format(ipython_version[0], ipython_version[1],
                                                                       ipython_version_major, ipython_version_minor))

###################################
## Check the lisp implementation ##
###################################

if config.lisp_implementation == "sbcl":
    try:
        sbcl_version_string = subprocess.check_output(["sbcl", "--version"]).decode()
    except FileNotFoundError:
        halt("Error: 'sbcl' executable not in PATH")
    except subprocess.CalledProcessError as e:
        halt("Error: {} from SBCL".format(e))

    #print("sbcl version string = {}".format(sbcl_version_string))

    import re
    m = re.match(r".*([0-9]+\.[0-9]+\.[0-9]+)", sbcl_version_string)
    if not m:
        halt("Error: issue with sbcl version string (please report)")

    config.sbcl_version = tuple([int(d) for d in m.group(1).split(".")])
    #print("sbcl version = {}".format(config.sbcl_version))
    if config.sbcl_version[0] < 1 or config.sbcl_version[1] < 2:
        halt("Error: require SBCL v1.2.x or above")

    print("... Kernel: using {}".format(sbcl_version_string))

elif config.lisp_implementation == "ccl":
    try:
        ccl_version_string = subprocess.check_output(["ccl", "-V"]).decode()
    except FileNotFoundError:
        halt("Error: 'ccl' executable not in PATH")
    except subprocess.CalledProcessError as e:
        halt("Error: {} from CCL".format(e))

    #print("ccl version string = {}".format(ccl_version_string))

    import re
    m = re.match(r".*([0-9]+\.[0-9]+)", ccl_version_string)
    if not m:
        halt("Error: issue with ccl version string (please report)")

    config.ccl_version = tuple([int(d) for d in m.group(1).split(".")])
    #print("ccl version = {}".format(config.ccl_version))
    if config.ccl_version[0] < 1 or config.ccl_version[1] < 10:
        halt("Error: require CCL v1.10 or above")

    print("... Kernel: using {}".format(ccl_version_string))

elif config.lisp_implementation == "ecl":
    halt("Error: ECL not (yet) supported")
elif config.lisp_implementation == "cmucl":
    halt("Error: CMUCL not (yet) supported")
elif config.lisp_implementation == "clisp":
    halt("Error: CLisp not (yet) supported")
else:
    halt("Error: Common Lisp implementation '{}' not supported".format(config.lisp_implementation))

##############################
## Installation of profile  ##
##############################

print("... profile customization")

custom_js_file = None

nb_try = 0
while not custom_js_file:
    try:
        custom_js_file = open(config.ipython_profile_dir + "/static/custom/custom.js", "r")
    except FileNotFoundError:
        # profile creation
        print("... create profile '{}'".format(config.ipython_profile_dir))
        ### XXX: Issue when running ipython for different commands multiple times
        ### (MultipleInstanceError) ... So run in a subprocess
        #IPython.start_ipython([config.ipython_executable,
        #                       "profile", "create",
        #                       "--profile-dir={}".format(config.ipython_profile_dir)])

        try:
            subprocess.check_call([config.ipython_executable,
                                  'profile', 'create',
                                  "--profile-dir={}".format(config.ipython_profile_dir)])
        except FileNotFoundError:
            halt("Error: '{}' executable not found".format(config.ipython_executable))
        except subprocess.CalledProcessError as e:
            halt("Error: {} from IPython".format(e))

    nb_try += 1
    if nb_try > 2:
        halt("Error: could not create profile (please report)")

## copy the custom js file

shutil.copy(config.fishbowl_startup_def_dir + "/profile/custom.js",
            config.ipython_profile_dir + "/static/custom/custom.js")

os.makedirs(config.ipython_profile_dir + "/static/components/codemirror/mode/commonlisp/", exist_ok=True)
shutil.copy(config.fishbowl_startup_def_dir + "/profile/commonlisp.js",
            config.ipython_profile_dir + "/static/components/codemirror/mode/commonlisp/commonlisp.js")

##############################
## Run the IPython command  ##
##############################

print("... launch frontend")

### XXX: strange MultipleInstanceError error raise, use subprocess
# IPython.start_ipython([config.ipython_executable, config.ipython_command,
#                        "--profile-dir={}".format(config.ipython_profile_dir),
#                        "--Session.key=b''",
#                        "--KernelManager.kernel_cmd=['sbcl', '--non-interactive', '--load', '{}/fishbowl.lisp', '{{connection_file}}']".format(config.fishbowl_startup_def_dir)])

if config.lisp_implementation == "sbcl":
    KERNEL_CMD = "--KernelManager.kernel_cmd=['sbcl', '--non-interactive',{1} '--load', '{0}/fishbowl.lisp', '{0}/src', '{2}', '{{connection_file}}']".format(config.fishbowl_startup_def_dir, "'--load', '{}',".format(config.lisp_preload) if config.lisp_preload else "", config.fishbowl_startup_run_dir)

elif config.lisp_implementation == "ccl":
    KERNEL_CMD = "--KernelManager.kernel_cmd=['ccl', '--batch',{1} '--load', '{0}/fishbowl.lisp', '--', '{0}/src', '{2}', '{{connection_file}}']".format(config.fishbowl_startup_def_dir,  "'--load', '{}',".format(config.lisp_preload) if config.lisp_preload else "", config.fishbowl_startup_run_dir)

else:
    halt("Error: unsupported lisp implementation '{}'".format(lisp_implementation))

# print("KERNEL_CMD = {}".format(KERNEL_CMD))

try:
    import signal
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    subprocess.check_call([config.ipython_executable,
                           config.ipython_command,
                           "--profile-dir={}".format(config.ipython_profile_dir),
                           "--Session.key=b''", KERNEL_CMD],
                          stdout=sys.stdout, stdin=sys.stdin, stderr=sys.stderr, shell=False)
except FileNotFoundError:
    halt("Error: '{}' executable not found".format(config.ipython_executable))
except subprocess.CalledProcessError as e:
    halt("Error: {} from IPython".format(e))

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2015 Tamas Nagy, Fatima Ugur, Ruilin Tian
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from __future__ import print_function, division, unicode_literals
import argparse, sys, os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

class HelpParser(argparse.ArgumentParser):
    """
    Like the normal parser except shows the help message on default
    instead of requiring the user to type -h to see it
    """
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

def main(directory):
    # import data as pandas dataframes, the contaminant columns are
    # converted to booleans so that the pandas parser is faster and less
    # grouchy
    psites = pd.read_table(os.path.join(directory, "Phospho (STY)Sites.txt"),
            converters={677: bool, 211:bool, 678:bool})
    prot_groups = pd.read_table(os.path.join(directory, "proteinGroups.txt"),
            converters={421: bool, 422:bool})

    # TODO: Actual implement some kind of analysis here
    yield

if __name__ == "__main__":
    parser = HelpParser(description='Analyze MS data for the Et0h group')
    parser.add_argument('-d','--dir', help='Path to the directory\
            containing the MaxQuant files', required=True)
    args = vars(parser.parse_args())

    if os.path.isdir(args["dir"]):
        main(args["dir"])
    else:
        sys.stderr.write('error: Not a valid directory\n')
        sys.exit(2)
<commit_msg>improve robustness by indexing using column names vs integer index<commit_after>#!/usr/bin/env python
#
# Copyright 2015 Tamas Nagy, Fatima Ugur, Ruilin Tian
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from __future__ import print_function, division, unicode_literals
import argparse, sys, os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

class HelpParser(argparse.ArgumentParser):
    """
    Like the normal parser except shows the help message on default
    instead of requiring the user to type -h to see it
    """
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

def main(directory):
    # import data as pandas dataframes, the contaminant columns are
    # converted to booleans so that the pandas parser is faster and less
    # grouchy
    psites = pd.read_table(os.path.join(directory, "Phospho (STY)Sites.txt"),
            converters={"Reverse": bool, "Diagnostic peak":bool,
                "Potential contaminant":bool})
    prot_groups = pd.read_table(os.path.join(directory, "proteinGroups.txt"),
            converters={"Reverse": bool, "Potential contaminant":bool})

    # TODO: Actual implement some kind of analysis here
    yield

if __name__ == "__main__":
    parser = HelpParser(description='Analyze MS data for the Et0h group')
    parser.add_argument('-d','--dir', help='Path to the directory\
            containing the MaxQuant files', required=True)
    args = vars(parser.parse_args())

    if os.path.isdir(args["dir"]):
        main(args["dir"])
    else:
        sys.stderr.write('error: Not a valid directory\n')
        sys.exit(2)
<|endoftext|>