<commit_before>"""

fs.remote
=========

Utilities for interfacing with remote filesystems

This module provides reusable utility functions that can be used to construct
FS subclasses interfacing with a remote filesystem.  These include:

  * RemoteFileBuffer:  a file-like object that locally buffers the contents of
                       a remote file, writing them back on flush() or close().

  * ConnectionManagerFS:  a WrapFS subclass that tracks the connection state
                          of a remote FS, and allows client code to wait for
                          a connection to be re-established.

   * CacheFS:  a WrapFS subclass that caces file and directory meta-data in
               memory, to speed access to a remote FS.

"""

import time
import copy

from fs.base import FS, threading
from fs.wrapfs import WrapFS, wrap_fs_methods
from fs.wrapfs.lazyfs import LazyFS
from fs.path import *
from fs.errors import *

try:
   from tempfile import SpooledTemporaryFile as TempFile
except ImportError:
   from tempfile import NamedTemporaryFile as TempFile

class RemoteFileBuffer(object):
    """File-like object providing buffer for local file operations.

    Instances of this class manage a local tempfile buffer corresponding
    to the contents of a remote file.  All reads and writes happen locally,
    with the content being copied to the remote file only on flush() or
    close().  Writes to the remote file are performed using the setcontents()
    method on the owning FS object.

    The intended use-case is for a remote filesystem (e.g. S3FS) to return
    instances of this class from its open() method, and to provide the
    file-uploading logic in its setcontents() method, as in the following
    pseudo-code::

        def open(self,path,mode="r"):
            rf = self._get_remote_file(path)
            return RemoteFileBuffer(self,path,mode,rf)

        def setcontents(self,path,file):
            self._put_remote_file(path,file)

    The current implementation reads the entire contents of the file into
    the buffer before returning.  Future implementations may pull data into
    the buffer on demand.
    """

    def __init__(self,fs,path,mode,rfile=None):
        """RemoteFileBuffer constructor.

        The owning filesystem, path and mode must be provided.  If the
        optional argument 'rfile' is provided, it must be a read()-able
        object or a string containing the initial file contents.
        """
        self.file = TempFile()
        self.fs = fs
        self.path = path
        self.mode = mode
        self.closed = False
        self._flushed = False
        if getattr(fs,"_lock",None) is not None:
            self._lock = fs._lock.__class__()
        else:
            self._lock = threading.RLock()
        if "r" in mode or "+" in mode or "a" in mode:
            if rfile is not None:
                if hasattr(rfile,"read"):
                    data = rfile.read(1024*256)
                    while data:
                        self.file.write(data)
                        data = rfile.read(1024*256)
                else:
                    self.file.write(str(rfile))
                if "a" not in mode:
                    self.file.seek(0)

    def __del__(self):
        if not self.closed:
            self.close()

    def __getattr__(self,name):
        file = self.__dict__['file']
        a = getattr(file, name)
        if not callable(a):
            return a
        @wraps(a)
        def call_with_lock(*args,**kwds):
            self._lock.acquire()
            try:
                if "write" in name:
                    self._flushed = False
                return a(*args,**kwds)
            finally:
                self._lock.release()
        setattr(self, name, call_with_lock)
        return call_with_lock

    def __enter__(self):
        self.file.__enter__()
        return self

    def __exit__(self,exc,value,tb):
        self.close()
        return False

    def __iter__(self):
        return iter(self.file)

    def truncate(self,size=None):
        self._lock.acquire()
        try:
            self.file.truncate(size)
            self.flush()
        finally:
            self._lock.release()

    def flush(self):
        self._lock.acquire()
        try:
            self.file.flush()
            if "w" in self.mode or "a" in self.mode or "+" in self.mode:
                if not self._flushed:
                    pos = self.file.tell()
                    self.file.seek(0)
                    self.fs.setcontents(self.path,self.file)
                    self.file.seek(pos)
                    self._flushed = True
        finally:
            self._lock.release()

    def close(self):
        self._lock.acquire()
        try:
            if not self.closed:
                self.closed = True
                if "w" in self.mode or "a" in self.mode or "+" in self.mode:
                    if not self._flushed:
                        self.file.seek(0)
                        self.file.seek(0)
                        self.fs.setcontents(self.path,self.file)
                self.file.close()
        finally:
            self._lock.release()

class ConnectionManagerFS(LazyFS):
    """FS wrapper providing simple connection management of a remote FS.

    The ConnectionManagerFS class is designed to wrap a remote FS object
    and provide some convenience methods for dealing with its remote
    connection state.

    The boolean attribute 'connected' indicates whether the remote fileystem
    has an active connection, and is initially True.  If any of the remote
    filesystem methods raises a RemoteConnectionError, 'connected' will
    switch to False and remain so until a successful remote method call.

    Application code can use the method 'wait_for_connection' to block
    until the connection is re-established.  Currently this reconnection
    is checked by a simple polling loop; eventually more sophisticated
    operating-system integration may be added.

    Since some remote FS classes can raise RemoteConnectionError during
    initialisation, this class makes use of lazy initialization. The
    remote FS can be specified as an FS instance, an FS subclass, or a
    (class,args) or (class,args,kwds) tuple. For example::

        >>> fs = ConnectionManagerFS(MyRemoteFS("http://www.example.com/"))
        Traceback (most recent call last):
            ...
        RemoteConnectionError: couldn't connect to "http://www.example.com/"
        >>> fs = ConnectionManagerFS((MyRemoteFS,["http://www.example.com/"]))
        >>> fs.connected
        False
        >>>

    """

    poll_interval = 1

    def __init__(self,wrapped_fs,poll_interval=None,connected=True):
        super(ConnectionManagerFS,self).__init__(wrapped_fs)
        if poll_interval is not None:
            self.poll_interval = poll_interval
        self._connection_cond = threading.Condition()
        self._poll_thread = None
        self._poll_sleeper = threading.Event()
        self.connected = connected

    def setcontents(self,path,data):
        return self.wrapped_fs.setcontents(path,data)

    def __getstate__(self):
        state = super(ConnectionManagerFS,self).__getstate__()
        del state["_connection_cond"]
        del state["_poll_sleeper"]
        state["_poll_thread"] = None
        return state

    def __setstate__(self,state):
        super(ConnectionManagerFS,self).__setstate__(state)
        self._connection_cond = threading.Condition()
        self._poll_sleeper = threading.Event()

    def wait_for_connection(self,timeout=None):
        self._connection_cond.acquire()
        try:
            if not self.connected:
                if not self._poll_thread:
                    target = self._poll_connection
                    self._poll_thread = threading.Thread(target=target)
                    self._poll_thread.start()
                self._connection_cond.wait(timeout)
        finally:
            self._connection_cond.release()

    def _poll_connection(self):
        while not self.connected and not self.closed:
            try:
                self.wrapped_fs.isdir("")
            except RemoteConnectionError:
                self._poll_sleeper.wait(self.poll_interval)
                self._poll_sleeper.clear()
            except FSError:
                break
            else:
                break
        self._connection_cond.acquire()
        try:
            if not self.closed:
                self.connected = True
            self._poll_thread = None
            self._connection_cond.notifyAll()
        finally:
            self._connection_cond.release()

    def close(self):
        if not self.closed:
            try:
                super(ConnectionManagerFS,self).close()
            except (RemoteConnectionError,):
                pass
            if self._poll_thread:
                self.connected = True
                self._poll_sleeper.set()
                self._poll_thread.join()
                self._poll_thread = None

def _ConnectionManagerFS_method_wrapper(func):
    """Method wrapper for ConnectionManagerFS.

    This method wrapper keeps an eye out for RemoteConnectionErrors and
    adjusts self.connected accordingly.
    """
    @wraps(func)
    def wrapper(self,*args,**kwds):
        try:
            result = func(self,*args,**kwds)
        except RemoteConnectionError:
            self.connected = False
            raise
        except FSError:
            self.connected = True
            raise
        else:
            self.connected = True
            return result
    return wrapper

wrap_fs_methods(_ConnectionManagerFS_method_wrapper,ConnectionManagerFS)

def _cached_method(func):
    """Method decorator that caches results for CacheFS."""
    @wraps(func)
    def wrapper(self,path="",*args,**kwds):
        try:
            (success,result) = self._cache_get(path,func.__name__,args,kwds)
        except KeyError:
            try:
                res = func(self,path,*args,**kwds)
            except Exception, e:
                self._cache_set(path,func.__name__,args,kwds,(False,e))
                raise
            else:
                self._cache_set(path,func.__name__,args,kwds,(True,res))
                return copy.copy(res)
        else:
            if not success:
                raise result
            else:
                return copy.copy(result)
    return wrapper

class CacheFS(WrapFS):
    """Simple wrapper to cache meta-data of a remote filesystems.

    This FS wrapper implements a simplistic cache that can help speed up
    access to a remote filesystem.  File and directory meta-data is cached
    but the actual file contents are not.
    """

    def __init__(self,fs,timeout=1):
        """CacheFS constructor.

        The optional argument 'timeout' specifies the cache timeout in
        seconds.  The default timeout is 1 second.  To prevent cache
        entries from ever timing out, set it to None.
        """
        self.timeout = timeout
        self._cache = {"":{}}
        super(CacheFS,self).__init__(fs)

    def _path_cache(self,path):
        cache = self._cache
        for name in iteratepath(path):
            cache = cache.setdefault(name,{"":{}})
        return cache

    def _cache_get(self,path,func,args,kwds):
        now = time.time()
        cache = self._path_cache(path)
        key = (tuple(args),tuple(sorted(kwds.iteritems())))
        (t,v) = cache[""][func][key]
        if self.timeout is not None:
            if t < now - self.timeout:
                raise KeyError
        return v

    def _cache_set(self,path,func,args,kwds,v):
        t = time.time()
        cache = self._path_cache(path)
        key = (tuple(args),tuple(sorted(kwds.iteritems())))
        cache[""].setdefault(func,{})[key] = (t,v)

    def _uncache(self,path,added=False,removed=False,unmoved=False):
        cache = self._cache
        names = list(iteratepath(path))
        # If it's not the root dir, also clear some items for ancestors
        if names:
            # Clear cached 'getinfo' and 'getsize' for all ancestors
            for name in names[:-1]:
                cache[""].pop("getinfo",None)
                cache[""].pop("getsize",None)
                cache = cache.get(name,None)
                if cache is None:
                    return
            # Adjust cached 'listdir' for parent directory.
            # TODO: account for whether it was added, removed, or unmoved
            cache[""].pop("getinfo",None)
            cache[""].pop("getsize",None)
            cache[""].pop("listdir",None)
        # Clear all cached info for the path itself.
        cache[names[-1]] = {"":{}}

    @_cached_method
    def exists(self,path):
        return super(CacheFS,self).exists(path)

    @_cached_method
    def isdir(self,path):
        return super(CacheFS,self).isdir(path)

    @_cached_method
    def isfile(self,path):
        return super(CacheFS,self).isfile(path)

    @_cached_method
    def listdir(self,path="",**kwds):
        return super(CacheFS,self).listdir(path,**kwds)

    @_cached_method
    def getinfo(self,path):
        return super(CacheFS,self).getinfo(path)

    @_cached_method
    def getsize(self,path):
        return super(CacheFS,self).getsize(path)

    @_cached_method
    def getxattr(self,path,name,default=None):
        return super(CacheFS,self).getxattr(path,name,default)

    @_cached_method
    def listxattrs(self,path):
        return super(CacheFS,self).listxattrs(path)

    def open(self,path,mode="r"):
        f = super(CacheFS,self).open(path,mode)
        self._uncache(path,unmoved=True)
        return f

    def setcontents(self,path,contents):
        res = super(CacheFS,self).setcontents(path,contents)
        self._uncache(path,unmoved=True)
        return res

    def getcontents(self,path):
        res = super(CacheFS,self).getcontents(path)
        self._uncache(path,unmoved=True)
        return res

    def makedir(self,path,**kwds):
        super(CacheFS,self).makedir(path,**kwds)
        self._uncache(path,added=True)

    def remove(self,path):
        super(CacheFS,self).remove(path)
        self._uncache(path,removed=True)

    def removedir(self,path,**kwds):
        super(CacheFS,self).removedir(path,**kwds)
        self._uncache(path,removed=True)

    def rename(self,src,dst):
        super(CacheFS,self).rename(src,dst)
        self._uncache(src,removed=True)
        self._uncache(dst,added=True)

    def copy(self,src,dst,**kwds):
        super(CacheFS,self).copy(src,dst,**kwds)
        self._uncache(dst,added=True)

    def copydir(self,src,dst,**kwds):
        super(CacheFS,self).copydir(src,dst,**kwds)
        self._uncache(dst,added=True)

    def move(self,src,dst,**kwds):
        super(CacheFS,self).move(src,dst,**kwds)
        self._uncache(src,removed=True)
        self._uncache(dst,added=True)

    def movedir(self,src,dst,**kwds):
        super(CacheFS,self).movedir(src,dst,**kwds)
        self._uncache(src,removed=True)
        self._uncache(dst,added=True)

    def setxattr(self,path,name,value):
        self._uncache(path,unmoved=True)
        return super(CacheFS,self).setxattr(path,name,value)

    def delxattr(self,path,name):
        self._uncache(path,unmoved=True)
        return super(CacheFS,self).delxattr(path,name)

<commit_msg>ensure max_size is always given to SpooledTemporaryFile<commit_after>"""

fs.remote
=========

Utilities for interfacing with remote filesystems

This module provides reusable utility functions that can be used to construct
FS subclasses interfacing with a remote filesystem.  These include:

  * RemoteFileBuffer:  a file-like object that locally buffers the contents of
                       a remote file, writing them back on flush() or close().

  * ConnectionManagerFS:  a WrapFS subclass that tracks the connection state
                          of a remote FS, and allows client code to wait for
                          a connection to be re-established.

   * CacheFS:  a WrapFS subclass that caches file and directory meta-data in
               memory, to speed access to a remote FS.

"""

import time
import copy

from fs.base import FS, threading
from fs.wrapfs import WrapFS, wrap_fs_methods
from fs.wrapfs.lazyfs import LazyFS
from fs.path import *
from fs.errors import *

try:
   from tempfile import SpooledTemporaryFile
except ImportError:
   from tempfile import NamedTemporaryFile
   class SpooledTemporaryFile(NamedTemporaryFile):
        def __init__(self,max_size=0,*args,**kwds):
            NamedTemporaryFile.__init__(self,*args,**kwds)

class RemoteFileBuffer(object):
    """File-like object providing buffer for local file operations.

    Instances of this class manage a local tempfile buffer corresponding
    to the contents of a remote file.  All reads and writes happen locally,
    with the content being copied to the remote file only on flush() or
    close().  Writes to the remote file are performed using the setcontents()
    method on the owning FS object.

    The intended use-case is for a remote filesystem (e.g. S3FS) to return
    instances of this class from its open() method, and to provide the
    file-uploading logic in its setcontents() method, as in the following
    pseudo-code::

        def open(self,path,mode="r"):
            rf = self._get_remote_file(path)
            return RemoteFileBuffer(self,path,mode,rf)

        def setcontents(self,path,file):
            self._put_remote_file(path,file)

    The current implementation reads the entire contents of the file into
    the buffer before returning.  Future implementations may pull data into
    the buffer on demand.
    """

    max_size_in_memory = 1024 * 8

    def __init__(self,fs,path,mode,rfile=None):
        """RemoteFileBuffer constructor.

        The owning filesystem, path and mode must be provided.  If the
        optional argument 'rfile' is provided, it must be a read()-able
        object or a string containing the initial file contents.
        """
        self.file = SpooledTemporyFile(max_size=self.max_size_in_memory)
        self.fs = fs
        self.path = path
        self.mode = mode
        self.closed = False
        self._flushed = False
        if getattr(fs,"_lock",None) is not None:
            self._lock = fs._lock.__class__()
        else:
            self._lock = threading.RLock()
        if "r" in mode or "+" in mode or "a" in mode:
            if rfile is not None:
                if hasattr(rfile,"read"):
                    data = rfile.read(1024*256)
                    while data:
                        self.file.write(data)
                        data = rfile.read(1024*256)
                else:
                    self.file.write(str(rfile))
                if "a" not in mode:
                    self.file.seek(0)

    def __del__(self):
        if not self.closed:
            self.close()

    def __getattr__(self,name):
        file = self.__dict__['file']
        a = getattr(file, name)
        if not callable(a):
            return a
        @wraps(a)
        def call_with_lock(*args,**kwds):
            self._lock.acquire()
            try:
                if "write" in name:
                    self._flushed = False
                return a(*args,**kwds)
            finally:
                self._lock.release()
        setattr(self, name, call_with_lock)
        return call_with_lock

    def __enter__(self):
        self.file.__enter__()
        return self

    def __exit__(self,exc,value,tb):
        self.close()
        return False

    def __iter__(self):
        return iter(self.file)

    def truncate(self,size=None):
        self._lock.acquire()
        try:
            self.file.truncate(size)
            self.flush()
        finally:
            self._lock.release()

    def flush(self):
        self._lock.acquire()
        try:
            self.file.flush()
            if "w" in self.mode or "a" in self.mode or "+" in self.mode:
                if not self._flushed:
                    pos = self.file.tell()
                    self.file.seek(0)
                    self.fs.setcontents(self.path,self.file)
                    self.file.seek(pos)
                    self._flushed = True
        finally:
            self._lock.release()

    def close(self):
        self._lock.acquire()
        try:
            if not self.closed:
                self.closed = True
                if "w" in self.mode or "a" in self.mode or "+" in self.mode:
                    if not self._flushed:
                        self.file.seek(0)
                        self.file.seek(0)
                        self.fs.setcontents(self.path,self.file)
                self.file.close()
        finally:
            self._lock.release()

class ConnectionManagerFS(LazyFS):
    """FS wrapper providing simple connection management of a remote FS.

    The ConnectionManagerFS class is designed to wrap a remote FS object
    and provide some convenience methods for dealing with its remote
    connection state.

    The boolean attribute 'connected' indicates whether the remote fileystem
    has an active connection, and is initially True.  If any of the remote
    filesystem methods raises a RemoteConnectionError, 'connected' will
    switch to False and remain so until a successful remote method call.

    Application code can use the method 'wait_for_connection' to block
    until the connection is re-established.  Currently this reconnection
    is checked by a simple polling loop; eventually more sophisticated
    operating-system integration may be added.

    Since some remote FS classes can raise RemoteConnectionError during
    initialisation, this class makes use of lazy initialization. The
    remote FS can be specified as an FS instance, an FS subclass, or a
    (class,args) or (class,args,kwds) tuple. For example::

        >>> fs = ConnectionManagerFS(MyRemoteFS("http://www.example.com/"))
        Traceback (most recent call last):
            ...
        RemoteConnectionError: couldn't connect to "http://www.example.com/"
        >>> fs = ConnectionManagerFS((MyRemoteFS,["http://www.example.com/"]))
        >>> fs.connected
        False
        >>>

    """

    poll_interval = 1

    def __init__(self,wrapped_fs,poll_interval=None,connected=True):
        super(ConnectionManagerFS,self).__init__(wrapped_fs)
        if poll_interval is not None:
            self.poll_interval = poll_interval
        self._connection_cond = threading.Condition()
        self._poll_thread = None
        self._poll_sleeper = threading.Event()
        self.connected = connected

    def setcontents(self,path,data):
        return self.wrapped_fs.setcontents(path,data)

    def __getstate__(self):
        state = super(ConnectionManagerFS,self).__getstate__()
        del state["_connection_cond"]
        del state["_poll_sleeper"]
        state["_poll_thread"] = None
        return state

    def __setstate__(self,state):
        super(ConnectionManagerFS,self).__setstate__(state)
        self._connection_cond = threading.Condition()
        self._poll_sleeper = threading.Event()

    def wait_for_connection(self,timeout=None):
        self._connection_cond.acquire()
        try:
            if not self.connected:
                if not self._poll_thread:
                    target = self._poll_connection
                    self._poll_thread = threading.Thread(target=target)
                    self._poll_thread.start()
                self._connection_cond.wait(timeout)
        finally:
            self._connection_cond.release()

    def _poll_connection(self):
        while not self.connected and not self.closed:
            try:
                self.wrapped_fs.isdir("")
            except RemoteConnectionError:
                self._poll_sleeper.wait(self.poll_interval)
                self._poll_sleeper.clear()
            except FSError:
                break
            else:
                break
        self._connection_cond.acquire()
        try:
            if not self.closed:
                self.connected = True
            self._poll_thread = None
            self._connection_cond.notifyAll()
        finally:
            self._connection_cond.release()

    def close(self):
        if not self.closed:
            try:
                super(ConnectionManagerFS,self).close()
            except (RemoteConnectionError,):
                pass
            if self._poll_thread:
                self.connected = True
                self._poll_sleeper.set()
                self._poll_thread.join()
                self._poll_thread = None

def _ConnectionManagerFS_method_wrapper(func):
    """Method wrapper for ConnectionManagerFS.

    This method wrapper keeps an eye out for RemoteConnectionErrors and
    adjusts self.connected accordingly.
    """
    @wraps(func)
    def wrapper(self,*args,**kwds):
        try:
            result = func(self,*args,**kwds)
        except RemoteConnectionError:
            self.connected = False
            raise
        except FSError:
            self.connected = True
            raise
        else:
            self.connected = True
            return result
    return wrapper

wrap_fs_methods(_ConnectionManagerFS_method_wrapper,ConnectionManagerFS)

def _cached_method(func):
    """Method decorator that caches results for CacheFS."""
    @wraps(func)
    def wrapper(self,path="",*args,**kwds):
        try:
            (success,result) = self._cache_get(path,func.__name__,args,kwds)
        except KeyError:
            try:
                res = func(self,path,*args,**kwds)
            except Exception, e:
                self._cache_set(path,func.__name__,args,kwds,(False,e))
                raise
            else:
                self._cache_set(path,func.__name__,args,kwds,(True,res))
                return copy.copy(res)
        else:
            if not success:
                raise result
            else:
                return copy.copy(result)
    return wrapper

class CacheFS(WrapFS):
    """Simple wrapper to cache meta-data of a remote filesystems.

    This FS wrapper implements a simplistic cache that can help speed up
    access to a remote filesystem.  File and directory meta-data is cached
    but the actual file contents are not.
    """

    def __init__(self,fs,timeout=1):
        """CacheFS constructor.

        The optional argument 'timeout' specifies the cache timeout in
        seconds.  The default timeout is 1 second.  To prevent cache
        entries from ever timing out, set it to None.
        """
        self.timeout = timeout
        self._cache = {"":{}}
        super(CacheFS,self).__init__(fs)

    def _path_cache(self,path):
        cache = self._cache
        for name in iteratepath(path):
            cache = cache.setdefault(name,{"":{}})
        return cache

    def _cache_get(self,path,func,args,kwds):
        now = time.time()
        cache = self._path_cache(path)
        key = (tuple(args),tuple(sorted(kwds.iteritems())))
        (t,v) = cache[""][func][key]
        if self.timeout is not None:
            if t < now - self.timeout:
                raise KeyError
        return v

    def _cache_set(self,path,func,args,kwds,v):
        t = time.time()
        cache = self._path_cache(path)
        key = (tuple(args),tuple(sorted(kwds.iteritems())))
        cache[""].setdefault(func,{})[key] = (t,v)

    def _uncache(self,path,added=False,removed=False,unmoved=False):
        cache = self._cache
        names = list(iteratepath(path))
        # If it's not the root dir, also clear some items for ancestors
        if names:
            # Clear cached 'getinfo' and 'getsize' for all ancestors
            for name in names[:-1]:
                cache[""].pop("getinfo",None)
                cache[""].pop("getsize",None)
                cache = cache.get(name,None)
                if cache is None:
                    return
            # Adjust cached 'listdir' for parent directory.
            # TODO: account for whether it was added, removed, or unmoved
            cache[""].pop("getinfo",None)
            cache[""].pop("getsize",None)
            cache[""].pop("listdir",None)
        # Clear all cached info for the path itself.
        cache[names[-1]] = {"":{}}

    @_cached_method
    def exists(self,path):
        return super(CacheFS,self).exists(path)

    @_cached_method
    def isdir(self,path):
        return super(CacheFS,self).isdir(path)

    @_cached_method
    def isfile(self,path):
        return super(CacheFS,self).isfile(path)

    @_cached_method
    def listdir(self,path="",**kwds):
        return super(CacheFS,self).listdir(path,**kwds)

    @_cached_method
    def getinfo(self,path):
        return super(CacheFS,self).getinfo(path)

    @_cached_method
    def getsize(self,path):
        return super(CacheFS,self).getsize(path)

    @_cached_method
    def getxattr(self,path,name,default=None):
        return super(CacheFS,self).getxattr(path,name,default)

    @_cached_method
    def listxattrs(self,path):
        return super(CacheFS,self).listxattrs(path)

    def open(self,path,mode="r"):
        f = super(CacheFS,self).open(path,mode)
        self._uncache(path,unmoved=True)
        return f

    def setcontents(self,path,contents):
        res = super(CacheFS,self).setcontents(path,contents)
        self._uncache(path,unmoved=True)
        return res

    def getcontents(self,path):
        res = super(CacheFS,self).getcontents(path)
        self._uncache(path,unmoved=True)
        return res

    def makedir(self,path,**kwds):
        super(CacheFS,self).makedir(path,**kwds)
        self._uncache(path,added=True)

    def remove(self,path):
        super(CacheFS,self).remove(path)
        self._uncache(path,removed=True)

    def removedir(self,path,**kwds):
        super(CacheFS,self).removedir(path,**kwds)
        self._uncache(path,removed=True)

    def rename(self,src,dst):
        super(CacheFS,self).rename(src,dst)
        self._uncache(src,removed=True)
        self._uncache(dst,added=True)

    def copy(self,src,dst,**kwds):
        super(CacheFS,self).copy(src,dst,**kwds)
        self._uncache(dst,added=True)

    def copydir(self,src,dst,**kwds):
        super(CacheFS,self).copydir(src,dst,**kwds)
        self._uncache(dst,added=True)

    def move(self,src,dst,**kwds):
        super(CacheFS,self).move(src,dst,**kwds)
        self._uncache(src,removed=True)
        self._uncache(dst,added=True)

    def movedir(self,src,dst,**kwds):
        super(CacheFS,self).movedir(src,dst,**kwds)
        self._uncache(src,removed=True)
        self._uncache(dst,added=True)

    def setxattr(self,path,name,value):
        self._uncache(path,unmoved=True)
        return super(CacheFS,self).setxattr(path,name,value)

    def delxattr(self,path,name):
        self._uncache(path,unmoved=True)
        return super(CacheFS,self).delxattr(path,name)

<|endoftext|>


<commit_before>import subprocess
import sys
import os
import setup_util

def start(args, logfile, errfile):
  setup_util.replace_text("onion/hello.c", "mysql_real_connect\(data.db\[i\], \".*\",", "mysql_real_connect(data.db[i], \"" + args.database_host + "\",")

  subprocess.call("rm *.o", cwd="onion", shell=True, stderr=errfile, stdout=logfile)
  subprocess.call("cp -R $IROOT/onion/* onion/onion", shell=True, stderr=errfile, stdout=logfile)
  subprocess.call("rm CMakeCache.txt", shell=True, cwd="onion/onion/build", stderr=errfile, stdout=logfile)
  subprocess.Popen("make && ./hello", shell=True, cwd="onion", stderr=errfile, stdout=logfile)
  return 0

def stop(logfile, errfile):

  p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
  out, err = p.communicate()
  for line in out.splitlines():
    if 'hello' in line:
      pid = int(line.split(None, 2)[1])
      os.kill(pid, 15)
  return 0
<commit_msg>Remove minor errors in onion<commit_after>import subprocess
import sys
import os
import setup_util

def start(args, logfile, errfile):
  setup_util.replace_text("onion/hello.c", "mysql_real_connect\(data.db\[i\], \".*\",", "mysql_real_connect(data.db[i], \"" + args.database_host + "\",")

  subprocess.call("rm -f *.o", cwd="onion", shell=True, stderr=errfile, stdout=logfile)
  subprocess.call("cp -R $IROOT/onion/ onion/onion", shell=True, stderr=errfile, stdout=logfile)
  subprocess.call("rm CMakeCache.txt", shell=True, cwd="onion/onion/build", stderr=errfile, stdout=logfile)
  subprocess.Popen("make && ./hello", shell=True, cwd="onion", stderr=errfile, stdout=logfile)
  return 0

def stop(logfile, errfile):

  p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
  out, err = p.communicate()
  for line in out.splitlines():
    if 'hello' in line:
      pid = int(line.split(None, 2)[1])
      os.kill(pid, 15)
  return 0
<|endoftext|>


<commit_before>"""

  fs.wrapfs:  class for wrapping an existing FS object with added functionality

This module provides the class WrapFS, a base class for objects that wrap
another FS object and provide some transformation of its contents.  It could
be very useful for implementing e.g. transparent encryption or compression
services.

As a simple example of how this class could be used, the 'HideDotFiles' class
implements the standard unix shell functionality of hiding dot files in
directory listings.

"""

from fnmatch import fnmatch

from fs.base import FS, threading, synchronize
from fs.errors import *

def rewrite_errors(func):
    @wraps(func)
    def wrapper(self,*args,**kwds):
        try:
            return func(self,*args,**kwds)
        except ResourceError, e:
            try:
                e.path = self._decode(e.path)
            except AttributeError:
                pass
            raise
    return wrapper

class WrapFS(FS):
    """FS that wraps another FS, providing translation etc.

    This class allows simple transforms to be applied to the names
    and/or contents of files in an FS.  It could be used to implement
    e.g. compression or encryption in a relatively painless manner.

    The following methods can be overridden to control how files are
    accessed in the underlying FS object:

        _file_wrap(file,mode):  called for each file that is opened from
                                the underlying FS; may return a modified
                                file-like object.

        _encode(path):  encode a path for access in the underlying FS

        _decode(path):  decode a path from the underlying FS

    If the required path translation proceeds one component at a time,
    it may be simpler to override the _encode_name() and _decode_name()
    methods.
    """

    def __init__(self,fs):
        super(WrapFS,self).__init__()
        try:
            self._lock = fs._lock
        except (AttributeError,FSError):
            self._lock = None
        self.wrapped_fs = fs

    def _file_wrap(self,f,mode):
        """Apply wrapping to an opened file."""
        return f

    def _encode_name(self,name):
        """Encode path component for the underlying FS."""
        return name

    def _decode_name(self,name):
        """Decode path component from the underlying FS."""
        return name

    def _encode(self,path):
        """Encode path for the underlying FS."""
        names = path.split("/")
        e_names = []
        for name in names:
            if name == "":
                e_names.append("")
            else:
                e_names.append(self._encode_name(name))
        return "/".join(e_names)

    def _decode(self,path):
        """Decode path from the underlying FS."""
        names = path.split("/")
        d_names = []
        for name in names:
            if name == "":
                d_names.append("")
            else:
                d_names.append(self._decode_name(name))
        return "/".join(d_names)

    def _adjust_mode(self,mode):
        """Adjust the mode used to open a file in the underlying FS.

        This method takes the mode given when opening a file, and should
        return a two-tuple giving the mode to be used in this FS as first
        item, and the mode to be used in the underlying FS as the second.

        An example of why this is needed is a WrapFS subclass that does
        transparent file compression - in this case files from the wrapped
        FS cannot be opened in append mode.
        """
        return (mode,mode)

    @rewrite_errors
    def getsyspath(self,path,allow_none=False):
        return self.wrapped_fs.getsyspath(self._encode(path),allow_none)

    @rewrite_errors
    def hassyspath(self,path):
        return self.wrapped_fs.hassyspath(self._encode(path))

    @rewrite_errors
    def open(self,path,mode="r"):
        (mode,wmode) = self._adjust_mode(mode)
        f = self.wrapped_fs.open(self._encode(path),wmode)
        return self._file_wrap(f,mode)

    @rewrite_errors
    def setcontents(self,path,contents):
        self.wrapped_fs.setcontents(self._encode(path),contents)

    @rewrite_errors
    def exists(self,path):
        return self.wrapped_fs.exists(self._encode(path))

    @rewrite_errors
    def isdir(self,path):
        return self.wrapped_fs.isdir(self._encode(path))

    @rewrite_errors
    def isfile(self,path):
        return self.wrapped_fs.isfile(self._encode(path))

    @rewrite_errors
    def listdir(self,path="",**kwds):
        wildcard = kwds.pop("wildcard","*")
        info = kwds.get("info",False)
        entries = []
        for e in self.wrapped_fs.listdir(self._encode(path),**kwds):
            if info:
                e = e.copy()
                e["name"] = self._decode(e["name"])
                if wildcard is not None and not fnmatch(e["name"],wildcard):
                    continue
            else:
                e = self._decode(e)
                if wildcard is not None and not fnmatch(e,wildcard):
                    continue
            entries.append(e)
        return entries

    @rewrite_errors
    def makedir(self,path,*args,**kwds):
        return self.wrapped_fs.makedir(self._encode(path),*args,**kwds)

    @rewrite_errors
    def remove(self,path):
        return self.wrapped_fs.remove(self._encode(path))

    @rewrite_errors
    def removedir(self,path,*args,**kwds):
        return self.wrapped_fs.removedir(self._encode(path),*args,**kwds)

    @rewrite_errors
    def rename(self,src,dst):
        return self.wrapped_fs.rename(self._encode(src),self._encode(dst))

    @rewrite_errors
    def getinfo(self,path):
        return self.wrapped_fs.getinfo(self._encode(path))

    @rewrite_errors
    def desc(self,path):
        return self.wrapped_fs.desc(self._encode(path))

    @rewrite_errors
    def copy(self,src,dst,**kwds):
        return self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def move(self,src,dst,**kwds):
        return self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def movedir(self,src,dst,**kwds):
        return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def copydir(self,src,dst,**kwds):
        return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def getxattr(self,path,name,default=None):
        try:
            return self.wrapped_fs.getxattr(self._encode(path),name,default)
        except AttributeError:
            raise UnsupportedError("getxattr")

    @rewrite_errors
    def setxattr(self,path,name,value):
        try:
            return self.wrapped_fs.setxattr(self._encode(path),name,value)
        except AttributeError:
            raise UnsupportedError("setxattr")

    @rewrite_errors
    def delxattr(self,path,name):
        try:
            return self.wrapped_fs.delxattr(self._encode(path),name)
        except AttributeError:
            raise UnsupportedError("delxattr")

    @rewrite_errors
    def listxattrs(self,path):
        try:
            return self.wrapped_fs.listxattrs(self._encode(path))
        except AttributeError:
            raise UnsupportedError("listxattrs")

    def __getattr__(self,attr):
        return getattr(self.wrapped_fs,attr)

    @rewrite_errors
    def close(self):
        if hasattr(self.wrapped_fs,"close"):
            self.wrapped_fs.close()

def wrap_fs_methods(decorator,cls=None):
    """Apply the given decorator to all FS methods on the given class.

    This function can be used in two ways.  When called with two arguments it
    applies the given function 'decorator' to each FS method of the given
    class.  When called with just a single argument, it creates and returns
    a class decorator which will do the same thing when applied.  So you can
    use it like this:

        wrap_fs_methods(mydecorator,MyFSClass)

    Or on more recent Python versions, like this:

        @wrap_fs_methods(mydecorator)
        class MyFSClass(FS):
            ...

    """
    methods = ("open","exists","isdir","isfile","listdir","makedir","remove",
               "removedir","rename","getinfo","copy","move","copydir",
               "movedir","close","getxattr","setxattr","delxattr","listxattrs")
    def apply_decorator(cls):
        for method_name in methods:
            method = getattr(cls,method_name,None)
            if method is not None:
                setattr(cls,method_name,decorator(method))
        return cls
    if cls is not None:
        return apply_decorator(cls)
    else:
        return apply_decorator

class HideDotFiles(WrapFS):
    """FS wrapper class that hides dot-files in directory listings.

    The listdir() function takes an extra keyword argument 'hidden'
    indicating whether hidden dotfiles shoud be included in the output.
    It is False by default.
    """

    def is_hidden(self,path):
        """Check whether the given path should be hidden."""
        return path and basename(path)[0] == "."

    def _encode(self,path):
        return path

    def _decode(self,path):
        return path

    def listdir(self,path="",**kwds):
        hidden = kwds.pop("hidden",True)
        entries = self.wrapped_fs.listdir(path,**kwds)
        if not hidden:
            entries = [e for e in entries if not self.is_hidden(e)]
        return entries

    def walk(self, path="/", wildcard=None, dir_wildcard=None, search="breadth",hidden=False):
        if search == "breadth":
            dirs = [path]
            while dirs:
                current_path = dirs.pop()
                paths = []
                for filename in self.listdir(current_path,hidden=hidden):
                    path = pathjoin(current_path, filename)
                    if self.isdir(path):
                        if dir_wildcard is not None:
                            if fnmatch(path, dir_wilcard):
                                dirs.append(path)
                        else:
                            dirs.append(path)
                    else:
                        if wildcard is not None:
                            if fnmatch(path, wildcard):
                                paths.append(filename)
                        else:
                            paths.append(filename)
                yield (current_path, paths)
        elif search == "depth":
            def recurse(recurse_path):
                for path in self.listdir(recurse_path, wildcard=dir_wildcard, full=True, dirs_only=True,hidden=hidden):
                    for p in recurse(path):
                        yield p
                yield (recurse_path, self.listdir(recurse_path, wildcard=wildcard, files_only=True,hidden=hidden))
            for p in recurse(path):
                yield p
        else:
            raise ValueError("Search should be 'breadth' or 'depth'")

    def isdirempty(self, path):
        path = normpath(path)
        iter_dir = iter(self.listdir(path,hidden=True))
        try:
            iter_dir.next()
        except StopIteration:
            return True
        return False

class LimitSize(WrapFS):

    def __init__(self,fs,max_size):
        self.max_size = max_size
        super(LimitSize,self).__init__(fs)
        self.cur_size = sum(self.getsize(f) for f in self.walkfiles())

    @synchronize
    def open(self,path,mode="r"):
        try:
            size = self.getsize(path)
        except ResourceNotFoundError:
            size = 0
        f = super(LimitSize,self).open(path,mode)
        if "w" in mode:
            self.cur_size -= size
            size = 0
        return LimitSizeFile(self,f,mode,size)

    def _acquire_space(self,size):
        new_size = self.cur_size + size
        if new_size > self.max_size:
            raise StorageSpaceError("write")
        self.cur_size = new_size

    #  Disable use of system-level paths, so that copy/copydir have to
    #  fall back to manual writes and pass through _acquire_space.
    getsyspath = FS.getsyspath
    copy = FS.copy
    copydir = FS.copydir

    @synchronize
    def remove(self,path):
        size = self.getsize(path)
        super(LimitSize,self).remove(path)
        self.cur_size -= size

    @synchronize
    def removedir(self,path,recursive=False,force=False):
        size = sum(self.getsize(f) for f in self.walkfiles(path))
        super(LimitSize,self).removedir(path,recursive=recursive,force=force)
        self.cur_size -= size

class LimitSizeFile(object):

    def __init__(self,fs,file,mode,size):
        self._lock = fs._lock
        self.fs = fs
        self.file = file
        self.mode = mode
        self.size = size
        self.closed = False

    @synchronize
    def write(self,data):
        pos = self.file.tell()
        if pos > self.size:
            self.size = pos
        diff = pos + len(data) - self.size
        if diff <= 0:
            self.file.write(data)
        else:
            self.fs._acquire_space(diff)
            self.file.write(data)
            self.size += diff

    def writelines(self,lines):
        for line in lines:
            self.write(line)

    @synchronize
    def truncate(self,size=None):
        pos = self.file.tell()
        if size is None:
            size = pos
        self.fs._acquire_space(size - self.size)
        self.file.truncate(size)
        self.size = size

    #  This is lifted straight from the stdlib's tempfile.py
    def __getattr__(self,name):
        file = self.__dict__['file']
        a = getattr(file, name)
        if not issubclass(type(a), type(0)):
            setattr(self, name, a)
        return a

    def __enter__(self):
        self.file.__enter__()
        return self

    def __exit__(self,exc,value,tb):
        self.close()
        return False

    def __iter__(self):
        return iter(self.file)

<commit_msg>wrap_fs_methods:  also direclty wrap setcontents()<commit_after>"""

  fs.wrapfs:  class for wrapping an existing FS object with added functionality

This module provides the class WrapFS, a base class for objects that wrap
another FS object and provide some transformation of its contents.  It could
be very useful for implementing e.g. transparent encryption or compression
services.

As a simple example of how this class could be used, the 'HideDotFiles' class
implements the standard unix shell functionality of hiding dot files in
directory listings.

"""

from fnmatch import fnmatch

from fs.base import FS, threading, synchronize
from fs.errors import *

def rewrite_errors(func):
    @wraps(func)
    def wrapper(self,*args,**kwds):
        try:
            return func(self,*args,**kwds)
        except ResourceError, e:
            try:
                e.path = self._decode(e.path)
            except AttributeError:
                pass
            raise
    return wrapper

class WrapFS(FS):
    """FS that wraps another FS, providing translation etc.

    This class allows simple transforms to be applied to the names
    and/or contents of files in an FS.  It could be used to implement
    e.g. compression or encryption in a relatively painless manner.

    The following methods can be overridden to control how files are
    accessed in the underlying FS object:

        _file_wrap(file,mode):  called for each file that is opened from
                                the underlying FS; may return a modified
                                file-like object.

        _encode(path):  encode a path for access in the underlying FS

        _decode(path):  decode a path from the underlying FS

    If the required path translation proceeds one component at a time,
    it may be simpler to override the _encode_name() and _decode_name()
    methods.
    """

    def __init__(self,fs):
        super(WrapFS,self).__init__()
        try:
            self._lock = fs._lock
        except (AttributeError,FSError):
            self._lock = None
        self.wrapped_fs = fs

    def _file_wrap(self,f,mode):
        """Apply wrapping to an opened file."""
        return f

    def _encode_name(self,name):
        """Encode path component for the underlying FS."""
        return name

    def _decode_name(self,name):
        """Decode path component from the underlying FS."""
        return name

    def _encode(self,path):
        """Encode path for the underlying FS."""
        names = path.split("/")
        e_names = []
        for name in names:
            if name == "":
                e_names.append("")
            else:
                e_names.append(self._encode_name(name))
        return "/".join(e_names)

    def _decode(self,path):
        """Decode path from the underlying FS."""
        names = path.split("/")
        d_names = []
        for name in names:
            if name == "":
                d_names.append("")
            else:
                d_names.append(self._decode_name(name))
        return "/".join(d_names)

    def _adjust_mode(self,mode):
        """Adjust the mode used to open a file in the underlying FS.

        This method takes the mode given when opening a file, and should
        return a two-tuple giving the mode to be used in this FS as first
        item, and the mode to be used in the underlying FS as the second.

        An example of why this is needed is a WrapFS subclass that does
        transparent file compression - in this case files from the wrapped
        FS cannot be opened in append mode.
        """
        return (mode,mode)

    @rewrite_errors
    def getsyspath(self,path,allow_none=False):
        return self.wrapped_fs.getsyspath(self._encode(path),allow_none)

    @rewrite_errors
    def hassyspath(self,path):
        return self.wrapped_fs.hassyspath(self._encode(path))

    @rewrite_errors
    def open(self,path,mode="r"):
        (mode,wmode) = self._adjust_mode(mode)
        f = self.wrapped_fs.open(self._encode(path),wmode)
        return self._file_wrap(f,mode)

    @rewrite_errors
    def setcontents(self,path,contents):
        self.wrapped_fs.setcontents(self._encode(path),contents)

    @rewrite_errors
    def exists(self,path):
        return self.wrapped_fs.exists(self._encode(path))

    @rewrite_errors
    def isdir(self,path):
        return self.wrapped_fs.isdir(self._encode(path))

    @rewrite_errors
    def isfile(self,path):
        return self.wrapped_fs.isfile(self._encode(path))

    @rewrite_errors
    def listdir(self,path="",**kwds):
        wildcard = kwds.pop("wildcard","*")
        info = kwds.get("info",False)
        entries = []
        for e in self.wrapped_fs.listdir(self._encode(path),**kwds):
            if info:
                e = e.copy()
                e["name"] = self._decode(e["name"])
                if wildcard is not None and not fnmatch(e["name"],wildcard):
                    continue
            else:
                e = self._decode(e)
                if wildcard is not None and not fnmatch(e,wildcard):
                    continue
            entries.append(e)
        return entries

    @rewrite_errors
    def makedir(self,path,*args,**kwds):
        return self.wrapped_fs.makedir(self._encode(path),*args,**kwds)

    @rewrite_errors
    def remove(self,path):
        return self.wrapped_fs.remove(self._encode(path))

    @rewrite_errors
    def removedir(self,path,*args,**kwds):
        return self.wrapped_fs.removedir(self._encode(path),*args,**kwds)

    @rewrite_errors
    def rename(self,src,dst):
        return self.wrapped_fs.rename(self._encode(src),self._encode(dst))

    @rewrite_errors
    def getinfo(self,path):
        return self.wrapped_fs.getinfo(self._encode(path))

    @rewrite_errors
    def desc(self,path):
        return self.wrapped_fs.desc(self._encode(path))

    @rewrite_errors
    def copy(self,src,dst,**kwds):
        return self.wrapped_fs.copy(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def move(self,src,dst,**kwds):
        return self.wrapped_fs.move(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def movedir(self,src,dst,**kwds):
        return self.wrapped_fs.movedir(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def copydir(self,src,dst,**kwds):
        return self.wrapped_fs.copydir(self._encode(src),self._encode(dst),**kwds)

    @rewrite_errors
    def getxattr(self,path,name,default=None):
        try:
            return self.wrapped_fs.getxattr(self._encode(path),name,default)
        except AttributeError:
            raise UnsupportedError("getxattr")

    @rewrite_errors
    def setxattr(self,path,name,value):
        try:
            return self.wrapped_fs.setxattr(self._encode(path),name,value)
        except AttributeError:
            raise UnsupportedError("setxattr")

    @rewrite_errors
    def delxattr(self,path,name):
        try:
            return self.wrapped_fs.delxattr(self._encode(path),name)
        except AttributeError:
            raise UnsupportedError("delxattr")

    @rewrite_errors
    def listxattrs(self,path):
        try:
            return self.wrapped_fs.listxattrs(self._encode(path))
        except AttributeError:
            raise UnsupportedError("listxattrs")

    def __getattr__(self,attr):
        return getattr(self.wrapped_fs,attr)

    @rewrite_errors
    def close(self):
        if hasattr(self.wrapped_fs,"close"):
            self.wrapped_fs.close()

def wrap_fs_methods(decorator,cls=None):
    """Apply the given decorator to all FS methods on the given class.

    This function can be used in two ways.  When called with two arguments it
    applies the given function 'decorator' to each FS method of the given
    class.  When called with just a single argument, it creates and returns
    a class decorator which will do the same thing when applied.  So you can
    use it like this:

        wrap_fs_methods(mydecorator,MyFSClass)

    Or on more recent Python versions, like this:

        @wrap_fs_methods(mydecorator)
        class MyFSClass(FS):
            ...

    """
    methods = ("open","exists","isdir","isfile","listdir","makedir","remove",
               "setcontents","removedir","rename","getinfo","copy","move",
               "copydir","movedir","close","getxattr","setxattr","delxattr",
               "listxattrs")

    def apply_decorator(cls):
        for method_name in methods:
            method = getattr(cls,method_name,None)
            if method is not None:
                setattr(cls,method_name,decorator(method))
        return cls
    if cls is not None:
        return apply_decorator(cls)
    else:
        return apply_decorator

class HideDotFiles(WrapFS):
    """FS wrapper class that hides dot-files in directory listings.

    The listdir() function takes an extra keyword argument 'hidden'
    indicating whether hidden dotfiles shoud be included in the output.
    It is False by default.
    """

    def is_hidden(self,path):
        """Check whether the given path should be hidden."""
        return path and basename(path)[0] == "."

    def _encode(self,path):
        return path

    def _decode(self,path):
        return path

    def listdir(self,path="",**kwds):
        hidden = kwds.pop("hidden",True)
        entries = self.wrapped_fs.listdir(path,**kwds)
        if not hidden:
            entries = [e for e in entries if not self.is_hidden(e)]
        return entries

    def walk(self, path="/", wildcard=None, dir_wildcard=None, search="breadth",hidden=False):
        if search == "breadth":
            dirs = [path]
            while dirs:
                current_path = dirs.pop()
                paths = []
                for filename in self.listdir(current_path,hidden=hidden):
                    path = pathjoin(current_path, filename)
                    if self.isdir(path):
                        if dir_wildcard is not None:
                            if fnmatch(path, dir_wilcard):
                                dirs.append(path)
                        else:
                            dirs.append(path)
                    else:
                        if wildcard is not None:
                            if fnmatch(path, wildcard):
                                paths.append(filename)
                        else:
                            paths.append(filename)
                yield (current_path, paths)
        elif search == "depth":
            def recurse(recurse_path):
                for path in self.listdir(recurse_path, wildcard=dir_wildcard, full=True, dirs_only=True,hidden=hidden):
                    for p in recurse(path):
                        yield p
                yield (recurse_path, self.listdir(recurse_path, wildcard=wildcard, files_only=True,hidden=hidden))
            for p in recurse(path):
                yield p
        else:
            raise ValueError("Search should be 'breadth' or 'depth'")

    def isdirempty(self, path):
        path = normpath(path)
        iter_dir = iter(self.listdir(path,hidden=True))
        try:
            iter_dir.next()
        except StopIteration:
            return True
        return False

class LimitSize(WrapFS):

    def __init__(self,fs,max_size):
        self.max_size = max_size
        super(LimitSize,self).__init__(fs)
        self.cur_size = sum(self.getsize(f) for f in self.walkfiles())

    @synchronize
    def open(self,path,mode="r"):
        try:
            size = self.getsize(path)
        except ResourceNotFoundError:
            size = 0
        f = super(LimitSize,self).open(path,mode)
        if "w" in mode:
            self.cur_size -= size
            size = 0
        return LimitSizeFile(self,f,mode,size)

    def _acquire_space(self,size):
        new_size = self.cur_size + size
        if new_size > self.max_size:
            raise StorageSpaceError("write")
        self.cur_size = new_size

    #  Disable use of system-level paths, so that copy/copydir have to
    #  fall back to manual writes and pass through _acquire_space.
    getsyspath = FS.getsyspath
    copy = FS.copy
    copydir = FS.copydir

    @synchronize
    def remove(self,path):
        size = self.getsize(path)
        super(LimitSize,self).remove(path)
        self.cur_size -= size

    @synchronize
    def removedir(self,path,recursive=False,force=False):
        size = sum(self.getsize(f) for f in self.walkfiles(path))
        super(LimitSize,self).removedir(path,recursive=recursive,force=force)
        self.cur_size -= size

class LimitSizeFile(object):

    def __init__(self,fs,file,mode,size):
        self._lock = fs._lock
        self.fs = fs
        self.file = file
        self.mode = mode
        self.size = size
        self.closed = False

    @synchronize
    def write(self,data):
        pos = self.file.tell()
        if pos > self.size:
            self.size = pos
        diff = pos + len(data) - self.size
        if diff <= 0:
            self.file.write(data)
        else:
            self.fs._acquire_space(diff)
            self.file.write(data)
            self.size += diff

    def writelines(self,lines):
        for line in lines:
            self.write(line)

    @synchronize
    def truncate(self,size=None):
        pos = self.file.tell()
        if size is None:
            size = pos
        self.fs._acquire_space(size - self.size)
        self.file.truncate(size)
        self.size = size

    #  This is lifted straight from the stdlib's tempfile.py
    def __getattr__(self,name):
        file = self.__dict__['file']
        a = getattr(file, name)
        if not issubclass(type(a), type(0)):
            setattr(self, name, a)
        return a

    def __enter__(self):
        self.file.__enter__()
        return self

    def __exit__(self,exc,value,tb):
        self.close()
        return False

    def __iter__(self):
        return iter(self.file)

<|endoftext|>


<commit_before>""" utils related to SurveyCoord objects"""

from frb.surveys.sdss import SDSS_Survey
from frb.surveys.des import DES_Survey
from frb.surveys.wise import WISE_Survey
from frb.surveys.decals import DECaL_Survey
from frb.surveys.psrcat import PSRCAT_Survey
from frb.surveys import heasarc
from frb.surveys.panstarrs import Pan_STARRS_Survey
from frb.surveys.nsc import NSC_Survey
from frb.surveys.vista import VISTA_Survey
from frb.surveys.catalog_utils import xmatch_and_merge_cats

from astropy.coordinates import SkyCoord
from astropy import units as u
from astropy.table import Table

import numpy as np
import warnings

optical_surveys = ['Pan-STARRS', 'WISE', 'SDSS', 'DES',  'DECaL', 'VISTA', 'NSC']
radio_surveys = ['NVSS', 'FIRST', 'WENSS', 'PSRCAT']
allowed_surveys = optical_surveys+radio_surveys

def load_survey_by_name(name, coord, radius, **kwargs):
    """
    Load up a Survey class object for the named survey
    allowed_surveys = ['SDSS', 'DES', 'NVSS', 'FIRST', 'WENSS', 'DECaL', 'PSRCAT', 'WISE', 'Pan-STARRS']

    Args:
        name (str): Name of the survey
        coord (astropy.coordiantes.SkyCoord): Coordinate to define survey around
        radius (astropy.units.Quanity): Outer radius of the survey
        **kwargs: Passed the Survey object

    Returns:
        frb.surveys.SurveyCoord: Child of this parent given by input survey name

    """

    if name not in allowed_surveys:
        raise IOError("Not ready for input survey.\n These are allowed: {}".format(allowed_surveys))

    # Do it
    if name == 'SDSS':
        survey = SDSS_Survey(coord, radius, **kwargs)
    elif name == 'DES':
        survey = DES_Survey(coord, radius, **kwargs)
    elif name == 'NVSS':
        survey = heasarc.NVSS_Survey(coord, radius, **kwargs)
    elif name == 'WENSS':
        survey = heasarc.WENSS_Survey(coord, radius, **kwargs)
    elif name == 'FIRST':
        survey = heasarc.FIRST_Survey(coord, radius, **kwargs)
    elif name == 'DECaL':
        survey = DECaL_Survey(coord, radius, **kwargs)
    elif name == 'WISE':
        survey = WISE_Survey(coord, radius, **kwargs)
    elif name == 'PSRCAT':
        survey = PSRCAT_Survey(coord, radius, **kwargs)
    elif name == 'Pan-STARRS':
        survey = Pan_STARRS_Survey(coord, radius,**kwargs)
    elif name == 'NSC':
        survey = NSC_Survey(coord, radius, **kwargs)
    elif name == 'VISTA':
        survey = VISTA_Survey(coord, radius, **kwargs)

    # Return
    return survey

def is_inside(surveyname:str, coord:SkyCoord)->bool:
    """
    Tests if a coordinate is within a survey footprint.
    Args:
        surveyname (str): Name of the survey
        coord (astropy.coordiantes.SkyCoord): Coordinate to check
    Returns:
        inside (bool): True if coord is within the footprint.
    """

    # Instantiate survey and run a cone search with 1 arcmin radius
    survey = load_survey_by_name(surveyname, coord, 1*u.arcmin)
    cat = survey.get_catalog()

    # Are there any objects in the returned catalog?
    if cat is None or len(cat) == 0:
        return False
    # is the coordinate in a region that's close to the edge but on the exterior?
    else:
        # Compute PAs of all returned objects
        cat_coords = SkyCoord(cat['ra'], cat['dec'], unit="deg")
        pa_list = coord.position_angle(cat_coords)

        # If the object is in the interior, then the max PA
        # difference is likely greater than 180 degrees.
        max_pa_diff = np.max(pa_list) - np.min(pa_list)
        if np.abs(max_pa_diff) <= 180*u.deg:
            # Are there too few objects? Warn the user if so.
            if len(cat)<=5:
                # Make sure there is no source at the search location.
                # if that's not the case, then no need to warn.
                separations = coord.separation(cat_coords)
                if np.min(separations)>2*u.arcsec:
                    warnings.warn("Only {} objects found in {} within 1'. Check location manually.".format(len(cat), surveyname),
                                  RuntimeWarning, stacklevel=2)
            return False
        else:
            return True

def in_which_survey(coord:SkyCoord)->dict:
    """
    Check if a particular coord is inside any
    survey that can be currently queried from
    `frb.surveys` module.
    Args:
        coord (astropy.coordiantes.SkyCoord): Coordinate to check
    Returns:
        inside (dict): A dict which tells which surveys the coordinate
            is inside.
    """
    # Loop through known surveys and check them one by one.
    inside = {}
    for surveyname in allowed_surveys:
        # Skip PSRCAT
        if surveyname == "PSRCAT":
            continue
        inside[surveyname] = is_inside(surveyname, coord)

    return inside

def search_all_surveys(coord:SkyCoord, radius:u.Quantity, include_radio:bool=False):
    """
    A method to query all allowed surveys and combine
    the results into one table.
    Args:
        coord (SkyCoord): Central coordinates of cone search.
        radius (Quantity): Search radius in angular units.
        include_radio (bool, optional): Want to include results from the HEASARC surveys?
            Include at your own risk. Untested. Might break in unexpected ways.

    Returns:
        combined_cat (Table): Table of merged query results.
    """

    # Start with an empty table
    combined_cat = Table()
    # Select surveys
    if include_radio: # Careful! NOT TESTED!
        surveys = allowed_surveys
    else:
        surveys = optical_surveys
    # Loop over them
    for surveyname in surveys:
        survey = load_survey_by_name(name=surveyname, coord=coord, radius=radius)
        try:
            survey.get_catalog()
        except ConnectionError:
            warnings.warn("Couldn't connect to {:s}. Skipping this for now.".format(surveyname), RuntimeWarning)
        # Did the survey return something?
        if (survey.catalog is not None) & (len(survey.catalog)>0):
            print("Found {:d} objects in {:s}".format(len(survey.catalog), surveyname))
            if len(combined_cat)==0:
                # First time
                combined_cat = survey.catalog
            else:
                # Combine otherwise
                # TODO: Need to deal with duplicate column names more elegantly.
                combined_cat = xmatch_and_merge_cats(combined_cat, survey.catalog)
        # No objects found?
        elif len(survey.catalog)==0:
            print("Empty table in "+surveyname)
            # Proper empty table
            if len(survey.catalog.colnames)>0:
                for colname in survey.catalog.colnames:
                    if colname in ['ra', 'dec']: # skip em'
                        continue
                    else: # pad
                        combined_cat[colname] = np.repeat(-999., len(combined_cat))
            else: # Broken stuff. Need to fix the survey
                continue

    return combined_cat

<commit_msg>search all surveys and make one giant table<commit_after>""" utils related to SurveyCoord objects"""

from urllib.error import HTTPError
from frb.surveys.sdss import SDSS_Survey
from frb.surveys.des import DES_Survey
from frb.surveys.wise import WISE_Survey
from frb.surveys.decals import DECaL_Survey
from frb.surveys.psrcat import PSRCAT_Survey
from frb.surveys import heasarc
from frb.surveys.panstarrs import Pan_STARRS_Survey
from frb.surveys.nsc import NSC_Survey
from frb.surveys.vista import VISTA_Survey
from frb.surveys.catalog_utils import xmatch_and_merge_cats

from astropy.coordinates import SkyCoord
from astropy import units as u
from astropy.table import Table

import numpy as np
import warnings

optical_surveys = ['Pan-STARRS', 'WISE', 'SDSS', 'DES',  'DECaL', 'VISTA', 'NSC']
radio_surveys = ['NVSS', 'FIRST', 'WENSS', 'PSRCAT']
allowed_surveys = optical_surveys+radio_surveys

def load_survey_by_name(name, coord, radius, **kwargs):
    """
    Load up a Survey class object for the named survey
    allowed_surveys = ['SDSS', 'DES', 'NVSS', 'FIRST', 'WENSS', 'DECaL', 'PSRCAT', 'WISE', 'Pan-STARRS']

    Args:
        name (str): Name of the survey
        coord (astropy.coordiantes.SkyCoord): Coordinate to define survey around
        radius (astropy.units.Quanity): Outer radius of the survey
        **kwargs: Passed the Survey object

    Returns:
        frb.surveys.SurveyCoord: Child of this parent given by input survey name

    """

    if name not in allowed_surveys:
        raise IOError("Not ready for input survey.\n These are allowed: {}".format(allowed_surveys))

    # Do it
    if name == 'SDSS':
        survey = SDSS_Survey(coord, radius, **kwargs)
    elif name == 'DES':
        survey = DES_Survey(coord, radius, **kwargs)
    elif name == 'NVSS':
        survey = heasarc.NVSS_Survey(coord, radius, **kwargs)
    elif name == 'WENSS':
        survey = heasarc.WENSS_Survey(coord, radius, **kwargs)
    elif name == 'FIRST':
        survey = heasarc.FIRST_Survey(coord, radius, **kwargs)
    elif name == 'DECaL':
        survey = DECaL_Survey(coord, radius, **kwargs)
    elif name == 'WISE':
        survey = WISE_Survey(coord, radius, **kwargs)
    elif name == 'PSRCAT':
        survey = PSRCAT_Survey(coord, radius, **kwargs)
    elif name == 'Pan-STARRS':
        survey = Pan_STARRS_Survey(coord, radius,**kwargs)
    elif name == 'NSC':
        survey = NSC_Survey(coord, radius, **kwargs)
    elif name == 'VISTA':
        survey = VISTA_Survey(coord, radius, **kwargs)

    # Return
    return survey

def is_inside(surveyname:str, coord:SkyCoord)->bool:
    """
    Tests if a coordinate is within a survey footprint.
    Args:
        surveyname (str): Name of the survey
        coord (astropy.coordiantes.SkyCoord): Coordinate to check
    Returns:
        inside (bool): True if coord is within the footprint.
    """

    # Instantiate survey and run a cone search with 1 arcmin radius
    survey = load_survey_by_name(surveyname, coord, 1*u.arcmin)
    cat = survey.get_catalog()

    # Are there any objects in the returned catalog?
    if cat is None or len(cat) == 0:
        return False
    # is the coordinate in a region that's close to the edge but on the exterior?
    else:
        # Compute PAs of all returned objects
        cat_coords = SkyCoord(cat['ra'], cat['dec'], unit="deg")
        pa_list = coord.position_angle(cat_coords)

        # If the object is in the interior, then the max PA
        # difference is likely greater than 180 degrees.
        max_pa_diff = np.max(pa_list) - np.min(pa_list)
        if np.abs(max_pa_diff) <= 180*u.deg:
            # Are there too few objects? Warn the user if so.
            if len(cat)<=5:
                # Make sure there is no source at the search location.
                # if that's not the case, then no need to warn.
                separations = coord.separation(cat_coords)
                if np.min(separations)>2*u.arcsec:
                    warnings.warn("Only {} objects found in {} within 1'. Check location manually.".format(len(cat), surveyname),
                                  RuntimeWarning, stacklevel=2)
            return False
        else:
            return True

def in_which_survey(coord:SkyCoord, optical_only:bool=True)->dict:
    """
    Check if a particular coord is inside any
    survey that can be currently queried from
    `frb.surveys` module.
    Args:
        coord (astropy.coordiantes.SkyCoord): Coordinate to check
    Returns:
        inside (dict): A dict which tells which surveys the coordinate
            is inside.
    """
    # Loop through known surveys and check them one by one.
    inside = {}
    if optical_only:
        all_surveys = optical_surveys
    else:
        all_surveys = allowed_surveys
    for surveyname in all_surveys:
        # Skip PSRCAT
        if surveyname == "PSRCAT":
            continue
        inside[surveyname] = is_inside(surveyname, coord)

    return inside

def search_all_surveys(coord:SkyCoord, radius:u.Quantity, include_radio:bool=False,
                       seed_cat:Table=None):
    """
    A method to query all allowed surveys and combine
    the results into one table.
    Args:
        coord (SkyCoord): Central coordinates of cone search.
        radius (Quantity): Search radius in angular units.
        include_radio (bool, optional): Want to include results from the HEASARC surveys?
            Include at your own risk. Untested. Might break in unexpected ways.
        seed_cat (Table, optional): If you'd like to merge the survey results
            with another photometry table that you already have.

    Returns:
        combined_cat (Table): Table of merged query results.
    """

    # Start with the seed table
    if seed_cat is not None:
        assert isinstance(seed_cat, Table), "The seed_cat must be an astropy table"
        assert np.all(np.isin(['ra','dec'],seed_cat.colnames)), "The seed catalog doesn't have 'ra' and/or 'dec' columns."
        combined_cat = seed_cat
    else:
        # Start with an empty table
        combined_cat = Table()
    # Select surveys
    if include_radio: # Careful! NOT TESTED!
        surveys = allowed_surveys
    else:
        surveys = optical_surveys
    # Loop over them
    for surveyname in surveys:
        if surveyname=='Pan-STARRS':
            if radius>0.5*u.deg:
                warnings.warn("Pan=STARRS doesn't allow cone searches wider than 0.5 deg. Skipping.", RuntimeWarning)
                continue
        survey = load_survey_by_name(name=surveyname, coord=coord, radius=radius)
        try:
            survey.get_catalog()
        except ConnectionError:
            warnings.warn("Couldn't connect to {:s}. Skipping this for now.".format(surveyname), RuntimeWarning)
        except HTTPError:
            warnings.warn("Couldn't connect to {:s}. Skipping this for now.".format(surveyname), RuntimeWarning)
        # Did the survey return something?
        if (survey.catalog is not None) & (len(survey.catalog)>0):
            print("Found {:d} objects in {:s}".format(len(survey.catalog), surveyname))
            if len(combined_cat)==0:
                # First time
                combined_cat = survey.catalog
            else:
                # Combine otherwise
                # TODO: Need to deal with duplicate column names more elegantly.
                combined_cat = xmatch_and_merge_cats(combined_cat, survey.catalog,)
        # No objects found?
        elif len(survey.catalog)==0:
            print("Empty table in "+surveyname)
            # Proper empty table
            #if len(survey.catalog.colnames)>0:
            #    for colname in survey.catalog.colnames:
            #        if colname in ['ra', 'dec']: # skip em'
            #            continue
            #        else: # pad
            #            combined_cat[colname] = np.repeat(-999., len(combined_cat))
            #else: # Broken stuff. Need to fix the survey
            #    continue

    return combined_cat

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Roastero, released under GPLv3

import time
import serial
import threading

from freshroastsr700 import pid
from freshroastsr700 import utils
from freshroastsr700 import exceptions

class freshroastsr700(object):
    """A class to interface with a freshroastsr700 coffee roaster."""
    def __init__(self, update_data_func=None, state_transition_func=None,
                 thermostat=False):
        """Create variables used to send in packets to the roaster. The update
        data function is called when a packet is opened. The state transistion
        function is used by the timer thread to know what to do next. See wiki
        for more information on packet structure and fields."""
        self.update_data_func = update_data_func
        self.state_transition_func = state_transition_func

        self._header = b'\xAA\xAA'
        self._temp_unit = b'\x61\x74'
        self._flags = b'\x63'
        self._current_state = b'\x02\x01'
        self._fan_speed = 1
        self._heat_setting = 0
        self._footer = b'\xAA\xFA'

        self.current_temp = 150
        self.time_remaining = 0
        self.total_time = 0

        self._cont = True

        if(thermostat is True):
            self._p = 4.000
            self._i = 0.045
            self._d = 2.200
            self._pid = pid.PID(self._p, self._i, self._d)
            self.target_temp = 150

            self.thermostat_thread = threading.Thread(target=self.thermostat)
            self.thermostat_thread.start()

    @property
    def fan_speed(self):
        """A getter method for _fan_speed."""
        return self._fan_speed

    @fan_speed.setter
    def fan_speed(self, value):
        """Verifies the value is between 1 and 9 inclusively."""
        if value not in range(1, 10):
            raise exceptions.RoasterValueError

        self._fan_speed = value

    @property
    def heat_setting(self):
        """A getter method for _heat_setting."""
        return self._heat_setting

    @heat_setting.setter
    def heat_setting(self, value):
        """Verifies that the heat setting is between 0 and 3."""
        if value not in range(0, 4):
            raise exceptions.RoasterValueError

        self._heat_setting = value

    def connect(self):
        """Connects to the roaster and creates communication thread."""
        port = utils.find_device('1A86:5523')
        self._ser = serial.Serial(
            port=port,
            baudrate=9600,
            bytesize=8,
            parity='N',
            stopbits=1.5,
            timeout=.25,
            xonxoff=False,
            rtscts=False,
            dsrdtr=False)

        self._initialize()

        self.comm_thread = threading.Thread(target=self.comm)
        self.comm_thread.start()
        self.time_thread = threading.Thread(target=self.timer)
        self.time_thread.start()

    def _initialize(self):
        """Sends the initialization packet to the roaster."""
        self._header = b'\xAA\x55'
        self._current_state = b'\x00\x00'
        s = self.generate_packet()
        self._ser.write(s)
        self._header = b'\xAA\xAA'
        self._current_state = b'\x02\x01'

        # The readline is used here to get the entirety of the current recipe
        # currently on the roaster.
        r = self._ser.readline()
        return r

    def auto_connect(self):
        """Starts a thread that will automatically connect to the roaster when
        it is plugged in."""
        self.connected = False
        self.auto_connect_thread = threading.Thread(target=self._auto_connect)
        self.auto_connect_thread.start()

    def _auto_connect(self):
        """Attempts to connect to the roaster every quarter of a second."""
        while(self._cont):
            try:
                self.connect()
                self.connected = True
                break
            except exceptions.RoasterLookupError:
                time.sleep(.25)

    def disconnect(self):
        """Stops the communication loop to the roaster. Note that this will not
        actually stop the roaster itself, but will allow the program to exit
        cleanly."""
        self._cont = False

    def comm(self):
        """Main communications loop to the roaster. If the packet is not 14
        bytes exactly, the packet will not be opened. If an update data
        function is available, it will be called when the packet is opened."""
        while(self._cont):
            s = self.generate_packet()
            try:
                self._ser.write(s)
            except serial.serialutil.SerialException:
                self._ser.close()
                self.auto_connect()
                return

            try:
                r = self._ser.read(14)
            except serial.serialutil.SerialException:
                self._ser.close()
                self.auto_connect()
                return

            if(r[-2:] == self._footer):
                temp = int.from_bytes(bytes(r[10:-2]), byteorder='big')

                if(temp == 65280):
                    self.current_temp = 150
                elif(temp > 550 or temp < 150):
                    self._initialize()
                    continue
                else:
                    self.current_temp = temp

                if(self.update_data_func is not None):
                    self.update_data_func(self)

            time.sleep(.25)

        self._ser.close()

    def timer(self):
        """Timer loop used to keep track of the time while roasting or
        cooling. If the time remaining reaches zero, the roaster will call the
        supplied state transistion function or the roaster will be set to
        the idle state."""
        while(self._cont):
            state = self.get_roaster_state()
            if(state == 'roasting' or state == 'cooling'):
                time.sleep(1)
                self.total_time += 1
                if(self.time_remaining > 0):
                    self.time_remaining -= 1
                else:
                    if(self.state_transition_func is not None):
                        self.state_transition_func(self, state)
                    else:
                        self.idle()

    def get_roaster_state(self):
        """Returns a string based upon the current state of the roaster. Will
        raise an exception if the state is unknown."""
        if(self._current_state == b'\x02\x01'):
            return 'idle'
        elif(self._current_state == b'\x04\x04'):
            return 'cooling'
        elif(self._current_state == b'\x08\x01'):
            return 'sleeping'
        elif(self._current_state == b'\x00\x00'):
            return 'connecting'
        elif(self._current_state == b'\x04\x02'):
            return 'roasting'
        else:
            return 'unknown'

    def generate_packet(self):
        """Generates a packet based upon the current class variables. Note that
        current temperature is not sent, as the original application sent zeros
        to the roaster for the current temperature."""
        roaster_time = utils.seconds_to_float(self.time_remaining)
        packet = (
            self._header +
            self._temp_unit +
            self._flags +
            self._current_state +
            self.fan_speed.to_bytes(1, byteorder='big') +
            int(float(roaster_time * 10)).to_bytes(1, byteorder='big') +
            self.heat_setting.to_bytes(1, byteorder='big') +
            b'\x00\x00' +
            self._footer)

        return packet

    def idle(self):
        """Sets the current state of the roaster to idle."""
        self._current_state = b'\x02\x01'

    def roast(self):
        """Sets the current state of the roaster to roast and begins
        roasting."""
        self._current_state = b'\x04\x02'

    def cool(self):
        """Sets the current state of the roaster to cool. The roaster expects
        that cool will be run after roast, and will not work as expected if ran
        before."""
        self._current_state = b'\x04\x04'

    def sleep(self):
        """Sets the current state of the roaster to sleep. Different than idle
        in that this will set double dashes on the roaster display rather than
        digits."""
        self._current_state = b'\x08\x01'

    def thermostat(self):
        """Utilizes a software PID controller to set the heat setting on the
        roaster given the current temperature and a target temperture."""
        while(self._cont):
            output = self._pid.update(self.current_temp, self.target_temp)

            if(self.target_temp >= 460):
                if(output >= 30):
                    self.heat_setting = 3
                else:
                    if(self.heat_setting == 2):
                        self.heat_setting = 3
                    else:
                        self.heat_setting = 2
            elif(self.target_temp >= 430):
                if(output >= 30):
                    self.heat_setting = 3
                elif(output >= 20):
                    self.heat_setting = 2
                else:
                    if(self.heat_setting == 1):
                        self.heat_setting = 2
                    else:
                        self.heat_setting = 1
            elif(self.target_temp >= 350):
                if(output >= 30):
                    self.heat_setting = 3
                elif(output >= 20):
                    self.heat_setting = 2
                elif(output >= 10):
                    self.heat_setting = 1
                else:
                    if(self.heat_setting == 0):
                        self.heat_setting = 1
                    else:
                        self.heat_setting = 0
            else:
                if(output >= 30):
                    self.heat_setting = 3
                elif(output >= 20):
                    self.heat_setting = 2
                elif(output >= 10):
                    self.heat_setting = 1
                else:
                    self.heat_setting = 0

            time.sleep(.25)
<commit_msg>Updated the way update data and state transition are called.<commit_after># -*- coding: utf-8 -*-
# Roastero, released under GPLv3

import time
import serial
import threading

from freshroastsr700 import pid
from freshroastsr700 import utils
from freshroastsr700 import exceptions

class freshroastsr700(object):
    """A class to interface with a freshroastsr700 coffee roaster."""
    def __init__(self, update_data_func=None, state_transition_func=None,
                 thermostat=False):
        """Create variables used to send in packets to the roaster. The update
        data function is called when a packet is opened. The state transistion
        function is used by the timer thread to know what to do next. See wiki
        for more information on packet structure and fields."""
        self.update_data_func = update_data_func
        self.state_transition_func = state_transition_func

        self._header = b'\xAA\xAA'
        self._temp_unit = b'\x61\x74'
        self._flags = b'\x63'
        self._current_state = b'\x02\x01'
        self._fan_speed = 1
        self._heat_setting = 0
        self._footer = b'\xAA\xFA'

        self.current_temp = 150
        self.time_remaining = 0
        self.total_time = 0

        self._cont = True

        if(thermostat is True):
            self._p = 4.000
            self._i = 0.045
            self._d = 2.200
            self._pid = pid.PID(self._p, self._i, self._d)
            self.target_temp = 150

            self.thermostat_thread = threading.Thread(target=self.thermostat)
            self.thermostat_thread.start()

    @property
    def fan_speed(self):
        """A getter method for _fan_speed."""
        return self._fan_speed

    @fan_speed.setter
    def fan_speed(self, value):
        """Verifies the value is between 1 and 9 inclusively."""
        if value not in range(1, 10):
            raise exceptions.RoasterValueError

        self._fan_speed = value

    @property
    def heat_setting(self):
        """A getter method for _heat_setting."""
        return self._heat_setting

    @heat_setting.setter
    def heat_setting(self, value):
        """Verifies that the heat setting is between 0 and 3."""
        if value not in range(0, 4):
            raise exceptions.RoasterValueError

        self._heat_setting = value

    def connect(self):
        """Connects to the roaster and creates communication thread."""
        port = utils.find_device('1A86:5523')
        self._ser = serial.Serial(
            port=port,
            baudrate=9600,
            bytesize=8,
            parity='N',
            stopbits=1.5,
            timeout=.25,
            xonxoff=False,
            rtscts=False,
            dsrdtr=False)

        self._initialize()

        self.comm_thread = threading.Thread(target=self.comm)
        self.comm_thread.start()
        self.time_thread = threading.Thread(target=self.timer)
        self.time_thread.start()

    def _initialize(self):
        """Sends the initialization packet to the roaster."""
        self._header = b'\xAA\x55'
        self._current_state = b'\x00\x00'
        s = self.generate_packet()
        self._ser.write(s)
        self._header = b'\xAA\xAA'
        self._current_state = b'\x02\x01'

        # The readline is used here to get the entirety of the current recipe
        # currently on the roaster.
        r = self._ser.readline()
        return r

    def auto_connect(self):
        """Starts a thread that will automatically connect to the roaster when
        it is plugged in."""
        self.connected = False
        self.auto_connect_thread = threading.Thread(target=self._auto_connect)
        self.auto_connect_thread.start()

    def _auto_connect(self):
        """Attempts to connect to the roaster every quarter of a second."""
        while(self._cont):
            try:
                self.connect()
                self.connected = True
                break
            except exceptions.RoasterLookupError:
                time.sleep(.25)

    def disconnect(self):
        """Stops the communication loop to the roaster. Note that this will not
        actually stop the roaster itself, but will allow the program to exit
        cleanly."""
        self._cont = False

    def comm(self):
        """Main communications loop to the roaster. If the packet is not 14
        bytes exactly, the packet will not be opened. If an update data
        function is available, it will be called when the packet is opened."""
        while(self._cont):
            s = self.generate_packet()
            try:
                self._ser.write(s)
            except serial.serialutil.SerialException:
                self._ser.close()
                self.auto_connect()
                return

            try:
                r = self._ser.read(14)
            except serial.serialutil.SerialException:
                self._ser.close()
                self.auto_connect()
                return

            if(r[-2:] == self._footer):
                temp = int.from_bytes(bytes(r[10:-2]), byteorder='big')

                if(temp == 65280):
                    self.current_temp = 150
                elif(temp > 550 or temp < 150):
                    self._initialize()
                    continue
                else:
                    self.current_temp = temp

                if(self.update_data_func is not None):
                    self.update_data_func()

            time.sleep(.25)

        self._ser.close()

    def timer(self):
        """Timer loop used to keep track of the time while roasting or
        cooling. If the time remaining reaches zero, the roaster will call the
        supplied state transistion function or the roaster will be set to
        the idle state."""
        while(self._cont):
            state = self.get_roaster_state()
            if(state == 'roasting' or state == 'cooling'):
                time.sleep(1)
                self.total_time += 1
                if(self.time_remaining > 0):
                    self.time_remaining -= 1
                else:
                    if(self.state_transition_func is not None):
                        self.state_transition_func()
                    else:
                        self.idle()

    def get_roaster_state(self):
        """Returns a string based upon the current state of the roaster. Will
        raise an exception if the state is unknown."""
        if(self._current_state == b'\x02\x01'):
            return 'idle'
        elif(self._current_state == b'\x04\x04'):
            return 'cooling'
        elif(self._current_state == b'\x08\x01'):
            return 'sleeping'
        elif(self._current_state == b'\x00\x00'):
            return 'connecting'
        elif(self._current_state == b'\x04\x02'):
            return 'roasting'
        else:
            return 'unknown'

    def generate_packet(self):
        """Generates a packet based upon the current class variables. Note that
        current temperature is not sent, as the original application sent zeros
        to the roaster for the current temperature."""
        roaster_time = utils.seconds_to_float(self.time_remaining)
        packet = (
            self._header +
            self._temp_unit +
            self._flags +
            self._current_state +
            self.fan_speed.to_bytes(1, byteorder='big') +
            int(float(roaster_time * 10)).to_bytes(1, byteorder='big') +
            self.heat_setting.to_bytes(1, byteorder='big') +
            b'\x00\x00' +
            self._footer)

        return packet

    def idle(self):
        """Sets the current state of the roaster to idle."""
        self._current_state = b'\x02\x01'

    def roast(self):
        """Sets the current state of the roaster to roast and begins
        roasting."""
        self._current_state = b'\x04\x02'

    def cool(self):
        """Sets the current state of the roaster to cool. The roaster expects
        that cool will be run after roast, and will not work as expected if ran
        before."""
        self._current_state = b'\x04\x04'

    def sleep(self):
        """Sets the current state of the roaster to sleep. Different than idle
        in that this will set double dashes on the roaster display rather than
        digits."""
        self._current_state = b'\x08\x01'

    def thermostat(self):
        """Utilizes a software PID controller to set the heat setting on the
        roaster given the current temperature and a target temperture."""
        while(self._cont):
            output = self._pid.update(self.current_temp, self.target_temp)

            if(self.target_temp >= 460):
                if(output >= 30):
                    self.heat_setting = 3
                else:
                    if(self.heat_setting == 2):
                        self.heat_setting = 3
                    else:
                        self.heat_setting = 2
            elif(self.target_temp >= 430):
                if(output >= 30):
                    self.heat_setting = 3
                elif(output >= 20):
                    self.heat_setting = 2
                else:
                    if(self.heat_setting == 1):
                        self.heat_setting = 2
                    else:
                        self.heat_setting = 1
            elif(self.target_temp >= 350):
                if(output >= 30):
                    self.heat_setting = 3
                elif(output >= 20):
                    self.heat_setting = 2
                elif(output >= 10):
                    self.heat_setting = 1
                else:
                    if(self.heat_setting == 0):
                        self.heat_setting = 1
                    else:
                        self.heat_setting = 0
            else:
                if(output >= 30):
                    self.heat_setting = 3
                elif(output >= 20):
                    self.heat_setting = 2
                elif(output >= 10):
                    self.heat_setting = 1
                else:
                    self.heat_setting = 0

            time.sleep(.25)
<|endoftext|>


<commit_before>def upload_to_path(instance, filename, folder):
    """
    Create and return path where files will be uploaded. This path has specific formation
    i.e. app_label/folder/filename
    :param instance: An instance of the model where the FileField is defined
    :param filename: The filename that was originally given to the file
    :param folder: The specific folder where files will be uploaded
    :return: path to upload files
    """
    return '{app_label}/{folder}/{filename}'.format(
        app_label=instance._meta.app_label,
        filename=filename,
        folder=folder
    )

def pretty_date(time):
    """
    Get a datetime object and return a pretty string like 'an hour ago',
    'Yesterday', '3 months ago', 'just now', etc
    """
    from datetime import datetime
    from django.utils.timezone import utc

    now = datetime.utcnow().replace(tzinfo=utc)
    diff = now - time
    second_diff = diff.seconds
    day_diff = diff.days

    if day_diff == 0:
        if second_diff < 10:
            return "JUST NOW"
        if second_diff < 60:
            return str(second_diff) + " SECONDS AGO"
        if second_diff < 120:
            return "1 MINUTE AGO"
        if second_diff < 3600:
            return str(second_diff / 60) + " MINUTES AGO"
        if second_diff < 7200:
            return "1 HOUR AGO"
        if second_diff < 86400:
            return str(second_diff / 3600) + " HOURS AGO"
    if day_diff == 1:
        return "YESTERDAY"
    if day_diff < 7:
        return str(day_diff) + " DAYS AGO"
    if day_diff < 14:
        return "1 WEEK AGO"
    if day_diff < 31:
        return str(day_diff / 7) + " WEEKS AGO"
    if day_diff < 60:
        return "1 MONTH AGO"
    if day_diff < 365:
        return str(day_diff / 30) + " MONTHS AGO"
    if day_diff < 730:
        return "1 YEAR AGO"
    return str(day_diff / 365) + " YEARS AGO"
<commit_msg>Update helpers.py<commit_after>from datetime import datetime
from django.utils.timezone import utc

def upload_to_path(instance, filename, folder):
    """
    Create and return path where files will be uploaded. This path has specific formation
    i.e. app_label/folder/filename
    :param instance: An instance of the model where the FileField is defined
    :param filename: The filename that was originally given to the file
    :param folder: The specific folder where files will be uploaded
    :return: path to upload files
    """
    return '{app_label}/{folder}/{filename}'.format(
        app_label=instance._meta.app_label,
        filename=filename,
        folder=folder
    )

def pretty_date(time):
    """
    :param time
    :return: datetime object in "time ago" format
    """
    now = datetime.utcnow().replace(tzinfo=utc)
    diff = now - time
    second_diff = diff.seconds
    day_diff = diff.days

    if day_diff == 0:
        if second_diff < 10:
            return "JUST NOW"
        if second_diff < 60:
            return str(second_diff) + " SECONDS AGO"
        if second_diff < 120:
            return "1 MINUTE AGO"
        if second_diff < 3600:
            return str(second_diff / 60) + " MINUTES AGO"
        if second_diff < 7200:
            return "1 HOUR AGO"
        if second_diff < 86400:
            return str(second_diff / 3600) + " HOURS AGO"
    if day_diff == 1:
        return "YESTERDAY"
    if day_diff < 7:
        return str(day_diff) + " DAYS AGO"
    if day_diff < 14:
        return "1 WEEK AGO"
    if day_diff < 31:
        return str(day_diff / 7) + " WEEKS AGO"
    if day_diff < 60:
        return "1 MONTH AGO"
    if day_diff < 365:
        return str(day_diff / 30) + " MONTHS AGO"
    if day_diff < 730:
        return "1 YEAR AGO"
    return str(day_diff / 365) + " YEARS AGO"
<|endoftext|>