<commit_before>from __future__ import division
import numpy as np
import time
import datetime
import copy
from .tree import *

class PTreeOpt(object):

    def __init__(self, f, feature_bounds, discrete_actions=False,
                 action_bounds=None, action_names=None, population_size=100,
                 mu=15, max_depth=4, mut_prob=0.9, cx_prob=0.9,
                 feature_names=None, discrete_features=None,
                 multiobj=False, epsilons=None):

        self.f = f
        self.num_features = len(feature_bounds)
        self.feature_bounds = feature_bounds
        self.discrete_actions = discrete_actions
        self.action_bounds = action_bounds
        self.action_names = action_names
        self.popsize = population_size
        self.mu = mu
        self.max_depth = max_depth
        self.mut_prob = mut_prob
        self.cx_prob = cx_prob
        self.feature_names = feature_names
        self.discrete_features = discrete_features
        self.multiobj = multiobj
        self.epsilons = epsilons

        if feature_names is not None and len(feature_names) != len(feature_bounds):
            raise ValueError(('feature_names and feature_bounds '
                              'must be the same length.'))

        if discrete_features is not None and len(discrete_features) != len(feature_bounds):
            raise ValueError(('discrete_features and feature_bounds '
                              'must be the same length.'))

        if discrete_actions:
            if action_names is None or action_bounds is not None:
                raise ValueError(('discrete_actions must be run with '
                                  'action_names, (which are strings), '
                                  'and not action_bounds.'))
        else:
            if action_bounds is None:
                raise ValueError(('Real-valued actions (which is the case by '
                                'default, discrete_actions=False) must include '
                                'action_bounds. Currently only one action is '
                                'supported, so bounds = [lower, upper].'))

        if mu > population_size:
            raise ValueError(('Number of parents (mu) cannot be greater than '
                              'the population_size.'))

    def iterate(self):

        # selection: find index numbers for parents
        if not self.multiobj:
            parents = self.select_truncation(self.objectives)

            if self.best_f is None or self.objectives[parents[0]] < self.best_f:
                self.best_f = self.objectives[parents[0]]
                self.best_P = self.population[parents[0]]
        else:
            parents = [self.binary_tournament(self.population, self.objectives)
                       for _ in range(self.mu)]

            if self.best_f is None:
                self.best_f = self.objectives[parents]
                self.best_P = self.population[parents]
            else:
                self.best_P, self.best_f = self.archive_sort(self.best_P,
                                                 self.best_f, self.population,
                                                 self.objectives)

        # first: mutate the parents, only keep child if it's better
        # changed 7/17/17 JDH: now mutate all except best parent (single-obj)
        children = set(range(self.popsize)) - set(parents)

        for i in parents[1:]:
            child = self.mutate(self.population[i])
            child.prune()
            self.population[i] = child

        # then crossover to develop the rest of the population
        for i in children:

            if np.random.rand() < self.cx_prob:
                P1, P2 = np.random.choice(
                    self.population[parents], 2, replace=False)
                child = self.crossover(P1, P2)[0]

                # bloat control
                while child.get_depth() > self.max_depth:
                    child = self.crossover(P1, P2)[0]

            else:  # replace with randomly chosen parent
                child = np.random.choice(self.population[parents], 1)[0]

            child = self.mutate(child)
            child.prune()
            self.population[i] = child

    def run(self, max_nfe=100, parallel=False, log_frequency=None):

        if parallel:
            from mpi4py import MPI
            comm = MPI.COMM_WORLD
            size = comm.Get_size()
            rank = comm.Get_rank()

        is_master = (not parallel) or (parallel and rank == 0)
        start_time = time.time()
        nfe, last_log = 0, 0

        if is_master:
            self.population = np.array(
                [self.random_tree() for _ in range(self.popsize)])
            self.best_f = None
            self.best_P = None

            if log_frequency:
                snapshots = {'nfe': [], 'time': [], 'best_f': [], 'best_P': []}
        else:
            self.population = None

        while nfe < max_nfe:

            for P in self.population:
                P.clear_count() # reset action counts to zero

            # evaluate objectives
            if not parallel:
                self.objectives = np.array(
                    [self.f(P) for P in self.population])
            else:
                if is_master:
                    chunks = np.array_split(self.population, size)
                else:
                    chunks = None

                local_Ps = comm.scatter(chunks, root=0)
                local_fs = [self.f(P) for P in local_Ps]
                objs = comm.gather(local_fs, root=0)
                comm.barrier()

                if is_master:
                    self.objectives = np.concatenate(objs)  # flatten list

            for P in self.population:
                P.normalize_count() # convert action count to percent

            nfe += self.popsize

            if is_master:
                self.iterate()

                if log_frequency is not None and nfe >= last_log + log_frequency:
                    elapsed = datetime.timedelta(
                        seconds=time.time() - start_time).seconds

                    if not self.multiobj:
                        print('%d\t%s\t%0.3f\t%s' %
                              (nfe, elapsed, self.best_f, self.best_P))
                    else:
                        print('# nfe = %d\n%s' % (nfe, self.best_f))
                        print(self.best_f.shape)
                    snapshots['nfe'].append(nfe)
                    snapshots['time'].append(elapsed)
                    snapshots['best_f'].append(self.best_f)
                    snapshots['best_P'].append(self.best_P)
                    last_log = nfe

        if is_master and log_frequency:
            return snapshots

    def random_tree(self, terminal_ratio=0.5):
        depth = np.random.randint(1, self.max_depth + 1)
        L = []
        S = [0]

        while S:
            current_depth = S.pop()

            # action node
            if current_depth == depth or (current_depth > 0 and\
                                          np.random.rand() < terminal_ratio):
                if self.discrete_actions:
                    L.append([str(np.random.choice(self.action_names))])
                else:
                    L.append([np.random.uniform(*self.action_bounds)])

            else:
                x = np.random.choice(self.num_features)
                v = np.random.uniform(*self.feature_bounds[x])
                L.append([x, v])
                S += [current_depth + 1] * 2

        T = PTree(L, self.feature_names, self.discrete_features)
        T.prune()
        return T

    def select_truncation(self, obj):
        return np.argsort(obj)[:self.mu]

    def crossover(self, P1, P2):
        P1, P2 = [copy.deepcopy(P) for P in (P1, P2)]
        # should use indices of ONLY feature nodes
        feature_ix1 = [i for i in range(P1.N) if P1.L[i].is_feature]
        feature_ix2 = [i for i in range(P2.N) if P2.L[i].is_feature]
        index1 = np.random.choice(feature_ix1)
        index2 = np.random.choice(feature_ix2)
        slice1 = P1.get_subtree(index1)
        slice2 = P2.get_subtree(index2)
        P1.L[slice1], P2.L[slice2] = P2.L[slice2], P1.L[slice1]
        P1.build()
        P2.build()
        return (P1, P2)

    def mutate(self, P, mutate_actions=True):
        P = copy.deepcopy(P)

        for item in P.L:
            if np.random.rand() < self.mut_prob:
                if item.is_feature:
                    low, high = self.feature_bounds[item.index]
                    if item.is_discrete:
                        item.threshold = np.random.randint(low, high+1)
                    else:
                        item.threshold = self.bounded_gaussian(
                            item.threshold, [low, high])
                elif mutate_actions:
                    if self.discrete_actions:
                        item.value = str(np.random.choice(self.action_names))
                    else:
                        item.value = self.bounded_gaussian(
                            item.value, self.action_bounds)

        return P

    def bounded_gaussian(self, x, bounds):
        # do mutation in normalized [0,1] to avoid sigma scaling issues
        lb, ub = bounds
        xnorm = (x - lb) / (ub - lb)
        x_trial = np.clip(xnorm + np.random.normal(0, scale=0.1), 0, 1)

        return lb + x_trial * (ub - lb)

    def dominates(self, a, b):
        # assumes minimization
        # a dominates b if it is <= in all objectives and < in at least one
        return (np.all(a <= b) and np.any(a < b))

    def same_box(self, a, b):
        if self.epsilons:
            a = a // self.epsilons
            b = b // self.epsilons
        return np.all(a == b)

    def binary_tournament(self, P, f):
        # select 1 parent from population P
        # (Luke Algorithm 99 p.138)
        i = np.random.randint(0, P.shape[0], 2)
        a, b = f[i[0]], f[i[1]]
        if self.dominates(a, b):
            return i[0]
        elif self.dominates(b, a):
            return i[1]
        else:
            return i[0] if np.random.rand() < 0.5 else i[1]

    def archive_sort(self, A, fA, P, fP):
        A = np.hstack((A, P))
        fA = np.vstack((fA, fP))
        N = len(A)
        keep = np.ones(N, dtype=bool)

        for i in range(N):
            for j in range(i + 1, N):
                if keep[j] and self.dominates(fA[i, :], fA[j, :]):  # \
                    keep[j] = False

                elif keep[i] and self.dominates(fA[j, :], fA[i, :]):
                    keep[i] = False

                elif self.same_box(fA[i, :], fA[j, :]):
                    keep[np.random.choice([i, j])] = False

        return (A[keep], fA[keep, :])
<commit_msg>fix parallel bug with action counts<commit_after>from __future__ import division
import numpy as np
import time
import datetime
import copy
from .tree import *

class PTreeOpt(object):

    def __init__(self, f, feature_bounds, discrete_actions=False,
                 action_bounds=None, action_names=None, population_size=100,
                 mu=15, max_depth=4, mut_prob=0.9, cx_prob=0.9,
                 feature_names=None, discrete_features=None,
                 multiobj=False, epsilons=None):

        self.f = f
        self.num_features = len(feature_bounds)
        self.feature_bounds = feature_bounds
        self.discrete_actions = discrete_actions
        self.action_bounds = action_bounds
        self.action_names = action_names
        self.popsize = population_size
        self.mu = mu
        self.max_depth = max_depth
        self.mut_prob = mut_prob
        self.cx_prob = cx_prob
        self.feature_names = feature_names
        self.discrete_features = discrete_features
        self.multiobj = multiobj
        self.epsilons = epsilons

        if feature_names is not None and len(feature_names) != len(feature_bounds):
            raise ValueError(('feature_names and feature_bounds '
                              'must be the same length.'))

        if discrete_features is not None and len(discrete_features) != len(feature_bounds):
            raise ValueError(('discrete_features and feature_bounds '
                              'must be the same length.'))

        if discrete_actions:
            if action_names is None or action_bounds is not None:
                raise ValueError(('discrete_actions must be run with '
                                  'action_names, (which are strings), '
                                  'and not action_bounds.'))
        else:
            if action_bounds is None:
                raise ValueError(('Real-valued actions (which is the case by '
                                'default, discrete_actions=False) must include '
                                'action_bounds. Currently only one action is '
                                'supported, so bounds = [lower, upper].'))

        if mu > population_size:
            raise ValueError(('Number of parents (mu) cannot be greater than '
                              'the population_size.'))

    def iterate(self):

        # selection: find index numbers for parents
        if not self.multiobj:

            parents = self.select_truncation(self.objectives)

            if self.best_f is None or self.objectives[parents[0]] < self.best_f:
                self.best_f = self.objectives[parents[0]]
                self.best_P = copy.deepcopy(self.population[parents[0]])

        else:
            parents = [self.binary_tournament(self.population, self.objectives)
                       for _ in range(self.mu)]

            if self.best_f is None:
                self.best_f = self.objectives[parents]
                self.best_P = self.population[parents]
            else:
                self.best_P, self.best_f = self.archive_sort(self.best_P,
                                                 self.best_f, self.population,
                                                 self.objectives)

        # first: mutate the parents, only keep child if it's better
        # changed 7/17/17 JDH: now mutate all except best parent (single-obj)
        children = set(range(self.popsize)) - set(parents)

        for i in parents[1:]:
            child = self.mutate(self.population[i])
            child.prune()
            self.population[i] = child

        # then crossover to develop the rest of the population
        for i in children:

            if np.random.rand() < self.cx_prob:
                P1, P2 = np.random.choice(
                    self.population[parents], 2, replace=False)
                child = self.crossover(P1, P2)[0]

                # bloat control
                while child.get_depth() > self.max_depth:
                    child = self.crossover(P1, P2)[0]

            else:  # replace with randomly chosen parent
                child = np.random.choice(self.population[parents], 1)[0]

            child = self.mutate(child)
            child.prune()
            self.population[i] = child

    def run(self, max_nfe=100, parallel=False, log_frequency=None):

        if parallel:
            from mpi4py import MPI
            comm = MPI.COMM_WORLD
            size = comm.Get_size()
            rank = comm.Get_rank()

        is_master = (not parallel) or (parallel and rank == 0)
        start_time = time.time()
        nfe, last_log = 0, 0

        if is_master:
            self.population = np.array(
                [self.random_tree() for _ in range(self.popsize)])
            self.best_f = None
            self.best_P = None

            if log_frequency:
                snapshots = {'nfe': [], 'time': [], 'best_f': [], 'best_P': []}
        else:
            self.population = None

        while nfe < max_nfe:

            if is_master:
                for P in self.population:
                    P.clear_count() # reset action counts to zero

            # evaluate objectives
            if not parallel:
                self.objectives = np.array(
                    [self.f(P) for P in self.population])
            else:
                if is_master:
                    chunks = np.array_split(self.population, size)
                else:
                    chunks = None

                local_Ps = comm.scatter(chunks, root=0)
                local_fs = [self.f(P) for P in local_Ps]
                objs = comm.gather(local_fs, root=0)
                temp_pop = comm.gather(local_Ps, root=0)
                comm.barrier()

                if is_master:
                    self.objectives = np.concatenate(objs)  # flatten list
                    self.population = np.concatenate(temp_pop)

            if is_master:
                for P in self.population:
                    P.normalize_count() # convert action count to percent

            nfe += self.popsize

            if is_master:
                self.iterate()

                if log_frequency is not None and nfe >= last_log + log_frequency:
                    elapsed = datetime.timedelta(
                        seconds=time.time() - start_time).seconds

                    if not self.multiobj:
                        print('%d\t%s\t%0.3f\t%s' %
                              (nfe, elapsed, self.best_f, self.best_P))
                    else:
                        print('# nfe = %d\n%s' % (nfe, self.best_f))
                        print(self.best_f.shape)
                    snapshots['nfe'].append(nfe)
                    snapshots['time'].append(elapsed)
                    snapshots['best_f'].append(self.best_f)
                    snapshots['best_P'].append(self.best_P)
                    last_log = nfe

        if is_master and log_frequency:
            return snapshots

    def random_tree(self, terminal_ratio=0.5):
        depth = np.random.randint(1, self.max_depth + 1)
        L = []
        S = [0]

        while S:
            current_depth = S.pop()

            # action node
            if current_depth == depth or (current_depth > 0 and\
                                          np.random.rand() < terminal_ratio):
                if self.discrete_actions:
                    L.append([str(np.random.choice(self.action_names))])
                else:
                    L.append([np.random.uniform(*self.action_bounds)])

            else:
                x = np.random.choice(self.num_features)
                v = np.random.uniform(*self.feature_bounds[x])
                L.append([x, v])
                S += [current_depth + 1] * 2

        T = PTree(L, self.feature_names, self.discrete_features)
        T.prune()
        return T

    def select_truncation(self, obj):
        return np.argsort(obj)[:self.mu]

    def crossover(self, P1, P2):
        P1, P2 = [copy.deepcopy(P) for P in (P1, P2)]
        # should use indices of ONLY feature nodes
        feature_ix1 = [i for i in range(P1.N) if P1.L[i].is_feature]
        feature_ix2 = [i for i in range(P2.N) if P2.L[i].is_feature]
        index1 = np.random.choice(feature_ix1)
        index2 = np.random.choice(feature_ix2)
        slice1 = P1.get_subtree(index1)
        slice2 = P2.get_subtree(index2)
        P1.L[slice1], P2.L[slice2] = P2.L[slice2], P1.L[slice1]
        P1.build()
        P2.build()
        return (P1, P2)

    def mutate(self, P, mutate_actions=True):
        P = copy.deepcopy(P)

        for item in P.L:
            if np.random.rand() < self.mut_prob:
                if item.is_feature:
                    low, high = self.feature_bounds[item.index]
                    if item.is_discrete:
                        item.threshold = np.random.randint(low, high+1)
                    else:
                        item.threshold = self.bounded_gaussian(
                            item.threshold, [low, high])
                elif mutate_actions:
                    if self.discrete_actions:
                        item.value = str(np.random.choice(self.action_names))
                    else:
                        item.value = self.bounded_gaussian(
                            item.value, self.action_bounds)

        return P

    def bounded_gaussian(self, x, bounds):
        # do mutation in normalized [0,1] to avoid sigma scaling issues
        lb, ub = bounds
        xnorm = (x - lb) / (ub - lb)
        x_trial = np.clip(xnorm + np.random.normal(0, scale=0.1), 0, 1)

        return lb + x_trial * (ub - lb)

    def dominates(self, a, b):
        # assumes minimization
        # a dominates b if it is <= in all objectives and < in at least one
        return (np.all(a <= b) and np.any(a < b))

    def same_box(self, a, b):
        if self.epsilons:
            a = a // self.epsilons
            b = b // self.epsilons
        return np.all(a == b)

    def binary_tournament(self, P, f):
        # select 1 parent from population P
        # (Luke Algorithm 99 p.138)
        i = np.random.randint(0, P.shape[0], 2)
        a, b = f[i[0]], f[i[1]]
        if self.dominates(a, b):
            return i[0]
        elif self.dominates(b, a):
            return i[1]
        else:
            return i[0] if np.random.rand() < 0.5 else i[1]

    def archive_sort(self, A, fA, P, fP):
        A = np.hstack((A, P))
        fA = np.vstack((fA, fP))
        N = len(A)
        keep = np.ones(N, dtype=bool)

        for i in range(N):
            for j in range(i + 1, N):
                if keep[j] and self.dominates(fA[i, :], fA[j, :]):  # \
                    keep[j] = False

                elif keep[i] and self.dominates(fA[j, :], fA[i, :]):
                    keep[i] = False

                elif self.same_box(fA[i, :], fA[j, :]):
                    keep[np.random.choice([i, j])] = False

        return (A[keep], fA[keep, :])
<|endoftext|>


<commit_before># import the necessary packages
# from pyimagesearch.tempimage import TempImage
# from dropbox.client import DropboxOAuth2FlowNoRedirect
# from dropbox.client import DropboxClient
from picamera.array import PiRGBArray
from picamera import PiCamera
import argparse
import warnings
import datetime
import imutils
import json
import time
import cv2

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-c", "--conf", required=True,
                help="path to the JSON configuration file")
args = vars(ap.parse_args())

# filter warnings, load the configuration and initialize the Dropbox
# client
warnings.filterwarnings("ignore")
conf = json.load(open(args["conf"]))
client = None

# initialize the camera and grab a reference to the raw camera capture
camera = PiCamera()
camera.resolution = tuple(conf["resolution"])
camera.framerate = conf["fps"]
rawCapture = PiRGBArray(camera, size=tuple(conf["resolution"]))

# allow the camera to warmup, then initialize the average frame, last
# uploaded timestamp, and frame motion counter
print "[INFO] warming up..."
time.sleep(conf["camera_warmup_time"])
avg = None
lastUploaded = datetime.datetime.now()
motionCounter = 0

# capture frames from the camera
for f in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
    # grab the raw NumPy array representing the image and initialize
    # the timestamp and occupied/unoccupied text
    frame = f.array
    timestamp = datetime.datetime.now()
    text = "Unoccupied"

    # resize the frame, convert it to grayscale, and blur it
    frame = imutils.resize(frame, width=500)
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    gray = cv2.GaussianBlur(gray, (21, 21), 0)

    # if the average frame is None, initialize it
    if avg is None:
        print "[INFO] starting background model..."
        avg = gray.copy().astype("float")
        rawCapture.truncate(0)
        continue

    # accumulate the weighted average between the current frame and
    # previous frames, then compute the difference between the current
    # frame and running average
    cv2.accumulateWeighted(gray, avg, 0.5)
    frameDelta = cv2.absdiff(gray, cv2.convertScaleAbs(avg))

    # threshold the delta image, dilate the thresholded image to fill
    # in holes, then find contours on thresholded image
    thresh = cv2.threshold(frameDelta, conf["delta_thresh"], 255,
                           cv2.THRESH_BINARY)[1]
    thresh = cv2.dilate(thresh, None, iterations=2)
    (cnts, _) = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                 cv2.CHAIN_APPROX_SIMPLE)

    # loop over the contours
    for c in cnts:
        # if the contour is too small, ignore it
        if cv2.contourArea(c) < conf["min_area"]:
            continue

        # compute the bounding box for the contour, draw it on the frame,
        # and update the text
        (x, y, w, h) = cv2.boundingRect(c)
        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
        text = "Occupied"

    # draw the text and timestamp on the frame
    ts = timestamp.strftime("%A %d %B %Y %I:%M:%S%p")
    cv2.putText(frame, "Room Status: {}".format(text), (10, 20),
                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
    cv2.putText(frame, ts, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX,
                0.35, (0, 0, 255), 1)

    # check to see if the room is occupied
    if text == "Occupied":
        # check to see if enough time has passed between uploads
        if (timestamp - lastUploaded).seconds >= conf["min_upload_seconds"]:
            # increment the motion counter
            motionCounter += 1

            # check to see if the number of frames with consistent motion is
            # high enough
            if motionCounter >= conf["min_motion_frames"]:
                # check to see if dropbox sohuld be used
                if conf["use_dropbox"]:
                    '''
                    # write the image to temporary file
                    t = TempImage()
                    cv2.imwrite(t.path, frame)

                    # upload the image to Dropbox and cleanup the tempory image
                    print "[UPLOAD] {}".format(ts)
                    path = "{base_path}/{timestamp}.jpg".format(
                        base_path=conf["dropbox_base_path"], timestamp=ts)
                    client.put_file(path, open(t.path, "rb"))
                    t.cleanup()
                    '''

                # update the last uploaded timestamp and reset the motion
                # counter
                lastUploaded = timestamp
                motionCounter = 0

    # otherwise, the room is not occupied
    else:
        motionCounter = 0

    # check to see if the frames should be displayed to screen
    if conf["show_video"]:
        # display the security feed
        cv2.imshow("Security Feed", frame)
        key = cv2.waitKey(1) & 0xFF

        # if the `q` key is pressed, break from the lop
        if key == ord("q"):
            break

    # clear the stream in preparation for the next frame
    rawCapture.truncate(0)

<commit_msg>Fixed findContours which was returning 3 parameters in OpenCV 3.1 but only 2 specified<commit_after># import the necessary packages
# from pyimagesearch.tempimage import TempImage
# from dropbox.client import DropboxOAuth2FlowNoRedirect
# from dropbox.client import DropboxClient
from picamera.array import PiRGBArray
from picamera import PiCamera
import argparse
import warnings
import datetime
import imutils
import json
import time
import cv2

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-c", "--conf", required=True,
                help="path to the JSON configuration file")
args = vars(ap.parse_args())

# filter warnings, load the configuration and initialize the Dropbox
# client
warnings.filterwarnings("ignore")
conf = json.load(open(args["conf"]))
client = None

# initialize the camera and grab a reference to the raw camera capture
camera = PiCamera()
camera.resolution = tuple(conf["resolution"])
camera.framerate = conf["fps"]
rawCapture = PiRGBArray(camera, size=tuple(conf["resolution"]))

# allow the camera to warmup, then initialize the average frame, last
# uploaded timestamp, and frame motion counter
print "[INFO] warming up..."
time.sleep(conf["camera_warmup_time"])
avg = None
lastUploaded = datetime.datetime.now()
motionCounter = 0

# capture frames from the camera
for f in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
    # grab the raw NumPy array representing the image and initialize
    # the timestamp and occupied/unoccupied text
    frame = f.array
    timestamp = datetime.datetime.now()
    text = "Unoccupied"

    # resize the frame, convert it to grayscale, and blur it
    frame = imutils.resize(frame, width=500)
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    gray = cv2.GaussianBlur(gray, (21, 21), 0)

    # if the average frame is None, initialize it
    if avg is None:
        print "[INFO] starting background model..."
        avg = gray.copy().astype("float")
        rawCapture.truncate(0)
        continue

    # accumulate the weighted average between the current frame and
    # previous frames, then compute the difference between the current
    # frame and running average
    cv2.accumulateWeighted(gray, avg, 0.5)
    frameDelta = cv2.absdiff(gray, cv2.convertScaleAbs(avg))

    # threshold the delta image, dilate the thresholded image to fill
    # in holes, then find contours on thresholded image
    thresh = cv2.threshold(frameDelta, conf["delta_thresh"], 255,
                           cv2.THRESH_BINARY)[1]
    thresh = cv2.dilate(thresh, None, iterations=2)
    (_, contours, _) = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    # loop over the contours
    for c in contours:
        # if the contour is too small, ignore it
        if cv2.contourArea(c) < conf["min_area"]:
            continue

        # compute the bounding box for the contour, draw it on the frame,
        # and update the text
        (x, y, w, h) = cv2.boundingRect(c)
        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
        text = "Occupied"

    # draw the text and timestamp on the frame
    ts = timestamp.strftime("%A %d %B %Y %I:%M:%S%p")
    cv2.putText(frame, "Room Status: {}".format(text), (10, 20),
                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
    cv2.putText(frame, ts, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX,
                0.35, (0, 0, 255), 1)

    # check to see if the room is occupied
    if text == "Occupied":
        # check to see if enough time has passed between uploads
        if (timestamp - lastUploaded).seconds >= conf["min_upload_seconds"]:
            # increment the motion counter
            motionCounter += 1

            # check to see if the number of frames with consistent motion is
            # high enough
            if motionCounter >= conf["min_motion_frames"]:
                # check to see if dropbox sohuld be used
                if conf["use_dropbox"]:
                    '''
                    # write the image to temporary file
                    t = TempImage()
                    cv2.imwrite(t.path, frame)

                    # upload the image to Dropbox and cleanup the tempory image
                    print "[UPLOAD] {}".format(ts)
                    path = "{base_path}/{timestamp}.jpg".format(
                        base_path=conf["dropbox_base_path"], timestamp=ts)
                    client.put_file(path, open(t.path, "rb"))
                    t.cleanup()
                    '''

                # update the last uploaded timestamp and reset the motion
                # counter
                lastUploaded = timestamp
                motionCounter = 0

    # otherwise, the room is not occupied
    else:
        motionCounter = 0

    # check to see if the frames should be displayed to screen
    if conf["show_video"]:
        # display the security feed
        cv2.imshow("Security Feed", frame)
        key = cv2.waitKey(1) & 0xFF

        # if the `q` key is pressed, break from the lop
        if key == ord("q"):
            break

    # clear the stream in preparation for the next frame
    rawCapture.truncate(0)

<|endoftext|>


<commit_before># Copyright: Damien Elmes <anki@ichi2.net>
# -*- coding: utf-8 -*-
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html

from aqt.qt import *
from aqt.utils import askUser, getOnlyText, openLink, showWarning, shortcut, \
    openHelp
from anki.utils import isMac, ids2str, fmtTimeSpan
import anki.js
from anki.errors import DeckRenameError
import aqt
from anki.sound import clearAudioQueue

class DeckBrowser(object):

    def __init__(self, mw):
        self.mw = mw
        self.web = mw.web
        self.bottom = aqt.toolbar.BottomBar(mw, mw.bottomWeb)
        self.scrollPos = QPoint(0, 0)

    def show(self):
        clearAudioQueue()
        self.web.setLinkHandler(self._linkHandler)
        self.web.setKeyHandler(None)
        self.mw.keyHandler = self._keyHandler
        self._renderPage()

    def refresh(self):
        self._renderPage()

    # Event handlers
    ##########################################################################

    def _linkHandler(self, url):
        if ":" in url:
            (cmd, arg) = url.split(":")
        else:
            cmd = url
        if cmd == "open":
            self._selDeck(arg)
        elif cmd == "opts":
            self._showOptions(arg)
        elif cmd == "shared":
            self._onShared()
        elif cmd == "import":
            self.mw.onImport()
        elif cmd == "lots":
            openHelp("using-decks-appropriately")
        elif cmd == "hidelots":
            self.mw.pm.profile['hideDeckLotsMsg'] = True
            self.refresh()
        elif cmd == "create":
            deck = getOnlyText(_("Name for deck:"))
            if deck:
                self.mw.col.decks.id(deck)
                self.refresh()
        elif cmd == "drag":
            draggedDeckDid, ontoDeckDid = arg.split(',')
            self._dragDeckOnto(draggedDeckDid, ontoDeckDid)
        elif cmd == "collapse":
            self._collapse(arg)

    def _keyHandler(self, evt):
        # currently does nothing
        key = unicode(evt.text())

    def _selDeck(self, did):
        self.scrollPos =  self.web.page().mainFrame().scrollPosition()
        self.mw.col.decks.select(did)
        self.mw.onOverview()

    # HTML generation
    ##########################################################################

    _dragIndicatorBorderWidth = "1px"

    _css = """
a.deck { color: #000; text-decoration: none; min-width: 5em;
         display:inline-block; }
a.deck:hover { text-decoration: underline; }
tr.deck td { border-bottom: %(width)s solid #e7e7e7; }
tr.top-level-drag-row td { border-bottom: %(width)s solid transparent; }
td { white-space: nowrap; }
tr.drag-hover td { border-bottom: %(width)s solid #aaa; }
body { margin: 1em; -webkit-user-select: none; }
.current { background-color: #e7e7e7; }
.decktd { min-width: 15em; }
.count { width: 6em; text-align: right; }
.collapse { color: #000; text-decoration:none; display:inline-block;
    width: 1em; }
.filtered { color: #00a !important; }
""" % dict(width=_dragIndicatorBorderWidth)

    _body = """

%(tree)s

%(stats)s
%(countwarn)s

"""

    def _renderPage(self, reuse=False):
        css = self.mw.sharedCSS + self._css
        if not reuse:
            self._dueTree = self.mw.col.sched.deckDueTree()
        tree = self._renderDeckTree(self._dueTree)
        stats = self._renderStats()
        op = self._oldPos()
        self.web.stdHtml(self._body%dict(
            tree=tree, stats=stats, countwarn=self._countWarn()), css=css,
                         js=anki.js.jquery+anki.js.ui, loadCB=lambda ok:\
                         self.web.page().mainFrame().setScrollPosition(op))
        self.web.key = "deckBrowser"
        self._drawButtons()

    def _oldPos(self):
        if self.web.key == "deckBrowser":
            return self.web.page().mainFrame().scrollPosition()
        else:
            return self.scrollPos

    def _renderStats(self):
        cards, thetime = self.mw.col.db.first("""
select count(), sum(time)/1000 from revlog
where id > ?""", (self.mw.col.sched.dayCutoff-86400)*1000)
        cards = cards or 0
        thetime = thetime or 0
        msgp1 = ngettext("%d card", "%d cards", cards) % cards
        buf = _("Studied %(a)s in %(b)s today.") % dict(a=msgp1,
                                                        b=fmtTimeSpan(thetime, unit=1))
        return buf

    def _countWarn(self):
        if (self.mw.col.decks.count() < 25 or
                self.mw.pm.profile.get("hideDeckLotsMsg")):
            return ""
        return "

"+(
            _("You have a lot of decks. Please see %(a)s. %(b)s") % dict(
                a=("%s" % _("this page")),
                b=("
(%s)" % (_("hide"))+
                    "</div")))

    def _renderDeckTree(self, nodes, depth=0):
        if not nodes:
            return ""
        if depth == 0:
            buf = """

%s	%s
%s
""" % (
            _("Deck"), _("Due"), _("New"))
            buf += self._topLevelDragRow()
        else:
            buf = ""
        for node in nodes:
            buf += self._deckRow(node, depth, len(nodes))
        if depth == 0:
            buf += self._topLevelDragRow()
        return buf

    def _deckRow(self, node, depth, cnt):
        name, did, due, lrn, new, children = node
        deck = self.mw.col.decks.get(did)
        if did == 1 and cnt > 1 and not children:
            # if the default deck is empty, hide it
            if not self.mw.col.db.scalar("select 1 from cards where did = 1"):
                return ""
        # parent toggled for collapsing
        for parent in self.mw.col.decks.parents(did):
            if parent['collapsed']:
                buff = ""
                return buff
        prefix = "-"
        if self.mw.col.decks.get(did)['collapsed']:
            prefix = "+"
        due += lrn
        def indent():
            return " "*6*depth
        if did == self.mw.col.conf['curDeck']:
            klass = 'deck current'
        else:
            klass = 'deck'
        buf = "
" % (klass, did)
        # deck link
        if children:
            collapse = "%s" % (did, prefix)
        else:
            collapse = ""
        if deck['dyn']:
            extraclass = "filtered"
        else:
            extraclass = ""
        buf += """

        %s%s%s	"""% (
            indent(), collapse, extraclass, did, name)
        # due counts
        def nonzeroColour(cnt, colour):
            if not cnt:
                colour = "#e0e0e0"
            if cnt >= 1000:
                cnt = "1000+"
            return "%s" % (colour, cnt)
        buf += "%s	%s	" % (
            nonzeroColour(due, "#007700"),
            nonzeroColour(new, "#000099"))
        # options
        buf += "%s
" % self.mw.button(
            link="opts:%d"%did, name="▾")
        # children
        buf += self._renderDeckTree(children, depth+1)
        return buf

    def _topLevelDragRow(self):
        return "

"

    def _dueImg(self, due, new):
        if due:
            i = "clock-icon"
        elif new:
            i = "plus-circle"
        else:
            i = "none"
        return '' % i

    # Options
    ##########################################################################

    def _showOptions(self, did):
        m = QMenu(self.mw)
        a = m.addAction(_("Rename"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._rename(did))
        a = m.addAction(_("Options"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._options(did))
        a = m.addAction(_("Export"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._export(did))
        a = m.addAction(_("Delete"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._delete(did))
        m.exec_(QCursor.pos())

    def _export(self, did):
        self.mw.onExport(did=did)

    def _rename(self, did):
        self.mw.checkpoint(_("Rename Deck"))
        deck = self.mw.col.decks.get(did)
        oldName = deck['name']
        newName = getOnlyText(_("New deck name:"), default=oldName)
        newName = newName.replace('"', "")
        if not newName or newName == oldName:
            return
        try:
            self.mw.col.decks.rename(deck, newName)
        except DeckRenameError, e:
            return showWarning(e.description)
        self.show()

    def _options(self, did):
        # select the deck first, because the dyn deck conf assumes the deck
        # we're editing is the current one
        self.mw.col.decks.select(did)
        self.mw.onDeckConf()

    def _collapse(self, did):
        self.mw.col.decks.collapse(did)
        self._renderPage(reuse=True)

    def _dragDeckOnto(self, draggedDeckDid, ontoDeckDid):
        try:
            self.mw.col.decks.renameForDragAndDrop(draggedDeckDid, ontoDeckDid)
        except DeckRenameError, e:
            return showWarning(e.description)

        self.show()

    def _delete(self, did):
        if str(did) == '1':
            return showWarning(_("The default deck can't be deleted."))
        self.mw.checkpoint(_("Delete Deck"))
        deck = self.mw.col.decks.get(did)
        if not deck['dyn']:
            dids = [did] + [r[1] for r in self.mw.col.decks.children(did)]
            cnt = self.mw.col.db.scalar(
                "select count() from cards where did in {0} or "
                "odid in {0}".format(ids2str(dids)))
            if cnt:
                extra = ngettext(" It has %d card.", " It has %d cards.", cnt) % cnt
            else:
                extra = None
        if deck['dyn'] or not extra or askUser(
            (_("Are you sure you wish to delete %s?") % deck['name']) +
            extra):
            self.mw.progress.start(immediate=True)
            self.mw.col.decks.rem(did, True)
            self.mw.progress.finish()
            self.show()

    # Top buttons
    ######################################################################

    def _drawButtons(self):
        links = [
            ["", "shared", _("Get Shared")],
            ["", "create", _("Create Deck")],
            ["Ctrl+I", "import", _("Import File")],
        ]
        buf = ""
        for b in links:
            if b[0]:
                b[0] = _("Shortcut key: %s") % shortcut(b[0])
            buf += """
<button title='%s' onclick='py.link(\"%s\");'>%s</button>""" % tuple(b)
        self.bottom.draw(buf)
        if isMac:
            size = 28
        else:
            size = 36 + self.mw.fontHeightDelta*3
        self.bottom.web.setFixedHeight(size)
        self.bottom.web.setLinkHandler(self._linkHandler)

    def _onShared(self):
        openLink(aqt.appShared+"decks/")
<commit_msg>use a different unicode arrow<commit_after># Copyright: Damien Elmes <anki@ichi2.net>
# -*- coding: utf-8 -*-
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html

from aqt.qt import *
from aqt.utils import askUser, getOnlyText, openLink, showWarning, shortcut, \
    openHelp
from anki.utils import isMac, ids2str, fmtTimeSpan
import anki.js
from anki.errors import DeckRenameError
import aqt
from anki.sound import clearAudioQueue

class DeckBrowser(object):

    def __init__(self, mw):
        self.mw = mw
        self.web = mw.web
        self.bottom = aqt.toolbar.BottomBar(mw, mw.bottomWeb)
        self.scrollPos = QPoint(0, 0)

    def show(self):
        clearAudioQueue()
        self.web.setLinkHandler(self._linkHandler)
        self.web.setKeyHandler(None)
        self.mw.keyHandler = self._keyHandler
        self._renderPage()

    def refresh(self):
        self._renderPage()

    # Event handlers
    ##########################################################################

    def _linkHandler(self, url):
        if ":" in url:
            (cmd, arg) = url.split(":")
        else:
            cmd = url
        if cmd == "open":
            self._selDeck(arg)
        elif cmd == "opts":
            self._showOptions(arg)
        elif cmd == "shared":
            self._onShared()
        elif cmd == "import":
            self.mw.onImport()
        elif cmd == "lots":
            openHelp("using-decks-appropriately")
        elif cmd == "hidelots":
            self.mw.pm.profile['hideDeckLotsMsg'] = True
            self.refresh()
        elif cmd == "create":
            deck = getOnlyText(_("Name for deck:"))
            if deck:
                self.mw.col.decks.id(deck)
                self.refresh()
        elif cmd == "drag":
            draggedDeckDid, ontoDeckDid = arg.split(',')
            self._dragDeckOnto(draggedDeckDid, ontoDeckDid)
        elif cmd == "collapse":
            self._collapse(arg)

    def _keyHandler(self, evt):
        # currently does nothing
        key = unicode(evt.text())

    def _selDeck(self, did):
        self.scrollPos =  self.web.page().mainFrame().scrollPosition()
        self.mw.col.decks.select(did)
        self.mw.onOverview()

    # HTML generation
    ##########################################################################

    _dragIndicatorBorderWidth = "1px"

    _css = """
a.deck { color: #000; text-decoration: none; min-width: 5em;
         display:inline-block; }
a.deck:hover { text-decoration: underline; }
tr.deck td { border-bottom: %(width)s solid #e7e7e7; }
tr.top-level-drag-row td { border-bottom: %(width)s solid transparent; }
td { white-space: nowrap; }
tr.drag-hover td { border-bottom: %(width)s solid #aaa; }
body { margin: 1em; -webkit-user-select: none; }
.current { background-color: #e7e7e7; }
.decktd { min-width: 15em; }
.count { width: 6em; text-align: right; }
.collapse { color: #000; text-decoration:none; display:inline-block;
    width: 1em; }
.filtered { color: #00a !important; }
""" % dict(width=_dragIndicatorBorderWidth)

    _body = """

%(tree)s

%(stats)s
%(countwarn)s

"""

    def _renderPage(self, reuse=False):
        css = self.mw.sharedCSS + self._css
        if not reuse:
            self._dueTree = self.mw.col.sched.deckDueTree()
        tree = self._renderDeckTree(self._dueTree)
        stats = self._renderStats()
        op = self._oldPos()
        self.web.stdHtml(self._body%dict(
            tree=tree, stats=stats, countwarn=self._countWarn()), css=css,
                         js=anki.js.jquery+anki.js.ui, loadCB=lambda ok:\
                         self.web.page().mainFrame().setScrollPosition(op))
        self.web.key = "deckBrowser"
        self._drawButtons()

    def _oldPos(self):
        if self.web.key == "deckBrowser":
            return self.web.page().mainFrame().scrollPosition()
        else:
            return self.scrollPos

    def _renderStats(self):
        cards, thetime = self.mw.col.db.first("""
select count(), sum(time)/1000 from revlog
where id > ?""", (self.mw.col.sched.dayCutoff-86400)*1000)
        cards = cards or 0
        thetime = thetime or 0
        msgp1 = ngettext("%d card", "%d cards", cards) % cards
        buf = _("Studied %(a)s in %(b)s today.") % dict(a=msgp1,
                                                        b=fmtTimeSpan(thetime, unit=1))
        return buf

    def _countWarn(self):
        if (self.mw.col.decks.count() < 25 or
                self.mw.pm.profile.get("hideDeckLotsMsg")):
            return ""
        return "

"+(
            _("You have a lot of decks. Please see %(a)s. %(b)s") % dict(
                a=("%s" % _("this page")),
                b=("
(%s)" % (_("hide"))+
                    "</div")))

    def _renderDeckTree(self, nodes, depth=0):
        if not nodes:
            return ""
        if depth == 0:
            buf = """

%s	%s
%s
""" % (
            _("Deck"), _("Due"), _("New"))
            buf += self._topLevelDragRow()
        else:
            buf = ""
        for node in nodes:
            buf += self._deckRow(node, depth, len(nodes))
        if depth == 0:
            buf += self._topLevelDragRow()
        return buf

    def _deckRow(self, node, depth, cnt):
        name, did, due, lrn, new, children = node
        deck = self.mw.col.decks.get(did)
        if did == 1 and cnt > 1 and not children:
            # if the default deck is empty, hide it
            if not self.mw.col.db.scalar("select 1 from cards where did = 1"):
                return ""
        # parent toggled for collapsing
        for parent in self.mw.col.decks.parents(did):
            if parent['collapsed']:
                buff = ""
                return buff
        prefix = "-"
        if self.mw.col.decks.get(did)['collapsed']:
            prefix = "+"
        due += lrn
        def indent():
            return " "*6*depth
        if did == self.mw.col.conf['curDeck']:
            klass = 'deck current'
        else:
            klass = 'deck'
        buf = "
" % (klass, did)
        # deck link
        if children:
            collapse = "%s" % (did, prefix)
        else:
            collapse = ""
        if deck['dyn']:
            extraclass = "filtered"
        else:
            extraclass = ""
        buf += """

        %s%s%s	"""% (
            indent(), collapse, extraclass, did, name)
        # due counts
        def nonzeroColour(cnt, colour):
            if not cnt:
                colour = "#e0e0e0"
            if cnt >= 1000:
                cnt = "1000+"
            return "%s" % (colour, cnt)
        buf += "%s	%s	" % (
            nonzeroColour(due, "#007700"),
            nonzeroColour(new, "#000099"))
        # options
        buf += "%s
" % self.mw.button(
            link="opts:%d"%did, name="▼")
        # children
        buf += self._renderDeckTree(children, depth+1)
        return buf

    def _topLevelDragRow(self):
        return "

"

    def _dueImg(self, due, new):
        if due:
            i = "clock-icon"
        elif new:
            i = "plus-circle"
        else:
            i = "none"
        return '' % i

    # Options
    ##########################################################################

    def _showOptions(self, did):
        m = QMenu(self.mw)
        a = m.addAction(_("Rename"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._rename(did))
        a = m.addAction(_("Options"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._options(did))
        a = m.addAction(_("Export"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._export(did))
        a = m.addAction(_("Delete"))
        a.connect(a, SIGNAL("triggered()"), lambda did=did: self._delete(did))
        m.exec_(QCursor.pos())

    def _export(self, did):
        self.mw.onExport(did=did)

    def _rename(self, did):
        self.mw.checkpoint(_("Rename Deck"))
        deck = self.mw.col.decks.get(did)
        oldName = deck['name']
        newName = getOnlyText(_("New deck name:"), default=oldName)
        newName = newName.replace('"', "")
        if not newName or newName == oldName:
            return
        try:
            self.mw.col.decks.rename(deck, newName)
        except DeckRenameError, e:
            return showWarning(e.description)
        self.show()

    def _options(self, did):
        # select the deck first, because the dyn deck conf assumes the deck
        # we're editing is the current one
        self.mw.col.decks.select(did)
        self.mw.onDeckConf()

    def _collapse(self, did):
        self.mw.col.decks.collapse(did)
        self._renderPage(reuse=True)

    def _dragDeckOnto(self, draggedDeckDid, ontoDeckDid):
        try:
            self.mw.col.decks.renameForDragAndDrop(draggedDeckDid, ontoDeckDid)
        except DeckRenameError, e:
            return showWarning(e.description)

        self.show()

    def _delete(self, did):
        if str(did) == '1':
            return showWarning(_("The default deck can't be deleted."))
        self.mw.checkpoint(_("Delete Deck"))
        deck = self.mw.col.decks.get(did)
        if not deck['dyn']:
            dids = [did] + [r[1] for r in self.mw.col.decks.children(did)]
            cnt = self.mw.col.db.scalar(
                "select count() from cards where did in {0} or "
                "odid in {0}".format(ids2str(dids)))
            if cnt:
                extra = ngettext(" It has %d card.", " It has %d cards.", cnt) % cnt
            else:
                extra = None
        if deck['dyn'] or not extra or askUser(
            (_("Are you sure you wish to delete %s?") % deck['name']) +
            extra):
            self.mw.progress.start(immediate=True)
            self.mw.col.decks.rem(did, True)
            self.mw.progress.finish()
            self.show()

    # Top buttons
    ######################################################################

    def _drawButtons(self):
        links = [
            ["", "shared", _("Get Shared")],
            ["", "create", _("Create Deck")],
            ["Ctrl+I", "import", _("Import File")],
        ]
        buf = ""
        for b in links:
            if b[0]:
                b[0] = _("Shortcut key: %s") % shortcut(b[0])
            buf += """
<button title='%s' onclick='py.link(\"%s\");'>%s</button>""" % tuple(b)
        self.bottom.draw(buf)
        if isMac:
            size = 28
        else:
            size = 36 + self.mw.fontHeightDelta*3
        self.bottom.web.setFixedHeight(size)
        self.bottom.web.setLinkHandler(self._linkHandler)

    def _onShared(self):
        openLink(aqt.appShared+"decks/")
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# PyarrFS - a RAR reading file system
# Copyright (c) 2010, 2011 Kristian Larsson <kristian@spritelink.net>
#
# This file is license under the X11/MIT license, please see the file COPYING
# distributed with PyarrFS for more details.
#

import os, sys
import errno
import fcntl
import re
import stat
import logging
import logging.handlers

import fuse
import rarfile

__version__         = '0.5.0'
__author__          = 'Kristian Larsson'
__author_email__    = 'kristian@spritelink.net'
__license__         = 'MIT'
__url__             = 'http://labs.spritelink.net/pyarrfs'

# log settings
log_format = "%(levelname)-8s %(message)s"
# TODO: allow log-level to be set with option on startup
#       -d should automatically set level to logging.DEBUG
log_level = logging.WARNING
# uncomment for debug
#log_level = logging.DEBUG
log_destinations = {}
# uncomment the following for debug output to stdout
# TODO: only output to STDOUT/STDERR if -f option is set, ie pyarrfs should
#       be run in foreground, or -d as in debug
#log_destinations['stdout'] = 1
# uncomment the following for debug output to syslog
# TODO: allow syslog loging to be enabled with option on startup
log_destinations['syslog'] = 1

logger = logging.getLogger()
logger.setLevel(log_level)
log_format = logging.Formatter(log_format)

if log_destinations.has_key('stdout'):
    log_stream = logging.StreamHandler()
    log_stream.setFormatter(log_format)
    logger.addHandler(log_stream)

if log_destinations.has_key('syslog') or 1==1:
    log_syslog = logging.handlers.SysLogHandler(address = '/dev/log')
    log_syslog.setFormatter(log_format)
    logger.addHandler(log_syslog)

fuse.fuse_python_api = (0, 2)

fuse.feature_assert('stateful_files', 'has_init')

class Pyarr(fuse.Fuse):
    def __init__(self, *args, **kw):
        fuse.Fuse.__init__(self, *args, **kw)
        self.root = '/'
        logger.info("init!")

    def fsinit(self):
        os.chdir(self.root)

    def access(self, path, mode):
        """Returns whether a user has access to performing
        """
        logger.info("access -- " + path)

        # test for write access, PyarrFS is incapable of do writes, so no axx
        if mode == os.W_OK:
            return -errno.EACCES

        # allow the rest
        # FIXME: do more granular access control, based on RAR file?
        return

        # this only works for files not inside a rar archive
        if not os.access("." + path, mode):
            return -errno.EACCES

    def getattr(self, path):
        logger.info("getattr -- " + str(path))
        if re.match('.*\.rar$', path):
            logging.debug("getattr: on rar archive for path " + str(path))
            original_stat = os.lstat("." + path)
            fake_stat = fuse.Stat()
            fake_stat.st_mode = stat.S_IFDIR | 0755
            fake_stat.st_ino = 0
            fake_stat.st_dev = 0
            fake_stat.st_rdev = 0
            fake_stat.st_nlink = 2
            fake_stat.st_uid = original_stat.st_uid
            fake_stat.st_gid = original_stat.st_gid
            fake_stat.st_size = 4096
            fake_stat.st_atime = original_stat.st_atime
            fake_stat.st_mtime = original_stat.st_mtime
            fake_stat.st_ctime = original_stat.st_ctime
            logging.debug("getattr: returning fake_stat for " + str(path))
            return fake_stat

        elif re.match('.*\.rar/.+', path):
            logging.debug("getattr: we need to check inside rar archive for path " + str(path))
            m = re.match('(.*\.rar)/(.+)', path)
            rar_file = m.group(1)
            rar_path = m.group(2)

            original_stat = os.lstat("." + rar_file)
            rf = rarfile.RarFile(rar_file, 'r', None, None, False)
            try:
                rfi = rf.getinfo(rar_path)
            except:
                # FIXME: add DEBUG log entry
                return -errno.ENOENT

            fake_stat = fuse.Stat()
            fake_stat.st_mode = stat.S_IFREG | 0444
            fake_stat.st_ino = 0
            fake_stat.st_dev = 0
            fake_stat.st_rdev = 0
            fake_stat.st_nlink = 1
            fake_stat.st_uid = original_stat.st_uid
            fake_stat.st_gid = original_stat.st_gid
            fake_stat.st_size = rfi.file_size
            fake_stat.st_blocks = (fake_stat.st_size + 511) / 512
            fake_stat.st_blksize = 4096

            import datetime
            (rft_year, rft_month, rft_day, rft_hour, rft_minute, rft_second) = rfi.date_time
            rft_time = datetime.datetime(rft_year, rft_month, rft_day, rft_hour, rft_minute, rft_second)
            fake_stat.st_atime = int(rft_time.strftime('%s'))
            fake_stat.st_mtime = int(rft_time.strftime('%s'))
            fake_stat.st_ctime = int(rft_time.strftime('%s'))
            logger.debug("getattr: returning fake_stat for " + str(rar_path) + " inside rar " + str(rar_file))
            return fake_stat

        logger.debug("getattr: returning normal os.lstat() for path " + str(path))
        return os.lstat("." + path)

    # get a directory listing
    # doesn't need changes in yarrfs compatibility mode
    def readdir(self, path, offset):
        logger.info("readdir -- path: " + str(path) + "  offset: " + str(offset) )
        dirent = [ '.', '..' ]

        if re.match('.*\.rar$', path):
            logger.debug("readdir: on rar archive, using rarfile")
            rf = rarfile.RarFile(path, 'r', None, None, False)
            for e in rf.namelist():
                dirent.append(str(e))
        else:
            logger.debug("readdir: normal dir, using os.listdir()")
            try:
                os.listdir("." + path)
            except:
                return

            for e in os.listdir("." + path):
                dirent.append(e)

        for e in dirent:
            yield fuse.Direntry(e)

    def readlink(self, path):
        logger.info("readlink -- " + path)
        return os.readlink("." + path)

    def statfs(self):
        logger.info("statfs -- " + path)
        return os.statvfs(".")

    class PyarrFile(object):
        def __init__(self, path, flags, *mode):
            # Enabling direct_io disables the kernels page cache.
            # Since the content of our RAR files should be pretty stable, we do
            # NOT enable this, ie we allow the kernel to cache all data.
            # This means we get both the .rar file and the file inside the RAR
            # archive in the block cache which might affect performance
            # negatively. On the other hand, the kernel could purge the .rar
            # file from its block cache and just keep the inside file.
            # It's likely mostly a performance thing. I'm guessing it's good
            # to let the kernel decide on what to cache and what not too, so we
            # set it to False. Some benchmarking should really be done to prove
            # either setting is better than the other
            self.direct_io = False
            # keep_cache means the kernel is allowed to cache content of a file
            # after its been closed an reopened. You only want to disable this
            # if the content of the file might change without the kernel knowing
            # about it, which would be typical for a networked file system.
            # That's not the case with PyarrFS so we enable it.
            self.keep_cache = True
            m = re.match('(.*\.rar)/(.+)', path)
            self.rar_file = m.group(1)
            self.rar_path = m.group(2)
            self.rf = rarfile.RarFile(self.rar_file, 'r', None, None, False)
            self.file = self.rf.open(self.rar_path)
            self.fd = 0

        def read(self, length, offset):
            self.file.seek(offset)
            return self.file.read(length)

        def release(self, flags):
            self.file.close()

    def main(self, *a, **kw):
        self.file_class = self.PyarrFile
        return fuse.Fuse.main(self, *a, **kw)

def main():
    usage = """
PyarrFS mirror the filesystem tree from some point on, allowing RAR archives to be treated as directories and files within those RAR archives to be read as regular files.

""" + fuse.Fuse.fusage

    server = Pyarr(version="PyarrFS 0.1",
                 usage=usage)

    # TODO: what does multithreaded really mean?
    #       just run many requests in parallel? We have no real dependancies
    #       between calls so we could probably enable this to increase
    #       throughput in environments with lots of simultaneous requests.
    #       Looks like it needs PyarrFile to be locked, what's the point then?
    server.multithreaded = False

    server.parse(values=server, errex=1)

    server.main()

if __name__ == '__main__':
    main()

#
# vim: et ts=4 :
#
<commit_msg>Implement own options to pyarr + fix logging<commit_after>#!/usr/bin/env python
#
# PyarrFS - a RAR reading file system
# Copyright (c) 2010, 2011 Kristian Larsson <kristian@spritelink.net>
#
# This file is license under the X11/MIT license, please see the file COPYING
# distributed with PyarrFS for more details.
#

import os, sys
import errno
import fcntl
import re
import stat
import logging
import logging.handlers

import fuse
import rarfile

__version__         = '0.5.0'
__author__          = 'Kristian Larsson'
__author_email__    = 'kristian@spritelink.net'
__license__         = 'MIT'
__url__             = 'http://labs.spritelink.net/pyarrfs'

# log settings
log_format = "%(levelname)-8s %(message)s"
logger = logging.getLogger()

fuse.fuse_python_api = (0, 2)

fuse.feature_assert('stateful_files', 'has_init')

class Pyarr(fuse.Fuse):
    def __init__(self, *args, **kw):
        logger.info("init!")

        fuse.Fuse.__init__(self, *args, **kw)

        self.debug = False
        self.foreground = False
        self.root = '/'

    def fsinit(self):
        os.chdir(self.root)

    def access(self, path, mode):
        """Returns whether a user has access to performing
        """
        logger.info("access -- " + path)

        # test for write access, PyarrFS is incapable of do writes, so no axx
        if mode == os.W_OK:
            return -errno.EACCES

        # allow the rest
        # FIXME: do more granular access control, based on RAR file?
        return

        # this only works for files not inside a rar archive
        if not os.access("." + path, mode):
            return -errno.EACCES

    def getattr(self, path):
        logger.info("getattr -- " + str(path))
        if re.match('.*\.rar$', path):
            logging.debug("getattr: on rar archive for path " + str(path))
            original_stat = os.lstat("." + path)
            fake_stat = fuse.Stat()
            fake_stat.st_mode = stat.S_IFDIR | 0755
            fake_stat.st_ino = 0
            fake_stat.st_dev = 0
            fake_stat.st_rdev = 0
            fake_stat.st_nlink = 2
            fake_stat.st_uid = original_stat.st_uid
            fake_stat.st_gid = original_stat.st_gid
            fake_stat.st_size = 4096
            fake_stat.st_atime = original_stat.st_atime
            fake_stat.st_mtime = original_stat.st_mtime
            fake_stat.st_ctime = original_stat.st_ctime
            logging.debug("getattr: returning fake_stat for " + str(path))
            return fake_stat

        elif re.match('.*\.rar/.+', path):
            logging.debug("getattr: we need to check inside rar archive for path " + str(path))
            m = re.match('(.*\.rar)/(.+)', path)
            rar_file = m.group(1)
            rar_path = m.group(2)

            original_stat = os.lstat("." + rar_file)
            rf = rarfile.RarFile(rar_file, 'r', None, None, False)
            try:
                rfi = rf.getinfo(rar_path)
            except:
                # FIXME: add DEBUG log entry
                return -errno.ENOENT

            fake_stat = fuse.Stat()
            fake_stat.st_mode = stat.S_IFREG | 0444
            fake_stat.st_ino = 0
            fake_stat.st_dev = 0
            fake_stat.st_rdev = 0
            fake_stat.st_nlink = 1
            fake_stat.st_uid = original_stat.st_uid
            fake_stat.st_gid = original_stat.st_gid
            fake_stat.st_size = rfi.file_size
            fake_stat.st_blocks = (fake_stat.st_size + 511) / 512
            fake_stat.st_blksize = 4096

            import datetime
            (rft_year, rft_month, rft_day, rft_hour, rft_minute, rft_second) = rfi.date_time
            rft_time = datetime.datetime(rft_year, rft_month, rft_day, rft_hour, rft_minute, rft_second)
            fake_stat.st_atime = int(rft_time.strftime('%s'))
            fake_stat.st_mtime = int(rft_time.strftime('%s'))
            fake_stat.st_ctime = int(rft_time.strftime('%s'))
            logger.debug("getattr: returning fake_stat for " + str(rar_path) + " inside rar " + str(rar_file))
            return fake_stat

        logger.debug("getattr: returning normal os.lstat() for path " + str(path))
        return os.lstat("." + path)

    # get a directory listing
    # doesn't need changes in yarrfs compatibility mode
    def readdir(self, path, offset):
        logger.info("readdir -- path: " + str(path) + "  offset: " + str(offset) )
        dirent = [ '.', '..' ]

        if re.match('.*\.rar$', path):
            logger.debug("readdir: on rar archive, using rarfile")
            rf = rarfile.RarFile(path, 'r', None, None, False)
            for e in rf.namelist():
                dirent.append(str(e))
        else:
            logger.debug("readdir: normal dir, using os.listdir()")
            try:
                os.listdir("." + path)
            except:
                return

            for e in os.listdir("." + path):
                dirent.append(e)

        for e in dirent:
            yield fuse.Direntry(e)

    def readlink(self, path):
        logger.info("readlink -- " + path)
        return os.readlink("." + path)

    def statfs(self):
        logger.info("statfs -- " + path)
        return os.statvfs(".")

    class PyarrFile(object):
        def __init__(self, path, flags, *mode):
            # Enabling direct_io disables the kernels page cache.
            # Since the content of our RAR files should be pretty stable, we do
            # NOT enable this, ie we allow the kernel to cache all data.
            # This means we get both the .rar file and the file inside the RAR
            # archive in the block cache which might affect performance
            # negatively. On the other hand, the kernel could purge the .rar
            # file from its block cache and just keep the inside file.
            # It's likely mostly a performance thing. I'm guessing it's good
            # to let the kernel decide on what to cache and what not too, so we
            # set it to False. Some benchmarking should really be done to prove
            # either setting is better than the other
            self.direct_io = False
            # keep_cache means the kernel is allowed to cache content of a file
            # after its been closed an reopened. You only want to disable this
            # if the content of the file might change without the kernel knowing
            # about it, which would be typical for a networked file system.
            # That's not the case with PyarrFS so we enable it.
            self.keep_cache = True
            m = re.match('(.*\.rar)/(.+)', path)
            self.rar_file = m.group(1)
            self.rar_path = m.group(2)
            self.rf = rarfile.RarFile(self.rar_file, 'r', None, None, False)
            self.file = self.rf.open(self.rar_path)
            self.fd = 0

        def read(self, length, offset):
            self.file.seek(offset)
            return self.file.read(length)

        def release(self, flags):
            self.file.close()

    def main(self, *a, **kw):
        self.file_class = self.PyarrFile
        return fuse.Fuse.main(self, *a, **kw)

def main():
    usage = """
PyarrFS mirror the filesystem tree from some point on, allowing RAR archives to be treated as directories and files within those RAR archives to be read as regular files.

""" + fuse.Fuse.fusage

    server = Pyarr(version="PyarrFS 0.1",
                 usage=usage)

    # TODO: what does multithreaded really mean?
    #       just run many requests in parallel? We have no real dependancies
    #       between calls so we could probably enable this to increase
    #       throughput in environments with lots of simultaneous requests.
    #       Looks like it needs PyarrFile to be locked, what's the point then?
    server.multithreaded = False

    server.parser.add_option('-r', '--root', dest='root', metavar="PATH", default=server.root, help="mirror filesystem from under PATH [default: %default]")
    server.parser.add_option('-D', '--pydebug', action='store_true', dest='pydebug', default=False, help="enable debug for just PyarrFS (not FUSE) (implies -f)")
    server.parse(values=server, errex=1)

    # always log to syslog
    log_syslog = logging.handlers.SysLogHandler(address = '/dev/log')
    log_syslog.setFormatter(logging.Formatter(log_format))
    log_syslog.setLevel(logging.WARNING)
    logger.addHandler(log_syslog)

    opts = {}
    # FIXME: why doesn't optdict work?
    for o in server.parser.fuse_args.optlist:
        opts[o] = True

    if server.pydebug:
        server.parser.fuse_args.modifiers['foreground'] = True

    if server.parser.fuse_args.modifiers['foreground'] or opts.has_key('debug') or server.pydebug:
        log_stream = logging.StreamHandler()
        log_stream.setFormatter(logging.Formatter("%(asctime)s: " + log_format))
        log_stream.setLevel(logging.DEBUG)
        if opts.has_key('debug') or server.pydebug:
            # FIXME: this isn't working, why? we only get WARNING messages
            log_stream.setLevel(logging.DEBUG)
        logger.addHandler(log_stream)

    # and go!
    server.main()

if __name__ == '__main__':
    main()

#
# vim: et ts=4 :
#
<|endoftext|>


<commit_before>#!/usr/local/bin/env python

#=============================================================================================
# MODULE DOCSTRING
#=============================================================================================

"""
YANK command-line interface (cli)

"""

#=============================================================================================
# MODULE IMPORTS
#=============================================================================================

import sys
import docopt

#=============================================================================================
# COMMAND-LINE INTERFACE
#=============================================================================================

usage = """
YANK

Usage:
  yank [-h | --help] [-c | --cite]
  yank selftest [-v | --verbose]
  yank platforms
  yank prepare binding amber --setupdir=DIRECTORY --ligand=DSLSTRING (-s=STORE | --store=STORE) [-i=NITER | --iterations=NITER] [--equilibrate=NEQUIL] [--restraints <restraint_type>] [--randomize-ligand] [--nbmethod=METHOD] [--gbsa=GBSA] [--constraints=CONSTRAINTS] [--temperature=TEMPERATURE] [--pressure=PRESSURE] [--minimize] [-v | --verbose]
  yank run (-s=STORE | --store=STORE) [-m | --mpi] [-i=NITER | --iterations=NITER] [--platform=PLATFORM] [--precision=PRECISION] [--phase=PHASE] [-o | --online-analysis] [-v | --verbose]
  yank status (-s=STORE | --store=STORE) [-v | --verbose]
  yank analyze (-s STORE | --store=STORE) [-v | --verbose]
  yank cleanup (-s=STORE | --store=STORE) [-v | --verbose]

Commands:
  selftest                      Run selftests.
  platforms                     List available OpenMM platforms.
  prepare binding amber         Set up binding free energy calculation using AMBER.
  run                           Run the calculation that has been set up
  status                        Get the current status
  analyze                       Analyze data
  cleanup                       Clean up (delete) run files.

General options:
  -h, --help                    Print command line help
  -c, --cite                    Print relevant citations
  -i=NITER, --iterations=NITER  Number of iterations to run
  --randomize-ligand            Randomize initial ligand positions if specified
  -v, --verbose                 Print verbose output
  -s=STORE, --store=STORE       Storage directory for NetCDF data files.
  -o, --online-analysis         Enable on-the-fly analysis
  --platform=PLATFORM           OpenMM Platform to use (Reference, CPU, OpenCL, CUDA)
  --precision=PRECISION         OpenMM Platform precision model to use (for CUDA or OpenCL only, one of {mixed, double, single})
  --equilibrate=NEQUIL          Number of equilibration iterations
  --minimize                    Minimize configurations before running simulation.
  -m, --mpi                     Use MPI to parallelize the calculation

Simulation options:
  --restraints=TYPE             Restraint type to add between protein and ligand in implicit solvent ('harmonic', 'flat-bottom') [default: 'flat-bottom']
  --gbsa=GBSA                   OpenMM GBSA model (HCT, OBC1, OBC2, GBn, GBn2)
  --nbmethod=METHOD             OpenMM nonbonded method (NoCutoff, CutoffPeriodic, PME, Ewald)
  --constraints=CONSTRAINTS     OpenMM constraints (None, HBonds, AllBonds, HAngles) [default: HBonds]
  --phase=PHASE                 Resume only specified phase of calculation ('solvent', 'complex')
  --temperature=TEMPERATURE     Temperature for simulation (in K, or simtk.unit readable string) [default: '298*kelvin']
  --pressure=PRESSURE           Pressure for simulation (in atm, or simtk.unit readable string) [default: '1*atmospheres']

Amber options:
  --setupdir=DIRECTORY          Setup directory to look for AMBER {receptor|ligand|complex}.{prmtop|inpcrd} files.
  --ligand=DSLSTRING            Specification of the ligand atoms according to MDTraj DSL syntax [default: 'resname MOL']

"""

# TODO: Add optional arguments that we can use to override sys.argv for testing purposes.
def main(argv=None):
    # Parse command-line arguments.
    from docopt import docopt
    import version
    args = docopt(usage, version=version.version, argv=argv)

    dispatched = False # Flag set to True if we have correctly dispatched a command.
    from . import commands # TODO: This would be clearer if we could do 'from yank import commands', but can't figure out how

    # Handle simple arguments.
    if args['--help']:
        print usage
        dispatched = True
    if args['--cite']:
        dispatched = commands.cite.dispatch(args)

    # Handle commands.
    command_list = ['selftest', 'platforms', 'prepare', 'run', 'status', 'analyze', 'cleanup'] # TODO: Build this list automagically by introspection of commands submodule.
    for command in command_list:
        if args[command]:
            dispatched = getattr(commands, command).dispatch(args)

    # If unsuccessful, print usage and exit with an error.
    if not dispatched:
        print usage
        return True

    # Indicate success.
    return False
<commit_msg>Removed quotes that were being interpreted as literals in docstring defaults.<commit_after>#!/usr/local/bin/env python

#=============================================================================================
# MODULE DOCSTRING
#=============================================================================================

"""
YANK command-line interface (cli)

"""

#=============================================================================================
# MODULE IMPORTS
#=============================================================================================

import sys
import docopt

#=============================================================================================
# COMMAND-LINE INTERFACE
#=============================================================================================

usage = """
YANK

Usage:
  yank [-h | --help] [-c | --cite]
  yank selftest [-v | --verbose]
  yank platforms
  yank prepare binding amber --setupdir=DIRECTORY --ligand=DSLSTRING (-s=STORE | --store=STORE) [-i=NITER | --iterations=NITER] [--equilibrate=NEQUIL] [--restraints <restraint_type>] [--randomize-ligand] [--nbmethod=METHOD] [--gbsa=GBSA] [--constraints=CONSTRAINTS] [--temperature=TEMPERATURE] [--pressure=PRESSURE] [--minimize] [-v | --verbose]
  yank run (-s=STORE | --store=STORE) [-m | --mpi] [-i=NITER | --iterations=NITER] [--platform=PLATFORM] [--precision=PRECISION] [--phase=PHASE] [-o | --online-analysis] [-v | --verbose]
  yank status (-s=STORE | --store=STORE) [-v | --verbose]
  yank analyze (-s STORE | --store=STORE) [-v | --verbose]
  yank cleanup (-s=STORE | --store=STORE) [-v | --verbose]

Commands:
  selftest                      Run selftests.
  platforms                     List available OpenMM platforms.
  prepare binding amber         Set up binding free energy calculation using AMBER.
  run                           Run the calculation that has been set up
  status                        Get the current status
  analyze                       Analyze data
  cleanup                       Clean up (delete) run files.

General options:
  -h, --help                    Print command line help
  -c, --cite                    Print relevant citations
  -i=NITER, --iterations=NITER  Number of iterations to run
  --randomize-ligand            Randomize initial ligand positions if specified
  -v, --verbose                 Print verbose output
  -s=STORE, --store=STORE       Storage directory for NetCDF data files.
  -o, --online-analysis         Enable on-the-fly analysis
  --platform=PLATFORM           OpenMM Platform to use (Reference, CPU, OpenCL, CUDA)
  --precision=PRECISION         OpenMM Platform precision model to use (for CUDA or OpenCL only, one of {mixed, double, single})
  --equilibrate=NEQUIL          Number of equilibration iterations
  --minimize                    Minimize configurations before running simulation.
  -m, --mpi                     Use MPI to parallelize the calculation

Simulation options:
  --restraints=TYPE             Restraint type to add between protein and ligand in implicit solvent (harmonic, flat-bottom) [default: flat-bottom]
  --gbsa=GBSA                   OpenMM GBSA model (HCT, OBC1, OBC2, GBn, GBn2)
  --nbmethod=METHOD             OpenMM nonbonded method (NoCutoff, CutoffPeriodic, PME, Ewald)
  --constraints=CONSTRAINTS     OpenMM constraints (None, HBonds, AllBonds, HAngles) [default: HBonds]
  --phase=PHASE                 Resume only specified phase of calculation (solvent, complex)
  --temperature=TEMPERATURE     Temperature for simulation (in K, or simtk.unit readable string) [default: 298*kelvin]
  --pressure=PRESSURE           Pressure for simulation (in atm, or simtk.unit readable string) [default: 1*atmospheres]

Amber options:
  --setupdir=DIRECTORY          Setup directory to look for AMBER {receptor|ligand|complex}.{prmtop|inpcrd} files.
  --ligand=DSLSTRING            Specification of the ligand atoms according to MDTraj DSL syntax [default: resname MOL]

"""

# TODO: Add optional arguments that we can use to override sys.argv for testing purposes.
def main(argv=None):
    # Parse command-line arguments.
    from docopt import docopt
    import version
    args = docopt(usage, version=version.version, argv=argv)

    dispatched = False # Flag set to True if we have correctly dispatched a command.
    from . import commands # TODO: This would be clearer if we could do 'from yank import commands', but can't figure out how

    # Handle simple arguments.
    if args['--help']:
        print usage
        dispatched = True
    if args['--cite']:
        dispatched = commands.cite.dispatch(args)

    # Handle commands.
    command_list = ['selftest', 'platforms', 'prepare', 'run', 'status', 'analyze', 'cleanup'] # TODO: Build this list automagically by introspection of commands submodule.
    for command in command_list:
        if args[command]:
            dispatched = getattr(commands, command).dispatch(args)

    # If unsuccessful, print usage and exit with an error.
    if not dispatched:
        print usage
        return True

    # Indicate success.
    return False
<|endoftext|>