<commit_before># Copyright 2010 Harry Bock <hbock@ele.uri.edu>
# See LICENSE for licensing information.

# Active test scheduler and circuit manager for TorBEL.

# We come from the __future__.
from __future__ import with_statement

import time
import random
import resource
import threading
from copy import copy
from collections import deque

from torbel import config
from torbel.controller import reactor
from torbel.logger import *
log = get_logger("torbel")

class TestScheduler:
    """ Abstract base class for all test schedulers. """
    controller = None
    name = "Abstract"
    def __init__(self, controller, max_pending_factor = 0.5):
        self.controller = controller
        self.terminated = False
        ## Circuit dictionaries.
        # Established circuits under test.
        self.circuits = {}
        # Circuits in the process of being built.
        self.pending_circuits = {}
        self.pending_circuit_lock = threading.RLock()
        self.pending_circuit_cond = threading.Condition(self.pending_circuit_lock)
        # Circuit failure metrics and bookkeeping.
        self.retry_routers = set()
        self.circuit_fail_count = 0
        self.circuit_retry_success_count = 0

        # Base max running circuits on the total number of file descriptors
        # we can have open (hard limit returned by getrlimit) and the maximum
        # number of file descriptors per circuit, adjusting for possible pending
        # circuits, TorCtl connection, stdin/out, and other files.
        max_files = resource.getrlimit(resource.RLIMIT_NOFILE)[1]

        circuit_limit = max_files / len(controller.test_ports)
        self.max_running_circuits = min(config.max_built_circuits, circuit_limit)
        self.max_pending_circuits = int(self.max_running_circuits * max_pending_factor)

        self.init()

    def init(self):
        """ Initialization routine for a custom scheduler.  Don't override
        __init__. """
        pass

    def next(self):
        """ Return a set of routers to be tested. May block until enough routers
        are deemed ready by the scheduler. """

        log.verbose2("Request more circuits. (%d pending, %d running).",
                     len(self.pending_circuits),
                     len(self.circuits))

        with self.pending_circuit_cond:
            # Block until we have less than ten circuits built or
            # waiting to be built.
            while len(self.pending_circuits) >= self.max_pending_circuits or \
                    len(self.circuits) >= self.max_running_circuits:
                self.pending_circuit_cond.wait(3.0)

                # We're done here.
                if self.terminated:
                    return []

        # Get what the child scheduler class wants to test.
        return list(self.fetch_next_tests())

    def new_consensus(self, cache):
        """ Called when a NEWCONSENSUS event occurs. cache is a dictionary
        of the entire consensus, keyed by router ID hash. """
        pass

    def new_descriptor(self, router):
        """ Called when a NEWDESC event occurs. router is the new descriptor
        RouterRecord object. """
        pass

    def fetch_next_tests(self):
        """ Scheduler-specific interface that returns a list of
        routers to retry and test.  TestScheduler.next() takes these
        results and performs rate-limiting, so it may not always test
        every router returned by this method.

        Return value is of the form (new_tests, retry_tests), where
        retry_tests is a list of routers that recently failed
        unexpectedly and should be tested again.
        """
        raise ValueError("You must implement fetch_next_tests()!")

    def retry_soon(self, router):
        """ Inidcate to the scheduler that the controller was not able
        to complete a stream test or circuit to router, but the result
        may indicate a temporary failure.  The scheduler should retry
        all tests to router as soon as possible."""
        self.retry_routers.add(router)

    def retry_later(self, router):
        """ Indicate to the scheduler that the controller was not able to
        complete a stream test due to a possibly temporary failure, and that
        it should retry at a longer interval than retry_soon. """
        # Default behavior is to use the retry_soon behavior unless
        # implemented otherwise.
        self.retry_soon(self, router)

    def stop(self):
        """ Stop the scheduler. """
        with self.pending_circuit_cond:
            self.pending_circuit_cond.notify()
            self.terminated = True

    def circ_pending(self, circ_id, router):
        with self.pending_circuit_lock:
            self.pending_circuits[circ_id] = router

    def circ_built(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.pending_circuits:
                router = self.pending_circuits[circ_id]
                del self.pending_circuits[circ_id]
                # Notify scheduler thread that we have
                # completed building a circuit and we could
                # need to pre-build more.
                self.pending_circuit_cond.notify()
            else:
                return

            # If we succeeded in building this router on retry,
            # reset its failure count to give it a clean slate.
            if router.retry:
                self.circuit_retry_success_count += 1
                router.retry = False
                log.verbose1("Retry for %s successful after %d failures (%d/%d %.2f%%)!",
                             router.nickname, router.circuit_failures,
                             self.circuit_retry_success_count,
                             self.circuit_fail_count + self.circuit_retry_success_count,
                             100 * float(self.circuit_retry_success_count) / \
                                 (self.circuit_fail_count + self.circuit_retry_success_count))
            router.circuit_failures = 0

            log.verbose1("Successfully built circuit %d for %s.",
                         circ_id, router.nickname)
            self.circuits[circ_id] = router
            self.controller.connect_test(router)

    def circ_closed(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.circuits:
                router = self.circuits[circ_id]
                # FINISHED = "The circuit has expired for being dirty or old."
                # (tor-spec.txt 5.4, "Tearing down circuits"). Treat this as
                # an error condition if we have not yet completed the test.
                if event.reason == "FINISHED":
                    if router.current_test and router.current_test.circ_id == circ_id:
                        self.circ_failed(event)
                        return

                log.verbose2("Closed circuit %d (%s).", circ_id,
                             self.circuits[circ_id].nickname)
                del self.circuits[circ_id]

            elif circ_id in self.pending_circuits:
                # Pending circuit closed before being built (can this happen?)
                log.debug("Pending circuit closed (%d)?", circ_id)
                router = self.pending_circuits[circ_id]
                del self.pending_circuits[circ_id]
                # Not technically an explicit failure, but the circuit is already
                # closed, so don't bother doing it again.
                self.controller.test_cleanup(router, circ_failed = True)
                self.pending_circuit_cond.notify()

    def circ_failed(self, event):
        circ_id = event.circ_id
        retry = False

        # We sometimes get a CIRC FAILED event after calling close_circuit,
        # so we should probably ignore these messages to make sure we don't
        # go in circles retrying the circuit build.
        if event.reason == "REQUESTED":
            return

        def cleanup_and_notify(router, retry = False):
            # Cleanup test results and notify the circuit thread.
            self.controller.test_cleanup(router, circ_failed = True)
            self.pending_circuit_cond.notify()

            if retry:
                # Append this router to our failure list, and let the scheduler
                # decide if testing should be re-tried.
                self.retry_soon(router)

        with self.pending_circuit_cond:
            if circ_id in self.circuits:
                log.debug("Established test circuit %d failed: %s", circ_id, event.reason)
                router = self.circuits[circ_id]
                router.circuit_failures += 1
                del self.circuits[circ_id]
                cleanup_and_notify(router, retry = True)

            elif circ_id in self.pending_circuits:
                # Circuit failed without being built.
                # Delete from pending_circuits and notify
                # CircuitBuilder that the pending_circuits dict
                # has changed.
                self.circuit_fail_count += 1
                router = self.pending_circuits[circ_id]
                if len(event.path) >= 1:
                    router.circuit_failures += 1
                    log.verbose1("Circ to %s failed (r:%s remr:%s). %d failures",
                                 router.nickname, event.reason, event.remote_reason,
                                 router.circuit_failures)
                else:
                    # We failed to extend to the entry guard.  This more than
                    # likely means we have a bad guard.  Remove this guard.
                    log.debug("Bad guard: circuit to %s failed (reason %s).",
                              router.nickname, event.reason)
                    if router.guard:
                        self.controller.remove_guard(router.guard)

                # Remove from pending circuits.
                del self.pending_circuits[circ_id]
                cleanup_and_notify(router, retry = True)

    def retry_candidates(self):
        """ Return a list of circuits that have recently failed and are candidates
        for retrying the test. """
        control = self.controller
        with self.pending_circuit_lock:
            max_retry = min(self.max_pending_circuits / 2,
                            len(self.retry_routers))

            # Look through the circuit failure queue and determine
            # which should be retried and which should wait until the next
            # run-through of testing.
            retry_set = set()
            retry_not_ready = []
            while len(retry_set) < max_retry and len(self.retry_routers) > 0:
                router = self.retry_routers.pop()
                if router.circuit_failures >= 3:
                    log.debug("%s: Too many failures.", router.nickname)
                elif router.is_ready():
                    retry_set.add(router)
                    router.retry = True
                # If a router is not ready to be retried (currently under test),
                # put it back on the retry list.
                else:
                    retry_not_ready.append(router)

            for router in retry_not_ready:
                self.retry_routers.add(router)

        return retry_set

    def print_stats(self):
        pass

class HammerScheduler(TestScheduler):
    """ The Hammer test scheduler hath no mercy. This scheduler will
    continually test every router it knows about.  Very good for
    stress-testing torbel and the Tor network itself, bad in practice."""
    name = "HAMMER"

    def fetch_next_tests(self):
        control = self.controller

        retry = self.retry_candidates()

        # Filter testable routers and sort them by the time their last test
        # started.
        with self.controller.consensus_cache_lock:
            ready = sorted(filter(lambda router: router.is_ready(),
                                  self.controller.router_cache.values()),
                           key = lambda r: r.last_test.start_time)

        # Only return up to self.max_pending_circuits routers to test.
        available_pending = self.max_pending_circuits - len(self.pending_circuits)
        return set(ready[:(available_pending - len(retry_list))]) | retry

class ConservativeScheduler(TestScheduler):
    """ Implement meeee! """
    name = "Conservative"
    def init(self):
        self.router_list = deque()
        self.n = 0
        self.new_router_lock = threading.RLock()
        self.new_router_cond = threading.Condition(self.new_router_lock)

    def new_consensus(self, cache):
        # Add NEWCONSENSUS data to our test cache.
        # TODO: This will cause lots of duplicate tests if we get a NEWCONSENSUS
        # soon after starting TorBEL, but it really shouldn't be a problem with
        # how fast we test.
        cache_values = copy(cache.values())
        random.shuffle(cache_values)
        with self.new_router_cond:
            for router in cache_values:
                self.router_list.append(router)

            self.new_router_cond.notify()

    def new_descriptor(self, router):
        # Append new descriptor to our list and notify a (possibly)
        # sleeping fetch_next_tests.
        with self.new_router_cond:
            self.router_list.append(router)
            self.new_router_cond.notify()

    def retry_soon(self, router):
        # Call parent class and notify possibly sleeping fetch_next_tests
        # call to get things rolling again.
        TestScheduler.retry_soon(self, router)
        with self.new_router_cond:
            self.new_router_cond.notify()

    def fetch_next_tests(self):
        testable = 0

        with self.new_router_cond:
            # Only return up to self.max_pending_circuits routers to test.
            while not self.terminated and testable == 0:
                # Boom, bail.
                if self.terminated:
                    return
                # Start with our retry candidates.
                test_set = self.retry_candidates()
                # Grab the number of available test circuits...
                with self.pending_circuit_lock:
                    available = self.max_pending_circuits - len(self.pending_circuits) - len(test_set)
                # If we have available circuits, grab as many routers to test as
                # possible.
                if available > 0:
                    for i in range(min(len(self.router_list), available)):
                        candidate = self.router_list.popleft()
                        test_set.add(candidate)
                # If we don't have any routers to test, sleep until we are
                # notified of new routers.
                testable = len(test_set)
                if testable == 0:
                    self.new_router_cond.wait()

        # General debugging stats for our test schedule progress.
        with self.controller.consensus_cache_lock:
            router_count = len(self.controller.router_cache)

        if self.n % 30 == 0:
            log.debug("Going to test %d routers. %.1f%% started (%d circs)!",
                      len(test_set),
                      100 * (router_count - len(self.router_list)) / float(router_count),
                      len(self.circuits))
        self.n += 1

        return sorted(list(test_set), key = lambda r: r.last_test.end_time)

    def retry_later(self, router):
        """ Retry router in five minutes. """
        reactor.callLater(5 * 60, lambda: retry_soon(router))

    def print_stats(self):
        TestScheduler.print_stats(self)
        with self.pending_circuit_cond:
            self.pending_circuit_cond.notify()
        #log.debug("new_router_lock.locked(): %s", self.new_router_lock.locked())
        log.debug("%d pending new tests, %d pending retries.",
                  len(self.router_list), len(self.retry_routers))
        log.debug("%d pending circuits, %d running circuits.",
                  len(self.pending_circuits),
                  len(self.circuits))

    def stop(self):
        # Notify new_router_cond first.
        TestScheduler.stop(self)
        with self.new_router_cond:
            self.new_router_cond.notify()
<commit_msg>Fix retry_later implementation in conservative scheduler.<commit_after># Copyright 2010 Harry Bock <hbock@ele.uri.edu>
# See LICENSE for licensing information.

# Active test scheduler and circuit manager for TorBEL.

# We come from the __future__.
from __future__ import with_statement

import time
import random
import resource
import threading
from copy import copy
from collections import deque

from torbel import config
from torbel.controller import reactor
from torbel.logger import *
log = get_logger("torbel")

class TestScheduler:
    """ Abstract base class for all test schedulers. """
    controller = None
    name = "Abstract"
    def __init__(self, controller, max_pending_factor = 0.5):
        self.controller = controller
        self.terminated = False
        ## Circuit dictionaries.
        # Established circuits under test.
        self.circuits = {}
        # Circuits in the process of being built.
        self.pending_circuits = {}
        self.pending_circuit_lock = threading.RLock()
        self.pending_circuit_cond = threading.Condition(self.pending_circuit_lock)
        # Circuit failure metrics and bookkeeping.
        self.retry_routers = set()
        self.circuit_fail_count = 0
        self.circuit_retry_success_count = 0

        # Base max running circuits on the total number of file descriptors
        # we can have open (hard limit returned by getrlimit) and the maximum
        # number of file descriptors per circuit, adjusting for possible pending
        # circuits, TorCtl connection, stdin/out, and other files.
        max_files = resource.getrlimit(resource.RLIMIT_NOFILE)[1]

        circuit_limit = max_files / len(controller.test_ports)
        self.max_running_circuits = min(config.max_built_circuits, circuit_limit)
        self.max_pending_circuits = int(self.max_running_circuits * max_pending_factor)

        self.init()

    def init(self):
        """ Initialization routine for a custom scheduler.  Don't override
        __init__. """
        pass

    def next(self):
        """ Return a set of routers to be tested. May block until enough routers
        are deemed ready by the scheduler. """

        log.verbose2("Request more circuits. (%d pending, %d running).",
                     len(self.pending_circuits),
                     len(self.circuits))

        with self.pending_circuit_cond:
            # Block until we have less than ten circuits built or
            # waiting to be built.
            while len(self.pending_circuits) >= self.max_pending_circuits or \
                    len(self.circuits) >= self.max_running_circuits:
                self.pending_circuit_cond.wait(3.0)

                # We're done here.
                if self.terminated:
                    return []

        # Get what the child scheduler class wants to test.
        return list(self.fetch_next_tests())

    def new_consensus(self, cache):
        """ Called when a NEWCONSENSUS event occurs. cache is a dictionary
        of the entire consensus, keyed by router ID hash. """
        pass

    def new_descriptor(self, router):
        """ Called when a NEWDESC event occurs. router is the new descriptor
        RouterRecord object. """
        pass

    def fetch_next_tests(self):
        """ Scheduler-specific interface that returns a list of
        routers to retry and test.  TestScheduler.next() takes these
        results and performs rate-limiting, so it may not always test
        every router returned by this method.

        Return value is of the form (new_tests, retry_tests), where
        retry_tests is a list of routers that recently failed
        unexpectedly and should be tested again.
        """
        raise ValueError("You must implement fetch_next_tests()!")

    def retry_soon(self, router):
        """ Inidcate to the scheduler that the controller was not able
        to complete a stream test or circuit to router, but the result
        may indicate a temporary failure.  The scheduler should retry
        all tests to router as soon as possible."""
        self.retry_routers.add(router)

    def retry_later(self, router):
        """ Indicate to the scheduler that the controller was not able to
        complete a stream test due to a possibly temporary failure, and that
        it should retry at a longer interval than retry_soon. """
        # Default behavior is to use the retry_soon behavior unless
        # implemented otherwise.
        self.retry_soon(self, router)

    def stop(self):
        """ Stop the scheduler. """
        with self.pending_circuit_cond:
            self.pending_circuit_cond.notify()
            self.terminated = True

    def circ_pending(self, circ_id, router):
        with self.pending_circuit_lock:
            self.pending_circuits[circ_id] = router

    def circ_built(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.pending_circuits:
                router = self.pending_circuits[circ_id]
                del self.pending_circuits[circ_id]
                # Notify scheduler thread that we have
                # completed building a circuit and we could
                # need to pre-build more.
                self.pending_circuit_cond.notify()
            else:
                return

            # If we succeeded in building this router on retry,
            # reset its failure count to give it a clean slate.
            if router.retry:
                self.circuit_retry_success_count += 1
                router.retry = False
                log.verbose1("Retry for %s successful after %d failures (%d/%d %.2f%%)!",
                             router.nickname, router.circuit_failures,
                             self.circuit_retry_success_count,
                             self.circuit_fail_count + self.circuit_retry_success_count,
                             100 * float(self.circuit_retry_success_count) / \
                                 (self.circuit_fail_count + self.circuit_retry_success_count))
            router.circuit_failures = 0

            log.verbose1("Successfully built circuit %d for %s.",
                         circ_id, router.nickname)
            self.circuits[circ_id] = router
            self.controller.connect_test(router)

    def circ_closed(self, event):
        circ_id = event.circ_id
        with self.pending_circuit_cond:
            if circ_id in self.circuits:
                router = self.circuits[circ_id]
                # FINISHED = "The circuit has expired for being dirty or old."
                # (tor-spec.txt 5.4, "Tearing down circuits"). Treat this as
                # an error condition if we have not yet completed the test.
                if event.reason == "FINISHED":
                    if router.current_test and router.current_test.circ_id == circ_id:
                        self.circ_failed(event)
                        return

                log.verbose2("Closed circuit %d (%s).", circ_id,
                             self.circuits[circ_id].nickname)
                del self.circuits[circ_id]

            elif circ_id in self.pending_circuits:
                # Pending circuit closed before being built (can this happen?)
                log.debug("Pending circuit closed (%d)?", circ_id)
                router = self.pending_circuits[circ_id]
                del self.pending_circuits[circ_id]
                # Not technically an explicit failure, but the circuit is already
                # closed, so don't bother doing it again.
                self.controller.test_cleanup(router, circ_failed = True)
                self.pending_circuit_cond.notify()

    def circ_failed(self, event):
        circ_id = event.circ_id
        retry = False

        # We sometimes get a CIRC FAILED event after calling close_circuit,
        # so we should probably ignore these messages to make sure we don't
        # go in circles retrying the circuit build.
        if event.reason == "REQUESTED":
            return

        def cleanup_and_notify(router, retry = False):
            # Cleanup test results and notify the circuit thread.
            self.controller.test_cleanup(router, circ_failed = True)
            self.pending_circuit_cond.notify()

            if retry:
                # Append this router to our failure list, and let the scheduler
                # decide if testing should be re-tried.
                self.retry_soon(router)

        with self.pending_circuit_cond:
            if circ_id in self.circuits:
                log.debug("Established test circuit %d failed: %s", circ_id, event.reason)
                router = self.circuits[circ_id]
                router.circuit_failures += 1
                del self.circuits[circ_id]
                cleanup_and_notify(router, retry = True)

            elif circ_id in self.pending_circuits:
                # Circuit failed without being built.
                # Delete from pending_circuits and notify
                # CircuitBuilder that the pending_circuits dict
                # has changed.
                self.circuit_fail_count += 1
                router = self.pending_circuits[circ_id]
                if len(event.path) >= 1:
                    router.circuit_failures += 1
                    log.verbose1("Circ to %s failed (r:%s remr:%s). %d failures",
                                 router.nickname, event.reason, event.remote_reason,
                                 router.circuit_failures)
                else:
                    # We failed to extend to the entry guard.  This more than
                    # likely means we have a bad guard.  Remove this guard.
                    log.debug("Bad guard: circuit to %s failed (reason %s).",
                              router.nickname, event.reason)
                    if router.guard:
                        self.controller.remove_guard(router.guard)

                # Remove from pending circuits.
                del self.pending_circuits[circ_id]
                cleanup_and_notify(router, retry = True)

    def retry_candidates(self):
        """ Return a list of circuits that have recently failed and are candidates
        for retrying the test. """
        control = self.controller
        with self.pending_circuit_lock:
            max_retry = min(self.max_pending_circuits / 2,
                            len(self.retry_routers))

            # Look through the circuit failure queue and determine
            # which should be retried and which should wait until the next
            # run-through of testing.
            retry_set = set()
            retry_not_ready = []
            while len(retry_set) < max_retry and len(self.retry_routers) > 0:
                router = self.retry_routers.pop()
                if router.circuit_failures >= 3:
                    log.debug("%s: Too many failures.", router.nickname)
                elif router.is_ready():
                    retry_set.add(router)
                    router.retry = True
                # If a router is not ready to be retried (currently under test),
                # put it back on the retry list.
                else:
                    retry_not_ready.append(router)

            for router in retry_not_ready:
                self.retry_routers.add(router)

        return retry_set

    def print_stats(self):
        pass

class HammerScheduler(TestScheduler):
    """ The Hammer test scheduler hath no mercy. This scheduler will
    continually test every router it knows about.  Very good for
    stress-testing torbel and the Tor network itself, bad in practice."""
    name = "HAMMER"

    def fetch_next_tests(self):
        control = self.controller

        retry = self.retry_candidates()

        # Filter testable routers and sort them by the time their last test
        # started.
        with self.controller.consensus_cache_lock:
            ready = sorted(filter(lambda router: router.is_ready(),
                                  self.controller.router_cache.values()),
                           key = lambda r: r.last_test.start_time)

        # Only return up to self.max_pending_circuits routers to test.
        available_pending = self.max_pending_circuits - len(self.pending_circuits)
        return set(ready[:(available_pending - len(retry_list))]) | retry

class ConservativeScheduler(TestScheduler):
    """ Implement meeee! """
    name = "Conservative"
    def init(self):
        self.router_list = deque()
        self.n = 0
        self.new_router_lock = threading.RLock()
        self.new_router_cond = threading.Condition(self.new_router_lock)

    def new_consensus(self, cache):
        # Add NEWCONSENSUS data to our test cache.
        # TODO: This will cause lots of duplicate tests if we get a NEWCONSENSUS
        # soon after starting TorBEL, but it really shouldn't be a problem with
        # how fast we test.
        cache_values = copy(cache.values())
        random.shuffle(cache_values)
        with self.new_router_cond:
            for router in cache_values:
                self.router_list.append(router)

            self.new_router_cond.notify()

    def new_descriptor(self, router):
        # Append new descriptor to our list and notify a (possibly)
        # sleeping fetch_next_tests.
        with self.new_router_cond:
            self.router_list.append(router)
            self.new_router_cond.notify()

    def retry_soon(self, router):
        # Call parent class and notify possibly sleeping fetch_next_tests
        # call to get things rolling again.
        TestScheduler.retry_soon(self, router)
        with self.new_router_cond:
            self.new_router_cond.notify()

    def fetch_next_tests(self):
        testable = 0

        with self.new_router_cond:
            # Only return up to self.max_pending_circuits routers to test.
            while not self.terminated and testable == 0:
                # Boom, bail.
                if self.terminated:
                    return
                # Start with our retry candidates.
                test_set = self.retry_candidates()
                # Grab the number of available test circuits...
                with self.pending_circuit_lock:
                    available = self.max_pending_circuits - len(self.pending_circuits) - len(test_set)
                # If we have available circuits, grab as many routers to test as
                # possible.
                if available > 0:
                    for i in range(min(len(self.router_list), available)):
                        candidate = self.router_list.popleft()
                        test_set.add(candidate)
                # If we don't have any routers to test, sleep until we are
                # notified of new routers.
                testable = len(test_set)
                if testable == 0:
                    self.new_router_cond.wait()

        # General debugging stats for our test schedule progress.
        with self.controller.consensus_cache_lock:
            router_count = len(self.controller.router_cache)

        if self.n % 30 == 0:
            log.debug("Going to test %d routers. %.1f%% started (%d circs)!",
                      len(test_set),
                      100 * (router_count - len(self.router_list)) / float(router_count),
                      len(self.circuits))
        self.n += 1

        return sorted(list(test_set), key = lambda r: r.last_test.end_time)

    def retry_later(self, router):
        """ Retry router in five minutes. """
        reactor.callLater(5 * 60, lambda: self.retry_soon(router))

    def print_stats(self):
        TestScheduler.print_stats(self)
        with self.pending_circuit_cond:
            self.pending_circuit_cond.notify()
        #log.debug("new_router_lock.locked(): %s", self.new_router_lock.locked())
        log.debug("%d pending new tests, %d pending retries.",
                  len(self.router_list), len(self.retry_routers))
        log.debug("%d pending circuits, %d running circuits.",
                  len(self.pending_circuits),
                  len(self.circuits))

    def stop(self):
        # Notify new_router_cond first.
        TestScheduler.stop(self)
        with self.new_router_cond:
            self.new_router_cond.notify()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Georgia Tech Spring 2014
#
# scheduler.py: script to manage experiments and data by finding
# appropriate clients, copying files into those directories, and
# potentially modifying the scheduling frequency information
#
# Note: you *MUST NOT* start more than 1 instance of this program at
# the same time. Doing so could cause race conditions in our scheduler
# and could result in your experiments not being run at the expected
# frequency

import argparse
import json
import os
import os.path

import centinel
import config
from centinel.models import Client

def parse_args():
    parser = argpars.ArgumentParser()
    country_help = ('Two letter country code of the country to run the'
                    'experiment in')
    parser.add_argument('--country', '-c', help=country_help, required=True)
    client_help = ("Number of clients in the country to run the measurement "
                   "on. If this is not specified, the experiment will be "
                   "scheduled on all clients in the country")
    parser.add_argument('--num-clients', '-n', help=client_help, default=None)
    data_help = ("Data file for the clients to use. Note that this must be "
                 "paired with an experiment file that has the same name "
                 "(specified at the top of the class in the experiment file)")
    parser.add_argument('--data', '-d', help=data_help, default=None)
    exp_help = ("Experiment file for the clients to use. This should be a "
                "Python file following the format as other experiments "
                "(see the example template or ping.py for how to do this)")
    parser.add_argument('--experiment', '-e', help=exp_help, default=None)
    freq_help = ("How often the experiment should be run in hours, i.e. "
                 "enter 4 to run the experiment every 4 hours. If no "
                 "frequency information is specified, the measurement will "
                 "run every hour")
    parser.add_argument('--frequency', '-f', help=freq_help, default=1)
    remove_help = ("Remove the experiment or data file with the specified "
                   "name from the target clients. Note that this option will "
                   "be applied to both data and experiments if both the -d "
                   "and -e options are specified")
    parser.add_argument('--remove', '-r', help=remove_help, default=False,
                        action='store_true')
    args = parser.parse_args()

    if args.frequency != 1 and args.experiment is None:
        parser.error("Specifying the frequency is only a valid option if you "
                     "are specifying an experiment to run. You must specify "
                     "a new experiment to schedule as well with the -e option")
    return args

def find_clients(country, num_clients):
    """Find num_clients active clients in the target country

    Params:
    country- two letter country code of target country
    num_clients- number of clients to get. If this is None, then we
        get all the active clients for that country

    Note: we define active here as having seen the client in the past
    month

    """
    clients = []
    potential_clients = Client.query.filter_by(country=country).all()
    month_diff = time
    for client in potential_clients:
        if client.last_seen < datetime.now() - month_diff:
            continue
        if num_clients is not None and len(clients) >= num_clients:
            continue
        clients.append(client.username)
    return clients

def copy_data(clients, data):
    """Copy the given data file so that it gets used by the clients

    Params:
    clients- the clients to copy the data file to
    data- the data file to copy into each user's directory

    """
    if not os.path.exists(data) and not remove:
        print "Error: invalid data file to copy from"
        return
    with open(data, 'r') as file_p:
        content = file_p.read()
    basename = os.path.basename(data)
    for client in clients:
        filename = os.path.join(config.data_dir, client, basename)
        with open(filename, 'w') as file_p:
            file_p.write(content)

def remove_data(clients, data):
    """Remove the given data file so that it does not get used by the clients

    Params:
    clients- the clients to copy the data file to
    data- the data file basename to remove

    """
    for client in clients:
        filename = os.path.join(config.data_dir, client, basename)
        if os.path.exists(filename):
            os.remove(filename)

def copy_exps(clients, exp):
    """Copy the given experiment so that it gets used by the clients

    Params:
    clients- the clients to copy the data file to
    exp- the experiment to copy into each user's directory

    """
    if not os.path.exists(exp):
        print "Error: invalid experiment to copy from"
        return
    with open(exp, 'r') as file_p:
        content = file_p.read()
    basename = os.path.basename(exp)
    for client in clients:
        filename = os.path.join(config.experiments_dir, client, basename)
        with open(filename, 'w') as file_p:
            file_p.write(content)

def remove_exps(clients, exp):
    """Remove the given experiment so that it does not get used by the clients

    Params:
    clients- the clients to copy the data file to
    exp- the experiment file basename to remove

    """
    for client in clients:
        filename = os.path.join(config.experiments_dir, client, basename)
        if os.path.exists(filename):
            os.remove(filename)

def copy_frequency(clients, freq, exp):
    """Schedule the given experiment to run at the frequency specified

    Params:
    clients- the clients to update the frequencies for
    freq- how many hours should elapse between runs, i.e. enter 1 to
        run every hour
    exp- the experiment to adjust the frequency for

    """
    exp = os.path.basename(exp)
    for clients in clients:
        # if the experiment doesn't exist for that user, then don't
        # adjust the frequency
        exp_file = os.path.join(config.experiments_dir, client, exp)
        if not os.path.exists(exp_file):
            continue
        filename = os.path.join(config.experiments_dir, client,
                                "scheduler.info")
        freqs = {}
        if os.path.exists(filename):
            with open(filename, 'r') as file_p:
                freq = json.load(file_p)
        freqs[exp] = {'last_seen': 0, 'frequency': freq}
        # Note: as mentioned in the first few introductory lines, this
        # section presents a race condition if another instance of the
        # scheduler is running at the same time and your experiment
        # may not be scheduled
        with open(filename, 'w') as file_p:
            json.dump(file_p, freqs)

def remove_frequency(clients, exp):
    """Remove the given experiment from the scheduler

    Params:
    clients- the clients to update the frequencies for
    exp- the experiment to adjust the frequency for

    """
    exp = os.path.basename(exp)
    for clients in clients:
        filename = os.path.join(config.experiments_dir, client,
                                "scheduler.info")
        freqs = {}
        if os.path.exists(filename):
            with open(filename, 'r') as file_p:
                freq = json.load(file_p)
        if freqs.get(exp) is not None:
            del freqs[exp]
        # Note: as mentioned in the first few introductory lines, this
        # section presents a race condition if another instance of the
        # scheduler is running at the same time and your experiment
        # may not be scheduled
        with open(filename, 'w') as file_p:
            json.dump(file_p, freqs)

if __name__ == "__main__":
    # Note: the argument parser takes care of default values for us,
    # so we don't need to specify default values
    args = parse_args()

    # lookup the clients/ probes to use
    clients = find_clients(args.country, args.num_clients)

    # print the clients and return if we are not copying any files over
    if args.experiment is None and args.data is None:
        print ("You have not specified a data file or experiment to run on "
               "the clients, so I will just print the clients usernames that "
               "you would have scheduled experiments on")
        for client in clients:
            print "{0}".format(client)
        return

    # copy the data files if necessary
    if args.data is not None:
        if args.remove:
            remove_data(clients, args.data)
        else:
            copy_data(clients, args.data)

    # copy the experiment files if necessary
    if args.experiment is not None:
        if args.remove:
            remove_exps(clients, args.experiment)
            remove_frequency(clients, args.experiment)
        else:
            copy_exps(clients, args.experiment)

    # add the frequency info as appropriate
    if args.frequency is not 1:
        copy_frequency(clients, args.frequency, args.experiment)
<commit_msg>added command line tool to schedule exps<commit_after>#!/usr/bin/env python
#
# Georgia Tech Spring 2014
#
# scheduler.py: script to manage experiments and data by finding
# appropriate clients, copying files into those directories, and
# potentially modifying the scheduling frequency information
#
# Note: you *MUST NOT* start more than 1 instance of this program at
# the same time. Doing so could cause race conditions in our scheduler
# and could result in your experiments not being run at the expected
# frequency

import argparse
from datetime import datetime, timedelta
import json
import os
import os.path
import sys

import config
from centinel.models import Client

def parse_args():
    parser = argparse.ArgumentParser()
    country_help = ('Two letter country code of the country to run the'
                    'experiment in')
    parser.add_argument('--country', '-c', help=country_help, required=True)
    client_help = ("Number of clients in the country to run the measurement "
                   "on. If this is not specified, the experiment will be "
                   "scheduled on all clients in the country")
    parser.add_argument('--num-clients', '-n', help=client_help, default=None)
    data_help = ("Data file for the clients to use. Note that this must be "
                 "paired with an experiment file that has the same name "
                 "(specified at the top of the class in the experiment file)")
    parser.add_argument('--data', '-d', help=data_help, default=None)
    exp_help = ("Experiment file for the clients to use. This should be a "
                "Python file following the format as other experiments "
                "(see the example template or ping.py for how to do this)")
    parser.add_argument('--experiment', '-e', help=exp_help, default=None)
    freq_help = ("How often the experiment should be run in hours, i.e. "
                 "enter 4 to run the experiment every 4 hours. If no "
                 "frequency information is specified, the measurement will "
                 "run every hour")
    parser.add_argument('--frequency', '-f', help=freq_help, default=1)
    remove_help = ("Remove the experiment or data file with the specified "
                   "name from the target clients. Note that this option will "
                   "be applied to both data and experiments if both the -d "
                   "and -e options are specified")
    parser.add_argument('--remove', '-r', help=remove_help, default=False,
                        action='store_true')
    args = parser.parse_args()

    if args.frequency != 1 and args.experiment is None:
        parser.error("Specifying the frequency is only a valid option if you "
                     "are specifying an experiment to run. You must specify "
                     "a new experiment to schedule as well with the -e option")
    return args

def find_clients(country, num_clients):
    """Find num_clients active clients in the target country

    Params:
    country- two letter country code of target country
    num_clients- number of clients to get. If this is None, then we
        get all the active clients for that country

    Note: we define active here as having seen the client in the past
    month

    """
    clients = []
    potential_clients = Client.query.filter_by(country=country).all()
    month_diff = timedelta(days=30)
    for client in potential_clients:
        if client.last_seen < datetime.now() - month_diff:
            continue
        if num_clients is not None and len(clients) >= num_clients:
            continue
        clients.append(client.username)
    return clients

def copy_data(clients, data):
    """Copy the given data file so that it gets used by the clients

    Params:
    clients- the clients to copy the data file to
    data- the data file to copy into each user's directory

    """
    if not os.path.exists(data):
        print "Error: invalid data file to copy from"
        return
    with open(data, 'r') as file_p:
        content = file_p.read()
    basename = os.path.basename(data)
    for client in clients:
        filename = os.path.join(config.inputs_dir, client, basename)
        with open(filename, 'w') as file_p:
            file_p.write(content)

def remove_data(clients, data):
    """Remove the given data file so that it does not get used by the clients

    Params:
    clients- the clients to copy the data file to
    data- the data file basename to remove

    """
    data = os.path.basename(data)
    for client in clients:
        filename = os.path.join(config.inputs_dir, client, data)
        if os.path.exists(filename):
            os.remove(filename)

def copy_exps(clients, exp):
    """Copy the given experiment so that it gets used by the clients

    Params:
    clients- the clients to copy the data file to
    exp- the experiment to copy into each user's directory

    """
    if not os.path.exists(exp):
        print "Error: invalid experiment to copy from"
        return
    with open(exp, 'r') as file_p:
        content = file_p.read()
    basename = os.path.basename(exp)
    for client in clients:
        filename = os.path.join(config.experiments_dir, client, basename)
        with open(filename, 'w') as file_p:
            file_p.write(content)

def remove_exps(clients, exp):
    """Remove the given experiment so that it does not get used by the clients

    Params:
    clients- the clients to copy the data file to
    exp- the experiment file basename to remove

    """
    basename = os.path.basename(exp)
    for client in clients:
        filename = os.path.join(config.experiments_dir, client, basename)
        if os.path.exists(filename):
            os.remove(filename)

def copy_frequency(clients, freq, exp):
    """Schedule the given experiment to run at the frequency specified

    Params:
    clients- the clients to update the frequencies for
    freq- how many hours should elapse between runs, i.e. enter 1 to
        run every hour
    exp- the experiment to adjust the frequency for

    """
    exp = os.path.basename(exp)
    for client in clients:
        # if the experiment doesn't exist for that user, then don't
        # adjust the frequency
        exp_file = os.path.join(config.experiments_dir, client, exp)
        if not os.path.exists(exp_file):
            continue
        filename = os.path.join(config.experiments_dir, client,
                                "scheduler.info")
        freqs = {}
        if os.path.exists(filename):
            with open(filename, 'r') as file_p:
                freqs = json.load(file_p)
        freqs[exp] = {'last_seen': 0, 'frequency': freq}
        # Note: as mentioned in the first few introductory lines, this
        # section presents a race condition if another instance of the
        # scheduler is running at the same time and your experiment
        # may not be scheduled
        with open(filename, 'w') as file_p:
            json.dump(freqs, file_p)

def remove_frequency(clients, exp):
    """Remove the given experiment from the scheduler

    Params:
    clients- the clients to update the frequencies for
    exp- the experiment to adjust the frequency for

    """
    exp = os.path.basename(exp)
    for client in clients:
        filename = os.path.join(config.experiments_dir, client,
                                "scheduler.info")
        freqs = {}
        if os.path.exists(filename):
            with open(filename, 'r') as file_p:
                freqs = json.load(file_p)
        if freqs.get(exp) is not None:
            del freqs[exp]
        if freqs == {}:
            os.remove(filename)
            return
        # Note: as mentioned in the first few introductory lines, this
        # section presents a race condition if another instance of the
        # scheduler is running at the same time and your experiment
        # may not be scheduled
        with open(filename, 'w') as file_p:
            json.dump(freqs, file_p)

if __name__ == "__main__":
    # Note: the argument parser takes care of default values for us,
    # so we don't need to specify default values
    args = parse_args()

    # lookup the clients/ probes to use
    clients = find_clients(args.country, args.num_clients)

    # print the clients and return if we are not copying any files over
    if args.experiment is None and args.data is None:
        print ("You have not specified a data file or experiment to run on "
               "the clients, so I will just print the clients usernames that "
               "you would have scheduled experiments on")
        for client in clients:
            print "{0}".format(client)
        sys.exit(0)

    # copy the data files if necessary
    if args.data is not None:
        if args.remove:
            remove_data(clients, args.data)
        else:
            copy_data(clients, args.data)

    # copy the experiment files if necessary
    if args.experiment is not None:
        if args.remove:
            remove_exps(clients, args.experiment)
            remove_frequency(clients, args.experiment)
        else:
            copy_exps(clients, args.experiment)

    # add the frequency info as appropriate
    if not args.remove and args.frequency is not 1:
        copy_frequency(clients, args.frequency, args.experiment)
<|endoftext|>


<commit_before>import asyncio
import discord
import os
from   PIL         import Image
from   discord.ext import commands
from   Cogs import Settings

# This is the Bot module - it contains things like nickname, status, etc

class Bot:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	def message(self, message):
		# Check the message and see if we should allow it - always yes.
		# This module doesn't need to cancel messages.
		return { 'Ignore' : False, 'Delete' : False}

	@commands.command(pass_context=True)
	async def nickname(self, ctx, name : str = None):
		"""Set the bot's nickname (admin-only)."""

		isAdmin = ctx.message.author.permissions_in(ctx.message.channel).administrator
		# Only allow admins to change server stats
		if not isAdmin:
			await self.bot.send_message(ctx.message.channel, 'You do not have sufficient privileges to access this command.')
			return

		# Let's get the bot's member in the current server
		botName = "{}#{}".format(self.bot.user.name, self.bot.user.discriminator)
		botMember = ctx.message.server.get_member_named(botName)
		await self.bot.change_nickname(botMember, name)

	@commands.command(pass_context=True)
	async def avatar(self, ctx, filename : str = None, sizeLimit : int = 8000000):
		"""Sets the bot's avatar (owner-only)."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		# Only allow owner to change server stats
		serverDict = self.settings.serverDict

		try:
			owner = serverDict['Owner']
		except KeyError:
			owner = None

		if owner == None:
			# No owner set
			msg = 'I have not been claimed, *yet*.'
			await self.bot.send_message(channel, msg)
			return
		else:
			if not author.id == owner:
				msg = 'You are not the *true* owner of me.  Only the rightful owner can change my avatar.'
				await self.bot.send_message(channel, msg)
				return
		if filename is None:
			await self.bot.edit_profile(avatar=None)
			return

		if not os.path.isfile('./{}'.format(filename)):
			# File doesn't exist
			msg = '*{}* doesn\'t exist in my working directory.'.format(filename)
			await self.bot.send_message(channel, msg)
			return

		# File exists - check if image
		img = Image.open(filename)
		ext = img.format

		if not ext:
			# File isn't a valid image
			msg = '*{}* isn\'t a valid image format.'.format(filename)
			await self.bot.send_message(channel, msg)
			return

		# Is an image PIL understands
		if not ext.lower == "png":
			# Not a PNG - let's convert
			filename = '{}.png'.format(filename)
			img.save(filename)
		# Should be a png here - let's check size
		# Let's make sure it's less than the passed limit

		imageSize = os.stat(filename)
		while int(imageSize.st_size) > sizeLimit:
			# Image is too big - resize
			myimage = Image.open(filename)
			xsize, ysize = myimage.size
			ratio = sizeLimit/int(imageSize.st_size)
			xsize *= ratio
			ysize *= ratio
			myimage = myimage.resize((int(xsize), int(ysize)), Image.ANTIALIAS)
			myimage.save(filename)
			imageSize = os.stat(filename)
		# Image is resized - let's save it
		img = Image.open(filename)
		ext = img.format
		img.close()

		with open(filename, 'rb') as f:
			newAvatar = f.read()
			await self.bot.edit_profile(avatar=newAvatar)

	@commands.command(pass_context=True)
	async def servers(self, ctx):
		"""Lists the number of servers I'm connected to!"""
		total = 0
		for server in self.bot.servers:
			total += 1
		msg = 'I am a part of *{}* servers!'.format(total)
		await self.bot.send_message(channel, msg)

	@commands.command(pass_context=True)
	async def playgame(self, ctx, game : str = None):
		"""Sets the playing status of the bot (owner-only)."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		# Only allow owner to change server stats
		serverDict = self.settings.serverDict

		try:
			owner = serverDict['Owner']
		except KeyError:
			owner = None

		if owner == None:
			# No owner set
			msg = 'I have not been claimed, *yet*.'
			await self.bot.send_message(channel, msg)
			return
		else:
			if not author.id == owner:
				msg = 'You are not the *true* owner of me.  Only the rightful owner can set my playing status.'
				await self.bot.send_message(channel, msg)
				return

		if game == None:
			await self.bot.change_presence(game=None)
			return

		await self.bot.change_presence(game=discord.Game(name=game))

	@commands.command(pass_context=True)
	async def setbotparts(self, ctx, parts : str = None):
		"""Set the bot's parts - can be a url, formatted text, or nothing to clear."""

		isAdmin = ctx.message.author.permissions_in(ctx.message.channel).administrator
		# Only allow admins to change server stats
		if not isAdmin:
			await self.bot.send_message(ctx.message.channel, 'You do not have sufficient privileges to access this command.')
			return

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		if not parts:
			parts = ""

		self.settings.setUserStat(self.bot.user, server, "Parts", parts)
		msg = '*{}\'s* parts have been set to:\n{}'.format(self.bot.user.name, parts)
		await self.bot.send_message(channel, msg)<commit_msg>Servers "channel" fix<commit_after>import asyncio
import discord
import os
from   PIL         import Image
from   discord.ext import commands
from   Cogs import Settings

# This is the Bot module - it contains things like nickname, status, etc

class Bot:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	def message(self, message):
		# Check the message and see if we should allow it - always yes.
		# This module doesn't need to cancel messages.
		return { 'Ignore' : False, 'Delete' : False}

	@commands.command(pass_context=True)
	async def nickname(self, ctx, name : str = None):
		"""Set the bot's nickname (admin-only)."""

		isAdmin = ctx.message.author.permissions_in(ctx.message.channel).administrator
		# Only allow admins to change server stats
		if not isAdmin:
			await self.bot.send_message(ctx.message.channel, 'You do not have sufficient privileges to access this command.')
			return

		# Let's get the bot's member in the current server
		botName = "{}#{}".format(self.bot.user.name, self.bot.user.discriminator)
		botMember = ctx.message.server.get_member_named(botName)
		await self.bot.change_nickname(botMember, name)

	@commands.command(pass_context=True)
	async def avatar(self, ctx, filename : str = None, sizeLimit : int = 8000000):
		"""Sets the bot's avatar (owner-only)."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		# Only allow owner to change server stats
		serverDict = self.settings.serverDict

		try:
			owner = serverDict['Owner']
		except KeyError:
			owner = None

		if owner == None:
			# No owner set
			msg = 'I have not been claimed, *yet*.'
			await self.bot.send_message(channel, msg)
			return
		else:
			if not author.id == owner:
				msg = 'You are not the *true* owner of me.  Only the rightful owner can change my avatar.'
				await self.bot.send_message(channel, msg)
				return
		if filename is None:
			await self.bot.edit_profile(avatar=None)
			return

		if not os.path.isfile('./{}'.format(filename)):
			# File doesn't exist
			msg = '*{}* doesn\'t exist in my working directory.'.format(filename)
			await self.bot.send_message(channel, msg)
			return

		# File exists - check if image
		img = Image.open(filename)
		ext = img.format

		if not ext:
			# File isn't a valid image
			msg = '*{}* isn\'t a valid image format.'.format(filename)
			await self.bot.send_message(channel, msg)
			return

		# Is an image PIL understands
		if not ext.lower == "png":
			# Not a PNG - let's convert
			filename = '{}.png'.format(filename)
			img.save(filename)
		# Should be a png here - let's check size
		# Let's make sure it's less than the passed limit

		imageSize = os.stat(filename)
		while int(imageSize.st_size) > sizeLimit:
			# Image is too big - resize
			myimage = Image.open(filename)
			xsize, ysize = myimage.size
			ratio = sizeLimit/int(imageSize.st_size)
			xsize *= ratio
			ysize *= ratio
			myimage = myimage.resize((int(xsize), int(ysize)), Image.ANTIALIAS)
			myimage.save(filename)
			imageSize = os.stat(filename)
		# Image is resized - let's save it
		img = Image.open(filename)
		ext = img.format
		img.close()

		with open(filename, 'rb') as f:
			newAvatar = f.read()
			await self.bot.edit_profile(avatar=newAvatar)

	@commands.command(pass_context=True)
	async def servers(self, ctx):
		"""Lists the number of servers I'm connected to!"""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		total = 0
		for server in self.bot.servers:
			total += 1
		msg = 'I am a part of *{}* servers!'.format(total)
		await self.bot.send_message(channel, msg)

	@commands.command(pass_context=True)
	async def playgame(self, ctx, game : str = None):
		"""Sets the playing status of the bot (owner-only)."""

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		# Only allow owner to change server stats
		serverDict = self.settings.serverDict

		try:
			owner = serverDict['Owner']
		except KeyError:
			owner = None

		if owner == None:
			# No owner set
			msg = 'I have not been claimed, *yet*.'
			await self.bot.send_message(channel, msg)
			return
		else:
			if not author.id == owner:
				msg = 'You are not the *true* owner of me.  Only the rightful owner can set my playing status.'
				await self.bot.send_message(channel, msg)
				return

		if game == None:
			await self.bot.change_presence(game=None)
			return

		await self.bot.change_presence(game=discord.Game(name=game))

	@commands.command(pass_context=True)
	async def setbotparts(self, ctx, parts : str = None):
		"""Set the bot's parts - can be a url, formatted text, or nothing to clear."""

		isAdmin = ctx.message.author.permissions_in(ctx.message.channel).administrator
		# Only allow admins to change server stats
		if not isAdmin:
			await self.bot.send_message(ctx.message.channel, 'You do not have sufficient privileges to access this command.')
			return

		channel = ctx.message.channel
		author  = ctx.message.author
		server  = ctx.message.server

		if not parts:
			parts = ""

		self.settings.setUserStat(self.bot.user, server, "Parts", parts)
		msg = '*{}\'s* parts have been set to:\n{}'.format(self.bot.user.name, parts)
		await self.bot.send_message(channel, msg)<|endoftext|>


<commit_before>#!/usr/bin/python
import re
import sys
import subprocess

DEBUG = (sys.argv[2] == "--macro")
input = "\n".join([l for l in open(sys.argv[1])])

# Evaluate str with node.
def nodejs(str):
  open("temp", 'w').write(header + str)
  return subprocess.check_output(["node", "temp"])

array_to_scheme = """
function to_scheme(item){
  if (!(item instanceof Array)) {
    return item;
  }
  var result = "(";
  for (var i = 0; i < item.length; i++){
    result += to_scheme(item[i]) + (i == item.length - 1 ? "" : " ");
  }
  result += ")";
  return result;
}
"""

def toscheme(str):
  return nodejs(array_to_scheme + "dump(to_scheme(%s));" % str)

class Node:
  known_macros = []
  macro_js = ""

  def __init__(self, name, args):
    assert isinstance(args, list)
    self.name = name
    self.args = args

  # Return scheme representation of self. This is exactly the same as what
  # we read in from the file.
  def toscheme(self):
    return "(%s " % self.name + " ".join([arg.toscheme() for arg in self.args]) + ")"

  def tojson(self):
    return "['%s', " % self.name + ", ".join([arg.tojson() for arg in self.args]) + "]"

  def tostr(self, indent):
    indentation = indent * "  "
    result = ""

    result = indentation + self.name
    for arg in self.args:
      result += "\n" + arg.tostr(indent + 1)

    return result

  def __repr__(self):
    return self.tostr(0)

  # Preliminary pass to compile all macros into JS.
  def compile_macro(self):
    if self.name == "defmacro":
      Node.known_macros.append(self.args[0].compile())
      return "\nfunction %s %s { %s } \n" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    return "".join([arg.compile_macro() for arg in self.args])

  # This pass converts all calls to macros into the resultant forms.
  def first_pass(self):
    if self.name in Node.known_macros:
      if DEBUG: print "Transforming %s\n\n" % (self.toscheme())
      macro_name = self.name
      # Construct JavaScript to call JS function and pass in args
      js = "dump(" + macro_name + "(" + ",".join([arg.tojson() for arg in self.args]) + "))"
      if DEBUG: print "Running %s\n\n" % (Node.macro_js + js)
      result = nodejs(Node.macro_js + js)
      # Result is now basically what we want, except it's JavaScript arrays.
      result = toscheme(result)
      new_node = parse(result, False)
      self.args = new_node.args
      self.name = new_node.name
      if DEBUG: print "Transformed to %s\n-----------" % (self.toscheme())
      return

    for node in self.args:
      node.first_pass()

  def compile(self):
    ops = ["+", "/", "*", "-", "||", "&&", "===", "!==", "!=", "==", "+=", "-=", "/=", "*=", "instanceof", "<", ">", "<=", ">=", "%"]
    unary = ["!", "+", "-", "new", "typeof", "++", "--"]

    name = self.name
    if name == "=":
      return "%s = %s" % (self.args[0].compile(), self.args[1].compile())
    elif name == "parenthesize":
      return "(%s)" % (self.args[0].compile())
    elif name == "brackets":
      return "{%s}" % (";\n".join([arg.compile() for arg in self.args]))
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "break":
      return "break;"
    elif name == "continue":
      return "continue"
    elif name == "defmacro":
      return ""
    elif name == "call":
      raise "CallCalled"
    elif name == "try":
      return "try {\n%s\n} catch %s  {\n%s} finally {\n%s}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile(), self.args[3].compile())
    elif name == "do":
      if len(self.args) == 0:
        return "(void 0)"
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "docomma":
      assert len(self.args) > 0
      return ",\n".join([arg.compile() for arg in self.args])
    elif name == "var":
      return "var %s" % ",".join([arg.compile() for arg in self.args])
    elif name == "[]":
      return "%s[%s]" % (self.args[0].compile(), self.args[1].compile())
    elif name == "[0]": #TODO: Remove for "[]"
      return "(%s)[0]" % self.args[0].compile()
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "==":
      return "%s == %s" % (self.args[0].compile(), self.args[1].compile())
    elif name == "while":
      return "while (%s) %s;" % (self.args[0].compile(), self.args[1].compile())
    elif name == "if":
      while len(self.args) < 3:
        self.args.append(Node("void", [Atom("0")])) # Append empty bodies to unfilled parts of the if

      return "if (%s) {%s;} else {%s;}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "ternary":
      return "%s ? %s : %s " % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
      # return ("if (%s) {%s;} else {%s;}") % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "function":
      return "function %s { %s }" % (self.args[0].compile(), ";".join([arg.compile() for arg in self.args[1:]]))
    elif name == "arglist":
      return "(" + ",".join([arg.compile() for arg in self.args]) + ")"

    for op in ops:
      if name == op and len(self.args) > 2:
        expr = (" " + op + " ").join([arg.compile() for arg in self.args if arg.compile().strip() != ""])
        return "(" + expr + ")"

      if name == op and len(self.args) == 2:
        return "(%s %s %s)" % (self.args[0].compile(), op, self.args[1].compile())

    for op in unary:
      if name == op and len(self.args) == 1:
        return "(%s %s)" % (op, self.args[0].compile())

    # A special case for stuff like ((function (arglist) 5))
    # where you immediately call an anonymous function.
    if self.name == "":
      return "(%s)()" % self.args[0].compile()
    else:
      return "%s(%s)" % (self.name, ", ".join([arg.compile() for arg in self.args]))

class Atom:
  def __init__(self, contents):
    self.contents = contents

  def tostr(self, indent):
    return indent * "  " + self.contents

  def toscheme(self):
    return self.contents

  def tojson(self):
    if isinstance(self.contents, str):
      return '"%s"' % self.contents
    else:
      return self.contents

  def compile(self):
    return self.contents

  def first_pass(self):
    pass

  def compile_macro(self):
    return ""

  def __repr__(self):
    return "Atom " + self.contents

def is_paren(str):
  return str == "(" or str == ")"

# Add quoting functionality: some basic syntactic sugar to make
# writing macros easier on the eyes (and brain...)
def desugar(tokens):
  result = []
  quoted = False
  depth = 0
  for token in tokens:
    if token == "`(":
      quoted = True
      result.append("(")
      result.append("list")
      depth = 1
    elif token == "(":
      result.append(token)
      if quoted: result.append("list")
      depth += 1
    elif token == ")":
      result.append(token)
      depth -= 1
      if depth == 0 and quoted:
        quoted = False
    elif token[0] == "~":
      if not quoted:
        raise "Unquote in nonquoted form?"
      result.append(token[1:]) #remove tilde, keep token.
    else:
      result.append('"%s"' % token if quoted else token)

  return result

def tokenize(string):
  in_string = False
  string_opener = ""
  tokens = [""]
  for ch in string:
    if (ch == "'" or ch == '"') and (string_opener == ch or string_opener == ""):
      string_opener = (ch if string_opener == "" else "")

      in_string = not in_string
      tokens[-1] += ch
      if not in_string: tokens.append("")
      continue

    if not in_string:
      if ch == " ":
        tokens.append("")
        continue

      if is_paren(ch):
        if tokens[-1] == "`": # Special case for quoted parens.
          tokens[-1] += ch
        else:
          tokens.append(ch)
        tokens.append("")
        continue

    tokens[-1] += ch

  tokens = [tok.strip() for tok in tokens[:-1] if tok.strip() != ""]
  tokens = desugar(tokens)
  return tokens

def parse(string, root=True):
  string = string.strip()
  if root: string = "(root " + string + ")"
  tokens = tokenize(string)

  def helper(tokens):
    node_name = tokens[1]
    node_args = [[]]
    # If we immediately call a function, make it an arg of a
    # special cased function with no name.
    if node_name == "(":
      node_name = ""
      tokens = tokens[1:-1]
    else:
      tokens = tokens[2:-1] #ignore (, name, and ).

    depth = 0
    for token in tokens:
      if is_paren(token):
        depth = depth + (1 if token == "(" else -1)
        node_args[-1].append(token)

        if depth == 0:
          node_args[-1] = helper(node_args[-1])
          node_args.append([])
        continue

      if depth == 0:
        node_args[-1] = Atom(token)
        node_args.append([])
      else:
        node_args[-1].append(token)

    node_args = node_args[:-1]
    return Node(node_name, node_args)

  return helper(tokens)

ast = parse(input)

output = sys.argv[1].split(".")[0] + ".js" #same name as input, but .js instead of .sc

# Header contains some basic lisp-y functions.
header = "".join([line for line in file("basic.sc")]) + "\n"
header = parse(header).compile()

Node.macro_js = ast.compile_macro()

ast.first_pass()
if sys.argv[2] == "--macro":
  open("macropass", "w").write(ast.toscheme())

open(output, 'w').write(ast.compile())
<commit_msg>Allow for recursive macros.<commit_after>#!/usr/bin/python
import re
import sys
import subprocess

DEBUG = (sys.argv[2] == "--macro")
input = "\n".join([l for l in open(sys.argv[1])])

# Evaluate str with node.
def nodejs(str):
  open("temp", 'w').write(header + str)
  return subprocess.check_output(["node", "temp"])

array_to_scheme = """
function to_scheme(item){
  if (!(item instanceof Array)) {
    return item;
  }
  var result = "(";
  for (var i = 0; i < item.length; i++){
    result += to_scheme(item[i]) + (i == item.length - 1 ? "" : " ");
  }
  result += ")";
  return result;
}
"""

def toscheme(str):
  return nodejs(array_to_scheme + "dump(to_scheme(%s));" % str)

class Node:
  known_macros = []
  macro_js = ""

  def __init__(self, name, args):
    assert isinstance(args, list)
    self.name = name
    self.args = args

  # Return scheme representation of self. This is exactly the same as what
  # we read in from the file.
  def toscheme(self):
    return "(%s " % self.name + " ".join([arg.toscheme() for arg in self.args]) + ")"

  def tojson(self):
    return "['%s', " % self.name + ", ".join([arg.tojson() for arg in self.args]) + "]"

  def tostr(self, indent):
    indentation = indent * "  "
    result = ""

    result = indentation + self.name
    for arg in self.args:
      result += "\n" + arg.tostr(indent + 1)

    return result

  def __repr__(self):
    return self.tostr(0)

  # Preliminary pass to compile all macros into JS.
  def compile_macro(self):
    if self.name == "defmacro":
      Node.known_macros.append(self.args[0].compile())
      return "\nfunction %s %s { %s } \n" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    return "".join([arg.compile_macro() for arg in self.args])

  # This pass converts all calls to macros into the resultant forms.
  def first_pass(self):

    # Allow recursive macros, disallow macros that call other macros.
    if self.name == "defmacro":
      return

    if self.name in Node.known_macros:
      if DEBUG: print "Transforming %s\n\n" % (self.toscheme())
      macro_name = self.name
      # Construct JavaScript to call JS function and pass in args
      js = "dump(" + macro_name + "(" + ",".join([arg.tojson() for arg in self.args]) + "))"
      if DEBUG: print "Running %s\n\n" % (Node.macro_js + js)
      result = nodejs(Node.macro_js + js)
      # Result is now basically what we want, except it's JavaScript arrays.
      result = toscheme(result)
      new_node = parse(result, False)
      self.args = new_node.args
      self.name = new_node.name
      if DEBUG: print "Transformed to %s\n-----------" % (self.toscheme())
      return

    for node in self.args:
      node.first_pass()

  def compile(self):
    ops = ["+", "/", "*", "-", "||", "&&", "===", "!==", "!=", "==", "+=", "-=", "/=", "*=", "instanceof", "<", ">", "<=", ">=", "%"]
    unary = ["!", "+", "-", "new", "typeof", "++", "--"]

    name = self.name
    if name == "=":
      return "%s = %s" % (self.args[0].compile(), self.args[1].compile())
    elif name == "parenthesize":
      return "(%s)" % (self.args[0].compile())
    elif name == "brackets":
      return "{%s}" % (";\n".join([arg.compile() for arg in self.args]))
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "break":
      return "break;"
    elif name == "continue":
      return "continue"
    elif name == "defmacro":
      return ""
    elif name == "call":
      raise "CallCalled"
    elif name == "try":
      return "try {\n%s\n} catch %s  {\n%s} finally {\n%s}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile(), self.args[3].compile())
    elif name == "do":
      if len(self.args) == 0:
        return "(void 0)"
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "docomma":
      assert len(self.args) > 0
      return ",\n".join([arg.compile() for arg in self.args])
    elif name == "var":
      return "var %s" % ",".join([arg.compile() for arg in self.args])
    elif name == "[]":
      return "%s[%s]" % (self.args[0].compile(), self.args[1].compile())
    elif name == "[0]": #TODO: Remove for "[]"
      return "(%s)[0]" % self.args[0].compile()
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "==":
      return "%s == %s" % (self.args[0].compile(), self.args[1].compile())
    elif name == "while":
      return "while (%s) %s;" % (self.args[0].compile(), self.args[1].compile())
    elif name == "if":
      while len(self.args) < 3:
        self.args.append(Node("void", [Atom("0")])) # Append empty bodies to unfilled parts of the if

      return "if (%s) {%s;} else {%s;}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "ternary":
      return "%s ? %s : %s " % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
      # return ("if (%s) {%s;} else {%s;}") % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "function":
      return "function %s { %s }" % (self.args[0].compile(), ";".join([arg.compile() for arg in self.args[1:]]))
    elif name == "arglist":
      return "(" + ",".join([arg.compile() for arg in self.args]) + ")"

    for op in ops:
      if name == op and len(self.args) > 2:
        expr = (" " + op + " ").join([arg.compile() for arg in self.args if arg.compile().strip() != ""])
        return "(" + expr + ")"

      if name == op and len(self.args) == 2:
        return "(%s %s %s)" % (self.args[0].compile(), op, self.args[1].compile())

    for op in unary:
      if name == op and len(self.args) == 1:
        return "(%s %s)" % (op, self.args[0].compile())

    # A special case for stuff like ((function (arglist) 5))
    # where you immediately call an anonymous function.
    if self.name == "":
      return "(%s)()" % self.args[0].compile()
    else:
      return "%s(%s)" % (self.name, ", ".join([arg.compile() for arg in self.args]))

class Atom:
  def __init__(self, contents):
    self.contents = contents

  def tostr(self, indent):
    return indent * "  " + self.contents

  def toscheme(self):
    return self.contents

  def tojson(self):
    if isinstance(self.contents, str):
      return '"%s"' % self.contents
    else:
      return self.contents

  def compile(self):
    return self.contents

  def first_pass(self):
    pass

  def compile_macro(self):
    return ""

  def __repr__(self):
    return "Atom " + self.contents

def is_paren(str):
  return str == "(" or str == ")"

# Add quoting functionality: some basic syntactic sugar to make
# writing macros easier on the eyes (and brain...)
def desugar(tokens):
  result = []
  quoted = False
  depth = 0
  for token in tokens:
    if token == "`(":
      quoted = True
      result.append("(")
      result.append("list")
      depth = 1
    elif token == "(":
      result.append(token)
      if quoted: result.append("list")
      depth += 1
    elif token == ")":
      result.append(token)
      depth -= 1
      if depth == 0 and quoted:
        quoted = False
    elif token[0] == "~":
      if not quoted:
        raise "Unquote in nonquoted form?"
      result.append(token[1:]) #remove tilde, keep token.
    else:
      result.append('"%s"' % token if quoted else token)

  return result

def tokenize(string):
  in_string = False
  string_opener = ""
  tokens = [""]
  for ch in string:
    if (ch == "'" or ch == '"') and (string_opener == ch or string_opener == ""):
      string_opener = (ch if string_opener == "" else "")

      in_string = not in_string
      tokens[-1] += ch
      if not in_string: tokens.append("")
      continue

    if not in_string:
      if ch == " ":
        tokens.append("")
        continue

      if is_paren(ch):
        if tokens[-1] == "`": # Special case for quoted parens.
          tokens[-1] += ch
        else:
          tokens.append(ch)
        tokens.append("")
        continue

    tokens[-1] += ch

  tokens = [tok.strip() for tok in tokens[:-1] if tok.strip() != ""]
  tokens = desugar(tokens)
  return tokens

def parse(string, root=True):
  string = string.strip()
  if root: string = "(root " + string + ")"
  tokens = tokenize(string)

  def helper(tokens):
    node_name = tokens[1]
    node_args = [[]]
    # If we immediately call a function, make it an arg of a
    # special cased function with no name.
    if node_name == "(":
      node_name = ""
      tokens = tokens[1:-1]
    else:
      tokens = tokens[2:-1] #ignore (, name, and ).

    depth = 0
    for token in tokens:
      if is_paren(token):
        depth = depth + (1 if token == "(" else -1)
        node_args[-1].append(token)

        if depth == 0:
          node_args[-1] = helper(node_args[-1])
          node_args.append([])
        continue

      if depth == 0:
        node_args[-1] = Atom(token)
        node_args.append([])
      else:
        node_args[-1].append(token)

    node_args = node_args[:-1]
    return Node(node_name, node_args)

  return helper(tokens)

ast = parse(input)

output = sys.argv[1].split(".")[0] + ".js" #same name as input, but .js instead of .sc

# Header contains some basic lisp-y functions.
header = "".join([line for line in file("basic.sc")]) + "\n"
header = parse(header).compile()

Node.macro_js = ast.compile_macro()

ast.first_pass()
if sys.argv[2] == "--macro":
  open("macropass", "w").write(ast.toscheme())

open(output, 'w').write(ast.compile())
<|endoftext|>


<commit_before>import base64
import os
import mimetypes

from pygeocoder import Geocoder

from tastypie import fields
from tastypie.authorization import Authorization, ReadOnlyAuthorization,\
    DjangoAuthorization
from tastypie.authentication import ApiKeyAuthentication
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from tastypie.contrib.gis.resources import ModelResource as GeoModelResource
from tastypie.fields import DictField
from tastypie.resources import ModelResource

from dataserver.authorization import GuardianAuthorization
from dataserver.authentication import AnonymousApiKeyAuthentication

from accounts.api import UserResource
from bucket.models import Bucket

from .models import (Map, DataLayer, TileLayer, Marker,
                     MarkerCategory, PostalAddress, Place)

class MapAuthorization(GuardianAuthorization):
    def __init__(self):
        super(MapAuthorization, self).__init__(
            create_permission_code="add_map",
            view_permission_code="view_map",
            update_permission_code="change_map",
            delete_permission_code="delete_map"
        )

    def read_detail(self, object_list, bundle):
        for obj in object_list:
            if obj.privacy != 'GROUP_RW_OTHERS_RO':
                return super(MapAuthorization, self).read_detail(object_list, bundle)

        return True

class MapResource(GeoModelResource):
    class Meta:
        queryset = Map.objects.all()
        resource_name = 'scout/map'
        authentication = AnonymousApiKeyAuthentication()
        authorization = MapAuthorization()

        always_return_data = True
        detail_uri_name = 'slug'

    created_by = fields.ToOneField('accounts.api.UserResource', 'created_by', readonly=True)
    data_layers = fields.ToManyField('scout.api.DataLayerResource', 'datalayers', full=True, null=True)
    tile_layer = fields.ForeignKey('scout.api.TileLayerResource', 'tilelayer', full=True)
    bucket = fields.ForeignKey('bucket.api.BucketResource', 'bucket', null=True, full=True)
    marker_categories = fields.ToManyField('scout.api.MarkerCategoryResource', 'marker_categories', null=True, full=True)

    def obj_create(self, bundle, **kwargs):
        bundle.obj = Map(bucket=Bucket.objects.create(created_by=bundle.request.user), created_by=bundle.request.user)
        bundle = self.full_hydrate(bundle)
        bundle.obj.save()

        return bundle

class MarkerCategoryResource(ModelResource):
    class Meta:
        queryset = MarkerCategory.objects.all()
        resource_name = 'scout/marker_category'

        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization() # FIXME

class TileLayerResource(GeoModelResource):
    class Meta:
        queryset = TileLayer.objects.all()
        resource_name = 'scout/tilelayer'

        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization() # FIXME

    maps = fields.ToManyField(MapResource, 'maps', null=True)

class DataLayerResource(ModelResource):
    class Meta:
        queryset = DataLayer.objects.all()
        resource_name = 'scout/datalayer'
        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization()

    markers = fields.ToManyField('scout.api.MarkerResource', 'markers', null=True, full=True)
    map = fields.ToOneField('scout.api.MapResource', 'map')
    json_mapping = fields.DictField(attribute='json_mapping')

class MarkerResource(GeoModelResource):
    class Meta:
        queryset = Marker.objects.all()
        resource_name = 'scout/marker'
        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True

    data_layer = fields.ToOneField(DataLayerResource, 'datalayer')
    created_by = fields.ToOneField(UserResource, 'created_by', full=True)
    category = fields.ToOneField(MarkerCategoryResource, 'category', full=True)

    def hydrate(self, bundle, request=None):
        if not bundle.obj.pk:
            bundle.data['created_by'] = bundle.request.user

        # Resolve position
        position = bundle.data['position']['coordinates']
        geo_results = Geocoder.reverse_geocode(position[0], position[1])
        if len(geo_results) > 0:
            bundle.data['address'] = geo_results[0]

        return bundle

class PostalAddressResource(ModelResource):
    class Meta:
        queryset = PostalAddress.objects.all()
        resource_name = 'scout/postaladdress'
        always_return_data = True
        authentication = AnonymousApiKeyAuthentication()
        authorization =  Authorization()

class PlaceResource(GeoModelResource):
    class Meta:
        queryset = Place.objects.all()
        resource_name = 'scout/place'
        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True

        filtering = {
            "geo": ALL
        }

    address = fields.ToOneField(PostalAddressResource, 'address')
<commit_msg>fixed authorizations<commit_after>import base64
import os
import mimetypes

from pygeocoder import Geocoder

from tastypie import fields
from tastypie.authorization import Authorization, ReadOnlyAuthorization,\
    DjangoAuthorization
from tastypie.authentication import ApiKeyAuthentication
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from tastypie.contrib.gis.resources import ModelResource as GeoModelResource
from tastypie.fields import DictField
from tastypie.resources import ModelResource

from dataserver.authorization import GuardianAuthorization
from dataserver.authentication import AnonymousApiKeyAuthentication

from accounts.api import UserResource
from bucket.models import Bucket

from .models import (Map, DataLayer, TileLayer, Marker,
                     MarkerCategory, PostalAddress, Place)

class MapAuthorization(GuardianAuthorization):
    def __init__(self):
        super(MapAuthorization, self).__init__(
            create_permission_code="add_map",
            view_permission_code="view_map",
            update_permission_code="change_map",
            delete_permission_code="delete_map"
        )

    def read_detail(self, object_list, bundle):
        for obj in object_list:
            if obj.privacy != 'GROUP_RW_OTHERS_RO':
                return super(MapAuthorization, self).read_detail(object_list, bundle)

        return True

class MapResource(GeoModelResource):
    class Meta:
        queryset = Map.objects.all()
        resource_name = 'scout/map'
        authentication = AnonymousApiKeyAuthentication()
        authorization = MapAuthorization()

        always_return_data = True
        detail_uri_name = 'slug'

    created_by = fields.ToOneField('accounts.api.UserResource', 'created_by', readonly=True)
    data_layers = fields.ToManyField('scout.api.DataLayerResource', 'datalayers', full=True, null=True)
    tile_layer = fields.ForeignKey('scout.api.TileLayerResource', 'tilelayer', full=True)
    bucket = fields.ForeignKey('bucket.api.BucketResource', 'bucket', null=True, full=True)
    marker_categories = fields.ToManyField('scout.api.MarkerCategoryResource', 'marker_categories', null=True, full=True)

    def obj_create(self, bundle, **kwargs):
        bundle.obj = Map(bucket=Bucket.objects.create(created_by=bundle.request.user), created_by=bundle.request.user)
        bundle = self.full_hydrate(bundle)
        bundle.obj.save()

        return bundle

class MarkerCategoryResource(ModelResource):
    class Meta:
        queryset = MarkerCategory.objects.all()
        resource_name = 'scout/marker_category'

        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization() # FIXME

class TileLayerResource(GeoModelResource):
    class Meta:
        queryset = TileLayer.objects.all()
        resource_name = 'scout/tilelayer'

        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization() # FIXME

    maps = fields.ToManyField(MapResource, 'maps', null=True)

class DataLayerResource(ModelResource):
    class Meta:
        queryset = DataLayer.objects.all()
        resource_name = 'scout/datalayer'
        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization()

    markers = fields.ToManyField('scout.api.MarkerResource', 'markers', null=True, full=True)
    map = fields.ToOneField('scout.api.MapResource', 'map')
    json_mapping = fields.DictField(attribute='json_mapping')

class MarkerResource(GeoModelResource):
    class Meta:
        queryset = Marker.objects.all()
        resource_name = 'scout/marker'
        authentication = AnonymousApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True

    data_layer = fields.ToOneField(DataLayerResource, 'datalayer')
    created_by = fields.ToOneField(UserResource, 'created_by', full=True)
    category = fields.ToOneField(MarkerCategoryResource, 'category', full=True)

    def hydrate(self, bundle, request=None):
        if not bundle.obj.pk:
            bundle.data['created_by'] = bundle.request.user

        # Resolve position
        position = bundle.data['position']['coordinates']
        geo_results = Geocoder.reverse_geocode(position[0], position[1])
        if len(geo_results) > 0:
            bundle.data['address'] = geo_results[0]

        return bundle

class PostalAddressResource(ModelResource):
    class Meta:
        queryset = PostalAddress.objects.all()
        resource_name = 'scout/postaladdress'
        always_return_data = True
        authentication = AnonymousApiKeyAuthentication()
        authorization =  DjangoAuthorization()

class PlaceResource(GeoModelResource):
    class Meta:
        queryset = Place.objects.all()
        resource_name = 'scout/place'
        authentication = AnonymousApiKeyAuthentication()
        authorization = DjangoAuthorization()
        always_return_data = True

        filtering = {
            "geo": ALL
        }

    address = fields.ToOneField(PostalAddressResource, 'address', null=True)
<|endoftext|>