<commit_before>import copy

from houston.system import System
from houston.generator.resource import ResourceUsage, ResourceLimits

class MissionGeneratorReport(object):
    """
    Used to provide a summary of a mission generation trial.
    """
    @staticmethod
    def from_json(jsn):
        assert isinstance(jsn, dict)

        jsn = jsn['report']
        history = [h['mission'] for h in jsn['history']]
        outcomes = {h['mission']: h['outcome'] for h in jsn['history']}
        failed = [f['mission'] for f in jsn['failed']]
        result = [Mission.from_json(m) for m in jsn['result']]

        system = System.from_json(jsn['settings']['system'])
        image = jsn['settings']['image']
        resource_usage = ResourceUsage.from_json(jsn['resources']['used'])
        resource_limits = ResourceLimits.from_json(jsn['resources']['limits'])

        return MissionGeneratorReport(system, image, history, outcomes, failed, resource_usage, resource_limits, result)

    def __init__(self, system, image, history, outcomes, failed, resource_usage, resource_limits, result):
        self.__system = system
        self.__image = image
        self.__history = history
        self.__outcomes = outcomes
        self.__failed = failed
        self.__resource_usage = resource_usage
        self.__resource_limits = resource_limits
        self.__result = result

    def outcome(self, mission):
        return self.__outcomes[mission]

    @property
    def system(self):
        return self.__system

    @property
    def image(self):
        return self.__image

    @property
    def outcomes(self):
        return copy.copy(self.__outcomes)

    @property
    def history(self):
        return self.__history[:]

    @property
    def resource_usage(self):
        return self.__resource_usage

    @property
    def resource_limits(self):
        return self.__resource_limits

    @property
    def result(self):
        return self.__result[:]

    def to_json(self):
        history = [(m, self.outcome(m)) for m in self.__history]
        history = [{'mission': m, 'outcome': o} for (m, o) in history]

        failed = [(m, self.outcome(m)) for m in self.__failed]
        failed = [{'mission': m, 'outcome': o} for (m, o) in failed]

        report = {
            'history': history,
            'failed': failed,
            'result': [m.to_json() for m in self.__result],
            'settings': {
                'system': self.system.to_json(),
                'image': self.image
            },
            'resources': {
                'used': self.resource_usage.to_json(),
                'limits': self.resource_limits.to_json()
            }
        }
        return {'report': report}
<commit_msg>fix import typo<commit_after>import copy

from houston.system import System
from houston.generator.resources import ResourceUsage, ResourceLimits

class MissionGeneratorReport(object):
    """
    Used to provide a summary of a mission generation trial.
    """
    @staticmethod
    def from_json(jsn):
        assert isinstance(jsn, dict)

        jsn = jsn['report']
        history = [h['mission'] for h in jsn['history']]
        outcomes = {h['mission']: h['outcome'] for h in jsn['history']}
        failed = [f['mission'] for f in jsn['failed']]
        result = [Mission.from_json(m) for m in jsn['result']]

        system = System.from_json(jsn['settings']['system'])
        image = jsn['settings']['image']
        resource_usage = ResourceUsage.from_json(jsn['resources']['used'])
        resource_limits = ResourceLimits.from_json(jsn['resources']['limits'])

        return MissionGeneratorReport(system, image, history, outcomes, failed, resource_usage, resource_limits, result)

    def __init__(self, system, image, history, outcomes, failed, resource_usage, resource_limits, result):
        self.__system = system
        self.__image = image
        self.__history = history
        self.__outcomes = outcomes
        self.__failed = failed
        self.__resource_usage = resource_usage
        self.__resource_limits = resource_limits
        self.__result = result

    def outcome(self, mission):
        return self.__outcomes[mission]

    @property
    def system(self):
        return self.__system

    @property
    def image(self):
        return self.__image

    @property
    def outcomes(self):
        return copy.copy(self.__outcomes)

    @property
    def history(self):
        return self.__history[:]

    @property
    def resource_usage(self):
        return self.__resource_usage

    @property
    def resource_limits(self):
        return self.__resource_limits

    @property
    def result(self):
        return self.__result[:]

    def to_json(self):
        history = [(m, self.outcome(m)) for m in self.__history]
        history = [{'mission': m, 'outcome': o} for (m, o) in history]

        failed = [(m, self.outcome(m)) for m in self.__failed]
        failed = [{'mission': m, 'outcome': o} for (m, o) in failed]

        report = {
            'history': history,
            'failed': failed,
            'result': [m.to_json() for m in self.__result],
            'settings': {
                'system': self.system.to_json(),
                'image': self.image
            },
            'resources': {
                'used': self.resource_usage.to_json(),
                'limits': self.resource_limits.to_json()
            }
        }
        return {'report': report}
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
import subprocess
import os

_environ = dict(os.environ)
os.environ["PATH"] += os.pathsep + './node_modules/phantomjs/lib'
subprocess.call("./bin/node ./node_modules/testem/testem.js ci", shell=True)
os.environ.clear()
os.environ.update(_environ)
<commit_msg>Fix path updating.<commit_after>#!/usr/bin/env python2.7
import subprocess
import os

_environ = dict(os.environ)
os.environ["PATH"] += os.environ["PATH"] + ':./node_modules/phantomjs/lib'
subprocess.call("./bin/node ./node_modules/testem/testem.js ci", shell=True)
os.environ.clear()
os.environ.update(_environ)
<|endoftext|>


<commit_before>#-- Content-Encoding: UTF-8 --
"""
Created on 29 févr. 2012

:author: Thomas Calmant
"""

# ------------------------------------------------------------------------------

from pelix.ipopo.decorators import ComponentFactory, Provides, Requires, \
    Validate, Invalidate, Instantiate

from base.javautils import to_jabsorb, from_jabsorb, JAVA_CLASS

import pelix.framework as pelix
import pelix.utilities as utilities

# ------------------------------------------------------------------------------

import fnmatch
import logging
import json
import os
import socket
import time
import threading
import sys

if sys.version_info[0] == 3:
    # Python 3
    import http.client as httplib

    def _to_string(data, encoding="UTF-8"):
        """
        Converts the given bytes array to a string
        """
        if type(data) is str:
            # Nothing to do
            return data

        return str(data, encoding)

else:
    # Python 2
    import httplib

    def _to_string(data, encoding="UTF-8"):
        """
        Converts the given bytes array to a string
        """
        return data.encode(encoding)

# ------------------------------------------------------------------------------

_logger = logging.getLogger(__name__)

SPECIAL_INTERNAL_ISOLATES_PREFIX = "org.psem2m.internals.isolates."
SPECIAL_ISOLATE_ID_FORKER = "%s%s" \
                                % (SPECIAL_INTERNAL_ISOLATES_PREFIX, "forker")

# ------------------------------------------------------------------------------

@ComponentFactory("IsolateDirectoryFactory")
@Instantiate("SignalDirectory")
@Provides("org.psem2m.IsolateDirectory")
@Requires("config", "org.psem2m.isolates.services.conf.ISvcConfig")
class IsolateDirectory(object):
    """
    Isolate directory
    """
    def __init__(self):
        """
        Constructor
        """
        self.config = None

    def is_valid_isolate(self, isolate_id):
        """
        Tests if the given isolate ID can be used in a "getAllXXX" method.
        Returns false if the isolate ID is the current one or the forker one.

        :param isolate_id: The isolate ID
        :return: True if the isolate ID can be used"
        """
        if not isolate_id:
            return None

        return isolate_id != SPECIAL_ISOLATE_ID_FORKER \
            and isolate_id != self.get_current_isolate_id()

    def _get_all_isolates(self):
        """
        Retrieves the list of all non-internal isolates, testing whether the ID
        starts with SPECIAL_ISOLATE_ID_FORKER or not.

        :return: All non-internal isolates ID (never None)
        """
        return set([isolate_id
            for isolate_id
            in self.config.get_application().get_isolate_ids()
            if self.is_valid_isolate(isolate_id) \
            and not isolate_id.startswith(SPECIAL_INTERNAL_ISOLATES_PREFIX)])

    def _get_all_monitors(self):
        """
        Retrieves the list of all internal isolates, except the forker, testing
        whether the ID starts with SPECIAL_ISOLATE_ID_FORKER or not.
        """
        return set((isolate_id
                for isolate_id
                in self.config.get_application().get_isolate_ids()
                if self.is_valid_isolate(isolate_id) \
                and isolate_id.startswith(SPECIAL_INTERNAL_ISOLATES_PREFIX)))

    def get_current_isolate_id(self):
        """
        Retrieves the host isolate ID

        :return: The current isolate ID
        """
        return os.getenv("PSEM2M_ISOLATE_ID", "<unknown>")

    def get_isolate(self, isolate_id):
        """
        Retrieves the (host, port) access tuple to the given isolate. Returns
        None if the isolate is unknown. Access to the current isolate and the
        forker can be returned by this method.

        :param isolate_id: An isolate ID
        :return: The (host, port) tuple to access the isolate, or None
        """
        isolate = self.config.get_application().get_isolate(isolate_id)
        if isolate is not None:
            return isolate.get_access()

        return None

    def get_isolates(self, isolates_ids):
        """
        Retrieves the access string of each of the given isolates.
        The parameter can be an array of isolate IDs or one of :

        * "*" or "ALL" : target monitors and isolates
        * "MONITORS" : target monitors only
        * "ISOLATES" : target isolates only

        Unknown isolates are ignored.
        Returns None if all given isolates are unknown.

        Current isolate and the forker (in PSEM2M) access URLs **must not**
        be returned by this method.

        :param isolates_ids: An array of isolate IDs or a target name
        :return: Access strings to the known isolates, null if none is known.
        """
        if not isolates_ids:
            # Don't work for nothing
            return None

        if utilities.is_string(isolates_ids):
            # Targets
            if isolates_ids == "*" or isolates_ids == "ALL":
                # Retrieve all isolates at once
                return self.get_isolates(set((isolate for isolate
                            in self.config.get_application().get_isolate_ids()
                            if self.is_valid_isolate(isolate))))

            elif isolates_ids == "MONITORS":
                # Retrieve all monitors
                return self.get_isolates(self._get_all_monitors())

            elif isolates_ids == "ISOLATES":
                # Retrieve all non-internal isolates
                return self.get_isolates(self._get_all_isolates())

            # Isolate
            url = self.get_isolate(isolates_ids)
            if url is not None:
                return [url]

            return None

        else:
            result = []

            for isolate in isolates_ids:
                # Standard work
                if self.is_valid_isolate(isolate):
                    url = self.get_isolate(isolate)
                    if url is not None:
                        result.append(url)

            if len(result) == 0:
                return None

            return result

    @Validate
    def validate(self, context):
        """
        Component validation
        """
        # Be sure that the configuration is fresh enough
        self.config.refresh()

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidation
        """
        self.config = None

# ------------------------------------------------------------------------------

def read_post_body(request_handler):
    """
    Reads the body of a POST request, returns an empty string on error
    """
    try :
        content_len = int(request_handler.headers.get('content-length'))
        return _to_string(request_handler.rfile.read(content_len))

    except:
        _logger.exception("Error reading POST body")

    return ""

@ComponentFactory("SignalReceiverFactory")
@Instantiate("SignalReceiver")
@Provides("org.psem2m.SignalReceiver")
@Requires("http", "HttpService")
class SignalReceiver(object):
    """
    PSEM2M Signals receiver
    """
    SERVLET_PATH = "/psem2m-signal-receiver/"

    def __init__(self):
        """
        Constructor
        """
        self.http = None
        self._listeners = {}

    @Validate
    def validate(self, context):
        """
        Component validated
        """
        self._listeners.clear()
        self.http.register_servlet(SignalReceiver.SERVLET_PATH, self)

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated
        """
        self.http.unregister_servlet(self)
        self._listeners.clear()

    def do_POST(self, handler):
        """
        Called when a POST request has been received
        """
        # Read the request
        signal_name = handler.path[len(SignalReceiver.SERVLET_PATH) - 1:]

        content_type = handler.headers.get('content-type')
        if content_type not in (None, 'application/json',
                                'application/x-www-form-urlencoded'):
            handler.send_response(500)

        else:
            # Handle it
            try:
                signal_data = from_jabsorb(json.loads(read_post_body(handler)))
                self.handle_received_signal(signal_name, signal_data)

            except:
                # Error
                _logger.exception("Error reading signal %s", signal_name)
                handler.send_response(500)

            else:
                # OK
                handler.send_response(200)

        # Send headers
        handler.send_header('Content-type', 'application/x-www-form-urlencoded')
        handler.send_header('Content-length', '0')
        handler.end_headers()

    def get_access_info(self):
        """
        Retrieves the (host, port) tuple to access this signal receiver

        :return: An (host, port) tuple
        """
        return (self.http.get_hostname(), self.http.get_port())

    def handle_received_signal(self, name, data):
        """
        Handles a received signal
        """
        for pattern in self._listeners.copy():
            if fnmatch.fnmatch(name, pattern):
                # Signal name matches the pattern
                listeners = self._listeners[pattern][:]
                for listener in listeners:
                    try:
                        # Notify the listener
                        listener.handle_received_signal(name, data)

                    except:
                        _logger.exception("Error notifying a signal listener")

    def register_listener(self, signal_pattern, listener):
        """
        Registers a signal listener
        """
        if not signal_pattern or listener is None:
            # Invalid listener
            return

        listeners = self._listeners.get(signal_pattern, None)
        if listeners is None:
            listeners = []
            self._listeners[signal_pattern] = listeners

        listeners.append(listener)

    def unregister_listener(self, signal_pattern, listener):
        """
        Unregisters a listener
        """
        if not signal_pattern or listener is None:
            # Invalid listener
            return

        listeners = self._listeners.get(signal_pattern, None)
        if listeners is not None:
            try:
                listeners.remove(listener)

            except ValueError:
                # Listener not found
                pass

# ------------------------------------------------------------------------------

@ComponentFactory("SignalSenderFactory")
@Instantiate("SignalSender")
@Provides("org.psem2m.SignalSender")
@Requires("directory", "org.psem2m.IsolateDirectory")
@Requires("local_recv", "org.psem2m.SignalReceiver")
class SignalSender(object):
    """
    PSEM2M Signals sender implementation
    """
    def __init__(self):
        """
        Constructor
        """
        self.directory = None

    def _internal_send(self, urls, name, data):
        """
        Sends the data
        """
        signal = {
            JAVA_CLASS: "org.psem2m.remotes.signals.http.HttpSignalData",
            "isolateSender": self.directory.get_current_isolate_id(),
            "senderHostName": "localhost",
            "signalContent": data,
            "timestamp": int(time.time() * 1000)
            }

        # Make a JSON form of a Jabsorb signal content
        json_signal = json.dumps(to_jabsorb(signal))

        # Start a new thread to send the signal
        thread = threading.Thread(target=self.__threaded_send,
                                  args=(urls, name, signal, json_signal))
        thread.start()

    def __threaded_send(self, urls, name, signal, json_signal):
        """
        Sends the data, in a separate thread
        """
        headers = {"Content-Type": "application/json"}

        for access in urls:

            if not access:
                continue

            host, port = access

            try:
                if host == "{local}":
                    _logger.debug("Local signal: %s", name)
                    self.local_recv.handle_received_signal(name, signal)

                else:
                    # 1 second timeout, to avoid useless waits
                    conn = httplib.HTTPConnection(host, port, timeout=1)

                    if name[0] == '/':
                        name = name[1:]

                    signal_url = "%s%s" % (SignalReceiver.SERVLET_PATH, name)

                    conn.request("POST", signal_url, json_signal, headers)
                    response = conn.getresponse()
                    if response.status != 200:
                        _logger.warn("Incorrect response for %s : %s %s",
                                     signal_url, response.status,
                                     response.reason)

            except socket.error as ex:
                # Socket error
                _logger.error("Error sending signal %s to %s : %s", name,
                              access, str(ex))

            except:
                # Other error...
                _logger.exception("Error sending signal %s to %s", name, access)

    def get_current_isolate_id(self):
        """
        Retrieves the current isolate ID
        """
        return self.directory.get_current_isolate_id()

    def send_data(self, target, name, data):
        """
        Sends a signal to the given target

        @param target: Target isolate
        @param name: Signal name
        @param data: Signal content
        """
        if target is None:
            _logger.warn("No target given")
            return

        urls = self.directory.get_isolates(target)
        if urls is None:
            _logger.warn("Unknown target(s) - '%s'", target)
            return

        self._internal_send(urls, name, data)

@ComponentFactory("SlaveAgentFactory")
@Instantiate("SlaveAgent")
@Requires("_receiver", "org.psem2m.SignalReceiver")
class MiniSlaveAgent(object):
    """
    Mini slave agent implementation : stops the framework when the stop signal
    has been received
    """
    ISOLATE_STOP_SIGNAL = "/psem2m/isolate/stop"

    def __init__(self):
        """
        Constructor
        """
        self._receiver = None
        self.context = None

    def handle_received_signal(self, name, data):
        """
        Handles a received signal
        """
        try:
            self.context.get_bundle(0).stop()
        except pelix.BundleException:
            _logger.exception("Error stopping the framework")

    @Validate
    def validate(self, context):
        """
        Component validated

        :param context: The bundle context
        """
        self._receiver.register_listener(MiniSlaveAgent.ISOLATE_STOP_SIGNAL,
                                         self)
        self.context = context

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated

        :param context: The bundle context
        """
        self._receiver.unregister_listener(MiniSlaveAgent.ISOLATE_STOP_SIGNAL,
                                           self)
        self.context = None
<commit_msg>Implémentation Python de InternalsDirectory<commit_after>#-- Content-Encoding: UTF-8 --
"""
Created on 29 févr. 2012

:author: Thomas Calmant
"""

# ------------------------------------------------------------------------------

from pelix.ipopo.decorators import ComponentFactory, Provides, Requires, \
    Validate, Invalidate, Instantiate

from base.javautils import to_jabsorb, from_jabsorb, JAVA_CLASS

import pelix.framework as pelix
import pelix.utilities as utilities

# ------------------------------------------------------------------------------

import fnmatch
import logging
import json
import os
import socket
import time
import threading
import sys

if sys.version_info[0] == 3:
    # Python 3
    import http.client as httplib

    def _to_string(data, encoding="UTF-8"):
        """
        Converts the given bytes array to a string
        """
        if type(data) is str:
            # Nothing to do
            return data

        return str(data, encoding)

else:
    # Python 2
    import httplib

    def _to_string(data, encoding="UTF-8"):
        """
        Converts the given bytes array to a string
        """
        return data.encode(encoding)

# ------------------------------------------------------------------------------

_logger = logging.getLogger(__name__)

SPECIAL_INTERNAL_ISOLATES_PREFIX = "org.psem2m.internals.isolates."
SPECIAL_ISOLATE_ID_FORKER = "%s%s" \
                                % (SPECIAL_INTERNAL_ISOLATES_PREFIX, "forker")

# ------------------------------------------------------------------------------

@ComponentFactory("IsolateDirectoryFactory")
@Instantiate("SignalDirectory")
@Provides("org.psem2m.IsolateDirectory")
@Requires("config", "org.psem2m.isolates.services.conf.ISvcConfig")
class IsolateDirectory(object):
    """
    Isolate directory
    """
    def __init__(self):
        """
        Constructor
        """
        self.config = None

    def is_valid_isolate(self, isolate_id):
        """
        Tests if the given isolate ID can be used in a "getAllXXX" method.
        Returns false if the isolate ID is the current one or the forker one.

        :param isolate_id: The isolate ID
        :return: True if the isolate ID can be used"
        """
        if not isolate_id:
            return None

        return isolate_id != SPECIAL_ISOLATE_ID_FORKER \
            and isolate_id != self.get_current_isolate_id()

    def _get_all_isolates(self):
        """
        Retrieves the list of all non-internal isolates, testing whether the ID
        starts with SPECIAL_ISOLATE_ID_FORKER or not.

        :return: All non-internal isolates ID (never None)
        """
        return set([isolate_id
            for isolate_id
            in self.config.get_application().get_isolate_ids()
            if self.is_valid_isolate(isolate_id) \
            and not isolate_id.startswith(SPECIAL_INTERNAL_ISOLATES_PREFIX)])

    def _get_all_monitors(self):
        """
        Retrieves the list of all internal isolates, except the forker, testing
        whether the ID starts with SPECIAL_ISOLATE_ID_FORKER or not.
        """
        return set((isolate_id
                for isolate_id
                in self.config.get_application().get_isolate_ids()
                if self.is_valid_isolate(isolate_id) \
                and isolate_id.startswith(SPECIAL_INTERNAL_ISOLATES_PREFIX)))

    def get_current_isolate_id(self):
        """
        Retrieves the host isolate ID

        :return: The current isolate ID
        """
        return os.getenv("PSEM2M_ISOLATE_ID", "<unknown>")

    def get_isolate(self, isolate_id):
        """
        Retrieves the (host, port) access tuple to the given isolate. Returns
        None if the isolate is unknown. Access to the current isolate and the
        forker can be returned by this method.

        :param isolate_id: An isolate ID
        :return: The (host, port) tuple to access the isolate, or None
        """
        isolate = self.config.get_application().get_isolate(isolate_id)
        if isolate is not None:
            return isolate.get_access()

        return None

    def get_isolates(self, isolates_ids):
        """
        Retrieves the access string of each of the given isolates.
        The parameter can be an array of isolate IDs or one of :

        * "*" or "ALL" : target monitors and isolates
        * "MONITORS" : target monitors only
        * "ISOLATES" : target isolates only

        Unknown isolates are ignored.
        Returns None if all given isolates are unknown.

        Current isolate and the forker (in PSEM2M) access URLs **must not**
        be returned by this method.

        :param isolates_ids: An array of isolate IDs or a target name
        :return: Access strings to the known isolates, null if none is known.
        """
        if not isolates_ids:
            # Don't work for nothing
            return None

        if utilities.is_string(isolates_ids):
            # Targets
            if isolates_ids == "*" or isolates_ids == "ALL":
                # Retrieve all isolates at once
                return self.get_isolates(set((isolate for isolate
                            in self.config.get_application().get_isolate_ids()
                            if self.is_valid_isolate(isolate))))

            elif isolates_ids == "MONITORS":
                # Retrieve all monitors
                return self.get_isolates(self._get_all_monitors())

            elif isolates_ids == "ISOLATES":
                # Retrieve all non-internal isolates
                return self.get_isolates(self._get_all_isolates())

            # Isolate
            url = self.get_isolate(isolates_ids)
            if url is not None:
                return [url]

            return None

        else:
            result = []

            for isolate in isolates_ids:
                # Standard work
                if self.is_valid_isolate(isolate):
                    url = self.get_isolate(isolate)
                    if url is not None:
                        result.append(url)

            if len(result) == 0:
                return None

            return result

    @Validate
    def validate(self, context):
        """
        Component validation
        """
        # Be sure that the configuration is fresh enough
        self.config.refresh()

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidation
        """
        self.config = None

# ------------------------------------------------------------------------------

@ComponentFactory("InternalDirectoryFactory")
@Instantiate("InternalSignalDirectory")
@Provides("org.psem2m.IsolateDirectory")
@Requires("finder", "org.psem2m.isolates.services.dirs.IFileFinderSvc")
class InternalDirectory(object):
    """
    Reads the internal-directory.js file
    """
    def __init__(self):
        """
        Constructor
        """
        self.finder = None
        self._aliases = {}
        self._directory = {}
        self._hosts = {}

    def get_current_isolate_id(self):
        """
        Retrieves the host isolate ID

        :return: The current isolate ID
        """
        return os.getenv("PSEM2M_ISOLATE_ID", "<unknown>")

    def get_isolate(self, isolate_id):
        """
        Retrieves the (host, port) access tuple to the given isolate. Returns
        None if the isolate is unknown. Access to the current isolate and the
        forker can be returned by this method.

        :param isolate_id: An isolate ID
        :return: The (host, port) tuple to access the isolate, or None
        """
        return self._directory.get(isolate_id, None)

    def get_isolates(self, isolates_ids):
        """
        Retrieves the access string of each of the given isolates.
        The parameter can be an array of isolate IDs or one of :

        * "*" or "ALL" : target monitors and isolates
        * "MONITORS" : target monitors only
        * "ISOLATES" : target isolates only

        Unknown isolates are ignored.
        Returns None if all given isolates are unknown.

        Current isolate and the forker (in PSEM2M) access URLs **must not**
        be returned by this method.

        :param isolates_ids: An array of isolate IDs or a target name
        :return: Access strings to the known isolates, null if none is known.
        """
        if not isolates_ids:
            # Don't work for nothing
            return None

        if utilities.is_string(isolates_ids):
            # Targets
            if isolates_ids == "*" or isolates_ids == "ALL":
                # Retrieve all isolates at once
                return self.get_isolates(set((isolate for isolate
                            in self._directory.keys()
                            if not isolate.startswith("org.psem2m.internals"))))

            elif isolates_ids == "MONITORS":
                # Retrieve all monitors
                return self.get_isolates(set((isolate for isolate
                            in self._directory.keys()
                            if isolate.startswith("org.psem2m.internals.isolates.monitor"))))

            # Isolate
            url = self.get_isolate(isolates_ids)
            if url is not None:
                return [url]

            return None

        else:
            result = []

            for isolate in isolates_ids:
                # Standard work
                if not isolate.startswith("org.psem2m.internals"):
                    url = self.get_isolate(isolate)
                    if url is not None:
                        result.append(url)

            if len(result) == 0:
                return None

            return result

    def __load_directory(self, filename):
        """
        Loads the internal directory
        """
        self._aliases.clear()
        self._directory.clear()
        self._hosts.clear()

        dirfile = self.finder.find(filename)
        if not dirfile:
            _logger.warn("No internal directory file found")
            return

        try:
            with open(dirfile[0], "r") as dirfp:
                root = json.load(dirfp)

        except IOError:
            _logger.exception("Error reading directory '%s'", dirfile)
            return

        for host in root:
            host_content = root[host]

            if utilities.is_string(host_content):
                # Alias
                self._aliases[host] = host_content

            else:
                # Isolates
                host_isolates = []

                for isolate in host_content:
                    self._directory[isolate] = (host, host_content[isolate])
                    host_isolates.append(isolate)

                self._hosts[host] = host_isolates

    @Validate
    def validate(self, context):
        """
        Component validation
        """
        self.__load_directory("conf/internal-directory.js")

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidation
        """
        self._aliases.clear()
        self._directory.clear()
        self._hosts.clear()

# ------------------------------------------------------------------------------

def read_post_body(request_handler):
    """
    Reads the body of a POST request, returns an empty string on error
    """
    try :
        content_len = int(request_handler.headers.get('content-length'))
        return _to_string(request_handler.rfile.read(content_len))

    except:
        _logger.exception("Error reading POST body")

    return ""

@ComponentFactory("SignalReceiverFactory")
@Instantiate("SignalReceiver")
@Provides("org.psem2m.SignalReceiver")
@Requires("http", "HttpService")
class SignalReceiver(object):
    """
    PSEM2M Signals receiver
    """
    SERVLET_PATH = "/psem2m-signal-receiver/"

    def __init__(self):
        """
        Constructor
        """
        self.http = None
        self._listeners = {}

    @Validate
    def validate(self, context):
        """
        Component validated
        """
        self._listeners.clear()
        self.http.register_servlet(SignalReceiver.SERVLET_PATH, self)

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated
        """
        self.http.unregister_servlet(self)
        self._listeners.clear()

    def do_POST(self, handler):
        """
        Called when a POST request has been received
        """
        # Read the request
        signal_name = handler.path[len(SignalReceiver.SERVLET_PATH) - 1:]

        content_type = handler.headers.get('content-type')
        if content_type not in (None, 'application/json',
                                'application/x-www-form-urlencoded'):
            handler.send_response(500)

        else:
            # Handle it
            try:
                signal_data = from_jabsorb(json.loads(read_post_body(handler)))
                self.handle_received_signal(signal_name, signal_data)

            except:
                # Error
                _logger.exception("Error reading signal %s", signal_name)
                handler.send_response(500)

            else:
                # OK
                handler.send_response(200)

        # Send headers
        handler.send_header('Content-type', 'application/x-www-form-urlencoded')
        handler.send_header('Content-length', '0')
        handler.end_headers()

    def get_access_info(self):
        """
        Retrieves the (host, port) tuple to access this signal receiver

        :return: An (host, port) tuple
        """
        return (self.http.get_hostname(), self.http.get_port())

    def handle_received_signal(self, name, data):
        """
        Handles a received signal
        """
        for pattern in self._listeners.copy():
            if fnmatch.fnmatch(name, pattern):
                # Signal name matches the pattern
                listeners = self._listeners[pattern][:]
                for listener in listeners:
                    try:
                        # Notify the listener
                        listener.handle_received_signal(name, data)

                    except:
                        _logger.exception("Error notifying a signal listener")

    def register_listener(self, signal_pattern, listener):
        """
        Registers a signal listener
        """
        if not signal_pattern or listener is None:
            # Invalid listener
            return

        listeners = self._listeners.get(signal_pattern, None)
        if listeners is None:
            listeners = []
            self._listeners[signal_pattern] = listeners

        listeners.append(listener)

    def unregister_listener(self, signal_pattern, listener):
        """
        Unregisters a listener
        """
        if not signal_pattern or listener is None:
            # Invalid listener
            return

        listeners = self._listeners.get(signal_pattern, None)
        if listeners is not None:
            try:
                listeners.remove(listener)

            except ValueError:
                # Listener not found
                pass

# ------------------------------------------------------------------------------

@ComponentFactory("SignalSenderFactory")
@Instantiate("SignalSender")
@Provides("org.psem2m.SignalSender")
@Requires("directories", "org.psem2m.IsolateDirectory", aggregate=True)
@Requires("local_recv", "org.psem2m.SignalReceiver")
class SignalSender(object):
    """
    PSEM2M Signals sender implementation
    """
    def __init__(self):
        """
        Constructor
        """
        self.directories = []

    def _internal_send(self, urls, name, data):
        """
        Sends the data
        """
        signal = {
            JAVA_CLASS: "org.psem2m.remotes.signals.http.HttpSignalData",
            "isolateSender": self.directories[0].get_current_isolate_id(),
            "senderHostName": "localhost",
            "signalContent": data,
            "timestamp": int(time.time() * 1000)
            }

        # Make a JSON form of a Jabsorb signal content
        json_signal = json.dumps(to_jabsorb(signal))

        # Start a new thread to send the signal
        thread = threading.Thread(target=self.__threaded_send,
                                  args=(urls, name, signal, json_signal))
        thread.start()

    def __threaded_send(self, urls, name, signal, json_signal):
        """
        Sends the data, in a separate thread
        """
        headers = {"Content-Type": "application/json"}

        for access in urls:

            if not access:
                continue

            host, port = access

            try:
                if host == "{local}":
                    _logger.debug("Local signal: %s", name)
                    self.local_recv.handle_received_signal(name, signal)

                else:
                    # 1 second timeout, to avoid useless waits
                    conn = httplib.HTTPConnection(host, port, timeout=1)

                    if name[0] == '/':
                        name = name[1:]

                    signal_url = "%s%s" % (SignalReceiver.SERVLET_PATH, name)

                    conn.request("POST", signal_url, json_signal, headers)
                    response = conn.getresponse()
                    if response.status != 200:
                        _logger.warn("Incorrect response for %s : %s %s",
                                     signal_url, response.status,
                                     response.reason)

            except socket.error as ex:
                # Socket error
                _logger.error("Error sending signal %s to %s : %s", name,
                              access, str(ex))

            except:
                # Other error...
                _logger.exception("Error sending signal %s to %s", name, access)

    def get_current_isolate_id(self):
        """
        Retrieves the current isolate ID
        """
        return self.directories[0].get_current_isolate_id()

    def send_data(self, target, name, data):
        """
        Sends a signal to the given target

        @param target: Target isolate
        @param name: Signal name
        @param data: Signal content
        """
        if target is None:
            _logger.warn("No target given")
            return

        for directory in self.directories:
            urls = directory.get_isolates(target)
            if urls is not None:
                self._internal_send(urls, name, data)
                break

        else:
            _logger.warn("Unknown target(s) - '%s'", target)

@ComponentFactory("SlaveAgentFactory")
@Instantiate("SlaveAgent")
@Requires("_receiver", "org.psem2m.SignalReceiver")
class MiniSlaveAgent(object):
    """
    Mini slave agent implementation : stops the framework when the stop signal
    has been received
    """
    ISOLATE_STOP_SIGNAL = "/psem2m/isolate/stop"

    def __init__(self):
        """
        Constructor
        """
        self._receiver = None
        self.context = None

    def handle_received_signal(self, name, data):
        """
        Handles a received signal
        """
        try:
            self.context.get_bundle(0).stop()
        except pelix.BundleException:
            _logger.exception("Error stopping the framework")

    @Validate
    def validate(self, context):
        """
        Component validated

        :param context: The bundle context
        """
        self._receiver.register_listener(MiniSlaveAgent.ISOLATE_STOP_SIGNAL,
                                         self)
        self.context = context

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated

        :param context: The bundle context
        """
        self._receiver.unregister_listener(MiniSlaveAgent.ISOLATE_STOP_SIGNAL,
                                           self)
        self.context = None
<|endoftext|>


<commit_before># This is based upon https://github.com/Aplopio/haystack-queryparser with local customizations

import re
import operator
from datetime import datetime, timedelta
from haystack.query import SQ
from haystack.inputs import Exact, Clean
from django.conf import settings
import logging
logger = logging.getLogger(__name__)

HAYSTACK_DEFAULT_OPERATOR = getattr(settings, 'HAYSTACK_DEFAULT_OPERATOR', 'AND')
# Enable what you need here.

class MatchingBracketsNotFoundError(Exception):
    """ malformed parenthetic expression """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return "Matching brackets were not found: "+self.value

class InequalityNotAllowedError(Exception):
    """ malformed inequality """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return self.value

class FieldNotRecognizedError(Exception):
    """ Attempt to use unregistered field """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return self.value

class MalformedDateError(Exception):
    """ date must be YYYY-MM-DD """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return self.value

def head(string):
    return string.split()[0]

def tail(string):
    return " ".join(string.split()[1:])

def parse_date(word):
    datetime_object = None
    for pattern in ['%m/%d/%Y', '%m/%Y', '%Y', '%Y-%m-%d', '%Y-%m']:
        try:
            datetime_object = datetime.strptime(word, pattern)
            return datetime_object
        except ValueError:
            pass
    raise MalformedDateError("'{}' is not a properly formatted date.".format(word))

class ParseSQ(object):
    """
    Parse a HydroShare query into SOLR form.
    """

    # Translation table for logical connectives
    OP = {
        'AND': operator.and_,
        'OR': operator.or_,
        'NOT': operator.inv,
        # aliases '+' (for include) and '-' (for NOT) removed 4/5/2019
        # due to potential collision with valid titles for resources.
    }

    # Translation table for inequalities
    HAYSTACK_INEQUALITY = {
        ':': '',
        '<': '__lt',
        '>': '__gt',
        '<=': '__lte',
        '>=': '__gte'
    }

    # fields known to SOLR that are reasonably searchable.
    # This omits unindexed fields and JSON fields.
    KNOWN_FIELDS = [
        'person',
        'author',
        'first_author',
        'short_id',
        'doi',
        'title',
        'abstract',
        'creator',
        'contributor',
        'subject',
        'availability',
        'replaced',
        'created',
        'modified',
        'organization',
        'publisher',
        'coverage_type',
        'east',
        'north',
        'northlimit',
        'eastlimit',
        'southlimit',
        'westlimit',
        'start_date',
        'end_date',
        'storage_type',
        'format',
        'identifier',
        'language',
        'source',
        'relation',
        'resource_type',
        'content_type',
        'comment',
        'owner_login',
        'owner',
        'geometry_type',
        'field_name',
        'field_type',
        'field_type_code',
        'variable',
        'variable_type',
        'variable_shape',
        'variable_descriptive_name',
        'variable_speciation',
        'site',
        'method',
        'quality_level',
        'data_source',
        'sample_medium',
        'units',
        'units_type'
    ]

    # This makes the query more friendly to humans
    # The names follow the dublin core names.
    # But humans know 'creator' as 'author'.
    REPLACE_BY = {
        'author': 'creator',
        'first_author': 'author'
    }

    INEQUALITY_FIELDS = [
        'created',
        'modified',
        'east',
        'north',
        'northlimit',
        'eastlimit',
        'southlimit',
        'westlimit',
        'start_date',
        'end_date',
    ]

    DATE_FIELDS = [
        'created',
        'modified',
        'start_date',
        'end_date'
    ]

    # # Optional more precise control of __exact keyword: disabled for now
    # Pattern_Field_Query = re.compile(r"^(\w+):(\w+)\s*", re.U)
    # Pattern_Field_Exact_Query = re.compile(r"^(\w+):\"(.+)\"\s*", re.U)

    # Paterns without more precise control of __exact keyword
    Pattern_Field_Query = re.compile(r"^(\w+)(:|[<>]=?)", re.U)
    Pattern_Normal_Query = re.compile(r"^(\w+)\s*", re.U)
    Pattern_Operator = re.compile(r"^(AND|OR|NOT)\s*", re.U)
    # '-', '+' removed 4/5/2019 to avoid potential collision with valid resource titles
    Pattern_Quoted_Text = re.compile(r"^\"([^\"]*)\"\s*", re.U)
    Pattern_Unquoted_Text = re.compile(r"^(\w*)\s*", re.U)

    def __init__(self, use_default=HAYSTACK_DEFAULT_OPERATOR,
                 handle_logic=False, handle_fields=False):
        self.Default_Operator = use_default
        self.handle_logic = handle_logic
        self.handle_fields = handle_fields

    @property
    def current(self):
        return self._current

    @current.setter
    def current(self, current):
        self._prev = self._current if current in ['NOT'] else None
        self._current = current

    def apply_operand(self, new_sq):
        if self.current in ['NOT']:
            new_sq = self.OP[self.current](new_sq)
            self.current = self._prev
        if self.sq:
            return self.OP[self.current](self.sq, new_sq)
        return new_sq

    def handle_field_query(self):
        mat = re.search(self.Pattern_Field_Query, self.query)
        search_field = mat.group(1)
        search_operator = mat.group(2)
        if search_field not in self.KNOWN_FIELDS:
            self.handle_normal_query()
            return
            # raise FieldNotRecognizedError(
            #     "Field name '{}' is not recognized."
            #     .format(search_field))
        if search_field in self.REPLACE_BY:
            search_field = self.REPLACE_BY[search_field]

        if search_operator != ':' and search_field not in self.INEQUALITY_FIELDS:
            raise InequalityNotAllowedError(
                "Inequality is not meaningful for '{}' qualifier."
                .format(search_field))
        # Strip the qualifier off the query, leaving only the match text
        self.query = re.sub(self.Pattern_Field_Query, '', self.query, 1)

        mat = re.search(self.Pattern_Quoted_Text, self.query)
        if mat:
            text_in_quotes = mat.group(1)
            if (search_operator != ':'):
                raise InequalityNotAllowedError(
                    "Inequality is not meaningful for quoted text \"{}\"."
                    .format(text_in_quotes))
            self.sq = self.apply_operand(SQ(**{search_field: Exact(text_in_quotes)}))
            # remove quoted text from query
            self.query = re.sub(self.Pattern_Quoted_Text, '', self.query, 1)
        else:  # no quotes
            word = head(self.query)  # This has no field specifier

            # Append __lt, __lte, etc to query as needed
            inequality_qualifier = self.HAYSTACK_INEQUALITY[search_operator]

            # Parse date in one of a limited number of common formats.
            # TODO: SOLR requires GMT; convert from GMT to local locale for date.
            if search_field in self.DATE_FIELDS:
                thisday_object = parse_date(word)
                thisday = thisday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                if search_operator == ':':
                    # limit creation date to one day by generating two inequalities
                    nextday_object = thisday_object + timedelta(days=1)
                    nextday = nextday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                    self.sq = self.apply_operand(SQ(**{search_field+'__gte': thisday}) &
                                                 SQ(**{search_field+'__lt': nextday}))
                elif search_operator == '<=':  # include whole day of target date
                    nextday_object = thisday_object + timedelta(days=1)
                    nextday = nextday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                    self.sq = self.apply_operand(
                        SQ(**{search_field+"__lt": nextday}))
                elif search_operator == '>':  # include whole day of target date
                    nextday_object = thisday_object + timedelta(days=1)
                    nextday = nextday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                    self.sq = self.apply_operand(
                        SQ(**{search_field+"__gte": nextday}))
                else:
                    self.sq = self.apply_operand(
                        SQ(**{search_field+inequality_qualifier: thisday}))
            else:
                self.sq = self.apply_operand(SQ(**{search_field+inequality_qualifier: word}))
            # remove unquoted text from query
            self.query = tail(self.query)

        self.current = self.Default_Operator

    def handle_brackets(self):
        no_brackets = 1
        i = 1
        assert self.query[0] == "("
        while no_brackets and i < len(self.query):
            if self.query[i] == ")":
                no_brackets -= 1
            elif self.query[i] == "(":
                no_brackets += 1
            i += 1
        if not no_brackets:
            parser = ParseSQ(use_default=self.Default_Operator,
                             handle_logic=self.handle_logic,
                             handle_fields=self.handle_fields)
            self.sq = self.apply_operand(parser.parse(self.query[1:i-1]))
        else:
            raise MatchingBracketsNotFoundError("Parentheses must match in '{}'."
                                                .format(self.query))
        self.query, self.current = self.query[i:], self.Default_Operator

    def handle_normal_query(self):
        word = head(self.query)
        word = word.replace('-', ' ')
        self.sq = self.apply_operand(SQ(content=Clean(word)))
        self.current = self.Default_Operator
        self.query = tail(self.query)

    def handle_operator_query(self):
        self.current = re.search(self.Pattern_Operator, self.query).group(1)
        self.query, n = re.subn(self.Pattern_Operator, '', self.query, 1)

    def handle_quoted_query(self):
        mat = re.search(self.Pattern_Quoted_Text, self.query)
        text_in_quotes = mat.group(1)
        # it seams that haystack exact only works if there is a space in the query.So adding a space
        # if not re.search(r'\s', text_in_quotes):
        #     text_in_quotes+=" "
        self.sq = self.apply_operand(SQ(content=Exact(text_in_quotes)))
        self.query, n = re.subn(self.Pattern_Quoted_Text, '', self.query, 1)
        self.current = self.Default_Operator

    def parse(self, query):
        """
        Parse a textual query into phrases, and interpret each phrase

        This can raise ValueError if the values passed are not valid.
        """
        self.query = query
        self.sq = SQ()
        self.current = self.Default_Operator
        while self.query:
            self.query = self.query.lstrip()
            if self.handle_fields and re.search(self.Pattern_Field_Query, self.query):
                self.handle_field_query()
            elif re.search(self.Pattern_Quoted_Text, self.query):
                self.handle_quoted_query()
            elif self.handle_logic and re.search(self.Pattern_Operator, self.query):
                self.handle_operator_query()
            elif re.search(self.Pattern_Normal_Query, self.query):
                self.handle_normal_query()
            elif self.handle_logic and self.query and self.query[0] == "(":
                self.handle_brackets()
            else:
                self.handle_normal_query()
        return self.sq
<commit_msg>Pass flake8<commit_after># This is based upon https://github.com/Aplopio/haystack-queryparser with local customizations

import re
import operator
from datetime import datetime, timedelta
from haystack.query import SQ
from haystack.inputs import Exact, Clean
from django.conf import settings
import logging
logger = logging.getLogger(__name__)

HAYSTACK_DEFAULT_OPERATOR = getattr(settings, 'HAYSTACK_DEFAULT_OPERATOR', 'AND')
# Enable what you need here.

class MatchingBracketsNotFoundError(Exception):
    """ malformed parenthetic expression """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return "Matching brackets were not found: "+self.value

class InequalityNotAllowedError(Exception):
    """ malformed inequality """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return self.value

class FieldNotRecognizedError(Exception):
    """ Attempt to use unregistered field """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return self.value

class MalformedDateError(Exception):
    """ date must be YYYY-MM-DD """

    def __init__(self, value=''):
        self.value = value

    def __str__(self):
        return self.value

def head(string):
    return string.split()[0]

def tail(string):
    return " ".join(string.split()[1:])

def parse_date(word):
    datetime_object = None
    for pattern in ['%m/%d/%Y', '%m/%Y', '%Y', '%Y-%m-%d', '%Y-%m']:
        try:
            datetime_object = datetime.strptime(word, pattern)
            return datetime_object
        except ValueError:
            pass
    raise MalformedDateError("'{}' is not a properly formatted date.".format(word))

class ParseSQ(object):
    """
    Parse a HydroShare query into SOLR form.
    """

    # Translation table for logical connectives
    OP = {
        'AND': operator.and_,
        'OR': operator.or_,
        'NOT': operator.inv,
        # aliases '+' (for include) and '-' (for NOT) removed 4/5/2019
        # due to potential collision with valid titles for resources.
    }

    # Translation table for inequalities
    HAYSTACK_INEQUALITY = {
        ':': '',
        '<': '__lt',
        '>': '__gt',
        '<=': '__lte',
        '>=': '__gte'
    }

    # fields known to SOLR that are reasonably searchable.
    # This omits unindexed fields and JSON fields.
    KNOWN_FIELDS = [
        'person',
        'author',
        'first_author',
        'short_id',
        'doi',
        'title',
        'abstract',
        'creator',
        'contributor',
        'subject',
        'availability',
        'replaced',
        'created',
        'modified',
        'organization',
        'publisher',
        'coverage_type',
        'east',
        'north',
        'northlimit',
        'eastlimit',
        'southlimit',
        'westlimit',
        'start_date',
        'end_date',
        'storage_type',
        'format',
        'identifier',
        'language',
        'source',
        'relation',
        'resource_type',
        'content_type',
        'comment',
        'owner_login',
        'owner',
        'geometry_type',
        'field_name',
        'field_type',
        'field_type_code',
        'variable',
        'variable_type',
        'variable_shape',
        'variable_descriptive_name',
        'variable_speciation',
        'site',
        'method',
        'quality_level',
        'data_source',
        'sample_medium',
        'units',
        'units_type'
    ]

    # This makes the query more friendly to humans
    # The names follow the dublin core names.
    # But humans know 'creator' as 'author'.
    REPLACE_BY = {
        'author': 'creator',
        'first_author': 'author'
    }

    INEQUALITY_FIELDS = [
        'created',
        'modified',
        'east',
        'north',
        'northlimit',
        'eastlimit',
        'southlimit',
        'westlimit',
        'start_date',
        'end_date',
    ]

    DATE_FIELDS = [
        'created',
        'modified',
        'start_date',
        'end_date'
    ]

    # # Optional more precise control of __exact keyword: disabled for now
    # Pattern_Field_Query = re.compile(r"^(\w+):(\w+)\s*", re.U)
    # Pattern_Field_Exact_Query = re.compile(r"^(\w+):\"(.+)\"\s*", re.U)

    # Paterns without more precise control of __exact keyword
    Pattern_Field_Query = re.compile(r"^(\w+)(:|[<>]=?)", re.U)
    Pattern_Normal_Query = re.compile(r"^(\w+)\s*", re.U)
    Pattern_Operator = re.compile(r"^(AND|OR|NOT)\s*", re.U)
    # '-', '+' removed 4/5/2019 to avoid potential collision with valid resource titles
    Pattern_Quoted_Text = re.compile(r"^\"([^\"]*)\"\s*", re.U)
    Pattern_Unquoted_Text = re.compile(r"^(\w*)\s*", re.U)

    def __init__(self, use_default=HAYSTACK_DEFAULT_OPERATOR,
                 handle_logic=False, handle_fields=False):
        self.Default_Operator = use_default
        self.handle_logic = handle_logic
        self.handle_fields = handle_fields

    @property
    def current(self):
        return self._current

    @current.setter
    def current(self, current):
        self._prev = self._current if current in ['NOT'] else None
        self._current = current

    def apply_operand(self, new_sq):
        if self.current in ['NOT']:
            new_sq = self.OP[self.current](new_sq)
            self.current = self._prev
        if self.sq:
            return self.OP[self.current](self.sq, new_sq)
        return new_sq

    def handle_field_query(self):
        mat = re.search(self.Pattern_Field_Query, self.query)
        search_field = mat.group(1)
        search_operator = mat.group(2)
        if search_field not in self.KNOWN_FIELDS:
            self.handle_normal_query()
            return
            # raise FieldNotRecognizedError(
            #     "Field name '{}' is not recognized."
            #     .format(search_field))
        if search_field in self.REPLACE_BY:
            search_field = self.REPLACE_BY[search_field]

        if search_operator != ':' and search_field not in self.INEQUALITY_FIELDS:
            raise InequalityNotAllowedError(
                "Inequality is not meaningful for '{}' qualifier."
                .format(search_field))
        # Strip the qualifier off the query, leaving only the match text
        self.query = re.sub(self.Pattern_Field_Query, '', self.query, 1)

        mat = re.search(self.Pattern_Quoted_Text, self.query)
        if mat:
            text_in_quotes = mat.group(1)
            if (search_operator != ':'):
                raise InequalityNotAllowedError(
                    "Inequality is not meaningful for quoted text \"{}\"."
                    .format(text_in_quotes))
            self.sq = self.apply_operand(SQ(**{search_field: Exact(text_in_quotes)}))
            # remove quoted text from query
            self.query = re.sub(self.Pattern_Quoted_Text, '', self.query, 1)
        else:  # no quotes
            word = head(self.query)  # This has no field specifier

            # Append __lt, __lte, etc to query as needed
            inequality_qualifier = self.HAYSTACK_INEQUALITY[search_operator]

            # Parse date in one of a limited number of common formats.
            # TODO: SOLR requires GMT; convert from GMT to local locale for date.
            if search_field in self.DATE_FIELDS:
                thisday_object = parse_date(word)
                thisday = thisday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                if search_operator == ':':
                    # limit creation date to one day by generating two inequalities
                    nextday_object = thisday_object + timedelta(days=1)
                    nextday = nextday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                    self.sq = self.apply_operand(SQ(**{search_field+'__gte': thisday}) &
                                                 SQ(**{search_field+'__lt': nextday}))
                elif search_operator == '<=':  # include whole day of target date
                    nextday_object = thisday_object + timedelta(days=1)
                    nextday = nextday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                    self.sq = self.apply_operand(
                        SQ(**{search_field+"__lt": nextday}))
                elif search_operator == '>':  # include whole day of target date
                    nextday_object = thisday_object + timedelta(days=1)
                    nextday = nextday_object.strftime("%Y-%m-%dT%H:%M:%SZ")
                    self.sq = self.apply_operand(
                        SQ(**{search_field+"__gte": nextday}))
                else:
                    self.sq = self.apply_operand(
                        SQ(**{search_field+inequality_qualifier: thisday}))
            else:
                self.sq = self.apply_operand(SQ(**{search_field+inequality_qualifier: word}))
            # remove unquoted text from query
            self.query = tail(self.query)

        self.current = self.Default_Operator

    def handle_brackets(self):
        no_brackets = 1
        i = 1
        assert self.query[0] == "("
        while no_brackets and i < len(self.query):
            if self.query[i] == ")":
                no_brackets -= 1
            elif self.query[i] == "(":
                no_brackets += 1
            i += 1
        if not no_brackets:
            parser = ParseSQ(use_default=self.Default_Operator,
                             handle_logic=self.handle_logic,
                             handle_fields=self.handle_fields)
            self.sq = self.apply_operand(parser.parse(self.query[1:i-1]))
        else:
            raise MatchingBracketsNotFoundError("Parentheses must match in '{}'."
                                                .format(self.query))
        self.query, self.current = self.query[i:], self.Default_Operator

    def handle_normal_query(self):
        word = head(self.query)
        word = word.replace('-', ' ')
        self.sq = self.apply_operand(SQ(content=Clean(word)))
        self.current = self.Default_Operator
        self.query = tail(self.query)

    def handle_operator_query(self):
        self.current = re.search(self.Pattern_Operator, self.query).group(1)
        self.query, n = re.subn(self.Pattern_Operator, '', self.query, 1)

    def handle_quoted_query(self):
        mat = re.search(self.Pattern_Quoted_Text, self.query)
        text_in_quotes = mat.group(1)
        # it seams that haystack exact only works if there is a space in the query.So adding a space
        # if not re.search(r'\s', text_in_quotes):
        #     text_in_quotes+=" "
        self.sq = self.apply_operand(SQ(content=Exact(text_in_quotes)))
        self.query, n = re.subn(self.Pattern_Quoted_Text, '', self.query, 1)
        self.current = self.Default_Operator

    def parse(self, query):
        """
        Parse a textual query into phrases, and interpret each phrase

        This can raise ValueError if the values passed are not valid.
        """
        self.query = query
        self.sq = SQ()
        self.current = self.Default_Operator
        while self.query:
            self.query = self.query.lstrip()
            if self.handle_fields and re.search(self.Pattern_Field_Query, self.query):
                self.handle_field_query()
            elif re.search(self.Pattern_Quoted_Text, self.query):
                self.handle_quoted_query()
            elif self.handle_logic and re.search(self.Pattern_Operator, self.query):
                self.handle_operator_query()
            elif re.search(self.Pattern_Normal_Query, self.query):
                self.handle_normal_query()
            elif self.handle_logic and self.query and self.query[0] == "(":
                self.handle_brackets()
            else:
                self.handle_normal_query()
        return self.sq
<|endoftext|>


<commit_before>#!/usr/bin/env python

import optparse
import sys

def main():
    empty_subtitle = '''1
00:00:00,000 --> 00:00:00,001

'''

    parser, infile_name, outfile_name = parse_options()

    with open(infile_name) as infile, open(outfile_name, 'w') as outfile:
        '''
        Add an empty subtitle at the beginning of the file to fix avconv subtitle
        offset issues. Also makes sure subtitle numbering starts at 1. Starting at 0
        causes avconv to fail importing the subtitle.
        '''
        outfile.write(empty_subtitle)

        # Renumber remaining subtitles
        subtitle_number = 2
        prev_line = ''
        for line in infile:
            line = line.strip()
            # Optionally reencode subtitles as utf8
            if parser.values.ensure_utf8 == True:
                try:
                    line = line.decode('utf8').encode('utf8')
                except UnicodeDecodeError:
                    line = line.decode('latin1').encode('utf8')
            if prev_line == '':
                line = str(subtitle_number)
                subtitle_number += 1
            '''
            \n is apparently platform-independent
            (https://docs.python.org/2/library/os.html#os.linesep)
            '''
            outfile.write('{0}\n'.format(line))
            prev_line = line

def parse_options():
    ''' set up and parse command line arguments
    '''

    # define a custom usage message
    usage = ('usage: %prog INPUT_FILE OUTPUT_FILE [options]\n'
    '\tWhere INPUT_FILE = path to SRT input file\n'
    '\tand OUTPUT_FILE = path to SRT output file')

    parser = optparse.OptionParser(usage=usage)

    # command line options to parse
    parser.add_option(
        '-u', '--ensure-utf8', action='store_true', dest='ensure_utf8',
        default=False, help='Try to ensure the output file is UTF8-encoded'
    )

    # parse the arguments
    (options, args) = parser.parse_args()

    if len(args) < 2:
        parser.print_help()
        sys.exit('Error: INPUT_FILE and OUTPUT_FILE are required')
    infile_name = args[0]
    outfile_name = args[1]

    return parser, infile_name, outfile_name

if __name__ == '__main__':
    main()
<commit_msg>Remove trailing spaces<commit_after>#!/usr/bin/env python

import optparse
import sys

def main():
    empty_subtitle = '''1
00:00:00,000 --> 00:00:00,001

'''

    parser, infile_name, outfile_name = parse_options()

    with open(infile_name) as infile, open(outfile_name, 'w') as outfile:
        '''
        Add an empty subtitle at the beginning of the file to fix avconv subtitle
        offset issues. Also makes sure subtitle numbering starts at 1. Starting at 0
        causes avconv to fail importing the subtitle.
        '''
        outfile.write(empty_subtitle)

        # Renumber remaining subtitles
        subtitle_number = 2
        prev_line = ''
        for line in infile:
            line = line.strip()
            # Optionally reencode subtitles as utf8
            if parser.values.ensure_utf8 == True:
                try:
                    line = line.decode('utf8').encode('utf8')
                except UnicodeDecodeError:
                    line = line.decode('latin1').encode('utf8')
            if prev_line == '':
                line = str(subtitle_number)
                subtitle_number += 1
            '''
            \n is apparently platform-independent
            (https://docs.python.org/2/library/os.html#os.linesep)
            '''
            outfile.write('{0}\n'.format(line))
            prev_line = line

def parse_options():
    ''' set up and parse command line arguments
    '''

    # define a custom usage message
    usage = ('usage: %prog INPUT_FILE OUTPUT_FILE [options]\n'
    '\tWhere INPUT_FILE = path to SRT input file\n'
    '\tand OUTPUT_FILE = path to SRT output file')

    parser = optparse.OptionParser(usage=usage)

    # command line options to parse
    parser.add_option(
        '-u', '--ensure-utf8', action='store_true', dest='ensure_utf8',
        default=False, help='Try to ensure the output file is UTF8-encoded'
    )

    # parse the arguments
    (options, args) = parser.parse_args()

    if len(args) < 2:
        parser.print_help()
        sys.exit('Error: INPUT_FILE and OUTPUT_FILE are required')
    infile_name = args[0]
    outfile_name = args[1]

    return parser, infile_name, outfile_name

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Basically, this is currently just a class of static utility methods to deal
with term identifiers.

Assume everything in this file is Waterloo-specific.
"""

import mongoengine as me
import logging

import rmc.shared.util as util

# TODO(david): Make naming consistent... prefix all methods with a verb (eg.
#     get).
# TODO(david): Lots of easy unit tests for methods here.

# TODO(jlfwong): From what I can tell, this isn't being used as an actual model
# any more - it's just being used as a collection of static methods
class Term(me.Document):

    SHORTLIST_TERM_ID = '9999_99'

    SEASONS = ['Winter', 'Spring', 'Fall']
    INVALID_TERM_MONTH = 13

    # TODO(mack): should month be 0 or 1 offset?
    # eg. 2011_05 => term that began May, 2011 => Spring 2011
    id = me.StringField(primary_key=True)

    @staticmethod
    def get_year_from_id(tid):
        return int(tid[:4])

    @property
    def year(self):
        return Term.get_year_from_id(self.id)

    @staticmethod
    def get_month_from_id(term_id):
        return int(term_id[5:])

    @staticmethod
    def season_from_id(tid):
        # currently specific to waterloo
        month = Term.get_month_from_id(tid)
        if (month == Term.INVALID_TERM_MONTH):
            return ''
        return Term.SEASONS[(month-1) / 4]

    @property
    def season(self):
        return Term.season_from_id(self.id)

    @staticmethod
    def name_from_id(tid):
        if tid == Term.SHORTLIST_TERM_ID:
            return 'Shortlist'
        else:
            return '%s %d' % (Term.season_from_id(tid),
                    Term.get_year_from_id(tid))

    @property
    def name(self):
        return Term.name_from_id(self.id)

    @staticmethod
    def get_id_from_year_month(year, month):
        if month not in [1, 5, 9, Term.INVALID_TERM_MONTH]:
            raise "Invalid month: %s" % month

        return '%s_%02d' % (year, month)

    @staticmethod
    def get_id_from_year_season(year, season):
        season = season.lower()
        month = Term.INVALID_TERM_MONTH
        for idx, ssn in enumerate(Term.SEASONS):
            if ssn.lower() == season:
                month = idx * 4 + 1

        if month == Term.INVALID_TERM_MONTH:
            logging.warn("Term: Invalid seasons '%s'. Using month %d in term id" % (season, Term.INVALID_TERM_MONTH))

        return Term.get_id_from_year_month(year, month)

    @staticmethod
    def id_from_name(name):
        try:
            season, year = name.split()
        except ValueError:
            logging.error("term.py: id_from_name(cls, '%s'). Fix me!" % (name))
            # Special place holder so we can correct it later
            return '8888_88'

        return Term.id_from_year_season(year, season)

    @classmethod
    def is_shortlist_term(cls, term_id):
        return term_id == Term.SHORTLIST_TERM_ID

    @staticmethod
    def get_current_term_id():
        return util.get_current_term_id()

    @staticmethod
    def get_next_term_id():
        return Term.get_next_term_id_from_term_id(Term.get_current_term_id())

    @staticmethod
    def get_next_term_id_from_year_month(year, month):
        year = int(year)
        month = int(month)

        if month == 9:
            year += 1
            month = 1
        else:
            month += 4

        return Term.get_id_from_year_month(year, month)

    @staticmethod
    def get_next_term_id_from_term_id(term_id):
        year = Term.get_year_from_id(term_id)
        month = Term.get_month_from_id(term_id)
        return Term.get_next_term_id_from_year_month(year, month)

    @staticmethod
    def get_quest_id_from_term_id(term_id):
        """Convert a term ID from our format to Quest's funky 4-digit code. Eg.
        2013_09 => 1139, 2014_01 => 1141.
        """
        year = Term.get_year_from_id(term_id)
        month = Term.get_month_from_id(term_id)
        return '%03d%d' % (year - 1900, month)

    @staticmethod
    def get_term_id_from_quest_id(quest_term_id):
        """Convert Quest's funky 4-digit term code to our term ID.
        Eg. 1139 => 2013_09, 1141 => 2014_01.
        """
        quest_term_id = int(quest_term_id)
        return '%04d_%02d' % (1900 + (quest_term_id / 10), quest_term_id % 10)
<commit_msg>fix broken schedule import<commit_after>"""Basically, this is currently just a class of static utility methods to deal
with term identifiers.

Assume everything in this file is Waterloo-specific.
"""

import mongoengine as me
import logging

import rmc.shared.util as util

# TODO(david): Make naming consistent... prefix all methods with a verb (eg.
#     get).
# TODO(david): Lots of easy unit tests for methods here.

# TODO(jlfwong): From what I can tell, this isn't being used as an actual model
# any more - it's just being used as a collection of static methods
class Term(me.Document):

    SHORTLIST_TERM_ID = '9999_99'

    SEASONS = ['Winter', 'Spring', 'Fall']
    INVALID_TERM_MONTH = 13

    # TODO(mack): should month be 0 or 1 offset?
    # eg. 2011_05 => term that began May, 2011 => Spring 2011
    id = me.StringField(primary_key=True)

    @staticmethod
    def get_year_from_id(tid):
        return int(tid[:4])

    @property
    def year(self):
        return Term.get_year_from_id(self.id)

    @staticmethod
    def get_month_from_id(term_id):
        return int(term_id[5:])

    @staticmethod
    def season_from_id(tid):
        # currently specific to waterloo
        month = Term.get_month_from_id(tid)
        if (month == Term.INVALID_TERM_MONTH):
            return ''
        return Term.SEASONS[(month-1) / 4]

    @property
    def season(self):
        return Term.season_from_id(self.id)

    @staticmethod
    def name_from_id(tid):
        if tid == Term.SHORTLIST_TERM_ID:
            return 'Shortlist'
        else:
            return '%s %d' % (Term.season_from_id(tid),
                    Term.get_year_from_id(tid))

    @property
    def name(self):
        return Term.name_from_id(self.id)

    @staticmethod
    def get_id_from_year_month(year, month):
        if month not in [1, 5, 9, Term.INVALID_TERM_MONTH]:
            raise "Invalid month: %s" % month

        return '%s_%02d' % (year, month)

    @staticmethod
    def get_id_from_year_season(year, season):
        season = season.lower()
        month = Term.INVALID_TERM_MONTH
        for idx, ssn in enumerate(Term.SEASONS):
            if ssn.lower() == season:
                month = idx * 4 + 1

        if month == Term.INVALID_TERM_MONTH:
            logging.warn("Term: Invalid seasons '%s'. Using month %d in term id" % (season, Term.INVALID_TERM_MONTH))

        return Term.get_id_from_year_month(year, month)

    @staticmethod
    def id_from_name(name):
        try:
            season, year = name.split()
        except ValueError:
            logging.error("term.py: id_from_name(cls, '%s'). Fix me!" % (name))
            # Special place holder so we can correct it later
            return '8888_88'

        return Term.get_id_from_year_season(year, season)

    @classmethod
    def is_shortlist_term(cls, term_id):
        return term_id == Term.SHORTLIST_TERM_ID

    @staticmethod
    def get_current_term_id():
        return util.get_current_term_id()

    @staticmethod
    def get_next_term_id():
        return Term.get_next_term_id_from_term_id(Term.get_current_term_id())

    @staticmethod
    def get_next_term_id_from_year_month(year, month):
        year = int(year)
        month = int(month)

        if month == 9:
            year += 1
            month = 1
        else:
            month += 4

        return Term.get_id_from_year_month(year, month)

    @staticmethod
    def get_next_term_id_from_term_id(term_id):
        year = Term.get_year_from_id(term_id)
        month = Term.get_month_from_id(term_id)
        return Term.get_next_term_id_from_year_month(year, month)

    @staticmethod
    def get_quest_id_from_term_id(term_id):
        """Convert a term ID from our format to Quest's funky 4-digit code. Eg.
        2013_09 => 1139, 2014_01 => 1141.
        """
        year = Term.get_year_from_id(term_id)
        month = Term.get_month_from_id(term_id)
        return '%03d%d' % (year - 1900, month)

    @staticmethod
    def get_term_id_from_quest_id(quest_term_id):
        """Convert Quest's funky 4-digit term code to our term ID.
        Eg. 1139 => 2013_09, 1141 => 2014_01.
        """
        quest_term_id = int(quest_term_id)
        return '%04d_%02d' % (1900 + (quest_term_id / 10), quest_term_id % 10)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import unittest
import uuid

from mock import patch

import requests
import responses
from six.moves.urllib import parse

from ..events import Event
from ..tracker import _AnalyticsHandler, Tracker
from ..utils import get_user_agent

class TestAnalyticsHandler(unittest.TestCase):

    def test_default_user_agent(self):
        # Given
        handler = _AnalyticsHandler()

        # Then
        user_agent = handler._session.headers['User-Agent']
        self.assertRegex(user_agent, r'^python-analytics/')
        self.assertEqual(user_agent, get_user_agent(None))

    def test_override_user_agent(self):
        # Given
        session = requests.Session()
        session.headers['User-Agent'] = 'MyAgent/1.0'
        handler = _AnalyticsHandler(session=session)

        # Then
        user_agent = handler._session.headers['User-Agent']
        self.assertRegex(
            user_agent, r'^python-analytics/[^ ]+ MyAgent/1.0')
        self.assertEqual(user_agent, get_user_agent('MyAgent/1.0'))

    @responses.activate
    def test_send_analytics(self):
        # Given
        responses.add(
            responses.POST,
            _AnalyticsHandler.target,
            status=200,
        )
        uid = str(uuid.uuid4())
        expected = {
            'v': ['1'],
            'tid': ['GA-ID'],
            'cid': [uid],
        }

        handler = _AnalyticsHandler()
        data = {
            'v': 1,
            'tid': 'GA-ID',
            'cid': uid,
        }

        # When
        handler.send(data)

        # Then
        self.assertEqual(len(responses.calls), 1)
        call, = responses.calls
        request, response = call
        sent_data = parse.parse_qs(request.body)
        self.assertEqual(sent_data, expected)

class TestTracker(unittest.TestCase):
    maxDiff = None

    @patch('uuid.uuid4')
    @responses.activate
    def test_tracker(self, uuid4):
        # Given
        responses.add(
            responses.POST,
            _AnalyticsHandler.target,
            status=200,
        )

        my_uuid = 'my-uuid'
        uuid4.return_value = my_uuid
        category = 'category'
        action = 'action'
        tracker = Tracker('GA-ID')
        event = Event(category=category, action=action)
        expected = {
            'v': ['1'],
            'tid': ['GA-ID'],
            'cid': [my_uuid],
            't': ['event'],
            'ec': [category],
            'ea': [action],
        }

        # When
        tracker.send(event)

        # Then
        self.assertEqual(len(responses.calls), 1)
        call, = responses.calls
        request, response = call
        sent_data = parse.parse_qs(request.body)
        self.assertEqual(sent_data, expected)
<commit_msg>Add python-2 compat code<commit_after>from __future__ import absolute_import, unicode_literals

import unittest
import uuid

from mock import patch

import requests
import responses
from six import PY2
from six.moves.urllib import parse

from ..events import Event
from ..tracker import _AnalyticsHandler, Tracker
from ..utils import get_user_agent

class TestAnalyticsHandler(unittest.TestCase):

    if PY2:
        assertRegex = unittest.TestCase.assertRegexpMatches

    def test_default_user_agent(self):
        # Given
        handler = _AnalyticsHandler()

        # Then
        user_agent = handler._session.headers['User-Agent']
        self.assertRegex(user_agent, r'^python-analytics/')
        self.assertEqual(user_agent, get_user_agent(None))

    def test_override_user_agent(self):
        # Given
        session = requests.Session()
        session.headers['User-Agent'] = 'MyAgent/1.0'
        handler = _AnalyticsHandler(session=session)

        # Then
        user_agent = handler._session.headers['User-Agent']
        self.assertRegex(
            user_agent, r'^python-analytics/[^ ]+ MyAgent/1.0')
        self.assertEqual(user_agent, get_user_agent('MyAgent/1.0'))

    @responses.activate
    def test_send_analytics(self):
        # Given
        responses.add(
            responses.POST,
            _AnalyticsHandler.target,
            status=200,
        )
        uid = str(uuid.uuid4())
        expected = {
            'v': ['1'],
            'tid': ['GA-ID'],
            'cid': [uid],
        }

        handler = _AnalyticsHandler()
        data = {
            'v': 1,
            'tid': 'GA-ID',
            'cid': uid,
        }

        # When
        handler.send(data)

        # Then
        self.assertEqual(len(responses.calls), 1)
        call, = responses.calls
        request, response = call
        sent_data = parse.parse_qs(request.body)
        self.assertEqual(sent_data, expected)

class TestTracker(unittest.TestCase):
    maxDiff = None

    @patch('uuid.uuid4')
    @responses.activate
    def test_tracker(self, uuid4):
        # Given
        responses.add(
            responses.POST,
            _AnalyticsHandler.target,
            status=200,
        )

        my_uuid = 'my-uuid'
        uuid4.return_value = my_uuid
        category = 'category'
        action = 'action'
        tracker = Tracker('GA-ID')
        event = Event(category=category, action=action)
        expected = {
            'v': ['1'],
            'tid': ['GA-ID'],
            'cid': [my_uuid],
            't': ['event'],
            'ec': [category],
            'ea': [action],
        }

        # When
        tracker.send(event)

        # Then
        self.assertEqual(len(responses.calls), 1)
        call, = responses.calls
        request, response = call
        sent_data = parse.parse_qs(request.body)
        self.assertEqual(sent_data, expected)
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Module containing the base test class and functions
used by the unittests.
"""

import os
import sys
import time
from functools import wraps

if sys.version_info[0:2] < (2, 7):
    import unittest2 as unittest
else:
    import unittest

try:
    import json
except ImportError:
    import simplejson as json

from nose.plugins.skip import SkipTest
from pyfarm.core.config import cfg

# Some initial configuration values before we load the models.  Some values,
# such as the table prefix are included here just so two tests don't step
# on each other (though in production it would be better to use a different DB).
cfg.update({
    "db.table_prefix": "pyfarm_unittest_%s_" % time.strftime("%M%d%Y%H%M%S"),
    "agent.min_port": 1025,
    "agent.max_port": 65535,
    "agent.min_cpus": 1,
    "agent.max_cpus": 2147483647,
    "agent.special_cpus": [0],
    "agent.min_ram": 32,
    "agent.max_ram": 2147483647,
    "agent.special_ram": [0],
    "job.priority": 500,
    "job.max_username_length": 254,
    "job.min_priority": 0,
    "job.max_priority": 1000,
    "job.batch": 1,
    "job.requeue": 1,
    "job.cpus": 4,
    "job.ram": 32})

# import all model objects into this space so relationships, foreign keys,
# and the the mapper won't have problems finding the required classes
from pyfarm.models.agent import AgentModel, AgentSoftwareModel, AgentTagsModel
from pyfarm.models.task import TaskModel
from pyfarm.models.job import JobModel, JobSoftwareModel, JobTagsModel

from pyfarm.models.core.app import db

def skip_on_ci(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        if "BUILDBOT_UUID" in os.environ or "TRAVIS" in os.environ:
            raise SkipTest
        return func(*args, **kwargs)
    return wrapper

class ModelTestCase(unittest.TestCase):
    ORIGINAL_ENVIRONMENT = dict(os.environ.data)

    def setUp(self):
        db.session.rollback()
        os.environ.clear()
        os.environ.update(self.ORIGINAL_ENVIRONMENT)
        db.create_all()

    def tearDown(self):
        db.session.remove()
        db.drop_all()
<commit_msg>fixing import<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Module containing the base test class and functions
used by the unittests.
"""

import os
import sys
import time
from functools import wraps

if sys.version_info[0:2] < (2, 7):
    import unittest2 as unittest
else:
    import unittest

try:
    import json
except ImportError:
    import simplejson as json

from nose.plugins.skip import SkipTest
from pyfarm.core.config import cfg

# Some initial configuration values before we load the models.  Some values,
# such as the table prefix are included here just so two tests don't step
# on each other (though in production it would be better to use a different DB).
cfg.update({
    "db.table_prefix": "pyfarm_unittest_%s_" % time.strftime("%M%d%Y%H%M%S"),
    "agent.min_port": 1025,
    "agent.max_port": 65535,
    "agent.min_cpus": 1,
    "agent.max_cpus": 2147483647,
    "agent.special_cpus": [0],
    "agent.min_ram": 32,
    "agent.max_ram": 2147483647,
    "agent.special_ram": [0],
    "job.priority": 500,
    "job.max_username_length": 254,
    "job.min_priority": 0,
    "job.max_priority": 1000,
    "job.batch": 1,
    "job.requeue": 1,
    "job.cpus": 4,
    "job.ram": 32})

# import all model objects into this space so relationships, foreign keys,
# and the the mapper won't have problems finding the required classes
from pyfarm.models.agent import AgentModel, AgentSoftwareModel, AgentTagsModel
from pyfarm.models.task import TaskModel
from pyfarm.models.job import JobModel, JobSoftwareModel, JobTagsModel

from pyfarm.master.application import db

def skip_on_ci(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        if "BUILDBOT_UUID" in os.environ or "TRAVIS" in os.environ:
            raise SkipTest
        return func(*args, **kwargs)
    return wrapper

class ModelTestCase(unittest.TestCase):
    ORIGINAL_ENVIRONMENT = dict(os.environ.data)

    def setUp(self):
        db.session.rollback()
        os.environ.clear()
        os.environ.update(self.ORIGINAL_ENVIRONMENT)
        db.create_all()

    def tearDown(self):
        db.session.remove()
        db.drop_all()
<|endoftext|>


<commit_before>"""
See documentation for the class Console defined here.
"""
from code import InteractiveConsole
import sys
import traceback

from wx.py.pseudo import PseudoFileIn, PseudoFileOut, PseudoFileErr

class Console(InteractiveConsole):
    """
    Console based on code.InteractiveConsole. You are supposed to run this
    console in a process you create with the `multiprocessing` package.
    It requires a parameter `queue_pack`, which you should create with
    `make_queue_pack()` in this package. You are supposed to feed the same
    queue pack into the Shell you will create for the two to be connected
    to each other.
    """

    def __init__(self, queue_pack, *args, **kwargs):
        super().__init__(self, *args, **kwargs)

        (self.input_queue,
         self.output_queue,
         self.runcode_finished_queue,
         self.runsource_return_queue) = queue_pack

        self.readfunc = self.input_queue.get
        self.writefunc = self.output_queue.put

        self.stdin = PseudoFileIn(self.readfunc)
        self.stdout = PseudoFileOut(self.writefunc)
        self.stderr = PseudoFileErr(self.writefunc)

    def raw_input(self, prompt=None):
        if prompt:
            self.write(prompt)
        return self.readfunc()

    def write(self, output):
        # self.log(output)
        return self.writefunc(output)

    def log(self, output):
        print(output)
        sys.stdout.flush()

    def push(self, command):
        more = self.runsource(command, self.filename)
        return more

    def showsyntaxerror(self, filename=None):
        ex_type, value, sys.last_traceback = sys.exc_info()
        sys.last_type = ex_type
        sys.last_value = value
        if filename and ex_type is SyntaxError:
            # Work hard to stuff the correct filename in the exception
            try:
                msg, (dummy_filename, lineno, offset, line) = value
            except ValueError:
                # Not the format we expect; leave it alone
                pass
            else:
                # Stuff in the right filename
                value = SyntaxError(msg, (filename, lineno, offset, line))
                sys.last_value = value
        ex_list = traceback.format_exception_only(ex_type, value)

        map(self.write, ex_list)

    def runsource(self, source, filename="<input>", symbol="single"):
        try:
            code = self.compile(source, filename, symbol)
        except (OverflowError, SyntaxError, ValueError):
            # Case 1
            self.showsyntaxerror(filename)

            self.runsource_return_queue.put(False)
            self.runcode_finished_queue.put(None)
            return False

        if code is None:
            # Case 2

            self.runsource_return_queue.put(True)
            self.runcode_finished_queue.put(None)
            return True

        # Case 3
        self.runsource_return_queue.put(False)

        stdin, stdout, stderr = \
            sys.stdin, sys.stdout, sys.stderr
        sys.stdin, sys.stdout, sys.stderr = \
            self.stdin, self.stdout, self.stderr

        try:
            self.runcode(code)
        finally:
            if sys.stdin == self.stdin:
                sys.stdin = stdin
            if sys.stdout == self.stdout:
                sys.stdout = stdout
            if sys.stderr == self.stderr:
                sys.stderr = stderr

        self.runcode_finished_queue.put(None)
        return False

    def interact(self, banner=None):
        try:
            sys.ps1
        except AttributeError:
            sys.ps1 = ">>> "
        try:
            sys.ps2
        except AttributeError:
            sys.ps2 = "... "
        """
        cprt = 'Type "help", "copyright", "credits" or "license" ' \
               'for more information.'
        if banner is None:
            self.write("Python %s on %s\n%s\n(%s)\n" %
                       (sys.version, sys.platform, cprt,
                        self.__class__.__name__))
        else:
            self.write("%s\n" % str(banner))
        """
        more = 0
        while True:
            try:
                if more:
                    pass  # prompt = sys.ps2
                else:
                    pass  # prompt = sys.ps1
                try:
                    line = self.raw_input()  # prompt)
                except EOFError:
                    self.write("\n")
                    break
                else:
                    # self.log(line.__repr__())
                    more = self.push(line)
            except KeyboardInterrupt:
                self.write("\nKeyboardInterrupt\n")
                self.resetbuffer()
                more = 0
<commit_msg>Fix TypeError: __init__() got multiple values for argument 'locals'<commit_after>"""
See documentation for the class Console defined here.
"""
from code import InteractiveConsole
import sys
import traceback

from wx.py.pseudo import PseudoFileIn, PseudoFileOut, PseudoFileErr

class Console(InteractiveConsole):
    """
    Console based on code.InteractiveConsole. You are supposed to run this
    console in a process you create with the `multiprocessing` package.
    It requires a parameter `queue_pack`, which you should create with
    `make_queue_pack()` in this package. You are supposed to feed the same
    queue pack into the Shell you will create for the two to be connected
    to each other.
    """

    def __init__(self, queue_pack, *args, **kwargs):
        InteractiveConsole.__init__(self, *args, **kwargs)

        (self.input_queue,
         self.output_queue,
         self.runcode_finished_queue,
         self.runsource_return_queue) = queue_pack

        self.readfunc = self.input_queue.get
        self.writefunc = self.output_queue.put

        self.stdin = PseudoFileIn(self.readfunc)
        self.stdout = PseudoFileOut(self.writefunc)
        self.stderr = PseudoFileErr(self.writefunc)

    def raw_input(self, prompt=None):
        if prompt:
            self.write(prompt)
        return self.readfunc()

    def write(self, output):
        # self.log(output)
        return self.writefunc(output)

    def log(self, output):
        print(output)
        sys.stdout.flush()

    def push(self, command):
        more = self.runsource(command, self.filename)
        return more

    def showsyntaxerror(self, filename=None):
        ex_type, value, sys.last_traceback = sys.exc_info()
        sys.last_type = ex_type
        sys.last_value = value
        if filename and ex_type is SyntaxError:
            # Work hard to stuff the correct filename in the exception
            try:
                msg, (dummy_filename, lineno, offset, line) = value
            except ValueError:
                # Not the format we expect; leave it alone
                pass
            else:
                # Stuff in the right filename
                value = SyntaxError(msg, (filename, lineno, offset, line))
                sys.last_value = value
        ex_list = traceback.format_exception_only(ex_type, value)

        map(self.write, ex_list)

    def runsource(self, source, filename="<input>", symbol="single"):
        try:
            code = self.compile(source, filename, symbol)
        except (OverflowError, SyntaxError, ValueError):
            # Case 1
            self.showsyntaxerror(filename)

            self.runsource_return_queue.put(False)
            self.runcode_finished_queue.put(None)
            return False

        if code is None:
            # Case 2

            self.runsource_return_queue.put(True)
            self.runcode_finished_queue.put(None)
            return True

        # Case 3
        self.runsource_return_queue.put(False)

        stdin, stdout, stderr = \
            sys.stdin, sys.stdout, sys.stderr
        sys.stdin, sys.stdout, sys.stderr = \
            self.stdin, self.stdout, self.stderr

        try:
            self.runcode(code)
        finally:
            if sys.stdin == self.stdin:
                sys.stdin = stdin
            if sys.stdout == self.stdout:
                sys.stdout = stdout
            if sys.stderr == self.stderr:
                sys.stderr = stderr

        self.runcode_finished_queue.put(None)
        return False

    def interact(self, banner=None):
        try:
            sys.ps1
        except AttributeError:
            sys.ps1 = ">>> "
        try:
            sys.ps2
        except AttributeError:
            sys.ps2 = "... "
        """
        cprt = 'Type "help", "copyright", "credits" or "license" ' \
               'for more information.'
        if banner is None:
            self.write("Python %s on %s\n%s\n(%s)\n" %
                       (sys.version, sys.platform, cprt,
                        self.__class__.__name__))
        else:
            self.write("%s\n" % str(banner))
        """
        more = 0
        while True:
            try:
                if more:
                    pass  # prompt = sys.ps2
                else:
                    pass  # prompt = sys.ps1
                try:
                    line = self.raw_input()  # prompt)
                except EOFError:
                    self.write("\n")
                    break
                else:
                    # self.log(line.__repr__())
                    more = self.push(line)
            except KeyboardInterrupt:
                self.write("\nKeyboardInterrupt\n")
                self.resetbuffer()
                more = 0
<|endoftext|>