<commit_before># Copyright (C) 2010-2014 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import sys
import time
import socket
import string
import random
import platform
import subprocess
import ConfigParser
from StringIO import StringIO
from zipfile import ZipFile
from SimpleXMLRPCServer import SimpleXMLRPCServer

BIND_IP = "0.0.0.0"
BIND_PORT = 8000

STATUS_INIT = 0x0001
STATUS_RUNNING = 0x0002
STATUS_COMPLETED = 0x0003
STATUS_FAILED = 0x0004
CURRENT_STATUS = STATUS_INIT

ERROR_MESSAGE = ""
ANALYZER_FOLDER = ""
RESULTS_FOLDER = ""

class Agent:
    """Cuckoo agent, it runs inside guest."""

    def __init__(self):
        self.system = platform.system().lower()
        self.analyzer_path = ""
        self.analyzer_pid = 0

    def _initialize(self):
        global ERROR_MESSAGE
        global ANALYZER_FOLDER

        if not ANALYZER_FOLDER:
            random.seed(time.time())
            container = "".join(random.choice(string.ascii_lowercase) for x in range(random.randint(5, 10)))

            if self.system == "windows":
                system_drive = os.environ["SYSTEMDRIVE"] + os.sep
                ANALYZER_FOLDER = os.path.join(system_drive, container)
            elif self.system == "linux" or self.system == "darwin":
                ANALYZER_FOLDER = os.path.join(os.environ["HOME"], container)
            else:
                ERROR_MESSAGE = "Unable to identify operating system"
                return False

            try:
                os.makedirs(ANALYZER_FOLDER)
            except OSError as e:
                ERROR_MESSAGE = e
                return False

        return True

    def get_status(self):
        """Get current status.
        @return: status.
        """
        return CURRENT_STATUS

    def get_error(self):
        """Get error message.
        @return: error message.
        """
        return str(ERROR_MESSAGE)

    def add_malware(self, data, name):
        """Get analysis data.
        @param data: analysis data.
        @param name: file name.
        @return: operation status.
        """
        global ERROR_MESSAGE
        data = data.data

        if self.system == "windows":
            root = os.environ["TEMP"]
        elif self.system == "linux" or self.system == "darwin":
            root = "/tmp"
        else:
            ERROR_MESSAGE = "Unable to write malware to disk because of " \
                            "failed identification of the operating system"
            return False

        file_path = os.path.join(root, name)

        try:
            with open(file_path, "wb") as malware:
                malware.write(data)
        except IOError as e:
            ERROR_MESSAGE = "Unable to write malware to disk: {0}".format(e)
            return False

        return True

    def add_config(self, options):
        """Creates analysis.conf file from current analysis options.
        @param options: current configuration options, dict format.
        @return: operation status.
        """
        global ERROR_MESSAGE

        if type(options) != dict:
            return False

        config = ConfigParser.RawConfigParser()
        config.add_section("analysis")

        try:
            for key, value in options.items():
                # Options can be UTF encoded.
                if isinstance(value, basestring):
                    try:
                        value = value.encode("utf-8")
                    except UnicodeEncodeError:
                        pass

                config.set("analysis", key, value)

            config_path = os.path.join(ANALYZER_FOLDER, "analysis.conf")

            with open(config_path, "wb") as config_file:
                config.write(config_file)
        except Exception as e:
            ERROR_MESSAGE = str(e)
            return False

        return True

    def add_analyzer(self, data):
        """Add analyzer.
        @param data: analyzer data.
        @return: operation status.
        """
        data = data.data

        if not self._initialize():
            return False

        try:
            zip_data = StringIO()
            zip_data.write(data)

            with ZipFile(zip_data, "r") as archive:
                archive.extractall(ANALYZER_FOLDER)
        finally:
            zip_data.close()

        self.analyzer_path = os.path.join(ANALYZER_FOLDER, "analyzer.py")

        return True

    def execute(self):
        """Execute analysis.
        @return: analyzer PID.
        """
        global ERROR_MESSAGE
        global CURRENT_STATUS

        if not self.analyzer_path or not os.path.exists(self.analyzer_path):
            return False

        try:
            proc = subprocess.Popen([sys.executable, self.analyzer_path],
                                    cwd=os.path.dirname(self.analyzer_path))
            self.analyzer_pid = proc.pid
        except OSError as e:
            ERROR_MESSAGE = str(e)
            return False

        CURRENT_STATUS = STATUS_RUNNING

        return self.analyzer_pid

    def complete(self, success=True, error="", results=""):
        """Complete analysis.
        @param success: success status.
        @param error: error status.
        """
        global ERROR_MESSAGE
        global CURRENT_STATUS
        global RESULTS_FOLDER

        if success:
            CURRENT_STATUS = STATUS_COMPLETED
        else:
            if error:
                ERROR_MESSAGE = str(error)

            CURRENT_STATUS = STATUS_FAILED

        RESULTS_FOLDER = results

        return True

if __name__ == "__main__":
    try:
        if not BIND_IP:
            BIND_IP = socket.gethostbyname(socket.gethostname())

        print("[+] Starting agent on %s:%s ..." % (BIND_IP, BIND_PORT))

        # Disable DNS lookup, by Scott D.
        def FakeGetFQDN(name=""):
            return name
        socket.getfqdn = FakeGetFQDN

        server = SimpleXMLRPCServer((BIND_IP, BIND_PORT), allow_none=True)
        server.register_instance(Agent())
        server.serve_forever()
    except KeyboardInterrupt:
        server.shutdown()
<commit_msg>small style changes<commit_after># Copyright (C) 2010-2014 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import sys
import time
import socket
import string
import random
import platform
import subprocess
import ConfigParser
from StringIO import StringIO
from zipfile import ZipFile
from SimpleXMLRPCServer import SimpleXMLRPCServer

BIND_IP = "0.0.0.0"
BIND_PORT = 8000

STATUS_INIT = 0x0001
STATUS_RUNNING = 0x0002
STATUS_COMPLETED = 0x0003
STATUS_FAILED = 0x0004
CURRENT_STATUS = STATUS_INIT

ERROR_MESSAGE = ""
ANALYZER_FOLDER = ""
RESULTS_FOLDER = ""

class Agent:
    """Cuckoo agent, it runs inside guest."""

    def __init__(self):
        self.system = platform.system().lower()
        self.analyzer_path = ""
        self.analyzer_pid = 0

    def _initialize(self):
        global ERROR_MESSAGE
        global ANALYZER_FOLDER

        if not ANALYZER_FOLDER:
            random.seed(time.time())
            container = "".join(random.choice(string.ascii_lowercase) for x in range(random.randint(5, 10)))

            if self.system == "windows":
                system_drive = os.environ["SYSTEMDRIVE"] + os.sep
                ANALYZER_FOLDER = os.path.join(system_drive, container)
            elif self.system == "linux" or self.system == "darwin":
                ANALYZER_FOLDER = os.path.join(os.environ["HOME"], container)
            else:
                ERROR_MESSAGE = "Unable to identify operating system"
                return False

            try:
                os.makedirs(ANALYZER_FOLDER)
            except OSError as e:
                ERROR_MESSAGE = e
                return False

        return True

    def get_status(self):
        """Get current status.
        @return: status.
        """
        return CURRENT_STATUS

    def get_error(self):
        """Get error message.
        @return: error message.
        """
        return str(ERROR_MESSAGE)

    def add_malware(self, data, name):
        """Get analysis data.
        @param data: analysis data.
        @param name: file name.
        @return: operation status.
        """
        global ERROR_MESSAGE
        data = data.data

        if self.system == "windows":
            root = os.environ["TEMP"]
        elif self.system == "linux" or self.system == "darwin":
            root = "/tmp"
        else:
            ERROR_MESSAGE = "Unable to write malware to disk because of " \
                            "failed identification of the operating system"
            return False

        file_path = os.path.join(root, name)

        try:
            with open(file_path, "wb") as sample:
                sample.write(data)
        except IOError as e:
            ERROR_MESSAGE = "Unable to write sample to disk: {0}".format(e)
            return False

        return True

    def add_config(self, options):
        """Creates analysis.conf file from current analysis options.
        @param options: current configuration options, dict format.
        @return: operation status.
        """
        global ERROR_MESSAGE

        if not isinstance(options, dict):
            return False

        config = ConfigParser.RawConfigParser()
        config.add_section("analysis")

        try:
            for key, value in options.items():
                # Options can be UTF encoded.
                if isinstance(value, basestring):
                    try:
                        value = value.encode("utf-8")
                    except UnicodeEncodeError:
                        pass

                config.set("analysis", key, value)

            config_path = os.path.join(ANALYZER_FOLDER, "analysis.conf")

            with open(config_path, "wb") as config_file:
                config.write(config_file)
        except Exception as e:
            ERROR_MESSAGE = str(e)
            return False

        return True

    def add_analyzer(self, data):
        """Add analyzer.
        @param data: analyzer data.
        @return: operation status.
        """
        data = data.data

        if not self._initialize():
            return False

        try:
            zip_data = StringIO()
            zip_data.write(data)

            with ZipFile(zip_data, "r") as archive:
                archive.extractall(ANALYZER_FOLDER)
        finally:
            zip_data.close()

        self.analyzer_path = os.path.join(ANALYZER_FOLDER, "analyzer.py")

        return True

    def execute(self):
        """Execute analysis.
        @return: analyzer PID.
        """
        global ERROR_MESSAGE
        global CURRENT_STATUS

        if not self.analyzer_path or not os.path.exists(self.analyzer_path):
            return False

        try:
            proc = subprocess.Popen([sys.executable, self.analyzer_path],
                                    cwd=os.path.dirname(self.analyzer_path))
            self.analyzer_pid = proc.pid
        except OSError as e:
            ERROR_MESSAGE = str(e)
            return False

        CURRENT_STATUS = STATUS_RUNNING

        return self.analyzer_pid

    def complete(self, success=True, error="", results=""):
        """Complete analysis.
        @param success: success status.
        @param error: error status.
        """
        global ERROR_MESSAGE
        global CURRENT_STATUS
        global RESULTS_FOLDER

        if success:
            CURRENT_STATUS = STATUS_COMPLETED
        else:
            if error:
                ERROR_MESSAGE = str(error)

            CURRENT_STATUS = STATUS_FAILED

        RESULTS_FOLDER = results

        return True

if __name__ == "__main__":
    try:
        if not BIND_IP:
            BIND_IP = socket.gethostbyname(socket.gethostname())

        print("[+] Starting agent on %s:%s ..." % (BIND_IP, BIND_PORT))

        # Disable DNS lookup, by Scott D.
        def FakeGetFQDN(name=""):
            return name

        socket.getfqdn = FakeGetFQDN

        server = SimpleXMLRPCServer((BIND_IP, BIND_PORT), allow_none=True)
        server.register_instance(Agent())
        server.serve_forever()
    except KeyboardInterrupt:
        server.shutdown()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import time

from proton import Message
from proton.handlers import MessagingHandler
from proton.reactor import Container
from utils import AGENT_BROADCAST_ADDRESS, STATE_FREE, STATE_CLIENT, STATE_SERVER, STATE_QUIESCE, \
    STATE_SERVER_CLOSING, DEPLOY_TYPE_CLIENT, DEPLOY_TYPE_SERVER, DEPLOY_TYPE_UNDEPLOY, DEPLOY_TYPE_QUIESCE

class Agent(MessagingHandler):
    """
    Our own custom message handler called Agent that implements reactive message handling semantics.
    An agent could be a client or a server
    """

    def __init__(self, url):
        super(Agent, self).__init__(prefetch=0)
        self.url = url
        self.senders = {}

        self.container = None
        self.connection = None

        # This receiver is mainly used is mainly used to receive client requests
        self.receiver = None

        # Used to receive command requests. This receives commands such a DEPLOY_AS_CLIENT, DEPLOY_AS_SERVER, QUIESCE
        self.command_receiver = None
        self.command_address = None

        # This like an abstract sender. You can supply the sender to this relay_sender and have it send messages.
        self.relay_sender = None
        self.sender = None

        self.desired_throughput = 0

        self.period = 0
        self.total_requests_received = 0
        self.outstanding_requests = 0
        self.sent = 0
        self.acknowledged = 0
        self.messages_to_send = 0
        self.backlog = 0

        # Default the state of the newly instantiated agent to FREE
        self.state = STATE_FREE
        self.address = None
        self.service_name = None

        self.work_queue = list()
        self.actual_throughput = []
        self.epoch_time_milliseconds_start = 0

    def calculate_actual_throughput(self):
        act_throughput = 0.0
        if self.actual_throughput:
            total_time = 0
            total_messages = 0
            for tup in self.actual_throughput:
                total_time += tup[0]
                total_messages += tup[1]
            if total_messages:
                act_throughput = (total_messages * 1000) / total_time

        return act_throughput

    def clear_stats(self):
        self.total_requests_received = 0
        self.sent = 0
        self.desired_throughput = 0
        self.outstanding_requests = 0
        self.actual_throughput = []
        self.backlog = 0
        self.state = STATE_FREE

    def get_stats(self):
        current_stats = dict() # Is creating a new dict everytime going to lead a memory leak?
        current_stats['total_requests_received'] = self.total_requests_received

        current_stats['state'] = self.state
        current_stats['sent'] = self.sent

        current_stats['service_address'] = self.service_name
        current_stats['desired_throughput'] = self.desired_throughput

        current_stats['backlog'] = self.backlog

        if self.container:
            current_stats['container_id'] = self.container._attrs['container_id']

        if self.command_address:
            current_stats['command_address'] = self.command_address

        current_stats['outstanding_requests'] = self.sent - self.acknowledged
        current_stats['actual_throughput'] = self.calculate_actual_throughput()

        return current_stats

    def set_agent_state(self, event):
        deploy_type = event.message.body.get('deploy_type')
        name = event.message.body.get('name')
        throughput = event.message.body.get('throughput')
        backlog = 0
        if event.message.body.get('backlog'):
            backlog = event.message.body.get('backlog')

        if self.state == STATE_FREE and deploy_type == DEPLOY_TYPE_SERVER:
            # Change the state of this agent to SERVER
            self.state = STATE_SERVER
            self.service_name = name
            self.backlog = backlog
            # Open receiver on name
            if throughput:
                self.desired_throughput = int(throughput)
            self.receiver = self.container.create_receiver(self.connection, source=name)
        elif self.state == STATE_FREE and deploy_type == DEPLOY_TYPE_CLIENT:
            # Change the state of this agent to CLIENT
            self.state = STATE_CLIENT
            self.service_name = name
            if throughput:
                self.desired_throughput = int(throughput)
            # Open sender to name
            self.sender = self.container.create_sender(self.connection, self.service_name)
        elif self.state == STATE_SERVER and deploy_type == DEPLOY_TYPE_UNDEPLOY:
            # Change the state of this agent to FREE
            self.state = STATE_SERVER_CLOSING
            # Stop issuing credit for receiver with name
            #TODO - Check examples to see how you can stop issuing credits
        elif self.state == STATE_CLIENT and deploy_type == DEPLOY_TYPE_UNDEPLOY:
            self.clear_stats()
            # Close sender 'name'
            self.sender.close()
        elif self.state == STATE_SERVER_CLOSING and deploy_type == DEPLOY_TYPE_UNDEPLOY:
            self.clear_stats()
            self.receiver.close()

    def calculate_backlog(self):
        actual_backlog = 0

        if self.receiver:
            actual_backlog = self.receiver.queued

        self.backlog = actual_backlog + len(self.work_queue)

    def send(self, to_address, body, correlation_id=None):
        sender = self.relay_sender or self.senders.get(to_address)

        if not sender:
            sender = self.container.create_sender(self.connection, to_address)
            self.senders[to_address] = sender

        sender.send(Message(address=to_address,
                            body=body,
                            correlation_id=correlation_id))

    def process_command(self, event):
        self.set_agent_state(event)

    def on_disconnected(self, event):
        pass

    def on_connection_closed(self, event):
        pass

    def on_session_closed(self, event):
        print 'Hello'

    def on_link_closed(self, event):
        if event.receiver and event.receiver == self.receiver and self.state == STATE_SERVER_CLOSING:
            self.clear_stats()

    def on_accepted(self, event):
        self.acknowledged += 1

    def on_start(self, event):
        self.connection = event.container.connect(self.url)
        self.container = event.container
        # self.broadcast_receiver = event.container.create_receiver(self.conn, AGENT_BROADCAST_ADDRESS)
        # This receiver is mainly used is mainly used to receive client requests
        self.command_receiver = event.container.create_receiver(self.connection, None, dynamic=True)

    def on_connection_opened(self, event):
        if event.connection.remote_offered_capabilities and 'ANONYMOUS-RELAY' in event.connection.remote_offered_capabilities:
            self.relay_sender = self.container.create_sender(self.connection, None)

    def calc_time_between_calls(self, message_count):
        time_between_calls = 0
        if not self.epoch_time_milliseconds_start:
            self.epoch_time_milliseconds_start = int(time.time() * 1000)
        else:
            current_time = int(time.time() * 1000)
            time_between_calls = current_time - self.epoch_time_milliseconds_start
            self.epoch_time_milliseconds_start = current_time

        time_between_calls_tuple = (time_between_calls, message_count)

        if len(self.actual_throughput) > 10:
            del(self.actual_throughput[0])

        self.actual_throughput.append(time_between_calls_tuple)

    def send_messages(self):
        if self.state == STATE_CLIENT:

            message_to_send = "Hello World"
            message_count = self.sender.credit
            if self.messages_to_send < message_count:
                message_count = self.messages_to_send

            self.calc_time_between_calls(message_count)

            for x in range(message_count):
                self.sent += 1
                self.send(self.service_name, message_to_send)

            self.messages_to_send -= message_count

    def on_sendable(self, event):
        self.send_messages()

    def on_link_opened(self, event):
        if event.receiver == self.command_receiver:
            event.receiver.flow(10)
            if event.receiver and event.receiver.remote_source:
                self.command_address = event.receiver.remote_source.address
        if event.receiver and event.receiver == self.receiver:
            if self.backlog:
                event.receiver.flow(int(self.backlog))

    def on_reactor_init(self, event):
        event.reactor.schedule(self.period, self)
        super(Agent, self).on_reactor_init(event)

    def on_timer_task(self, event):
        if self.relay_sender and self.period == 9:
            self.calculate_backlog()
            self.send(AGENT_BROADCAST_ADDRESS, self.get_stats())
            self.period = 0
        else:
            self.period += 1

        if self.state == STATE_CLIENT:
            self.messages_to_send = self.desired_throughput / 10
            self.send_messages()
        elif self.state == STATE_SERVER or self.state == STATE_SERVER_CLOSING:
            messages_to_process = self.desired_throughput / 10

            called = False
            for x in range(messages_to_process):
                if self.state == STATE_SERVER:
                    self.receiver.flow(1)
                elif self.state == STATE_SERVER_CLOSING and self.backlog == 0:
                    self.state = STATE_FREE
                try:
                    if self.work_queue:
                        del(self.work_queue[x])
                        if not called:
                            self.calc_time_between_calls(messages_to_process)
                            called = True
                except:
                    pass

        event.reactor.schedule(0.1, self)

    def on_message(self, event):
        """
        Invoked when messages are received.
        :param event:
        :return:
        """
        # We will only process if the receiver is present in our list of expected receivers.
        if event.receiver == self.command_receiver:
            self.set_agent_state(event)
            event.receiver.flow(1)
        elif event.receiver == self.receiver:
            # Increment the total_requests_received by 1 and add simply add the message to out local work_queue
            self.total_requests_received += 1
            self.work_queue.append(event.message.body)

try:
    agent = Agent("0.0.0.0:5672")
    container = Container(agent)
    container.run()
    # Container(Agent("0.0.0.0:5672")).run()
except KeyboardInterrupt:
    container.stop()

<commit_msg>Added code to store the delivery in the work queue and settle the delivery<commit_after>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

import time

from proton import Message, Delivery
from proton.handlers import MessagingHandler
from proton.reactor import Container
from utils import AGENT_BROADCAST_ADDRESS, STATE_FREE, STATE_CLIENT, STATE_SERVER, STATE_QUIESCE, \
    STATE_SERVER_CLOSING, DEPLOY_TYPE_CLIENT, DEPLOY_TYPE_SERVER, DEPLOY_TYPE_UNDEPLOY, DEPLOY_TYPE_QUIESCE

class Agent(MessagingHandler):
    """
    Our own custom message handler called Agent that implements reactive message handling semantics.
    An agent could be a client or a server
    """

    def __init__(self, url):
        super(Agent, self).__init__(prefetch=0, auto_accept=False)
        self.url = url
        self.senders = {}

        self.container = None
        self.connection = None

        # This receiver is mainly used is mainly used to receive client requests
        self.receiver = None

        # Used to receive command requests. This receives commands such a DEPLOY_AS_CLIENT, DEPLOY_AS_SERVER, QUIESCE
        self.command_receiver = None
        self.command_address = None

        # This like an abstract sender. You can supply the sender to this relay_sender and have it send messages.
        self.relay_sender = None
        self.sender = None

        self.desired_throughput = 0

        self.period = 0
        self.total_requests_received = 0
        self.outstanding_requests = 0
        self.sent = 0
        self.acknowledged = 0
        self.messages_to_send = 0
        self.backlog = 0
        self.allowed_backlog = 0

        # Default the state of the newly instantiated agent to FREE
        self.state = STATE_FREE
        self.address = None
        self.service_name = None

        self.work_queue = list()
        self.actual_throughput = []
        self.epoch_time_milliseconds_start = 0

    def calculate_actual_throughput(self):
        act_throughput = 0.0
        if self.actual_throughput:
            total_time = 0
            total_messages = 0
            for tup in self.actual_throughput:
                total_time += tup[0]
                total_messages += tup[1]
            if total_messages and total_time:
                act_throughput = (total_messages * 1000) / total_time

        return act_throughput

    def clear_stats(self):
        self.total_requests_received = 0
        self.sent = 0
        self.desired_throughput = 0
        self.outstanding_requests = 0
        self.actual_throughput = []
        self.backlog = 0
        self.state = STATE_FREE

    def get_stats(self):
        current_stats = dict() # Is creating a new dict everytime going to lead a memory leak?
        current_stats['total_requests_received'] = self.total_requests_received

        current_stats['state'] = self.state
        current_stats['sent'] = self.sent

        current_stats['service_address'] = self.service_name
        current_stats['desired_throughput'] = self.desired_throughput

        current_stats['backlog'] = self.backlog

        if self.container:
            current_stats['container_id'] = self.container._attrs['container_id']

        if self.command_address:
            current_stats['command_address'] = self.command_address

        current_stats['outstanding_requests'] = self.sent - self.acknowledged
        current_stats['actual_throughput'] = self.calculate_actual_throughput()

        return current_stats

    def set_agent_state(self, event):
        deploy_type = event.message.body.get('deploy_type')
        name = event.message.body.get('name')
        throughput = event.message.body.get('throughput')
        backlog = 0
        if event.message.body.get('backlog'):
            backlog = event.message.body.get('backlog')

        if self.state == STATE_FREE and deploy_type == DEPLOY_TYPE_SERVER:
            # Change the state of this agent to SERVER
            self.state = STATE_SERVER
            self.service_name = name
            self.allowed_backlog = backlog
            # Open receiver on name
            if throughput:
                self.desired_throughput = int(throughput)
            self.receiver = self.container.create_receiver(self.connection, source=name)
        elif self.state == STATE_FREE and deploy_type == DEPLOY_TYPE_CLIENT:
            # Change the state of this agent to CLIENT
            self.state = STATE_CLIENT
            self.service_name = name
            if throughput:
                self.desired_throughput = int(throughput)
            # Open sender to name
            self.sender = self.container.create_sender(self.connection, self.service_name)
        elif self.state == STATE_SERVER and deploy_type == DEPLOY_TYPE_UNDEPLOY:
            # Change the state of this agent to FREE
            self.state = STATE_SERVER_CLOSING
            # Stop issuing credit for receiver with name
            #TODO - Check examples to see how you can stop issuing credits
        elif self.state == STATE_CLIENT and deploy_type == DEPLOY_TYPE_UNDEPLOY:
            self.clear_stats()
            # Close sender 'name'
            self.sender.close()
        elif self.state == STATE_SERVER_CLOSING and deploy_type == DEPLOY_TYPE_UNDEPLOY:
            self.clear_stats()
            self.receiver.close()

    def calculate_backlog(self):
        actual_backlog = 0

        if self.receiver:
            actual_backlog = self.receiver.queued

        self.backlog = actual_backlog + len(self.work_queue)

    def send(self, to_address, body, correlation_id=None):
        sender = self.relay_sender or self.senders.get(to_address)

        if not sender:
            sender = self.container.create_sender(self.connection, to_address)
            self.senders[to_address] = sender

        sender.send(Message(address=to_address,
                            body=body,
                            correlation_id=correlation_id))

    def process_command(self, event):
        self.set_agent_state(event)

    def on_disconnected(self, event):
        pass

    def on_connection_closed(self, event):
        pass

    def on_session_closed(self, event):
        print 'Hello'

    def on_link_closed(self, event):
        if event.receiver and event.receiver == self.receiver and self.state == STATE_SERVER_CLOSING:
            self.clear_stats()

    def on_accepted(self, event):
        self.acknowledged += 1

    def on_start(self, event):
        self.connection = event.container.connect(self.url)
        self.container = event.container
        # self.broadcast_receiver = event.container.create_receiver(self.conn, AGENT_BROADCAST_ADDRESS)
        # This receiver is mainly used is mainly used to receive client requests
        self.command_receiver = event.container.create_receiver(self.connection, None, dynamic=True)

    def on_connection_opened(self, event):
        if event.connection.remote_offered_capabilities and 'ANONYMOUS-RELAY' in event.connection.remote_offered_capabilities:
            self.relay_sender = self.container.create_sender(self.connection, None)

    def calc_time_between_calls(self, message_count):
        time_between_calls = 0
        if not self.epoch_time_milliseconds_start:
            self.epoch_time_milliseconds_start = int(time.time() * 1000)
        else:
            current_time = int(time.time() * 1000)
            time_between_calls = current_time - self.epoch_time_milliseconds_start
            self.epoch_time_milliseconds_start = current_time

        time_between_calls_tuple = (time_between_calls, message_count)

        if len(self.actual_throughput) > 10:
            del(self.actual_throughput[0])

        self.actual_throughput.append(time_between_calls_tuple)

    def send_messages(self):
        if self.state == STATE_CLIENT:

            message_to_send = "Hello World"
            message_count = self.sender.credit
            if self.messages_to_send < message_count:
                message_count = self.messages_to_send

            self.calc_time_between_calls(message_count)

            for x in range(message_count):
                self.sent += 1
                self.send(self.service_name, message_to_send)

            self.messages_to_send -= message_count

    def on_sendable(self, event):
        self.send_messages()

    def on_link_opened(self, event):
        if event.receiver == self.command_receiver:
            event.receiver.flow(10)
            if event.receiver and event.receiver.remote_source:
                self.command_address = event.receiver.remote_source.address
        if event.receiver and event.receiver == self.receiver:
            if self.allowed_backlog:
                event.receiver.flow(int(self.allowed_backlog))

    def on_reactor_init(self, event):
        event.reactor.schedule(self.period, self)
        super(Agent, self).on_reactor_init(event)

    def on_timer_task(self, event):
        if self.relay_sender and self.period == 9:
            self.calculate_backlog()
            self.send(AGENT_BROADCAST_ADDRESS, self.get_stats())
            self.period = 0
        else:
            self.period += 1

        if self.state == STATE_CLIENT:
            self.messages_to_send = self.desired_throughput / 10
            self.send_messages()
        elif self.state == STATE_SERVER or self.state == STATE_SERVER_CLOSING:
            messages_to_process = self.desired_throughput / 10

            called = False
            for x in range(messages_to_process):

                if self.state == STATE_SERVER_CLOSING and self.backlog == 0:
                    self.state = STATE_FREE
                    self.clear_stats()

                try:
                    if self.work_queue:
                        delivery = self.work_queue[x]

                        delivery.update(Delivery.ACCEPTED)
                        delivery.settle()

                        del(self.work_queue[x])

                        if self.state == STATE_SERVER:
                            self.receiver.flow(1)

                        if not called:
                            self.calc_time_between_calls(messages_to_process)
                            called = True
                except:
                    pass

        event.reactor.schedule(0.1, self)

    def on_message(self, event):
        # We will only process if the receiver is present in our list of expected receivers.
        if event.receiver == self.command_receiver:
            self.set_agent_state(event)
            event.receiver.flow(1)
        elif event.receiver == self.receiver:
            # Increment the total_requests_received by 1 and add simply add the message to out local work_queue
            self.total_requests_received += 1
            self.work_queue.append(event.delivery)

try:
    agent = Agent("0.0.0.0:5672")
    container = Container(agent)
    container.run()
    # Container(Agent("0.0.0.0:5672")).run()
except KeyboardInterrupt:
    container.stop()

<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
from sklearn import tree

train_url = "trainingData/train.csv"
train = pd.read_csv(train_url)

test_url = "trainingData/test.csv"
test = pd.read_csv(test_url)

train["Sex"][train["Sex"] != "male"] = 1 # all females and empty entries will be 1
train["Sex"][train["Sex"] == "male"] = 0

#Impute the Embarked variable
train["Embarked"] = train["Embarked"].fillna("S")

#Convert the Embarked classes to integer form
train["Embarked"][train["Embarked"] == "S"] = 0
train["Embarked"][train["Embarked"] == "C"] = 1
train["Embarked"][train["Embarked"] == "Q"] = 2

train["Age"] = train["Age"].fillna(train["Age"].median())

# Adding a feature to improve prediction:
train["family_size"] = train["SibSp"].values + train["Parch"].values + 1

#cross_val = train[800:]
#train = train[1:800]

feature_names = ["Pclass", "Sex", "Age", "Fare", "SibSp", "Parch", "family_size"]

# to control overfitting:
max_depth = 10
min_samples_split = 5

target = train["Survived"].values
features_one = train[feature_names].values

#Fitting the selected features in a decision tree
my_tree_one = tree.DecisionTreeClassifier(max_depth = max_depth, min_samples_split = min_samples_split, random_state = 1)
my_tree_one = my_tree_one.fit(features_one, target)

#cross_target = cross_val["Survived"]
#cross_features = cross_val[feature_names].values
#print(my_tree_one.score(cross_features , cross_target))

# -------------------------------------------------------------------------------
# ------------------------------- Test cases ------------------------------------
# -------------------------------------------------------------------------------
# Impute the missing value with the median
test.Fare[152] = test.Fare.median()

test["Sex"][test["Sex"] != "male"] = 1 # all females and empty entries will be 1
test["Sex"][test["Sex"] == "male"] = 0

#Impute the Embarked variable
test["Embarked"] = test["Embarked"].fillna("S")

#Convert the Embarked classes to integer form
test["Embarked"][test["Embarked"] == "S"] = 0
test["Embarked"][test["Embarked"] == "C"] = 1
test["Embarked"][test["Embarked"] == "Q"] = 2

test["Age"] = test["Age"].fillna(test["Age"].median())

test["family_size"] = test["SibSp"].values + test["Parch"].values + 1
# Extract the features from the test set: Pclass, Sex, Age, and Fare.
test_features = test[feature_names].values

# Make your prediction using the test set
my_prediction = my_tree_one.predict(test_features)

PassengerId = np.array(test["PassengerId"]).astype(int)
my_solution = pd.DataFrame(my_prediction, PassengerId, columns = ["Survived"])

my_solution.to_csv("myPrediction.csv", index_label = ["PassengerId"])
<commit_msg>use Random forest instead of decision trees<commit_after>import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier

train_url = "trainingData/train.csv"
train = pd.read_csv(train_url)

test_url = "trainingData/test.csv"
test = pd.read_csv(test_url)

train["Sex"][train["Sex"] != "male"] = 1 # all females and empty entries will be 1
train["Sex"][train["Sex"] == "male"] = 0

#Impute the Embarked variable
train["Embarked"] = train["Embarked"].fillna("S")

#Convert the Embarked classes to integer form
train["Embarked"][train["Embarked"] == "S"] = 0
train["Embarked"][train["Embarked"] == "C"] = 1
train["Embarked"][train["Embarked"] == "Q"] = 2

train["Age"] = train["Age"].fillna(train["Age"].median())

# Adding a feature to improve prediction:
# train["family_size"] = train["SibSp"].values + train["Parch"].values + 1

feature_names = ["Pclass", "Sex", "Age", "Fare", "SibSp", "Parch", "Embarked"]

target = train["Survived"].values
features_one = train[feature_names].values

#Fitting the selected features in a decision tree
max_depth = 10
min_samples_split = 2
my_forest = RandomForestClassifier(max_depth = max_depth, min_samples_split = min_samples_split, n_estimators = 100, random_state = 1)
my_forest = my_forest.fit(features_one, target)

# -------------------------------------------------------------------------------
# ------------------------------- Test cases ------------------------------------
# -------------------------------------------------------------------------------
# Impute the missing value with the median
test.Fare[152] = test.Fare.median()

test["Sex"][test["Sex"] != "male"] = 1 # all females and empty entries will be 1
test["Sex"][test["Sex"] == "male"] = 0

#Impute the Embarked variable
test["Embarked"] = test["Embarked"].fillna("S")

#Convert the Embarked classes to integer form
test["Embarked"][test["Embarked"] == "S"] = 0
test["Embarked"][test["Embarked"] == "C"] = 1
test["Embarked"][test["Embarked"] == "Q"] = 2

test["Age"] = test["Age"].fillna(test["Age"].median())

test["family_size"] = test["SibSp"].values + test["Parch"].values + 1
# Extract the features from the test set: Pclass, Sex, Age, and Fare.
test_features = test[feature_names].values

# Make your prediction using the test set
my_prediction = my_forest.predict(test_features)

PassengerId = np.array(test["PassengerId"]).astype(int)
my_solution = pd.DataFrame(my_prediction, PassengerId, columns = ["Survived"])

my_solution.to_csv("myPrediction.csv", index_label = ["PassengerId"])
<|endoftext|>


<commit_before>from collections import deque

import numpy as np

from .. import utils

__all__ = [
    "get_all_layers",
    "get_all_layers_old",
    "get_output",
    "get_all_params",
    "get_all_bias_params",
    "get_all_non_bias_params",
    "count_params",
    "get_all_param_values",
    "set_all_param_values",
]

def get_all_layers(layer):
    """
    This function gathers all layers below one or more given :class:`Layer`
    instances, including the given layer(s). Its main use is to collect all
    layers of a network just given the output layer(s). The layers are
    guaranteed to be returned in a topological order: a layer in the result
    list is always preceded by all layers its input depends on.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> get_all_layers(l1) == [l_in, l1]
        True
        >>> l2 = DenseLayer(l_in, num_units=10)
        >>> get_all_layers([l2, l1]) == [l_in, l2, l1]
        True
        >>> get_all_layers([l1, l2]) == [l_in, l1, l2]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all layers feeding
            into it, or a list of :class:`Layer` instances.

    :returns:
        - layers : list
            a list of :class:`Layer` instances feeding into the given
            instance(s) either directly or indirectly, and the given
            instance(s) themselves, in topological order.
    """
    import warnings
    warnings.warn("get_all_layers() has been changed to return layers in "
                  "topological order. The former implementation is still "
                  "available as get_all_layers_old(), but will be removed "
                  "before the first release of Lasagne. To ignore this "
                  "warning, use `warnings.filterwarnings('ignore', "
                  "'.*topo.*')`.")

    # We perform a depth-first search. We add a layer to the result list only
    # after adding all its incoming layers (if any) or when detecting a cycle.
    # We use a LIFO stack to avoid ever running into recursion depth limits.
    try:
        queue = deque(layer)
    except TypeError:
        queue = deque([layer])
    seen = set()
    done = set()
    result = []

    while queue:
        # Peek at the leftmost node in the queue.
        layer = queue[0]
        if layer is None:
            # Some node had an input_layer set to `None`. Just ignore it.
            queue.popleft()
        elif layer not in seen:
            # We haven't seen this node yet: Mark it and queue all incomings
            # to be processed first. If there are no incomings, the node will
            # be appended to the result list in the next iteration.
            seen.add(layer)
            if hasattr(layer, 'input_layers'):
                queue.extendleft(reversed(layer.input_layers))
            elif hasattr(layer, 'input_layer'):
                queue.appendleft(layer.input_layer)
        else:
            # We've been here before: Either we've finished all its incomings,
            # or we've detected a cycle. In both cases, we remove the layer
            # from the queue and append it to the result list.
            queue.popleft()
            if layer not in done:
                result.append(layer)
                done.add(layer)

    return result

def get_all_layers_old(layer):
    """
    Earlier implementation of `get_all_layers()` that does a breadth-first
    search. Kept here to ease converting old models that rely on the order
    of get_all_layers() or get_all_params(). Will be removed before the
    first release of Lasagne.
    """
    if isinstance(layer, (list, tuple)):
        layers = list(layer)
    else:
        layers = [layer]
    layers_to_expand = list(layers)
    while len(layers_to_expand) > 0:
        current_layer = layers_to_expand.pop(0)
        children = []

        if hasattr(current_layer, 'input_layers'):
            children = current_layer.input_layers
        elif hasattr(current_layer, 'input_layer'):
            children = [current_layer.input_layer]

        # filter the layers that have already been visited, and remove None
        # elements (for layers without incoming layers)
        children = [child for child in children
                    if child not in layers and
                    child is not None]
        layers_to_expand.extend(children)
        layers.extend(children)

    return layers

def get_output(layer_or_layers, inputs=None, **kwargs):
    """
    Computes the output of the network at one or more given layers.
    Optionally, you can define the input(s) to propagate through the network
    instead of using the input variable(s) associated with the network's
    input layer(s).

    :parameters:
        - layer_or_layers : Layer or list
            the :class:`Layer` instance for which to compute the output
            expressions, or a list of :class:`Layer` instances.
        - inputs : None, Theano expression, numpy array, or dict
            If None, uses the input variables associated with the
            :class:`InputLayer` instances.
            If a Theano expression, this defines the input for a single
            :class:`InputLayer` instance. Will throw a ValueError if there
            are multiple :class:`InputLayer` instances.
            If a numpy array, this will be wrapped as a Theano constant
            and used just like a Theano expression.
            If a dictionary, any :class:`Layer` instance (including the
            input layers) can be mapped to a Theano expression or numpy
            array to use instead of its regular output.

    :returns:
        - output : Theano expression or list
            the output of the given layer(s) for the given network input

    :note:
        Depending on your network architecture, `get_output([l1, l2])` may
        be crucially different from `[get_output(l1), get_output(l2)]`. Only
        the former ensures that the output expressions depend on the same
        intermediate expressions. For example, when `l1` and `l2` depend on
        a common dropout layer, the former will use the same dropout mask for
        both, while the latter will use two different dropout masks.
    """
    from .input import InputLayer
    from .base import MultipleInputsLayer
    # obtain topological ordering of all layers the output layer(s) depend on
    all_layers = get_all_layers(layer_or_layers)
    # initialize layer-to-expression mapping from given input(s)
    if isinstance(inputs, dict):
        all_outputs = dict((layer, utils.as_theano_expression(expr))
                           for layer, expr in inputs.items())
    elif inputs is not None:
        inputs = utils.as_theano_expression(inputs)
        all_outputs = dict((layer, inputs)
                           for layer in all_layers
                           if isinstance(layer, InputLayer))
        if len(all_outputs) > 1:
            raise ValueError("get_output() was called with a single input "
                             "expression on a network with multiple input "
                             "layers. Please call it with a dictionary of "
                             "input expressions instead.")
    else:
        all_outputs = dict((layer, layer.input_var)
                           for layer in all_layers
                           if isinstance(layer, InputLayer))
    # update layer-to-expression mapping by propagating the inputs
    for layer in all_layers:
        if layer not in all_outputs:
            if isinstance(layer, MultipleInputsLayer):
                layer_inputs = [all_outputs[input_layer]
                                for input_layer in layer.input_layers]
            else:
                layer_inputs = all_outputs[layer.input_layer]
            all_outputs[layer] = layer.get_output_for(layer_inputs, **kwargs)
    # return the output(s) of the requested layer(s) only
    try:
        return [all_outputs[layer] for layer in layer_or_layers]
    except TypeError:
        return all_outputs[layer_or_layers]

def get_all_params(layer):
    """
    This function gathers all learnable parameters of all layers below one
    or more given :class:`Layer` instances, including the layer(s) itself.
    Its main use is to collect all parameters of a network just given the
    output layer(s).

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_params = get_all_params(l1)
        >>> all_params == [l1.W, l1.b]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all parameters,
            or a list of :class:`Layer` instances.

    :returns:
        - params : list
            a list of Theano shared variables representing the parameters.
    """
    layers = get_all_layers(layer)
    params = sum([l.get_params() for l in layers], [])
    return utils.unique(params)

def get_all_bias_params(layer):
    """
    This function gathers all learnable bias parameters of all layers below one
    or more given :class:`Layer` instances, including the layer(s) itself.

    This is useful for situations where the biases should be treated
    separately from other parameters, e.g. they are typically excluded from
    L2 regularization.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_params = get_all_bias_params(l1)
        >>> all_params == [l1.b]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all bias
            parameters, or a list of :class:`Layer` instances.

    :returns:
        - params : list
            a list of Theano shared variables representing the bias parameters.

    """
    layers = get_all_layers(layer)
    params = sum([l.get_bias_params() for l in layers], [])
    return utils.unique(params)

def get_all_non_bias_params(layer):
    """
    This function gathers all learnable non-bias parameters of all layers below
    one or more given :class:`Layer` instances, including the layer(s) itself.

    This is useful for situations where the biases should be treated
    separately from other parameters, e.g. they are typically excluded from
    L2 regularization.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_params = get_all_non_bias_params(l1)
        >>> all_params == [l1.W]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all non-bias
            parameters, or a list of :class:`Layer` instances.

    :returns:
        - params : list
            a list of Theano shared variables representing the non-bias
            parameters.

    """
    all_params = get_all_params(layer)
    all_bias_params = get_all_bias_params(layer)
    return [p for p in all_params if p not in all_bias_params]

def count_params(layer):
    """
    This function counts all learnable parameters (i.e. the number of scalar
    values) of all layers below one or more given :class:`Layer` instances,
    including the layer(s) itself.

    This is useful to compare the capacity of various network architectures.
    All parameters returned by the :class:`Layer`s' `get_params` methods are
    counted, including biases.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> param_count = count_params(l1)
        >>> param_count
        1050
        >>> param_count == 20 * 50 + 50  # 20 input * 50 units + 50 biases
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to count the parameters,
            or a list of :class:`Layer` instances.
    :returns:
        - count : int
            the total number of learnable parameters.

    """
    params = get_all_params(layer)
    shapes = [p.get_value().shape for p in params]
    counts = [np.prod(shape) for shape in shapes]
    return sum(counts)

def get_all_param_values(layer):
    """
    This function returns the values of the parameters of all layers below one
    or more given :class:`Layer` instances, including the layer(s) itself.

    This function can be used in conjunction with set_all_param_values to save
    and restore model parameters.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_param_values = get_all_param_values(l1)
        >>> (all_param_values[0] == l1.W.get_value()).all()
        True
        >>> (all_param_values[1] == l1.b.get_value()).all()
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all parameter
            values, or a list of :class:`Layer` instances.

    :returns:
        - param_values : list of numpy.array
            a list of numpy arrays representing the parameter values.
    """
    params = get_all_params(layer)
    return [p.get_value() for p in params]

def set_all_param_values(layer, values):
    """
    Given a list of numpy arrays, this function sets the parameters of all
    layers below one or more given :class:`Layer` instances (including the
    layer(s) itself) to the given values.

    This function can be used in conjunction with get_all_param_values to save
    and restore model parameters.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_param_values = get_all_param_values(l1)
        >>> # all_param_values is now [l1.W.get_value(), l1.b.get_value()]
        >>> # ...
        >>> set_all_param_values(l1, all_param_values)
        >>> # the parameter values are restored.

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to set all parameter
            values, or a list of :class:`Layer` instances.
        - values : list of numpy.array
            a list of numpy arrays representing the parameter values,
            must match the number of parameters
    """
    params = get_all_params(layer)
    if len(params) != len(values):
        raise ValueError("mismatch: got %d values to set %d parameters" %
                         (len(values), len(params)))
    for p, v in zip(params, values):
        p.set_value(v)
<commit_msg>Catch free-floating layers in get_outputs()<commit_after>from collections import deque

import numpy as np

from .. import utils

__all__ = [
    "get_all_layers",
    "get_all_layers_old",
    "get_output",
    "get_all_params",
    "get_all_bias_params",
    "get_all_non_bias_params",
    "count_params",
    "get_all_param_values",
    "set_all_param_values",
]

def get_all_layers(layer):
    """
    This function gathers all layers below one or more given :class:`Layer`
    instances, including the given layer(s). Its main use is to collect all
    layers of a network just given the output layer(s). The layers are
    guaranteed to be returned in a topological order: a layer in the result
    list is always preceded by all layers its input depends on.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> get_all_layers(l1) == [l_in, l1]
        True
        >>> l2 = DenseLayer(l_in, num_units=10)
        >>> get_all_layers([l2, l1]) == [l_in, l2, l1]
        True
        >>> get_all_layers([l1, l2]) == [l_in, l1, l2]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all layers feeding
            into it, or a list of :class:`Layer` instances.

    :returns:
        - layers : list
            a list of :class:`Layer` instances feeding into the given
            instance(s) either directly or indirectly, and the given
            instance(s) themselves, in topological order.
    """
    import warnings
    warnings.warn("get_all_layers() has been changed to return layers in "
                  "topological order. The former implementation is still "
                  "available as get_all_layers_old(), but will be removed "
                  "before the first release of Lasagne. To ignore this "
                  "warning, use `warnings.filterwarnings('ignore', "
                  "'.*topo.*')`.")

    # We perform a depth-first search. We add a layer to the result list only
    # after adding all its incoming layers (if any) or when detecting a cycle.
    # We use a LIFO stack to avoid ever running into recursion depth limits.
    try:
        queue = deque(layer)
    except TypeError:
        queue = deque([layer])
    seen = set()
    done = set()
    result = []

    while queue:
        # Peek at the leftmost node in the queue.
        layer = queue[0]
        if layer is None:
            # Some node had an input_layer set to `None`. Just ignore it.
            queue.popleft()
        elif layer not in seen:
            # We haven't seen this node yet: Mark it and queue all incomings
            # to be processed first. If there are no incomings, the node will
            # be appended to the result list in the next iteration.
            seen.add(layer)
            if hasattr(layer, 'input_layers'):
                queue.extendleft(reversed(layer.input_layers))
            elif hasattr(layer, 'input_layer'):
                queue.appendleft(layer.input_layer)
        else:
            # We've been here before: Either we've finished all its incomings,
            # or we've detected a cycle. In both cases, we remove the layer
            # from the queue and append it to the result list.
            queue.popleft()
            if layer not in done:
                result.append(layer)
                done.add(layer)

    return result

def get_all_layers_old(layer):
    """
    Earlier implementation of `get_all_layers()` that does a breadth-first
    search. Kept here to ease converting old models that rely on the order
    of get_all_layers() or get_all_params(). Will be removed before the
    first release of Lasagne.
    """
    if isinstance(layer, (list, tuple)):
        layers = list(layer)
    else:
        layers = [layer]
    layers_to_expand = list(layers)
    while len(layers_to_expand) > 0:
        current_layer = layers_to_expand.pop(0)
        children = []

        if hasattr(current_layer, 'input_layers'):
            children = current_layer.input_layers
        elif hasattr(current_layer, 'input_layer'):
            children = [current_layer.input_layer]

        # filter the layers that have already been visited, and remove None
        # elements (for layers without incoming layers)
        children = [child for child in children
                    if child not in layers and
                    child is not None]
        layers_to_expand.extend(children)
        layers.extend(children)

    return layers

def get_output(layer_or_layers, inputs=None, **kwargs):
    """
    Computes the output of the network at one or more given layers.
    Optionally, you can define the input(s) to propagate through the network
    instead of using the input variable(s) associated with the network's
    input layer(s).

    :parameters:
        - layer_or_layers : Layer or list
            the :class:`Layer` instance for which to compute the output
            expressions, or a list of :class:`Layer` instances.
        - inputs : None, Theano expression, numpy array, or dict
            If None, uses the input variables associated with the
            :class:`InputLayer` instances.
            If a Theano expression, this defines the input for a single
            :class:`InputLayer` instance. Will throw a ValueError if there
            are multiple :class:`InputLayer` instances.
            If a numpy array, this will be wrapped as a Theano constant
            and used just like a Theano expression.
            If a dictionary, any :class:`Layer` instance (including the
            input layers) can be mapped to a Theano expression or numpy
            array to use instead of its regular output.

    :returns:
        - output : Theano expression or list
            the output of the given layer(s) for the given network input

    :note:
        Depending on your network architecture, `get_output([l1, l2])` may
        be crucially different from `[get_output(l1), get_output(l2)]`. Only
        the former ensures that the output expressions depend on the same
        intermediate expressions. For example, when `l1` and `l2` depend on
        a common dropout layer, the former will use the same dropout mask for
        both, while the latter will use two different dropout masks.
    """
    from .input import InputLayer
    from .base import MultipleInputsLayer
    # obtain topological ordering of all layers the output layer(s) depend on
    all_layers = get_all_layers(layer_or_layers)
    # initialize layer-to-expression mapping from given input(s)
    if isinstance(inputs, dict):
        all_outputs = dict((layer, utils.as_theano_expression(expr))
                           for layer, expr in inputs.items())
    elif inputs is not None:
        inputs = utils.as_theano_expression(inputs)
        all_outputs = dict((layer, inputs)
                           for layer in all_layers
                           if isinstance(layer, InputLayer))
        if len(all_outputs) > 1:
            raise ValueError("get_output() was called with a single input "
                             "expression on a network with multiple input "
                             "layers. Please call it with a dictionary of "
                             "input expressions instead.")
    else:
        all_outputs = dict((layer, layer.input_var)
                           for layer in all_layers
                           if isinstance(layer, InputLayer))
    # update layer-to-expression mapping by propagating the inputs
    for layer in all_layers:
        if layer not in all_outputs:
            try:
                if isinstance(layer, MultipleInputsLayer):
                    layer_inputs = [all_outputs[input_layer]
                                    for input_layer in layer.input_layers]
                else:
                    layer_inputs = all_outputs[layer.input_layer]
            except KeyError:
                # one of the input_layer attributes must have been `None`
                raise ValueError("get_output() was called without giving an "
                                 "input expression for the free-floating "
                                 "layer %r. Please call it with a dictionary "
                                 "mapping this layer to an input expression."
                                 % layer)
            all_outputs[layer] = layer.get_output_for(layer_inputs, **kwargs)
    # return the output(s) of the requested layer(s) only
    try:
        return [all_outputs[layer] for layer in layer_or_layers]
    except TypeError:
        return all_outputs[layer_or_layers]

def get_all_params(layer):
    """
    This function gathers all learnable parameters of all layers below one
    or more given :class:`Layer` instances, including the layer(s) itself.
    Its main use is to collect all parameters of a network just given the
    output layer(s).

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_params = get_all_params(l1)
        >>> all_params == [l1.W, l1.b]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all parameters,
            or a list of :class:`Layer` instances.

    :returns:
        - params : list
            a list of Theano shared variables representing the parameters.
    """
    layers = get_all_layers(layer)
    params = sum([l.get_params() for l in layers], [])
    return utils.unique(params)

def get_all_bias_params(layer):
    """
    This function gathers all learnable bias parameters of all layers below one
    or more given :class:`Layer` instances, including the layer(s) itself.

    This is useful for situations where the biases should be treated
    separately from other parameters, e.g. they are typically excluded from
    L2 regularization.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_params = get_all_bias_params(l1)
        >>> all_params == [l1.b]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all bias
            parameters, or a list of :class:`Layer` instances.

    :returns:
        - params : list
            a list of Theano shared variables representing the bias parameters.

    """
    layers = get_all_layers(layer)
    params = sum([l.get_bias_params() for l in layers], [])
    return utils.unique(params)

def get_all_non_bias_params(layer):
    """
    This function gathers all learnable non-bias parameters of all layers below
    one or more given :class:`Layer` instances, including the layer(s) itself.

    This is useful for situations where the biases should be treated
    separately from other parameters, e.g. they are typically excluded from
    L2 regularization.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_params = get_all_non_bias_params(l1)
        >>> all_params == [l1.W]
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all non-bias
            parameters, or a list of :class:`Layer` instances.

    :returns:
        - params : list
            a list of Theano shared variables representing the non-bias
            parameters.

    """
    all_params = get_all_params(layer)
    all_bias_params = get_all_bias_params(layer)
    return [p for p in all_params if p not in all_bias_params]

def count_params(layer):
    """
    This function counts all learnable parameters (i.e. the number of scalar
    values) of all layers below one or more given :class:`Layer` instances,
    including the layer(s) itself.

    This is useful to compare the capacity of various network architectures.
    All parameters returned by the :class:`Layer`s' `get_params` methods are
    counted, including biases.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> param_count = count_params(l1)
        >>> param_count
        1050
        >>> param_count == 20 * 50 + 50  # 20 input * 50 units + 50 biases
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to count the parameters,
            or a list of :class:`Layer` instances.
    :returns:
        - count : int
            the total number of learnable parameters.

    """
    params = get_all_params(layer)
    shapes = [p.get_value().shape for p in params]
    counts = [np.prod(shape) for shape in shapes]
    return sum(counts)

def get_all_param_values(layer):
    """
    This function returns the values of the parameters of all layers below one
    or more given :class:`Layer` instances, including the layer(s) itself.

    This function can be used in conjunction with set_all_param_values to save
    and restore model parameters.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_param_values = get_all_param_values(l1)
        >>> (all_param_values[0] == l1.W.get_value()).all()
        True
        >>> (all_param_values[1] == l1.b.get_value()).all()
        True

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to gather all parameter
            values, or a list of :class:`Layer` instances.

    :returns:
        - param_values : list of numpy.array
            a list of numpy arrays representing the parameter values.
    """
    params = get_all_params(layer)
    return [p.get_value() for p in params]

def set_all_param_values(layer, values):
    """
    Given a list of numpy arrays, this function sets the parameters of all
    layers below one or more given :class:`Layer` instances (including the
    layer(s) itself) to the given values.

    This function can be used in conjunction with get_all_param_values to save
    and restore model parameters.

    :usage:
        >>> from lasagne.layers import InputLayer, DenseLayer
        >>> l_in = InputLayer((100, 20))
        >>> l1 = DenseLayer(l_in, num_units=50)
        >>> all_param_values = get_all_param_values(l1)
        >>> # all_param_values is now [l1.W.get_value(), l1.b.get_value()]
        >>> # ...
        >>> set_all_param_values(l1, all_param_values)
        >>> # the parameter values are restored.

    :parameters:
        - layer : Layer or list
            the :class:`Layer` instance for which to set all parameter
            values, or a list of :class:`Layer` instances.
        - values : list of numpy.array
            a list of numpy arrays representing the parameter values,
            must match the number of parameters
    """
    params = get_all_params(layer)
    if len(params) != len(values):
        raise ValueError("mismatch: got %d values to set %d parameters" %
                         (len(values), len(params)))
    for p, v in zip(params, values):
        p.set_value(v)
<|endoftext|>


<commit_before>#!/usr/bin/python3

import clang3.cindex
from clang3.cindex import Index
from clang3.cindex import conf
from clang3.cindex import TokenGroup
from ailearner import RNN

from sys import argv

class Formatter:
	def __init__(self, model='model.dat'):
		if not conf.loaded:
			conf.set_library_path("/usr/lib/llvm-3.8/lib")
		self.rnn = RNN.load(model) #[0-127] 128 total ascii characters, all non-ascii will be mapped onto 0
		self.modelfile = model

	def tokenize(self, filepath):
		exclude_decls_from_pch = 1
		display_diagnostics = 0

		index = Index.create(exclude_decls_from_pch)
		if not index:
			print("Can't create clang index")
			return None

		tu = index.parse(filepath.encode('utf-8'))
		return tu

	def load_original(self, filepath):
		original = ''
		with open(filepath, 'r') as f:
			for line in f:
				original += line

		return original

	def writeout_tokens(self, tu, original, filepath, debug = False):
		if not tu:
			print('tu is empty!!')
			return None

		self.rnn.reset_prediction()
		cursor = tu.cursor
		rng = cursor.extent
		with open(filepath, 'wb') as f:
			last_offset  = 0
			next_char_prediction = 0
			last_token = None
			last_char = ''
			for token in TokenGroup.get_tokens(tu, rng):
				out = ''
				inp = last_char
				original_white_spaces = ''
				predicted_white_spaces = ''

				if debug:
					out = str('tok = ' + str(token.spelling) +
						 " -> [" + str(token.extent.start.line) + ': ' + str(token.extent.start.column) + ', ' + str(token.extent.end.column) + "]" +
						" at {" + str(token.extent.start.offset) + ", " + str(token.extent.end.offset) + "}\n")

					f.write(out.encode('utf-8'))
					continue

				if last_offset != token.extent.start.offset:
					# white spaces before current token
					original_white_spaces = original[last_offset:token.extent.start.offset]

				# forming input sequence
				inp += original_white_spaces
				inp += token.spelling.decode('utf-8')

				# inserting predicted white spaces
				for ch in inp[:-1]:
					next_char_prediction = self.rnn.predict_char(ch)
					if next_char_prediction.isspace():
						predicted_white_spaces += next_char_prediction
					else:
						break
				last_char = inp[-1]

				# forming output sequence
				out += predicted_white_spaces
				out += token.spelling.decode('utf-8')
				# write it out
				f.write(out.encode('utf-8'))
				# next iteration prepare
				last_token = token
				last_offset = token.extent.end.offset

			# write out last CR
			f.write('\n'.encode('utf-8'))

	def get_model_number(self):
		import re
		return int(re.findall('\d+', self.modelfile)[-1])

	def get_output_file(self, filename):
		import os
		filename, ext = os.path.splitext(filename)
		model_number = self.get_model_number()
		return '%s.%d%s' % (filename, model_number, ext)

	def format_file(self, filepath, predict = None):
		tu = self.tokenize(filepath)
		if not tu:
			print("Can't parse source file " + filepath)
			return False

		original = self.load_original(filepath)
		out_filepath = self.get_output_file(filepath)
		self.writeout_tokens(tu, original, out_filepath)
		return True

def usage():
	print("Usage: " + argv[0] + " <source file> [model.dat]")

def main():
	if len(argv) != 2 and len(argv) != 3:
		usage()
		exit(-1)

	model = 'model.dat'
	if len(argv) == 3:
		model = argv[2]

	conf.set_library_path("/usr/lib/llvm-3.8/lib")
	fmt = Formatter(model)
	if not fmt.rnn:
		print("can't load model.dat!!! Please generate one using ailearner.py")
		exit(-1)

	fmt.format_file(argv[1])

if __name__ == '__main__':
	main()
<commit_msg>fixed loop of rnn feeding with characters<commit_after>#!/usr/bin/python3

import clang3.cindex
from clang3.cindex import Index
from clang3.cindex import conf
from clang3.cindex import TokenGroup
from ailearner import RNN

from sys import argv

class Formatter:
	def __init__(self, model='model.dat'):
		if not conf.loaded:
			conf.set_library_path("/usr/lib/llvm-3.8/lib")
		self.rnn = RNN.load(model) #[0-127] 128 total ascii characters, all non-ascii will be mapped onto 0
		self.modelfile = model

	def tokenize(self, filepath):
		exclude_decls_from_pch = 1
		display_diagnostics = 0

		index = Index.create(exclude_decls_from_pch)
		if not index:
			print("Can't create clang index")
			return None

		tu = index.parse(filepath.encode('utf-8'))
		return tu

	def load_original(self, filepath):
		original = ''
		with open(filepath, 'r') as f:
			for line in f:
				original += line

		return original

	def writeout_tokens(self, tu, original, filepath, debug = False):
		if not tu:
			print('tu is empty!!')
			return None

		self.rnn.reset_prediction()
		cursor = tu.cursor
		rng = cursor.extent
		with open(filepath, 'wb') as f, open('inp.debug', 'w') as f_inp, open('out.debug', 'w') as f_out:
			last_offset  = 0
			next_char_prediction = 0
			last_token = None
			last_char = ''
			for token in TokenGroup.get_tokens(tu, rng):
				out = ''
				inp = last_char
				original_white_spaces = ''
				predicted_white_spaces = ''

				if debug:
					out = str('tok = ' + str(token.spelling) +
						 " -> [" + str(token.extent.start.line) + ': ' + str(token.extent.start.column) + ', ' + str(token.extent.end.column) + "]" +
						" at {" + str(token.extent.start.offset) + ", " + str(token.extent.end.offset) + "}\n")

					f.write(out.encode('utf-8'))
					continue

				if last_offset != token.extent.start.offset:
					# white spaces before current token
					original_white_spaces = original[last_offset:token.extent.start.offset]

				# forming input sequence
				inp += original_white_spaces
				inp += token.spelling.decode('utf-8')

				# inserting predicted white spaces
				continuous_whitespaces = True
				for ch in inp[:-1]:
					f_inp.write(ch)
					next_char_prediction = self.rnn.predict_char(ch)
					f_out.write(next_char_prediction)
					if continuous_whitespaces and next_char_prediction.isspace():
						predicted_white_spaces += next_char_prediction
					else:
						continuous_whitespaces = False
				last_char = inp[-1]

				# forming output sequence
				out += predicted_white_spaces
				out += token.spelling.decode('utf-8')
				# write it out
				f.write(out.encode('utf-8'))
				# next iteration prepare
				last_token = token
				last_offset = token.extent.end.offset

			# write out last CR
			f.write('\n'.encode('utf-8'))

	def get_model_number(self):
		import re
		return int(re.findall('\d+', self.modelfile)[-1])

	def get_output_file(self, filename):
		import os
		filename, ext = os.path.splitext(filename)
		model_number = self.get_model_number()
		return '%s.%d%s' % (filename, model_number, ext)

	def format_file(self, filepath, predict = None):
		tu = self.tokenize(filepath)
		if not tu:
			print("Can't parse source file " + filepath)
			return False

		original = self.load_original(filepath)
		out_filepath = self.get_output_file(filepath)
		self.writeout_tokens(tu, original, out_filepath)
		return True

def usage():
	print("Usage: " + argv[0] + " <source file> [model.dat]")

def main():
	if len(argv) != 2 and len(argv) != 3:
		usage()
		exit(-1)

	model = 'model.dat'
	if len(argv) == 3:
		model = argv[2]

	conf.set_library_path("/usr/lib/llvm-3.8/lib")
	fmt = Formatter(model)
	if not fmt.rnn:
		print("can't load model.dat!!! Please generate one using ailearner.py")
		exit(-1)

	fmt.format_file(argv[1])

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2013 Piston Cloud Computing, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from __future__ import with_statement

import os
import sys
sys.path.append("./")

from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig

# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)

# add your model's MetaData object here
# for 'autogenerate' support
# from refstack.models import db
# target_metadata = db.metadata
target_metadata = None

# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.

def run_migrations_offline():
    """Run migrations in 'offline' mode.

    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.

    Calls to context.execute() here emit the given string to the
    script output.

    """
    url = config.get_main_option("sqlalchemy.url")
    context.configure(url=url)

    with context.begin_transaction():
        context.run_migrations()

def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context.

    """
    engine = engine_from_config(
                config.get_section(config.config_ini_section),
                prefix='sqlalchemy.',
                poolclass=pool.NullPool)

    connection = engine.connect()
    context.configure(
                connection=connection,
                target_metadata=target_metadata
                )

    try:
        with context.begin_transaction():
            context.run_migrations()
    finally:
        connection.close()

if context.is_offline_mode():
    run_migrations_offline()
else:
    run_migrations_online()
<commit_msg>pep8 fixes<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2013 Piston Cloud Computing, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from __future__ import with_statement
import sys
sys.path.append("./")
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig

# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config

# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)

# add your model's MetaData object here
# for 'autogenerate' support
# from refstack.models import db
# target_metadata = db.metadata
target_metadata = None

# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.

def run_migrations_offline():
    """Run migrations in 'offline' mode.

    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.

    Calls to context.execute() here emit the given string to the
    script output.

    """
    url = config.get_main_option("sqlalchemy.url")
    context.configure(url=url)

    with context.begin_transaction():
        context.run_migrations()

def run_migrations_online():
    """Run migrations in 'online' mode.

    In this scenario we need to create an Engine
    and associate a connection with the context."""
    engine = engine_from_config(
        config.get_section(config.config_ini_section),
        prefix='sqlalchemy.',
        poolclass=pool.NullPool)

    connection = engine.connect()
    context.configure(
        connection=connection,
        target_metadata=target_metadata)

    try:
        with context.begin_transaction():
            context.run_migrations()
    finally:
        connection.close()

if context.is_offline_mode():
    run_migrations_offline()
else:
    run_migrations_online()
<|endoftext|>


<commit_before>"""
Find a peak position in 1D array.

Support a is an array of length n.
If a is an array of length 1, a[0] is a peak.
In general, a[k] is a peak iff a[k] > a[k - 1] and a[k] > a[k + 1].
If a[0] > a[1], then a[0] is a peak.
If a[n - 1] > a[n - 2], then a[n - 1] is a peak.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def peak_1d_iter(arr):
    """Find peak by naive iteration.

    Time complexity: O(n).
    Space complexity: O(1).
    """
    # Iterate to check element is greater than its left & right neighbors.
    for i in range(len(arr)):
        if ((i == 0 or arr[i - 1] <= arr[i]) and
            (i == len(arr) - 1 or arr[i] >= arr[i + 1])):
            return i

def _binary_search_recur_helper(arr, left, right):
    """Helper function for peak_1d_binary_search_recur()."""
    if right - left == 0:
        return left
    else:
        mid = (left + right + 1) // 2

        if arr[mid - 1] > arr[mid]:
            # If mid's left > mid, search left part.
            return _binary_search_recur_helper(arr, left, mid - 1)
        else:
            # Otherwise, search right part.
            return _binary_search_recur_helper(arr, mid, right)

def peak_1d_binary_search_recur(arr):
    """Find peak by recursive binary search algorithm.

    Time complexity: O(logn).
    Space complexity: O(1).
    """
    return _binary_search_recur_helper(arr, 0, len(arr) - 1)

def peak_1d_binary_search_iter(arr):
    """Find peak by iterative binary search algorithm.

    Time complexity: O(logn).
    Space complexity: O(1).
    """
    left, right = 0, len(arr) - 1

    while left < right:
        mid = (left + right + 1) // 2

        if arr[mid - 1] > arr[mid]:
            # If mid's left > mid, search left part.
            right = mid - 1
        else:
            # Otherwise, search right part.
            left = mid

    # For left = right.
    return left

def main():
    import time
    import numpy as np

    # Array of length 5 with peak at 3.
    arr = [0, 1, 2, 4, 3]
    print('arr', arr)

    start_time = time.time()
    print('By peak_1d_iter(): {}'.format(peak_1d_iter(arr)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By peak_1d_binary_search_recur(): {}'
          .format(peak_1d_binary_search_recur(arr)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By peak_1d_binary_search_iter(): {}'
          .format(peak_1d_binary_search_iter(arr)))
    print('Time: {}'.format(time.time() - start_time))

    np.random.seed(71)
    arr = np.random.permutation(10)
    print('arr', arr)

    start_time = time.time()
    print('By peak_1d_iter(): {}'.format(peak_1d_iter(arr)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By peak_1d_binary_search_recur(): {}'
          .format(peak_1d_binary_search_recur(arr)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By peak_1d_binary_search_iter(): {}'
          .format(peak_1d_binary_search_iter(arr)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<commit_msg>Refactor 3 sols<commit_after>"""
Find a peak position in 1D array.

Support nums is an array of length n.
In general, nums[k] is a peak iff
nums[k] > nums[k - 1] and nums[k] > nums[k + 1].
If nums[0] > nums[1], then nums[0] is a peak.
If nums[n - 2] < nums[n - 1], then nums[n - 1] is a peak.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def peak_1d_brute_force(nums):
    """Find peak by naive iteration.

    Time complexity: O(n).
    Space complexity: O(1).
    """
    # Iterate to check element is greater than its neighbors.
    for i in range(len(nums)):
        if ((i == 0 or nums[i] >= nums[i - 1]) and
            (i == len(nums) - 1 or nums[i] >= nums[i + 1])):
            return i

def _binary_search_recur(nums, left, right):
    """Helper function for peak_1d_binary_search_recur()."""
    if right - left == 0:
        return left
    else:
        mid = left + (right - left) // 2

        if nums[mid] < nums[mid + 1]:
            # If mid < mid's right, search right part.
            return _binary_search_recur(nums, mid + 1, right)
        else:
            # Otherwise, search left part.
            return _binary_search_recur(nums, left, mid)

def peak_1d_binary_search_recur(nums):
    """Find peak by recursive binary search algorithm.

    Time complexity: O(logn).
    Space complexity: O(1).
    """
    return _binary_search_recur(nums, 0, len(nums) - 1)

def peak_1d_binary_search_iter(nums):
    """Find peak by iterative binary search algorithm.

    Time complexity: O(logn).
    Space complexity: O(1).
    """
    left, right = 0, len(nums) - 1

    while left < right:
        mid = left + (right - left) // 2

        if nums[mid] < nums[mid + 1]:
            # If mid < mid's right, search right part.
            left = mid + 1
        else:
            # Otherwise, search left part.
            right = mid

    # For left = right.
    return left

def main():
    import time
    import numpy as np

    # numsay of length 5 with peak at 3.
    nums = [0, 1, 2, 4, 3]
    print('nums', nums)

    start_time = time.time()
    print('By brute force: {}'.format(peak_1d_brute_force(nums)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By recur binary search: {}'
          .format(peak_1d_binary_search_recur(nums)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By iter binary search: {}'
          .format(peak_1d_binary_search_iter(nums)))
    print('Time: {}'.format(time.time() - start_time))

    np.random.seed(71)
    nums = np.random.permutation(10)
    print('nums', nums)

    start_time = time.time()
    print('By brute force: {}'.format(peak_1d_brute_force(nums)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By recur binary search: {}'
          .format(peak_1d_binary_search_recur(nums)))
    print('Time: {}'.format(time.time() - start_time))

    start_time = time.time()
    print('By iter binary search: {}'
          .format(peak_1d_binary_search_iter(nums)))
    print('Time: {}'.format(time.time() - start_time))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Find a peak in 2D array.

Support a is an array of length (m, n).
If a is an array of length (1, 1), a[0][0] is a peak.
In general, a[i][j] is a peak iff
  a[i][j] >= a[i][j - 1] and a[i][j] >= a[i][j + 1] and
  a[i][j] >= a[i + 1][j] and a[i][j] >= a[i - 1][j]
Similarly for corner cases, a[i][j], i = 0 or m - 1, or j = 0 or n -1.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def _max_1d(arr):
    """Find max in 1D array.

    Time complexity: O(m).
    Space complexity: O(1).
    """
    max_col = 0
    max_item = arr[0]
    for i in range(1, len(arr)):
        if arr[i] > max_item:
            max_col = i
            max_item = arr[i]
    return max_col, max_item

def peak_2d(arr, start_row, end_row):
    """Find peak in 2D array (nxm) by divide & conquer algorithm.

    Procedure:
    - Find max element in mid row to get max column.
    - Find peak in max column.

    Time complexity: O(m*log(n)).
    Space complexity: O(1).
    """
    if end_row - start_row == 0:
        # For last one row, find the max item.
        _, max_item = _max_1d(arr)
        return max_item
    else:
        mid_row = start_row + (end_row - start_row) // 2
        max_col, _ = _max_1d(arr[mid_row])
        if arr[mid_row - 1][max_col] > arr[mid_row][max_col]:
            # Binary search the upper half.
            return peak_2d(arr, start_row, mid_row - 1)
        elif arr[mid_row + 1][max_col] > arr[mid_row][max_col]:
            # Binary search the bottom half.
            return peak_2d(arr, mid_row + 1, end_row)
        else:
            # Find 2D peak.
            return arr[mid_row][max_col]

def main():
    # 2D array with peak 21 at pos (2, 3).
    arr = [[10, 8, 10, 10, 7],
           [14, 13, 12, 11, 9],
           [15, 9, 11, 21, 20],
           [16, 17, 19, 20, 18]]

    start_row, end_row = 0, len(arr) - 1
    print('Peak: {}'.format(peak_2d(arr, start_row, end_row)))

if __name__ == '__main__':
    main()
<commit_msg>Refactor to intuitive binary search sol<commit_after>"""
Find a peak in 2D array.

Support a is an array of length (m, n).
If a is an array of length (1, 1), a[0][0] is a peak.
In general, a[i][j] is a peak iff
  a[i][j] >= a[i][j - 1] and a[i][j] >= a[i][j + 1] and
  a[i][j] >= a[i + 1][j] and a[i][j] >= a[i - 1][j]
Similarly for corner cases, a[i][j], i = 0 or m - 1, or j = 0 or n -1.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

def _max_1d(nums):
    """Find max in 1D array.

    Time complexity: O(m).
    Space complexity: O(1).
    """
    max_col = 0
    max_item = nums[0]
    for i in range(1, len(nums)):
        if nums[i] > max_item:
            max_col = i
            max_item = nums[i]
    return max_col, max_item

def _peak_2d_recur(nums, start_row, end_row):
    if end_row - start_row == 0:
        # For last one row, find the max item.
        max_col, max_item = _max_1d(nums)
        return max_item
    else:
        mid_row = start_row + (end_row - start_row) // 2
        max_col, max_item = _max_1d(nums[mid_row])

        if nums[mid_row][max_col] < nums[mid_row + 1][max_col]:
            # Binary search in bottom half.
            return _peak_2d_recur(nums, mid_row + 1, end_row)
        elif nums[mid_row][max_col] < nums[mid_row - 1][max_col]:
            # Binary search in upper half.
            return _peak_2d_recur(nums, start_row, mid_row - 1)
        else:
            # Find 2D peak.
            return nums[mid_row][max_col]

def peak_2d(nums):
    """Find peak in 2D array (nxm) by divide & conquer algorithm.

    Procedure:
    - Find max element in mid row to get max column.
    - Find peak in max column.

    Time complexity: O(m*log(n)).
    Space complexity: O(1).
    """
    start_row, end_row = 0, len(nums) - 1
    return _peak_2d_recur(nums, start_row, end_row)

def main():
    # 2D array with peak 21 at pos (2, 3).
    nums = [[10, 8, 10, 10, 7],
           [14, 13, 12, 11, 9],
           [15, 9, 11, 21, 20],
           [16, 17, 19, 20, 18]]
    print(peak_2d(nums))

if __name__ == '__main__':
    main()
<|endoftext|>