<commit_before>import tkinter as tk
from tkinter.messagebox import askyesno
from tkinter.messagebox import askokcancel
from tkinter.messagebox import showwarning

class MessageBox(object):

    # ask question UI
    def ask_ques(self, title='Question', string='Do you want to save?', icontype='info'):
        # icontype can be "error", "warning", "info"
        self.root = tk.Tk()
        self.center()
        self.root.withdraw()
        result = askokcancel(title, string, icon=icontype)
        self.root.destroy()
        self.root.mainloop()
        return result

    def alert(self, title='Alert', string='Please at least add one target'):
        self.root = tk.Tk()
        self.root.withdraw()
        showwarning(title, string)
        self.root.destroy()
        self.root.mainloop()

    def center(self):
        self.root.update_idletasks()
        w = self.root.winfo_screenwidth()
        h = self.root.winfo_screenheight()
        size = tuple(int(_) for _ in self.root.geometry().split('+')[0].split('x'))
        x = w/2 - size[0]/2
        y = h/2 - size[1]/2
        self.root.geometry("%dx%d+%d+%d" % (size + (x, y)))<commit_msg>add: instruction box<commit_after>import tkinter as tk
from tkinter import ttk
from tkinter.messagebox import askyesno
from tkinter.messagebox import askokcancel
from tkinter.messagebox import showwarning

class MessageBox(object):

    # ask question UI
    def ask_ques(self, title='Question', string='Do you want to save?', icontype='info'):
        # icontype can be "error", "warning", "info"
        self.root = tk.Tk()
        self.center()
        self.root.withdraw()
        result = askokcancel(title, string, icon=icontype)
        self.root.destroy()
        self.root.mainloop()
        return result

    def alert(self, title='Alert', string='Please at least add one target'):
        self.root = tk.Tk()
        self.root.withdraw()
        showwarning(title, string)
        self.root.destroy()
        self.root.mainloop()

    def center(self):
        self.root.update_idletasks()
        w = self.root.winfo_screenwidth()
        h = self.root.winfo_screenheight()
        size = tuple(int(_) for _ in self.root.geometry().split('+')[0].split('x'))
        x = w/2 - size[0]/2
        y = h/2 - size[1]/2
        self.root.geometry("%dx%d+%d+%d" % (size + (x, y)))

class Instruction(object):

    def __init__(self, title='Settings'):
        super().__init__()
        self.title = title
        self.__hotkey = []
        self.__action = []

    def reset(self):
        self.__hotkey = []
        self.__action = []

    def row_append(self, hotkey, description):
        self.__hotkey.append(hotkey)
        self.__action.append(description)

    def show(self):
        self.window = tk.Tk()
        self.window.title(self.title)
        self.window.resizable(0, 0)

        hotkey = ttk.LabelFrame(self.window, text='Hotkey')
        hotkey.grid(column=0, row=len(self.__hotkey))
        for i, k in enumerate(self.__hotkey):
            ttk.Label(hotkey, text=k).grid(column=0, row=i)

        action = ttk.LabelFrame(self.window, text='Action')
        action.grid(column=1, row=len(self.__action))
        for i, description in enumerate(self.__action):
            ttk.Label(action, text=description).grid(column=1, row=i, sticky=tk.W)

        self.window.mainloop()
<|endoftext|>


<commit_before>from nacl.exceptions import CryptoError
from scapy.all import Ether, sendp, sniff, bind_layers
from time import time, sleep
import sched
from threading import Thread
from argparse import ArgumentParser
from random import sample
from atexit import register
from pyric.pyw import macget, getcard

from pyadtn.message_store import DataStore
from pyadtn.key_manager import KeyManager
from pyadtn.aDTN_packet import aDTNPacket, aDTNInnerPacket
from pyadtn.utils import log_network, log_debug

ETHERTYPE = 0xcafe
FILTER = "ether proto 0xcafe and not ether src "
SNIFF_TIMEOUT = 5

class aDTN:
    """
    Receives and sends aDTN packets.
    Keys used for encrypting and decrypting the packets are stored in a KeyManager.
    Received payload is stored in a MessageStore instance.
    """
    def __init__(self, batch_size, sending_interval, wireless_interface, data_store):
        """
        Initialize an aDTN instance and its respective key manager and message store, as well as a sending message pool
        from which the next sending batch gets generated.

        Define aDTNInnerPacket to be the payload of aDTNPacket. Define aDTNPacket to be the payload of Ethernet frames
        of type 0xcafe.

        Set up a scheduler to handle message sending.
        Define a thread to handle received messages.

        The wireless interface should be previously set to ad-hoc mode and its ESSID should be the same in other devices
        running aDTN.
        :param batch_size: number of packets to transmit at each sending operation
        :param sending_interval: number of seconds between two sending operations
        :param wireless_interface: wireless interface to send and receive packets
        """
        self._batch_size = batch_size
        self._sending_freq = sending_interval
        self._wireless_interface = wireless_interface
        self._km = KeyManager()
        self.data_store = DataStore(data_store)
        self._sending_pool = []
        self._scheduler = sched.scheduler(time, sleep)
        self._sending = None
        self._sniffing = None
        self._thread_send = None
        self._thread_receive = None
        self._sent_pkt_counter = None
        self._received_pkt_counter = None
        self._decrypted_pkt_counter = None
        self._start_time = None
        self._mac_address = macget(getcard(wireless_interface))
        bind_layers(aDTNPacket, aDTNInnerPacket)
        bind_layers(Ether, aDTNPacket, type=ETHERTYPE)
        log_debug("MAC address in use: {}".format(self._mac_address))

    def _prepare_sending_pool(self):
        """
        Refill the sending pool with packets if its length drops below the sending batch size. Packets contain
        encrypted messages from the message store. If there are not enough messages to be sent, fake packets are
        generated until the sending pool is full.
        """
        if len(self._sending_pool) < self._batch_size:
            to_send = self.data_store.get_data()[:self._batch_size]
            for message in to_send:
                for key in self._km.keys.values():
                    packet = aDTNPacket(key=key) / aDTNInnerPacket() / message
                    self._sending_pool.append(packet)
            while len(self._sending_pool) < self._batch_size:
                fake_key = self._km.get_fake_key()
                packet = aDTNPacket(key=fake_key) / aDTNInnerPacket()
                self._sending_pool.append(packet)

    def _send(self):
        """
        Send a batch of randomly selected packets from the sending pool, then ensure the sending pool gets refilled if
        necessary. The packets are encapsulated in an Ethernet frame of type 0xcafe and removed from the sending pool,
        and finally broadcast in a batch.
        This function reschedules itself to occur every sending_freq seconds.
        """
        self._scheduler.enter(self._sending_freq, 1, self._send)
        log_debug("Sending scheduler queue length: {}".format(len(self._scheduler.queue)))
        if self._sending:
            batch = []
            s = sample(self._sending_pool, self._batch_size)
            for pkt in s:
                batch.append(Ether(dst="ff:ff:ff:ff:ff:ff", src=self._mac_address, type=ETHERTYPE) / pkt)
                self._sending_pool.remove(pkt)
            t_before = time()
            sendp(batch, iface=self._wireless_interface, verbose=False)
            t_after = time()
            self._sent_pkt_counter += len(batch)
            log_network("snt {} in {}s".format(len(batch), t_after - t_before))
            self._prepare_sending_pool()

    def _process(self, frame):
        """
        Process a received frame by attempting to decrypt its payload - the aDTN packet - with every key in the key
        store. If a decryption succeeds, the extracted message is stored in the message store, otherwise the next key is
        used. If all decryptions fail, the packet is discarded.
        :param frame: Ethernet frame containing an aDTN packet
        """
        payload = frame.payload.load
        self._received_pkt_counter += 1
        for key in self._km.keys.values():
            try:
                ap = aDTNPacket(key=key)
                ap.dissect(payload)
                msg = ap.payload.payload.load.decode('utf-8')
                self._decrypted_pkt_counter += 1
                self.data_store.add_object(msg)
                log_network("rcv OK")
                return
            except CryptoError:
                pass
            except ValueError:
                log_debug("invalid nonce")
        log_network("rcv NO_KEY")

    def _sniff(self):
        """ Wrapper for packet sniffing. """
        while True:
            if self._sniffing is False:
                return
            sniff(iface=self._wireless_interface, prn=self._process, filter=FILTER + self._mac_address, store=0,
                  timeout=SNIFF_TIMEOUT)

    def start(self):
        """
        Run the aDTN network functionality in two threads, one for sending and the other for receiving. Received
        Ethernet frames are filtered for ethertype and processed if they match the 0xcafe type. The sending thread runs
        a scheduler for periodic sending of aDTN packets.
        """
        self._start_time = time()
        self._sent_pkt_counter = 0
        self._received_pkt_counter = 0
        self._decrypted_pkt_counter = 0
        self._prepare_sending_pool()
        self._scheduler.enter(self._sending_freq, 1, self._send)
        self._sniffing = True
        self._thread_receive = Thread(target=self._sniff, name="ReceivingThread")
        self._sending = True
        self._thread_send = Thread(target=self._scheduler.run, name="SendingThread", kwargs={"blocking": True})
        log_network("start-{}-{}".format(self._batch_size, self._sending_freq))
        self._thread_receive.start()
        sleep(5)
        self._thread_send.start()

    def stop(self):
        """
        Stop aDTN. Make sure the two threads created at start are finished properly.
        """
        self._sending = False
        try:
            while not self._scheduler.empty():
                event = self._scheduler.queue.pop()
                self._scheduler.cancel(event)
            # By now the scheduler has run empty, so join the thread:
            self._thread_send.join()
            sleep(5)
            # Now we just have to join the receiving thread to stop aDTN completely:
            self._sniffing = False
            self._thread_receive.join()
            log_network("stop")
        except ValueError:  # In case the popped event started running in the meantime...
            log_debug("Scheduler is not empty, retry stopping.")
            self.stop()  # ...call the stop function once more.

def parse_args():
    """ Parse command line arguments.
    :return: arguments received via the command line
    """
    parser = ArgumentParser(description='Run an aDTN simulation instance.')
    parser.add_argument('batch_size', type=int, help='how many messages to send in a batch')
    parser.add_argument('sending_interval', type=float, help='interval (in s) between sending a batch')
    parser.add_argument('wireless_interface', type=str, help='name of the wireless interface')
    parser.add_argument('data_store', type=str, default=None, help="file storing the data objects")
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_args()
    adtn = aDTN(args.batch_size, args.sending_interval, args.wireless_interface, args.data_store)
    register(aDTN.stop, adtn)
    adtn.start()
<commit_msg>log sending activity<commit_after>from nacl.exceptions import CryptoError
from scapy.all import Ether, sendp, sniff, bind_layers
from time import time, sleep
import sched
from threading import Thread
from argparse import ArgumentParser
from random import sample
from atexit import register
from pyric.pyw import macget, getcard

from pyadtn.message_store import DataStore
from pyadtn.key_manager import KeyManager
from pyadtn.aDTN_packet import aDTNPacket, aDTNInnerPacket
from pyadtn.utils import log_network, log_debug

ETHERTYPE = 0xcafe
FILTER = "ether proto 0xcafe and not ether src "
SNIFF_TIMEOUT = 5

class aDTN:
    """
    Receives and sends aDTN packets.
    Keys used for encrypting and decrypting the packets are stored in a KeyManager.
    Received payload is stored in a MessageStore instance.
    """
    def __init__(self, batch_size, sending_interval, wireless_interface, data_store):
        """
        Initialize an aDTN instance and its respective key manager and message store, as well as a sending message pool
        from which the next sending batch gets generated.

        Define aDTNInnerPacket to be the payload of aDTNPacket. Define aDTNPacket to be the payload of Ethernet frames
        of type 0xcafe.

        Set up a scheduler to handle message sending.
        Define a thread to handle received messages.

        The wireless interface should be previously set to ad-hoc mode and its ESSID should be the same in other devices
        running aDTN.
        :param batch_size: number of packets to transmit at each sending operation
        :param sending_interval: number of seconds between two sending operations
        :param wireless_interface: wireless interface to send and receive packets
        """
        self._batch_size = batch_size
        self._sending_freq = sending_interval
        self._wireless_interface = wireless_interface
        self._km = KeyManager()
        self.data_store = DataStore(data_store)
        self._sending_pool = []
        self._scheduler = sched.scheduler(time, sleep)
        self._sending = None
        self._sniffing = None
        self._thread_send = None
        self._thread_receive = None
        self._sent_pkt_counter = None
        self._received_pkt_counter = None
        self._decrypted_pkt_counter = None
        self._start_time = None
        self._mac_address = macget(getcard(wireless_interface))
        bind_layers(aDTNPacket, aDTNInnerPacket)
        bind_layers(Ether, aDTNPacket, type=ETHERTYPE)
        log_debug("MAC address in use: {}".format(self._mac_address))
        self._stats_file_name = '{}_{}.stats'.format(batch_size, sending_interval)

    def _prepare_sending_pool(self):
        """
        Refill the sending pool with packets if its length drops below the sending batch size. Packets contain
        encrypted messages from the message store. If there are not enough messages to be sent, fake packets are
        generated until the sending pool is full.
        """
        if len(self._sending_pool) < self._batch_size:
            to_send = self.data_store.get_data()[:self._batch_size]
            for message in to_send:
                for key in self._km.keys.values():
                    packet = aDTNPacket(key=key) / aDTNInnerPacket() / message
                    self._sending_pool.append(packet)
            while len(self._sending_pool) < self._batch_size:
                fake_key = self._km.get_fake_key()
                packet = aDTNPacket(key=fake_key) / aDTNInnerPacket()
                self._sending_pool.append(packet)

    def _send(self):
        """
        Send a batch of randomly selected packets from the sending pool, then ensure the sending pool gets refilled if
        necessary. The packets are encapsulated in an Ethernet frame of type 0xcafe and removed from the sending pool,
        and finally broadcast in a batch.
        This function reschedules itself to occur every sending_freq seconds.
        """
        self._scheduler.enter(self._sending_freq, 1, self._send)
        log_debug("Sending scheduler queue length: {}".format(len(self._scheduler.queue)))
        if self._sending:
            batch = []
            s = sample(self._sending_pool, self._batch_size)
            for pkt in s:
                batch.append(Ether(dst="ff:ff:ff:ff:ff:ff", src=self._mac_address, type=ETHERTYPE) / pkt)
                self._sending_pool.remove(pkt)
            t_before = time()
            sendp(batch, iface=self._wireless_interface, verbose=False)
            t_after = time()
            with open(self._stats_file_name, 'a') as stats_file:
                stats_file.write('{},{},{}\n'.format(t_before, t_after, len(batch)))
            self._sent_pkt_counter += len(batch)
            log_network("snt {} in {}s".format(len(batch), t_after - t_before))
            self._prepare_sending_pool()

    def _process(self, frame):
        """
        Process a received frame by attempting to decrypt its payload - the aDTN packet - with every key in the key
        store. If a decryption succeeds, the extracted message is stored in the message store, otherwise the next key is
        used. If all decryptions fail, the packet is discarded.
        :param frame: Ethernet frame containing an aDTN packet
        """
        payload = frame.payload.load
        self._received_pkt_counter += 1
        for key in self._km.keys.values():
            try:
                ap = aDTNPacket(key=key)
                ap.dissect(payload)
                msg = ap.payload.payload.load.decode('utf-8')
                self._decrypted_pkt_counter += 1
                self.data_store.add_object(msg)
                log_network("rcv OK")
                return
            except CryptoError:
                pass
            except ValueError:
                log_debug("invalid nonce")
        log_network("rcv NO_KEY")

    def _sniff(self):
        """ Wrapper for packet sniffing. """
        while True:
            if self._sniffing is False:
                return
            sniff(iface=self._wireless_interface, prn=self._process, filter=FILTER + self._mac_address, store=0,
                  timeout=SNIFF_TIMEOUT)

    def start(self):
        """
        Run the aDTN network functionality in two threads, one for sending and the other for receiving. Received
        Ethernet frames are filtered for ethertype and processed if they match the 0xcafe type. The sending thread runs
        a scheduler for periodic sending of aDTN packets.
        """
        self._start_time = time()
        self._sent_pkt_counter = 0
        self._received_pkt_counter = 0
        self._decrypted_pkt_counter = 0
        self._prepare_sending_pool()
        self._scheduler.enter(self._sending_freq, 1, self._send)
        self._sniffing = True
        self._thread_receive = Thread(target=self._sniff, name="ReceivingThread")
        self._sending = True
        self._thread_send = Thread(target=self._scheduler.run, name="SendingThread", kwargs={"blocking": True})
        log_network("start-{}-{}".format(self._batch_size, self._sending_freq))
        self._thread_receive.start()
        sleep(5)
        self._thread_send.start()

    def stop(self):
        """
        Stop aDTN. Make sure the two threads created at start are finished properly.
        """
        self._sending = False
        try:
            while not self._scheduler.empty():
                event = self._scheduler.queue.pop()
                self._scheduler.cancel(event)
            # By now the scheduler has run empty, so join the thread:
            self._thread_send.join()
            sleep(5)
            # Now we just have to join the receiving thread to stop aDTN completely:
            self._sniffing = False
            self._thread_receive.join()
            log_network("stop")
        except ValueError:  # In case the popped event started running in the meantime...
            log_debug("Scheduler is not empty, retry stopping.")
            self.stop()  # ...call the stop function once more.

def parse_args():
    """ Parse command line arguments.
    :return: arguments received via the command line
    """
    parser = ArgumentParser(description='Run an aDTN simulation instance.')
    parser.add_argument('batch_size', type=int, help='how many messages to send in a batch')
    parser.add_argument('sending_interval', type=float, help='interval (in s) between sending a batch')
    parser.add_argument('wireless_interface', type=str, help='name of the wireless interface')
    parser.add_argument('data_store', type=str, default=None, help="file storing the data objects")
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_args()
    adtn = aDTN(args.batch_size, args.sending_interval, args.wireless_interface, args.data_store)
    register(aDTN.stop, adtn)
    adtn.start()
<|endoftext|>


<commit_before>from nacl.exceptions import CryptoError
from scapy.all import Ether, sendp, sniff, bind_layers
import time
import sched
from threading import Thread
from argparse import ArgumentParser
from random import sample
from atexit import register
from logging import basicConfig, debug, DEBUG

from pyadtn.message_store import DataStore
from pyadtn.key_manager import KeyManager
from pyadtn.aDTN_packet import aDTNPacket, aDTNInnerPacket

ETHERTYPE = "0xcafe"
FILTER = "ether proto " + ETHERTYPE
SNIFF_TIMEOUT = 5

basicConfig(filename='aDTN.log', level=DEBUG,
            format='[%(relativeCreated)8d] %(message)s', )

class aDTN:
    """
    Receives and sends aDTN packets.
    Keys used for encrypting and decrypting the packets are stored in a KeyManager.
    Received payload is stored in a MessageStore instance.
    """
    def __init__(self, batch_size, sending_freq, wireless_interface, data_store):
        """
        Initialize an aDTN instance and its respective key manager and message store, as well as a sending message pool
        from which the next sending batch gets generated.

        Define aDTNInnerPacket to be the payload of aDTNPacket. Define aDTNPacket to be the payload of Ethernet frames
        of type 0xcafe.

        Set up a scheduler to handle message sending.
        Define a thread to handle received messages.

        The wireless interface should be previously set to ad-hoc mode and its ESSID should be the same in other devices
        running aDTN.
        :param batch_size: number of packets to transmit at each sending operation
        :param sending_freq: number of seconds between two sending operations
        :param wireless_interface: wireless interface to send and receive packets
        """
        self._batch_size = batch_size
        self._sending_freq = sending_freq
        self._wireless_interface = wireless_interface
        self._km = KeyManager()
        self.data_store = DataStore(data_store)
        self._sending_pool = []
        self._scheduler = sched.scheduler(time.time, time.sleep)
        self._sending = None
        self._sniffing = None
        self._thread_send = None
        self._thread_receive = None
        self._sent_pkt_counter = None
        self._received_pkt_counter = None
        self._decrypted_pkt_counter = None
        self._start_time = None
        bind_layers(aDTNPacket, aDTNInnerPacket)
        bind_layers(Ether, aDTNPacket, type=0xcafe)

    def _prepare_sending_pool(self):
        """
        Refill the sending pool with packets if its length drops below the sending batch size. Packets contain
        encrypted messages from the message store. If there are not enough messages to be sent, fake packets are
        generated until the sending pool is full.
        """
        if len(self._sending_pool) < self._batch_size:
            to_send = self.data_store.get_data()[:self._batch_size]
            for message in to_send:
                for key in self._km.keys.values():
                    packet = aDTNPacket(key=key) / aDTNInnerPacket() / message
                    self._sending_pool.append(packet)
            while len(self._sending_pool) < self._batch_size:
                fake_key = self._km.get_fake_key()
                self._sending_pool.append((aDTNPacket(key=fake_key) / aDTNInnerPacket()))

    def _send(self):
        """
        Send a batch of randomly selected packets from the sending pool, then ensure the sending pool gets refilled if
        necessary. The packets are encapsulated in an Ethernet frame of type 0xcafe and removed from the sending pool,
        and finally broadcast in a batch.
        This function reschedules itself to occur every sending_freq seconds.
        """
        self._scheduler.enter(self._sending_freq, 1, self._send)
        if self._sending:
            batch = []
            s = sample(self._sending_pool, self._batch_size)
            for pkt in s:
                batch.append(Ether(dst="ff:ff:ff:ff:ff:ff", type=ETHERTYPE) / pkt)
                self._sending_pool.remove(pkt)
            sendp(batch, iface=self._wireless_interface)
            self._sent_pkt_counter += len(batch)
            self._prepare_sending_pool()

    def _process(self, frame):
        """
        Process a received frame by attempting to decrypt its payload - the aDTN packet - with every key in the key
        store. If a decryption succeeds, the extracted message is stored in the message store, otherwise the next key is
        used. If all decryptions fail, the packet is discarded.
        :param frame: Ethernet frame containing an aDTN packet
        """
        payload = frame.payload.load
        self._received_pkt_counter += 1
        for key in self._km.keys.values():
            try:
                ap = aDTNPacket(key=key)
                ap.dissect(payload)
                msg = ap.payload.payload.load.decode('utf-8')
                self._decrypted_pkt_counter += 1
                self.data_store.add_object(msg)
                return
            except CryptoError:
                pass

    def _sniff(self):
        """ Wrapper for packet sniffing. """
        while True:
            if self._sniffing is False:
                return
            sniff(iface=self._wireless_interface, prn=self._process, filter=FILTER, store=0, timeout=SNIFF_TIMEOUT)

    def start(self):
        """
        Run the aDTN network functionality in two threads, one for sending and the other for receiving. Received
        Ethernet frames are filtered for ethertype and processed if they match the 0xcafe type. The sending thread runs
        a scheduler for periodic sending of aDTN packets.
        """
        self._start_time = time.time()
        self._sent_pkt_counter = 0
        self._received_pkt_counter = 0
        self._decrypted_pkt_counter = 0
        self._prepare_sending_pool()
        self._scheduler.enter(self._sending_freq, 1, self._send)
        self._sending = True
        self._thread_send = Thread(target=self._scheduler.run, kwargs={"blocking": True})
        self._thread_send.start()
        self._sniffing = True
        self._thread_receive = Thread(target=self._sniff)
        self._thread_receive.start()

    def stop(self):
        """
        Stop aDTN. Make sure the two threads created at start are finished properly.
        """
        self._sending = False
        try:
            while not self._scheduler.empty():
                event = self._scheduler.queue.pop()
                self._scheduler.cancel(event)
        except ValueError:  # In case the popped event started running in the meantime...
            self.stop()  # ...call the stop function once more.
        # By now the scheduler has run empty, so join the thread:
        self._thread_send.join()
        # Now we just have to join the receiving thread to stop aDTN completely:
        self._sniffing = False
        self._thread_receive.join()
        stop_time = time.time()
        uptime = stop_time - self._start_time
        debug("Running time: {} -- sent: {}, received: {}, decrypted: {}, batch size: {}, sending freq.: {}.".format(
            uptime, self._sent_pkt_counter, self._received_pkt_counter, self._decrypted_pkt_counter, self._batch_size,
            self._sending_freq))

def parse_args():
    """ Parse command line arguments.
    :return: arguments received via the command line
    """
    parser = ArgumentParser(description='Run an aDTN simulation instance.')
    parser.add_argument('batch_size', type=int, help='how many messages to send in a batch')
    parser.add_argument('sending_freq', type=int, help='interval (in s) between sending a batch')
    parser.add_argument('wireless_interface', type=str, help='name of the wireless interface')
    parser.add_argument('data_store', type=str, default=None, help="file storing the data objects")
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_args()
    adtn = aDTN(args.batch_size, args.sending_freq, args.wireless_interface, args.data_store)
    register(aDTN.stop, adtn)
    adtn.start()
<commit_msg>use constant ethertype when binding layers<commit_after>from nacl.exceptions import CryptoError
from scapy.all import Ether, sendp, sniff, bind_layers
import time
import sched
from threading import Thread
from argparse import ArgumentParser
from random import sample
from atexit import register
from logging import basicConfig, debug, DEBUG

from pyadtn.message_store import DataStore
from pyadtn.key_manager import KeyManager
from pyadtn.aDTN_packet import aDTNPacket, aDTNInnerPacket

ETHERTYPE = "0xcafe"
FILTER = "ether proto " + ETHERTYPE
SNIFF_TIMEOUT = 5

basicConfig(filename='aDTN.log', level=DEBUG,
            format='[%(relativeCreated)8d] %(message)s', )

class aDTN:
    """
    Receives and sends aDTN packets.
    Keys used for encrypting and decrypting the packets are stored in a KeyManager.
    Received payload is stored in a MessageStore instance.
    """
    def __init__(self, batch_size, sending_freq, wireless_interface, data_store):
        """
        Initialize an aDTN instance and its respective key manager and message store, as well as a sending message pool
        from which the next sending batch gets generated.

        Define aDTNInnerPacket to be the payload of aDTNPacket. Define aDTNPacket to be the payload of Ethernet frames
        of type 0xcafe.

        Set up a scheduler to handle message sending.
        Define a thread to handle received messages.

        The wireless interface should be previously set to ad-hoc mode and its ESSID should be the same in other devices
        running aDTN.
        :param batch_size: number of packets to transmit at each sending operation
        :param sending_freq: number of seconds between two sending operations
        :param wireless_interface: wireless interface to send and receive packets
        """
        self._batch_size = batch_size
        self._sending_freq = sending_freq
        self._wireless_interface = wireless_interface
        self._km = KeyManager()
        self.data_store = DataStore(data_store)
        self._sending_pool = []
        self._scheduler = sched.scheduler(time.time, time.sleep)
        self._sending = None
        self._sniffing = None
        self._thread_send = None
        self._thread_receive = None
        self._sent_pkt_counter = None
        self._received_pkt_counter = None
        self._decrypted_pkt_counter = None
        self._start_time = None
        bind_layers(aDTNPacket, aDTNInnerPacket)
        bind_layers(Ether, aDTNPacket, type=ETHERTYPE)

    def _prepare_sending_pool(self):
        """
        Refill the sending pool with packets if its length drops below the sending batch size. Packets contain
        encrypted messages from the message store. If there are not enough messages to be sent, fake packets are
        generated until the sending pool is full.
        """
        if len(self._sending_pool) < self._batch_size:
            to_send = self.data_store.get_data()[:self._batch_size]
            for message in to_send:
                for key in self._km.keys.values():
                    packet = aDTNPacket(key=key) / aDTNInnerPacket() / message
                    self._sending_pool.append(packet)
            while len(self._sending_pool) < self._batch_size:
                fake_key = self._km.get_fake_key()
                self._sending_pool.append((aDTNPacket(key=fake_key) / aDTNInnerPacket()))

    def _send(self):
        """
        Send a batch of randomly selected packets from the sending pool, then ensure the sending pool gets refilled if
        necessary. The packets are encapsulated in an Ethernet frame of type 0xcafe and removed from the sending pool,
        and finally broadcast in a batch.
        This function reschedules itself to occur every sending_freq seconds.
        """
        self._scheduler.enter(self._sending_freq, 1, self._send)
        if self._sending:
            batch = []
            s = sample(self._sending_pool, self._batch_size)
            for pkt in s:
                batch.append(Ether(dst="ff:ff:ff:ff:ff:ff", type=ETHERTYPE) / pkt)
                self._sending_pool.remove(pkt)
            sendp(batch, iface=self._wireless_interface)
            self._sent_pkt_counter += len(batch)
            self._prepare_sending_pool()

    def _process(self, frame):
        """
        Process a received frame by attempting to decrypt its payload - the aDTN packet - with every key in the key
        store. If a decryption succeeds, the extracted message is stored in the message store, otherwise the next key is
        used. If all decryptions fail, the packet is discarded.
        :param frame: Ethernet frame containing an aDTN packet
        """
        payload = frame.payload.load
        self._received_pkt_counter += 1
        for key in self._km.keys.values():
            try:
                ap = aDTNPacket(key=key)
                ap.dissect(payload)
                msg = ap.payload.payload.load.decode('utf-8')
                self._decrypted_pkt_counter += 1
                self.data_store.add_object(msg)
                return
            except CryptoError:
                pass

    def _sniff(self):
        """ Wrapper for packet sniffing. """
        while True:
            if self._sniffing is False:
                return
            sniff(iface=self._wireless_interface, prn=self._process, filter=FILTER, store=0, timeout=SNIFF_TIMEOUT)

    def start(self):
        """
        Run the aDTN network functionality in two threads, one for sending and the other for receiving. Received
        Ethernet frames are filtered for ethertype and processed if they match the 0xcafe type. The sending thread runs
        a scheduler for periodic sending of aDTN packets.
        """
        self._start_time = time.time()
        self._sent_pkt_counter = 0
        self._received_pkt_counter = 0
        self._decrypted_pkt_counter = 0
        self._prepare_sending_pool()
        self._scheduler.enter(self._sending_freq, 1, self._send)
        self._sending = True
        self._thread_send = Thread(target=self._scheduler.run, kwargs={"blocking": True})
        self._thread_send.start()
        self._sniffing = True
        self._thread_receive = Thread(target=self._sniff)
        self._thread_receive.start()

    def stop(self):
        """
        Stop aDTN. Make sure the two threads created at start are finished properly.
        """
        self._sending = False
        try:
            while not self._scheduler.empty():
                event = self._scheduler.queue.pop()
                self._scheduler.cancel(event)
        except ValueError:  # In case the popped event started running in the meantime...
            self.stop()  # ...call the stop function once more.
        # By now the scheduler has run empty, so join the thread:
        self._thread_send.join()
        # Now we just have to join the receiving thread to stop aDTN completely:
        self._sniffing = False
        self._thread_receive.join()
        stop_time = time.time()
        uptime = stop_time - self._start_time
        debug("Running time: {} -- sent: {}, received: {}, decrypted: {}, batch size: {}, sending freq.: {}.".format(
            uptime, self._sent_pkt_counter, self._received_pkt_counter, self._decrypted_pkt_counter, self._batch_size,
            self._sending_freq))

def parse_args():
    """ Parse command line arguments.
    :return: arguments received via the command line
    """
    parser = ArgumentParser(description='Run an aDTN simulation instance.')
    parser.add_argument('batch_size', type=int, help='how many messages to send in a batch')
    parser.add_argument('sending_freq', type=int, help='interval (in s) between sending a batch')
    parser.add_argument('wireless_interface', type=str, help='name of the wireless interface')
    parser.add_argument('data_store', type=str, default=None, help="file storing the data objects")
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_args()
    adtn = aDTN(args.batch_size, args.sending_freq, args.wireless_interface, args.data_store)
    register(aDTN.stop, adtn)
    adtn.start()
<|endoftext|>


<commit_before>import os
import uuid
import json
import random

import web
from jinja2 import Environment, Template, FileSystemLoader

from config import settings
from blast import get_blast_databases, run_blast, get_sequence_from_database, process_blast_result, get_blast_database_from_title

NBLAST = ('blastn', 'tblastn', 'tblastx',)
PBLAST = ('blastp', 'blastx',)

settings = settings()

'''
urls = (
    '{}'.format(settings.get('BASE_URL')), 'index',
    '{}results/(.*)/'.format(settings.get('BASE_URL')), 'results',
    '{}run/(.*)/'.format(settings.get('BASE_URL')), 'start_blast',
    '{}status/(.*)/'.format(settings.get('BASE_URL')), 'status',
    '{}sequence/(.*)/(.*)/'.format(settings.get('BASE_URL')), 'sequence',
    '{}multisequence'.format(settings.get('BASE_URL')), 'multisequence',
    )
'''

#app = web.application(urls, globals())
#application = app.wsgifunc()

def render_template(template_name, base_template=None, **context):
    extensions = context.pop('extensions', [])
    globals = context.pop('globals', {})

    if base_template is not None and base_template != 'base.html':
        context['layout_template'] = Template(open(base_template).read())
    else:
        context['layout_template'] = 'base.html'

    location = os.path.join(os.path.dirname(__file__), 'templates')

    jinja_env = Environment(
        loader=FileSystemLoader(location),
        extensions=extensions,
        )

    jinja_env.globals.update(globals)

    return jinja_env.get_template(template_name).render(context)

class index:
    def GET(self):
        databases = get_blast_databases(settings.get('BLAST_EXECUTABLE_PATH'), settings.get('BLAST_DATABASE_PATH'))
        programs = {'nucleotide': NBLAST, 'protein': PBLAST}
        qs = web.input()
        identifier = qs.get('settings')
        db = ''
        pgrm = ''
        seq = ''
        adv = ''
        if identifier is not None:
            with open('{0}{1}.json'.format(settings.get('BLAST_RESULTS_PATH'), identifier)) as sp:
                opt = json.loads(sp.read())
                db = opt['database']
                pgrm = opt['program']
                seq = opt['sequence']
                adv = opt['advanced']
        return render_template('index.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title="pyBlast | Enter sequence to BLAST",
            databases = databases,
            programs = programs,
            uuid = uuid.uuid4(),
            db = db,
            pgrm = pgrm,
            sequence = seq,
            advanced = adv,
            rndm = random.random(),
        )

class results:
    def POST(self, identifier):
        options = web.input()
        return render_template('results_container.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title="pyBlast | BLAST results",
            identifier = identifier,
            options = json.dumps(options),
            execute = True,
            rndm = random.random(),
        )

    def GET(self, identifier):
        return render_template('results_container.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title="pyBlast | BLAST results",
            identifier = identifier,
            rndm = random.random(),
        )

class start_blast:
    def POST(self, identifier):
        options = web.input()
        web.header('Content-type', 'application/json')
        try:
            database = get_blast_database_from_title(settings.get('BLAST_EXECUTABLE_PATH'),settings.get('BLAST_DATABASE_PATH'),options.database)
            #database = settings.get('BLAST_DATABASE_PATH')+options.database
            program = settings.get('BLAST_EXECUTABLE_PATH')+options.program
            sequence = options.sequence
            advanced = options.advanced
            out, err = run_blast(database, program, settings.get('BLAST_RESULTS_PATH'), identifier, sequence, advanced)
            with open('{0}/{1}.json'.format(settings.get('BLAST_RESULTS_PATH'), identifier), 'w+') as sp:
                sp.write(json.dumps({
                    'database': options.database,
                    'program': options.program,
                    'sequence': sequence,
                    'advanced': advanced,
                }))
            return json.dumps({'running': True, 'out': out, 'err': err})
        except KeyError:
            web.ctx.status = 400
            return json.dumps({'running': False})

class status:
    def GET(self, identifier):
        web.header('Content-type', 'application/json')
        file_loc = "{0}{1}.xml".format(settings.get('BLAST_RESULTS_PATH'), identifier)
        qs = web.input()
        cutoff = qs.get('cutoff')
        with open(file_loc) as results:
            if cutoff is not None:
                processed = process_blast_result(results.read(), cutoff=float(cutoff))
            else:
                processed = process_blast_result(results.read())

            results_page = render_template('results.html',
                settings = settings,
                results = processed,
                uuid = uuid.uuid4(),
                identifier = identifier,
            )
            return json.dumps({'active': False, 'results': processed, 'results_page': results_page})
        return json.dumps({'active': False, 'err': 'There has been an error'})

class sequence:
    def GET(self, db, sequence):
        seqs = sequence.split(',')
        results = []
        for s in seqs:
            results.append([s, get_sequence_from_database(settings.get('BLAST_EXECUTABLE_PATH'), settings.get('BLAST_DATABASE_PATH')+db, s)])
        return render_template('sequences.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title = 'pyBlast | Retrieved sequences',
            results = results,
        )

class multisequence:
    def GET(self):
        qs = web.input(sequences=[])
        seqs = u','.join(qs.sequences)
        s = sequence()
        return s.GET(qs.db, seqs)

app = None

def run(overidden_settings={}, run_server=False):
    settings.merge(overidden_settings)
    urls = (
        '{}'.format(settings.get('BASE_URL')), 'index',
        '{}results/(.*)/'.format(settings.get('BASE_URL')), 'results',
        '{}run/(.*)/'.format(settings.get('BASE_URL')), 'start_blast',
        '{}status/(.*)/'.format(settings.get('BASE_URL')), 'status',
        '{}sequence/(.*)/(.*)/'.format(settings.get('BASE_URL')), 'sequence',
        '{}multisequence'.format(settings.get('BASE_URL')), 'multisequence',
        )
    app = web.application(urls, globals())
    if run_server:
        app.run()
    else:
        return app.wsgifunc()

if __name__ == '__main__':
    app.run()
<commit_msg>Fix multisequence url issue<commit_after>import os
import uuid
import json
import random

import web
from jinja2 import Environment, Template, FileSystemLoader

from config import settings
from blast import get_blast_databases, run_blast, get_sequence_from_database, process_blast_result, get_blast_database_from_title

NBLAST = ('blastn', 'tblastn', 'tblastx',)
PBLAST = ('blastp', 'blastx',)

settings = settings()

'''
urls = (
    '{}'.format(settings.get('BASE_URL')), 'index',
    '{}results/(.*)/'.format(settings.get('BASE_URL')), 'results',
    '{}run/(.*)/'.format(settings.get('BASE_URL')), 'start_blast',
    '{}status/(.*)/'.format(settings.get('BASE_URL')), 'status',
    '{}sequence/(.*)/(.*)/'.format(settings.get('BASE_URL')), 'sequence',
    '{}multisequence'.format(settings.get('BASE_URL')), 'multisequence',
    )
'''

#app = web.application(urls, globals())
#application = app.wsgifunc()

def render_template(template_name, base_template=None, **context):
    extensions = context.pop('extensions', [])
    globals = context.pop('globals', {})

    if base_template is not None and base_template != 'base.html':
        context['layout_template'] = Template(open(base_template).read())
    else:
        context['layout_template'] = 'base.html'

    location = os.path.join(os.path.dirname(__file__), 'templates')

    jinja_env = Environment(
        loader=FileSystemLoader(location),
        extensions=extensions,
        )

    jinja_env.globals.update(globals)

    return jinja_env.get_template(template_name).render(context)

class index:
    def GET(self):
        databases = get_blast_databases(settings.get('BLAST_EXECUTABLE_PATH'), settings.get('BLAST_DATABASE_PATH'))
        programs = {'nucleotide': NBLAST, 'protein': PBLAST}
        qs = web.input()
        identifier = qs.get('settings')
        db = ''
        pgrm = ''
        seq = ''
        adv = ''
        if identifier is not None:
            with open('{0}{1}.json'.format(settings.get('BLAST_RESULTS_PATH'), identifier)) as sp:
                opt = json.loads(sp.read())
                db = opt['database']
                pgrm = opt['program']
                seq = opt['sequence']
                adv = opt['advanced']
        return render_template('index.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title="pyBlast | Enter sequence to BLAST",
            databases = databases,
            programs = programs,
            uuid = uuid.uuid4(),
            db = db,
            pgrm = pgrm,
            sequence = seq,
            advanced = adv,
            rndm = random.random(),
        )

class results:
    def POST(self, identifier):
        options = web.input()
        return render_template('results_container.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title="pyBlast | BLAST results",
            identifier = identifier,
            options = json.dumps(options),
            execute = True,
            rndm = random.random(),
        )

    def GET(self, identifier):
        return render_template('results_container.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title="pyBlast | BLAST results",
            identifier = identifier,
            rndm = random.random(),
        )

class start_blast:
    def POST(self, identifier):
        options = web.input()
        web.header('Content-type', 'application/json')
        try:
            database = get_blast_database_from_title(settings.get('BLAST_EXECUTABLE_PATH'),settings.get('BLAST_DATABASE_PATH'),options.database)
            #database = settings.get('BLAST_DATABASE_PATH')+options.database
            program = settings.get('BLAST_EXECUTABLE_PATH')+options.program
            sequence = options.sequence
            advanced = options.advanced
            out, err = run_blast(database, program, settings.get('BLAST_RESULTS_PATH'), identifier, sequence, advanced)
            with open('{0}/{1}.json'.format(settings.get('BLAST_RESULTS_PATH'), identifier), 'w+') as sp:
                sp.write(json.dumps({
                    'database': options.database,
                    'program': options.program,
                    'sequence': sequence,
                    'advanced': advanced,
                }))
            return json.dumps({'running': True, 'out': out, 'err': err})
        except KeyError:
            web.ctx.status = 400
            return json.dumps({'running': False})

class status:
    def GET(self, identifier):
        web.header('Content-type', 'application/json')
        file_loc = "{0}{1}.xml".format(settings.get('BLAST_RESULTS_PATH'), identifier)
        qs = web.input()
        cutoff = qs.get('cutoff')
        with open(file_loc) as results:
            if cutoff is not None:
                processed = process_blast_result(results.read(), cutoff=float(cutoff))
            else:
                processed = process_blast_result(results.read())

            results_page = render_template('results.html',
                settings = settings,
                results = processed,
                uuid = uuid.uuid4(),
                identifier = identifier,
            )
            return json.dumps({'active': False, 'results': processed, 'results_page': results_page})
        return json.dumps({'active': False, 'err': 'There has been an error'})

class sequence:
    def GET(self, db, sequence):
        seqs = sequence.split(',')
        results = []
        for s in seqs:
            results.append([s, get_sequence_from_database(settings.get('BLAST_EXECUTABLE_PATH'), settings.get('BLAST_DATABASE_PATH')+db, s)])
        return render_template('sequences.html',
            base_template = settings.get('BASE_TEMPLATE'),
            settings = settings,
            title = 'pyBlast | Retrieved sequences',
            results = results,
        )

class multisequence:
    def GET(self):
        qs = web.input(sequences=[])
        seqs = u','.join(qs.sequences)
        s = sequence()
        return s.GET(qs.db, seqs)

app = None

def run(overidden_settings={}, run_server=False):
    settings.merge(overidden_settings)
    urls = (
        '{}'.format(settings.get('BASE_URL')), 'index',
        '{}results/(.*)/'.format(settings.get('BASE_URL')), 'results',
        '{}run/(.*)/'.format(settings.get('BASE_URL')), 'start_blast',
        '{}status/(.*)/'.format(settings.get('BASE_URL')), 'status',
        '{}sequence/(.*)/(.*)/'.format(settings.get('BASE_URL')), 'sequence',
        '{}multisequence/'.format(settings.get('BASE_URL')), 'multisequence',
        )
    app = web.application(urls, globals())
    if run_server:
        app.run()
    else:
        return app.wsgifunc()

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 12-09-2017 16:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This file contains different models for the execution of subprocesses / external processes, e.g. via the command line
"""

import abc

# Execution of commands
class CommandLineRunnerFactory:
    pass

class CommandLineRunner(metaclass=abc.ABCMeta):
    def __init__(self):
        self.command = None
        self.timeout = None
        self._stdout = b' '
        self._stderr = b' '

class CommandLineRunnerAsThread(CommandLineRunner):
    pass

class CommandLineRunnerOnHpc(CommandLineRunner):
    pass
<commit_msg>getter for stdout<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 12-09-2017 16:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This file contains different models for the execution of subprocesses / external processes, e.g. via the command line
"""

import abc

# Execution of commands
class CommandLineRunnerFactory:
    pass

class CommandLineRunner(metaclass=abc.ABCMeta):
    def __init__(self):
        self.command = None
        self.timeout = None
        self._stdout = b' '
        self._stderr = b' '

    def get_stdout(self):
        pass

class CommandLineRunnerAsThread(CommandLineRunner):
    pass

class CommandLineRunnerOnHpc(CommandLineRunner):
    pass
<|endoftext|>


<commit_before>import os

target_arch = 'arm'
android_api_level = os.getenv('ANDROID_VER')

# Python optional modules.
# Available:
#  tinycc - Tiny cc compiler
#  bzip2 - enable the bz2 module and the bzip2 codec
#  xz - enable the lzma module and the lzma codec
#  openssl - enable the ssl module and SSL/TLS support for sockets
#  readline - enable the readline module and command history/the like in the REPL
#  ncurses - enable the curses module
#  sqlite - enable the sqlite3 module
#  gdbm - enable the dbm/gdbm modules
#  libffi - enable the ctypes module
#  zlib - enable the zlib module
#  expat - enable the pyexpat module
#  tools - some handy utility scripts from ./devscripts
packages = ('openssl', 'ncurses', 'readline', 'bzip2', 'xz', 'zlib', 'sqlite', 'gdbm', 'libffi', 'expat', 'tools')
# 3rd Python modules.
#py_packages = ('openblas','numpy','scipy')
py_packages = ('scipy',)

skip_build_py  = os.path.exists(".skip_build_py")
skip_build_py_module  = os.path.exists(".skip_build_py_module")

use_bintray = False
bintray_username = 'qpython-android'
bintray_repo = 'qpython3-core'
<commit_msg>Bump v1.1.0 Support numpy/scipy for Python 3.6.6<commit_after>import os

target_arch = 'arm'
android_api_level = os.getenv('ANDROID_VER')

# Python optional modules.
# Available:
#  tinycc - Tiny cc compiler
#  bzip2 - enable the bz2 module and the bzip2 codec
#  xz - enable the lzma module and the lzma codec
#  openssl - enable the ssl module and SSL/TLS support for sockets
#  readline - enable the readline module and command history/the like in the REPL
#  ncurses - enable the curses module
#  sqlite - enable the sqlite3 module
#  gdbm - enable the dbm/gdbm modules
#  libffi - enable the ctypes module
#  zlib - enable the zlib module
#  expat - enable the pyexpat module
#  tools - some handy utility scripts from ./devscripts
packages = ('openssl', 'ncurses', 'readline', 'bzip2', 'xz', 'zlib', 'sqlite', 'gdbm', 'libffi', 'expat', 'tools')
# 3rd Python modules.
py_packages = ('openblas','numpy','scipy')
#py_packages = ('scipy',)

skip_build_py  = os.path.exists(".skip_build_py")
skip_build_py_module  = os.path.exists(".skip_build_py_module")

use_bintray = False
bintray_username = 'qpython-android'
bintray_repo = 'qpython3-core'
<|endoftext|>


<commit_before>"""
Class to read DBF files.
"""

import os
import datetime
import collections

from .struct_parser import StructParser
from .field_parser import FieldParser
from .common import parse_string
from .ifiles import ifind
from .fpt import FPT
from .codepages import guess_encoding
from .dbversions import get_dbversion_string

DBFHeader = StructParser(
    'DBFHeader',
    '<BBBBLHHHBBLLLBBH',
    ['dbversion',
     'year',
     'month',
     'day',
     'numrecords',
     'headerlen',
     'recordlen',
     'reserved1',
     'incomplete_transaction',
     'encryption_flag',
     'free_record_thread',
     'reserved2',
     'reserved3',
     'mdx_flag',
     'language_driver',
     'reserved4',
     ])

DBFField = StructParser(
    'DBFField',
    '<11scLBBHBBBB7sB',
    ['name',
     'type',
     'address',
     'length',
     'decimal_count',
     'reserved1',
     'workarea_id',
     'reserved2',
     'reserved3',
     'set_fields_flag',
     'reserved4',
     'index_field_flag',
     ])

FieldValue = collections.namedtuple('Field', 'name value')

def expand_year(year):
    """Convert 2-digit year to 4-digit year."""

    if year < 80:
        return 2000 + year
    else:
        return 1900 + year

class RecordIterator(object):
    def __init__(self, table, record_type):
        self._record_type = record_type
        self._table = table

    def __iter__(self):
        for record in self._table._iter_records(self._record_type):
            yield record

    def __len__(self):
        return self._table._count_records(self._record_type)

class FakeMemoFile(object):
    def __getitem__(self, i):
        return ''

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False

_FAKE_MEMOFILE = FakeMemoFile()

class Table(list):
    """
    Class to read DBF files.
    """

    def __init__(self, filename,
                 encoding=None,
                 ignorecase=True,
                 lowernames=False,
                 parserclass=FieldParser,
                 recfactory=dict,
                 load=False,
                 raw=False):

        self.encoding = encoding
        self.ignorecase = ignorecase
        self.lowernames = lowernames
        self.parserclass = parserclass
        self.recfactory = recfactory
        self.raw = raw

        # Name part before .dbf is the table name
        self.name = os.path.basename(filename)
        self.name = os.path.splitext(self.name)[0]
        self.name = self.name.lower()

        self._deleted = None
        self.loaded = False

        if ignorecase:
            self.filename = ifind(filename)
            if not self.filename:
                raise IOError('No such file: {!r}'.format(filename))
        else:
            self.filename = filename
        # Filled in by self._read_headers()
        self.memofilename = None
        self.header = None
        self.fields = []       # namedtuples
        self.field_names = []  # strings

        with open(self.filename, mode='rb') as infile:
            self._read_headers(infile)
            self._field_parser = self.parserclass(self.encoding)

            self._check_headers()

            self.date = datetime.date(expand_year(self.header.year),
                                      self.header.month,
                                      self.header.day)

        self.dbversion = get_dbversion_string(self.header.dbversion)

        if load:
            self.load()
        else:
            self.unload()

    def _get_memofile(self):
        if self.memofilename and not self.raw:
            return FPT(self.memofilename)
        else:
            return _FAKE_MEMOFILE

    def load(self):
        if not self.loaded:
            self[:] = self._iter_records(b' ')
            self._deleted = list(self._iter_records(b'*'))
            self.loaded = True

    def unload(self):
        # self.loaded is not checked here because this
        # is called by __init__() where self.loaded=False.
        # Also, unloading twice has no consequences.
        del self[:]
        self._deleted = None
        self.loaded = False

    @property
    def deleted(self):
        if self.loaded:
            return self._deleted
        else:
            return RecordIterator(self, b'*')

    def _read_headers(self, infile):
        #
        # Todo: more checks
        # http://www.clicketyclick.dk/databases/xbase/format/dbf_check.html#CHECK_DBF
        #
        self.header = DBFHeader.read(infile)

        if self.encoding is None:
            try:
                self.encoding = guess_encoding(self.header.language_driver)
            except LookupError as err:
                self.encoding = 'latin1'

        #
        # Read field headers
        #
        while 1:
            sep = infile.read(1)
            if sep in (b'\r', b'\n', b''):
                # End of field headers
                break

            fh = DBFField.read(infile, prepend=sep)
            # We need to fix the name and type

            fieldname = parse_string(fh.name, self.encoding)
            if self.lowernames:
                fieldname = fieldname.lower()
            fieldtype = parse_string(fh.type, self.encoding)

            fh = fh._replace(name=fieldname,
                             type=fieldtype)

            self.field_names.append(fh.name)

            self.fields.append(fh)

        if len(self.fields) < 1:
            message = 'dbf file must have at least one field: {!r}'
            raise ValueError(message.format(self.filename))

        # Check for memo file
        field_types = set([field.type for field in self.fields])
        if 'M' in field_types:
            fn = os.path.splitext(self.filename)[0] + '.fpt'
            match = ifind(self.filename, ext='.fpt')
            if match:
                self.memofilename = match
            else:
                # Todo: warn and return field as byte string?
                raise IOError('Missing memo file: {!r}'.format(fn))

    def _check_headers(self):
        """Check headers for possible format errors."""
        for field in self.fields:

            if field.type == '0' and field.length != 1:
                message = 'Field of type 0 must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'I' and field.length != 4:
                message = 'Field type I must have length 4 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'L' and field.length != 1:
                message = 'Field type L must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif not self._field_parser.field_type_supported(field.type):
                # Todo: return as byte string?
                raise ValueError('Unknown field type: {!r}'.format(field.type))

    def _read_record(self, infile, memofile):
        items = []  # List of Field
        for field in self.fields:
            value = infile.read(field.length)
            if not self.raw:
                value = self._field_parser.parse(field, value)

                #
                # Decoding memo fields requires a little more
                # trickery.
                #
                if field.type == 'M' and value is not None:
                    memo = memofile[value]
                    if memo.type == 'memo':
                        # Decode to unicode
                        value = parse_string(memo.data, self.encoding)
                    else:
                        # Byte string
                        value = memo.data

            items.append(FieldValue(name=field.name, value=value))

        return self.recfactory(items)

    def _skip_record(self, infile):
        infile.seek(self.header.recordlen - 1, 1)

    def _count_records(self, record_type=b' '):
        count = 0

        with open(self.filename, 'rb') as infile:
            infile.seek(self.header.headerlen, 0)
            while True:
                sep = infile.read(1)
                if sep == record_type:
                    count += 1
                    self._skip_record(infile)
                elif sep in (b'\x1a', b''):
                    # End of records.
                    break
                else:
                    self._skip_record(infile)

        return count

    def _iter_records(self, record_type=b' '):
        with open(self.filename, 'rb') as infile, \
              self._get_memofile() as memofile:
            # Skip to first record.
            infile.seek(self.header.headerlen, 0)

            while True:
                sep = infile.read(1)

                if sep == record_type:
                    yield self._read_record(infile, memofile)
                elif sep in (b'\x1a', b''):
                    # End of records.
                    break
                else:
                    self._skip_record(infile)

    def __iter__(self):
        if self.loaded:
            for record in list.__iter__(self):
                yield record
        else:
            for record in self._iter_records():
                yield record

    def __len__(self):
        if self.loaded:
            return list.__len__(self)
        else:
            return self._count_records()

    def __repr__(self):
        if self.loaded:
            return list.__repr__(self)
        else:
            return '<Unloaded DBF table {!r}>'.format(self.filename)
<commit_msg>Added some comments.<commit_after>"""
Class to read DBF files.
"""

import os
import datetime
import collections

from .struct_parser import StructParser
from .field_parser import FieldParser
from .common import parse_string
from .ifiles import ifind
from .fpt import FPT
from .codepages import guess_encoding
from .dbversions import get_dbversion_string

DBFHeader = StructParser(
    'DBFHeader',
    '<BBBBLHHHBBLLLBBH',
    ['dbversion',
     'year',
     'month',
     'day',
     'numrecords',
     'headerlen',
     'recordlen',
     'reserved1',
     'incomplete_transaction',
     'encryption_flag',
     'free_record_thread',
     'reserved2',
     'reserved3',
     'mdx_flag',
     'language_driver',
     'reserved4',
     ])

DBFField = StructParser(
    'DBFField',
    '<11scLBBHBBBB7sB',
    ['name',
     'type',
     'address',
     'length',
     'decimal_count',
     'reserved1',
     'workarea_id',
     'reserved2',
     'reserved3',
     'set_fields_flag',
     'reserved4',
     'index_field_flag',
     ])

FieldValue = collections.namedtuple('Field', 'name value')

def expand_year(year):
    """Convert 2-digit year to 4-digit year."""

    if year < 80:
        return 2000 + year
    else:
        return 1900 + year

class RecordIterator(object):
    def __init__(self, table, record_type):
        self._record_type = record_type
        self._table = table

    def __iter__(self):
        for record in self._table._iter_records(self._record_type):
            yield record

    def __len__(self):
        return self._table._count_records(self._record_type)

class FakeMemoFile(object):
    def __getitem__(self, i):
        return ''

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False

_FAKE_MEMOFILE = FakeMemoFile()

class Table(list):
    """
    Class to read DBF files.
    """

    def __init__(self, filename,
                 encoding=None,
                 ignorecase=True,
                 lowernames=False,
                 parserclass=FieldParser,
                 recfactory=dict,
                 load=False,
                 raw=False):

        self.encoding = encoding
        self.ignorecase = ignorecase
        self.lowernames = lowernames
        self.parserclass = parserclass
        self.recfactory = recfactory
        self.raw = raw

        # Name part before .dbf is the table name
        self.name = os.path.basename(filename)
        self.name = os.path.splitext(self.name)[0]
        self.name = self.name.lower()

        self._deleted = None
        self.loaded = False

        if ignorecase:
            self.filename = ifind(filename)
            if not self.filename:
                raise IOError('No such file: {!r}'.format(filename))
        else:
            self.filename = filename
        # Filled in by self._read_headers()
        self.memofilename = None
        self.header = None
        self.fields = []       # namedtuples
        self.field_names = []  # strings

        with open(self.filename, mode='rb') as infile:
            self._read_headers(infile)
            self._field_parser = self.parserclass(self.encoding)

            self._check_headers()

            self.date = datetime.date(expand_year(self.header.year),
                                      self.header.month,
                                      self.header.day)

        self.dbversion = get_dbversion_string(self.header.dbversion)

        if load:
            self.load()
        else:
            self.unload()

    def _get_memofile(self):
        if self.memofilename and not self.raw:
            return FPT(self.memofilename)
        else:
            return _FAKE_MEMOFILE

    def load(self):
        if not self.loaded:
            self[:] = self._iter_records(b' ')
            self._deleted = list(self._iter_records(b'*'))
            self.loaded = True

    def unload(self):
        # self.loaded is not checked here because this
        # is called by __init__() where self.loaded=False.
        # Also, unloading twice has no consequences.
        del self[:]
        self._deleted = None
        self.loaded = False

    @property
    def deleted(self):
        if self.loaded:
            return self._deleted
        else:
            return RecordIterator(self, b'*')

    def _read_headers(self, infile):
        #
        # Todo: more checks
        # http://www.clicketyclick.dk/databases/xbase/format/dbf_check.html#CHECK_DBF
        #
        self.header = DBFHeader.read(infile)

        if self.encoding is None:
            try:
                self.encoding = guess_encoding(self.header.language_driver)
            except LookupError as err:
                self.encoding = 'latin1'

        #
        # Read field headers
        #
        while 1:
            sep = infile.read(1)
            if sep in (b'\r', b'\n', b''):
                # End of field headers
                break

            fh = DBFField.read(infile, prepend=sep)
            # We need to fix the name and type

            fieldname = parse_string(fh.name, self.encoding)
            if self.lowernames:
                fieldname = fieldname.lower()
            fieldtype = parse_string(fh.type, self.encoding)

            fh = fh._replace(name=fieldname,
                             type=fieldtype)

            self.field_names.append(fh.name)

            self.fields.append(fh)

        if len(self.fields) < 1:
            message = 'dbf file must have at least one field: {!r}'
            raise ValueError(message.format(self.filename))

        # Check for memo file
        field_types = set([field.type for field in self.fields])
        if 'M' in field_types:
            fn = os.path.splitext(self.filename)[0] + '.fpt'
            match = ifind(self.filename, ext='.fpt')
            if match:
                self.memofilename = match
            else:
                # Todo: warn and return field as byte string?
                raise IOError('Missing memo file: {!r}'.format(fn))

    def _check_headers(self):
        """Check headers for possible format errors."""
        for field in self.fields:

            if field.type == '0' and field.length != 1:
                message = 'Field of type 0 must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'I' and field.length != 4:
                message = 'Field type I must have length 4 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'L' and field.length != 1:
                message = 'Field type L must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif not self._field_parser.field_type_supported(field.type):
                # Todo: return as byte string?
                raise ValueError('Unknown field type: {!r}'.format(field.type))

    def _read_record(self, infile, memofile):
        items = []  # List of Field
        for field in self.fields:
            value = infile.read(field.length)
            if not self.raw:
                value = self._field_parser.parse(field, value)

                #
                # Decoding memo fields requires a little more
                # trickery.
                #
                if field.type == 'M' and value is not None:
                    memo = memofile[value]
                    if memo.type == 'memo':
                        # Decode to unicode
                        value = parse_string(memo.data, self.encoding)
                    else:
                        # Byte string
                        value = memo.data

            items.append(FieldValue(name=field.name, value=value))

        return self.recfactory(items)

    def _skip_record(self, infile):
        # -1 for the record separator which was already read.
        infile.seek(self.header.recordlen - 1, 1)

    def _count_records(self, record_type=b' '):
        count = 0

        with open(self.filename, 'rb') as infile:
            # Skip to first record.
            infile.seek(self.header.headerlen, 0)

            while True:
                sep = infile.read(1)
                if sep == record_type:
                    count += 1
                    self._skip_record(infile)
                elif sep in (b'\x1a', b''):
                    # End of records.
                    break
                else:
                    self._skip_record(infile)

        return count

    def _iter_records(self, record_type=b' '):
        with open(self.filename, 'rb') as infile, \
             self._get_memofile() as memofile:
            # Skip to first record.
            infile.seek(self.header.headerlen, 0)

            while True:
                sep = infile.read(1)

                if sep == record_type:
                    yield self._read_record(infile, memofile)
                elif sep in (b'\x1a', b''):
                    # End of records.
                    break
                else:
                    self._skip_record(infile)

    def __iter__(self):
        if self.loaded:
            for record in list.__iter__(self):
                yield record
        else:
            for record in self._iter_records():
                yield record

    def __len__(self):
        if self.loaded:
            return list.__len__(self)
        else:
            return self._count_records()

    def __repr__(self):
        if self.loaded:
            return list.__repr__(self)
        else:
            return '<Unloaded DBF table {!r}>'.format(self.filename)
<|endoftext|>


<commit_before>import numpy as np
import multiprocessing
from .. import train

class Task(object):
    def __init__(self, column_indices):
        self.column_indices = column_indices

class Result(object):
    def __init__(self, column_indices, score):
        self.column_indices = column_indices
        self.score = score

class Worker(multiprocessing.Process, train.Train):
    def __init__(self, task_queue, result_queue, x, y, folds, solver, scorer):
        multiprocessing.Process.__init__(self)
        train.Train.__init__(self)

        # queues for inter-process communication
        self.task_queue = task_queue
        self.result_queue = result_queue

        # data
        self.x = x
        self.y = y
        self.folds = folds
        self.number_of_folds = len(folds)
        self.solver = solver
        self.scorer = scorer

    def _get_fold(self, fold):
        # get number rows
        l_indices = np.shape(self.x)[0]

        # get indices
        row_indices_for_validation = self.folds[fold]
        row_indices_for_training = [s for s in xrange(0, l_indices) if s not in row_indices_for_validation]

        return row_indices_for_training, row_indices_for_validation

    def _train(self, col_indices_for_inputs, row_indices):
        # get x and y
        x = self.x[np.ix_(row_indices, col_indices_for_inputs)]
        y = self.y[row_indices]

        # run linear regression
        fit = self.solver.calculate_parameters(x, y)

        return fit

    def _validate(self, col_indices_for_inputs, row_indices, fit):
        # get x and y
        x = self.x[np.ix_(row_indices, col_indices_for_inputs)]
        y = self.y[row_indices]

        # generate predictions
        predicted_y = self.solver.apply_parameters(x, fit)

        # create tuples
        validation = np.concatenate((y, predicted_y), 1)

        return self.scorer.score(validation)

    def _score(self, col_indices_for_inputs):
        score = 0.

        # for each fold in k-fold-cross-validation
        for fold in xrange(self.number_of_folds):
            # get indices
            (row_indices_for_training, row_indices_for_validation) = self._get_fold(fold)

            # train and get fit
            fit = self._train(col_indices_for_inputs, row_indices_for_training)

            # validation score
            score += self._validate(col_indices_for_inputs, row_indices_for_validation, fit)

        # average MSE
        return score / float(self.number_of_folds)

    def _run_task(self, task):
        # score potential columns
        score = self._score( task.column_indices )

        # make result
        return Result(task.column_indices, score)

    def run(self):
        while True:
            next_task = self.task_queue.get()

            # no next task, break
            if next_task is None:
                # remove empty task from task queue
                self.task_queue.task_done()

                break

            # run task
            self.result_queue.put(self._run_task(next_task))

            # remove from task queue, and add result
            self.task_queue.task_done()
<commit_msg>Necessary changes in parallel trainer to work with last commit.<commit_after>import numpy as np
import multiprocessing
from .. import train

class Task(object):
    def __init__(self, column_indices):
        self.column_indices = column_indices

class Result(object):
    def __init__(self, column_indices, score):
        self.column_indices = column_indices
        self.score = score

class Worker(multiprocessing.Process, train.Train):
    def __init__(self, task_queue, result_queue, x, y, folds, solver, scorer):
        multiprocessing.Process.__init__(self)
        train.Train.__init__(self, x=x, y=y, solver=solver, scorer=scorer, number_of_folds=len(folds))

        # queues for inter-process communication
        self.task_queue = task_queue
        self.result_queue = result_queue

        # store folds
        self.folds = folds

    def _get_fold(self, fold):
        # get number rows
        l_indices = np.shape(self.x)[0]

        # get indices
        row_indices_for_validation = self.folds[fold]
        row_indices_for_training = [s for s in xrange(0, l_indices) if s not in row_indices_for_validation]

        return row_indices_for_training, row_indices_for_validation

    def _train(self, col_indices_for_inputs, row_indices):
        # get x and y
        x = self.x[np.ix_(row_indices, col_indices_for_inputs)]
        y = self.y[row_indices]

        # run linear regression
        fit = self.solver.calculate_parameters(x, y)

        return fit

    def _validate(self, col_indices_for_inputs, row_indices, fit):
        # get x and y
        x = self.x[np.ix_(row_indices, col_indices_for_inputs)]
        y = self.y[row_indices]

        # generate predictions
        predicted_y = self.solver.apply_parameters(x, fit)

        # create tuples
        validation = np.concatenate((y, predicted_y), 1)

        return self.scorer.score(validation)

    def _score(self, col_indices_for_inputs):
        score = 0.

        # for each fold in k-fold-cross-validation
        for fold in xrange(self.number_of_folds):
            # get indices
            (row_indices_for_training, row_indices_for_validation) = self._get_fold(fold)

            # train and get fit
            fit = self._train(col_indices_for_inputs, row_indices_for_training)

            # validation score
            score += self._validate(col_indices_for_inputs, row_indices_for_validation, fit)

        # average MSE
        return score / float(self.number_of_folds)

    def _run_task(self, task):
        # score potential columns
        score = self._score( task.column_indices )

        # make result
        return Result(task.column_indices, score)

    def run(self):
        while True:
            next_task = self.task_queue.get()

            # no next task, break
            if next_task is None:
                # remove empty task from task queue
                self.task_queue.task_done()

                break

            # run task
            self.result_queue.put(self._run_task(next_task))

            # remove from task queue, and add result
            self.task_queue.task_done()
<|endoftext|>


<commit_before># bot.py
#
# Pubsub Client Bot - Listens to pubsub events for new blog entries.
#
import time
import datetime
from twisted.python import log
from twisted.internet import defer, task, reactor
from twisted.words.xish import domish, xpath
from wokkel.pubsub import PubSubClient, Item
from wokkel import disco
from twisted.words.protocols.jabber.jid import internJID

from wokkel.xmppim import AvailablePresence
# use cheetah for templating
from Cheetah.Template import Template

class PubSub2Blog(object):
    """ Handles PubSub event items and turns them to formated static files.

    The templates are extremly simple for now and can be changed later.
    """
    queue_interval = 2.0

    def __init__(self, path):
        self.path = path
        self.www_path = path+'/www'
        self.template_path = path+'/templates'

        self.atom_queue  = []
        self.index_queue = []

        self.processing_atom  = False
        self.processing_index = False

        ## tasks to process queues
        self.task = task.LoopingCall(self._processQueues)
        self.task.start(self.queue_interval) # run every N seconds

    def _processQueue(self, queue_name, ext='.html'):
        """ Process a queue with the given name.
        """
        queue = getattr(self, queue_name+'_queue')
        queue_status = getattr(self, 'processing_'+queue_name, False)

        if not queue_status and len(queue)>0:
            setattr(self, 'processing_'+queue_name, True)
            id, entry_file_name, entries = queue.pop()
            args = {
                'id': id,
                'entries': entries,
                'updated':str(datetime.datetime.utcnow()),
                    }

            html = self.template(queue_name+ext, args)
            file_name = self.www_path+'/' + queue_name + ext
            self._writeFile(file_name, html)

            reactor.callLater(1.0, setattr, self, 'processing_'+queue_name, True)

    def _processQueues(self):
        self._processQueue('atom', '.xml')
        self._processQueue('index')

    def template(self, file, vars):
        """ Process a template
        """
        template = open(self.template_path+'/'+file, 'r').read()
        return str(Template(template, searchList=[vars]))

    def atom2hash(self, elem):
        """ Convert an atom domish element to a dictionary for template processing.
        """
        args = {}
        args['id'] = str(elem.id)
        id, args['idname'] = str(elem.id).split(":",1)
        args['categories'] = []
        cats = domish.generateElementsNamed(elem.children, 'category')
        for cat in cats:
            args['categories'].append(cat['term'])

        if elem.content:
            args['content'] = str(elem.content)
        elif elem.title:
            args['content'] = str(elem.title)

        if elem.title:
            args['title'] = str(elem.title)
        elif elem.id:
            args['title'] = str(elem.id)
        else:
            args['title'] = 'No Title'

        return args

    def _writeFile(self, file_name, html):
        # open up file
        ef = open(file_name, 'w')
        ef.write(html)
        ef.close()

    def updateEntry(self, blog_id, args):
        """update the entry on disk.

        """
        html = self.template('entry.html', args)
        id, file_name = blog_id.split(":", 1)
        file_name = self.www_path+'/archive/' + file_name + ".html"
        self._writeFile(file_name, html)

    def updateAtom(self, blog_id, entries):
        """
        """
        id, file_name = blog_id.split(":", 1)
        # push on queue
        self.atom_queue.append((id, file_name, entries))

    def updateIndex(self, blog_id, entries):
        """
        """
        id, file_name = blog_id.split(":", 1)
        # push on queue
        self.index_queue.append((id, file_name, entries))

class Bot(PubSubClient):
    """The listener bot to manage the static html representing thetofu.com

    In addition to the standard XMPPHandler behavior, it also provides
    getJid() and publish().
    """
    admins = []

    def __init__(self, blog, service=None, node=None):
	PubSubClient.__init__(self)
        self.blog = blog
	self.service = service
	self.nodeIdentifier = node

    def connectionInitialized(self):
        """When connection is establised send presence and subscribe to the node
        that the bot is listening to.
        """
        self.send(AvailablePresence())
        self.subscribe(self.service, self.nodeIdentifier, self.getJid().userhostJID())

        PubSubClient.connectionInitialized(self)

    @defer.inlineCallbacks
    def itemsReceived(self, item_event):
        """Gather items, convert to html and send the files to there proper location.
        """

        for item in item_event.items:
            if item.name != 'item': # TODO - handle retract and other events
                continue
            item_id = item.getAttribute('id', str(time.time()))

            date_pub = xpath.queryForNodes("/item/entry/published", item)
            published = datetime.datetime.now()
            if date_pub:
                published = str(date_pub[0])

            blog_id = None
            blog_ids = xpath.queryForNodes("/item/entry/id", item)
            if blog_ids:
                blog_id = str(blog_ids[0])

            # create entry
            args = self.blog.atom2hash(item.entry)
            self.blog.updateEntry(blog_id, args)
            # grab last 10 items
            ret_items = yield self.items(self.service, self.nodeIdentifier, 10)
            last_items = []
            last_ids   = []
            for ri in ret_items:
                rargs = self.blog.atom2hash(ri.entry)
                if rargs['id'] not in last_ids:
                    last_items.append(rargs)
                    last_ids.append(args['id'])
            # update index
            self.blog.updateIndex(blog_id, last_items)
            # update atom
            self.blog.updateAtom(blog_id, last_items)

    def getJid(self):
	"""Return the JID the connection is authenticed as."""

	return self.xmlstream.authenticator.jid

<commit_msg>make sure file permissions are correct to read<commit_after># bot.py
#
# Pubsub Client Bot - Listens to pubsub events for new blog entries.
#
import time
import datetime
import os
from twisted.python import log
from twisted.internet import defer, task, reactor
from twisted.words.xish import domish, xpath
from wokkel.pubsub import PubSubClient, Item
from wokkel import disco
from twisted.words.protocols.jabber.jid import internJID

from wokkel.xmppim import AvailablePresence
# use cheetah for templating
from Cheetah.Template import Template

class PubSub2Blog(object):
    """ Handles PubSub event items and turns them to formated static files.

    The templates are extremly simple for now and can be changed later.
    """
    queue_interval = 2.0

    def __init__(self, path):
        self.path = path
        self.www_path = path+'/www'
        self.template_path = path+'/templates'

        self.atom_queue  = []
        self.index_queue = []

        self.processing_atom  = False
        self.processing_index = False

        ## tasks to process queues
        self.task = task.LoopingCall(self._processQueues)
        self.task.start(self.queue_interval) # run every N seconds

    def _processQueue(self, queue_name, ext='.html'):
        """ Process a queue with the given name.
        """
        queue = getattr(self, queue_name+'_queue')
        queue_status = getattr(self, 'processing_'+queue_name, False)

        if not queue_status and len(queue)>0:
            setattr(self, 'processing_'+queue_name, True)
            id, entry_file_name, entries = queue.pop()
            args = {
                'id': id,
                'entries': entries,
                'updated':str(datetime.datetime.utcnow()),
                    }

            html = self.template(queue_name+ext, args)
            file_name = self.www_path+'/' + queue_name + ext
            self._writeFile(file_name, html)

            reactor.callLater(1.0, setattr, self, 'processing_'+queue_name, True)

    def _processQueues(self):
        self._processQueue('atom', '.xml')
        self._processQueue('index')

    def template(self, file, vars):
        """ Process a template
        """
        template = open(self.template_path+'/'+file, 'r').read()
        return str(Template(template, searchList=[vars]))

    def atom2hash(self, elem):
        """ Convert an atom domish element to a dictionary for template processing.
        """
        args = {}
        args['id'] = str(elem.id)
        id, args['idname'] = str(elem.id).split(":",1)
        args['categories'] = []
        cats = domish.generateElementsNamed(elem.children, 'category')
        for cat in cats:
            args['categories'].append(cat['term'])

        if elem.content:
            args['content'] = str(elem.content)
        elif elem.title:
            args['content'] = str(elem.title)

        if elem.title:
            args['title'] = str(elem.title)
        elif elem.id:
            args['title'] = str(elem.id)
        else:
            args['title'] = 'No Title'

        return args

    def _writeFile(self, file_name, html):
        # open up file
        ef = open(file_name, 'w')
        os.fchmod(ef, 644)
        ef.write(html)
        ef.close()

    def updateEntry(self, blog_id, args):
        """update the entry on disk.

        """
        html = self.template('entry.html', args)
        id, file_name = blog_id.split(":", 1)
        file_name = self.www_path+'/archive/' + file_name + ".html"
        self._writeFile(file_name, html)

    def updateAtom(self, blog_id, entries):
        """
        """
        id, file_name = blog_id.split(":", 1)
        # push on queue
        self.atom_queue.append((id, file_name, entries))

    def updateIndex(self, blog_id, entries):
        """
        """
        id, file_name = blog_id.split(":", 1)
        # push on queue
        self.index_queue.append((id, file_name, entries))

class Bot(PubSubClient):
    """The listener bot to manage the static html representing thetofu.com

    In addition to the standard XMPPHandler behavior, it also provides
    getJid() and publish().
    """
    admins = []

    def __init__(self, blog, service=None, node=None):
	PubSubClient.__init__(self)
        self.blog = blog
	self.service = service
	self.nodeIdentifier = node

    def connectionInitialized(self):
        """When connection is establised send presence and subscribe to the node
        that the bot is listening to.
        """
        self.send(AvailablePresence())
        self.subscribe(self.service, self.nodeIdentifier, self.getJid().userhostJID())

        PubSubClient.connectionInitialized(self)

    @defer.inlineCallbacks
    def itemsReceived(self, item_event):
        """Gather items, convert to html and send the files to there proper location.
        """

        for item in item_event.items:
            if item.name != 'item': # TODO - handle retract and other events
                continue
            item_id = item.getAttribute('id', str(time.time()))

            date_pub = xpath.queryForNodes("/item/entry/published", item)
            published = datetime.datetime.now()
            if date_pub:
                published = str(date_pub[0])

            blog_id = None
            blog_ids = xpath.queryForNodes("/item/entry/id", item)
            if blog_ids:
                blog_id = str(blog_ids[0])

            # create entry
            args = self.blog.atom2hash(item.entry)
            self.blog.updateEntry(blog_id, args)
            # grab last 10 items
            ret_items = yield self.items(self.service, self.nodeIdentifier, 10)
            last_items = []
            last_ids   = []
            for ri in ret_items:
                rargs = self.blog.atom2hash(ri.entry)
                if rargs['id'] not in last_ids:
                    last_items.append(rargs)
                    last_ids.append(args['id'])
            # update index
            self.blog.updateIndex(blog_id, last_items)
            # update atom
            self.blog.updateAtom(blog_id, last_items)

    def getJid(self):
	"""Return the JID the connection is authenticed as."""

	return self.xmlstream.authenticator.jid

<|endoftext|>


<commit_before>from bgl import *
import bge
import aud

from .widget import *

class Video(Widget):
	"""Widget for displaying video only (i.e., no sound)"""

	def __init__(self, parent, name, vid, play_audio=False, repeat=0, aspect=None, size=[1, 1], pos=[0, 0],
				sub_theme='', options=BGUI_DEFAULT):
		"""
		:param parent: the widget's parent
		:param name: the name of the widget
		:param vid: the video to use for the widget
		:param play_audio: play the audio track of the video
		:param repeat: how many times to repeat the video (-1 = infinite)
		:param aspect: constrain the widget size to a specified aspect ratio
		:param size: a tuple containing the width and height
		:param pos: a tuple containing the x and y position
		:param sub_theme: name of a sub_theme defined in the theme file (similar to CSS classes)
		:param options: various other options

		"""

		Widget.__init__(self, parent, name, aspect, size, pos, sub_theme, options)

		# Generate a texture

		id_buf = Buffer(GL_INT, 1)
		glGenTextures(1, id_buf)

		self.tex_id = id_buf.to_list()[0] if hasattr(id_buf, "to_list") else id_buf.list[0]

		# Bind and load the texture data
		glBindTexture(GL_TEXTURE_2D, self.tex_id)
		video = bge.texture.VideoFFmpeg(vid)
		video.repeat = repeat
		video.play()
		im_buf = video.image

		if im_buf:
			# Setup some parameters
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)

			glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE)

			# Upload the texture data
			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, video.size[0], video.size[1],
							0, GL_RGBA, GL_UNSIGNED_BYTE, im_buf)
		else:
			print("Unable to load the video:", vid)

		if play_audio:
			f = aud.Factory(vid)
			self.aud_handle = aud.device().play(f)
		else:
			self.aud_handle = None

		# Store the video for later
		self.video = video

	def play(self, start, end, use_frames=True, fps=None):
		start = float(start)
		end = float(end)

		if use_frames:
			if not fps:
				fps = self.video.framerate
				print("Using fps:", fps)
			start /= fps
			end /= fps

		if start == end:
			end += 0.1
		self.video.stop()
		self.video.range = [start, end]
		self.video.play()

	def _cleanup(self):
		id_buf = Buffer(GL_INT, 1)
		id_buf[0] = self.tex_id
		glDeleteTextures(1, id_buf)

		if self.aud_handle:
			self.aud_handle.stop()

		Widget._cleanup(self)

	def _draw(self):
		"""Draws the video frame"""

		# Enable textures and alpha blending
		glEnable(GL_TEXTURE_2D)

		glEnable(GL_BLEND)
		glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)

		# Bind the texture
		glBindTexture(GL_TEXTURE_2D, self.tex_id)
		texco = [(0, 0), (1, 0), (1, 1), (0, 1)]

		# Upload the next frame to the graphics
		im_buf = self.video.image

		if im_buf:
			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.video.size[0], self.video.size[1],
							0, GL_RGBA, GL_UNSIGNED_BYTE, im_buf)

		# Draw the textured quad
		glColor4f(1, 1, 1, 1)

		glBegin(GL_QUADS)
		for i in range(4):
			glTexCoord2f(texco[i][0], texco[i][1])
			glVertex2f(self.gl_position[i][0], self.gl_position[i][1])
		glEnd()

		glBindTexture(GL_TEXTURE_2D, 0)

		# Invalidate the image
		self.video.refresh()

		# Now draw the children
		Widget._draw(self)<commit_msg>The Video widget is now a subclass of Image, which reduces code duplication.<commit_after>from bgl import *
import bge
import aud

from .widget import *
from .image import Image

class Video(Image):
	"""Widget for displaying video only (i.e., no sound)"""

	def __init__(self, parent, name, vid, play_audio=False, repeat=0, aspect=None, size=[1, 1], pos=[0, 0],
				sub_theme='', options=BGUI_DEFAULT):
		"""
		:param parent: the widget's parent
		:param name: the name of the widget
		:param vid: the video to use for the widget
		:param play_audio: play the audio track of the video
		:param repeat: how many times to repeat the video (-1 = infinite)
		:param aspect: constrain the widget size to a specified aspect ratio
		:param size: a tuple containing the width and height
		:param pos: a tuple containing the x and y position
		:param sub_theme: name of a sub_theme defined in the theme file (similar to CSS classes)
		:param options: various other options

		"""

		Image.__init__(self, parent, name, None, aspect, size, pos, sub_theme=sub_theme, options=options)

		# Bind and load the texture data
		glBindTexture(GL_TEXTURE_2D, self.tex_id)
		video = bge.texture.VideoFFmpeg(vid)
		video.repeat = repeat
		video.play()
		im_buf = video.image

		if im_buf:
			# Setup some parameters
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)

			glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE)

			# Upload the texture data
			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, video.size[0], video.size[1],
							0, GL_RGBA, GL_UNSIGNED_BYTE, im_buf)
		else:
			print("Unable to load the video:", vid)

		if play_audio:
			f = aud.Factory(vid)
			self.aud_handle = aud.device().play(f)
		else:
			self.aud_handle = None

		# Store the video for later
		self.video = video

	def play(self, start, end, use_frames=True, fps=None):
		start = float(start)
		end = float(end)

		if use_frames:
			if not fps:
				fps = self.video.framerate
				print("Using fps:", fps)
			start /= fps
			end /= fps

		if start == end:
			end += 0.1
		self.video.stop()
		self.video.range = [start, end]
		self.video.play()

	def _cleanup(self):
		if self.aud_handle:
			self.aud_handle.stop()

		Image._cleanup(self)

	def update_image(self, img):
		"""This does nothing on a Video widget"""

		# This breaks the Liskov substitution principle, but I think the way to solve
		# that is to change the Image interface a bit to avoid the problem.

		Image.update_image(self, None)

	def _draw(self):
		"""Draws the video frame"""

		# Upload the next frame to the graphics
		im_buf = self.video.image

		if im_buf:
			glBindTexture(GL_TEXTURE_2D, self.tex_id)
			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.video.size[0], self.video.size[1],
							0, GL_RGBA, GL_UNSIGNED_BYTE, im_buf)

		# Draw the textured quad
		Image._draw(self)

		# Invalidate the image
		self.video.refresh()
		<|endoftext|>