<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

import argparse
import sys
import struct
import os
import elftools
from distutils.version import LooseVersion
from elftools.elf.elffile import ELFFile
from elftools.elf.sections import SymbolTableSection

if LooseVersion(elftools.__version__) < LooseVersion('0.24'):
    sys.stderr.write("pyelftools is out of date, need version 0.24 or later\n")
    sys.exit(1)

def debug(text):
    if not args.verbose:
        return
    sys.stdout.write(os.path.basename(sys.argv[0]) + ": " + text + "\n")

def error(text):
    sys.stderr.write(os.path.basename(sys.argv[0]) + ": " + text + "\n")
    sys.exit(1)

gdt_pd_fmt = "<HIH"

FLAGS_GRAN = 1 << 7  # page granularity
ACCESS_EX = 1 << 3  # executable
ACCESS_DC = 1 << 2  # direction/conforming
ACCESS_RW = 1 << 1  # read or write permission

# 6 byte pseudo descriptor, but we're going to actually use this as the
# zero descriptor and return 8 bytes

def create_gdt_pseudo_desc(addr, size):
    debug("create pseudo decriptor: %x %x" % (addr, size))
    # ...and take back one byte for the Intel god whose Ark this is...
    size = size - 1
    return struct.pack(gdt_pd_fmt, size, addr, 0)

# Limit argument always in bytes
def chop_base_limit(base, limit):
    base_lo = base & 0xFFFF
    base_mid = (base >> 16) & 0xFF
    base_hi = (base >> 24) & 0xFF

    limit_lo = limit & 0xFFFF
    limit_hi = (limit >> 16) & 0xF

    return (base_lo, base_mid, base_hi, limit_lo, limit_hi)

gdt_ent_fmt = "<HHBBBB"

def create_code_data_entry(base, limit, dpl, flags, access):
    debug("create code or data entry: %x %x %x %x %x" %
          (base, limit, dpl, flags, access))

    base_lo, base_mid, base_hi, limit_lo, limit_hi = chop_base_limit(base,
                                                                     limit)

    # This is a valid descriptor
    present = 1

    # 32-bit protected mode
    size = 1

    # 1 = code or data, 0 = system type
    desc_type = 1

    # Just set accessed to 1 already so the CPU doesn't need it update it,
    # prevents freakouts if the GDT is in ROM, we don't care about this
    # bit in the OS
    accessed = 1

    access = access | (present << 7) | (dpl << 5) | (desc_type << 4) | accessed
    flags = flags | (size << 6) | limit_hi

    return struct.pack(gdt_ent_fmt, limit_lo, base_lo, base_mid,
                       access, flags, base_hi)

def create_tss_entry(base, limit, dpl):
    debug("create TSS entry: %x %x %x" % (base, limit, dpl))
    present = 1

    base_lo, base_mid, base_hi, limit_lo, limit_hi, = chop_base_limit(base,
                                                                      limit)

    type_code = 0x9  # non-busy 32-bit TSS descriptor
    gran = 0

    flags = (gran << 7) | limit_hi
    type_byte = ((present << 7) | (dpl << 5) | type_code)

    return struct.pack(gdt_ent_fmt, limit_lo, base_lo, base_mid,
                       type_byte, flags, base_hi)

def get_symbols(obj):
    for section in obj.iter_sections():
        if isinstance(section, SymbolTableSection):
            return {sym.name: sym.entry.st_value
                    for sym in section.iter_symbols()}

    raise LookupError("Could not find symbol table")

def parse_args():
    global args
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("-k", "--kernel", required=True,
                        help="Zephyr kernel image")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Print extra debugging information")
    parser.add_argument("-o", "--output-gdt", required=True,
                        help="output GDT binary")
    args = parser.parse_args()
    if "VERBOSE" in os.environ:
        args.verbose = 1

def main():
    parse_args()

    with open(args.kernel, "rb") as fp:
        kernel = ELFFile(fp)
        syms = get_symbols(kernel)

    # NOTE: use-cases are extremely limited; we always have a basic flat
    # code/data segments. If we are doing stack protection, we are going to
    # have two TSS to manage the main task and the special task for double
    # fault exception handling
    if "CONFIG_USERSPACE" in syms:
        num_entries = 7
    elif "CONFIG_HW_STACK_PROTECTION" in syms:
        num_entries = 5
    else:
        num_entries = 3

    gdt_base = syms["_gdt"]

    with open(args.output_gdt, "wb") as fp:
        # The pseudo descriptor is stuffed into the NULL descriptor
        # since the CPU never looks at it
        fp.write(create_gdt_pseudo_desc(gdt_base, num_entries * 8))

        # Selector 0x08: code descriptor
        fp.write(create_code_data_entry(0, 0xFFFFF, 0,
                                        FLAGS_GRAN, ACCESS_EX | ACCESS_RW))

        # Selector 0x10: data descriptor
        fp.write(create_code_data_entry(0, 0xFFFFF, 0,
                                        FLAGS_GRAN, ACCESS_RW))

        if num_entries >= 5:
            main_tss = syms["_main_tss"]
            df_tss = syms["_df_tss"]

            # Selector 0x18: main TSS
            fp.write(create_tss_entry(main_tss, 0x67, 0))

            # Selector 0x20: double-fault TSS
            fp.write(create_tss_entry(df_tss, 0x67, 0))

        if num_entries == 7:
            # Selector 0x28: code descriptor, dpl = 3
            fp.write(create_code_data_entry(0, 0xFFFFF, 3,
                                            FLAGS_GRAN, ACCESS_EX | ACCESS_RW))

            # Selector 0x30: data descriptor, dpl = 3
            fp.write(create_code_data_entry(0, 0xFFFFF, 3,
                                            FLAGS_GRAN, ACCESS_RW))

if __name__ == "__main__":
    main()
<commit_msg>x86: gen_gdt.py: expand doctsring<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

"""Generate a Global Descriptor Table (GDT) for x86 CPUs.

For additional detail on GDT and x86 memory management, please
consult the IA Architecture SW Developer Manual, vol. 3.

This script accepts as input the zephyr_prebuilt.elf binary,
which is a link of the Zephyr kernel without various build-time
generated data structures (such as the GDT) inserted into it.
This kernel image has been properly padded such that inserting
these data structures will not disturb the memory addresses of
other symbols.

The output is a GDT whose contents depend on the kernel
configuration. With no memory protection features enabled,
we generate flat 32-bit code and data segments. If hardware-
based stack overflow protection or userspace is enabled,
we additionally create descriptors for the main and double-
fault IA tasks, needed for userspace privilege elevation and
double-fault handling. If userspace is enabled, we also create
flat code/data segments for ring 3 execution.
"""

import argparse
import sys
import struct
import os
import elftools
from distutils.version import LooseVersion
from elftools.elf.elffile import ELFFile
from elftools.elf.sections import SymbolTableSection

if LooseVersion(elftools.__version__) < LooseVersion('0.24'):
    sys.stderr.write("pyelftools is out of date, need version 0.24 or later\n")
    sys.exit(1)

def debug(text):
    if not args.verbose:
        return
    sys.stdout.write(os.path.basename(sys.argv[0]) + ": " + text + "\n")

def error(text):
    sys.stderr.write(os.path.basename(sys.argv[0]) + ": " + text + "\n")
    sys.exit(1)

gdt_pd_fmt = "<HIH"

FLAGS_GRAN = 1 << 7  # page granularity
ACCESS_EX = 1 << 3  # executable
ACCESS_DC = 1 << 2  # direction/conforming
ACCESS_RW = 1 << 1  # read or write permission

# 6 byte pseudo descriptor, but we're going to actually use this as the
# zero descriptor and return 8 bytes

def create_gdt_pseudo_desc(addr, size):
    debug("create pseudo decriptor: %x %x" % (addr, size))
    # ...and take back one byte for the Intel god whose Ark this is...
    size = size - 1
    return struct.pack(gdt_pd_fmt, size, addr, 0)

# Limit argument always in bytes
def chop_base_limit(base, limit):
    base_lo = base & 0xFFFF
    base_mid = (base >> 16) & 0xFF
    base_hi = (base >> 24) & 0xFF

    limit_lo = limit & 0xFFFF
    limit_hi = (limit >> 16) & 0xF

    return (base_lo, base_mid, base_hi, limit_lo, limit_hi)

gdt_ent_fmt = "<HHBBBB"

def create_code_data_entry(base, limit, dpl, flags, access):
    debug("create code or data entry: %x %x %x %x %x" %
          (base, limit, dpl, flags, access))

    base_lo, base_mid, base_hi, limit_lo, limit_hi = chop_base_limit(base,
                                                                     limit)

    # This is a valid descriptor
    present = 1

    # 32-bit protected mode
    size = 1

    # 1 = code or data, 0 = system type
    desc_type = 1

    # Just set accessed to 1 already so the CPU doesn't need it update it,
    # prevents freakouts if the GDT is in ROM, we don't care about this
    # bit in the OS
    accessed = 1

    access = access | (present << 7) | (dpl << 5) | (desc_type << 4) | accessed
    flags = flags | (size << 6) | limit_hi

    return struct.pack(gdt_ent_fmt, limit_lo, base_lo, base_mid,
                       access, flags, base_hi)

def create_tss_entry(base, limit, dpl):
    debug("create TSS entry: %x %x %x" % (base, limit, dpl))
    present = 1

    base_lo, base_mid, base_hi, limit_lo, limit_hi, = chop_base_limit(base,
                                                                      limit)

    type_code = 0x9  # non-busy 32-bit TSS descriptor
    gran = 0

    flags = (gran << 7) | limit_hi
    type_byte = ((present << 7) | (dpl << 5) | type_code)

    return struct.pack(gdt_ent_fmt, limit_lo, base_lo, base_mid,
                       type_byte, flags, base_hi)

def get_symbols(obj):
    for section in obj.iter_sections():
        if isinstance(section, SymbolTableSection):
            return {sym.name: sym.entry.st_value
                    for sym in section.iter_symbols()}

    raise LookupError("Could not find symbol table")

def parse_args():
    global args
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("-k", "--kernel", required=True,
                        help="Zephyr kernel image")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Print extra debugging information")
    parser.add_argument("-o", "--output-gdt", required=True,
                        help="output GDT binary")
    args = parser.parse_args()
    if "VERBOSE" in os.environ:
        args.verbose = 1

def main():
    parse_args()

    with open(args.kernel, "rb") as fp:
        kernel = ELFFile(fp)
        syms = get_symbols(kernel)

    # NOTE: use-cases are extremely limited; we always have a basic flat
    # code/data segments. If we are doing stack protection, we are going to
    # have two TSS to manage the main task and the special task for double
    # fault exception handling
    if "CONFIG_USERSPACE" in syms:
        num_entries = 7
    elif "CONFIG_HW_STACK_PROTECTION" in syms:
        num_entries = 5
    else:
        num_entries = 3

    gdt_base = syms["_gdt"]

    with open(args.output_gdt, "wb") as fp:
        # The pseudo descriptor is stuffed into the NULL descriptor
        # since the CPU never looks at it
        fp.write(create_gdt_pseudo_desc(gdt_base, num_entries * 8))

        # Selector 0x08: code descriptor
        fp.write(create_code_data_entry(0, 0xFFFFF, 0,
                                        FLAGS_GRAN, ACCESS_EX | ACCESS_RW))

        # Selector 0x10: data descriptor
        fp.write(create_code_data_entry(0, 0xFFFFF, 0,
                                        FLAGS_GRAN, ACCESS_RW))

        if num_entries >= 5:
            main_tss = syms["_main_tss"]
            df_tss = syms["_df_tss"]

            # Selector 0x18: main TSS
            fp.write(create_tss_entry(main_tss, 0x67, 0))

            # Selector 0x20: double-fault TSS
            fp.write(create_tss_entry(df_tss, 0x67, 0))

        if num_entries == 7:
            # Selector 0x28: code descriptor, dpl = 3
            fp.write(create_code_data_entry(0, 0xFFFFF, 3,
                                            FLAGS_GRAN, ACCESS_EX | ACCESS_RW))

            # Selector 0x30: data descriptor, dpl = 3
            fp.write(create_code_data_entry(0, 0xFFFFF, 3,
                                            FLAGS_GRAN, ACCESS_RW))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>
import os
import subprocess

def tcpdump_start(x, arg_d):
    _, _, _ = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
            "mkdir /tmp/net-applet-shuffler")
    # connect to host and start tcpdump
    # https://wiki.ubuntuusers.de/tcpdump/
    # tcpdump -i [interface] [protocol] -n -s 0 -w
    # /tmp/net-applet-shuffler/[filename] '[filter(e.g. dst port 20000)]
    _, _, exit_code = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
            "tcpdump -i {} -n -s 0 -w /tmp/net-applet-shuffler/dump_{}.pcap "
            "'{}'".format(arg_d["host_interface"], arg_d["applet_id"],
                          arg_d["filter"]))

    if exit_code != 0:
        x.p.msg("error: tcpdump could not be started at host {}\n"
                "failed params:\n".format(arg_d["host_name"]))
        x.p.msg("tcpdump -i {} -n -s 0 -w "
                "/tmp/net-applet-shuffler/dump_{}.pcap '{}'".format
                (arg_d["host_interface"], arg_d["applet_id"], arg_d["filter"]))
        return False

    # retrieve tcpdump pid
    # there will be two processes running, tcpdump and the file writer
    # the dump will have a sudo in front and end the writer gracefully,
    # if ended gracefully
    pid_tcpdump = "0"
    stdout = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
                        "ps -ef | grep tcpdump")
    stdout_decoded = stdout[0].decode("utf-8")
    for x in stdout_decoded.splitlines():
        # unique identifier
        if "sudo tcpdump -i {} -n -s 0 -w /tmp/net-applet-shuffler/tcpdump_{} " \
           "'{}'".format(arg_d["host_interface"], arg_d["applet_id"],
                         arg_d["filter"]) in x:
            pid_tcpdump = x.split()[1]

    # save pid to file
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"], "touch "
            "/tmp/net-applet-shuffler/tcpdump_{}".format(arg_d["applet_id"]))
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"], "sh -c \"echo '{}' > "
            "/tmp/net-applet-shuffler/tcpdump_{}\"".format(pid_tcpdump,
                                                           arg_d["applet_id"]))

    return True

def transfer_dumpfile(x, arg_d):

    # make local directories
    os.makedirs(arg_d["local_file_name"])
    # use secure copy to retrieve the dump file
    process = subprocess.Popen("scp {}@{}:/tmp/net-applet-shuffler/dump_{}.pcap"
            " {}".format(arg_d["host_user"], arg_d["host_ip"],
            arg_d["applet_id"], arg_d["local_file_name"]).split(),
            stdout=subprocess.PIPE)
    # block until file transfer is done
    _, _ = process.communicate()
    if process.returncode != 0:
        x.p.msg("error transferring the dumpfile dump_{}.pcap from host {"
                "}".format(arg_d["applet_id"], arg_d["host_name"]))

        return False

    # clean up dumpfile on host
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"], "rm "
            "/tmp/net-applet-shuffler/dump_{}.pcpa".format(arg_d["applet_id"]))

    return True

def tcpdump_stop(x, arg_d):
    # retrieve pid
    stdout, _, exit_code = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
            "echo \"$(</tmp/net-applet-shuffler/tcpdump_{})\""
            .format(arg_d["applet_id"]))
    pid_tcpdump = stdout.decode("utf-8").splitlines()[0]

    if exit_code != 0:
        x.p.msg("error: tcpdump pid at host {} could not be retrieved\n"
                "failed params:\n".format(arg_d["host_name"]))
        x.p.msg("echo \"$(</tmp/net-applet-shuffler/tcpdump_{})\""
                .format(arg_d["applet_id"]))

        return False

    # end tcpdump gracefully
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
               "kill -2 {}".format(pid_tcpdump))
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
               "kill {}".format(pid_tcpdump))
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
               "rm /tmp/net-applet-shuffler/tcpdump_{"
               "}".format(arg_d["applet_id"]))

    if not transfer_dumpfile(x, arg_d):

        return False

    return True

def main(x, conf, args):
    if not len(args) >= 4:
        x.p.msg("wrong usage. use: [hostname] id:[number] mode:[start|stop] "
                "local-file-name:\"[filename]\" filter:\"[filter]\"\n")
        return False

    # arguments dictionary
    arg_d = {}
    arg_d["host_name"] = args[0]
    arg_d["applet_id"] = args[1].split(":")[1]
    arg_d["applet_mode"] = args[2].split(":")[1]
    arg_d["local_file_name"] = args[3].split("\"")[1]
    arg_d["filter"] = args[4].split("\"")[1]
    # retrieve: host ip, host user name
    arg_d["host_ip"] = conf['boxes'][arg_d["host_name"]]["interfaces"][0]['ip-address']
    arg_d["host_interface"] = conf['boxes'][arg_d["host_name"]][0]['name']
    arg_d["host_user"] = conf['boxes'][arg_d["host_name"]]['user']

    # applet mode:
    # start: start the tcpdump
    if arg_d["applet_mode"] == "start":
        if tcpdump_start(x, arg_d):

            return True

        return False

    # applet mode:
    # stop: stop the tcpdump
    # blocks until file is transferred
    if arg_d["applet_mode"] == "stop":
        if tcpdump_stop(x, arg_d):

            return True

        return False
<commit_msg>small improvements<commit_after>
import os
import subprocess

def tcpdump_start(x, arg_d):
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
            "mkdir /tmp/net-applet-shuffler")
    # connect to host and start tcpdump
    # https://wiki.ubuntuusers.de/tcpdump/
    # tcpdump -i [interface] [protocol] -n -s 0 -w
    # /tmp/net-applet-shuffler/[filename] '[filter(e.g. dst port 20000)]
    _, _, exit_code = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
            "tcpdump -i {} -n -s 0 -w /tmp/net-applet-shuffler/dump_{}.pcap "
            "'{}'".format(arg_d["host_interface"], arg_d["applet_id"],
                          arg_d["filter"]))

    if exit_code != 0:
        x.p.msg("error: tcpdump could not be started at host {}\n"
                "failed params:\n".format(arg_d["host_name"]))
        x.p.msg("tcpdump -i {} -n -s 0 -w "
                "/tmp/net-applet-shuffler/dump_{}.pcap '{}'".format
                (arg_d["host_interface"], arg_d["applet_id"], arg_d["filter"]))
        return False

    # retrieve tcpdump pid
    # there will be two processes running, tcpdump and the file writer
    # the dump will have a sudo in front and end the writer gracefully,
    # if ended gracefully
    pid_tcpdump = "0"
    stdout = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
                        "ps -ef | grep tcpdump")
    stdout_decoded = stdout[0].decode("utf-8")
    for x in stdout_decoded.splitlines():
        # unique identifier
        if "sudo tcpdump -i {} -n -s 0 -w /tmp/net-applet-shuffler/tcpdump_{} " \
           "'{}'".format(arg_d["host_interface"], arg_d["applet_id"],
                         arg_d["filter"]) in x:
            pid_tcpdump = x.split()[1]

    # save pid to file
    path_to_tcpdump_pid = "/tmp/net-applet-shuffler/tcpdump_{" \
                          "}".format(arg_d["applet_id"])
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"], "touch {"
            "}".format(path_to_tcpdump_pid))
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"], "sh -c \"echo '{}' > "
            "{}\"".format(pid_tcpdump, path_to_tcpdump_pid))

    return True

def transfer_dumpfile(x, arg_d):

    # make local directories
    os.makedirs(arg_d["local_file_name"])
    # use secure copy to retrieve the dump file
    process = subprocess.Popen("scp {}@{}:/tmp/net-applet-shuffler/dump_{}.pcap"
            " {}".format(arg_d["host_user"], arg_d["host_ip"],
            arg_d["applet_id"], arg_d["local_file_name"]).split(),
            stdout=subprocess.PIPE)
    # block until file transfer is done
    process.communicate()
    if process.returncode != 0:
        x.p.msg("error transferring the dumpfile dump_{}.pcap from host {"
                "}".format(arg_d["applet_id"], arg_d["host_name"]))

        return False

    # clean up dumpfile on host
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"], "rm "
            "/tmp/net-applet-shuffler/dump_{}.pcpa".format(arg_d["applet_id"]))

    return True

def tcpdump_stop(x, arg_d):
    # retrieve pid
    stdout, _, exit_code = x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
            "echo \"$(</tmp/net-applet-shuffler/tcpdump_{})\""
            .format(arg_d["applet_id"]))

    if exit_code != 0:
        x.p.msg("error: tcpdump pid at host {} could not be retrieved\n"
                "failed params:\n".format(arg_d["host_name"]))
        x.p.msg("echo \"$(</tmp/net-applet-shuffler/tcpdump_{})\""
                .format(arg_d["applet_id"]))

        return False

    pid_tcpdump = stdout.decode("utf-8").splitlines()[0]
    # end tcpdump gracefully
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
               "kill -2 {}".format(pid_tcpdump))
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
               "kill {}".format(pid_tcpdump))
    x.ssh.exec(arg_d["host_ip"], arg_d["host_user"],
               "rm /tmp/net-applet-shuffler/tcpdump_{"
               "}".format(arg_d["applet_id"]))

    if not transfer_dumpfile(x, arg_d):

        return False

    return True

def main(x, conf, args):
    if not len(args) >= 4:
        x.p.msg("wrong usage. use: [hostname] id:[number] mode:[start|stop] "
                "local-file-name:\"[filename]\" filter:\"[filter]\"\n")
        return False

    # arguments dictionary
    arg_d = {}
    arg_d["host_name"] = args[0]
    arg_d["applet_id"] = args[1].split(":")[1]
    arg_d["applet_mode"] = args[2].split(":")[1]
    arg_d["local_file_name"] = args[3].split("\"")[1]
    arg_d["filter"] = args[4].split("\"")[1]
    # retrieve: host ip, host user name
    arg_d["host_ip"] = conf['boxes'][arg_d["host_name"]]["interfaces"][0]['ip-address']
    arg_d["host_interface"] = conf['boxes'][arg_d["host_name"]][0]['name']
    arg_d["host_user"] = conf['boxes'][arg_d["host_name"]]['user']

    # applet mode:
    # start: start the tcpdump
    if arg_d["applet_mode"] == "start":
        if tcpdump_start(x, arg_d):

            return True

        return False

    # applet mode:
    # stop: stop the tcpdump
    # blocks until file is transferred
    if arg_d["applet_mode"] == "stop":
        if tcpdump_stop(x, arg_d):

            return True

        return False
<|endoftext|>


<commit_before>### Implementation of the Lightsweeper floor
from LSRealTile import LSRealTile
from LSRealTile import lsOpen

import time
import os
import Colors
import Shapes
import pickle
from Move import Move

#handles all communications with RealTile objects, serving as the interface to the
#actual lightsweeper floor. thus updates are pushed to it (display) and also pulled from it
#(sensor changes)
class LSRealFloor():
    COLS = 8
    ROWS = 6
    SENSOR_THRESHOLD = 15
    sharedSerials = list()

    def __init__(self, rows=ROWS, cols=COLS, serials=None):
        self.rows = rows
        self.cols = cols
        print("RealFloor init", rows, cols)

        tilepile = lsOpen()

        for port in tilepile.lsMatrix:
            self.sharedSerials.append(port)
        print(repr(self.sharedSerials))  #debugging

  # Old code:
#        if serials is None:
#            # top-left, A1,2,3
#            comPort1 = "COM5"
#            # top-right, A4,5,6
#            comPort2 = "COM12"
#            # bottom-left, B1,2,3
#            comPort3 = "COM13"
#            # bottom-right, B4,5,6
#            comPort4 = "COM15"
#            availPorts = list(serial_ports())
#            print("Available serial ports:" + str(availPorts))
#            print("connecting to ", comPort1, comPort2, comPort3, comPort4)
#            serial1 = None
#            serial2 = None
#            serial3 = None
#            serial4 = None
#            try:
#                serial1 = Serial(comPort1, 19200, timeout=0.005)
#            except IOError:
#                print("Could not open", comPort1)
#            try:
#                serial2 = Serial(comPort2, 19200, timeout=0.005)
#            except IOError:
#               print("Could not open", comPort2)
#            try:
#                serial3 = Serial(comPort3, 19200, timeout=0.005)
#            except IOError:
#                print("Could not open", comPort3)
#            try:
#                serial4 = Serial(comPort4, 19200, timeout=0.005)
#            except IOError:
#                print("Could not open", comPort4)
#            self.sharedSerials = [serial1, serial2, serial3, serial4]
#            print("finished with COM ports")
#        else:
#            self.sharedSerials = serials

        self.addressToRowColumn = {}
        # make all the rows
        self.tileRows = []
        print("Using Jay-Daddy's pickling system")
        picklefile = open('Bfloor', 'r')
        #print(pickle)
        config = pickle.Unpickler(picklefile)
        print(config.load())
        i = 0
        for row in range(rows):
            tiles = []
            self.tileAddresses = []
            for col in range(cols):
                line = ""
                while line is "":
                    line = (pickle.readline()).strip('()\n').replace('\'','')
                line = tuple(line.split(','))
                # the COM entry should just be the number 1, 2, 3, or 4 instead of the COM port.
                # 1 being top-left, 4 bottom-right
                comNumber = int(line[2]) - 1
                tile = LSRealTile(self.sharedSerials[comNumber])
                tile.comNumber = comNumber
                address = int(line[3])
                tile.assignAddress(address)
                self.addressToRowColumn[(address,comNumber)] = (row, col)
                tile.setColor(Colors.WHITE)
                tile.setShape(Shapes.ZERO)
                #print("address assigned:", tile.getAddress())
                #print("test getAddress", tile.getAddress())
                i += 1
                #assign address
                tiles.append(tile)
            self.tileRows.append(tiles)
        return

    def setAllColor(self, color):
        for row in self.tileRows:
            for tile in row:
                tile.setColor(color)

    def set(self, row, col, shape, color):
        tile = self.tileRows[row][col];
        tile.setColor(color)
        tile.setShape(shape)

    def setColor(self, row, col, color):
        tile = self.tileRows[row][col]
        tile.setColor(color)

    def setShape(self, row, col, shape):
        tile = self.tileRows[row][col]
        tile.setShape(shape)

    def setSegmentsCustom(self, row, col, segments):
        tile = self.tileRows[row][col]
        tile.setSegmentsCustom(segments)

    def RAINBOWMODE(self, updateFrequency = 0.4):
        self.pollSensors()
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.RED)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.YELLOW)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.GREEN)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.CYAN)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.BLUE)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.VIOLET)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.WHITE)
                tile.setShape(126)
        wait(updateFrequency)

    def printAddresses(self):
        s = ""
        for row in range(0,self.rows):
            for col in range(0, self.cols):
                s += str(self.tileRows[row][col].getAddress()) + " "
            #print(s)
            s = ""

    def pollSensors(self):
        sensorsChanged = []
        tiles = self._getTileList(0,0)
        for tile in tiles:
            val = tile.sensorStatus()
            if val < self.SENSOR_THRESHOLD:
                rowCol = self.addressToRowColumn[(tile.address, tile.comNumber)]
                move = Move(rowCol[0], rowCol[1], val)
                sensorsChanged.append(move)
        return sensorsChanged

    def _getTileList(self,row,column):
        tileList = []
        #whole floor
        # whole floor
        if row < 1 and column < 1:
            for tileRow in self.tileRows:
                for tile in tileRow:
                    tileList.append(tile)
                    count = len(tileList)
        # whole row
        elif column < 1:
            tileRow = self.tileRows[row-1]
            for tile in tileRow:
                tileList.append(tile)
                count = len(tileList)
        # whole column
        elif row < 1:
            for tileRow in self.tileRows:
                tile = tileRow[column-1]
                tileList.append(tile)
                count = len(tileList)
        # single tile
        else:
            tileRow = self.tileRows[row-1]
            tileList = [tileRow[column-1]]
        return tileList

    def clearboard(self):
        zeroTile = LSRealTile(self.sharedSerials[0])
        zeroTile.assignAddress(0)
        zeroTile.blank()
        zeroTile = LSRealTile(self.sharedSerials[1])
        zeroTile.assignAddress(0)
        zeroTile.blank()
        zeroTile = LSRealTile(self.sharedSerials[2])
        zeroTile.assignAddress(0)
        zeroTile.blank()
        zeroTile = LSRealTile(self.sharedSerials[3])
        zeroTile.assignAddress(0)
        zeroTile.blank()

    def showboard(self):
        return

    def refreshboard(self):
        return

    def resetboard(self):
        return

    def purgetile(self,tile):
        return False

    def clock(self):
        return

def wait(seconds):
    # self.pollSensors()
    currentTime = time.time()
    while time.time() - currentTime < seconds:
        pass

if __name__ == "__main__":
    print("todo: testing RealFloor")
    floor = LSRealFloor(3, 3)
    #call not implemented yet
    #floor.setSegmentsCustom(0, 0, [Colors.RED, Colors.YELLOW, Colors.GREEN, Colors.CYAN, Colors.BLUE, Colors.VIOLET, Colors.WHITE])
    floor.RAINBOWMODE(0.1)
    print("Floor should be displaying things now")
    floor.RAINBOWMODE(0.1)
    floor.RAINBOWMODE(0.1)
    while True:
        floor.RAINBOWMODE(1)
<commit_msg>Modifying LSRealFloor.py to use new json config (work in progress)<commit_after>### Implementation of the Lightsweeper floor
from LSRealTile import LSRealTile
from LSRealTile import lsOpen

import time
import os
import Colors
import Shapes
import json
from Move import Move

#handles all communications with RealTile objects, serving as the interface to the
#actual lightsweeper floor. thus updates are pushed to it (display) and also pulled from it
#(sensor changes)
class LSRealFloor():
    COLS = 8
    ROWS = 6
    SENSOR_THRESHOLD = 15
    sharedSerials = list()

    def __init__(self, rows=ROWS, cols=COLS, serials=None):
        fileName = 'Bfloor' # Testing
        self.rows = rows
        self.cols = cols
        print("RealFloor init", rows, cols)

        tilepile = lsOpen()

        for port in tilepile.lsMatrix:
            self.sharedSerials.append(port)
        print(repr(self.sharedSerials))  #debugging

  # Old code:
#        if serials is None:
#            # top-left, A1,2,3
#            comPort1 = "COM5"
#            # top-right, A4,5,6
#            comPort2 = "COM12"
#            # bottom-left, B1,2,3
#            comPort3 = "COM13"
#            # bottom-right, B4,5,6
#            comPort4 = "COM15"
#            availPorts = list(serial_ports())
#            print("Available serial ports:" + str(availPorts))
#            print("connecting to ", comPort1, comPort2, comPort3, comPort4)
#            serial1 = None
#            serial2 = None
#            serial3 = None
#            serial4 = None
#            try:
#                serial1 = Serial(comPort1, 19200, timeout=0.005)
#            except IOError:
#                print("Could not open", comPort1)
#            try:
#                serial2 = Serial(comPort2, 19200, timeout=0.005)
#            except IOError:
#               print("Could not open", comPort2)
#            try:
#                serial3 = Serial(comPort3, 19200, timeout=0.005)
#            except IOError:
#                print("Could not open", comPort3)
#            try:
#                serial4 = Serial(comPort4, 19200, timeout=0.005)
#            except IOError:
#                print("Could not open", comPort4)
#            self.sharedSerials = [serial1, serial2, serial3, serial4]
#            print("finished with COM ports")
#        else:
#            self.sharedSerials = serials

        self.addressToRowColumn = {}
        # make all the rows
        self.tileRows = []
        print("Using Jay-Daddy's pickling system")
        with open(fileName) as configFile:
            config = json.load(configFile)
        for (row, col, port, addr) in config:
            tile = LSRealTile(tilepile.lsSerial(port))
            tile.assignAddress(addr)
            tile.setColor(Colors.WHITE) # Debugging
            tile.setShape(Shapes.ZERO)  # Debugging
            print ("Address assigned: " + str(tile.getAddress())) # Debugging
            input() # debugging

 # Old code:
#        for row in range(rows):
#            tiles = []
#            self.tileAddresses = []
#            for col in range(cols):
#                line = ""
#                while line is "":
#                    line = (pickle.readline()).strip('()\n').replace('\'','')
#                line = tuple(line.split(','))
#                # the COM entry should just be the number 1, 2, 3, or 4 instead of the COM port.
#                # 1 being top-left, 4 bottom-right
#                comNumber = int(line[2]) - 1
#                tile = LSRealTile(self.sharedSerials[comNumber])
#                tile.comNumber = comNumber
#                address = int(line[3])
#                tile.assignAddress(address)
#                self.addressToRowColumn[(address,comNumber)] = (row, col)
#                tile.setColor(Colors.WHITE)
#                tile.setShape(Shapes.ZERO)
#                #print("address assigned:", tile.getAddress())
#                #print("test getAddress", tile.getAddress())
#                i += 1
#                #assign address
#                tiles.append(tile)
#            self.tileRows.append(tiles)

        return

    def setAllColor(self, color):
        for row in self.tileRows:
            for tile in row:
                tile.setColor(color)

    def set(self, row, col, shape, color):
        tile = self.tileRows[row][col];
        tile.setColor(color)
        tile.setShape(shape)

    def setColor(self, row, col, color):
        tile = self.tileRows[row][col]
        tile.setColor(color)

    def setShape(self, row, col, shape):
        tile = self.tileRows[row][col]
        tile.setShape(shape)

    def setSegmentsCustom(self, row, col, segments):
        tile = self.tileRows[row][col]
        tile.setSegmentsCustom(segments)

    def RAINBOWMODE(self, updateFrequency = 0.4):
        self.pollSensors()
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.RED)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.YELLOW)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.GREEN)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.CYAN)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.BLUE)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.VIOLET)
                tile.setShape(126)
        wait(updateFrequency)
        for row in self.tileRows:
            for tile in row:
                tile.setColor(Colors.WHITE)
                tile.setShape(126)
        wait(updateFrequency)

    def printAddresses(self):
        s = ""
        for row in range(0,self.rows):
            for col in range(0, self.cols):
                s += str(self.tileRows[row][col].getAddress()) + " "
            #print(s)
            s = ""

    def pollSensors(self):
        sensorsChanged = []
        tiles = self._getTileList(0,0)
        for tile in tiles:
            val = tile.sensorStatus()
            if val < self.SENSOR_THRESHOLD:
                rowCol = self.addressToRowColumn[(tile.address, tile.comNumber)]
                move = Move(rowCol[0], rowCol[1], val)
                sensorsChanged.append(move)
        return sensorsChanged

    def _getTileList(self,row,column):
        tileList = []
        #whole floor
        # whole floor
        if row < 1 and column < 1:
            for tileRow in self.tileRows:
                for tile in tileRow:
                    tileList.append(tile)
                    count = len(tileList)
        # whole row
        elif column < 1:
            tileRow = self.tileRows[row-1]
            for tile in tileRow:
                tileList.append(tile)
                count = len(tileList)
        # whole column
        elif row < 1:
            for tileRow in self.tileRows:
                tile = tileRow[column-1]
                tileList.append(tile)
                count = len(tileList)
        # single tile
        else:
            tileRow = self.tileRows[row-1]
            tileList = [tileRow[column-1]]
        return tileList

    def clearboard(self):
        zeroTile = LSRealTile(self.sharedSerials[0])
        zeroTile.assignAddress(0)
        zeroTile.blank()
        zeroTile = LSRealTile(self.sharedSerials[1])
        zeroTile.assignAddress(0)
        zeroTile.blank()
        zeroTile = LSRealTile(self.sharedSerials[2])
        zeroTile.assignAddress(0)
        zeroTile.blank()
        zeroTile = LSRealTile(self.sharedSerials[3])
        zeroTile.assignAddress(0)
        zeroTile.blank()

    def showboard(self):
        return

    def refreshboard(self):
        return

    def resetboard(self):
        return

    def purgetile(self,tile):
        return False

    def clock(self):
        return

def wait(seconds):
    # self.pollSensors()
    currentTime = time.time()
    while time.time() - currentTime < seconds:
        pass

if __name__ == "__main__":
    print("todo: testing RealFloor")
    floor = LSRealFloor(3, 3)
    #call not implemented yet
    #floor.setSegmentsCustom(0, 0, [Colors.RED, Colors.YELLOW, Colors.GREEN, Colors.CYAN, Colors.BLUE, Colors.VIOLET, Colors.WHITE])
    floor.RAINBOWMODE(0.1)
    print("Floor should be displaying things now")
    floor.RAINBOWMODE(0.1)
    floor.RAINBOWMODE(0.1)
    while True:
        floor.RAINBOWMODE(1)
<|endoftext|>


<commit_before>"""
Parallel & Distributed Algorithms - laboratory

Examples:

- Launch 8 workers with default parameter values:
    > python arir.py 8

- Launch 12 workers with custom parameter values:
    > python arir.py 12 --shared-memory-size 128 --delay-connect 2.0 --delay-transmit 0.5 --delay-process 0.75
"""

__author__ = 'moorglade'

import multiprocessing
import time
import datetime
import sys
import argparse
import random
import math

def _parse_args():
    parser = argparse.ArgumentParser()

    # specify command line options
    parser.add_argument(
        'n_workers',
        help='number of workers in the distributed system',
        type=int
    )
    parser.add_argument(
        'submatrix-size',
        help='single square sub-matrix size',
        type=int,
        default=3
    )
    parser.add_argument(
        'should-count-square',
        help='should compute square root of matrix A or AxB multiplication',
        type=bool,
        default=True
    )
    parser.add_argument(
        '--shared-memory-size',
        help='size of the shared memory array [number of ints]',
        type=int,
        default=16
    )
    parser.add_argument(
        '--delay-connect',
        help='network connection delay [s]',
        type=float,
        default=0.1
    )
    parser.add_argument(
        '--delay-transmit',
        help='network transmission delay [s]',
        type=float,
        default=0.1
    )
    parser.add_argument(
        '--delay-process',
        help='processing delay [s]',
        type=float,
        default=0.1
    )

    return argparse.Namespace(**{
        key.replace('-', '_'): value
        for key, value in vars(parser.parse_args()).items()
        })

class DistributedSystem(object):
    def __init__(self, configuration):
        object.__init__(self)

        shared = SharedState(configuration.n_workers, configuration.shared_memory_size)
        network = Network(configuration)

        self.__workers = [
            Worker(worker_id, configuration, shared, network.get_endpoint(worker_id))
            for worker_id in range(configuration.n_workers)
            ]

    def run(self):
        print 'Launching {} workers...'.format(len(self.__workers))
        start = datetime.datetime.now()

        for worker in self.__workers:
            worker.start()

        print 'Waiting for the workers to terminate...'
        for worker in self.__workers:
            worker.join()

        stop = datetime.datetime.now()
        print 'All workers terminated.'

        print 'Processing took {} seconds.'.format((stop - start).total_seconds())

class SharedState(object):
    def __init__(self, n_workers, shared_memory_size):
        object.__init__(self)
        self.__barrier = Barrier(n_workers)
        self.__memory = multiprocessing.Array('i', shared_memory_size)

    @property
    def barrier(self):
        return self.__barrier

    @property
    def memory(self):
        return self.__memory

class Barrier(object):
    def __init__(self, n):
        object.__init__(self)
        self.__counter = multiprocessing.Value('i', 0, lock=False)
        self.__n = n
        self.__condition = multiprocessing.Condition()

    def wait(self):
        with self.__condition:
            self.__counter.value += 1

            if self.__counter.value == self.__n:
                self.__counter.value = 0
                self.__condition.notify_all()

            else:
                self.__condition.wait()

class SharedMemory(object):
    def __init__(self, shared_memory_size):
        object.__init__(self)
        self.__array = multiprocessing.Array('i', shared_memory_size)

class Network(object):
    any_id = -1

    def __init__(self, configuration):
        object.__init__(self)
        channels = [NetworkChannel(configuration) for _ in range(configuration.n_workers)]
        self.__endpoints = [NetworkEndpoint(channel_id, channels) for channel_id in range(configuration.n_workers)]

    def get_endpoint(self, index):
        return self.__endpoints[index]

class NetworkChannel(object):
    def __init__(self, configuration):
        self.__configuration = configuration

        self.__source_id = multiprocessing.Value('i', Network.any_id, lock=False)
        self.__queue = multiprocessing.Queue(maxsize=1)
        self.__enter_lock = multiprocessing.Lock()
        self.__exit_lock = multiprocessing.Lock()
        self.__enter_lock.acquire()
        self.__exit_lock.acquire()

    def send(self, source_id, data):
        while True:
            self.__enter_lock.acquire()

            if self.__source_id.value in [source_id, Network.any_id]:
                self.__source_id.value = source_id
                self.__queue.put(data)
                time.sleep(self.__configuration.delay_connect + len(data) * self.__configuration.delay_transmit)
                self.__exit_lock.release()
                break

            else:
                self.__enter_lock.release()

    def receive(self, source_id=Network.any_id):
        self.__source_id.value = source_id

        self.__enter_lock.release()
        data = self.__queue.get()
        self.__exit_lock.acquire()

        return self.__source_id.value, data

class NetworkEndpoint(object):
    def __init__(self, channel_id, channels):
        self.__channels = channels
        self.__my_id = channel_id
        self.__my_channel = self.__channels[self.__my_id]

    def send(self, destination_id, data):
        if destination_id == self.__my_id:
            raise RuntimeError('Worker {} tried to send data to itself.'.format(self.__my_id))

        self.__channels[destination_id].send(self.__my_id, data)

    def receive(self, worker_id=Network.any_id):
        return self.__my_channel.receive(worker_id)

class Matrix(object):
    def __init__(self, size, is_counting_square):
        self.dataA = [[random.randint(-2048, 2048) for x in range(size)] for x in range(size)]
        if is_counting_square:
            self.dataB = self.dataA
        else:
            self.dataB = [[random.randint(-2048, 2048) for x in range(size)] for x in range(size)]
        self.dataC = [[0 for x in range(size)] for x in range(size)]

    @staticmethod
    def get_data_of_shift(matrix, direction):
        if direction == 'left':
            return matrix[0]
        elif direction == 'right':
            return matrix[-1]
        elif direction == 'up':
            return list(zip(*matrix)[0])
        elif direction == 'down':
            return list(zip(*matrix)[-1])

        return None

    @staticmethod
    def multiply(matrix_a, matrix_b):
        return [[sum(a*b for a, b in zip(X_row, Y_col)) for Y_col in zip(*matrix_b)] for X_row in matrix_a]

class Worker(multiprocessing.Process):
    def __init__(self, worker_id, configuration, shared, network_endpoint):
        multiprocessing.Process.__init__(self)

        self.__worker_id = worker_id
        self.__configuration = configuration
        self.__shared = shared
        self.__network_endpoint = network_endpoint

        self.carts = Worker.get_carts_layout(configuration.n_workers)
        self.matrix = Matrix(configuration.submatrix_size, configuration.should_count_square)

    @property
    def _n_workers(self):
        return self.__configuration.n_workers

    @property
    def _sub_matrix_size(self):
        return self.__configuration.submatrix_size

    @staticmethod
    def _generate_random_data(length):
        return [random.randint(-2048, 2048) for _ in range(length)]

    def _barrier(self):
        self.__shared.barrier.wait()

    def _send(self, worker_id, data):
        self.__network_endpoint.send(worker_id, data)

    def _receive(self, worker_id=Network.any_id):
        return self.__network_endpoint.receive(worker_id)

    def _log(self, message):
        print '[WORKER {}] {}'.format(self.__worker_id, message)

    # My methods

    # Getting current cart id
    def get_cart_row_id(self):
        return self.get_my_cart_coordinates()[0][1]

    def get_cart_column_id(self):
        return self.get_my_cart_coordinates()[0][0]

    # Returns list of matches, so assuming there is only one, we need to take first element
    def get_my_cart_coordinates(self):
        return [(ind, self.carts[ind].index(self.__worker_id)) for ind in xrange(len(self.carts)) if self.__worker_id in self.carts[ind]]

    # Shifting of meshes between carts
    # Constitutive name - we shift columns, left
    def left_circular_shift_row(self):
        my_column_id = self.get_cart_column_id()
        last_column_id = int(math.sqrt(self._n_workers)) - 1

        if my_column_id == 0:
            # Last column
            my_send_partner = last_column_id
            my_receive_partner = my_column_id + 1
        elif my_column_id == last_column_id:
            # first column
            my_send_partner = my_column_id - 1
            my_receive_partner = 0
        else:
            my_send_partner = my_column_id - 1
            my_receive_partner = my_column_id + 1

        # Sending and receiving at once.. will it work ?!
        self._send(my_send_partner, self.matrix.dataA)
        received_data = self._receive(my_receive_partner)

        self.matrix.dataA = received_data

    # Shifting of meshes between carts
    # Constitutive name - we shift rows, up
    def upward_circular_shift_column(self):
        my_row_id = self.get_cart_row_id()
        last_row_id = int(math.sqrt(self._n_workers)) - 1

        if my_row_id == 0:
            # Last column
            my_send_partner = last_row_id
            my_receive_partner = my_row_id + 1
        elif my_row_id == last_row_id:
            # first column
            my_send_partner = my_row_id - 1
            my_receive_partner = 0
        else:
            my_send_partner = my_row_id - 1
            my_receive_partner = my_row_id + 1

        # Sending and receiving at once.. will it work ?!
        self._send(my_send_partner, self.matrix.dataB)
        received_data = self._receive(my_receive_partner)

        self.matrix.dataB = received_data

    @staticmethod
    def get_carts_layout(number_of_carts):
        carts = [[0 for y in range(int(math.sqrt(number_of_carts)))] for x in range(int(math.sqrt(number_of_carts)))]

        counter = 0
        for x in carts:
            for y in carts:
                carts[x,y] = counter
                counter += 1

        return carts

    def run(self):
        self._log('Started.')

        self._log('Terminated.')

def main():
    random.seed()
    configuration = _parse_args()
    system = DistributedSystem(configuration)
    system.run()

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Minor bugs fixes<commit_after>"""
Parallel & Distributed Algorithms - laboratory

Examples:

- Launch 8 workers with default parameter values:
    > python arir.py 8

- Launch 12 workers with custom parameter values:
    > python arir.py 12 --shared-memory-size 128 --delay-connect 2.0 --delay-transmit 0.5 --delay-process 0.75
"""

__author__ = 'moorglade'

import multiprocessing
import time
import datetime
import sys
import argparse
import random
import math

def _parse_args():
    parser = argparse.ArgumentParser()

    # specify command line options
    parser.add_argument(
        'n_workers',
        help='number of workers in the distributed system',
        type=int
    )
    parser.add_argument(
        '--shared-memory-size',
        help='size of the shared memory array [number of ints]',
        type=int,
        default=16
    )
    parser.add_argument(
        '--delay-connect',
        help='network connection delay [s]',
        type=float,
        default=0.1
    )
    parser.add_argument(
        '--delay-transmit',
        help='network transmission delay [s]',
        type=float,
        default=0.1
    )
    parser.add_argument(
        '--delay-process',
        help='processing delay [s]',
        type=float,
        default=0.1
    )
    parser.add_argument(
        '--submatrix-size',
        help='single square sub-matrix size',
        type=int,
        default=3
    )
    parser.add_argument(
        '--should-count-square',
        help='should compute square root of matrix A or AxB multiplication',
        type=bool,
        default=True
    )

    return argparse.Namespace(**{
        key.replace('-', '_'): value
        for key, value in vars(parser.parse_args()).items()
        })

class DistributedSystem(object):
    def __init__(self, configuration):
        object.__init__(self)

        shared = SharedState(configuration.n_workers, configuration.shared_memory_size)
        network = Network(configuration)

        self.__workers = [
            Worker(worker_id, configuration, shared, network.get_endpoint(worker_id))
            for worker_id in range(configuration.n_workers)
            ]

    def run(self):
        print 'Launching {} workers...'.format(len(self.__workers))
        start = datetime.datetime.now()

        for worker in self.__workers:
            worker.start()

        print 'Waiting for the workers to terminate...'
        for worker in self.__workers:
            worker.join()

        stop = datetime.datetime.now()
        print 'All workers terminated.'

        print 'Processing took {} seconds.'.format((stop - start).total_seconds())

class SharedState(object):
    def __init__(self, n_workers, shared_memory_size):
        object.__init__(self)
        self.__barrier = Barrier(n_workers)
        self.__memory = multiprocessing.Array('i', shared_memory_size)

    @property
    def barrier(self):
        return self.__barrier

    @property
    def memory(self):
        return self.__memory

class Barrier(object):
    def __init__(self, n):
        object.__init__(self)
        self.__counter = multiprocessing.Value('i', 0, lock=False)
        self.__n = n
        self.__condition = multiprocessing.Condition()

    def wait(self):
        with self.__condition:
            self.__counter.value += 1

            if self.__counter.value == self.__n:
                self.__counter.value = 0
                self.__condition.notify_all()

            else:
                self.__condition.wait()

class SharedMemory(object):
    def __init__(self, shared_memory_size):
        object.__init__(self)
        self.__array = multiprocessing.Array('i', shared_memory_size)

class Network(object):
    any_id = -1

    def __init__(self, configuration):
        object.__init__(self)
        channels = [NetworkChannel(configuration) for _ in range(configuration.n_workers)]
        self.__endpoints = [NetworkEndpoint(channel_id, channels) for channel_id in range(configuration.n_workers)]

    def get_endpoint(self, index):
        return self.__endpoints[index]

class NetworkChannel(object):
    def __init__(self, configuration):
        self.__configuration = configuration

        self.__source_id = multiprocessing.Value('i', Network.any_id, lock=False)
        self.__queue = multiprocessing.Queue(maxsize=1)
        self.__enter_lock = multiprocessing.Lock()
        self.__exit_lock = multiprocessing.Lock()
        self.__enter_lock.acquire()
        self.__exit_lock.acquire()

    def send(self, source_id, data):
        while True:
            self.__enter_lock.acquire()

            if self.__source_id.value in [source_id, Network.any_id]:
                self.__source_id.value = source_id
                self.__queue.put(data)
                time.sleep(self.__configuration.delay_connect + len(data) * self.__configuration.delay_transmit)
                self.__exit_lock.release()
                break

            else:
                self.__enter_lock.release()

    def receive(self, source_id=Network.any_id):
        self.__source_id.value = source_id

        self.__enter_lock.release()
        data = self.__queue.get()
        self.__exit_lock.acquire()

        return self.__source_id.value, data

class NetworkEndpoint(object):
    def __init__(self, channel_id, channels):
        self.__channels = channels
        self.__my_id = channel_id
        self.__my_channel = self.__channels[self.__my_id]

    def send(self, destination_id, data):
        if destination_id == self.__my_id:
            raise RuntimeError('Worker {} tried to send data to itself.'.format(self.__my_id))

        self.__channels[destination_id].send(self.__my_id, data)

    def receive(self, worker_id=Network.any_id):
        return self.__my_channel.receive(worker_id)

class Matrix(object):
    def __init__(self, size, is_counting_square):
        self.dataA = [[random.randint(-2048, 2048) for x in range(size)] for x in range(size)]
        if is_counting_square:
            self.dataB = self.dataA
        else:
            self.dataB = [[random.randint(-2048, 2048) for x in range(size)] for x in range(size)]
        self.dataC = [[0 for x in range(size)] for x in range(size)]

    @staticmethod
    def get_data_of_shift(matrix, direction):
        if direction == 'left':
            return matrix[0]
        elif direction == 'right':
            return matrix[-1]
        elif direction == 'up':
            return list(zip(*matrix)[0])
        elif direction == 'down':
            return list(zip(*matrix)[-1])

        return None

    @staticmethod
    def multiply(matrix_a, matrix_b):
        return [[sum(a*b for a, b in zip(X_row, Y_col)) for Y_col in zip(*matrix_b)] for X_row in matrix_a]

class Worker(multiprocessing.Process):
    def __init__(self, worker_id, configuration, shared, network_endpoint):
        multiprocessing.Process.__init__(self)

        self.__worker_id = worker_id
        self.__configuration = configuration
        self.__shared = shared
        self.__network_endpoint = network_endpoint

        self.carts = Worker.get_carts_layout(configuration.n_workers)
        self.matrix = Matrix(configuration.submatrix_size, configuration.should_count_square)

    @property
    def _n_workers(self):
        return self.__configuration.n_workers

    @property
    def _sub_matrix_size(self):
        return self.__configuration.submatrix_size

    @staticmethod
    def _generate_random_data(length):
        return [random.randint(-2048, 2048) for _ in range(length)]

    def _barrier(self):
        self.__shared.barrier.wait()

    def _send(self, worker_id, data):
        self.__network_endpoint.send(worker_id, data)

    def _receive(self, worker_id=Network.any_id):
        return self.__network_endpoint.receive(worker_id)

    def _log(self, message):
        print '[WORKER {}] {}'.format(self.__worker_id, message)

    # My methods

    # Getting current cart id
    def get_cart_row_id(self):
        return self.get_my_cart_coordinates()[0][1]

    def get_cart_column_id(self):
        return self.get_my_cart_coordinates()[0][0]

    # Returns list of matches, so assuming there is only one, we need to take first element
    def get_my_cart_coordinates(self):
        return [(ind, self.carts[ind].index(self.__worker_id)) for ind in xrange(len(self.carts)) if self.__worker_id in self.carts[ind]]

    # Shifting of meshes between carts
    # Constitutive name - we shift columns, left
    def left_circular_shift_row(self):
        my_column_id = self.get_cart_column_id()
        last_column_id = int(math.sqrt(self._n_workers)) - 1

        if my_column_id == 0:
            # Last column
            my_send_partner = last_column_id
            my_receive_partner = my_column_id + 1
        elif my_column_id == last_column_id:
            # first column
            my_send_partner = my_column_id - 1
            my_receive_partner = 0
        else:
            my_send_partner = my_column_id - 1
            my_receive_partner = my_column_id + 1

        # Sending and receiving at once.. will it work ?!
        self._send(my_send_partner, self.matrix.dataA)
        received_data = self._receive(my_receive_partner)

        self.matrix.dataA = received_data

    # Shifting of meshes between carts
    # Constitutive name - we shift rows, up
    def upward_circular_shift_column(self):
        my_row_id = self.get_cart_row_id()
        last_row_id = int(math.sqrt(self._n_workers)) - 1

        if my_row_id == 0:
            # Last column
            my_send_partner = last_row_id
            my_receive_partner = my_row_id + 1
        elif my_row_id == last_row_id:
            # first column
            my_send_partner = my_row_id - 1
            my_receive_partner = 0
        else:
            my_send_partner = my_row_id - 1
            my_receive_partner = my_row_id + 1

        # Sending and receiving at once.. will it work ?!
        self._send(my_send_partner, self.matrix.dataB)
        received_data = self._receive(my_receive_partner)

        self.matrix.dataB = received_data

    @staticmethod
    def get_carts_layout(number_of_carts):
        carts = [[0 for y in range(int(math.sqrt(number_of_carts)))] for x in range(int(math.sqrt(number_of_carts)))]

        counter = 0
        for x in range(len(carts)):
            for y in range(len(carts)):
                carts[x][y] = counter
                counter += 1

        return carts

    def run(self):
        self._log('Started.')
        print(repr(self.matrix.dataA))
        self.left_circular_shift_row()
        print(repr(self.matrix.dataA))
        self._log('Terminated.')

def main():
    random.seed()
    configuration = _parse_args()
    system = DistributedSystem(configuration)
    system.run()

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

'Functions to set up and more easily use deform_bootstrap_extra with Pyramid.'

from __future__ import (absolute_import, division, print_function,
    unicode_literals)
from pkg_resources import resource_filename
import deform as d
from pyramid.httpexceptions import HTTPUnauthorized
from pyramid.i18n import get_localizer
from pyramid.threadlocal import get_current_request

from pyramid.i18n import TranslationStringFactory
_ = TranslationStringFactory('deform_bootstrap_extra')
del TranslationStringFactory

def translator(term):
    return get_localizer(get_current_request()).translate(term)

already_setup = False

def setup_for_pyramid(config, translator=translator, template_dirs=(
                      'deform_bootstrap_extra:templates',
                      'deform_bootstrap:templates',
                      'deform:templates')):
    '''Set deform up for i18n and give its template loader the correct
    directory hierarchy.

    This includes deform_bootstrap, so the app developer must not.
    '''
    global already_setup
    if already_setup:
        return
    from .. import monkeypatch_colander
    monkeypatch_colander()

    config.add_translation_dirs('colander:locale', 'deform:locale',)
    #    'deform_bootstrap_extra:locale')
    config.add_static_view('deform', 'deform:static')
    config.add_static_view('deform_bootstrap_extra',
        'deform_bootstrap_extra:static')
    config.include('deform_bootstrap')
    dirs = tuple([resource_filename(*dir.split(':'))
        for dir in template_dirs])
    d.Form.set_zpt_renderer(dirs, translator=translator)
    already_setup = True

def button(title=_('Submit'), name=None, icon=None):
    '''Conveniently generate a Deform button while setting its
    ``name`` attribute, translating the label and capitalizing it.

    The button may also have a bootstrap icon.
    '''
    b = d.Button(title=translator(title).capitalize(),
        name=name or title.lower())
    b.icon = icon
    return b

def verify_csrf(func):
    '''Decorator that checks the CSRF token. TODO: Use and test.'''
    def wrapper(*a, **kw):
        request = get_current_request()
        if request.params.get('_csrf_') == request.session.get_csrf_token():
            return func(*a, **kw)
        else:
            raise HTTPUnauthorized('You do not pass our CSRF protection.')
    return wrapper
<commit_msg>Avoid resource_filename() which doesn't work in appengine<commit_after># -*- coding: utf-8 -*-

'Functions to set up and more easily use deform_bootstrap_extra with Pyramid.'

from __future__ import (absolute_import, division, print_function,
    unicode_literals)
import deform as d
# from pkg_resources import resource_filename  # does not work in appengine 177
from pyramid.asset import abspath_from_asset_spec
from pyramid.httpexceptions import HTTPUnauthorized
from pyramid.i18n import get_localizer
from pyramid.threadlocal import get_current_request

from pyramid.i18n import TranslationStringFactory
_ = TranslationStringFactory('deform_bootstrap_extra')
del TranslationStringFactory

def translator(term):
    return get_localizer(get_current_request()).translate(term)

already_setup = False

def setup_for_pyramid(config, translator=translator, template_dirs=(
                      'deform_bootstrap_extra:templates',
                      'deform_bootstrap:templates',
                      'deform:templates')):
    '''Set deform up for i18n and give its template loader the correct
    directory hierarchy.

    This includes deform_bootstrap, so the app developer must not.
    '''
    global already_setup
    if already_setup:
        return
    from .. import monkeypatch_colander
    monkeypatch_colander()

    config.add_translation_dirs('colander:locale', 'deform:locale',)
    #    'deform_bootstrap_extra:locale')
    config.add_static_view('deform', 'deform:static')
    config.add_static_view('deform_bootstrap_extra',
        'deform_bootstrap_extra:static')
    config.include('deform_bootstrap')
    # dirs = tuple([resource_filename(*dir.split(':'))
    #     for dir in template_dirs])
    dirs = tuple([abspath_from_asset_spec(dir) for dir in template_dirs])
    d.Form.set_zpt_renderer(dirs, translator=translator)
    already_setup = True

def button(title=_('Submit'), name=None, icon=None):
    '''Conveniently generate a Deform button while setting its
    ``name`` attribute, translating the label and capitalizing it.

    The button may also have a bootstrap icon.
    '''
    b = d.Button(title=translator(title).capitalize(),
        name=name or title.lower())
    b.icon = icon
    return b

def verify_csrf(func):
    '''Decorator that checks the CSRF token. TODO: Use and test.'''
    def wrapper(*a, **kw):
        request = get_current_request()
        if request.params.get('_csrf_') == request.session.get_csrf_token():
            return func(*a, **kw)
        else:
            raise HTTPUnauthorized('You do not pass our CSRF protection.')
    return wrapper
<|endoftext|>


<commit_before># Copyright 2007 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.

DON'T USE THIS MODULE DIRECTLY!  The classes here should be imported
via collections; they are defined here only to alleviate certain
bootstrapping issues.  Unit tests are in test_collections.
"""

from abc import ABCMeta, abstractmethod

__all__ = ["Hashable", "Iterable", "Iterator",
           "Sized", "Container", "Callable",
           "Set", "MutableSet",
           "Mapping", "MutableMapping",
           "MappingView", "KeysView", "ItemsView", "ValuesView",
           "Sequence", "MutableSequence",
           ]

### ONE-TRICK PONIES ###

class Hashable:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __hash__(self):
        return 0

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Hashable:
            for B in C.__mro__:
                if "__hash__" in B.__dict__:
                    if B.__dict__["__hash__"]:
                        return True
                    break
        return NotImplemented

class Iterable:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __iter__(self):
        while False:
            yield None

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Iterable:
            if any("__iter__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

Iterable.register(str)

class Iterator:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __next__(self):
        raise StopIteration

    def __iter__(self):
        return self

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Iterator:
            if any("next" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

class Sized:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __len__(self):
        return 0

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Sized:
            if any("__len__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

class Container:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __contains__(self, x):
        return False

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Container:
            if any("__contains__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

class Callable:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __contains__(self, x):
        return False

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Callable:
            if any("__call__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

### SETS ###

class Set:
    __metaclass__ = ABCMeta

    """A set is a finite, iterable container.

    This class provides concrete generic implementations of all
    methods except for __contains__, __iter__ and __len__.

    To override the comparisons (presumably for speed, as the
    semantics are fixed), all you have to do is redefine __le__ and
    then the other operations will automatically follow suit.
    """

    @abstractmethod
    def __contains__(self, value):
        return False

    @abstractmethod
    def __iter__(self):
        while False:
            yield None

    @abstractmethod
    def __len__(self):
        return 0

    def __le__(self, other):
        if not isinstance(other, Set):
            return NotImplemented
        if len(self) > len(other):
            return False
        for elem in self:
            if elem not in other:
                return False
        return True

    def __lt__(self, other):
        if not isinstance(other, Set):
            return NotImplemented
        return len(self) < len(other) and self.__le__(other)

    def __eq__(self, other):
        if not isinstance(other, Set):
            return NotImplemented
        return len(self) == len(other) and self.__le__(other)

    @classmethod
    def _from_iterable(cls, it):
        return frozenset(it)

    def __and__(self, other):
        if not isinstance(other, Iterable):
            return NotImplemented
        return self._from_iterable(value for value in other if value in self)

    def isdisjoint(self, other):
        for value in other:
            if value in self:
                return False
        return True

    def __or__(self, other):
        if not isinstance(other, Iterable):
            return NotImplemented
        return self._from_iterable(itertools.chain(self, other))

    def __sub__(self, other):
        if not isinstance(other, Set):
            if not isinstance(other, Iterable):
                return NotImplemented
            other = self._from_iterable(other)
        return self._from_iterable(value for value in self
                                   if value not in other)

    def __xor__(self, other):
        if not isinstance(other, Set):
            if not isinstance(other, Iterable):
                return NotImplemented
            other = self._from_iterable(other)
        return (self - other) | (other - self)

    def _hash(self):
        """Compute the hash value of a set.

        Note that we don't define __hash__: not all sets are hashable.
        But if you define a hashable set type, its __hash__ should
        call this function.

        This must be compatible __eq__.

        All sets ought to compare equal if they contain the same
        elements, regardless of how they are implemented, and
        regardless of the order of the elements; so there's not much
        freedom for __eq__ or __hash__.  We match the algorithm used
        by the built-in frozenset type.
        """
        MAX = sys.maxint
        MASK = 2 * MAX + 1
        n = len(self)
        h = 1927868237 * (n + 1)
        h &= MASK
        for x in self:
            hx = hash(x)
            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
            h &= MASK
        h = h * 69069 + 907133923
        h &= MASK
        if h > MAX:
            h -= MASK + 1
        if h == -1:
            h = 590923713
        return h

Set.register(frozenset)

class MutableSet(Set):

    @abstractmethod
    def add(self, value):
        """Return True if it was added, False if already there."""
        raise NotImplementedError

    @abstractmethod
    def discard(self, value):
        """Return True if it was deleted, False if not there."""
        raise NotImplementedError

    def remove(self, value):
        """Remove an element. If not a member, raise a KeyError."""
        if value not in self:
            raise KeyError(value)
        self.discard(value)

    def pop(self):
        """Return the popped value.  Raise KeyError if empty."""
        it = iter(self)
        try:
            value = it.__next__()
        except StopIteration:
            raise KeyError
        self.discard(value)
        return value

    def toggle(self, value):
        """Return True if it was added, False if deleted."""
        # XXX This implementation is not thread-safe
        if value in self:
            self.discard(value)
            return False
        else:
            self.add(value)
            return True

    def clear(self):
        """This is slow (creates N new iterators!) but effective."""
        try:
            while True:
                self.pop()
        except KeyError:
            pass

    def __ior__(self, it):
        for value in it:
            self.add(value)
        return self

    def __iand__(self, c):
        for value in self:
            if value not in c:
                self.discard(value)
        return self

    def __ixor__(self, it):
        # This calls toggle(), so if that is overridded, we call the override
        for value in it:
            self.toggle(it)
        return self

    def __isub__(self, it):
        for value in it:
            self.discard(value)
        return self

MutableSet.register(set)

### MAPPINGS ###

class Mapping:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __getitem__(self, key):
        raise KeyError

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    @abstractmethod
    def __len__(self):
        return 0

    @abstractmethod
    def __iter__(self):
        while False:
            yield None

    def keys(self):
        return KeysView(self)

    def items(self):
        return ItemsView(self)

    def values(self):
        return ValuesView(self)

class MappingView:
    __metaclass__ = ABCMeta

    def __init__(self, mapping):
        self._mapping = mapping

    def __len__(self):
        return len(self._mapping)

class KeysView(MappingView, Set):

    def __contains__(self, key):
        return key in self._mapping

    def __iter__(self):
        for key in self._mapping:
            yield key

KeysView.register(type({}.keys()))

class ItemsView(MappingView, Set):

    def __contains__(self, item):
        key, value = item
        try:
            v = self._mapping[key]
        except KeyError:
            return False
        else:
            return v == value

    def __iter__(self):
        for key in self._mapping:
            yield (key, self._mapping[key])

ItemsView.register(type({}.items()))

class ValuesView(MappingView):

    def __contains__(self, value):
        for key in self._mapping:
            if value == self._mapping[key]:
                return True
        return False

    def __iter__(self):
        for key in self._mapping:
            yield self._mapping[key]

ValuesView.register(type({}.values()))

class MutableMapping(Mapping):

    @abstractmethod
    def __setitem__(self, key, value):
        raise KeyError

    @abstractmethod
    def __delitem__(self, key):
        raise KeyError

    __marker = object()

    def pop(self, key, default=__marker):
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def popitem(self):
        try:
            key = next(iter(self))
        except StopIteration:
            raise KeyError
        value = self[key]
        del self[key]
        return key, value

    def clear(self):
        try:
            while True:
                self.popitem()
        except KeyError:
            pass

    def update(self, other=(), **kwds):
        if isinstance(other, Mapping):
            for key in other:
                self[key] = other[key]
        elif hasattr(other, "keys"):
            for key in other.keys():
                self[key] = other[key]
        else:
            for key, value in other:
                self[key] = value
        for key, value in kwds.items():
            self[key] = value

MutableMapping.register(dict)

### SEQUENCES ###

class Sequence:
    __metaclass__ = ABCMeta

    """All the operations on a read-only sequence.

    Concrete subclasses must override __new__ or __init__,
    __getitem__, and __len__.
    """

    @abstractmethod
    def __getitem__(self, index):
        raise IndexError

    @abstractmethod
    def __len__(self):
        return 0

    def __iter__(self):
        i = 0
        while True:
            try:
                v = self[i]
            except IndexError:
                break
            yield v
            i += 1

    def __contains__(self, value):
        for v in self:
            if v == value:
                return True
        return False

    def __reversed__(self):
        for i in reversed(range(len(self))):
            yield self[i]

    def index(self, value):
        for i, v in enumerate(self):
            if v == value:
                return i
        raise ValueError

    def count(self, value):
        return sum(1 for v in self if v == value)

Sequence.register(tuple)
Sequence.register(basestring)
Sequence.register(buffer)

class MutableSequence(Sequence):

    @abstractmethod
    def __setitem__(self, index, value):
        raise IndexError

    @abstractmethod
    def __delitem__(self, index):
        raise IndexError

    @abstractmethod
    def insert(self, index, value):
        raise IndexError

    def append(self, value):
        self.insert(len(self), value)

    def reverse(self):
        n = len(self)
        for i in range(n//2):
            self[i], self[n-i-1] = self[n-i-1], self[i]

    def extend(self, values):
        for v in values:
            self.append(v)

    def pop(self, index=-1):
        v = self[index]
        del self[index]
        return v

    def remove(self, value):
        del self[self.index(value)]

    def __iadd__(self, values):
        self.extend(values)

MutableSequence.register(list)
<commit_msg>Fix defect in __ixor__ which would get the wrong answer if the input iterable had a duplicate element (two calls to toggle() reverse each other).  Borrow the correct code from sets.py.<commit_after># Copyright 2007 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.

DON'T USE THIS MODULE DIRECTLY!  The classes here should be imported
via collections; they are defined here only to alleviate certain
bootstrapping issues.  Unit tests are in test_collections.
"""

from abc import ABCMeta, abstractmethod

__all__ = ["Hashable", "Iterable", "Iterator",
           "Sized", "Container", "Callable",
           "Set", "MutableSet",
           "Mapping", "MutableMapping",
           "MappingView", "KeysView", "ItemsView", "ValuesView",
           "Sequence", "MutableSequence",
           ]

### ONE-TRICK PONIES ###

class Hashable:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __hash__(self):
        return 0

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Hashable:
            for B in C.__mro__:
                if "__hash__" in B.__dict__:
                    if B.__dict__["__hash__"]:
                        return True
                    break
        return NotImplemented

class Iterable:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __iter__(self):
        while False:
            yield None

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Iterable:
            if any("__iter__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

Iterable.register(str)

class Iterator:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __next__(self):
        raise StopIteration

    def __iter__(self):
        return self

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Iterator:
            if any("next" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

class Sized:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __len__(self):
        return 0

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Sized:
            if any("__len__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

class Container:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __contains__(self, x):
        return False

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Container:
            if any("__contains__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

class Callable:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __contains__(self, x):
        return False

    @classmethod
    def __subclasshook__(cls, C):
        if cls is Callable:
            if any("__call__" in B.__dict__ for B in C.__mro__):
                return True
        return NotImplemented

### SETS ###

class Set:
    __metaclass__ = ABCMeta

    """A set is a finite, iterable container.

    This class provides concrete generic implementations of all
    methods except for __contains__, __iter__ and __len__.

    To override the comparisons (presumably for speed, as the
    semantics are fixed), all you have to do is redefine __le__ and
    then the other operations will automatically follow suit.
    """

    @abstractmethod
    def __contains__(self, value):
        return False

    @abstractmethod
    def __iter__(self):
        while False:
            yield None

    @abstractmethod
    def __len__(self):
        return 0

    def __le__(self, other):
        if not isinstance(other, Set):
            return NotImplemented
        if len(self) > len(other):
            return False
        for elem in self:
            if elem not in other:
                return False
        return True

    def __lt__(self, other):
        if not isinstance(other, Set):
            return NotImplemented
        return len(self) < len(other) and self.__le__(other)

    def __eq__(self, other):
        if not isinstance(other, Set):
            return NotImplemented
        return len(self) == len(other) and self.__le__(other)

    @classmethod
    def _from_iterable(cls, it):
        return frozenset(it)

    def __and__(self, other):
        if not isinstance(other, Iterable):
            return NotImplemented
        return self._from_iterable(value for value in other if value in self)

    def isdisjoint(self, other):
        for value in other:
            if value in self:
                return False
        return True

    def __or__(self, other):
        if not isinstance(other, Iterable):
            return NotImplemented
        return self._from_iterable(itertools.chain(self, other))

    def __sub__(self, other):
        if not isinstance(other, Set):
            if not isinstance(other, Iterable):
                return NotImplemented
            other = self._from_iterable(other)
        return self._from_iterable(value for value in self
                                   if value not in other)

    def __xor__(self, other):
        if not isinstance(other, Set):
            if not isinstance(other, Iterable):
                return NotImplemented
            other = self._from_iterable(other)
        return (self - other) | (other - self)

    def _hash(self):
        """Compute the hash value of a set.

        Note that we don't define __hash__: not all sets are hashable.
        But if you define a hashable set type, its __hash__ should
        call this function.

        This must be compatible __eq__.

        All sets ought to compare equal if they contain the same
        elements, regardless of how they are implemented, and
        regardless of the order of the elements; so there's not much
        freedom for __eq__ or __hash__.  We match the algorithm used
        by the built-in frozenset type.
        """
        MAX = sys.maxint
        MASK = 2 * MAX + 1
        n = len(self)
        h = 1927868237 * (n + 1)
        h &= MASK
        for x in self:
            hx = hash(x)
            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
            h &= MASK
        h = h * 69069 + 907133923
        h &= MASK
        if h > MAX:
            h -= MASK + 1
        if h == -1:
            h = 590923713
        return h

Set.register(frozenset)

class MutableSet(Set):

    @abstractmethod
    def add(self, value):
        """Return True if it was added, False if already there."""
        raise NotImplementedError

    @abstractmethod
    def discard(self, value):
        """Return True if it was deleted, False if not there."""
        raise NotImplementedError

    def remove(self, value):
        """Remove an element. If not a member, raise a KeyError."""
        if value not in self:
            raise KeyError(value)
        self.discard(value)

    def pop(self):
        """Return the popped value.  Raise KeyError if empty."""
        it = iter(self)
        try:
            value = it.__next__()
        except StopIteration:
            raise KeyError
        self.discard(value)
        return value

    def clear(self):
        """This is slow (creates N new iterators!) but effective."""
        try:
            while True:
                self.pop()
        except KeyError:
            pass

    def __ior__(self, it):
        for value in it:
            self.add(value)
        return self

    def __iand__(self, c):
        for value in self:
            if value not in c:
                self.discard(value)
        return self

    def __ixor__(self, it):
        if not isinstance(it, Set):
            it = self._from_iterable(it)
        for value in it:
            if value in self:
                self.discard(value)
            else:
                self.add(value)

    def __isub__(self, it):
        for value in it:
            self.discard(value)
        return self

MutableSet.register(set)

### MAPPINGS ###

class Mapping:
    __metaclass__ = ABCMeta

    @abstractmethod
    def __getitem__(self, key):
        raise KeyError

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __contains__(self, key):
        try:
            self[key]
        except KeyError:
            return False
        else:
            return True

    @abstractmethod
    def __len__(self):
        return 0

    @abstractmethod
    def __iter__(self):
        while False:
            yield None

    def keys(self):
        return KeysView(self)

    def items(self):
        return ItemsView(self)

    def values(self):
        return ValuesView(self)

class MappingView:
    __metaclass__ = ABCMeta

    def __init__(self, mapping):
        self._mapping = mapping

    def __len__(self):
        return len(self._mapping)

class KeysView(MappingView, Set):

    def __contains__(self, key):
        return key in self._mapping

    def __iter__(self):
        for key in self._mapping:
            yield key

KeysView.register(type({}.keys()))

class ItemsView(MappingView, Set):

    def __contains__(self, item):
        key, value = item
        try:
            v = self._mapping[key]
        except KeyError:
            return False
        else:
            return v == value

    def __iter__(self):
        for key in self._mapping:
            yield (key, self._mapping[key])

ItemsView.register(type({}.items()))

class ValuesView(MappingView):

    def __contains__(self, value):
        for key in self._mapping:
            if value == self._mapping[key]:
                return True
        return False

    def __iter__(self):
        for key in self._mapping:
            yield self._mapping[key]

ValuesView.register(type({}.values()))

class MutableMapping(Mapping):

    @abstractmethod
    def __setitem__(self, key, value):
        raise KeyError

    @abstractmethod
    def __delitem__(self, key):
        raise KeyError

    __marker = object()

    def pop(self, key, default=__marker):
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def popitem(self):
        try:
            key = next(iter(self))
        except StopIteration:
            raise KeyError
        value = self[key]
        del self[key]
        return key, value

    def clear(self):
        try:
            while True:
                self.popitem()
        except KeyError:
            pass

    def update(self, other=(), **kwds):
        if isinstance(other, Mapping):
            for key in other:
                self[key] = other[key]
        elif hasattr(other, "keys"):
            for key in other.keys():
                self[key] = other[key]
        else:
            for key, value in other:
                self[key] = value
        for key, value in kwds.items():
            self[key] = value

MutableMapping.register(dict)

### SEQUENCES ###

class Sequence:
    __metaclass__ = ABCMeta

    """All the operations on a read-only sequence.

    Concrete subclasses must override __new__ or __init__,
    __getitem__, and __len__.
    """

    @abstractmethod
    def __getitem__(self, index):
        raise IndexError

    @abstractmethod
    def __len__(self):
        return 0

    def __iter__(self):
        i = 0
        while True:
            try:
                v = self[i]
            except IndexError:
                break
            yield v
            i += 1

    def __contains__(self, value):
        for v in self:
            if v == value:
                return True
        return False

    def __reversed__(self):
        for i in reversed(range(len(self))):
            yield self[i]

    def index(self, value):
        for i, v in enumerate(self):
            if v == value:
                return i
        raise ValueError

    def count(self, value):
        return sum(1 for v in self if v == value)

Sequence.register(tuple)
Sequence.register(basestring)
Sequence.register(buffer)

class MutableSequence(Sequence):

    @abstractmethod
    def __setitem__(self, index, value):
        raise IndexError

    @abstractmethod
    def __delitem__(self, index):
        raise IndexError

    @abstractmethod
    def insert(self, index, value):
        raise IndexError

    def append(self, value):
        self.insert(len(self), value)

    def reverse(self):
        n = len(self)
        for i in range(n//2):
            self[i], self[n-i-1] = self[n-i-1], self[i]

    def extend(self, values):
        for v in values:
            self.append(v)

    def pop(self, index=-1):
        v = self[index]
        del self[index]
        return v

    def remove(self, value):
        del self[self.index(value)]

    def __iadd__(self, values):
        self.extend(values)

MutableSequence.register(list)
<|endoftext|>