<commit_before># -*- coding: utf-8 -*-

"""BlinkyTape Python communication library.

  This code assumes stock serialLoop() in the firmware.

  Commands are issued in 3-byte blocks, with pixel data
  encoded in RGB triplets in range 0-254, sent sequentially
  and a triplet ending with a 255 causes the accumulated pixel
  data to display (a show command).

  Note that with the stock firmware changing the maximum brightness
  over serial communication is impossible.
"""

import serial
import os.path
import subprocess
import re
from time import sleep

# For Python3 support- always run strings through a bytes converter
import sys
if sys.version_info < (3,):
    def encode(x):
        return x
else:
    import codecs
    def encode(x):
        return codecs.latin_1_encode(x)[0]

class BlinkyTape(object):
    min_colour_val = 0
    max_colour_val = 254

    def __init__(self, port=None, ledCount=60, buffered=True):
        """Creates a BlinkyTape object and opens the port.

        Parameters:
          port
            Required, port name as accepted by PySerial library:
            http://pyserial.sourceforge.net/pyserial_api.html#serial.Serial
            It is the same port name that is used in Arduino IDE.
            Ex.: COM5 (Windows), /dev/ttyACM0 (Linux).
          ledCount
            Optional, total number of LEDs to work with,
            defaults to 60 LEDs. The limit is enforced and an
            attempt to send more pixel data will throw an exception.
          buffered
            Optional, enabled by default. If enabled, will buffer
            pixel data until a show command is issued. If disabled,
            the data will be sent in byte triplets as expected by firmware,
            with immediate flush of the serial buffers (slower).

        """
        self.port = self.__identify_port(port)
        print "port=%s" % self.port
        self.ledCount = ledCount
        self.position = 0
        self.buffered = buffered
        self.buf = ""
        self.serial = serial.Serial(self.port, 115200)
        self.show()  # Flush any incomplete data

    def __identify_port(self, port=None):
        if port is not None:
            return port
        elif os.name == 'posix': # covers all *nix machines, including Linux?
            if not os.path.exists( '/dev/serial') or not os.path.exists('/dev/serial/by-id'):
                raise RuntimeError("Assuming this is a *nix machine, cannot find the device path for BlinkyTape via 'ls -l /dev/serial/by-id'. Is it connected?")
            ls_l_output = subprocess.check_output(['ls', '-l', '/dev/serial/by-id'])
            m = re.search( 'BlinkyTape\S+\s+->\s+\S+(/tty\S+)', ls_l_output)
            if m == None:
                raise RuntimeError("Assuming this is a *nix machine, cannot find the device path for BlinkyTape via 'ls -l /dev/serial/by-id'. Is it connected?")
            device_path = '/dev' + m.group(1)
            return device_path
        elif os.name == 'nt': # covers all windows machines?
            cmd = "wmic path Win32_SerialPort Where 'Caption like \"%" + "BlinkyTape%\"' Get DeviceID" # Clumsy, but can't work out how else to avoid %B going weird
            com_name_output = subprocess.check_output(cmd)
            m = re.search( '(COM\d+)', com_name_output)
            if m == None:
                raise RuntimeError("Assuming this is a Windows machine, cannot find a matching COM port for BlinkyTape. Is it connected?")
            com_name = m.group(1)
            return com_name
        else:
            raise RuntimeError("Cannot establish what kind OS is running, so don't know where to look for the BlinkyTape connection.")

    def send_list(self, colors):
        if len(colors) > self.ledCount:
            raise RuntimeError("Attempting to set pixel outside range!")
        for r, g, b in colors:
            self.sendPixel(r, g, b)
        self.show()

    def colour_chr(self, n):
        if n < self.min_colour_val:
            n = self.min_colour_val
        elif n > self.max_colour_val:
            n = self.max_colour_val
        return chr(n)

    def send_list(self, colors):
        data = ""
        for r, g, b in colors:
            data += self.colour_chr(r) + self.colour_chr(g) + self.colour_chr(b)
        self.serial.write(encode(data))
        self.show()

    def sendPixel(self, r, g, b):
        """Sends the next pixel data triplet in RGB format.

        Values are clamped to 0-254 automatically.

        Throws a RuntimeException if [ledCount] pixels are already set.
        """
        data = self.colour_chr(r) + self.colour_chr(g) + self.colour_chr(b)
        if self.position < self.ledCount:
            if self.buffered:
                self.buf += data
            else:
                self.serial.write(encode(data))
                self.serial.flush()
            self.position += 1
        else:
            raise RuntimeError("Attempting to set pixel outside range!")

    def show(self):
        """Sends the command(s) to display all accumulated pixel data.

        Resets the next pixel position to 0, flushes the serial buffer,
        and discards any accumulated responses from BlinkyTape.
        """
        control = chr(255)
        if self.buffered:
            # Fix an OS X specific bug where sending more than 383 bytes of data at once
            # hangs the BlinkyTape controller. Why this is???
            # TODO: Test me on other platforms
            CHUNK_SIZE = 300

            self.buf += control
            for i in range(0, len(self.buf), CHUNK_SIZE):
                self.serial.write(encode(self.buf[i:i+CHUNK_SIZE]))
                self.serial.flush()

            self.buf = ""
        else:
            self.serial.write(encode(control))
        self.serial.flush()
        self.serial.flushInput()  # Clear responses from BlinkyTape, if any
        self.position = 0

    def displayColor(self, r, g, b):
        """Fills [ledCount] pixels with RGB color and shows it."""
        for i in range(0, self.ledCount):
            self.sendPixel(r, g, b)
        self.show()

    def resetToBootloader(self):
        """Initiates a reset on BlinkyTape.

        Note that it will be disconnected.
        """
        self.serial.setBaudrate(1200)
        self.close()

    def close(self):
        """Safely closes the serial port."""
        self.serial.close()

    def clear_all(self):
        for x in range(self.ledCount):
            self.sendPixel(0, 0, 0)
        self.show()

# Example code

if __name__ == "__main__":

    import glob
    import optparse

    parser = optparse.OptionParser()
    parser.add_option("-p", "--port", dest="portname",
                      help="serial port (ex: /dev/ttyUSB0)", default=None)
    parser.add_option("-c", "--ledcount", dest="ledcount",
                      help="number of LEDs attached", type="int", default=60)
    parser.add_option("-b", action="store_true", default="True", dest="buffered")
    parser.add_option("-u", action="store_false", dest="buffered")

    (options, args) = parser.parse_args()

    bt = BlinkyTape(options.portname, options.ledcount, options.buffered)

    while True:
        bt.displayColor(255, 0, 0)
        sleep(0.5)
        bt.displayColor(0, 255, 0)
        sleep(0.5)
        bt.displayColor(0, 0, 255)
        sleep(0.5)
        bt.displayColor(255, 255, 255)
        sleep(0.5)
        bt.displayColor(0, 0, 0)
        sleep(0.5)
<commit_msg>tweaked port detection for windows 10<commit_after># -*- coding: utf-8 -*-

"""BlinkyTape Python communication library.

  This code assumes stock serialLoop() in the firmware.

  Commands are issued in 3-byte blocks, with pixel data
  encoded in RGB triplets in range 0-254, sent sequentially
  and a triplet ending with a 255 causes the accumulated pixel
  data to display (a show command).

  Note that with the stock firmware changing the maximum brightness
  over serial communication is impossible.
"""

import serial
import os.path
import subprocess
import re
from time import sleep

# For Python3 support- always run strings through a bytes converter
import sys
if sys.version_info < (3,):
    def encode(x):
        return x
else:
    import codecs
    def encode(x):
        return codecs.latin_1_encode(x)[0]

class BlinkyTape(object):
    min_colour_val = 0
    max_colour_val = 254

    def __init__(self, port=None, ledCount=60, buffered=True):
        """Creates a BlinkyTape object and opens the port.

        Parameters:
          port
            Required, port name as accepted by PySerial library:
            http://pyserial.sourceforge.net/pyserial_api.html#serial.Serial
            It is the same port name that is used in Arduino IDE.
            Ex.: COM5 (Windows), /dev/ttyACM0 (Linux).
          ledCount
            Optional, total number of LEDs to work with,
            defaults to 60 LEDs. The limit is enforced and an
            attempt to send more pixel data will throw an exception.
          buffered
            Optional, enabled by default. If enabled, will buffer
            pixel data until a show command is issued. If disabled,
            the data will be sent in byte triplets as expected by firmware,
            with immediate flush of the serial buffers (slower).

        """
        self.port = self.__identify_port(port)
        print "port=%s" % self.port
        self.ledCount = ledCount
        self.position = 0
        self.buffered = buffered
        self.buf = ""
        self.serial = serial.Serial(self.port, 115200)
        self.show()  # Flush any incomplete data

    def __identify_port(self, port=None):
        if port is not None:
            return port
        elif os.name == 'posix': # covers all *nix machines, including Linux?
            if not os.path.exists( '/dev/serial') or not os.path.exists('/dev/serial/by-id'):
                raise RuntimeError("Assuming this is a *nix machine, cannot find the device path for BlinkyTape via 'ls -l /dev/serial/by-id'. Is it connected?")
            ls_l_output = subprocess.check_output(['ls', '-l', '/dev/serial/by-id'])
            m = re.search( 'BlinkyTape\S+\s+->\s+\S+(/tty\S+)', ls_l_output)
            if m == None:
                raise RuntimeError("Assuming this is a *nix machine, cannot find the device path for BlinkyTape via 'ls -l /dev/serial/by-id'. Is it connected?")
            device_path = '/dev' + m.group(1)
            return device_path
        elif os.name == 'nt': # covers all windows machines?
            cmd = "wmic path Win32_SerialPort Where 'Caption like \"%" + "BlinkyTape%\"' Get DeviceID" # Clumsy, but can't work out how else to avoid %B going weird
            com_name_output = subprocess.check_output(cmd)
            m = re.search( '(COM\d+)', com_name_output)
            if m == None: # but in Windows 10, the name BlinkyTape does not seem to appear, so just look for any device at all. Assuming this might fail if you have more than one device attached. Sorry. I hate the wmic thing.
                cmd2 = "wmic path Win32_SerialPort Get DeviceID"
                com_name_output2 = subprocess.check_output(cmd2)
                m = re.search( '(COM\d+)', com_name_output2)
            if m == None:
                raise RuntimeError("Assuming this is a Windows machine, cannot find a matching COM port for BlinkyTape. Is it connected?")
            com_name = m.group(1)
            return com_name
        else:
            raise RuntimeError("Cannot establish what kind OS is running, so don't know where to look for the BlinkyTape connection.")

    def send_list(self, colors):
        if len(colors) > self.ledCount:
            raise RuntimeError("Attempting to set pixel outside range!")
        for r, g, b in colors:
            self.sendPixel(r, g, b)
        self.show()

    def colour_chr(self, n):
        if n < self.min_colour_val:
            n = self.min_colour_val
        elif n > self.max_colour_val:
            n = self.max_colour_val
        return chr(n)

    def send_list(self, colors):
        data = ""
        for r, g, b in colors:
            data += self.colour_chr(r) + self.colour_chr(g) + self.colour_chr(b)
        self.serial.write(encode(data))
        self.show()

    def sendPixel(self, r, g, b):
        """Sends the next pixel data triplet in RGB format.

        Values are clamped to 0-254 automatically.

        Throws a RuntimeException if [ledCount] pixels are already set.
        """
        data = self.colour_chr(r) + self.colour_chr(g) + self.colour_chr(b)
        if self.position < self.ledCount:
            if self.buffered:
                self.buf += data
            else:
                self.serial.write(encode(data))
                self.serial.flush()
            self.position += 1
        else:
            raise RuntimeError("Attempting to set pixel outside range!")

    def show(self):
        """Sends the command(s) to display all accumulated pixel data.

        Resets the next pixel position to 0, flushes the serial buffer,
        and discards any accumulated responses from BlinkyTape.
        """
        control = chr(255)
        if self.buffered:
            # Fix an OS X specific bug where sending more than 383 bytes of data at once
            # hangs the BlinkyTape controller. Why this is???
            # TODO: Test me on other platforms
            CHUNK_SIZE = 300

            self.buf += control
            for i in range(0, len(self.buf), CHUNK_SIZE):
                self.serial.write(encode(self.buf[i:i+CHUNK_SIZE]))
                self.serial.flush()

            self.buf = ""
        else:
            self.serial.write(encode(control))
        self.serial.flush()
        self.serial.flushInput()  # Clear responses from BlinkyTape, if any
        self.position = 0

    def displayColor(self, r, g, b):
        """Fills [ledCount] pixels with RGB color and shows it."""
        for i in range(0, self.ledCount):
            self.sendPixel(r, g, b)
        self.show()

    def resetToBootloader(self):
        """Initiates a reset on BlinkyTape.

        Note that it will be disconnected.
        """
        self.serial.setBaudrate(1200)
        self.close()

    def close(self):
        """Safely closes the serial port."""
        self.serial.close()

    def clear_all(self):
        for x in range(self.ledCount):
            self.sendPixel(0, 0, 0)
        self.show()

# Example code

if __name__ == "__main__":

    import glob
    import optparse

    parser = optparse.OptionParser()
    parser.add_option("-p", "--port", dest="portname",
                      help="serial port (ex: /dev/ttyUSB0)", default=None)
    parser.add_option("-c", "--ledcount", dest="ledcount",
                      help="number of LEDs attached", type="int", default=60)
    parser.add_option("-b", action="store_true", default="True", dest="buffered")
    parser.add_option("-u", action="store_false", dest="buffered")

    (options, args) = parser.parse_args()

    bt = BlinkyTape(options.portname, options.ledcount, options.buffered)

    while True:
        bt.displayColor(255, 0, 0)
        sleep(0.5)
        bt.displayColor(0, 255, 0)
        sleep(0.5)
        bt.displayColor(0, 0, 255)
        sleep(0.5)
        bt.displayColor(255, 255, 255)
        sleep(0.5)
        bt.displayColor(0, 0, 0)
        sleep(0.5)
<|endoftext|>


<commit_before>from pymongo import MongoClient
import datetime

def prune_extents(removed, collection):
    for exnode in removed:
        collection.remove({"parent": exnode})

def prune_exnodes(collection):
    removed = []

    for exnode in collection.find({"mode": "file"}):
        do_remove = True
        for extent in exnode["extents"]:
            try:
                if datetime.datetime.strptime(extent["lifetimes"][0]["end"], "%Y-%m-%d %H:%M:%S") >= datetime.datetime.utcnow():
                    do_remove = False
                    break

            except Exception as exp:
                print("Bad extent {extent}: Removing".format(extent = extent["id"]))

        if do_remove:
            print("removing: Exnode[{uid}]".format(uid = exnode["id"]))
            removed.append(exnode["id"])

    remove_cmd = {"id": {"$in": removed}}
    collection.remove(remove_cmd)
    return removed

def prune_directories(collection):
    root = collection.find({"parent": None})

    for exnode in root:
        children = search_children(exnode, collection)

def search_children(exnode, collection):
    if exnode["mode"] == "file":
        return True

    contains_data = False
    children = collection.find({ "parent": exnode["id"] })

    for child in children:
        contains_data = contains_data | search_children(child, collection)

    if not contains_data:
        collection.remove({"id": exnode["id"]})
        print "Removing Directory: {0}".format(exnode["name"])

    return contains_data

def main():
    client = MongoClient()
    db = client["unis_db"]
    exnodes = db.exnodes
    extents = db.extents

    removed = prune_exnodes(exnodes)
    prune_extents(removed, extents)
    prune_directories(exnodes)

if __name__ == "__main__":
    main()
<commit_msg>Updating prune script so it actually prunes exnodes without embedded extents.<commit_after>from pymongo import MongoClient
import datetime

def prune_extents(removed, collection):
    for exnode in removed["refs"]:
        collection.remove({"parent.href": exnode})

def prune_exnodes(collection, extcoll):
    removed = {"refs": [],
               "ids": []}

    for exnode in collection.find({"mode": "file"}):
        do_remove = True

        if "extents" not in exnode:
            exnode["extents"] = []
            extents = extcoll.find({"parent.href": exnode["selfRef"]})
            for e in extents:
                exnode["extents"].append(e)

        if not len(exnode["extents"]):
            print("Skipping exnode with no extents! [{uid} {name}]".format(uid=exnode["id"], name=exnode["name"]))
            continue

        for extent in exnode["extents"]:
            try:
                if datetime.datetime.strptime(extent["lifetimes"][0]["end"], "%Y-%m-%d %H:%M:%S") >= datetime.datetime.utcnow():
                    do_remove = False
                    break

            except Exception as exp:
                print("Bad extent {extent}: Removing".format(extent = extent["id"]))

        if do_remove:
            print("removing: Exnode[{uid} {name}]".format(uid=exnode["id"], name=exnode["name"]))
            removed["refs"].append(exnode["selfRef"])
            removed["ids"].append(exnode["id"])

    remove_cmd = {"id": {"$in": removed["ids"]}}
    collection.remove(remove_cmd)
    return removed

def prune_directories(collection):
    root = collection.find({"parent": None})

    for exnode in root:
        children = search_children(exnode, collection)

def search_children(exnode, collection):
    if exnode["mode"] == "file":
        return True

    contains_data = False
    children = collection.find({ "parent.href": exnode["selfRef"] })

    for child in children:
        contains_data = contains_data | search_children(child, collection)

    if not contains_data:
        collection.remove({"id": exnode["id"]})
        print "Removing Directory: {0}".format(exnode["name"])

    return contains_data

def main():
    client = MongoClient()
    db = client["exnode_db"]
    exnodes = db.exnodes
    extents = db.extents

    removed = prune_exnodes(exnodes, extents)
    prune_extents(removed, extents)
    prune_directories(exnodes)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Identify and anonymize IP addresses."""

import ipaddress
import logging
import regex

from hashlib import md5
from six import u

class tree_node():
    """Simple binary tree with a value, left node, and right node.

    This is used for holding a translation from original to anonymized IP
    addresses.
    """

    def __init__(self, value):
        """Initialize new node."""
        self.left = None
        self.right = None
        self.value = value

    def dump_to_file(self, file_out, depth=0, input_addr=0, output_addr=0):
        """Recursively traverse tree and write translations to output file."""
        # Root node value does not contribute to output_addr, so only update
        # output_addr for nodes after root (depth > 0)
        if depth > 0:
            output_addr = (output_addr << 1) + self.value

        # Only dump nodes at max depth (32) i.e. full 32bit anonymization
        if depth == 32:
            org_ip_str = str(ipaddress.IPv4Address(input_addr))
            new_ip_str = str(ipaddress.IPv4Address(output_addr))
            logging.debug('dumped {}\t{}'.format(org_ip_str, new_ip_str))
            file_out.write('{}\t{}\n'.format(org_ip_str, new_ip_str))
            return

        depth += 1
        if self.left is not None:
            left_path = (input_addr << 1)
            self.left.dump_to_file(file_out, depth, left_path, output_addr)
        if self.right is not None:
            right_path = (input_addr << 1) + 1
            self.right.dump_to_file(file_out, depth, right_path, output_addr)

    def preserve_ipv4_class(self):
        """Initialize tree to preserve IPv4 classes (call only on root node)."""
        node = self
        # IP classes are defined by the number of leading 1's in the address up
        # to the fourth 1, so setup the tree to preserve those
        for i in range(0, 4):
            node.left = tree_node(0)
            node.right = tree_node(1)
            node = node.right

def anonymize_ip_addr(my_ip_tree, line, salt):
    """Replace each IP address in the line with an anonymized IP address.

    Quad-octets that look like masks will be left unchanged.  That is, any
    quad-octet that consists solely of an initial group of 1s followed by 0s
    or initial 0s followed by 1s will be unchanged.
    """
    pattern = '((\d{1,3})\.\d{1,3}\.\d{1,3}\.\d{1,3})(?=/(\d{1,3}))?'
    matches = regex.findall(pattern, line)
    if matches is None:
        return line

    # Escape existing curly braces since string.format will be used to insert
    # anonymized IP addresses
    new_line = line.replace('{', '{{')
    new_line = new_line.replace('}', '}}')
    new_line = regex.sub(pattern, '{}', new_line)

    ip_addrs = []
    for match in matches:
        ip_str = match[0]
        first_octet = match[1]
        ip_int = _ip_to_int(ip_str)
        if _is_mask(ip_int):
            logging.debug("Skipping mask {}".format(ip_str))
            ip_addrs.append(ip_str)
        elif int(first_octet) >= 224:
            # TODO: consider just removing this and anonymizing (if preserving
            # class), but skipping anything in IP class D and class E for now
            logging.debug("Skipping addresses reserved for multicast and R&D {}"
                          .format(ip_str))
            ip_addrs.append(ip_str)
        else:
            new_ip = _convert_to_anon_ip(my_ip_tree, ip_int, salt)
            new_ip_str = str(ipaddress.IPv4Address(new_ip))
            ip_addrs.append(new_ip_str)
            logging.debug("Replaced {} with {}".format(ip_str, new_ip_str))

    return new_line.format(*ip_addrs)

def _convert_to_anon_ip(node, ip_int, salt):
    """Anonymize an IP address using an existing IP tree root node.

    The bits of a given source IP address define a branch in the binary tree,
    where each source bit selects an edge (1=right, 0=left) from the previous
    node and the value at the next node is the anonymized bit.  This process
    is repeated until all prefix bits are exhausted.  The values at each node
    are pseudorandomly generated as needed and are the inverse of their sibling.
    """
    new_ip_int = 0

    for i in range(31, -1, -1):
        # msb is the next bit to anonymize
        msb = (ip_int >> i) & 1
        if node.left is None:
            # Use a hash as a deterministic pseudorandom bit generator here
            # so address anonymization is the same regardless of order/context
            preceding_bits = '{:032b}'.format(ip_int)[:31 - i]
            last_hash_digit = md5((salt + preceding_bits).encode()).hexdigest()[-1]
            last_hash_bit = int(last_hash_digit, 16) & 1

            # Go ahead and populate both left and right nodes, sacrificing
            # space to simplify control flow
            node.left = tree_node(last_hash_bit)
            node.right = tree_node(1 - last_hash_bit)
        if msb:
            node = node.right
        else:
            node = node.left
        new_ip_int |= node.value << i
    return new_ip_int

def _ip_to_int(ip_str):
    """Convert an IP address string to integer representation."""
    # Need to strip leading zeros so ipaddress does not assume octal notation
    ip_str = regex.sub('0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)',
                       r'\1.\2.\3.\4', ip_str)
    ip_int = int(ipaddress.IPv4Address(u(ip_str)))
    return int(ip_int)

def _is_mask(possible_mask_int):
    """Determine if the input int is a mask or not.

    If the binary representation starts with all 1s and ends with all 0s
    (or starts with 0s and ends with 1s), then we assume it is a mask.

    Counting the number of times consecutive bits do not match (transitions)
    gives us a reasonable idea of whether or not something is a mask.  With 0
    or 1 transitions, assume the value is a mask.
    e.g. 1100 has only one place where consecutive bits don't match
         0000 has zero
         0110 has two
         0101 has three
    """
    prev_bit = possible_mask_int & 1
    delta_count = 0
    for pos in range(1, 32):
        cur_bit = (possible_mask_int >> pos) & 1
        delta_count += (prev_bit ^ cur_bit)
        prev_bit = cur_bit
    return (delta_count < 2)
<commit_msg>comment<commit_after>"""Identify and anonymize IP addresses."""

import ipaddress
import logging
import regex

from hashlib import md5
from six import u

class tree_node():
    """Simple binary tree with a value, left node, and right node.

    This is used for holding a translation from original to anonymized IP
    addresses.
    """

    def __init__(self, value):
        """Initialize new node."""
        self.left = None
        self.right = None
        self.value = value

    def dump_to_file(self, file_out, depth=0, input_addr=0, output_addr=0):
        """Recursively traverse tree and write translations to output file."""
        # Root node value does not contribute to output_addr, so only update
        # output_addr for nodes after root (depth > 0)
        if depth > 0:
            output_addr = (output_addr << 1) + self.value

        # Only dump nodes at max depth (32) i.e. full 32bit anonymization
        if depth == 32:
            org_ip_str = str(ipaddress.IPv4Address(input_addr))
            new_ip_str = str(ipaddress.IPv4Address(output_addr))
            logging.debug('dumped {}\t{}'.format(org_ip_str, new_ip_str))
            file_out.write('{}\t{}\n'.format(org_ip_str, new_ip_str))
            return

        depth += 1
        if self.left is not None:
            left_path = (input_addr << 1)
            self.left.dump_to_file(file_out, depth, left_path, output_addr)
        if self.right is not None:
            right_path = (input_addr << 1) + 1
            self.right.dump_to_file(file_out, depth, right_path, output_addr)

    def preserve_ipv4_class(self):
        """Initialize tree to preserve IPv4 classes (call only on root node)."""
        node = self
        # IP classes are defined by the number of leading 1's in the address up
        # to the fourth 1, so setup the tree to preserve those
        for i in range(0, 4):
            node.left = tree_node(0)
            node.right = tree_node(1)
            node = node.right

def anonymize_ip_addr(my_ip_tree, line, salt):
    """Replace each IP address in the line with an anonymized IP address.

    Quad-octets that look like masks will be left unchanged.  That is, any
    quad-octet that consists solely of an initial group of 1s followed by 0s
    or initial 0s followed by 1s will be unchanged.
    """
    pattern = '((\d{1,3})\.\d{1,3}\.\d{1,3}\.\d{1,3})(?=/(\d{1,3}))?'
    matches = regex.findall(pattern, line)
    if matches is None:
        return line

    # Escape existing curly braces since string.format will be used to insert
    # anonymized IP addresses
    new_line = line.replace('{', '{{')
    new_line = new_line.replace('}', '}}')
    new_line = regex.sub(pattern, '{}', new_line)

    ip_addrs = []
    for match in matches:
        ip_str = match[0]
        first_octet = match[1]
        ip_int = _ip_to_int(ip_str)
        if _is_mask(ip_int):
            logging.debug("Skipping mask {}".format(ip_str))
            ip_addrs.append(ip_str)
        elif int(first_octet) >= 224:
            # TODO: consider just removing this and anonymizing (if preserving
            # class), but skipping anything in IP class D and class E for now
            logging.debug("Skipping addresses reserved for multicast and R&D {}"
                          .format(ip_str))
            ip_addrs.append(ip_str)
        else:
            new_ip = _convert_to_anon_ip(my_ip_tree, ip_int, salt)
            new_ip_str = str(ipaddress.IPv4Address(new_ip))
            ip_addrs.append(new_ip_str)
            logging.debug("Replaced {} with {}".format(ip_str, new_ip_str))

    return new_line.format(*ip_addrs)

def _convert_to_anon_ip(node, ip_int, salt):
    """Anonymize an IP address using an existing IP tree root node.

    The bits of a given source IP address define a branch in the binary tree,
    where each source bit selects an edge (1=right, 0=left) from the previous
    node and the value at the next node is the anonymized bit.  This process
    is repeated until all prefix bits are exhausted.  The values at each node
    are pseudorandomly generated as needed and are the inverse of their sibling.
    """
    new_ip_int = 0

    for i in range(31, -1, -1):
        # msb is the next bit to anonymize
        msb = (ip_int >> i) & 1
        if node.left is None:
            # Use a hash as a deterministic pseudorandom bit generator here
            # so address anonymization is the same regardless of order/context

            # This is a string of all source bits preceding the one to be anonymized
            preceding_bits = '{:032b}'.format(ip_int)[:31 - i]
            last_hash_digit = md5((salt + preceding_bits).encode()).hexdigest()[-1]
            last_hash_bit = int(last_hash_digit, 16) & 1

            # Go ahead and populate both left and right nodes, sacrificing
            # space to simplify control flow
            node.left = tree_node(last_hash_bit)
            node.right = tree_node(1 - last_hash_bit)
        if msb:
            node = node.right
        else:
            node = node.left
        new_ip_int |= node.value << i
    return new_ip_int

def _ip_to_int(ip_str):
    """Convert an IP address string to integer representation."""
    # Need to strip leading zeros so ipaddress does not assume octal notation
    ip_str = regex.sub('0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)',
                       r'\1.\2.\3.\4', ip_str)
    ip_int = int(ipaddress.IPv4Address(u(ip_str)))
    return int(ip_int)

def _is_mask(possible_mask_int):
    """Determine if the input int is a mask or not.

    If the binary representation starts with all 1s and ends with all 0s
    (or starts with 0s and ends with 1s), then we assume it is a mask.

    Counting the number of times consecutive bits do not match (transitions)
    gives us a reasonable idea of whether or not something is a mask.  With 0
    or 1 transitions, assume the value is a mask.
    e.g. 1100 has only one place where consecutive bits don't match
         0000 has zero
         0110 has two
         0101 has three
    """
    prev_bit = possible_mask_int & 1
    delta_count = 0
    for pos in range(1, 32):
        cur_bit = (possible_mask_int >> pos) & 1
        delta_count += (prev_bit ^ cur_bit)
        prev_bit = cur_bit
    return (delta_count < 2)
<|endoftext|>


<commit_before>import string

class InvertedIndex:
    def __init__(self, common_words, documents=None):
        self.__index = {}  # a dict mapping words to TermInfo
        self.__stats = {}  # a dict mapping doc IDs to DocStats
        self.common_words = common_words
        if documents is not None:
            for doc in documents:
                self.add_document(doc)

    def add_document(self, document):
        """
        Parameters:
            document : a document (must respond to `doc_id` and `get_full_text`).

            Document IDs must be unique in the index.
        Raises:
            ValueError: when trying to insert a document whose ID already exists in the index
        """
        stats = document_word_count(document, self.common_words)
        if document.doc_id in self.__stats:
            raise ValueError("Document ID {0} is already present in this index".format(document.doc_id))
        for word, count in stats.frequency.items():
            assert(count > 0)
            if word in self.__index:
                self.__index[word].add_document(document.doc_id, count)
            else:
                self.__index[word] = TermInfo()
        self.__stats[document.doc_id] = stats

    @property
    def words(self):
        """
        Returns:
            list of str: the words in this index dictionnary (e.g. the indexed keywords)
        """
        return self.__index.keys()

    @property
    def doc_ids(self):
        """
        Returns:
            list of int: a list of the IDs of the documents indexed in this index
        """
        return self.__stats.keys()

    def get_weights(self, doc_id):
        """
        Parameter:
            doc_id (int): a document ID. This document must be present in the index.
        Returns:
            A dict mapping the words in the given document to their weights.
        """
        return self.__stats[doc_id].weights

class TermInfo:
    def __init__(self):
        self.docs = {} # doc_id, freq

    def add_document(self, doc_id, count):
        if doc_id in self.docs:
            self.docs[doc_id] += count
        else:
            self.docs[doc_id] = count

class DocStats:
    """Various statistics about a document.
    """

    def __init__(self, frequency, nb_words):
        """
        Parameters:
            frequency (dict): a dictionnary mapping words to their number of occurences
            nb_words (int): the number of words in the document.
        """
        self.frequency = frequency
        self.nb_words = nb_words

    @property
    def weights(self):
        """
        Returns:
            a dictionnary mapping each word in the document to a weight
        """
        out = {}
        for word in self.frequency.keys():
            out[word] = self.frequency[word] / self.nb_words
        return out

def document_word_count(document, common_words):
    """
    Parameters:
        document (Document)
        common_words (list of str)
    Returns:
        A dictionnary:
        - keys: words inside the `document`, except the `common_words`
        - values: number of occurences of the word in the `document`
    Example:
        >>> document = CacmDocument(2382,"aujourd \n toto \n tata","il fait \n tyty","toto \n tata tyty")
        >>> common_words= ["aujourd", "il", "fait", "ca", "mot", "ok"]
        >>> document_statistics(document,common_words)
        { "toto":2, "tata":2, "tyty":2 }
    """
    word_list = get_word_list(document)
    return count_occurences(remove_stop_words(word_list, common_words))

def count_occurences(word_list):
    """
    Parameter:
        word_list (list of str)
    Returns:
        A dictionnary:
        - keys : each word in the `word_list`
        - values : number of occurences of the word in the `word_list`
    Example:
        >>> statistics(["toto", "tata", "tyty", "toto", "toto", "tata"]) == DocStats({"toto":3,"tata":2,"tyty":1},9)
        True
    """
    out = {}
    for word in word_list:
        if word not in out.keys():
            out[word] = 1
        else:
            out[word] = out[word] + 1
    return DocStats(out, len(word_list))

def _punctuation_to_space(character):
    """
    Parameter:
        character (char)
    Returns:
        char: a space if `character` was a punctuation character, `character` otherwise
    Example:
        >>> _punctuation_to_space("-") == " "
        True
    """
    separators = string.punctuation
    if character in separators:
        return " "
    return character

def tokenize(line):
    """
    Parameter:
        line (str)
    Returns:
        a list a words without ponctuation or spaces
    Example:
        >>> tokenize("aujourd'hui (lundi) il fait beau -")
        ["aujourd", "hui", "lundi", "il", "fait", "beau"]
    """
    cleaned = ''.join([_punctuation_to_space(c) for c in list(line)])
    tokens = [token for token in cleaned.split(" ") if token not in string.whitespace]
    return tokens
def normalize(term):
    """
    Parameter:
        term (str)
    Returns:
        string: `term` converted to lowercase
    Example:
        >>> normalize("tEsT")
        test
    """
    return term.lower()

def remove_stop_words(word_list, common_words):
    """
    Parameters:
        word_list (list of str)
        common_words (list of str)
    Returns:
        list of str: all the words in `word_list` that do not belong to `common_words`
    Example:
        remove_stop_words([tata", "titi"], ["titi"]) returns ["tata"]
    """
    return [normalize(word) for word in word_list if word not in common_words]

def get_word_list(document):
    """
    Parameter:
        document (CacmDocument)
    Returns:
        list of str: all the words inside a document, without the punctation
    Example:
        >>> get_word_list(document) returns ["aujourd", "toto", "tata","il", "fait", "tyty","toto","tata","tyty"] == CacmDocument(2382,"aujourd \n toto \n tata","il fait \n tyty","toto \n tata tyty")
        True
    """
    word_list = []
    for line in document.get_full_text().split("\n"):
        word_list = word_list + tokenize(line)
    return word_list

def document_size(document):
    """
    Parameter:
        document (Document)
    Returns:
        int: the number of words in the document
    Example:
        >>> document_size(CacmDocument(2382,"aujourd \n toto \n tata","il fait \n tyty","toto \n tata tyty"))
        9
    """
    size = 0
    for word in get_word_list(document):
        size = size + 1
    return size
<commit_msg>Fix ignoring the first time a word is added in the index.<commit_after>import string

class InvertedIndex:
    def __init__(self, common_words, documents=None):
        self.__index = {}  # a dict mapping words to TermInfo
        self.__stats = {}  # a dict mapping doc IDs to DocStats
        self.common_words = common_words
        if documents is not None:
            for doc in documents:
                self.add_document(doc)

    def add_document(self, document):
        """
        Parameters:
            document : a document (must respond to `doc_id` and `get_full_text`).

            Document IDs must be unique in the index.
        Raises:
            ValueError: when trying to insert a document whose ID already exists in the index
        """
        stats = document_word_count(document, self.common_words)
        if document.doc_id in self.__stats:
            raise ValueError("Document ID {0} is already present in this index".format(document.doc_id))
        for word, count in stats.frequency.items():
            assert(count > 0)
            if word not in self.__index:
                self.__index[word] = TermInfo()
            self.__index[word].add_document(document.doc_id, count)
        self.__stats[document.doc_id] = stats

    @property
    def words(self):
        """
        Returns:
            list of str: the words in this index dictionnary (e.g. the indexed keywords)
        """
        return self.__index.keys()

    @property
    def doc_ids(self):
        """
        Returns:
            list of int: a list of the IDs of the documents indexed in this index
        """
        return self.__stats.keys()

    def get_weights(self, doc_id):
        """
        Parameter:
            doc_id (int): a document ID. This document must be present in the index.
        Returns:
            A dict mapping the words in the given document to their weights.
        """
        return self.__stats[doc_id].weights

class TermInfo:
    def __init__(self):
        self.docs = {} # doc_id, freq

    def add_document(self, doc_id, count):
        if doc_id in self.docs:
            self.docs[doc_id] += count
        else:
            self.docs[doc_id] = count

class DocStats:
    """Various statistics about a document.
    """

    def __init__(self, frequency, nb_words):
        """
        Parameters:
            frequency (dict): a dictionnary mapping words to their number of occurences
            nb_words (int): the number of words in the document.
        """
        self.frequency = frequency
        self.nb_words = nb_words

    @property
    def weights(self):
        """
        Returns:
            a dictionnary mapping each word in the document to a weight
        """
        out = {}
        for word in self.frequency.keys():
            out[word] = self.frequency[word] / self.nb_words
        return out

def document_word_count(document, common_words):
    """
    Parameters:
        document (Document)
        common_words (list of str)
    Returns:
        A dictionnary:
        - keys: words inside the `document`, except the `common_words`
        - values: number of occurences of the word in the `document`
    Example:
        >>> document = CacmDocument(2382,"aujourd \n toto \n tata","il fait \n tyty","toto \n tata tyty")
        >>> common_words= ["aujourd", "il", "fait", "ca", "mot", "ok"]
        >>> document_statistics(document,common_words)
        { "toto":2, "tata":2, "tyty":2 }
    """
    word_list = get_word_list(document)
    return count_occurences(remove_stop_words(word_list, common_words))

def count_occurences(word_list):
    """
    Parameter:
        word_list (list of str)
    Returns:
        A dictionnary:
        - keys : each word in the `word_list`
        - values : number of occurences of the word in the `word_list`
    Example:
        >>> statistics(["toto", "tata", "tyty", "toto", "toto", "tata"]) == DocStats({"toto":3,"tata":2,"tyty":1},9)
        True
    """
    out = {}
    for word in word_list:
        if word not in out.keys():
            out[word] = 1
        else:
            out[word] = out[word] + 1
    return DocStats(out, len(word_list))

def _punctuation_to_space(character):
    """
    Parameter:
        character (char)
    Returns:
        char: a space if `character` was a punctuation character, `character` otherwise
    Example:
        >>> _punctuation_to_space("-") == " "
        True
    """
    separators = string.punctuation
    if character in separators:
        return " "
    return character

def tokenize(line):
    """
    Parameter:
        line (str)
    Returns:
        a list a words without ponctuation or spaces
    Example:
        >>> tokenize("aujourd'hui (lundi) il fait beau -")
        ["aujourd", "hui", "lundi", "il", "fait", "beau"]
    """
    cleaned = ''.join([_punctuation_to_space(c) for c in list(line)])
    tokens = [token for token in cleaned.split(" ") if token not in string.whitespace]
    return tokens
def normalize(term):
    """
    Parameter:
        term (str)
    Returns:
        string: `term` converted to lowercase
    Example:
        >>> normalize("tEsT")
        test
    """
    return term.lower()

def remove_stop_words(word_list, common_words):
    """
    Parameters:
        word_list (list of str)
        common_words (list of str)
    Returns:
        list of str: all the words in `word_list` that do not belong to `common_words`
    Example:
        remove_stop_words([tata", "titi"], ["titi"]) returns ["tata"]
    """
    return [normalize(word) for word in word_list if word not in common_words]

def get_word_list(document):
    """
    Parameter:
        document (CacmDocument)
    Returns:
        list of str: all the words inside a document, without the punctation
    Example:
        >>> get_word_list(document) returns ["aujourd", "toto", "tata","il", "fait", "tyty","toto","tata","tyty"] == CacmDocument(2382,"aujourd \n toto \n tata","il fait \n tyty","toto \n tata tyty")
        True
    """
    word_list = []
    for line in document.get_full_text().split("\n"):
        word_list = word_list + tokenize(line)
    return word_list

def document_size(document):
    """
    Parameter:
        document (Document)
    Returns:
        int: the number of words in the document
    Example:
        >>> document_size(CacmDocument(2382,"aujourd \n toto \n tata","il fait \n tyty","toto \n tata tyty"))
        9
    """
    size = 0
    for word in get_word_list(document):
        size = size + 1
    return size
<|endoftext|>


<commit_before>#
# Copyright (c) 2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

import os

from conary.deps import deps
from conary.repository import changeset
from conary.repository import filecontents
from conary import files
from conary import trove
from conary import versions

def makePathId():
    """returns 16 random bytes, for use as a pathId"""
    return os.urandom(16)

class ClientNewTrove:
    def _createTroves(self, troveAndPathList):
        cs = changeset.ChangeSet()
        troveList = [ x[0] for x in troveAndPathList ]
        previousVersionMap = self._targetNewTroves(troveList)
        self._addAllNewFiles(cs, troveAndPathList, previousVersionMap)
        for trove in troveList:
            trove.computeDigests()
            trvCs = trove.diff(None, absolute=True)[0]
            cs.newTrove(trvCs)
        return cs

    def createSourceTrove(self, name, label, upstreamVersion, pathDict,
                          changeLog, factory = None):
        """
            Create a source trove.
            @param name: trove name of source components.
            @type name: str
            @param label: trove label
            @type label: str
            @param upstreamVersion: upstream version of source component
            @type upstreamVersion: str
            @param pathDict: Dictionary mapping path strings to
            conaryclient.filetypes._File objects, which represent the contents
            of each file
            @type pathDict: dict(str: conaryclient.filetypes._File)
            @param changeLog: Change log associated with this source trove.
            @type changeLog: changelog.ChangeLog
            @param factory: designate a factory associated with this source
            trove.
            @type factory: str
        """
        if not name.endswith(':source'):
            raise RuntimeError('Only source components allowed')
        versionStr = '/%s/%s-1' % (label, upstreamVersion)
        version = versions.VersionFromString(versionStr).copy()
        version.resetTimeStamps()
        troveObj = trove.Trove(name, version, deps.Flavor(),
                               changeLog = changeLog)
        troveObj.setFactory(factory)
        return self._createTroves([(troveObj, pathDict)])

    def _targetNewTroves(self, troveList):
        repos = self.getRepos()
        previousVersionMap = {}
        versionDict = {}
        troveSpecs = {}
        trovesSeen = set()
        for troveObj in troveList:
            name, version, flavor = troveObj.getNameVersionFlavor()
            if not name.endswith(':source'):
                raise RuntimeError('Only source components allowed')
            versionSpec = '%s/%s' % (version.trailingLabel(),
                                     version.trailingRevision().getVersion())
            if (name, versionSpec) in trovesSeen:
                raise RuntimeError('Cannot create multiple versions of %s with same version' % name)

            trovesSeen.add((name, versionSpec))
            troveSpecs[name, version, None] = troveObj

        results = repos.findTroves(None, troveSpecs, None, allowMissing=True)
        for troveSpec, troveObj in troveSpecs.iteritems():
            tupList = results.get(troveSpec, [])
            if tupList:
                assert(len(tupList) == 1)
                latestVersion = tupList[0][1]
                newVersion = latestVersion.copy()
                newVersion.incrementSourceCount()
                troveObj.changeVersion(newVersion)
                previousVersionMap[troveObj.getNameVersionFlavor()] = tupList[0]
            else:
                version = troveObj.getVersion()
                branch = version.branch()
                upstreamVer = version.trailingRevision().getVersion()
                revision = versions.Revision('%s-1' % upstreamVer)
                newVersion = branch.createVersion(revision)
                troveObj.changeVersion(newVersion)
        return previousVersionMap

    def _addAllNewFiles(self, cs, troveAndPathList, previousVersionMap):
        repos = self.getRepos()
        existingTroves = repos.getTroves(previousVersionMap.values(),
                                         withFiles=True)
        troveDict = dict(zip(previousVersionMap.values(), existingTroves))
        for trove, pathDict in troveAndPathList:
            existingTroveTup = previousVersionMap.get(
                                        trove.getNameVersionFlavor(), None)
            if existingTroveTup:
                existingTrove = troveDict[existingTroveTup]
            else:
                existingTrove = None
            self._addNewFiles(cs, trove, pathDict, existingTrove)

    def _removeOldPathIds(self, troveObj):
        allPathIds = [x[0] for x in  troveObj.iterFileList()]
        for pathId in allPathIds:
            troveObj.removePath(pathId)

    def _addNewFiles(self, cs, trove, pathDict, existingTrove):
        existingPaths = {}
        pathIds = {}
        if existingTrove:
            for pathId, path, fileId, fileVer in existingTrove.iterFileList():
                existingPaths[path] = (fileId, pathId, fileVer)
        self._removeOldPathIds(trove)

        for path, fileObj in pathDict.iteritems():
            if path in existingPaths:
                oldFileId, pathId, oldFileVer = existingPaths[path]
            else:
                pathId = makePathId()
                oldFileId = oldFileVer = None
            f = fileObj.get(pathId)
            f.flags.isSource(set = True)
            newFileId = f.fileId()
            if oldFileId == newFileId:
                newFileVer = oldFileVer
            else:
                newFileVer = trove.getVersion()
                cs.addFile(None, newFileId, f.freeze())

                contentType = changeset.ChangedFileTypes.file
                contents = hasattr(fileObj, 'contents') and fileObj.contents
                if contents:
                    cs.addFileContents(pathId, newFileId, contentType,
                            contents, cfgFile=False)

            trove.addFile(pathId, path, newFileVer, newFileId)

    def getFilesFromTrove(self, name, version, flavor, fileList=None, trv=None):
        repos = self.getRepos()
        if trv is None:
            trv = repos.getTrove(name, version, flavor, withFiles=True)
        results = {}

        filesToGet = []
        paths = []
        if fileList is None:
            fileObs = repos.getFileVersions([(x[0], x[2], x[3]) \
                    for x in trv.iterFileList()])
            for idx, (pathId, path, fileId, fileVer) in \
                    enumerate(trv.iterFileList()):
                fileObj = fileObs[idx]
                if fileObj.hasContents:
                    filesToGet.append((fileId, fileVer))
                    paths.append((path))
        else:
            for pathId, path, fileId, fileVer in trv.iterFileList():
                if path in fileList:
                    filesToGet.append((fileId, fileVer))
                    paths.append((path))
        fileContents = repos.getFileContents(filesToGet)
        for (contents, path) in zip(fileContents, paths):
            results[path] = contents.get()
        return results

<commit_msg>fix source count bug (CNY-2498)<commit_after>#
# Copyright (c) 2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

import os

from conary.deps import deps
from conary.repository import changeset
from conary.repository import filecontents
from conary import files
from conary import trove
from conary import versions

def makePathId():
    """returns 16 random bytes, for use as a pathId"""
    return os.urandom(16)

class ClientNewTrove:
    def _createTroves(self, troveAndPathList):
        cs = changeset.ChangeSet()
        troveList = [ x[0] for x in troveAndPathList ]
        previousVersionMap = self._targetNewTroves(troveList)
        self._addAllNewFiles(cs, troveAndPathList, previousVersionMap)
        for trove in troveList:
            trove.computeDigests()
            trvCs = trove.diff(None, absolute=True)[0]
            cs.newTrove(trvCs)
        return cs

    def createSourceTrove(self, name, label, upstreamVersion, pathDict,
                          changeLog, factory = None):
        """
            Create a source trove.
            @param name: trove name of source components.
            @type name: str
            @param label: trove label
            @type label: str
            @param upstreamVersion: upstream version of source component
            @type upstreamVersion: str
            @param pathDict: Dictionary mapping path strings to
            conaryclient.filetypes._File objects, which represent the contents
            of each file
            @type pathDict: dict(str: conaryclient.filetypes._File)
            @param changeLog: Change log associated with this source trove.
            @type changeLog: changelog.ChangeLog
            @param factory: designate a factory associated with this source
            trove.
            @type factory: str
        """
        if not name.endswith(':source'):
            raise RuntimeError('Only source components allowed')
        versionStr = '/%s/%s-1' % (label, upstreamVersion)
        version = versions.VersionFromString(versionStr).copy()
        version.resetTimeStamps()
        troveObj = trove.Trove(name, version, deps.Flavor(),
                               changeLog = changeLog)
        troveObj.setFactory(factory)
        return self._createTroves([(troveObj, pathDict)])

    def _targetNewTroves(self, troveList):
        repos = self.getRepos()
        previousVersionMap = {}
        versionDict = {}
        troveSpecs = {}
        trovesSeen = set()
        for troveObj in troveList:
            name, version, flavor = troveObj.getNameVersionFlavor()
            if not name.endswith(':source'):
                raise RuntimeError('Only source components allowed')
            versionSpec = '%s/%s' % (version.trailingLabel(),
                                     version.trailingRevision().getVersion())
            if (name, versionSpec) in trovesSeen:
                raise RuntimeError('Cannot create multiple versions of %s with same version' % name)

            trovesSeen.add((name, versionSpec))
            troveSpecs[name, versionSpec, None] = troveObj

        results = repos.findTroves(None, troveSpecs, None, allowMissing=True)
        for troveSpec, troveObj in troveSpecs.iteritems():
            tupList = results.get(troveSpec, [])
            if tupList:
                assert(len(tupList) == 1)
                latestVersion = tupList[0][1]
                newVersion = latestVersion.copy()
                newVersion.incrementSourceCount()
                troveObj.changeVersion(newVersion)
                previousVersionMap[troveObj.getNameVersionFlavor()] = tupList[0]
            else:
                version = troveObj.getVersion()
                branch = version.branch()
                upstreamVer = version.trailingRevision().getVersion()
                revision = versions.Revision('%s-1' % upstreamVer)
                newVersion = branch.createVersion(revision)
                troveObj.changeVersion(newVersion)
        return previousVersionMap

    def _addAllNewFiles(self, cs, troveAndPathList, previousVersionMap):
        repos = self.getRepos()
        existingTroves = repos.getTroves(previousVersionMap.values(),
                                         withFiles=True)
        troveDict = dict(zip(previousVersionMap.values(), existingTroves))
        for trove, pathDict in troveAndPathList:
            existingTroveTup = previousVersionMap.get(
                                        trove.getNameVersionFlavor(), None)
            if existingTroveTup:
                existingTrove = troveDict[existingTroveTup]
            else:
                existingTrove = None
            self._addNewFiles(cs, trove, pathDict, existingTrove)

    def _removeOldPathIds(self, troveObj):
        allPathIds = [x[0] for x in  troveObj.iterFileList()]
        for pathId in allPathIds:
            troveObj.removePath(pathId)

    def _addNewFiles(self, cs, trove, pathDict, existingTrove):
        existingPaths = {}
        pathIds = {}
        if existingTrove:
            for pathId, path, fileId, fileVer in existingTrove.iterFileList():
                existingPaths[path] = (fileId, pathId, fileVer)
        self._removeOldPathIds(trove)

        for path, fileObj in pathDict.iteritems():
            if path in existingPaths:
                oldFileId, pathId, oldFileVer = existingPaths[path]
            else:
                pathId = makePathId()
                oldFileId = oldFileVer = None
            f = fileObj.get(pathId)
            f.flags.isSource(set = True)
            newFileId = f.fileId()
            if oldFileId == newFileId:
                newFileVer = oldFileVer
            else:
                newFileVer = trove.getVersion()
                cs.addFile(None, newFileId, f.freeze())

                contentType = changeset.ChangedFileTypes.file
                contents = hasattr(fileObj, 'contents') and fileObj.contents
                if contents:
                    cs.addFileContents(pathId, newFileId, contentType,
                            contents, cfgFile=False)

            trove.addFile(pathId, path, newFileVer, newFileId)

    def getFilesFromTrove(self, name, version, flavor, fileList=None, trv=None):
        repos = self.getRepos()
        if trv is None:
            trv = repos.getTrove(name, version, flavor, withFiles=True)
        results = {}

        filesToGet = []
        paths = []
        if fileList is None:
            fileObs = repos.getFileVersions([(x[0], x[2], x[3]) \
                    for x in trv.iterFileList()])
            for idx, (pathId, path, fileId, fileVer) in \
                    enumerate(trv.iterFileList()):
                fileObj = fileObs[idx]
                if fileObj.hasContents:
                    filesToGet.append((fileId, fileVer))
                    paths.append((path))
        else:
            for pathId, path, fileId, fileVer in trv.iterFileList():
                if path in fileList:
                    filesToGet.append((fileId, fileVer))
                    paths.append((path))
        fileContents = repos.getFileContents(filesToGet)
        for (contents, path) in zip(fileContents, paths):
            results[path] = contents.get()
        return results

<|endoftext|>


<commit_before>import struct
from asyncio import IncompleteReadError

from nacl.secret import SecretBox

from .util import split_chunks, inc_nonce

HEADER_LENGTH = 2 + 16 + 16
MAX_SEGMENT_SIZE = 4 * 1024
TERMINATION_HEADER = (b'\x00' * 18)

def get_stream_pair(reader, writer, **kwargs):
    """Return a tuple with `(unbox_stream, box_stream)` (reader/writer).

    :return: (:class:`secret_handshake.boxstream.UnboxStream`,
              :class:`secret_handshake.boxstream.BoxStream`) """
    box_args = {
        'key': kwargs['encrypt_key'],
        'nonce': kwargs['encrypt_nonce'],
    }
    unbox_args = {
        'key': kwargs['decrypt_key'],
        'nonce': kwargs['decrypt_nonce'],
    }
    return UnboxStream(reader, **unbox_args), BoxStream(writer, **box_args)

class UnboxStream(object):
    def __init__(self, reader, key, nonce):
        self.reader = reader
        self.key = key
        self.nonce = nonce
        self.closed = False

    async def read(self):
        try:
            data = await self.reader.readexactly(HEADER_LENGTH)
        except IncompleteReadError:
            self.closed = True
            return None

        box = SecretBox(self.key)

        header = box.decrypt(data, self.nonce)

        if header == TERMINATION_HEADER:
            self.closed = True
            return None

        length = struct.unpack('>H', header[:2])[0]
        mac = header[2:]

        data = await self.reader.readexactly(length)

        body = box.decrypt(mac + data, inc_nonce(self.nonce))

        self.nonce = inc_nonce(inc_nonce(self.nonce))
        return body

    async def __aiter__(self):
        while True:
            data = await self.read()
            if data is None:
                return
            yield data

class BoxStream(object):
    def __init__(self, writer, key, nonce):
        self.writer = writer
        self.key = key
        self.box = SecretBox(self.key)
        self.nonce = nonce

    def write(self, data):

        # XXX: This nonce logic is almost for sure wrong

        for chunk in split_chunks(data, MAX_SEGMENT_SIZE):
            body = self.box.encrypt(chunk, inc_nonce(self.nonce))[24:]
            header = struct.pack('>H', len(body) - 16) + body[:16]

            hdrbox = self.box.encrypt(header, self.nonce)[24:]
            self.writer.write(hdrbox)

            self.nonce = inc_nonce(inc_nonce(self.nonce))
            self.writer.write(body[16:])

    def close(self):
        self.writer.write(self.box.encrypt(b'\x00' * 18, self.nonce)[24:])
<commit_msg>Remove FIXME<commit_after>import struct
from asyncio import IncompleteReadError

from nacl.secret import SecretBox

from .util import split_chunks, inc_nonce

HEADER_LENGTH = 2 + 16 + 16
MAX_SEGMENT_SIZE = 4 * 1024
TERMINATION_HEADER = (b'\x00' * 18)

def get_stream_pair(reader, writer, **kwargs):
    """Return a tuple with `(unbox_stream, box_stream)` (reader/writer).

    :return: (:class:`secret_handshake.boxstream.UnboxStream`,
              :class:`secret_handshake.boxstream.BoxStream`) """
    box_args = {
        'key': kwargs['encrypt_key'],
        'nonce': kwargs['encrypt_nonce'],
    }
    unbox_args = {
        'key': kwargs['decrypt_key'],
        'nonce': kwargs['decrypt_nonce'],
    }
    return UnboxStream(reader, **unbox_args), BoxStream(writer, **box_args)

class UnboxStream(object):
    def __init__(self, reader, key, nonce):
        self.reader = reader
        self.key = key
        self.nonce = nonce
        self.closed = False

    async def read(self):
        try:
            data = await self.reader.readexactly(HEADER_LENGTH)
        except IncompleteReadError:
            self.closed = True
            return None

        box = SecretBox(self.key)

        header = box.decrypt(data, self.nonce)

        if header == TERMINATION_HEADER:
            self.closed = True
            return None

        length = struct.unpack('>H', header[:2])[0]
        mac = header[2:]

        data = await self.reader.readexactly(length)

        body = box.decrypt(mac + data, inc_nonce(self.nonce))

        self.nonce = inc_nonce(inc_nonce(self.nonce))
        return body

    async def __aiter__(self):
        while True:
            data = await self.read()
            if data is None:
                return
            yield data

class BoxStream(object):
    def __init__(self, writer, key, nonce):
        self.writer = writer
        self.key = key
        self.box = SecretBox(self.key)
        self.nonce = nonce

    def write(self, data):
        for chunk in split_chunks(data, MAX_SEGMENT_SIZE):
            body = self.box.encrypt(chunk, inc_nonce(self.nonce))[24:]
            header = struct.pack('>H', len(body) - 16) + body[:16]

            hdrbox = self.box.encrypt(header, self.nonce)[24:]
            self.writer.write(hdrbox)

            self.nonce = inc_nonce(inc_nonce(self.nonce))
            self.writer.write(body[16:])

    def close(self):
        self.writer.write(self.box.encrypt(b'\x00' * 18, self.nonce)[24:])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
from xml.etree import ElementTree
import urllib
import json

TITLE    = 'Sarpurinn'
PREFIX   = '/video/sarpurinn'
ART      = 'art-default.jpg'
ICON     = 'icon-default.png'
STREAM_URL = 'http://smooth.ruv.cache.is/'
INFO_URL = "http://ruv.is/sarpurinn"
SARP_STOR_DAYS = 31

def Start(): # Initialize the plug-in

  Plugin.AddViewGroup("Details", viewMode="InfoList", mediaType="items")
  Plugin.AddViewGroup("List", viewMode="List", mediaType="items")

# Setup the default attributes for the ObjectContainer
  ObjectContainer.title1 = TITLE
  ObjectContainer.view_group = 'Details'
  ObjectContainer.art = R(ART)

# Setup the default attributes for the other objects
#DirectoryObject.thumb = R(ICON)
#DirectoryObject.art = R(ART)
#VideoClipObject.thumb = R(ICON)
#VideoClipObject.art = R(ART)

#####################################################################
@handler(PREFIX, TITLE)
def MainMenu():
  oc = ObjectContainer()
  oc.add(DirectoryObject(key=Callback(LiveMenu), title="Live", thumb = R(ICON)))
  oc.add(DirectoryObject(key=Callback(SarpMenu), title="Sarpurinn", thumb = R(ICON)))

  return oc

@route(PREFIX + '/createliveobject', include_container = bool)
def CreateLiveObject(url, title, summary, thumb = None, vidCodec = None, audCodec = None, media_container = None, vidRes = None, include_container=False, *args, **kwargs):

	video_object = VideoClipObject(
		key = Callback(CreateLiveObject, url = url, title = title, summary = summary, thumb = thumb, vidCodec = vidCodec, audCodec = audCodec, media_container = media_container, vidRes = vidRes, include_container = True),
		rating_key = url, ### ???????
		title = title,
		summary = summary,
		thumb = thumb,
		items = [
			MediaObject(
				parts = [
					PartObject(
						key = HTTPLiveStreamURL(Callback(PlayVideoLive, url = url))
					)
				],
				video_codec = vidCodec, #VideoCodec.H264,
				audio_codec = audCodec, #AudioCodec.AAC,
				video_resolution = vidRes,
				audio_channels = 2,
				container = media_container, #Container.MP4,
				optimized_for_streaming = True
			)
		]
	)

	if include_container:
		return ObjectContainer(objects = [video_object])
	else:
		return video_object

@indirect
@route(PREFIX + '/playvideolive.m3u8')
def PlayVideoLive(url):
	return IndirectResponse(VideoClipObject, key=url)

@route(PREFIX, "/livemenu")
def LiveMenu():
	oc = ObjectContainer()
	oc.add(CreateLiveObject(
		url = "http://ruvruv-live.hls.adaptive.level3.net/ruv/ruv/index/stream4.m3u8",
		title = "RÚV",
		summary = "Bein útsending RÚV",
		thumb = R("ruv.png"), #Callback(Thumb, url=thumb),
		vidCodec = VideoCodec.H264,
		audCodec = AudioCodec.AAC,
		media_container = Container.MP4,
		vidRes = "720",
		include_container=False
		)
	)
	oc.add(CreateLiveObject(
		url = "http://ruvruv2-live.hls.adaptive.level3.net/ruv/ruv2/index/stream5.m3u8",
		title = "RÚV 2",
		summary = "Bein útsending RÚV 2",
		thumb = R("ruv2.png"), #Callback(Thumb, url=thumb),
		vidCodec = VideoCodec.H264,
		audCodec = AudioCodec.AAC,
		media_container = Container.MP4,
		vidRes = "1080",
		include_container=False
		)
	)
	return oc

@route(PREFIX + '/createvideoobject', include_container = bool)
def CreateVideoObject(url, title, summary, pid, thumb = None, vidCodec = None, audCodec = None, media_container = None, vidRes = None, include_container=False, *args, **kwargs):

	video_object = VideoClipObject(
		key = Callback(CreateVideoObject, url = url, title = title, summary = summary, pid = pid, thumb = thumb, vidCodec = vidCodec, audCodec = audCodec, media_container = media_container, vidRes = vidRes, include_container = True),
		rating_key = url, ### ???????
		title = title,
		summary = summary,
		thumb = thumb,
		items = [
			MediaObject(
				parts = [
					PartObject(
						key = Callback(PlaySarpVideo, url = url, pid = pid)
					)
				],
				video_codec = vidCodec, #VideoCodec.H264,
				audio_codec = audCodec, #AudioCodec.AAC,
				video_resolution = vidRes,
				audio_channels = 2,
				container = media_container, #Container.MP4,
				optimized_for_streaming = True
			)
		]
	)

	if include_container:
		return ObjectContainer(objects = [video_object])
	else:
		return video_object

@indirect
@route(PREFIX + '/playsarpvideo')
def PlaySarpVideo(url,pid):
	URLS = [
		'opid/{0}R{1}.mp4',
		'lokad/{0}R{1}.mp4',
		'lokad/{0}M{1}.mp4'
	]
	vid_url = ""
	if ("lokad" in url):
		for nr in range(30):
			url_test = url + pid + "R" + str(nr) + ".mp4"
			Log("URL TEST: "+url_test)
			if (urllib.urlopen(url_test).getcode() == 200):
				vid_url = url_test
				break
	else:
		for url_ending in URLS:
			for nr in range(30):
				url_test = STREAM_URL + url_ending.format(pid, nr)
				Log("URL TEST: "+url_test)
				if (urllib.urlopen(url_test).getcode() == 200):
					vid_url = url_test
					break

	if (vid_url == ""):
		Log("Not found on server")
		return None

	Log("Final url: " + vid_url)
	return IndirectResponse(VideoClipObject, key=vid_url)

# Get the TV schedule for specified day
@route(PREFIX, "/getschedule")
def GetSchedule(dags):
	schedule = {}
	#schedule['date'] = datetime.date.today()

	url = "http://muninn.ruv.is/files/xml/ruv/" + dags + "/"
	r = urllib.urlopen(url)
	if (r.getcode() != 200):
		print "Could not get schedule"
		return None
	schedule_xml = ElementTree.fromstring(r.read())

	if (schedule_xml is None):
		print "Could not get schedule"
		return None

	for child in schedule_xml.iter("service"):
		# if (not child.tag == "service"):
		#	continue
		#Log(child)
		for entry_xml in child.iter('event'):
			entry = {}
			entry['title'] = entry_xml.find('title').text
			entry['pid'] = entry_xml.get('event-id')
			entry['showtime'] = entry_xml.get('start-time')
			entry['duration'] = entry_xml.get('duration')
			entry['sid'] = entry_xml.get('serie-id')

			# if marked then definitely "/lokad/"
			isl = entry_xml.get('mark')
			if (isl == "yes"):
				entry['isl'] = True
			elif( isl == "no"):
				entry['isl'] = False

			details_basic = entry_xml.find('description')
			if( not details_basic is None):
				entry['desc'] = details_basic.text

			entry_org_title = entry_xml.find('original-title')
			if( not entry_org_title is None ):
				entry['original-title'] = entry_org_title.text

			# If the series id is nothing then it is not a show (e.g. dagskrárlok)
			if( not entry['sid'] ):
				continue

			cat = entry_xml.find('category')
			if(  not cat is None  ):
				entry['catid'] = cat.get('value')
				entry['cat'] = cat.text

			ep = entry_xml.find('episode')
			if( not ep is None ):
				entry['ep_num'] = ep.get('number')
				entry['ep_total'] = ep.get('number-of-episodes')
			if( int(entry['ep_total']) > 1 ):
				# Append the episode number to the show title if it is a real multi-episode show
				entry['title'] = entry['title'] + " ("+entry['ep_num']+" af "+entry['ep_total']+")"
			else:
				# If it isn't a multi episode show then append the date to the title (to avoid overwriting files)
				entry['title'] = str(entry['title'])+ " (" + entry['showtime'][:16] + ")"

			schedule[entry['pid']] = entry
	return schedule

@route(PREFIX, "/daysmenu")
def DaysMenu():
	oc = ObjectContainer()
	oc.title1 = TITLE
	oc.title2 = unicode("Veldu dag")
	oc.add(DirectoryObject(key=Callback(SarpMenu), title= unicode("Í dag")))

	for d in range(1,SARP_STOR_DAYS+1):
		dagur = datetime.date.today() - datetime.timedelta(days=d)
		oc.add(DirectoryObject(key=Callback(SarpMenu, dags = str(dagur)), title=str(dagur)))
	return oc

@route(PREFIX, "/sarpmenu")
def SarpMenu(dags = None):
	other = "Fyrri dagar"
	if dags is not None:
		other = unicode("Aðrir dagar")
	else:
		dags = str(datetime.date.today())

	# get schedule
	schedule = GetSchedule(dags)
	#schedule = sorted(schedule, key=itemgetter('showtime'), reverse=True)
	oc = ObjectContainer()
	oc.title2 = dags
	oc.add(DirectoryObject(key=Callback(DaysMenu), title=other, thumb = R(ICON)))
	Log(len(schedule.items()))
	for key, schedule_item in schedule.items():
		if (not 'pid' in schedule_item):
			continue
		titill = schedule_item['title']
		#date = datetime.datetime.strptime(schedule_item['showtime'], '%Y-%m-%d %H:%M:%S')
		desc = schedule_item["desc"]
		#duration = schedule_item["duration"]
		item_pid = schedule_item["pid"]
		preUrl = ""
		Log(pid)
		if (schedule_item['isl']):
			preUrl = "lokad/"

		oc.add(CreateVideoObject(
			url = STREAM_URL + preUrl,
			title = titill,
			summary = desc,
			pid = item_pid,
			thumb = R(ICON), #Callback(Thumb, url=thumb),
			vidCodec = VideoCodec.H264,
			audCodec = AudioCodec.AAC,
			media_container = Container.MP4,
			vidRes = "576",
			include_container=False
			)
		)

	# oc.add(VideoClipObject(
		# url = STREAM_URL + "lokad/4897620",
		# title = "Rembrandt",
		# summary = "Skemmtilegt",
		# thumb = R(ICON), #Callback(Thumb, url=thumb),
		# duration = 5*60*1000,
		# #  originally_available_at = date
		# )
	# )

	return oc

def Thumb(url):

  try:
    data = HTTP.Request(url, cacheTime = CACHE_1MONTH).content
    return DataObject(data, 'image/jpeg')
  except:
    return Redirect(R(ICON)) <commit_msg>sarp video test fix<commit_after># -*- coding: utf-8 -*-
import datetime
from xml.etree import ElementTree
import urllib
import json

TITLE    = 'Sarpurinn'
PREFIX   = '/video/sarpurinn'
ART      = 'art-default.jpg'
ICON     = 'icon-default.png'
STREAM_URL = 'http://smooth.ruv.cache.is/'
INFO_URL = "http://ruv.is/sarpurinn"
SARP_STOR_DAYS = 31

def Start(): # Initialize the plug-in

  Plugin.AddViewGroup("Details", viewMode="InfoList", mediaType="items")
  Plugin.AddViewGroup("List", viewMode="List", mediaType="items")

# Setup the default attributes for the ObjectContainer
  ObjectContainer.title1 = TITLE
  ObjectContainer.view_group = 'Details'
  ObjectContainer.art = R(ART)

# Setup the default attributes for the other objects
#DirectoryObject.thumb = R(ICON)
#DirectoryObject.art = R(ART)
#VideoClipObject.thumb = R(ICON)
#VideoClipObject.art = R(ART)

#####################################################################
@handler(PREFIX, TITLE)
def MainMenu():
  oc = ObjectContainer()
  oc.add(DirectoryObject(key=Callback(LiveMenu), title="Live", thumb = R(ICON)))
  oc.add(DirectoryObject(key=Callback(SarpMenu), title="Sarpurinn", thumb = R(ICON)))

  return oc

@route(PREFIX + '/createliveobject', include_container = bool)
def CreateLiveObject(url, title, summary, thumb = None, vidCodec = None, audCodec = None, media_container = None, vidRes = None, include_container=False, *args, **kwargs):

	video_object = VideoClipObject(
		key = Callback(CreateLiveObject, url = url, title = title, summary = summary, thumb = thumb, vidCodec = vidCodec, audCodec = audCodec, media_container = media_container, vidRes = vidRes, include_container = True),
		rating_key = url, ### ???????
		title = title,
		summary = summary,
		thumb = thumb,
		items = [
			MediaObject(
				parts = [
					PartObject(
						key = HTTPLiveStreamURL(Callback(PlayVideoLive, url = url))
					)
				],
				video_codec = vidCodec, #VideoCodec.H264,
				audio_codec = audCodec, #AudioCodec.AAC,
				video_resolution = vidRes,
				audio_channels = 2,
				container = media_container, #Container.MP4,
				optimized_for_streaming = True
			)
		]
	)

	if include_container:
		return ObjectContainer(objects = [video_object])
	else:
		return video_object

@indirect
@route(PREFIX + '/playvideolive.m3u8')
def PlayVideoLive(url):
	return IndirectResponse(VideoClipObject, key=url)

@route(PREFIX, "/livemenu")
def LiveMenu():
	oc = ObjectContainer()
	oc.add(CreateLiveObject(
		url = "http://ruvruv-live.hls.adaptive.level3.net/ruv/ruv/index/stream4.m3u8",
		title = "RÚV",
		summary = "Bein útsending RÚV",
		thumb = R("ruv.png"), #Callback(Thumb, url=thumb),
		vidCodec = VideoCodec.H264,
		audCodec = AudioCodec.AAC,
		media_container = Container.MP4,
		vidRes = "720",
		include_container=False
		)
	)
	oc.add(CreateLiveObject(
		url = "http://ruvruv2-live.hls.adaptive.level3.net/ruv/ruv2/index/stream5.m3u8",
		title = "RÚV 2",
		summary = "Bein útsending RÚV 2",
		thumb = R("ruv2.png"), #Callback(Thumb, url=thumb),
		vidCodec = VideoCodec.H264,
		audCodec = AudioCodec.AAC,
		media_container = Container.MP4,
		vidRes = "1080",
		include_container=False
		)
	)
	return oc

@route(PREFIX + '/createvideoobject', include_container = bool)
def CreateVideoObject(url, title, summary, pid, thumb = None, vidCodec = None, audCodec = None, media_container = None, vidRes = None, include_container=False, *args, **kwargs):

	video_object = VideoClipObject(
		key = Callback(CreateVideoObject, url = url, title = title, summary = summary, pid = pid, thumb = thumb, vidCodec = vidCodec, audCodec = audCodec, media_container = media_container, vidRes = vidRes, include_container = True),
		rating_key = url, ### ???????
		title = title,
		summary = summary,
		thumb = thumb,
		items = [
			MediaObject(
				parts = [
					PartObject(
						key = Callback(PlaySarpVideo, url = url, pid = pid)
					)
				],
				video_codec = vidCodec, #VideoCodec.H264,
				audio_codec = audCodec, #AudioCodec.AAC,
				video_resolution = vidRes,
				audio_channels = 2,
				container = media_container, #Container.MP4,
				optimized_for_streaming = True
			)
		]
	)

	if include_container:
		return ObjectContainer(objects = [video_object])
	else:
		return video_object

@indirect
@route(PREFIX + '/playsarpvideo')
def PlaySarpVideo(url,pid):
	URLS = [
		'opid/{0}R{1}.mp4',
		'lokad/{0}R{1}.mp4',
		'lokad/{0}M{1}.mp4'
	]
	vid_url = ""
	if ("lokad" in url):
		for nr in range(30):
			url_test = url + pid + "R" + str(nr) + ".mp4"
			Log("URL TEST: "+url_test)
			if (urllib.urlopen(url_test).getcode() == 200):
				vid_url = url_test
				break
	else:
		for url_ending in URLS:
			for nr in range(30):
				url_test = STREAM_URL + url_ending.format(pid, nr)
				Log("URL TEST: "+url_test)
				if (urllib.urlopen(url_test).getcode() == 200):
					vid_url = url_test
					break

	if (vid_url == ""):
		Log("Not found on server")
		return None

	Log("Final url: " + vid_url)
	return IndirectResponse(VideoClipObject, key=vid_url)

# Get the TV schedule for specified day
@route(PREFIX, "/getschedule")
def GetSchedule(dags):
	schedule = {}
	#schedule['date'] = datetime.date.today()

	url = "http://muninn.ruv.is/files/xml/ruv/" + dags + "/"
	r = urllib.urlopen(url)
	if (r.getcode() != 200):
		print "Could not get schedule"
		return None
	schedule_xml = ElementTree.fromstring(r.read())

	if (schedule_xml is None):
		print "Could not get schedule"
		return None

	for child in schedule_xml.iter("service"):
		# if (not child.tag == "service"):
		#	continue
		#Log(child)
		for entry_xml in child.iter('event'):
			entry = {}
			entry['title'] = entry_xml.find('title').text
			entry['pid'] = entry_xml.get('event-id')
			entry['showtime'] = entry_xml.get('start-time')
			entry['duration'] = entry_xml.get('duration')
			entry['sid'] = entry_xml.get('serie-id')

			# if marked then definitely "/lokad/"
			isl = entry_xml.get('mark')
			if (isl == "yes"):
				entry['isl'] = True
			elif( isl == "no"):
				entry['isl'] = False

			details_basic = entry_xml.find('description')
			if( not details_basic is None):
				entry['desc'] = details_basic.text

			entry_org_title = entry_xml.find('original-title')
			if( not entry_org_title is None ):
				entry['original-title'] = entry_org_title.text

			# If the series id is nothing then it is not a show (e.g. dagskrárlok)
			if( not entry['sid'] ):
				continue

			cat = entry_xml.find('category')
			if(  not cat is None  ):
				entry['catid'] = cat.get('value')
				entry['cat'] = cat.text

			ep = entry_xml.find('episode')
			if( not ep is None ):
				entry['ep_num'] = ep.get('number')
				entry['ep_total'] = ep.get('number-of-episodes')
			if( int(entry['ep_total']) > 1 ):
				# Append the episode number to the show title if it is a real multi-episode show
				entry['title'] = entry['title'] + " ("+entry['ep_num']+" af "+entry['ep_total']+")"
			else:
				# If it isn't a multi episode show then append the date to the title (to avoid overwriting files)
				entry['title'] = str(entry['title'])+ " (" + entry['showtime'][:16] + ")"

			schedule[entry['pid']] = entry
	return schedule

@route(PREFIX, "/daysmenu")
def DaysMenu():
	oc = ObjectContainer()
	oc.title1 = TITLE
	oc.title2 = unicode("Veldu dag")
	oc.add(DirectoryObject(key=Callback(SarpMenu), title= unicode("Í dag")))

	for d in range(1,SARP_STOR_DAYS+1):
		dagur = datetime.date.today() - datetime.timedelta(days=d)
		oc.add(DirectoryObject(key=Callback(SarpMenu, dags = str(dagur)), title=str(dagur)))
	return oc

@route(PREFIX, "/sarpmenu")
def SarpMenu(dags = None):
	other = "Fyrri dagar"
	if dags is not None:
		other = unicode("Aðrir dagar")
	else:
		dags = str(datetime.date.today())

	# get schedule
	schedule = GetSchedule(dags)
	#schedule = sorted(schedule, key=itemgetter('showtime'), reverse=True)
	oc = ObjectContainer()
	oc.title2 = dags
	oc.add(DirectoryObject(key=Callback(DaysMenu), title=other, thumb = R(ICON)))
	Log(len(schedule.items()))
	for key, schedule_item in schedule.items():
		if (not 'pid' in schedule_item):
			continue
		titill = schedule_item['title']
		#date = datetime.datetime.strptime(schedule_item['showtime'], '%Y-%m-%d %H:%M:%S')
		desc = schedule_item["desc"]
		#duration = schedule_item["duration"]
		item_pid = schedule_item["pid"]
		preUrl = ""
		Log(item_pid)
		if (schedule_item['isl']):
			preUrl = "lokad/"

		oc.add(CreateVideoObject(
			url = STREAM_URL + preUrl,
			title = titill,
			summary = desc,
			pid = item_pid,
			thumb = R(ICON), #Callback(Thumb, url=thumb),
			vidCodec = VideoCodec.H264,
			audCodec = AudioCodec.AAC,
			media_container = Container.MP4,
			vidRes = "576",
			include_container=False
			)
		)

	# oc.add(VideoClipObject(
		# url = STREAM_URL + "lokad/4897620",
		# title = "Rembrandt",
		# summary = "Skemmtilegt",
		# thumb = R(ICON), #Callback(Thumb, url=thumb),
		# duration = 5*60*1000,
		# #  originally_available_at = date
		# )
	# )

	return oc

def Thumb(url):

  try:
    data = HTTP.Request(url, cacheTime = CACHE_1MONTH).content
    return DataObject(data, 'image/jpeg')
  except:
    return Redirect(R(ICON)) <|endoftext|>