<commit_before>#!/usr/bin/env python2

import re
import glob

class AutoVivification(dict):
    """Implementation of perl's autovivification feature."""
    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

conffile = "./amalg.conf"

def importfile(filename, keyword):
    files = glob.iglob(filename)
    combined = ""

    for onefile in files:
        #infile = open(filename, 'r')
        infile = open(onefile, 'r')

        for line in infile:
            #print "%s" % line.rstrip()
            #print "%s" % line.strip() # removes whitespace on left and right
            result = re.match('\s*%s\s+(\S+);' % keyword, line.strip() )
            #result = re.match('(include.*)', line.strip(), re.I | re.U )
            if result:
                combined += "#"+line+"\n"
                #print "#include %s " % result.group(1)
                combined += importfile(result.group(1),keyword)
            else:
                combined += line
                #print line.rstrip()
    return combined

#infile = open(onefile, 'r')

#wholeconfig = infile.readlines()
#print wholeconfig

# list structure
# [ server stanza { line : { listen: [ ], server_name : [ ], root { location : path } } } ]

def parse_nginx_config(wholeconfig):
    stanza_count = 0
    server_start = 0
    location_start = 0
    linenum = 0
    nginx_stanzas = {} #AutoVivification()
    for line in wholeconfig.splitlines():
        linenum += 1
        # this doesn't do well if you open and close a stanza on the same line
        if len(re.findall('{',line)) > 0 and len(re.findall('}',line)) > 0:
            print "This script does not consistently support opening { and closing } stanzas on the same line."
        stanza_count+=len(re.findall('{',line))
        stanza_count-=len(re.findall('}',line))
        result = re.match('^\s*server', line.strip() )
        if result:
            server_start = stanza_count
            server_line = str(linenum)
            if not server_line in nginx_stanzas:
                nginx_stanzas[server_line] = { }
        # are we in a server block, and not a child stanza of the server block? is so, look for keywords
        # this is so we don't print the root directive for location as an example. That might be useful, but isn't implemented at this time.
        if server_start == stanza_count:
            # we are in a server block
            #result = re.match('\s*(listen|server|root)', line.strip())
            result = re.match('\s*(listen|server_name|root)\s*(.*)', line.strip("\s\t;"))
            if result:
                #print line.strip()
                if result.group(1)=="listen":
                    if not result.group(1) in nginx_stanzas[server_line]:
                        nginx_stanzas[server_line][result.group(1)] = []
                    nginx_stanzas[server_line][result.group(1)] += [result.group(2)]
                    #print "listen %s" % result.group(2)
                if result.group(1)=="server_name":
                    if not result.group(1) in nginx_stanzas[server_line]:
                        nginx_stanzas[server_line][result.group(1)] = []
                    nginx_stanzas[server_line][result.group(1)] += result.group(2).split()
                    #print "server_name %s" % result.group(2)
                if result.group(1)=="root":
                    if not result.group(1) in nginx_stanzas[server_line]:
                        nginx_stanzas[server_line][result.group(1)] = []
                    nginx_stanzas[server_line][result.group(1)] += result.group(2).split()
                    #print "root %s" % result.group(2)
        # if the server block is bigger than the current stanza, we have left the server stanza we were in
        # if server_start > stanza_count and server_start > 0: # The lowest stanza_count goes is 0, so it is redundant
        if server_start > stanza_count:
            # we are no longer in the server { block
            server_start = 0
            #print ""
    return nginx_stanzas

wholeconfig = importfile(conffile,"include")
nginx_stanzas = parse_nginx_config(wholeconfig)
print "%r" % nginx_stanzas

<commit_msg>Corrected a regexp match for server, that matched server* and should only match server {. Added a loop to step through the dict and print what it contains instead of just dumping the whole thing.<commit_after>#!/usr/bin/env python2

import re
import glob

conffile = "./amalg.conf"

class AutoVivification(dict):
    """Implementation of perl's autovivification feature."""
    def __getitem__(self, item):
        try:
            return dict.__getitem__(self, item)
        except KeyError:
            value = self[item] = type(self)()
            return value

def importfile(filename, keyword):
    files = glob.iglob(filename)
    combined = ""

    for onefile in files:
        #infile = open(filename, 'r')
        infile = open(onefile, 'r')

        for line in infile:
            #print "%s" % line.rstrip()
            #print "%s" % line.strip() # removes whitespace on left and right
            result = re.match('\s*%s\s+(\S+);' % keyword, line.strip() )
            #result = re.match('(include.*)', line.strip(), re.I | re.U )
            if result:
                combined += "#"+line+"\n"
                #print "#include %s " % result.group(1)
                combined += importfile(result.group(1),keyword)
            else:
                combined += line
                #print line.rstrip()
    return combined

def parse_nginx_config(wholeconfig):
    """
    list structure
    [ server stanza { line : { listen: [ ], server_name : [ ], root { location : path } } } ]
    """
    stanza_count = 0
    server_start = 0
    location_start = 0
    linenum = 0
    nginx_stanzas = {} #AutoVivification()
    for line in wholeconfig.splitlines():
        linenum += 1
        # this doesn't do well if you open and close a stanza on the same line
        if len(re.findall('{',line)) > 0 and len(re.findall('}',line)) > 0:
            print "This script does not consistently support opening { and closing } stanzas on the same line."
        stanza_count+=len(re.findall('{',line))
        stanza_count-=len(re.findall('}',line))
        result = re.match('^\s*server\s', line.strip() )
        if result:
            server_start = stanza_count
            server_line = str(linenum)
            if not server_line in nginx_stanzas:
                nginx_stanzas[server_line] = { }
        # are we in a server block, and not a child stanza of the server block? is so, look for keywords
        # this is so we don't print the root directive for location as an example. That might be useful, but isn't implemented at this time.
        if server_start == stanza_count:
            # we are in a server block
            #result = re.match('\s*(listen|server|root)', line.strip())
            result = re.match('\s*(listen|server_name|root)\s*(.*)', line.strip("\s\t;"))
            if result:
                #print line.strip()
                if result.group(1)=="listen":
                    if not result.group(1) in nginx_stanzas[server_line]:
                        nginx_stanzas[server_line][result.group(1)] = []
                    nginx_stanzas[server_line][result.group(1)] += [result.group(2)]
                    #print "listen %s" % result.group(2)
                if result.group(1)=="server_name":
                    if not result.group(1) in nginx_stanzas[server_line]:
                        nginx_stanzas[server_line][result.group(1)] = []
                    nginx_stanzas[server_line][result.group(1)] += result.group(2).split()
                    #print "server_name %s" % result.group(2)
                if result.group(1)=="root":
                    #if not result.group(1) in nginx_stanzas[server_line]:
                    #    nginx_stanzas[server_line][result.group(1)] = {}
                    nginx_stanzas[server_line][result.group(1)] = result.group(2)
                    #print "root %s" % result.group(2)
        # if the server block is bigger than the current stanza, we have left the server stanza we were in
        # if server_start > stanza_count and server_start > 0: # The lowest stanza_count goes is 0, so it is redundant
        if server_start > stanza_count:
            # we are no longer in the server { block
            server_start = 0
            #print ""
    return nginx_stanzas

wholeconfig = importfile(conffile,"include")
nginx_stanzas = parse_nginx_config(wholeconfig)
#print "%r" % nginx_stanzas

for one in nginx_stanzas:
    print "%s %s" % (one,nginx_stanzas[one])
    <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals, division
import re
import math
import six

from nibble.util import operator_same_class

@six.python_2_unicode_compatible
class Information(object):
    """
    Represents a quantity of digital information.
    """

    PARSE_REGEX = re.compile(r'(\d+(?:\.\d+)?|\.\d+)(?: +)?(\w+)')

    BITS = 1
    NIBBLES = 4
    BYTES = 8

    # binary, bytes
    KIBIBYTES = BYTES * 1024
    MEBIBYTES = BYTES * 1024 ** 2
    GIBIBYTES = BYTES * 1024 ** 3
    TEBIBYTES = BYTES * 1024 ** 4
    PEBIBYTES = BYTES * 1024 ** 5
    EXBIBYTES = BYTES * 1024 ** 6
    ZEBIBYTES = BYTES * 1024 ** 7
    YOBIBYTES = BYTES * 1024 ** 8

    # decimal, bytes
    KILOBYTES = BYTES * 1000
    MEGABYTES = BYTES * 1000 ** 2
    GIGABYTES = BYTES * 1000 ** 3
    TERABYTES = BYTES * 1000 ** 4
    PETABYTES = BYTES * 1000 ** 5
    EXABYTES = BYTES * 1000 ** 6
    ZETTABYTES = BYTES * 1000 ** 7
    YOTTABYTES = BYTES * 1000 ** 8

    # binary, bits
    KIBIBITS = 1024
    MEBIBITS = 1024 ** 2
    GIBIBITS = 1024 ** 3
    TEBIBITS = 1024 ** 4
    PEBIBITS = 1024 ** 5
    EXBIBITS = 1024 ** 6
    ZEBIBITS = 1024 ** 7
    YOBIBITS = 1024 ** 8

    # decimal, bits
    KILOBITS = 1000
    MEGABITS = 1000 ** 2
    GIGABITS = 1000 ** 3
    TERABITS = 1000 ** 4
    PETABITS = 1000 ** 5
    EXABITS = 1000 ** 6
    ZETTABITS = 1000 ** 7
    YOTTABITS = 1000 ** 8

    _SYMBOLS = {
        'b': 1,
        'B': BYTES,
        'N': NIBBLES,
        'n': NIBBLES,

        'K': KIBIBYTES,
        'KiB': KIBIBYTES,
        'kiB': KIBIBYTES,
        'M': MEBIBYTES,
        'MiB': MEBIBYTES,
        'miB': MEBIBYTES,
        'G': GIBIBYTES,
        'GiB': GIBIBYTES,
        'giB': GIBIBYTES,
        'T': TEBIBYTES,
        'TiB': TEBIBYTES,
        'tiB': TEBIBYTES,
        'P': PEBIBYTES,
        'PiB': PEBIBYTES,
        'piB': PEBIBYTES,
        'E': EXBIBYTES,
        'EiB': EXBIBYTES,
        'eiB': EXBIBYTES,
        'Z': ZEBIBYTES,
        'ZiB': ZEBIBYTES,
        'ziB': ZEBIBYTES,
        'Y': YOBIBYTES,
        'YiB': YOBIBYTES,
        'yiB': YOBIBYTES,

        'KB': KILOBYTES,
        'kB': KILOBYTES,
        'MB': MEGABYTES,
        'mB': MEGABYTES,
        'GB': GIGABYTES,
        'gB': GIGABYTES,
        'TB': TERABYTES,
        'tB': TERABYTES,
        'PB': PETABYTES,
        'pB': PETABYTES,
        'EB': EXABYTES,
        'eB': EXABYTES,
        'ZB': ZETTABYTES,
        'zB': ZETTABYTES,
        'YB': YOTTABYTES,
        'yB': YOTTABYTES,

        'Kib': KIBIBITS,
        'kib': KIBIBITS,
        'Mib': MEBIBITS,
        'mib': MEBIBITS,
        'Gib': GIBIBITS,
        'gib': GIBIBITS,
        'Tib': TEBIBITS,
        'tib': TEBIBITS,
        'Pib': PEBIBITS,
        'pib': PEBIBITS,
        'Eib': EXBIBITS,
        'eib': EXBIBITS,
        'Zib': ZEBIBITS,
        'zib': ZEBIBITS,
        'Yib': YOBIBITS,
        'yib': YOBIBITS,

        'Kb': KILOBITS,
        'kb': KILOBITS,
        'Mb': MEGABITS,
        'mb': MEGABITS,
        'Gb': GIGABITS,
        'gb': GIGABITS,
        'Tb': TERABITS,
        'tb': TERABITS,
        'Pb': PETABITS,
        'pb': PETABITS,
        'Eb': EXABITS,
        'eb': EXABITS,
        'Zb': ZETTABITS,
        'zb': ZETTABITS,
        'Yb': YOTTABITS,
        'yb': YOTTABITS
    }

    def __init__(self, quantity, unit=BITS):
        """
        Initialise a new information object.

        :param quantity: The number of the unit.
        :param unit: The size of the unit in bits, e.g. MiB = 8388608 bits.
                     Defaults to bits.
        """
        self.bits = math.ceil(quantity * unit)

    @classmethod
    def parse(cls, string):
        """
        Get an object representing an information string, e.g. "12TiB" or "9 n".

        :param string: The information string.
        :return: The parsed quantity of information.
        :raises ValueError: If the string could not be parsed. Check the message
                            for the reason why.
        """
        result = cls.PARSE_REGEX.match(string.strip())
        if not result:
            raise ValueError(
                'Unable to parse information string: {0}'.format(string))

        quantity_str = result.group(1)
        try:
            quantity = float(quantity_str)
            if quantity.is_integer():
                quantity = int(quantity)
        except ValueError:
            raise ValueError(
                'Unable to parse quantity number: {0}'.format(quantity_str))

        unit_str = result.group(2)
        if unit_str not in cls._SYMBOLS:
            raise ValueError(
                'Unrecognised information unit symbol: {0}'.format(unit_str))
        unit = cls._SYMBOLS[unit_str]

        return Information(quantity, unit)

    @operator_same_class
    def __lt__(self, other):
        return self.bits < other.bits

    def __le__(self, other):
        return self < other or self == other

    @operator_same_class
    def __eq__(self, other):
        return other.bits == self.bits

    def __ne__(self, other):
        return not other == self

    def __ge__(self, other):
        return self == other or self > other

    @operator_same_class
    def __gt__(self, other):
        return self.bits > other.bits

    @operator_same_class
    def __add__(self, other):
        return Information(self.bits + other.bits)

    @operator_same_class
    def __sub__(self, other):
        if self.bits - other.bits < 0:
            raise ArithmeticError(
                'Cannot have a negative amount of information')
        return Information(self.bits - other.bits)

    @operator_same_class
    def __mul__(self, other):
        return Information(self.bits * other.bits)

    @operator_same_class
    def __truediv__(self, other):
        return Information(self.bits / other.bits)

    def __repr__(self):
        return '<Information({0})>'.format(self.bits)

    def __str__(self):
        return '{0:,} bits'.format(self.bits)
<commit_msg>Caught out by `math.ceil` returning a `float`... again<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals, division
import re
import math
import six

from nibble.util import operator_same_class

@six.python_2_unicode_compatible
class Information(object):
    """
    Represents a quantity of digital information.
    """

    PARSE_REGEX = re.compile(r'(\d+(?:\.\d+)?|\.\d+)(?: +)?(\w+)')

    BITS = 1
    NIBBLES = 4
    BYTES = 8

    # binary, bytes
    KIBIBYTES = BYTES * 1024
    MEBIBYTES = BYTES * 1024 ** 2
    GIBIBYTES = BYTES * 1024 ** 3
    TEBIBYTES = BYTES * 1024 ** 4
    PEBIBYTES = BYTES * 1024 ** 5
    EXBIBYTES = BYTES * 1024 ** 6
    ZEBIBYTES = BYTES * 1024 ** 7
    YOBIBYTES = BYTES * 1024 ** 8

    # decimal, bytes
    KILOBYTES = BYTES * 1000
    MEGABYTES = BYTES * 1000 ** 2
    GIGABYTES = BYTES * 1000 ** 3
    TERABYTES = BYTES * 1000 ** 4
    PETABYTES = BYTES * 1000 ** 5
    EXABYTES = BYTES * 1000 ** 6
    ZETTABYTES = BYTES * 1000 ** 7
    YOTTABYTES = BYTES * 1000 ** 8

    # binary, bits
    KIBIBITS = 1024
    MEBIBITS = 1024 ** 2
    GIBIBITS = 1024 ** 3
    TEBIBITS = 1024 ** 4
    PEBIBITS = 1024 ** 5
    EXBIBITS = 1024 ** 6
    ZEBIBITS = 1024 ** 7
    YOBIBITS = 1024 ** 8

    # decimal, bits
    KILOBITS = 1000
    MEGABITS = 1000 ** 2
    GIGABITS = 1000 ** 3
    TERABITS = 1000 ** 4
    PETABITS = 1000 ** 5
    EXABITS = 1000 ** 6
    ZETTABITS = 1000 ** 7
    YOTTABITS = 1000 ** 8

    _SYMBOLS = {
        'b': 1,
        'B': BYTES,
        'N': NIBBLES,
        'n': NIBBLES,

        'K': KIBIBYTES,
        'KiB': KIBIBYTES,
        'kiB': KIBIBYTES,
        'M': MEBIBYTES,
        'MiB': MEBIBYTES,
        'miB': MEBIBYTES,
        'G': GIBIBYTES,
        'GiB': GIBIBYTES,
        'giB': GIBIBYTES,
        'T': TEBIBYTES,
        'TiB': TEBIBYTES,
        'tiB': TEBIBYTES,
        'P': PEBIBYTES,
        'PiB': PEBIBYTES,
        'piB': PEBIBYTES,
        'E': EXBIBYTES,
        'EiB': EXBIBYTES,
        'eiB': EXBIBYTES,
        'Z': ZEBIBYTES,
        'ZiB': ZEBIBYTES,
        'ziB': ZEBIBYTES,
        'Y': YOBIBYTES,
        'YiB': YOBIBYTES,
        'yiB': YOBIBYTES,

        'KB': KILOBYTES,
        'kB': KILOBYTES,
        'MB': MEGABYTES,
        'mB': MEGABYTES,
        'GB': GIGABYTES,
        'gB': GIGABYTES,
        'TB': TERABYTES,
        'tB': TERABYTES,
        'PB': PETABYTES,
        'pB': PETABYTES,
        'EB': EXABYTES,
        'eB': EXABYTES,
        'ZB': ZETTABYTES,
        'zB': ZETTABYTES,
        'YB': YOTTABYTES,
        'yB': YOTTABYTES,

        'Kib': KIBIBITS,
        'kib': KIBIBITS,
        'Mib': MEBIBITS,
        'mib': MEBIBITS,
        'Gib': GIBIBITS,
        'gib': GIBIBITS,
        'Tib': TEBIBITS,
        'tib': TEBIBITS,
        'Pib': PEBIBITS,
        'pib': PEBIBITS,
        'Eib': EXBIBITS,
        'eib': EXBIBITS,
        'Zib': ZEBIBITS,
        'zib': ZEBIBITS,
        'Yib': YOBIBITS,
        'yib': YOBIBITS,

        'Kb': KILOBITS,
        'kb': KILOBITS,
        'Mb': MEGABITS,
        'mb': MEGABITS,
        'Gb': GIGABITS,
        'gb': GIGABITS,
        'Tb': TERABITS,
        'tb': TERABITS,
        'Pb': PETABITS,
        'pb': PETABITS,
        'Eb': EXABITS,
        'eb': EXABITS,
        'Zb': ZETTABITS,
        'zb': ZETTABITS,
        'Yb': YOTTABITS,
        'yb': YOTTABITS
    }

    def __init__(self, quantity, unit=BITS):
        """
        Initialise a new information object.

        :param quantity: The number of the unit.
        :param unit: The size of the unit in bits, e.g. MiB = 8388608 bits.
                     Defaults to bits.
        """
        self.bits = int(math.ceil(quantity * unit))

    @classmethod
    def parse(cls, string):
        """
        Get an object representing an information string, e.g. "12TiB" or "9 n".

        :param string: The information string.
        :return: The parsed quantity of information.
        :raises ValueError: If the string could not be parsed. Check the message
                            for the reason why.
        """
        result = cls.PARSE_REGEX.match(string.strip())
        if not result:
            raise ValueError(
                'Unable to parse information string: {0}'.format(string))

        quantity_str = result.group(1)
        try:
            quantity = float(quantity_str)
            if quantity.is_integer():
                quantity = int(quantity)
        except ValueError:
            raise ValueError(
                'Unable to parse quantity number: {0}'.format(quantity_str))

        unit_str = result.group(2)
        if unit_str not in cls._SYMBOLS:
            raise ValueError(
                'Unrecognised information unit symbol: {0}'.format(unit_str))
        unit = cls._SYMBOLS[unit_str]

        return Information(quantity, unit)

    @operator_same_class
    def __lt__(self, other):
        return self.bits < other.bits

    def __le__(self, other):
        return self < other or self == other

    @operator_same_class
    def __eq__(self, other):
        return other.bits == self.bits

    def __ne__(self, other):
        return not other == self

    def __ge__(self, other):
        return self == other or self > other

    @operator_same_class
    def __gt__(self, other):
        return self.bits > other.bits

    @operator_same_class
    def __add__(self, other):
        return Information(self.bits + other.bits)

    @operator_same_class
    def __sub__(self, other):
        if self.bits - other.bits < 0:
            raise ArithmeticError(
                'Cannot have a negative amount of information')
        return Information(self.bits - other.bits)

    @operator_same_class
    def __mul__(self, other):
        return Information(self.bits * other.bits)

    @operator_same_class
    def __truediv__(self, other):
        return Information(self.bits / other.bits)

    def __repr__(self):
        return '<Information({0})>'.format(self.bits)

    def __str__(self):
        return '{0:,} bits'.format(self.bits)
<|endoftext|>


<commit_before>
import os
import time

from threading import Thread

LOCAL_NET_PATH = os.path.dirname(os.path.realpath(__file__))
REMOTE_NET_PATH = "/tmp/net-applet-shuffler"

def controller_thread(x, arg_d):
    # assemble arguments
    # 1. applet_id
    # 2. name_source
    # 3. user_source
    # 4. ip_source_data
    # 5. port_source
    # 6. name_dest
    # 7. user_dest
    # 8. ip_dest_data
    # 9. port_dest
    # 10. netserver_port
    # 11. flow_length
    # 12. flow_offset
    # 13. ip_source_control
    # 14. ip_dest_control
    arguments_string = " ".join((arg_d["applet_id"], arg_d["name_source"],
                                 arg_d["user_source"], arg_d["ip_source_data"],
                                 arg_d["port_source"], arg_d["name_dest"],
                                 arg_d["user_dest"], arg_d["ip_dest_data"],
                                 arg_d["port_dest"], arg_d["netserver_port"],
                                 arg_d["flow_length"], arg_d["flow_offset"],
                                 arg_d["ip_source_control"],
                                 arg_d["ip_dest_control"]))
    # check if netperf controller is already on source
    exit_code = x.ssh.exec(arg_d["ip_source_control"], arg_d["user_source"],
                           "test -f {}/netperf-controller.py"
                           .format(REMOTE_NET_PATH))
    # if not, copy it to source
    if not exit_code == 0:
        x.ssh.copy_to(arg_d["user_source"], arg_d["ip_source_control"],
                      LOCAL_NET_PATH, REMOTE_NET_PATH, "netperf-controller.py",
                      "netperf-controller.py")

    x.ssh.exec(arg_d["ip_source_control"], arg_d["user_source"],
               "python3.5 {}/{} {}".format(REMOTE_NET_PATH,
                                           "netperf-controller.py",
                                           arguments_string))

def main(x, conf, args):

    if not len(args) == 8:
        x.p.err("wrong usage. use: [name] sink:[name] id:[id] source_port:"
                "[port] sink_port:[port] length:[bytes|seconds] "
                "flow_offset:[seconds] netserver:[port]\n")
        return False
    # arguments dictionary
    arg_d = dict()
    try:
        arg_d["name_source"] = args[0]
        arg_d["name_dest"] = args[1].split(":")[1]
        arg_d["applet_id"] = args[2].split(":")[1]
        arg_d["port_source"] = args[3].split(":")[1]
        arg_d["port_dest"] = args[4].split(":")[1]
        arg_d["flow_length"] = args[5].split(":")[1]
        arg_d["flow_offset"] = args[6].split(":")[1]
        arg_d["netserver_port"] = args[7].split(":")[1]
    except IndexError:
        x.p.err("error: wrong usage\n")
        return False
    # retrieve: source ip, source user name, destination ip, destination user
    # name
    arg_d["ip_source_data"] = conf.get_data_ip(arg_d["name_source"])
    arg_d["ip_source_control"] = conf.get_control_ip(arg_d["name_source"])
    arg_d["user_source"] = conf.get_user(arg_d["name_source"])
    arg_d["ip_dest_data"] = conf.get_data_ip(arg_d["name_dest"])
    arg_d["ip_dest_control"] = conf.get_control_ip(arg_d["name_dest"])
    arg_d["user_dest"] = conf.get_user(arg_d["name_dest"])

    # potentially start parallel netperf instances
    # note: at this point, the distribution of the applet will be done, which
    # is probably not the best way to do (can cause unwanted time offsets)
    x.p.msg("Starting netperf applet on host {}\n".format(arg_d["user_source"]))
    contr_thread = Thread(target=controller_thread, args=(x, arg_d, ))
    contr_thread.daemon = True
    contr_thread.start()
    time.sleep(1)

    return True
<commit_msg>refining fix for not working with exec-applet<commit_after>
import os
import time

from threading import Thread

LOCAL_NET_PATH = os.path.dirname(os.path.realpath(__file__))
REMOTE_NET_PATH = "/tmp/net-applet-shuffler"
# fix for exec-applet
CONTROLLER_STARTED = False

def controller_thread(x, arg_d):
    # assemble arguments
    # 1. applet_id
    # 2. name_source
    # 3. user_source
    # 4. ip_source_data
    # 5. port_source
    # 6. name_dest
    # 7. user_dest
    # 8. ip_dest_data
    # 9. port_dest
    # 10. netserver_port
    # 11. flow_length
    # 12. flow_offset
    # 13. ip_source_control
    # 14. ip_dest_control
    arguments_string = " ".join((arg_d["applet_id"], arg_d["name_source"],
                                 arg_d["user_source"], arg_d["ip_source_data"],
                                 arg_d["port_source"], arg_d["name_dest"],
                                 arg_d["user_dest"], arg_d["ip_dest_data"],
                                 arg_d["port_dest"], arg_d["netserver_port"],
                                 arg_d["flow_length"], arg_d["flow_offset"],
                                 arg_d["ip_source_control"],
                                 arg_d["ip_dest_control"]))
    # check if netperf controller is already on source
    exit_code = x.ssh.exec(arg_d["ip_source_control"], arg_d["user_source"],
                           "test -f {}/netperf-controller.py"
                           .format(REMOTE_NET_PATH))
    # if not, copy it to source
    if not exit_code == 0:
        x.ssh.copy_to(arg_d["user_source"], arg_d["ip_source_control"],
                      LOCAL_NET_PATH, REMOTE_NET_PATH, "netperf-controller.py",
                      "netperf-controller.py")

    x.ssh.exec(arg_d["ip_source_control"], arg_d["user_source"],
               "python3.5 {}/{} {}".format(REMOTE_NET_PATH,
                                           "netperf-controller.py",
                                           arguments_string))
    global CONTROLLER_STARTED
    CONTROLLER_STARTED = True

def main(x, conf, args):

    if not len(args) == 8:
        x.p.err("wrong usage. use: [name] sink:[name] id:[id] source_port:"
                "[port] sink_port:[port] length:[bytes|seconds] "
                "flow_offset:[seconds] netserver:[port]\n")
        return False
    # arguments dictionary
    arg_d = dict()
    try:
        arg_d["name_source"] = args[0]
        arg_d["name_dest"] = args[1].split(":")[1]
        arg_d["applet_id"] = args[2].split(":")[1]
        arg_d["port_source"] = args[3].split(":")[1]
        arg_d["port_dest"] = args[4].split(":")[1]
        arg_d["flow_length"] = args[5].split(":")[1]
        arg_d["flow_offset"] = args[6].split(":")[1]
        arg_d["netserver_port"] = args[7].split(":")[1]
    except IndexError:
        x.p.err("error: wrong usage\n")
        return False
    # retrieve: source ip, source user name, destination ip, destination user
    # name
    arg_d["ip_source_data"] = conf.get_data_ip(arg_d["name_source"])
    arg_d["ip_source_control"] = conf.get_control_ip(arg_d["name_source"])
    arg_d["user_source"] = conf.get_user(arg_d["name_source"])
    arg_d["ip_dest_data"] = conf.get_data_ip(arg_d["name_dest"])
    arg_d["ip_dest_control"] = conf.get_control_ip(arg_d["name_dest"])
    arg_d["user_dest"] = conf.get_user(arg_d["name_dest"])

    # potentially start parallel netperf instances
    # note: at this point, the distribution of the applet will be done, which
    # is probably not the best way to do (can cause unwanted time offsets)
    x.p.msg("Starting netperf applet on host {}\n".format(arg_d["user_source"]))
    contr_thread = Thread(target=controller_thread, args=(x, arg_d, ))
    contr_thread.daemon = True
    contr_thread.start()
    while not CONTROLLER_STARTED:
        time.sleep(1)

    return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from nimp.commands._command import *
from nimp.utilities.ue3 import *
from nimp.utilities.ue4 import *
from nimp.utilities.file_mapper import *
from nimp.utilities.ps3 import *

#-------------------------------------------------------------------------------
class ShipCommand(Command):
    def __init__(self):
        Command.__init__(self, 'ship', 'Generate shippable loose files in target directory')

    #---------------------------------------------------------------------------
    def configure_arguments(self, env, parser):
        parser.add_argument('-c',
                            '--configuration',
                            help    = 'Configuration to publish',
                            metavar = '<configuration>')

        parser.add_argument('-p',
                            '--platform',
                            help    = 'Platforms to publish',
                            metavar = '<platform>')

        parser.add_argument('-r',
                            '--revision',
                            help = 'revision',
                            metavar = '<revision>')

        parser.add_argument('--dlc',
                            help    = 'DLCs to ship',
                            metavar = '<dlc>',
                            default = None)

        # FIXME: env.is_ue4 is not present at this time, but we want it
        if hasattr(env, 'project_type') and env.project_type is 'UE4':
            parser.add_argument('--destination',
                                help    = 'Destination Directory',
                                metavar = '<DIR>')

        if hasattr(env, 'project_type') and env.project_type is 'UE3':
            parser.add_argument('--loose-files-directory',
                                help    = 'Loose files destination Directory',
                                metavar = '<LOOSE_FILES_DIRECTORY>')

            parser.add_argument('--packages-directory',
                                help    = 'Packages destination Directory',
                                metavar = '<PACKAGES_DIRECTORY>')

        parser.add_argument('--no-cook',
                            help = 'Do not cook, assume everything is in place',
                            action = "store_true",
                            default = False)

        return True

    #---------------------------------------------------------------------------
    def run(self, env):

        # Unreal Engine 4
        if env.is_ue4:
            loose_dir = env.format(env.destination) if env.destination else env.format(env.publish_ship)
            if 0 != call_process(env.root_dir,
                                 ["./Engine/Binaries/DotNET/AutomationTool.exe",
                                  "BuildCookRun",
                                  "-nocompileeditor", "-nop4",
                                  env.format("-project={game}/{game}.uproject"),
                                  "-cook", "-stage", "-archive",
                                  "-archivedirectory=%s" % sanitize_path(loose_dir),
                                  "-package",
                                  "-clientconfig=Shipping",
                                  "-ue4exe=UE4Editor-Cmd.exe",
                                  "-pak",
                                  "-prereqs",
                                  "-nodebuginfo",
                                  env.format("-targetplatform={ue4_cook_platform}"),
                                  "-utf8output"]):
                return False
            return True
        # Unreal Engine 3
        if env.is_ue3:

            loose_dir = env.format(env.loose_files_directory) if env.loose_files_directory else env.publish_ship

            log_notification("Looking for master in %s" % (env.format(env.publish_master)))

            ship_game = (env.dlc == 'main')
            ship_incremental = os.path.exists(env.format(env.publish_master))

            if not env.no_cook:
                log_notification("Performing UE3 ship… game:%d incremental:%d" % (ship_game, ship_incremental))

                if ship_incremental and not ship_game:
                    log_error("Building a DLC patch is still not implemented")
                    return False

                # If cooking a DLC or a patch, we first need the original cook;
                # either the master cook (when shipping a game patch) or one of
                # the latest cooks done by buildbot
                if ship_incremental or not ship_game:
                    log_notification("Deploying original cook…")
                    master_files = env.map_files()
                    if ship_game:
                        master_files_source = master_files.src(env.publish_master) \
                                                          .recursive() \
                                                          .files()
                    else:
                        master_files_source = master_files.override(dlc = 'main', revision = 'HEAD') \
                                                          .src(env.publish_cook) \
                                                          .recursive() \
                                                          .files()
                    if not all_map(robocopy, master_files()):
                        return False

                # Now we can cook the desired maps
                maps = env.cook_maps[env.dlc.lower()]

                log_notification("Cooking maps: %s" % (" ".join(maps)))
                if not ue3_cook(env.game,
                                maps,
                                env.languages,
                                env.dlc,
                                env.ue3_cook_platform,
                                env.configuration,
                                incremental = ship_game and ship_incremental):
                    return False

                # If necessary, overwrite files we didn’t want to patch from the master again
                if ship_game and ship_incremental:
                    log_notification("Redeploying master cook ignoring patched files…")
                    patch_files = env.map_files()
                    patch_files.src(env.publish_master).override(step = 'patching').load_set("patch")
                    files_to_exclude = [src for src, *args in patch_files()]

                    for f in files_to_exclude:
                        log_notification("Will exclude: %s" % (f))

                    master_files_source.exclude_ignore_case(*files_to_exclude)
                    if not all_map(robocopy, master_files()):
                        return False

            if True:
                # Publish cooked directory to env.publish_cook
                if ship_game and hasattr(env, 'revision'):
                    cook_files = env.map_files()
                    cook_files.to(env.publish_cook).load_set("cook")
                    if not all_map(robocopy, cook_files()):
                        return False

            if True:
                # Publish final files to env.publish_ship
                if ship_game:
                    # PS3-specific step
                    if env.is_ps3:
                        ue3_generate_ps3_binaries(env)

                    if not ship_incremental:
                        # FIXME: implement this!
                        log_warning("Publishing full game is not implemented")
                    else:
                        log_notification("Copying files to output directory…")
                        patch_files = env.map_files()
                        patch_files.to(loose_dir).override(step = 'deploy').load_set("patch")
                        if not all_map(robocopy, patch_files()):
                            return False

                        # Win32-specific step
                        if env.is_win32:
                            ue3_fix_pc_ini(env, os.path.join(loose_dir, "Episode01"))
                else:
                    log_notification("Copying DLC to output directory…")
                    dlc_files = env.map_files()
                    dlc_files.to(loose_dir).load_set("dlc")

                    if not all_map(robocopy, dlc_files()):
                        return False

                if env.is_ps3:
                    ps3_generate_edata(env, loose_dir)
            return True

        # Error!
        log_error("Invalid project type %s" % (env.project_type))
        return False

<commit_msg>Sanitize path<commit_after># -*- coding: utf-8 -*-

from nimp.commands._command import *
from nimp.utilities.ue3 import *
from nimp.utilities.ue4 import *
from nimp.utilities.file_mapper import *
from nimp.utilities.ps3 import *

#-------------------------------------------------------------------------------
class ShipCommand(Command):
    def __init__(self):
        Command.__init__(self, 'ship', 'Generate shippable loose files in target directory')

    #---------------------------------------------------------------------------
    def configure_arguments(self, env, parser):
        parser.add_argument('-c',
                            '--configuration',
                            help    = 'Configuration to publish',
                            metavar = '<configuration>')

        parser.add_argument('-p',
                            '--platform',
                            help    = 'Platforms to publish',
                            metavar = '<platform>')

        parser.add_argument('-r',
                            '--revision',
                            help = 'revision',
                            metavar = '<revision>')

        parser.add_argument('--dlc',
                            help    = 'DLCs to ship',
                            metavar = '<dlc>',
                            default = None)

        # FIXME: env.is_ue4 is not present at this time, but we want it
        if hasattr(env, 'project_type') and env.project_type is 'UE4':
            parser.add_argument('--destination',
                                help    = 'Destination Directory',
                                metavar = '<DIR>')

        if hasattr(env, 'project_type') and env.project_type is 'UE3':
            parser.add_argument('--loose-files-directory',
                                help    = 'Loose files destination Directory',
                                metavar = '<LOOSE_FILES_DIRECTORY>')

            parser.add_argument('--packages-directory',
                                help    = 'Packages destination Directory',
                                metavar = '<PACKAGES_DIRECTORY>')

        parser.add_argument('--no-cook',
                            help = 'Do not cook, assume everything is in place',
                            action = "store_true",
                            default = False)

        return True

    #---------------------------------------------------------------------------
    def run(self, env):

        # Unreal Engine 4
        if env.is_ue4:
            loose_dir = env.format(env.destination) if env.destination else env.format(env.publish_ship)
            if 0 != call_process(env.root_dir,
                                 ["./Engine/Binaries/DotNET/AutomationTool.exe",
                                  "BuildCookRun",
                                  "-nocompileeditor", "-nop4",
                                  sanitize_path(env.format("-project={game}/{game}.uproject")),
                                  "-cook", "-stage", "-archive",
                                  "-archivedirectory=%s" % sanitize_path(loose_dir),
                                  "-package",
                                  "-clientconfig=Shipping",
                                  "-ue4exe=UE4Editor-Cmd.exe",
                                  "-pak",
                                  "-prereqs",
                                  "-nodebuginfo",
                                  env.format("-targetplatform={ue4_cook_platform}"),
                                  "-utf8output"]):
                return False
            return True
        # Unreal Engine 3
        if env.is_ue3:

            loose_dir = env.format(env.loose_files_directory) if env.loose_files_directory else env.publish_ship

            log_notification("Looking for master in %s" % (env.format(env.publish_master)))

            ship_game = (env.dlc == 'main')
            ship_incremental = os.path.exists(env.format(env.publish_master))

            if not env.no_cook:
                log_notification("Performing UE3 ship… game:%d incremental:%d" % (ship_game, ship_incremental))

                if ship_incremental and not ship_game:
                    log_error("Building a DLC patch is still not implemented")
                    return False

                # If cooking a DLC or a patch, we first need the original cook;
                # either the master cook (when shipping a game patch) or one of
                # the latest cooks done by buildbot
                if ship_incremental or not ship_game:
                    log_notification("Deploying original cook…")
                    master_files = env.map_files()
                    if ship_game:
                        master_files_source = master_files.src(env.publish_master) \
                                                          .recursive() \
                                                          .files()
                    else:
                        master_files_source = master_files.override(dlc = 'main', revision = 'HEAD') \
                                                          .src(env.publish_cook) \
                                                          .recursive() \
                                                          .files()
                    if not all_map(robocopy, master_files()):
                        return False

                # Now we can cook the desired maps
                maps = env.cook_maps[env.dlc.lower()]

                log_notification("Cooking maps: %s" % (" ".join(maps)))
                if not ue3_cook(env.game,
                                maps,
                                env.languages,
                                env.dlc,
                                env.ue3_cook_platform,
                                env.configuration,
                                incremental = ship_game and ship_incremental):
                    return False

                # If necessary, overwrite files we didn’t want to patch from the master again
                if ship_game and ship_incremental:
                    log_notification("Redeploying master cook ignoring patched files…")
                    patch_files = env.map_files()
                    patch_files.src(env.publish_master).override(step = 'patching').load_set("patch")
                    files_to_exclude = [src for src, *args in patch_files()]

                    for f in files_to_exclude:
                        log_notification("Will exclude: %s" % (f))

                    master_files_source.exclude_ignore_case(*files_to_exclude)
                    if not all_map(robocopy, master_files()):
                        return False

            if True:
                # Publish cooked directory to env.publish_cook
                if ship_game and hasattr(env, 'revision'):
                    cook_files = env.map_files()
                    cook_files.to(env.publish_cook).load_set("cook")
                    if not all_map(robocopy, cook_files()):
                        return False

            if True:
                # Publish final files to env.publish_ship
                if ship_game:
                    # PS3-specific step
                    if env.is_ps3:
                        ue3_generate_ps3_binaries(env)

                    if not ship_incremental:
                        # FIXME: implement this!
                        log_warning("Publishing full game is not implemented")
                    else:
                        log_notification("Copying files to output directory…")
                        patch_files = env.map_files()
                        patch_files.to(loose_dir).override(step = 'deploy').load_set("patch")
                        if not all_map(robocopy, patch_files()):
                            return False

                        # Win32-specific step
                        if env.is_win32:
                            ue3_fix_pc_ini(env, os.path.join(loose_dir, "Episode01"))
                else:
                    log_notification("Copying DLC to output directory…")
                    dlc_files = env.map_files()
                    dlc_files.to(loose_dir).load_set("dlc")

                    if not all_map(robocopy, dlc_files()):
                        return False

                if env.is_ps3:
                    ps3_generate_edata(env, loose_dir)
            return True

        # Error!
        log_error("Invalid project type %s" % (env.project_type))
        return False

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#-------------------------------------------------------------------------------
import socket
import random
import string
import time
import contextlib
import os

from nimp.utilities.build import *
from nimp.utilities.deployment import *
from nimp.utilities.file_mapper import *
from nimp.utilities.perforce import *
from nimp.utilities.system import *

#---------------------------------------------------------------------------
def ue4_build(env):
    if not env.ue4_build_configuration:
        log_error("Invalid empty value for configuration")
        return False

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if env.fastbuild:
        os.environ['UBT_bAllowFastBuild'] = 'true'
        # FIXME: it should not be our job to set this
        os.environ['UBT_bUseUnityBuild'] = 'false'

    # The project file generation requires RPCUtility and Ionic.Zip.Reduced very early
    if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/RPCUtility/RPCUtility.sln'),
                   'Any CPU', 'Development', None, '11', 'Build'):
        log_error("Could not build RPCUtility")
        return False

    # HACK: For some reason nothing copies this file on OS X
    if platform.system() == 'Darwin':
        robocopy(env.format('{root_dir}/Engine/Binaries/ThirdParty/Ionic/Ionic.Zip.Reduced.dll'),
                 env.format('{root_dir}/Engine/Binaries/DotNET/Ionic.Zip.Reduced.dll'))

    # HACK: We also need this on Windows
    if is_windows():
        robocopy(env.format('{root_dir}/Engine/Source/ThirdParty/FBX/2014.2.1/lib/vs2012/x64/release/libfbxsdk.dll'),
                 env.format('{root_dir}/Engine/Binaries/Win64/libfbxsdk.dll'))

    # Bootstrap if necessary
    if hasattr(env, 'bootstrap') and env.bootstrap:
        # Now generate project files
        if _ue4_generate_project(env) != 0:
            log_error("Error generating UE4 project files")
            return False

    # The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '12'
    try:
        for line in open(env.format(env.solution)):
            if '# Visual Studio 2012' in line:
                vs_version = '11'
                break
    except:
        pass

    # The main solution file
    solution = env.format(env.solution)

    # We’ll try to build all tools even in case of failure
    result = True

    # List of tools to build
    tools = []

    if env.target == 'tools':

        tools += [ 'UnrealFrontend',
                   'UnrealFileServer',
                   'ShaderCompileWorker', ]

        if env.platform != 'mac':
            tools += [ 'UnrealLightmass', ] # doesn’t build (yet?)

        if env.platform == 'linux':
            tools += [ 'CrossCompilerTool', ]

        if env.platform == 'win64':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'PS4DevKitUtil',
                       'PS4MapFileUtil',
                       'XboxOnePDBFileUtil',
                       'SymbolDebugger',
                       'UnrealPak', ]

    # Some tools are necessary even when not building tools...
    if env.platform == 'ps4':
        if 'PS4MapFileUtil' not in tools: tools += [ 'PS4MapFileUtil' ]

    if env.platform == 'xboxone':
        if 'XboxOnePDBFileUtil' not in tools: tools += [ 'XboxOnePDBFileUtil' ]

    # Build tools from the main solution
    for tool in tools:
        if not _ue4_build_project(env, solution, tool,
                                  'Mac' if env.platform == 'mac'
                                  else 'Linux' if env.platform == 'linux'
                                  else 'Win64',
                                  'Development', vs_version, 'Build'):
            log_error("Could not build %s" % (tool, ))
            result = False

    # Build tools from other solutions or with other flags
    if env.target == 'tools':

        if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln'),
                       'Any CPU', 'Development', None, vs_version, 'Build'):
            log_error("Could not build NetworkProfiler")
            result = False

        if env.platform != 'mac':
            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln'),
                           'Any CPU', 'Development', None, vs_version, 'Build'):
                log_error("Could not build UnrealSwarm")
                result = False

            if not vsbuild(env.format('{root_dir}/Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln'),
                           'Any CPU', 'Development', None, vs_version, 'Build'):
                log_error("Could not build SwarmInterface")
                result = False

        # These tools seem to be Windows only for now
        if env.platform == 'win64':

            if not _ue4_build_project(env, solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                log_error("Could not build BootstrapPackagedGame")
                result = False

            if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln'),
                           'x64', 'Development', None, '11', 'Build'):
                log_error("Could not build XboxOnePackageNameUtil")
                result = False

    if not result:
        return result

    if env.target == 'game':
        if not _ue4_build_project(env, solution, env.game, env.ue4_build_platform,
                                  env.ue4_build_configuration, vs_version, 'Build'):
            return False

    if env.target == 'editor':
        if not _ue4_build_project(env, solution, 'UE4Editor', env.ue4_build_platform,
                                  env.ue4_build_configuration + ' Editor', vs_version, 'Build'):
            return False

    return True

#
# Generate UE4 project files
#

def _ue4_generate_project(env):
    if is_windows():
        return call_process(env.root_dir, ['cmd', '/c', 'GenerateProjectFiles.bat'])
    else:
        return call_process(env.root_dir, ['/bin/sh', './GenerateProjectFiles.sh'])

#
# Helper commands for configuration sanitising
#

def get_ue4_build_config(config):
    d = { "debug"    : "Debug",
          "devel"    : "Development",
          "test"     : "Test",
          "shipping" : "Shipping", }
    if config not in d:
        log_warning('Unsupported UE4 build config “%s”' % (config))
        return None
    return d[config]

def get_ue4_build_platform(platform):
    d = { "ps4"     : "PS4",
          "xboxone" : "XboxOne",
          "win64"   : "Win64",
          "win32"   : "Win32",
          "linux"   : "Linux",
          "mac"     : "Mac", }
    if platform not in d:
        log_warning('Unsupported UE4 build platform “%s”' % (platform))
        return None
    return d[platform]

def get_ue4_cook_platform(platform):
    d = { "ps4"     : "PS4",
          "xboxone" : "XBoxOne",
          "win64"   : "Win64",
          "win32"   : "Win32",
          "linux"   : "Linux", }
    if platform not in d:
        log_warning('Unsupported UE4 cook platform “%s”' % (platform))
        return None
    return d[platform]

#---------------------------------------------------------------------------
def _ue4_build_project(env, sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if is_windows():
        return vsbuild(sln_file, build_platform, configuration,
                       project, vs_version, target)

    return call_process(env.root_dir,
                        ['/bin/sh', './Engine/Build/BatchFiles/%s/Build.sh' % (build_platform),
                         project, build_platform, configuration]) == 0

#---------------------------------------------------------------------------
def ue4_commandlet(env, commandlet, *args):
    if is_windows():
        exe = 'Engine/Binaries/Win64/UE4Editor.exe'
    elif platform.system() == 'Darwin':
        exe = 'Engine/Binaries/Mac/UE4Editor'
    else:
        exe = 'Engine/Binaries/Linux/UE4Editor'

    cmdline = [sanitize_path(os.path.join(env.format(env.root_dir), exe)),
               env.game,
               '-run=%s' % commandlet]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return call_process('.', cmdline) == 0

<commit_msg>Reverted bb6856a10dc8f50a98614da7b22c37f395648bc4<commit_after># -*- coding: utf-8 -*-

#-------------------------------------------------------------------------------
import socket
import random
import string
import time
import contextlib
import os

from nimp.utilities.build import *
from nimp.utilities.deployment import *
from nimp.utilities.file_mapper import *
from nimp.utilities.perforce import *
from nimp.utilities.system import *

#---------------------------------------------------------------------------
def ue4_build(env):
    if not env.ue4_build_configuration:
        log_error("Invalid empty value for configuration")
        return False

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if env.fastbuild:
        os.environ['UBT_bAllowFastBuild'] = 'true'
        # FIXME: it should not be our job to set this
        os.environ['UBT_bUseUnityBuild'] = 'false'

    # The project file generation requires RPCUtility and Ionic.Zip.Reduced very early
    if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/RPCUtility/RPCUtility.sln'),
                   'Any CPU', 'Development', None, '11', 'Build'):
        log_error("Could not build RPCUtility")
        return False

    # HACK: For some reason nothing copies this file on OS X
    if platform.system() == 'Darwin':
        robocopy(env.format('{root_dir}/Engine/Binaries/ThirdParty/Ionic/Ionic.Zip.Reduced.dll'),
                 env.format('{root_dir}/Engine/Binaries/DotNET/Ionic.Zip.Reduced.dll'))

    # HACK: We also need this on Windows
    if is_windows():
        robocopy(env.format('{root_dir}/Engine/Source/ThirdParty/FBX/2014.2.1/lib/vs2012/x64/release/libfbxsdk.dll'),
                 env.format('{root_dir}/Engine/Binaries/Win64/libfbxsdk.dll'))

    # Bootstrap if necessary
    if hasattr(env, 'bootstrap') and env.bootstrap:
        # Now generate project files
        if _ue4_generate_project(env) != 0:
            log_error("Error generating UE4 project files")
            return False

    # The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '12'
    try:
        for line in open(env.format(env.solution)):
            if '# Visual Studio 2012' in line:
                vs_version = '11'
                break
    except:
        pass

    # The main solution file
    solution = env.format(env.solution)

    # We’ll try to build all tools even in case of failure
    result = True

    # List of tools to build
    tools = []

    if env.target == 'tools':

        tools += [ 'UnrealFrontend',
                   'UnrealFileServer',
                   'ShaderCompileWorker', ]

        if env.platform != 'mac':
            tools += [ 'UnrealLightmass', ] # doesn’t build (yet?)

        if env.platform == 'linux':
            tools += [ 'CrossCompilerTool', ]

        if env.platform == 'win64':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'PS4DevKitUtil',
                       'PS4MapFileUtil',
                       'XboxOnePDBFileUtil',
                       'SymbolDebugger',
                       'UnrealPak', ]

    # Some tools are necessary even when not building tools...
    if env.platform == 'ps4':
        if 'PS4MapFileUtil' not in tools: tools += [ 'PS4MapFileUtil' ]

    if env.platform == 'xboxone':
        if 'XboxOnePDBFileUtil' not in tools: tools += [ 'XboxOnePDBFileUtil' ]

    # Build tools from the main solution
    for tool in tools:
        if not _ue4_build_project(env, solution, tool,
                                  'Mac' if env.platform == 'mac'
                                  else 'Linux' if env.platform == 'linux'
                                  else 'Win64',
                                  'Development', vs_version, 'Build'):
            log_error("Could not build %s" % (tool, ))
            result = False

    # Build tools from other solutions or with other flags
    if env.target == 'tools':

        if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln'),
                       'Any CPU', 'Development', None, vs_version, 'Build'):
            log_error("Could not build NetworkProfiler")
            result = False

        if env.platform != 'mac':
            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln'),
                           'Any CPU', 'Development', None, vs_version, 'Build'):
                log_error("Could not build UnrealSwarm")
                result = False

            if not vsbuild(env.format('{root_dir}/Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln'),
                           'Any CPU', 'Development', None, vs_version, 'Build'):
                log_error("Could not build SwarmInterface")
                result = False

        # These tools seem to be Windows only for now
        if env.platform == 'win64':

            if not _ue4_build_project(env, solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                log_error("Could not build BootstrapPackagedGame")
                result = False

            if not vsbuild(env.format('{root_dir}/Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln'),
                           'x64', 'Development', None, '11', 'Build'):
                log_error("Could not build XboxOnePackageNameUtil")
                result = False

    if not result:
        return result

    if env.target == 'game':
        if not _ue4_build_project(env, solution, env.game, env.ue4_build_platform,
                                  env.ue4_build_configuration, vs_version, 'Build'):
            return False

    if env.target == 'editor':
        if not _ue4_build_project(env, solution, env.game, env.ue4_build_platform,
                                  env.ue4_build_configuration + ' Editor', vs_version, 'Build'):
            return False

    return True

#
# Generate UE4 project files
#

def _ue4_generate_project(env):
    if is_windows():
        return call_process(env.root_dir, ['cmd', '/c', 'GenerateProjectFiles.bat'])
    else:
        return call_process(env.root_dir, ['/bin/sh', './GenerateProjectFiles.sh'])

#
# Helper commands for configuration sanitising
#

def get_ue4_build_config(config):
    d = { "debug"    : "Debug",
          "devel"    : "Development",
          "test"     : "Test",
          "shipping" : "Shipping", }
    if config not in d:
        log_warning('Unsupported UE4 build config “%s”' % (config))
        return None
    return d[config]

def get_ue4_build_platform(platform):
    d = { "ps4"     : "PS4",
          "xboxone" : "XboxOne",
          "win64"   : "Win64",
          "win32"   : "Win32",
          "linux"   : "Linux",
          "mac"     : "Mac", }
    if platform not in d:
        log_warning('Unsupported UE4 build platform “%s”' % (platform))
        return None
    return d[platform]

def get_ue4_cook_platform(platform):
    d = { "ps4"     : "PS4",
          "xboxone" : "XBoxOne",
          "win64"   : "Win64",
          "win32"   : "Win32",
          "linux"   : "Linux", }
    if platform not in d:
        log_warning('Unsupported UE4 cook platform “%s”' % (platform))
        return None
    return d[platform]

#---------------------------------------------------------------------------
def _ue4_build_project(env, sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if is_windows():
        return vsbuild(sln_file, build_platform, configuration,
                       project, vs_version, target)

    return call_process(env.root_dir,
                        ['/bin/sh', './Engine/Build/BatchFiles/%s/Build.sh' % (build_platform),
                         project, build_platform, configuration]) == 0

#---------------------------------------------------------------------------
def ue4_commandlet(env, commandlet, *args):
    if is_windows():
        exe = 'Engine/Binaries/Win64/UE4Editor.exe'
    elif platform.system() == 'Darwin':
        exe = 'Engine/Binaries/Mac/UE4Editor'
    else:
        exe = 'Engine/Binaries/Linux/UE4Editor'

    cmdline = [sanitize_path(os.path.join(env.format(env.root_dir), exe)),
               env.game,
               '-run=%s' % commandlet]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return call_process('.', cmdline) == 0

<|endoftext|>


<commit_before>"""
Copyright 2009 55 Minutes (http://www.55minutes.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os, time
from urllib import pathname2url as p2url

from data_storage import ModuleVars
from html_module_detail import html_module_detail
from html_module_errors import html_module_errors
from html_module_excludes import html_module_excludes
from templates import default_module_index as module_index

def html_report(outdir, modules, excludes=None, errors=None):
    """
    Creates an ``index.html`` in the specified ``outdir``. Also attempts to create
    a ``modules`` subdirectory to create module detail html pages, which are named
    `module.__name__` + '.html'.

    It uses `templates.default_module_index` to create the index page. The template
    contains the following sections which need to be rendered and assembled into
    `index.html`.

    TOP: Contains the HTML declaration and head information, as well as the
         inline stylesheet. It doesn't require any variables.

    CONTENT_HEADER: The header portion of the body. Requires the following variable:
                    * ``%(test_timestamp)s``

    CONTENT_BODY: A table of contents to the different module detail pages, with some
                  basic stats. Requires the following variables:
                  * ``%(module_stats)s`` This is the actual content of the table and is
                    generated by looping through the modules we want to report on and
                    concatenanting together rendered ``MODULE_STAT`` template (see below).
                  * ``%(total_lines)d``
                  * ``%(total_executed)d``
                  * ``%(total_excluded)d``
                  * ``%(overall_covered)0.1f``

    EXCEPTIONS_LINK: Link to the excludes and errors index page which shows
                     packages and modules which were not part of the coverage
                     analysis. Requires the following variable:
                     * ``%(exceptions_link)s`` Link to the index page.
                     * ``%(exceptions_desc)s`` Describe the exception.

    ERRORS_LINK: Link to the errors index page which shows packages and modules which
                 had problems being imported. Requires the following variable:
                 * ``%(errors_link)s`` Link to the index page.

    BOTTOM: Just a closing ````

    MODULE_STAT: Used to assemble the content of ``%(module_stats)s`` for ``CONTENT_BODY``.
                 Requires the following variables:
                 * ``%(severity)s`` (normal, warning, critical) used as CSS class identifier
                   to style the coverage percentage.
                 * ``%(module_link)s``
                 * ``%(module_name)s``
                 * ``%(total_count)d``
                 * ``%(executed_count)d``
                 * ``%(excluded_count)d``
                 * ``%(percent_covered)0.1f``
    """
    # TODO: More robust directory checking and creation
    outdir = os.path.abspath(outdir)
    test_timestamp = time.strftime('%a %Y-%m-%d %H:%M %Z')
    m_subdirname = 'modules'
    m_dir = os.path.join(outdir, m_subdirname)
    try:
        os.mkdir(m_dir)
    except OSError:
        pass

    total_lines = 0
    total_executed = 0
    total_excluded = 0
    total_stmts = 0
    module_stats = list()
    m_names = modules.keys()
    m_names.sort()
    for n in m_names:
        m_vars = ModuleVars(n, modules[n])
        if not m_vars.total_count:
            excludes.append(m_vars.module_name)
            del modules[n]
            continue
        m_vars.module_link = p2url(os.path.join(m_subdirname, m_vars.module_name + '.html'))
        module_stats.append(module_index.MODULE_STAT %m_vars.__dict__)
        total_lines += m_vars.total_count
        total_executed += m_vars.executed_count
        total_excluded += m_vars.excluded_count
        total_stmts += len(m_vars.stmts)
    module_stats = os.linesep.join(module_stats)
    overall_covered = float(total_executed)/total_stmts*100

    m_names = modules.keys()
    m_names.sort()
    i = 0
    for i, n in enumerate(m_names):
        m_vars = ModuleVars(n)
        nav = dict(up_link=p2url(os.path.join('..', 'index.html')),
                   up_label='index')
        if i > 0:
            m = ModuleVars(m_names[i-1])
            nav['prev_link'] = os.path.basename(m.module_link)
            nav['prev_label'] = m.module_name
        if i+1 < len(modules):
            m = ModuleVars(m_names[i+1])
            nav['next_link'] = os.path.basename(m.module_link)
            nav['next_label'] = m.module_name
        html_module_detail(
            os.path.join(m_dir, m_vars.module_name + '.html'), n, nav)

    fo = file(os.path.join(outdir, 'index.html'), 'wb+')
    print >>fo, module_index.TOP
    print >>fo, module_index.CONTENT_HEADER %vars()
    print >>fo, module_index.CONTENT_BODY %vars()
    if excludes:
        _file = 'excludes.html'
        exceptions_link = _file
        exception_desc = "Excluded packages and modules"
        print >>fo, module_index.EXCEPTIONS_LINK %vars()
        html_module_excludes(os.path.join(outdir, _file), excludes)
    if errors:
        _file = 'errors.html'
        exceptions_link = _file
        exception_desc = "Error packages and modules"
        print >>fo, module_index.EXCEPTIONS_LINK %vars()
        html_module_errors(os.path.join(outdir, _file), errors)
    print >>fo, module_index.BOTTOM
    fo.close()

<commit_msg>Caught a division by zero error.<commit_after>"""
Copyright 2009 55 Minutes (http://www.55minutes.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os, time
from urllib import pathname2url as p2url

from data_storage import ModuleVars
from html_module_detail import html_module_detail
from html_module_errors import html_module_errors
from html_module_excludes import html_module_excludes
from templates import default_module_index as module_index

def html_report(outdir, modules, excludes=None, errors=None):
    """
    Creates an ``index.html`` in the specified ``outdir``. Also attempts to create
    a ``modules`` subdirectory to create module detail html pages, which are named
    `module.__name__` + '.html'.

    It uses `templates.default_module_index` to create the index page. The template
    contains the following sections which need to be rendered and assembled into
    `index.html`.

    TOP: Contains the HTML declaration and head information, as well as the
         inline stylesheet. It doesn't require any variables.

    CONTENT_HEADER: The header portion of the body. Requires the following variable:
                    * ``%(test_timestamp)s``

    CONTENT_BODY: A table of contents to the different module detail pages, with some
                  basic stats. Requires the following variables:
                  * ``%(module_stats)s`` This is the actual content of the table and is
                    generated by looping through the modules we want to report on and
                    concatenanting together rendered ``MODULE_STAT`` template (see below).
                  * ``%(total_lines)d``
                  * ``%(total_executed)d``
                  * ``%(total_excluded)d``
                  * ``%(overall_covered)0.1f``

    EXCEPTIONS_LINK: Link to the excludes and errors index page which shows
                     packages and modules which were not part of the coverage
                     analysis. Requires the following variable:
                     * ``%(exceptions_link)s`` Link to the index page.
                     * ``%(exceptions_desc)s`` Describe the exception.

    ERRORS_LINK: Link to the errors index page which shows packages and modules which
                 had problems being imported. Requires the following variable:
                 * ``%(errors_link)s`` Link to the index page.

    BOTTOM: Just a closing ````

    MODULE_STAT: Used to assemble the content of ``%(module_stats)s`` for ``CONTENT_BODY``.
                 Requires the following variables:
                 * ``%(severity)s`` (normal, warning, critical) used as CSS class identifier
                   to style the coverage percentage.
                 * ``%(module_link)s``
                 * ``%(module_name)s``
                 * ``%(total_count)d``
                 * ``%(executed_count)d``
                 * ``%(excluded_count)d``
                 * ``%(percent_covered)0.1f``
    """
    # TODO: More robust directory checking and creation
    outdir = os.path.abspath(outdir)
    test_timestamp = time.strftime('%a %Y-%m-%d %H:%M %Z')
    m_subdirname = 'modules'
    m_dir = os.path.join(outdir, m_subdirname)
    try:
        os.mkdir(m_dir)
    except OSError:
        pass

    total_lines = 0
    total_executed = 0
    total_excluded = 0
    total_stmts = 0
    module_stats = list()
    m_names = modules.keys()
    m_names.sort()
    for n in m_names:
        m_vars = ModuleVars(n, modules[n])
        if not m_vars.total_count:
            excludes.append(m_vars.module_name)
            del modules[n]
            continue
        m_vars.module_link = p2url(os.path.join(m_subdirname, m_vars.module_name + '.html'))
        module_stats.append(module_index.MODULE_STAT %m_vars.__dict__)
        total_lines += m_vars.total_count
        total_executed += m_vars.executed_count
        total_excluded += m_vars.excluded_count
        total_stmts += len(m_vars.stmts)
    module_stats = os.linesep.join(module_stats)
    if total_stmts:
        overall_covered = float(total_executed)/total_stmts*100
    else:
        overall_covered = 0.0

    m_names = modules.keys()
    m_names.sort()
    i = 0
    for i, n in enumerate(m_names):
        m_vars = ModuleVars(n)
        nav = dict(up_link=p2url(os.path.join('..', 'index.html')),
                   up_label='index')
        if i > 0:
            m = ModuleVars(m_names[i-1])
            nav['prev_link'] = os.path.basename(m.module_link)
            nav['prev_label'] = m.module_name
        if i+1 < len(modules):
            m = ModuleVars(m_names[i+1])
            nav['next_link'] = os.path.basename(m.module_link)
            nav['next_label'] = m.module_name
        html_module_detail(
            os.path.join(m_dir, m_vars.module_name + '.html'), n, nav)

    fo = file(os.path.join(outdir, 'index.html'), 'wb+')
    print >>fo, module_index.TOP
    print >>fo, module_index.CONTENT_HEADER %vars()
    print >>fo, module_index.CONTENT_BODY %vars()
    if excludes:
        _file = 'excludes.html'
        exceptions_link = _file
        exception_desc = "Excluded packages and modules"
        print >>fo, module_index.EXCEPTIONS_LINK %vars()
        html_module_excludes(os.path.join(outdir, _file), excludes)
    if errors:
        _file = 'errors.html'
        exceptions_link = _file
        exception_desc = "Error packages and modules"
        print >>fo, module_index.EXCEPTIONS_LINK %vars()
        html_module_errors(os.path.join(outdir, _file), errors)
    print >>fo, module_index.BOTTOM
    fo.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python
from lxml import etree
import sqlite3
import os
import datetime
from shows_hosts_with_open_port_and_service_desc import parse_opts

__program__ = 'python_convert_nmap_xml_to_sqlite_db'
___author__ = 'dave b. <db@d1b.org>'
__license__ = 'GPL v2'

class nmap_xml_to_sqlite:
	def __init__(self, filename, store_p=os.path.expanduser('~/.nmap_pdb/')):
		self.filename = filename
		self.store_p = store_p
		self._db_name = "nmap.db"
		self.conn = None
		self.cursor = None

	def create_store_dir(self):
		""" create the store directory if it doesn't exist """
		if not os.path.exists(self.store_p):
			os.mkdir(self.store_p)

	def connect_to_db(self):
		""" connect to the database """
		self.conn = sqlite3.connect(self.store_p + self._db_name)
		self.cursor = self.conn.cursor()

	def create_db(self):
		""" create the database tables if they don't exist """
		self.cursor.execute("""create table if not exists
		hosts(addr text, hostname text, scan_time datetime,
		unique(addr, hostname, scan_time))""")
		self.cursor.execute("""create table if not exists
		open_port (addr text, port integer, product text,
		protocol text, scan_time datetime, name text,
		servicefp text, version text,
		unique(protocol, port, addr, scan_time))""")
		self.cursor.execute("""create table if not exists scan
		(scan_time datetime, args text, unique (scan_time, args))""")

	def insert_scan_into_db(self, time_of_scan, args):
		""" insert a scan into the database """
		sql_statement = """insert or ignore into scan (scan_time, args) VALUES (?, ?) """
		self.cursor.execute(sql_statement, (time_of_scan, args))

	def insert_host_into_db(self, addr, hostname, time_of_scan):
		""" insert a host into the database """
		sql_statement = """insert or ignore into hosts (addr, hostname, scan_time) VALUES (?, ?, ?) """
		self.cursor.execute(sql_statement, (addr, hostname, time_of_scan))

	def insert_port_into_db(self, addr, protocol, serv_d, time_of_scan):
		""" insert a port into the database """
		sql_statement = """insert or ignore into open_port (addr, port, product, protocol, scan_time,
		name, servicefp, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"""
		self.cursor.execute(sql_statement, (addr, serv_d["portid"], serv_d["product"], \
			protocol, time_of_scan, serv_d["name"], serv_d["servicefp"], serv_d["version"] ))

	def insert_all_scan_info_into_db(self):
		"""
			XXX: make this method cleaner!
			insert every host that has open ports in the nmap xml file and
			a description for it (the port) into the database
		"""
		self._doc = etree.parse(self.filename)
		time_of_scan, args = "", ""
		for x in self._doc.xpath("//nmaprun"):
			time_of_scan = datetime.datetime.fromtimestamp(float(x.attrib['start']))
			args = x.attrib['args']

		self.insert_scan_into_db(time_of_scan, args)

		for x in self._doc.xpath("//host"):
			hostname = "" #this will be the value of the last hostname node's name element
			address = ""
			desc = ""
			protocol = ""
			for host_n in x.xpath("hostnames/hostname/@name"):
				hostname = host_n
			for addr in x.xpath("address/@addr"):
				address = addr
			self.insert_host_into_db(address, hostname, time_of_scan)

			for open_p in x.xpath("ports/port[state[@state='open']]"):
				protocol = open_p.attrib['protocol']
				wrap_service_dict = self._service_wrap_attrib(list(open_p)[1].attrib)
				wrap_service_dict["portid"] = open_p.attrib["portid"]
				self.insert_port_into_db(address, protocol, wrap_service_dict, time_of_scan)

	def _service_wrap_attrib(self, child_attrib):
		""" some fields are optional - so enter a blank value for a key if it doesn't exist  """
		wraped_dict_result = {}
		for key in ["version", "product", "name", "servicefp"]:
			if key in child_attrib.keys():
				wraped_dict_result[key] = child_attrib[key]
			else:
				wraped_dict_result[key] = ""
		return wraped_dict_result

	def close_and_commit_to_db(self):
		""" commit to the database and close the cursor """
		self.conn.commit()
		self.cursor.close()

def main():
	filename = parse_opts()
	s = nmap_xml_to_sqlite(filename)
	s.create_store_dir()
	s.connect_to_db()
	s.create_db()
	s.insert_all_scan_info_into_db()
	s.close_and_commit_to_db()

if __name__ == "__main__":
	main()
<commit_msg>correct the 'wraped' variable name spelling<commit_after>#!/usr/bin/env python
from lxml import etree
import sqlite3
import os
import datetime
from shows_hosts_with_open_port_and_service_desc import parse_opts

__program__ = 'python_convert_nmap_xml_to_sqlite_db'
___author__ = 'dave b. <db@d1b.org>'
__license__ = 'GPL v2'

class nmap_xml_to_sqlite:
	def __init__(self, filename, store_p=os.path.expanduser('~/.nmap_pdb/')):
		self.filename = filename
		self.store_p = store_p
		self._db_name = "nmap.db"
		self.conn = None
		self.cursor = None

	def create_store_dir(self):
		""" create the store directory if it doesn't exist """
		if not os.path.exists(self.store_p):
			os.mkdir(self.store_p)

	def connect_to_db(self):
		""" connect to the database """
		self.conn = sqlite3.connect(self.store_p + self._db_name)
		self.cursor = self.conn.cursor()

	def create_db(self):
		""" create the database tables if they don't exist """
		self.cursor.execute("""create table if not exists
		hosts(addr text, hostname text, scan_time datetime,
		unique(addr, hostname, scan_time))""")
		self.cursor.execute("""create table if not exists
		open_port (addr text, port integer, product text,
		protocol text, scan_time datetime, name text,
		servicefp text, version text,
		unique(protocol, port, addr, scan_time))""")
		self.cursor.execute("""create table if not exists scan
		(scan_time datetime, args text, unique (scan_time, args))""")

	def insert_scan_into_db(self, time_of_scan, args):
		""" insert a scan into the database """
		sql_statement = """insert or ignore into scan (scan_time, args) VALUES (?, ?) """
		self.cursor.execute(sql_statement, (time_of_scan, args))

	def insert_host_into_db(self, addr, hostname, time_of_scan):
		""" insert a host into the database """
		sql_statement = """insert or ignore into hosts (addr, hostname, scan_time) VALUES (?, ?, ?) """
		self.cursor.execute(sql_statement, (addr, hostname, time_of_scan))

	def insert_port_into_db(self, addr, protocol, serv_d, time_of_scan):
		""" insert a port into the database """
		sql_statement = """insert or ignore into open_port (addr, port, product, protocol, scan_time,
		name, servicefp, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"""
		self.cursor.execute(sql_statement, (addr, serv_d["portid"], serv_d["product"], \
			protocol, time_of_scan, serv_d["name"], serv_d["servicefp"], serv_d["version"] ))

	def insert_all_scan_info_into_db(self):
		"""
			XXX: make this method cleaner!
			insert every host that has open ports in the nmap xml file and
			a description for it (the port) into the database
		"""
		self._doc = etree.parse(self.filename)
		time_of_scan, args = "", ""
		for x in self._doc.xpath("//nmaprun"):
			time_of_scan = datetime.datetime.fromtimestamp(float(x.attrib['start']))
			args = x.attrib['args']

		self.insert_scan_into_db(time_of_scan, args)

		for x in self._doc.xpath("//host"):
			hostname = "" #this will be the value of the last hostname node's name element
			address = ""
			desc = ""
			protocol = ""
			for host_n in x.xpath("hostnames/hostname/@name"):
				hostname = host_n
			for addr in x.xpath("address/@addr"):
				address = addr
			self.insert_host_into_db(address, hostname, time_of_scan)

			for open_p in x.xpath("ports/port[state[@state='open']]"):
				protocol = open_p.attrib['protocol']
				wrap_service_dict = self._service_wrap_attrib(list(open_p)[1].attrib)
				wrap_service_dict["portid"] = open_p.attrib["portid"]
				self.insert_port_into_db(address, protocol, wrap_service_dict, time_of_scan)

	def _service_wrap_attrib(self, child_attrib):
		""" some fields are optional - so enter a blank value for a key if it doesn't exist  """
		wrapped_dict_result = {}
		for key in ["version", "product", "name", "servicefp"]:
			if key in child_attrib.keys():
				wrapped_dict_result[key] = child_attrib[key]
			else:
				wrapped_dict_result[key] = ""
		return wrapped_dict_result

	def close_and_commit_to_db(self):
		""" commit to the database and close the cursor """
		self.conn.commit()
		self.cursor.close()

def main():
	filename = parse_opts()
	s = nmap_xml_to_sqlite(filename)
	s.create_store_dir()
	s.connect_to_db()
	s.create_db()
	s.insert_all_scan_info_into_db()
	s.close_and_commit_to_db()

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>from movielens import *

import numpy as np
import pickle
import easygui
import random
import os.path

NO_OF_RECOMMENDATIONS = 10

def load_from_dataset(utility_matrix):
    user = []
    item = []
    ratings = []
    d = Dataset()
    d.load_users("data/u.user", user)
    d.load_items("data/u.item", item)
    d.load_ratings("data/u.base", ratings)
    movie_genre = []
    for movie in item:
        movie_genre.append([movie.unknown, movie.action, movie.adventure, movie.animation, movie.childrens, movie.comedy,
                            movie.crime, movie.documentary, movie.drama, movie.fantasy, movie.film_noir, movie.horror,
                            movie.musical, movie.mystery, movie.romance, movie.sci_fi, movie.thriller, movie.war, movie.western])
    movie_genre = np.array(movie_genre)

    # Find the average rating for each user and stores it in the user's object
    for i in range(0, len(user)):
        x = utility_matrix[i]
        user[i].avg_r = sum(a for a in x if a > 0) / sum(a > 0 for a in x)

    return user, item, movie_genre, ratings

def pcs(x, y, ut, user):
    num = 0
    den1 = 0
    den2 = 0
    A = ut[x - 1]
    B = ut[y - 1]
    num = sum((a - user[x - 1].avg_r) * (b - user[y - 1].avg_r) for a, b in zip(A, B) if a > 0 and b > 0)
    den1 = sum((a - user[x - 1].avg_r) ** 2 for a in A if a > 0)
    den2 = sum((b - user[y - 1].avg_r) ** 2 for b in B if b > 0)
    den = (den1 ** 0.5) * (den2 ** 0.5)
    if den == 0:
        return 0
    else:
        return num / den

def pop_movies(movies, movie_clusters):

	top_movies = []

	with open("avg_rating.pkl", "r") as fp:
		avg_rating = pickle.load(fp)

	for movie, rating, cluster in zip(movies, avg_rating, movie_clusters.labels_):

		# Threshold for average is low due to sparse dataset
		if rating > 0.1:
			top_movies.append(movie)
	return random.sample(top_movies, 100)

def update_recommendations(stored_rating, new_rating, alpha):
	# Update the average rating to include information abour latest preference
    # Alpha determines relative importance
	updated_rating = (alpha-1) * stored_rating + alpha * new_rating
	return updated_rating

def recomm_main(utility_matrix, avg_ratings, demographics, pcs_matrix):
    user, item, movie_genre, ratings = load_from_dataset(utility_matrix)
    n_users = len(user)
    n_items = len(item)

    with open("cluster.pkl", "r") as fp:
        cluster = pickle.load(fp)

    ask = pop_movies(item, cluster)

    print "Please rate the following movies (1-5):\nFill in 0 if you have not seen it:"
    k=0
    for movie in ask:
        print movie.title + ": "
        a = int(input())
        if a==0:
            continue
        if avg_ratings[cluster.labels_[movie.id - 1]] != 0:
            # avg_ratings[cluster.labels_[movie.id - 1]] = (avg_ratings[cluster.labels_[movie.id - 1]] + a) / 2
            avg_ratings[cluster.labels_[movie.id - 1]] = update_recommendations(avg_ratings[cluster.labels_[movie.id - 1]], a, 0.5)
        else:
            avg_ratings[cluster.labels_[movie.id - 1]] = a
        k = k+1
        if k == 10:
            break

    utility_new = np.vstack((utility_matrix, avg_ratings))

    user.append(demographics)

    print "Finding users which have similar preferences."
    for i in range(0, n_users + 1):
        if i != 943:
            pcs_matrix[i] = pcs(944, i + 1, utility_new, user)
    user_index = []
    for i in user:
        user_index.append(i.id - 1)
    user_index = user_index[:943]
    user_index = np.array(user_index)

    top_similar = [x for (y, x) in sorted(zip(pcs_matrix, user_index), key=lambda pair: pair[0], reverse=True)]
    top_5 = top_similar[:5]

    top_5_cluster = []

    for i in range(0, 5):
        maxi = 0
        maxe = 0
        for j in range(0, 19):
            if maxe < utility_matrix[top_5[i]][j]:
                maxe = utility_matrix[top_5[i]][j]
                maxi = j
        top_5_cluster.append(maxi)
    #print top_5_cluster

    res = {}
    for i in range(len(top_5_cluster)):
        if top_5_cluster[i] not in res.keys():
            res[top_5_cluster[i]] = len(top_5_cluster) - i
        else:
            res[top_5_cluster[i]] += len(top_5_cluster) - i

    top_cluster = res.keys()[0]
    movies_in_top_cluster = []

    for i in item:
        if cluster.labels_[i.id - 1] == top_cluster:
            movies_in_top_cluster.append(i)

    movie_dict = {movie.id: [0, 0, 0, 0, 0] for movie in movies_in_top_cluster}
    for movie in movies_in_top_cluster:
        for j in ratings:
            if j.user_id in top_5 and j.item_id == movie.id:
                movie_dict[movie.id][j.rating - 1] += 1

    recommended_movies = None
    movie_sums = []
    for movie in movie_dict:
        total = 0
        for i, j in zip(range(0, NO_OF_RECOMMENDATIONS), movie_dict[movie]):
            total += i * j
        movie_sums.append(total)
    recommended_movies = sorted(zip(movie_dict.keys(), movie_sums), key=lambda x: x[1], reverse=True)# print recommended_movies[:5]
    final_recommendations=[]
    for i in item:
        if i.id in [recommended_movies[k][0] for k in range(NO_OF_RECOMMENDATIONS)]:
            final_recommendations.append(i)
            print i.title
    return (avg_ratings, demographics, pcs_matrix, final_recommendations)

def rate_recomm(utility_matrix, avg_ratings, demographics, pcs_matrix, recommendations):
    with open("cluster.pkl", "r") as fp:
        cluster = pickle.load(fp)
    user, item, movie_genre, ratings = load_from_dataset(utility_matrix)
    user.append(demographics)
    n_users = len(user)
    n_items = len(item)
    c = 0
    for i in recommendations:
        print i.title
        r = input("Enter your rating\n")
        if r>3.5:
            c+=1

        avg_ratings[cluster.labels_[i.id - 1]] = update_recommendations(avg_rating[cluster.labels_[i.id - 1]], r, 0.5)

    print "Precision of predictions : ",c/5.0
    for i in range(0, n_users):
        if i!=943:
            utility_new = np.vstack((utility_matrix, avg_ratings))
            pcs_matrix[i] = pcs(944, i + 1, utility_new, user)
    return avg_ratings, pcs_matrix

def UI_main():
    username=raw_input("\nEnter username\n")
    if os.path.exists(username + ".pkl"):
        print "Old User"
        flag=True
        o=pickle.load(open(username+".pkl", "rb"))
        avg_ratings=o.avg_ratings
        demographics=o.demographics
        pcs_matrix=o.pcs
        recommendations=o.recommendations
    else:
        print "New User"
        flag=False
        age = input("Enter your age: ")
        gender = raw_input("M/F: ")
        pin = input("Enter pincode: ")
        profession = easygui.choicebox(msg="Select your profession",choices=["administrator",
"artist",
"doctor",
"educator",
"engineer",
"entertainment",
"executive",
"healthcare",
"homemaker",
"lawyer",
"librarian",
"marketing",
"none",
"other",
"programmer",
"retired",
"salesman",
"scientist",
"student",
"technician",
"writer",]
)
        avg_ratings = np.zeros(19)
        demographics= User(944, age,  profession, pin, gender)
        pcs_matrix = np.zeros(943)
        recommendations=[]
    utility_matrix = pickle.load(open("utility_matrix.pkl", "rb"))
    while(True):
        ch=input("\nEnter:\n 1 for getting recommendations\n 2 for rating past recommendations\n 3 to exit\n")
        if int(ch)==1:
            avg_ratings, demographics, pcs_matrix, recommendations = recomm_main(utility_matrix, avg_ratings, demographics,pcs_matrix)
        elif int(ch)==2:
            avg_ratings, pcs_matrix = rate_recomm(utility_matrix, avg_ratings, demographics, pcs_matrix, recommendations)
        else:
            break

    pickle.dump(NewUser(username, avg_ratings, demographics, pcs_matrix, recommendations), open(username+".pkl", "wb"))

if __name__=="__main__":
    UI_main()
<commit_msg>Add reciprocal rank<commit_after>from movielens import *

import numpy as np
import pickle
import easygui
import random
import os.path

NO_OF_RECOMMENDATIONS = 10
RECIPROCAL_RANKING_THRESHOLD = 3

def load_from_dataset(utility_matrix):
    user = []
    item = []
    ratings = []
    d = Dataset()
    d.load_users("data/u.user", user)
    d.load_items("data/u.item", item)
    d.load_ratings("data/u.base", ratings)
    movie_genre = []
    for movie in item:
        movie_genre.append([movie.unknown, movie.action, movie.adventure, movie.animation, movie.childrens, movie.comedy,
                            movie.crime, movie.documentary, movie.drama, movie.fantasy, movie.film_noir, movie.horror,
                            movie.musical, movie.mystery, movie.romance, movie.sci_fi, movie.thriller, movie.war, movie.western])
    movie_genre = np.array(movie_genre)

    # Find the average rating for each user and stores it in the user's object
    for i in range(0, len(user)):
        x = utility_matrix[i]
        user[i].avg_r = sum(a for a in x if a > 0) / sum(a > 0 for a in x)

    return user, item, movie_genre, ratings

def pcs(x, y, ut, user):
    num = 0
    den1 = 0
    den2 = 0
    A = ut[x - 1]
    B = ut[y - 1]
    num = sum((a - user[x - 1].avg_r) * (b - user[y - 1].avg_r) for a, b in zip(A, B) if a > 0 and b > 0)
    den1 = sum((a - user[x - 1].avg_r) ** 2 for a in A if a > 0)
    den2 = sum((b - user[y - 1].avg_r) ** 2 for b in B if b > 0)
    den = (den1 ** 0.5) * (den2 ** 0.5)
    if den == 0:
        return 0
    else:
        return num / den

def pop_movies(movies, movie_clusters):

	top_movies = []

	with open("avg_rating.pkl", "r") as fp:
		avg_rating = pickle.load(fp)

	for movie, rating, cluster in zip(movies, avg_rating, movie_clusters.labels_):

		# Threshold for average is low due to sparse dataset
		if rating > 0.1:
			top_movies.append(movie)
	return random.sample(top_movies, 100)

def update_recommendations(stored_rating, new_rating, alpha):
	# Update the average rating to include information abour latest preference
    # Alpha determines relative importance
	updated_rating = (alpha-1) * stored_rating + alpha * new_rating
	return updated_rating

def recomm_main(utility_matrix, avg_ratings, demographics, pcs_matrix):
    user, item, movie_genre, ratings = load_from_dataset(utility_matrix)
    n_users = len(user)
    n_items = len(item)

    with open("cluster.pkl", "r") as fp:
        cluster = pickle.load(fp)

    ask = pop_movies(item, cluster)

    print "Please rate the following movies (1-5):\nFill in 0 if you have not seen it:"
    k=0
    for movie in ask:
        print movie.title + ": "
        a = int(input())
        if a==0:
            continue
        if avg_ratings[cluster.labels_[movie.id - 1]] != 0:
            # avg_ratings[cluster.labels_[movie.id - 1]] = (avg_ratings[cluster.labels_[movie.id - 1]] + a) / 2
            avg_ratings[cluster.labels_[movie.id - 1]] = update_recommendations(avg_ratings[cluster.labels_[movie.id - 1]], a, 0.5)
        else:
            avg_ratings[cluster.labels_[movie.id - 1]] = a
        k = k+1
        if k == 10:
            break

    utility_new = np.vstack((utility_matrix, avg_ratings))

    user.append(demographics)

    print "Finding users which have similar preferences."
    for i in range(0, n_users + 1):
        if i != 943:
            pcs_matrix[i] = pcs(944, i + 1, utility_new, user)
    user_index = []
    for i in user:
        user_index.append(i.id - 1)
    user_index = user_index[:943]
    user_index = np.array(user_index)

    top_similar = [x for (y, x) in sorted(zip(pcs_matrix, user_index), key=lambda pair: pair[0], reverse=True)]
    top_5 = top_similar[:5]

    top_5_cluster = []

    for i in range(0, 5):
        maxi = 0
        maxe = 0
        for j in range(0, 19):
            if maxe < utility_matrix[top_5[i]][j]:
                maxe = utility_matrix[top_5[i]][j]
                maxi = j
        top_5_cluster.append(maxi)
    #print top_5_cluster

    res = {}
    for i in range(len(top_5_cluster)):
        if top_5_cluster[i] not in res.keys():
            res[top_5_cluster[i]] = len(top_5_cluster) - i
        else:
            res[top_5_cluster[i]] += len(top_5_cluster) - i

    top_cluster = res.keys()[0]
    movies_in_top_cluster = []

    for i in item:
        if cluster.labels_[i.id - 1] == top_cluster:
            movies_in_top_cluster.append(i)

    movie_dict = {movie.id: [0, 0, 0, 0, 0] for movie in movies_in_top_cluster}
    for movie in movies_in_top_cluster:
        for j in ratings:
            if j.user_id in top_5 and j.item_id == movie.id:
                movie_dict[movie.id][j.rating - 1] += 1

    recommended_movies = None
    movie_sums = []
    for movie in movie_dict:
        total = 0
        for i, j in zip(range(0, NO_OF_RECOMMENDATIONS), movie_dict[movie]):
            total += i * j
        movie_sums.append(total)
    recommended_movies = sorted(zip(movie_dict.keys(), movie_sums), key=lambda x: x[1], reverse=True)# print recommended_movies[:5]
    final_recommendations=[]
    for i in item:
        if i.id in [recommended_movies[k][0] for k in range(NO_OF_RECOMMENDATIONS)]:
            final_recommendations.append(i)
            print i.title
    return (avg_ratings, demographics, pcs_matrix, final_recommendations)

def rate_recomm(utility_matrix, avg_ratings, demographics, pcs_matrix, recommendations):
    with open("cluster.pkl", "r") as fp:
        cluster = pickle.load(fp)
    user, item, movie_genre, ratings = load_from_dataset(utility_matrix)
    user.append(demographics)
    n_users = len(user)
    n_items = len(item)
    c = 0
    rel_idx = -1
    idx = 1
    pending_recom=[]
    for i in recommendations:
        print i.title
        r = input("Enter your rating, if you've not seen the movie provide 0\n")
        if r==0:
            pending_recom.append(i)
            continue
        if r>3.5:
            c+=1
            if rel_idx == -1:
                rel_idx = idx
        avg_ratings[cluster.labels_[i.id - 1]] = (avg_ratings[cluster.labels_[i.id - 1]] + r) / 2
        idx += 1
    reciprocal_ranking = 0

    if rel_idx <= RECIPROCAL_RANKING_THRESHOLD and rel_idx != -1:
        reciprocal_ranking = 1.0/(rel_idx)

    print "Precision of predictions : ",c/float(len(recommendations))
    print "Reciprocal Ranking : ", reciprocal_ranking
    for i in range(0, n_users):
        if i!=943:
            utility_new = np.vstack((utility_matrix, avg_ratings))
            pcs_matrix[i] = pcs(944, i + 1, utility_new, user)
    return avg_ratings, pcs_matrix, pending_recom

def UI_main():
    username=raw_input("\nEnter username\n")
    if os.path.exists(username + ".pkl"):
        print "Old User"
        flag=True
        o=pickle.load(open(username+".pkl", "rb"))
        avg_ratings=o.avg_ratings
        demographics=o.demographics
        pcs_matrix=o.pcs
        recommendations=o.recommendations
    else:
        print "New User"
        flag=False
        age = input("Enter your age: ")
        gender = raw_input("M/F: ")
        pin = input("Enter pincode: ")
        profession = easygui.choicebox(msg="Select your profession",choices=["administrator",
"artist",
"doctor",
"educator",
"engineer",
"entertainment",
"executive",
"healthcare",
"homemaker",
"lawyer",
"librarian",
"marketing",
"none",
"other",
"programmer",
"retired",
"salesman",
"scientist",
"student",
"technician",
"writer",]
)
        avg_ratings = np.zeros(19)
        demographics= User(944, age,  profession, pin, gender)
        pcs_matrix = np.zeros(943)
        recommendations=[]
    utility_matrix = pickle.load(open("utility_matrix.pkl", "rb"))
    while(True):
        ch=input("\nEnter:\n 1 for getting recommendations\n 2 for rating past recommendations\n 3 to exit\n")
        if int(ch)==1:
            avg_ratings, demographics, pcs_matrix, recommendations = recomm_main(utility_matrix, avg_ratings, demographics,pcs_matrix)
        elif int(ch)==2:
            if recommendations:
                # avg_ratings, pcs_matrix = rate_recomm(utility_matrix, avg_ratings, demographics, pcs_matrix, recommendations)
                avg_ratings, pcs_matrix, recommendations = rate_recomm(utility_matrix, avg_ratings, demographics, pcs_matrix, recommendations)

            else:
                print "Please get more recommendations"
        else:
            break

    pickle.dump(NewUser(username, avg_ratings, demographics, pcs_matrix, recommendations), open(username+".pkl", "wb"))

if __name__=="__main__":
    UI_main()
<|endoftext|>