<commit_before>from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import irc_lower

class NickServAlias(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], " ".join(data["params"])])

class ChanServAlias(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], " ".join(data["params"])])

class BidServAlias(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], " ".join(data["params"])])

class NSIdentifyCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "IDENTIFY {}".format(" ".join(data["params"]))])

class NSGhostCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "GHOST {}".format(" ".join(data["params"]))])

class NSLoginCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "LOGIN {}".format(" ".join(data["params"]))])

class NSLogoutCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "LOGOUT {}".format(" ".join(data["params"]))])

class NSDropCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "DROP {}".format(" ".join(data["params"]))])

class NSNicklistCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "NICKLIST {}".format(" ".join(data["params"]))])

class NSAccountCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "ACCOUNT {}".format(" ".join(data["params"]))])

class NSCertCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "CERT {}".format(" ".join(data["params"]))])

class CSRegisterCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], "REGISTER {}".format(" ".join(data["params"]))])

class CSAccessCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], "ACCESS {}".format(" ".join(data["params"]))])

class CSCdropCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], "CDROP {}".format(" ".join(data["params"]))])

class BSStartCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "START {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSStopCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "STOP {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSBidCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "BID {}".format(" ".join(data["params"]))])

class BSRevertCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "REVERT {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSOnceCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "ONCE {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSTwiceCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "TWICE {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSSoldCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "SOLD {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSHighbidderCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "HIGHBIDDER {}".format(" ".join(data["params"]))])

class BSCurrentAuctionCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "CURRENTAUCTION {}".format(" ".join(data["params"]))])

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd
        self.serviceServer = None
        self.blockedUsers = set()

    def spawn(self):
        if "services_nickserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_nickserv_nick"] = "NickServ"
        if "services_chanserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_chanserv_nick"] = "ChanServ"
        if "services_bidserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_bidserv_nick"] = "BidServ"
        return {
            "commands": {
                "NICKSERV": NickServAlias(),
                "NS": NickServAlias(),
                "CHANSERV": ChanServAlias(),
                "CS": ChanServAlias(),
                "BIDSERV": BidServAlias(),
                "BS": BidServAlias(),

                "IDENTIFY": NSIdentifyCommand(),
                "ID": NSIdentifyCommand(),
                "GHOST": NSGhostCommand(),
                "LOGIN": NSLoginCommand(),
                "LOGOUT": NSLogoutCommand(),
                "DROP": NSDropCommand(),
                "NICKLIST": NSNicklistCommand(),
                "ACCOUNT": NSAccountCommand(),
                "CERT": NSCertCommand(),

                "REGISTER": CSRegisterCommand(),
                "ACCESS": CSAccessCommand(),
                "CDROP": CSCdropCommand(),

                "START": BSStartCommand(),
                "STOP": BSStopCommand(),
                "BID": BSBidCommand(),
                "REVERT": BSRevertCommand(),
                "ONCE": BSOnceCommand(),
                "TWICE": BSTwiceCommand(),
                "SOLD": BSSoldCommand(),
                "HIGHBIDDER": BSHighbidderCommand(),
                "CURRENTAUCTION": BSCurrentAuctionCommand()
            },
            "actions": {
                "commandpermission": self.commandPermission,
                "netsplit": self.onNetsplit
            },
            "server": {
                "ServiceServer": self.noteServer,
                "ServiceBlockUser": self.addBlock,
                "ServiceUnblockUser": self.removeBlock
            }
        }

    def commandPermission(self, user, cmd, data):
        if cmd == "NICK" and data["nick"] in [ self.ircd.servconfig["services_nickserv_nick"], self.ircd.servconfig["services_chanserv_nick"], self.ircd.servconfig["services_bidserv_nick"] ]:
            user.sendMessage(irc.ERR_ERRONEUSNICKNAME, data["nick"], ":Invalid nickname: Reserved for Services")
            return {}
        if self.ircd.servconfig["services_nickserv_nick"] not in self.ircd.users:
            return data
        nickserv = self.ircd.users[self.ircd.servconfig["services_nickserv_nick"]]
        if user not in self.blockedUsers:
            return data
        if cmd == "PRIVMSG":
            to_nickserv = False
            for u in data["targetuser"]:
                if irc_lower(u.nickname) == irc_lower(nickserv.nickname):
                    to_nickserv = True
                    break
            if to_nickserv:
                data["targetuser"] = [nickserv]
                data["targetchan"] = []
                data["chanmod"] = []
                return data
            user.sendMessage("NOTICE", ":You cannot message anyone other than NickServ until you identify or change nicks.", prefix=nickserv.prefix())
            return {}
        if cmd in [ "PING", "PONG", "NICK", "QUIT", "NS", "NICKSERV", "LOGIN", "ID", "IDENTIFY" ]:
            return data
        user.sendMessage("NOTICE", ":You cannot use the command \x02{}\x02 until you identify or change nicks.".format(cmd), prefix=nickserv.prefix())
        return {}

    def onNetsplit(self, name):
        if name == self.serviceServer:
            self.blockedUsers = set()
            self.serviceServer = None

    def noteServer(self, command, args):
        self.serviceServer = args[0]

    def addBlock(self, command, args):
        if args[0] not in self.ircd.userid:
            return
        self.blockedUsers.add(self.ircd.userid[args[0]])

    def removeBlock(self, command, args):
        if args[0] not in self.ircd.userid:
            return
        self.blockedUsers.discard(self.ircd.userid[args[0]])<commit_msg>Add operserv config option to db_services_leaf<commit_after>from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import irc_lower

class NickServAlias(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], " ".join(data["params"])])

class ChanServAlias(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], " ".join(data["params"])])

class BidServAlias(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], " ".join(data["params"])])

class NSIdentifyCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "IDENTIFY {}".format(" ".join(data["params"]))])

class NSGhostCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "GHOST {}".format(" ".join(data["params"]))])

class NSLoginCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "LOGIN {}".format(" ".join(data["params"]))])

class NSLogoutCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "LOGOUT {}".format(" ".join(data["params"]))])

class NSDropCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "DROP {}".format(" ".join(data["params"]))])

class NSNicklistCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "NICKLIST {}".format(" ".join(data["params"]))])

class NSAccountCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "ACCOUNT {}".format(" ".join(data["params"]))])

class NSCertCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_nickserv_nick"], "CERT {}".format(" ".join(data["params"]))])

class CSRegisterCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], "REGISTER {}".format(" ".join(data["params"]))])

class CSAccessCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], "ACCESS {}".format(" ".join(data["params"]))])

class CSCdropCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_chanserv_nick"], "CDROP {}".format(" ".join(data["params"]))])

class BSStartCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "START {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSStopCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "STOP {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSBidCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "BID {}".format(" ".join(data["params"]))])

class BSRevertCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "REVERT {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSOnceCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "ONCE {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSTwiceCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "TWICE {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSSoldCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "SOLD {}".format(" ".join(data["params"]))])

    def processParams(self, user, params):
        if "o" not in user.mode:
            user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
            return {}
        return {
            "user": user,
            "params": params
        }

class BSHighbidderCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "HIGHBIDDER {}".format(" ".join(data["params"]))])

class BSCurrentAuctionCommand(Command):
    def onUse(self, user, data):
        user.handleCommand("PRIVMSG", None, [self.ircd.servconfig["services_bidserv_nick"], "CURRENTAUCTION {}".format(" ".join(data["params"]))])

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd
        self.serviceServer = None
        self.blockedUsers = set()

    def spawn(self):
        if "services_nickserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_nickserv_nick"] = "NickServ"
        if "services_chanserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_chanserv_nick"] = "ChanServ"
        if "services_bidserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_bidserv_nick"] = "BidServ"
        if "services_operserv_nick" not in self.ircd.servconfig:
            self.ircd.servconfig["services_operserv_nick"] = "OperServ"
        return {
            "commands": {
                "NICKSERV": NickServAlias(),
                "NS": NickServAlias(),
                "CHANSERV": ChanServAlias(),
                "CS": ChanServAlias(),
                "BIDSERV": BidServAlias(),
                "BS": BidServAlias(),

                "IDENTIFY": NSIdentifyCommand(),
                "ID": NSIdentifyCommand(),
                "GHOST": NSGhostCommand(),
                "LOGIN": NSLoginCommand(),
                "LOGOUT": NSLogoutCommand(),
                "DROP": NSDropCommand(),
                "NICKLIST": NSNicklistCommand(),
                "ACCOUNT": NSAccountCommand(),
                "CERT": NSCertCommand(),

                "REGISTER": CSRegisterCommand(),
                "ACCESS": CSAccessCommand(),
                "CDROP": CSCdropCommand(),

                "START": BSStartCommand(),
                "STOP": BSStopCommand(),
                "BID": BSBidCommand(),
                "REVERT": BSRevertCommand(),
                "ONCE": BSOnceCommand(),
                "TWICE": BSTwiceCommand(),
                "SOLD": BSSoldCommand(),
                "HIGHBIDDER": BSHighbidderCommand(),
                "CURRENTAUCTION": BSCurrentAuctionCommand()
            },
            "actions": {
                "commandpermission": self.commandPermission,
                "netsplit": self.onNetsplit
            },
            "server": {
                "ServiceServer": self.noteServer,
                "ServiceBlockUser": self.addBlock,
                "ServiceUnblockUser": self.removeBlock
            }
        }

    def commandPermission(self, user, cmd, data):
        if cmd == "NICK" and data["nick"] in [ self.ircd.servconfig["services_nickserv_nick"], self.ircd.servconfig["services_chanserv_nick"], self.ircd.servconfig["services_bidserv_nick"] ]:
            user.sendMessage(irc.ERR_ERRONEUSNICKNAME, data["nick"], ":Invalid nickname: Reserved for Services")
            return {}
        if self.ircd.servconfig["services_nickserv_nick"] not in self.ircd.users:
            return data
        nickserv = self.ircd.users[self.ircd.servconfig["services_nickserv_nick"]]
        if user not in self.blockedUsers:
            return data
        if cmd == "PRIVMSG":
            to_nickserv = False
            for u in data["targetuser"]:
                if irc_lower(u.nickname) == irc_lower(nickserv.nickname):
                    to_nickserv = True
                    break
            if to_nickserv:
                data["targetuser"] = [nickserv]
                data["targetchan"] = []
                data["chanmod"] = []
                return data
            user.sendMessage("NOTICE", ":You cannot message anyone other than NickServ until you identify or change nicks.", prefix=nickserv.prefix())
            return {}
        if cmd in [ "PING", "PONG", "NICK", "QUIT", "NS", "NICKSERV", "LOGIN", "ID", "IDENTIFY" ]:
            return data
        user.sendMessage("NOTICE", ":You cannot use the command \x02{}\x02 until you identify or change nicks.".format(cmd), prefix=nickserv.prefix())
        return {}

    def onNetsplit(self, name):
        if name == self.serviceServer:
            self.blockedUsers = set()
            self.serviceServer = None

    def noteServer(self, command, args):
        self.serviceServer = args[0]

    def addBlock(self, command, args):
        if args[0] not in self.ircd.userid:
            return
        self.blockedUsers.add(self.ircd.userid[args[0]])

    def removeBlock(self, command, args):
        if args[0] not in self.ircd.userid:
            return
        self.blockedUsers.discard(self.ircd.userid[args[0]])<|endoftext|>


<commit_before># Copyright (C) 2020 FireEye, Inc. All Rights Reserved.

import collections

import six
import tabulate

import capa.render.utils as rutils

tabulate.PRESERVE_WHITESPACE = True

def width(s, character_count):
    """pad the given string to at least `character_count`"""
    if len(s) < character_count:
        return s + " " * (character_count - len(s))
    else:
        return s

def render_meta(doc, ostream):
    rows = [
        (width("md5", 22), width(doc["meta"]["sample"]["md5"], 82)),
        ("path", doc["meta"]["sample"]["path"]),
    ]

    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
    ostream.write("\n")

def render_capabilities(doc, ostream):
    """
    example::

        +-------------------------------------------------------+-------------------------------------------------+
        | CAPABILITY                                            | NAMESPACE                                       |
        |-------------------------------------------------------+-------------------------------------------------|
        | check for OutputDebugString error (2 matches)         | anti-analysis/anti-debugging/debugger-detection |
        | read and send data from client to server              | c2/file-transfer                                |
        | ...                                                   | ...                                             |
        +-------------------------------------------------------+-------------------------------------------------+
    """
    rows = []
    for rule in rutils.capability_rules(doc):
        count = len(rule["matches"])
        if count == 1:
            capability = rutils.bold(rule["meta"]["name"])
        else:
            capability = "%s (%d matches)" % (rutils.bold(rule["meta"]["name"]), count)
        rows.append((capability, rule["meta"]["namespace"]))

    if rows:
        ostream.write(
            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
        )
        ostream.write("\n")
    else:
        ostream.writeln(rutils.bold("no capabilities found"))

def render_attack(doc, ostream):
    """
    example::

        +------------------------+----------------------------------------------------------------------+
        | ATT&CK Tactic          | ATT&CK Technique                                                     |
        |------------------------+----------------------------------------------------------------------|
        | DEFENSE EVASION        | Obfuscated Files or Information [T1027]                              |
        | DISCOVERY              | Query Registry [T1012]                                               |
        |                        | System Information Discovery [T1082]                                 |
        | EXECUTION              | Command and Scripting Interpreter::Windows Command Shell [T1059.003] |
        |                        | Shared Modules [T1129]                                               |
        | EXFILTRATION           | Exfiltration Over C2 Channel [T1041]                                 |
        | PERSISTENCE            | Create or Modify System Process::Windows Service [T1543.003]         |
        +------------------------+----------------------------------------------------------------------+
    """
    tactics = collections.defaultdict(set)
    for rule in rutils.capability_rules(doc):
        if not rule["meta"].get("att&ck"):
            continue

        for attack in rule["meta"]["att&ck"]:
            tactic, _, rest = attack.partition("::")
            if "::" in rest:
                technique, _, rest = rest.partition("::")
                subtechnique, _, id = rest.rpartition(" ")
                tactics[tactic].add((technique, subtechnique, id))
            else:
                technique, _, id = rest.rpartition(" ")
                tactics[tactic].add((technique, id))

    rows = []
    for tactic, techniques in sorted(tactics.items()):
        inner_rows = []
        for spec in sorted(techniques):
            if len(spec) == 2:
                technique, id = spec
                inner_rows.append("%s %s" % (rutils.bold(technique), id))
            elif len(spec) == 3:
                technique, subtechnique, id = spec
                inner_rows.append("%s::%s %s" % (rutils.bold(technique), subtechnique, id))
            else:
                raise RuntimeError("unexpected ATT&CK spec format")
        rows.append((rutils.bold(tactic.upper()), "\n".join(inner_rows),))

    if rows:
        ostream.write(
            tabulate.tabulate(
                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
            )
        )
        ostream.write("\n")

def render_default(doc):
    ostream = rutils.StringIO()

    render_meta(doc, ostream)
    ostream.write("\n")
    render_attack(doc, ostream)
    ostream.write("\n")
    render_capabilities(doc, ostream)

    return ostream.getvalue()
<commit_msg>Update default.py<commit_after># Copyright (C) 2020 FireEye, Inc. All Rights Reserved.

import collections

import six
import tabulate

import capa.render.utils as rutils

tabulate.PRESERVE_WHITESPACE = True

def width(s, character_count):
    """pad the given string to at least `character_count`"""
    if len(s) < character_count:
        return s + " " * (character_count - len(s))
    else:
        return s

def render_meta(doc, ostream):
    rows = [
        (width("md5", 22), width(doc["meta"]["sample"]["md5"], 82)),
        (width("sha1", 22), width(doc["meta"]["sample"]["sha1"], 82)),
        (width("sha256", 22), width(doc["meta"]["sample"]["sha256"], 82)),
        ("path", doc["meta"]["sample"]["path"]),
    ]

    ostream.write(tabulate.tabulate(rows, tablefmt="psql"))
    ostream.write("\n")

def render_capabilities(doc, ostream):
    """
    example::

        +-------------------------------------------------------+-------------------------------------------------+
        | CAPABILITY                                            | NAMESPACE                                       |
        |-------------------------------------------------------+-------------------------------------------------|
        | check for OutputDebugString error (2 matches)         | anti-analysis/anti-debugging/debugger-detection |
        | read and send data from client to server              | c2/file-transfer                                |
        | ...                                                   | ...                                             |
        +-------------------------------------------------------+-------------------------------------------------+
    """
    rows = []
    for rule in rutils.capability_rules(doc):
        count = len(rule["matches"])
        if count == 1:
            capability = rutils.bold(rule["meta"]["name"])
        else:
            capability = "%s (%d matches)" % (rutils.bold(rule["meta"]["name"]), count)
        rows.append((capability, rule["meta"]["namespace"]))

    if rows:
        ostream.write(
            tabulate.tabulate(rows, headers=[width("CAPABILITY", 50), width("NAMESPACE", 50)], tablefmt="psql")
        )
        ostream.write("\n")
    else:
        ostream.writeln(rutils.bold("no capabilities found"))

def render_attack(doc, ostream):
    """
    example::

        +------------------------+----------------------------------------------------------------------+
        | ATT&CK Tactic          | ATT&CK Technique                                                     |
        |------------------------+----------------------------------------------------------------------|
        | DEFENSE EVASION        | Obfuscated Files or Information [T1027]                              |
        | DISCOVERY              | Query Registry [T1012]                                               |
        |                        | System Information Discovery [T1082]                                 |
        | EXECUTION              | Command and Scripting Interpreter::Windows Command Shell [T1059.003] |
        |                        | Shared Modules [T1129]                                               |
        | EXFILTRATION           | Exfiltration Over C2 Channel [T1041]                                 |
        | PERSISTENCE            | Create or Modify System Process::Windows Service [T1543.003]         |
        +------------------------+----------------------------------------------------------------------+
    """
    tactics = collections.defaultdict(set)
    for rule in rutils.capability_rules(doc):
        if not rule["meta"].get("att&ck"):
            continue

        for attack in rule["meta"]["att&ck"]:
            tactic, _, rest = attack.partition("::")
            if "::" in rest:
                technique, _, rest = rest.partition("::")
                subtechnique, _, id = rest.rpartition(" ")
                tactics[tactic].add((technique, subtechnique, id))
            else:
                technique, _, id = rest.rpartition(" ")
                tactics[tactic].add((technique, id))

    rows = []
    for tactic, techniques in sorted(tactics.items()):
        inner_rows = []
        for spec in sorted(techniques):
            if len(spec) == 2:
                technique, id = spec
                inner_rows.append("%s %s" % (rutils.bold(technique), id))
            elif len(spec) == 3:
                technique, subtechnique, id = spec
                inner_rows.append("%s::%s %s" % (rutils.bold(technique), subtechnique, id))
            else:
                raise RuntimeError("unexpected ATT&CK spec format")
        rows.append((rutils.bold(tactic.upper()), "\n".join(inner_rows),))

    if rows:
        ostream.write(
            tabulate.tabulate(
                rows, headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], tablefmt="psql"
            )
        )
        ostream.write("\n")

def render_default(doc):
    ostream = rutils.StringIO()

    render_meta(doc, ostream)
    ostream.write("\n")
    render_attack(doc, ostream)
    ostream.write("\n")
    render_capabilities(doc, ostream)

    return ostream.getvalue()
<|endoftext|>


<commit_before>#!/usr/bin/python

jobs_xml = """\
<?xml version="1.0"?>
<jobs count="3" end_index="2" filter_by="" full_collection="http://testserver/api/jobs/" id="http://testserver/api/jobs/" limit="10" next_page="" num_pages="1" order_by="" per_page="10" previous_page="" start_index="0">
  <job id="http://testserver/api/jobs/1">
    <job_description>System registration</job_description>
    <time_updated>2010-09-16T13:36:25.939154+00:00</time_updated>
    <job_state>Running</job_state>
    <job_type>system registration</job_type>
    <time_created>2010-09-16T13:36:25.939042+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid001</job_uuid>
    <job_id>1</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/2">
    <job_description>System synchronization</job_description>
    <time_updated>2010-09-16T13:36:25.943043+00:00</time_updated>
    <job_state>Running</job_state>
    <job_type>system poll</job_type>
    <time_created>2010-09-16T13:36:25.942952+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid002</job_uuid>
    <job_id>2</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/3">
    <job_description>On-demand system synchronization</job_description>
    <time_updated>2010-09-16T13:36:25.946773+00:00</time_updated>
    <job_state>Running</job_state>
    <job_type>immediate system poll</job_type>
    <time_created>2010-09-16T13:36:25.946675+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid003</job_uuid>
    <job_id>3</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
</jobs>"""

job_xml = """\
<?xml version="1.0"?>
<job id="http://testserver/api/jobs/1">
  <job_description>System registration</job_description>
  <time_updated>2010-09-16T13:53:18.402208+00:00</time_updated>
  <job_state>Running</job_state>
  <job_type>system registration</job_type>
  <time_created>2010-09-16T13:53:18.402105+00:00</time_created>
  <systems/>
  <job_uuid>rmakeuuid001</job_uuid>
  <job_id>1</job_id>
  <status_code>100</status_code>
  <status_text>Initializing</status_text>
</job>"""

job_states_xml = """\
<?xml version="1.0"?>
<job_states count="4" end_index="3" filter_by="" full_collection="http://testserver/api/job_states" id="http://testserver/api/job_states;start_index=0;limit=10" limit="10" next_page="" num_pages="1" order_by="" per_page="10" previous_page="" start_index="0">
  <job_state id="http://testserver/api/job_states/1">
    <job_state_id>1</job_state_id>
    <jobs id="http://testserver/api/job_states/1/jobs"/>
    <name>Queued</name>
  </job_state>
  <job_state id="http://testserver/api/job_states/2">
    <job_state_id>2</job_state_id>
    <jobs id="http://testserver/api/job_states/2/jobs"/>
    <name>Running</name>
  </job_state>
  <job_state id="http://testserver/api/job_states/3">
    <job_state_id>3</job_state_id>
    <jobs id="http://testserver/api/job_states/3/jobs"/>
    <name>Completed</name>
  </job_state>
  <job_state id="http://testserver/api/job_states/4">
    <job_state_id>4</job_state_id>
    <jobs id="http://testserver/api/job_states/4/jobs"/>
    <name>Failed</name>
  </job_state>
</job_states>"""

job_state_xml = """\
<?xml version="1.0"?>
<job_state id="http://testserver/api/job_states/1">
  <job_state_id>1</job_state_id>
  <jobs id="http://testserver/api/job_states/1/jobs"/>
  <name>Queued</name>
</job_state>"""

systems_jobs_xml = """\
<?xml version="1.0"?>
<jobs count="3" end_index="2" filter_by="" full_collection="http://testserver/api/inventory/systems/3/jobs/" id="http://testserver/api/inventory/systems/3/jobs/" limit="10" next_page="" num_pages="1" order_by="" per_page="10" previous_page="" start_index="0">
  <job id="http://testserver/api/jobs/1">
    <job_description>System registration</job_description>
    <time_updated>2010-09-16T20:13:13.325788+00:00</time_updated>
    <job_state>Running</job_state>
    <job_type>system registration</job_type>
    <time_created>2010-09-16T20:13:13.325686+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid001</job_uuid>
    <job_id>1</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/2">
    <job_description>System synchronization</job_description>
    <time_updated>2010-09-16T20:13:13.334487+00:00</time_updated>
    <job_state>Running</job_state>
    <job_type>system poll</job_type>
    <time_created>2010-09-16T20:13:13.334392+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid002</job_uuid>
    <job_id>2</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/3">
    <job_description>On-demand system synchronization</job_description>
    <time_updated>2010-09-16T20:13:13.339408+00:00</time_updated>
    <job_state>Running</job_state>
    <job_type>immediate system poll</job_type>
    <time_created>2010-09-16T20:13:13.339318+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid003</job_uuid>
    <job_id>3</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
</jobs>"""

<commit_msg>Fix jobs tests<commit_after>#!/usr/bin/python

jobs_xml = """\
<?xml version="1.0"?>
<jobs count="3" end_index="2" filter_by="" full_collection="http://testserver/api/jobs/" id="http://testserver/api/jobs/" limit="10" next_page="" num_pages="1" order_by="" per_page="10" previous_page="" start_index="0">
  <job id="http://testserver/api/jobs/1">
    <job_description>System registration</job_description>
    <time_updated>2010-09-16T13:36:25.939154+00:00</time_updated>
    <job_state id="http://testserver/api/job_states/2">Running</job_state>
    <job_type>system registration</job_type>
    <time_created>2010-09-16T13:36:25.939042+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid001</job_uuid>
    <job_id>1</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/2">
    <job_description>System synchronization</job_description>
    <time_updated>2010-09-16T13:36:25.943043+00:00</time_updated>
    <job_state id="http://testserver/api/job_states/2">Running</job_state>
    <job_type>system poll</job_type>
    <time_created>2010-09-16T13:36:25.942952+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid002</job_uuid>
    <job_id>2</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/3">
    <job_description>On-demand system synchronization</job_description>
    <time_updated>2010-09-16T13:36:25.946773+00:00</time_updated>
    <job_state id="http://testserver/api/job_states/2">Running</job_state>
    <job_type>immediate system poll</job_type>
    <time_created>2010-09-16T13:36:25.946675+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid003</job_uuid>
    <job_id>3</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
</jobs>"""

job_xml = """\
<?xml version="1.0"?>
<job id="http://testserver/api/jobs/1">
  <job_description>System registration</job_description>
  <time_updated>2010-09-16T13:53:18.402208+00:00</time_updated>
  <job_state id="http://testserver/api/job_states/2">Running</job_state>
  <job_type>system registration</job_type>
  <time_created>2010-09-16T13:53:18.402105+00:00</time_created>
  <systems/>
  <job_uuid>rmakeuuid001</job_uuid>
  <job_id>1</job_id>
  <status_code>100</status_code>
  <status_text>Initializing</status_text>
</job>"""

job_states_xml = """\
<?xml version="1.0"?>
<job_states count="4" end_index="3" filter_by="" full_collection="http://testserver/api/job_states" id="http://testserver/api/job_states;start_index=0;limit=10" limit="10" next_page="" num_pages="1" order_by="" per_page="10" previous_page="" start_index="0">
  <job_state id="http://testserver/api/job_states/1">
    <job_state_id>1</job_state_id>
    <jobs id="http://testserver/api/job_states/1/jobs"/>
    <name>Queued</name>
  </job_state>
  <job_state id="http://testserver/api/job_states/2">
    <job_state_id>2</job_state_id>
    <jobs id="http://testserver/api/job_states/2/jobs"/>
    <name>Running</name>
  </job_state>
  <job_state id="http://testserver/api/job_states/3">
    <job_state_id>3</job_state_id>
    <jobs id="http://testserver/api/job_states/3/jobs"/>
    <name>Completed</name>
  </job_state>
  <job_state id="http://testserver/api/job_states/4">
    <job_state_id>4</job_state_id>
    <jobs id="http://testserver/api/job_states/4/jobs"/>
    <name>Failed</name>
  </job_state>
</job_states>"""

job_state_xml = """\
<?xml version="1.0"?>
<job_state id="http://testserver/api/job_states/1">
  <job_state_id>1</job_state_id>
  <jobs id="http://testserver/api/job_states/1/jobs"/>
  <name>Queued</name>
</job_state>"""

systems_jobs_xml = """\
<?xml version="1.0"?>
<jobs count="3" end_index="2" filter_by="" full_collection="http://testserver/api/inventory/systems/3/jobs/" id="http://testserver/api/inventory/systems/3/jobs/" limit="10" next_page="" num_pages="1" order_by="" per_page="10" previous_page="" start_index="0">
  <job id="http://testserver/api/jobs/1">
    <job_description>System registration</job_description>
    <time_updated>2010-09-16T20:13:13.325788+00:00</time_updated>
    <job_state id="http://testserver/api/job_states/2">Running</job_state>
    <job_type>system registration</job_type>
    <time_created>2010-09-16T20:13:13.325686+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid001</job_uuid>
    <job_id>1</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/2">
    <job_description>System synchronization</job_description>
    <time_updated>2010-09-16T20:13:13.334487+00:00</time_updated>
    <job_state id="http://testserver/api/job_states/2">Running</job_state>
    <job_type>system poll</job_type>
    <time_created>2010-09-16T20:13:13.334392+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid002</job_uuid>
    <job_id>2</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
  <job id="http://testserver/api/jobs/3">
    <job_description>On-demand system synchronization</job_description>
    <time_updated>2010-09-16T20:13:13.339408+00:00</time_updated>
    <job_state id="http://testserver/api/job_states/2">Running</job_state>
    <job_type>immediate system poll</job_type>
    <time_created>2010-09-16T20:13:13.339318+00:00</time_created>
    <systems/>
    <job_uuid>rmakeuuid003</job_uuid>
    <job_id>3</job_id>
    <status_code>100</status_code>
    <status_text>Initializing</status_text>
  </job>
</jobs>"""

<|endoftext|>


<commit_before>from __future__ import print_function
import requests
from websocket import create_connection
import time
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
import json
import dill
from tqdm import tqdm
import sys
import io
from zipfile import ZipFile
from os import path, remove
from .settings import settings

def statusCheck(res):
    if res.status_code != 200:
        print(res.text)
        sys.exit()

def contact_server():
    if settings.LOCAL:
        serverType = 'local'
    else:
        serverType = 'gpu'
    r = requests.post('http://%s/api/gpu/checkAvailability' % settings.CATALEARN_URL,
                      data={'username': settings.API_KEY,
                            'type': serverType})
    statusCheck(r)
    res = r.json()

    jobHash = res['jobHash']
    idle = res['idle']
    instanceId = res['instanceId']

    if not idle:
        print("Starting server, this will take about 20 seconds")
        while True:
            r = requests.post('http://%s/api/gpu/checkStatus' % settings.CATALEARN_URL,
                              data={'instanceId': instanceId})
            statusCheck(r)
            res = r.json()
            if res['started']:
                break
            time.sleep(3)
            print('.', end='')
        print()

    r = requests.post('http://%s/api/gpu/runJob' % settings.CATALEARN_URL,
                      data={'hash': jobHash})
    statusCheck(r)
    res = r.json()
    gpuIp = res['ip']
    wsPort = res['ws_port']
    return (gpuIp, wsPort, jobHash)

def upload_data(gpuIp, jobHash):
    url = 'http://%s:%s/runJobDecorator' % (gpuIp, settings.GPU_PORT)
    print("Uploading data")

    fileSize = path.getsize('uploads.pkl')
    pbar = tqdm(total=fileSize, unit='B', unit_scale=True)

    def callback(monitor):
        progress = monitor.bytes_read - callback.last_bytes_read
        pbar.update(progress)
        callback.last_bytes_read = monitor.bytes_read
    callback.last_bytes_read = 0

    with open('uploads.pkl', 'rb') as file:
        data = {
            'file': ('uploads.pkl', file, 'application/octet-stream'),
            'hash': jobHash
        }
        encoder = MultipartEncoder(
            fields=data
        )
        monitor = MultipartEncoderMonitor(encoder, callback)
        try:
            r = requests.post(url, data=monitor, headers={
                'Content-Type': monitor.content_type})
            pbar.close()
            statusCheck(r)
            remove('uploads.pkl')
            return True
        except:
            pbar.close()  # need to close before printing anything
            print('Upload cancelled')
            remove('uploads.pkl')
            return False

def stream_output(gpuIp, wsPort, jobHash):

    gpuUrl = 'ws://%s:%s' % (gpuIp, wsPort)
    ws = create_connection(gpuUrl)
    outUrl = None
    ws.send(jobHash)
    try:
        while True:
            msg = ws.recv()
            msgJson = json.loads(msg)
            if 'end' not in msgJson:
                print(msgJson['message'], end='')
            else:
                if 'downloadUrl' in msgJson:
                    outUrl = msgJson['downloadUrl']
                else:
                    outUrl = None
                break
    except KeyboardInterrupt:
        print('\nJob interrupted')
    finally:
        ws.close()
        return outUrl

def get_result(outUrl, jobHash):

    print("Downloading result")
    r = requests.post(outUrl, data={'hash': jobHash})
    statusCheck(r)

    totalSize = int(r.headers.get('content-length', 0))
    with open('download.zip', 'wb') as f:
        pbar = tqdm(total=totalSize, unit='B', unit_scale=True)
        chunckSize = 32768
        for data in r.iter_content(chunk_size=chunckSize):
            f.write(data)
            pbar.update(chunckSize)
        pbar.close()

    z = ZipFile(io.BytesIO(r.content))
    z.extractall()
    newFiles = z.namelist()

    result = None
    if path.isfile(jobHash):
        with open(jobHash, "rb") as f:

            # import_all()  # Hack: a workaround for dill's pickling problem
            result = dill.load(f)
            # unimport_all()
            print("Done!")
            remove(jobHash)
            newFiles.remove(jobHash)

    remove('download.zip')

    printedNameList = str(newFiles)[1:-1]
    if len(newFiles) == 1:
        print('New file: %s' % printedNameList)
    elif len(newFiles) > 1:
        print('New files: %s' % printedNameList)

    return result

def get_time_and_credit(jobHash):
    r = requests.post('http://%s/api/gpu/getTimeAndCredit' % settings.CATALEARN_URL,
                      data={'hash': jobHash})
    statusCheck(r)
    res = r.json()
    jobDuration = res['time']
    remainingCredits = res['credits']
    print('The job took %s minutes, you have %s minutes of credit remaining' % (
        jobDuration, remainingCredits))
<commit_msg>improved plural logic<commit_after>from __future__ import print_function
import requests
from websocket import create_connection
import time
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
import json
import dill
from tqdm import tqdm
import sys
import io
from zipfile import ZipFile
from os import path, remove
from .settings import settings

def statusCheck(res):
    if res.status_code != 200:
        print(res.text)
        sys.exit()

def contact_server():
    if settings.LOCAL:
        serverType = 'local'
    else:
        serverType = 'gpu'
    r = requests.post('http://%s/api/gpu/checkAvailability' % settings.CATALEARN_URL,
                      data={'username': settings.API_KEY,
                            'type': serverType})
    statusCheck(r)
    res = r.json()

    jobHash = res['jobHash']
    idle = res['idle']
    instanceId = res['instanceId']

    if not idle:
        print("Starting server, this will take about 20 seconds")
        while True:
            r = requests.post('http://%s/api/gpu/checkStatus' % settings.CATALEARN_URL,
                              data={'instanceId': instanceId})
            statusCheck(r)
            res = r.json()
            if res['started']:
                break
            time.sleep(3)
            print('.', end='')
        print()

    r = requests.post('http://%s/api/gpu/runJob' % settings.CATALEARN_URL,
                      data={'hash': jobHash})
    statusCheck(r)
    res = r.json()
    gpuIp = res['ip']
    wsPort = res['ws_port']
    return (gpuIp, wsPort, jobHash)

def upload_data(gpuIp, jobHash):
    url = 'http://%s:%s/runJobDecorator' % (gpuIp, settings.GPU_PORT)
    print("Uploading data")

    fileSize = path.getsize('uploads.pkl')
    pbar = tqdm(total=fileSize, unit='B', unit_scale=True)

    def callback(monitor):
        progress = monitor.bytes_read - callback.last_bytes_read
        pbar.update(progress)
        callback.last_bytes_read = monitor.bytes_read
    callback.last_bytes_read = 0

    with open('uploads.pkl', 'rb') as file:
        data = {
            'file': ('uploads.pkl', file, 'application/octet-stream'),
            'hash': jobHash
        }
        encoder = MultipartEncoder(
            fields=data
        )
        monitor = MultipartEncoderMonitor(encoder, callback)
        try:
            r = requests.post(url, data=monitor, headers={
                'Content-Type': monitor.content_type})
            pbar.close()
            statusCheck(r)
            remove('uploads.pkl')
            return True
        except:
            pbar.close()  # need to close before printing anything
            print('Upload cancelled')
            remove('uploads.pkl')
            return False

def stream_output(gpuIp, wsPort, jobHash):

    gpuUrl = 'ws://%s:%s' % (gpuIp, wsPort)
    ws = create_connection(gpuUrl)
    outUrl = None
    ws.send(jobHash)
    try:
        while True:
            msg = ws.recv()
            msgJson = json.loads(msg)
            if 'end' not in msgJson:
                print(msgJson['message'], end='')
            else:
                if 'downloadUrl' in msgJson:
                    outUrl = msgJson['downloadUrl']
                else:
                    outUrl = None
                break
    except KeyboardInterrupt:
        print('\nJob interrupted')
    finally:
        ws.close()
        return outUrl

def get_result(outUrl, jobHash):

    print("Downloading result")
    r = requests.post(outUrl, data={'hash': jobHash})
    statusCheck(r)

    totalSize = int(r.headers.get('content-length', 0))
    with open('download.zip', 'wb') as f:
        pbar = tqdm(total=totalSize, unit='B', unit_scale=True)
        chunckSize = 32768
        for data in r.iter_content(chunk_size=chunckSize):
            f.write(data)
            pbar.update(chunckSize)
        pbar.close()

    z = ZipFile(io.BytesIO(r.content))
    z.extractall()
    newFiles = z.namelist()

    result = None
    if path.isfile(jobHash):
        with open(jobHash, "rb") as f:

            # import_all()  # Hack: a workaround for dill's pickling problem
            result = dill.load(f)
            # unimport_all()
            print("Done!")
            remove(jobHash)
            newFiles.remove(jobHash)

    remove('download.zip')

    printedNameList = str(newFiles)[1:-1]
    if len(newFiles) > 0:
        print('New file%s: %s' % ('' if len(newFiles) == 1 else 's', printedNameList))

    return result

def get_time_and_credit(jobHash):
    r = requests.post('http://%s/api/gpu/getTimeAndCredit' % settings.CATALEARN_URL,
                      data={'hash': jobHash})
    statusCheck(r)
    res = r.json()
    jobDuration = res['time']
    remainingCredits = res['credits']
    print('%s minute%s used, you have %s minute%s of credit remaining' % (
        jobDuration, '' if jobDuration <= 1 else 's',
        remainingCredits, '' if remainingCredits <= 1 else 's'))
<|endoftext|>


<commit_before>from spec import Spec, skip, eq_, ok_

from invoke.vendor.etcaetera.adapter import File, Adapter

from invoke.config import Config

def loads_path(c, path):
    files = [x for x in c._config.adapters if isinstance(x, File)]
    paths = [x.filepath for x in files]
    found = any(x == path for x in paths)
    ok_(found, "{0!r} not found, file adapters: {1!r}".format(path, paths))

class Config_(Spec):
    class init:
        "__init__"

        def can_be_empty(self):
            eq_(Config().__class__, Config) # derp

        def configure_global_location_prefix(self):
            # This is a bit funky but more useful than just replicating the
            # same test farther down?
            c = Config(global_prefix='meh')
            loads_path(c, 'meh.yaml')

        def default_global_prefix_is_etc(self):
            # TODO: make this work on Windows somehow without being a total
            # tautology? heh.
            c = Config()
            loads_path(c, '/etc/invoke.yaml')

        def configure_user_location_prefix(self):
            skip()

        def default_local_prefix_is_homedir(self):
            skip()

        def unknown_kwargs_turn_into_top_level_defaults(self):
            c = Config(foo='bar')
            eq_(c._config.defaults['foo'], 'bar')

        def accepts_explicit_adapter_override_list(self):
            c = Config(adapters=[])
            # Slightly encapsulation-breaking. Meh.
            # (Our) Config objs always start with a Defaults.
            eq_(len(c._config.adapters), 1)

        def does_not_trigger_config_loading(self):
            # Cuz automatic loading could potentially be surprising.
            # Meh-tastic no-exception-raised test.
            class DummyAdapter(Adapter):
                def load(self, *args, **kwargs):
                    raise Exception("I shouldn't have been called!")
            c = Config(adapters=[DummyAdapter()])

    class basic_API:
        "Basic API components"

        def load_method_is_an_thing(self):
            # Kinda duplicative but meh, nice and explicit.
            c = Config(default='value')
            c.load()
            eq_(c.default, 'value')

        def can_set_defaults_after_initialization(self):
            # Something light which wraps self._config.defaults[k] = v
            c = Config()
            c.set_defaults({'foo': 'bar'})
            c.load()
            eq_(c.foo, 'bar')

        def set_defaults_overrides_convenience_kwargs(self):
            c = Config(foo='bar')
            c.set_defaults({'biz': 'baz'})
            c.load()
            ok_('foo' not in c, "Expected {0!r} not to include 'foo'!")
            ok_('biz' in c, "Expected {0!r} to include 'biz'!")

        def allows_dict_and_attr_access(self):
            # TODO: combine with tests for Context probably
            skip()

        def nested_dict_values_also_allow_dual_access(self):
            # TODO: ditto
            skip()

        def attr_access_has_useful_errr_msg(self):
            c = Config()
            c.load()
            try:
                c.nope
            except AttributeError as e:
                expected = """
No attribute or config key found for 'nope'

Valid real attributes: ['keys', 'load', 'set_defaults']

Valid keys: []""".lstrip()
                eq_(str(e), expected)
            else:
                assert False, "Didn't get an AttributeError on bad key!"

        def loaded_keys_are_not_case_munged(self):
            # Looks tautological, but ensures we're suppressing etcaetera's
            # default UPPERCASE_EVERYTHING behavior
            d = {'FOO': 'bar', 'biz': 'baz', 'Boz': 'buzz'}
            c = Config(**d)
            c.load()
            for x in d:
                err = "Expected to find {0!r} in {1!r}, didn't"
                ok_(x in c, err.format(x, c.keys()))

        def is_iterable_like_dict(self):
            def expect(c, expected):
                eq_(set(c.keys()), expected)
                eq_(set(list(c)), expected)
            c = Config(a=1, b=2)
            expect(c, set())
            c.load()
            expect(c, set(['a', 'b']))

        def supports_membership_testing_like_dict(self):
            c = Config(foo='bar')
            c.load()
            ok_('foo' in c, "Unable to find 'foo' in {0!r}".format(c))

    class system_global:
        "Systemwide conf file"
        def yaml_first(self):
            c = Config(global_prefix='tests/_support/configs/global')
            c.load()
            eq_(c.hooray, 'configuration')

        def json_if_no_yaml(self):
            skip()

        def python_if_no_json_or_yaml(self):
            skip()

    class user_specific:
        "User-specific conf file"
        def yaml_first(self):
            skip()

        def json_if_no_yaml(self):
            skip()

        def python_if_no_json_or_yaml(self):
            skip()

    class project_specific:
        "Local-to-project conf file"
        def yaml_first(self):
            skip()

        def json_if_no_yaml(self):
            skip()

        def python_if_no_json_or_yaml(self):
            skip()

    def honors_conf_file_flag(self):
        skip()

    class env_vars:
        "Environment variables"
        def base_case(self):
            # FOO=bar
            skip()

        def throws_error_on_undefined_settings(self):
            skip()

        def underscores_top_level(self):
            # FOO_BAR=biz => {'foo_bar': 'biz'}
            skip()

        def underscores_nested(self):
            # FOO_BAR=biz => {'foo': {'bar': 'biz'}}
            skip()

        def both_underscores(self):
            # FOO_BAR_BIZ=baz => {'foo_bar': {'biz': 'baz'}}
            skip()

    class hierarchy:
        "Config hierarchy in effect"
        def systemwide_overrides_collection(self):
            skip()

        def user_overrides_systemwide(self):
            skip()

        def user_overrides_collection(self):
            skip()

        def project_overrides_user(self):
            skip()

        def project_overrides_systemwide(self):
            skip()

        def project_overrides_collection(self):
            skip()

        def env_vars_override_project(self):
            skip()

        def env_vars_override_user(self):
            skip()

        def env_vars_override_systemwide(self):
            skip()

        def env_vars_override_collection(self):
            skip()

        def runtime_overrides_env_vars(self):
            skip()

        def runtime_overrides_project(self):
            skip()

        def runtime_overrides_user(self):
            skip()

        def runtime_overrides_systemwide(self):
            skip()

        def runtime_overrides_collection(self):
            skip()

        def e_flag_overrides_all(self):
            "-e overrides run.echo for all other layers"
            skip()

<commit_msg>Failing tests re: user path<commit_after>from spec import Spec, skip, eq_, ok_

from invoke.vendor.etcaetera.adapter import File, Adapter

from invoke.config import Config

def loads_path(c, path):
    files = [x for x in c._config.adapters if isinstance(x, File)]
    paths = [x.filepath for x in files]
    found = any(x == path for x in paths)
    ok_(found, "{0!r} not found, file adapters: {1!r}".format(path, paths))

class Config_(Spec):
    class init:
        "__init__"

        def can_be_empty(self):
            eq_(Config().__class__, Config) # derp

        def configure_global_location_prefix(self):
            # This is a bit funky but more useful than just replicating the
            # same test farther down?
            c = Config(global_prefix='meh')
            loads_path(c, 'meh.yaml')

        def default_global_prefix_is_etc(self):
            # TODO: make this work on Windows somehow without being a total
            # tautology? heh.
            loads_path(Config(), '/etc/invoke.yaml')

        def configure_user_location_prefix(self):
            c = Config(user_prefix='whatever')
            loads_path(c, 'whatever.yaml')

        def default_local_prefix_is_homedir(self):
            loads_path(Config(), '~/.invoke.yaml')

        def unknown_kwargs_turn_into_top_level_defaults(self):
            c = Config(foo='bar')
            eq_(c._config.defaults['foo'], 'bar')

        def accepts_explicit_adapter_override_list(self):
            c = Config(adapters=[])
            # Slightly encapsulation-breaking. Meh.
            # (Our) Config objs always start with a Defaults.
            eq_(len(c._config.adapters), 1)

        def does_not_trigger_config_loading(self):
            # Cuz automatic loading could potentially be surprising.
            # Meh-tastic no-exception-raised test.
            class DummyAdapter(Adapter):
                def load(self, *args, **kwargs):
                    raise Exception("I shouldn't have been called!")
            c = Config(adapters=[DummyAdapter()])

    class basic_API:
        "Basic API components"

        def load_method_is_an_thing(self):
            # Kinda duplicative but meh, nice and explicit.
            c = Config(default='value')
            c.load()
            eq_(c.default, 'value')

        def can_set_defaults_after_initialization(self):
            # Something light which wraps self._config.defaults[k] = v
            c = Config()
            c.set_defaults({'foo': 'bar'})
            c.load()
            eq_(c.foo, 'bar')

        def set_defaults_overrides_convenience_kwargs(self):
            c = Config(foo='bar')
            c.set_defaults({'biz': 'baz'})
            c.load()
            ok_('foo' not in c, "Expected {0!r} not to include 'foo'!")
            ok_('biz' in c, "Expected {0!r} to include 'biz'!")

        def allows_dict_and_attr_access(self):
            # TODO: combine with tests for Context probably
            skip()

        def nested_dict_values_also_allow_dual_access(self):
            # TODO: ditto
            skip()

        def attr_access_has_useful_errr_msg(self):
            c = Config()
            c.load()
            try:
                c.nope
            except AttributeError as e:
                expected = """
No attribute or config key found for 'nope'

Valid real attributes: ['keys', 'load', 'set_defaults']

Valid keys: []""".lstrip()
                eq_(str(e), expected)
            else:
                assert False, "Didn't get an AttributeError on bad key!"

        def loaded_keys_are_not_case_munged(self):
            # Looks tautological, but ensures we're suppressing etcaetera's
            # default UPPERCASE_EVERYTHING behavior
            d = {'FOO': 'bar', 'biz': 'baz', 'Boz': 'buzz'}
            c = Config(**d)
            c.load()
            for x in d:
                err = "Expected to find {0!r} in {1!r}, didn't"
                ok_(x in c, err.format(x, c.keys()))

        def is_iterable_like_dict(self):
            def expect(c, expected):
                eq_(set(c.keys()), expected)
                eq_(set(list(c)), expected)
            c = Config(a=1, b=2)
            expect(c, set())
            c.load()
            expect(c, set(['a', 'b']))

        def supports_membership_testing_like_dict(self):
            c = Config(foo='bar')
            c.load()
            ok_('foo' in c, "Unable to find 'foo' in {0!r}".format(c))

    class system_global:
        "Systemwide conf file"
        def yaml_first(self):
            c = Config(global_prefix='tests/_support/configs/global')
            c.load()
            eq_(c.hooray, 'configuration')

        def json_if_no_yaml(self):
            skip()

        def python_if_no_json_or_yaml(self):
            skip()

    class user_specific:
        "User-specific conf file"
        def yaml_first(self):
            skip()

        def json_if_no_yaml(self):
            skip()

        def python_if_no_json_or_yaml(self):
            skip()

    class project_specific:
        "Local-to-project conf file"
        def yaml_first(self):
            skip()

        def json_if_no_yaml(self):
            skip()

        def python_if_no_json_or_yaml(self):
            skip()

    def honors_conf_file_flag(self):
        skip()

    class env_vars:
        "Environment variables"
        def base_case(self):
            # FOO=bar
            skip()

        def throws_error_on_undefined_settings(self):
            skip()

        def underscores_top_level(self):
            # FOO_BAR=biz => {'foo_bar': 'biz'}
            skip()

        def underscores_nested(self):
            # FOO_BAR=biz => {'foo': {'bar': 'biz'}}
            skip()

        def both_underscores(self):
            # FOO_BAR_BIZ=baz => {'foo_bar': {'biz': 'baz'}}
            skip()

    class hierarchy:
        "Config hierarchy in effect"
        def systemwide_overrides_collection(self):
            skip()

        def user_overrides_systemwide(self):
            skip()

        def user_overrides_collection(self):
            skip()

        def project_overrides_user(self):
            skip()

        def project_overrides_systemwide(self):
            skip()

        def project_overrides_collection(self):
            skip()

        def env_vars_override_project(self):
            skip()

        def env_vars_override_user(self):
            skip()

        def env_vars_override_systemwide(self):
            skip()

        def env_vars_override_collection(self):
            skip()

        def runtime_overrides_env_vars(self):
            skip()

        def runtime_overrides_project(self):
            skip()

        def runtime_overrides_user(self):
            skip()

        def runtime_overrides_systemwide(self):
            skip()

        def runtime_overrides_collection(self):
            skip()

        def e_flag_overrides_all(self):
            "-e overrides run.echo for all other layers"
            skip()

<|endoftext|>


<commit_before>from . import _vmray  # noqa

__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
           'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',
           'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
           'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep',
           'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon',
           'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
           'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',
           'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
           'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
           'qrcode', 'ocr']
<commit_msg>chg: [init] removed trailing whitespace<commit_after>from . import _vmray  # noqa

__all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'circl_passivessl',
           'countrycode', 'cve', 'dns', 'btc_steroids', 'domaintools', 'eupi',
           'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal',
           'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep',
           'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon',
           'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl',
           'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator',
           'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io',
           'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors',
           'qrcode', 'ocr']
<|endoftext|>


<commit_before>from __future__ import generators

############## urlparse.parse_qsl ###########################################

try:
    from urlparse import parse_qsl
except ImportError:
    from cgi import parse_qsl

############## __builtin__.all ##############################################

try:
    all([True])
    all = all
except NameError:
    def all(iterable):
        for item in iterable:
            if not item:
                return False
        return True

############## __builtin__.any ##############################################

try:
    any([True])
    any = any
except NameError:
    def any(iterable):
        for item in iterable:
            if item:
                return True
        return False

############## collections.OrderedDict ######################################

import weakref
try:
    from collections import MutableMapping
except ImportError:
    from UserDict import DictMixin as MutableMapping
from itertools import imap as _imap
from operator import eq as _eq

class _Link(object):
    """Doubly linked list."""
    __slots__ = 'prev', 'next', 'key', '__weakref__'

class OrderedDict(dict, MutableMapping):
    """Dictionary that remembers insertion order"""
    # An inherited dict maps keys to values.
    # The inherited dict provides __getitem__, __len__, __contains__, and get.
    # The remaining methods are order-aware.
    # Big-O running times for all methods are the same as for regular
    # dictionaries.

    # The internal self.__map dictionary maps keys to links in a doubly
    # linked list.
    # The circular doubly linked list starts and ends with a sentinel element.
    # The sentinel element never gets deleted (this simplifies the algorithm).
    # The prev/next links are weakref proxies (to prevent circular
    # references).
    # Individual links are kept alive by the hard reference in self.__map.
    # Those hard references disappear when a key is deleted from
    # an OrderedDict.

    __marker = object()

    def __init__(self, *args, **kwds):
        """Initialize an ordered dictionary.

        Signature is the same as for regular dictionaries, but keyword
        arguments are not recommended because their insertion order is
        arbitrary.

        """
        if len(args) > 1:
            raise TypeError("expected at most 1 arguments, got %d" % (
                                len(args)))
        try:
            self.__root
        except AttributeError:
            # sentinel node for the doubly linked list
            self.__root = root = _Link()
            root.prev = root.next = root
            self.__map = {}
        self.update(*args, **kwds)

    def clear(self):
        "od.clear() -> None.  Remove all items from od."
        root = self.__root
        root.prev = root.next = root
        self.__map.clear()
        dict.clear(self)

    def __setitem__(self, key, value):
        "od.__setitem__(i, y) <==> od[i]=y"
        # Setting a new item creates a new link which goes at the end of the
        # linked list, and the inherited dictionary is updated with the new
        # key/value pair.
        if key not in self:
            self.__map[key] = link = _Link()
            root = self.__root
            last = root.prev
            link.prev, link.next, link.key = last, root, key
            last.next = root.prev = weakref.proxy(link)
        dict.__setitem__(self, key, value)

    def __delitem__(self, key):
        """od.__delitem__(y) <==> del od[y]"""
        # Deleting an existing item uses self.__map to find the
        # link which is then removed by updating the links in the
        # predecessor and successor nodes.
        dict.__delitem__(self, key)
        link = self.__map.pop(key)
        link.prev.next = link.next
        link.next.prev = link.prev

    def __iter__(self):
        """od.__iter__() <==> iter(od)"""
        # Traverse the linked list in order.
        root = self.__root
        curr = root.next
        while curr is not root:
            yield curr.key
            curr = curr.next

    def __reversed__(self):
        """od.__reversed__() <==> reversed(od)"""
        # Traverse the linked list in reverse order.
        root = self.__root
        curr = root.prev
        while curr is not root:
            yield curr.key
            curr = curr.prev

    def __reduce__(self):
        """Return state information for pickling"""
        items = [[k, self[k]] for k in self]
        tmp = self.__map, self.__root
        del(self.__map, self.__root)
        inst_dict = vars(self).copy()
        self.__map, self.__root = tmp
        if inst_dict:
            return (self.__class__, (items,), inst_dict)
        return self.__class__, (items,)

    def setdefault(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            self[key] = default
        return default

    def update(self, other=(), **kwds):
        if isinstance(other, dict):
            for key in other:
                self[key] = other[key]
        elif hasattr(other, "keys"):
            for key in other.keys():
                self[key] = other[key]
        else:
            for key, value in other:
                self[key] = value
        for key, value in kwds.items():
            self[key] = value

    def pop(self, key, default=__marker):
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def values(self):
        return [self[key] for key in self]

    def items(self):
        return [(key, self[key]) for key in self]

    def itervalues(self):
        for key in self:
            yield self[key]

    def iteritems(self):
        for key in self:
            yield (key, self[key])

    def iterkeys(self):
        return iter(self)

    def keys(self):
        return list(self)

    def popitem(self, last=True):
        """od.popitem() -> (k, v)

        Return and remove a (key, value) pair.
        Pairs are returned in LIFO order if last is true or FIFO
        order if false.

        """
        if not self:
            raise KeyError('dictionary is empty')
        key = (last and reversed(self) or iter(self)).next()
        value = self.pop(key)
        return key, value

    def __repr__(self):
        "od.__repr__() <==> repr(od)"
        if not self:
            return '%s()' % (self.__class__.__name__,)
        return '%s(%r)' % (self.__class__.__name__, self.items())

    def copy(self):
        "od.copy() -> a shallow copy of od"
        return self.__class__(self)

    @classmethod
    def fromkeys(cls, iterable, value=None):
        """OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
        and values equal to v (which defaults to None)."""
        d = cls()
        for key in iterable:
            d[key] = value
        return d

    def __eq__(self, other):
        """od.__eq__(y) <==> od==y.  Comparison to another OD is
        order-sensitive while comparison to a regular mapping
        is order-insensitive."""
        if isinstance(other, OrderedDict):
            return len(self) == len(other) and \
                   all(_imap(_eq, self.iteritems(), other.iteritems()))
        return dict.__eq__(self, other)

    def __ne__(self, other):
        return not (self == other)

############## collections.defaultdict ######################################

try:
    from collections import defaultdict
except ImportError:
    # Written by Jason Kirtland, taken from Python Cookbook:
    # <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/523034>
    class defaultdict(dict):

        def __init__(self, default_factory=None, *args, **kwargs):
            dict.__init__(self, *args, **kwargs)
            self.default_factory = default_factory

        def __getitem__(self, key):
            try:
                return dict.__getitem__(self, key)
            except KeyError:
                return self.__missing__(key)

        def __missing__(self, key):
            if self.default_factory is None:
                raise KeyError(key)
            self[key] = value = self.default_factory()
            return value

        def __reduce__(self):
            f = self.default_factory
            args = f is None and tuple() or f
            return type(self), args, None, None, self.iteritems()

        def copy(self):
            return self.__copy__()

        def __copy__(self):
            return type(self)(self.default_factory, self)

        def __deepcopy__(self):
            import copy
            return type(self)(self.default_factory,
                        copy.deepcopy(self.items()))

        def __repr__(self):
            return "defaultdict(%s, %s)" % (self.default_factory,
                                            dict.__repr__(self))
    import collections
    collections.defaultdict = defaultdict # Pickle needs this.

############## logging.LoggerAdapter ########################################

try:
    from logging import LoggerAdapter
except ImportError:
    class LoggerAdapter(object):

        def __init__(self, logger, extra):
            self.logger = logger
            self.extra = extra

        def process(self, msg, kwargs):
            kwargs["extra"] = self.extra
            return msg, kwargs

        def debug(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.debug(msg, *args, **kwargs)

        def info(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.info(msg, *args, **kwargs)

        def warning(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.warning(msg, *args, **kwargs)

        def warn(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.warn(msg, *args, **kwargs)

        def error(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.error(msg, *args, **kwargs)

        def exception(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            kwargs["exc_info"] = 1
            self.logger.error(msg, *args, **kwargs)

        def critical(self, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.critical(msg, *args, **kwargs)

        def log(self, level, msg, *args, **kwargs):
            msg, kwargs = self.process(msg, kwargs)
            self.logger.log(level, msg, *args, **kwargs)

        def isEnabledFor(self, level, *args, **kwargs):
            return self.logger.isEnabledFor(level, *args, **kwargs)

############## itertools.izip_longest #######################################

try:
    from itertools import izip_longest
except ImportError:
    import itertools
    def izip_longest(*args, **kwds):
        fillvalue = kwds.get("fillvalue")

        def sentinel(counter=([fillvalue] * (len(args) - 1)).pop):
            yield counter() # yields the fillvalue, or raises IndexError

        fillers = itertools.repeat(fillvalue)
        iters = [itertools.chain(it, sentinel(), fillers)
                    for it in args]
        try:
            for tup in itertools.izip(*iters):
                yield tup
        except IndexError:
            pass

############## itertools.chain.from_iterable ################################
from itertools import chain

def _compat_chain_from_iterable(iterables):
    for it in iterables:
        for element in it:
            yield element

try:
    chain_from_iterable = getattr(chain, "from_iterable")
except AttributeError:
    chain_from_iterable = _compat_chain_from_iterable
<commit_msg>utils.compat.LoggerAdapter: Support extra argument in Python 2.4.<commit_after>from __future__ import generators

############## urlparse.parse_qsl ###########################################

try:
    from urlparse import parse_qsl
except ImportError:
    from cgi import parse_qsl

############## __builtin__.all ##############################################

try:
    all([True])
    all = all
except NameError:
    def all(iterable):
        for item in iterable:
            if not item:
                return False
        return True

############## __builtin__.any ##############################################

try:
    any([True])
    any = any
except NameError:
    def any(iterable):
        for item in iterable:
            if item:
                return True
        return False

############## collections.OrderedDict ######################################

import weakref
try:
    from collections import MutableMapping
except ImportError:
    from UserDict import DictMixin as MutableMapping
from itertools import imap as _imap
from operator import eq as _eq

class _Link(object):
    """Doubly linked list."""
    __slots__ = 'prev', 'next', 'key', '__weakref__'

class OrderedDict(dict, MutableMapping):
    """Dictionary that remembers insertion order"""
    # An inherited dict maps keys to values.
    # The inherited dict provides __getitem__, __len__, __contains__, and get.
    # The remaining methods are order-aware.
    # Big-O running times for all methods are the same as for regular
    # dictionaries.

    # The internal self.__map dictionary maps keys to links in a doubly
    # linked list.
    # The circular doubly linked list starts and ends with a sentinel element.
    # The sentinel element never gets deleted (this simplifies the algorithm).
    # The prev/next links are weakref proxies (to prevent circular
    # references).
    # Individual links are kept alive by the hard reference in self.__map.
    # Those hard references disappear when a key is deleted from
    # an OrderedDict.

    __marker = object()

    def __init__(self, *args, **kwds):
        """Initialize an ordered dictionary.

        Signature is the same as for regular dictionaries, but keyword
        arguments are not recommended because their insertion order is
        arbitrary.

        """
        if len(args) > 1:
            raise TypeError("expected at most 1 arguments, got %d" % (
                                len(args)))
        try:
            self.__root
        except AttributeError:
            # sentinel node for the doubly linked list
            self.__root = root = _Link()
            root.prev = root.next = root
            self.__map = {}
        self.update(*args, **kwds)

    def clear(self):
        "od.clear() -> None.  Remove all items from od."
        root = self.__root
        root.prev = root.next = root
        self.__map.clear()
        dict.clear(self)

    def __setitem__(self, key, value):
        "od.__setitem__(i, y) <==> od[i]=y"
        # Setting a new item creates a new link which goes at the end of the
        # linked list, and the inherited dictionary is updated with the new
        # key/value pair.
        if key not in self:
            self.__map[key] = link = _Link()
            root = self.__root
            last = root.prev
            link.prev, link.next, link.key = last, root, key
            last.next = root.prev = weakref.proxy(link)
        dict.__setitem__(self, key, value)

    def __delitem__(self, key):
        """od.__delitem__(y) <==> del od[y]"""
        # Deleting an existing item uses self.__map to find the
        # link which is then removed by updating the links in the
        # predecessor and successor nodes.
        dict.__delitem__(self, key)
        link = self.__map.pop(key)
        link.prev.next = link.next
        link.next.prev = link.prev

    def __iter__(self):
        """od.__iter__() <==> iter(od)"""
        # Traverse the linked list in order.
        root = self.__root
        curr = root.next
        while curr is not root:
            yield curr.key
            curr = curr.next

    def __reversed__(self):
        """od.__reversed__() <==> reversed(od)"""
        # Traverse the linked list in reverse order.
        root = self.__root
        curr = root.prev
        while curr is not root:
            yield curr.key
            curr = curr.prev

    def __reduce__(self):
        """Return state information for pickling"""
        items = [[k, self[k]] for k in self]
        tmp = self.__map, self.__root
        del(self.__map, self.__root)
        inst_dict = vars(self).copy()
        self.__map, self.__root = tmp
        if inst_dict:
            return (self.__class__, (items,), inst_dict)
        return self.__class__, (items,)

    def setdefault(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            self[key] = default
        return default

    def update(self, other=(), **kwds):
        if isinstance(other, dict):
            for key in other:
                self[key] = other[key]
        elif hasattr(other, "keys"):
            for key in other.keys():
                self[key] = other[key]
        else:
            for key, value in other:
                self[key] = value
        for key, value in kwds.items():
            self[key] = value

    def pop(self, key, default=__marker):
        try:
            value = self[key]
        except KeyError:
            if default is self.__marker:
                raise
            return default
        else:
            del self[key]
            return value

    def values(self):
        return [self[key] for key in self]

    def items(self):
        return [(key, self[key]) for key in self]

    def itervalues(self):
        for key in self:
            yield self[key]

    def iteritems(self):
        for key in self:
            yield (key, self[key])

    def iterkeys(self):
        return iter(self)

    def keys(self):
        return list(self)

    def popitem(self, last=True):
        """od.popitem() -> (k, v)

        Return and remove a (key, value) pair.
        Pairs are returned in LIFO order if last is true or FIFO
        order if false.

        """
        if not self:
            raise KeyError('dictionary is empty')
        key = (last and reversed(self) or iter(self)).next()
        value = self.pop(key)
        return key, value

    def __repr__(self):
        "od.__repr__() <==> repr(od)"
        if not self:
            return '%s()' % (self.__class__.__name__,)
        return '%s(%r)' % (self.__class__.__name__, self.items())

    def copy(self):
        "od.copy() -> a shallow copy of od"
        return self.__class__(self)

    @classmethod
    def fromkeys(cls, iterable, value=None):
        """OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
        and values equal to v (which defaults to None)."""
        d = cls()
        for key in iterable:
            d[key] = value
        return d

    def __eq__(self, other):
        """od.__eq__(y) <==> od==y.  Comparison to another OD is
        order-sensitive while comparison to a regular mapping
        is order-insensitive."""
        if isinstance(other, OrderedDict):
            return len(self) == len(other) and \
                   all(_imap(_eq, self.iteritems(), other.iteritems()))
        return dict.__eq__(self, other)

    def __ne__(self, other):
        return not (self == other)

############## collections.defaultdict ######################################

try:
    from collections import defaultdict
except ImportError:
    # Written by Jason Kirtland, taken from Python Cookbook:
    # <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/523034>
    class defaultdict(dict):

        def __init__(self, default_factory=None, *args, **kwargs):
            dict.__init__(self, *args, **kwargs)
            self.default_factory = default_factory

        def __getitem__(self, key):
            try:
                return dict.__getitem__(self, key)
            except KeyError:
                return self.__missing__(key)

        def __missing__(self, key):
            if self.default_factory is None:
                raise KeyError(key)
            self[key] = value = self.default_factory()
            return value

        def __reduce__(self):
            f = self.default_factory
            args = f is None and tuple() or f
            return type(self), args, None, None, self.iteritems()

        def copy(self):
            return self.__copy__()

        def __copy__(self):
            return type(self)(self.default_factory, self)

        def __deepcopy__(self):
            import copy
            return type(self)(self.default_factory,
                        copy.deepcopy(self.items()))

        def __repr__(self):
            return "defaultdict(%s, %s)" % (self.default_factory,
                                            dict.__repr__(self))
    import collections
    collections.defaultdict = defaultdict # Pickle needs this.

############## logging.LoggerAdapter ########################################

try:
    from logging import LoggerAdapter
except ImportError:
    class LoggerAdapter(object):

        def __init__(self, logger, extra):
            self.logger = logger
            self.extra = extra

        def setLevel(self, level):
            self.level = logging._checkLevel(level)

        def process(self, msg, kwargs):
            kwargs["extra"] = self.extra
            return msg, kwargs

        def debug(self, msg, *args, **kwargs):
            self.log(logging.DEBUG, msg, args, **kwargs)

        def info(self, msg, *args, **kwargs):
            self.log(logging.INFO, msg, *args, **kwargs)

        def warning(self, msg, *args, **kwargs):
            self.log(logging.WARNING, msg, *args, **kwargs)
        warn = warning

        def error(self, msg, *args, **kwargs):
            self.log(logging.ERROR, msg, *args, **kwargs)

        def exception(self, msg, *args, **kwargs):
            kwargs.setdefault("exc_info", 1)
            self.error(msg, *args, **kwargs)

        def critical(self, msg, *args, **kwargs):
            self.log(logging.CRITICAL, msg, *args, **kwargs)
        fatal = critical

        def log(self, level, msg, *args, **kwargs):
            if self.logger.isEnabledFor(level):
                msg, kwargs = self.process(msg, kwargs)
                self._log(level, msg, *args, **kwargs)

        def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
                func=None, extra=None):
            rv = logging.LogRecord(name, level, fn, lno,
                                   msg, args, exc_info, func)
            if extra is not None:
                for key, value in extra.items():
                    if key in ("message", "asctime") or key in rv.__dict__:
                        raise KeyError(
                                "Attempt to override %r in LogRecord" % key)
                    rv.__dict__[key] = value
            return rv

        def _log(self, level, msg, args, exc_info=None, extra=None):
            defcaller = "(unknown file)", 0, "(unknown function)"
            if logging._srcfile:
                # IronPython doesn't track Python frames, so findCaller
                # throws an exception on some versions of IronPython.
                # We trap it here so that IronPython can use logging.
                try:
                    fn, lno, func = self.logger.findCaller()
                except ValueError:
                    fn, lno, func = defcaller
            else:
                fn, lno, func = defcaller
            if exc_info:
                if not isinstance(exc_info, tuple):
                    exc_info = sys.exc_info()
            record = self.makeRecord(self.logger.name, level, fn, lno, msg,
                                     args, exc_info, func, extra)
            self.logger.handle(record)

        def isEnabledFor(self, level, *args, **kwargs):
            return self.logger.isEnabledFor(level, *args, **kwargs)

        def addHandler(self, hdlr):
            self.logger.addHandler(hdlr)

        def removeHandler(self, hdlr):
            self.logger.removeHandler(hdlr)

############## itertools.izip_longest #######################################

try:
    from itertools import izip_longest
except ImportError:
    import itertools
    def izip_longest(*args, **kwds):
        fillvalue = kwds.get("fillvalue")

        def sentinel(counter=([fillvalue] * (len(args) - 1)).pop):
            yield counter() # yields the fillvalue, or raises IndexError

        fillers = itertools.repeat(fillvalue)
        iters = [itertools.chain(it, sentinel(), fillers)
                    for it in args]
        try:
            for tup in itertools.izip(*iters):
                yield tup
        except IndexError:
            pass

############## itertools.chain.from_iterable ################################
from itertools import chain

def _compat_chain_from_iterable(iterables):
    for it in iterables:
        for element in it:
            yield element

try:
    chain_from_iterable = getattr(chain, "from_iterable")
except AttributeError:
    chain_from_iterable = _compat_chain_from_iterable
<|endoftext|>