<commit_before>import os
import sys

import sublime
import sublime_plugin

lib_path = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'libs'))
sys.path.append(lib_path)

import clang.cindex

# globals
shared_settings = None
syntax_checker = None

class Settings:
    def __init__(self):
        self.settings = sublime.load_settings('Clime.sublime-settings')

    def get(self, key_name):
        return self.settings.get(key_name)

    def has(self, key_name):
        return self.settings.has(key_name)

class TranslationUnit:
    def __init__(self, view, index):
        self.view = view
        self.index = index
        self.error_regions = []
        self.errors_by_line = {}
        self.translation_unit = self.index.parse(view.file_name())
        self.calculate_error_regions()

    def reparse(self):
        self.translation_unit.reparse(unsaved_files=[(self.view.file_name(), self.view.substr(sublime.Region(0, self.view.size())))])
        self.calculate_error_regions()

    def calculate_error_regions(self):
        self.error_regions = []
        self.errors_by_line = {}
        for diagnostic in self.translation_unit.diagnostics:
            self.error_regions.append(self.view.full_line(self.view.text_point(diagnostic.location.line - 1, diagnostic.location.column - 1)))
            self.errors_by_line[diagnostic.location.line - 1] = diagnostic.spelling

    def error_for_line(self, line_number):
        if line_number in self.errors_by_line:
            return self.errors_by_line[line_number]
        return None

class SyntaxChecker:
    def __init__(self):
        self.index = clang.cindex.Index.create()
        self.translation_units = {}

    def create_translation_unit(self, view):
        if view.id() in self.translation_units:
            return True

        file_name, file_extension = os.path.splitext(view.file_name() or '')
        if not file_extension in shared_settings.get('file_extensions'):
            return False

        self.translation_units[view.id()] = TranslationUnit(view, self.index)
        self.show_diagnostics(view)
        return True

    def reparse_translation_unit(self, view):
        if not view.id() in self.translation_units:
            return

        self.translation_units[view.id()].reparse()
        self.show_diagnostics(view)

    def show_diagnostics(self, view):
        if not view.id() in self.translation_units:
            return;

        view.erase_regions('clime_errors')
        error_regions = self.translation_units[view.id()].error_regions
        if error_regions:
            view.add_regions('clime_errors', error_regions, 'error', 'cross', sublime.DRAW_SQUIGGLY_UNDERLINE | sublime.DRAW_NO_FILL | sublime.DRAW_NO_OUTLINE | sublime.PERSISTENT)
            self.show_status(view)

    def show_status(self, view):
        if not view.id() in self.translation_units:
            return;

        selection = view.sel()

        # don't attempt to show status if the user is using multi select
        if len(selection) > 1:
            view.erase_status('clime_error')
            return

        row, col = view.rowcol(selection[0].begin())

        error_status = self.translation_units[view.id()].error_for_line(row)
        if (error_status):
            view.set_status('clime_error', error_status)
        else:
            view.erase_status('clime_error')

def plugin_loaded():
    global shared_settings
    global translation_unit_manager
    global syntax_checker

    shared_settings = Settings()

    if not (clang.cindex.Config.library_path or clang.cindex.Config.library_file):
        if shared_settings.has('libclang_path'):
            clang.cindex.Config.set_library_path(shared_settings.get('libclang_path'))
        elif sublime.platform() == 'osx':
            clang.cindex.Config.set_library_file('/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/libclang.dylib')

    syntax_checker = SyntaxChecker()

class ClimeSyntaxCheckCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        syntax_checker.reparse_translation_unit(self.view)

class ClimeEventListener(sublime_plugin.EventListener):
    def __init__(self):
        self.pendingSyntaxChecks = set()
        self.should_check_syntax = {}

    def on_modified_async(self, view):
        if not view.id() in self.should_check_syntax:
            self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)
        if self.should_check_syntax[view.id()]:
            self.run_syntax_check(view)

    def on_save_async(self, view):
        self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)

    def on_load_async(self, view):
        self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)

    def on_selection_modified_async(self, view):
        syntax_checker.show_status(view)

    def run_syntax_check(self, view):
        if view.id() in self.pendingSyntaxChecks or not view.file_name():
            return

        self.pendingSyntaxChecks.add(view.id())
        sublime.set_timeout(lambda: self.run_syntax_check_callback(view), 1000)

    def run_syntax_check_callback(self, view):
        self.pendingSyntaxChecks.discard(view.id())
        view.run_command('clime_syntax_check')

<commit_msg>Create translation unit on activated event to handle already open files<commit_after>import os
import sys

import sublime
import sublime_plugin

lib_path = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'libs'))
sys.path.append(lib_path)

import clang.cindex

# globals
shared_settings = None
syntax_checker = None

class Settings:
    def __init__(self):
        self.settings = sublime.load_settings('Clime.sublime-settings')

    def get(self, key_name):
        return self.settings.get(key_name)

    def has(self, key_name):
        return self.settings.has(key_name)

class TranslationUnit:
    def __init__(self, view, index):
        self.view = view
        self.index = index
        self.error_regions = []
        self.errors_by_line = {}
        self.translation_unit = self.index.parse(view.file_name())
        self.calculate_error_regions()

    def reparse(self):
        self.translation_unit.reparse(unsaved_files=[(self.view.file_name(), self.view.substr(sublime.Region(0, self.view.size())))])
        self.calculate_error_regions()

    def calculate_error_regions(self):
        self.error_regions = []
        self.errors_by_line = {}
        for diagnostic in self.translation_unit.diagnostics:
            self.error_regions.append(self.view.full_line(self.view.text_point(diagnostic.location.line - 1, diagnostic.location.column - 1)))
            self.errors_by_line[diagnostic.location.line - 1] = diagnostic.spelling

    def error_for_line(self, line_number):
        if line_number in self.errors_by_line:
            return self.errors_by_line[line_number]
        return None

class SyntaxChecker:
    def __init__(self):
        self.index = clang.cindex.Index.create()
        self.translation_units = {}

    def create_translation_unit(self, view):
        if view.id() in self.translation_units:
            return True

        file_name, file_extension = os.path.splitext(view.file_name() or '')
        if not file_extension in shared_settings.get('file_extensions'):
            return False

        self.translation_units[view.id()] = TranslationUnit(view, self.index)
        self.show_diagnostics(view)
        return True

    def reparse_translation_unit(self, view):
        if not view.id() in self.translation_units:
            return

        self.translation_units[view.id()].reparse()
        self.show_diagnostics(view)

    def show_diagnostics(self, view):
        if not view.id() in self.translation_units:
            return;

        view.erase_regions('clime_errors')
        error_regions = self.translation_units[view.id()].error_regions
        if error_regions:
            view.add_regions('clime_errors', error_regions, 'error', 'cross', sublime.DRAW_SQUIGGLY_UNDERLINE | sublime.DRAW_NO_FILL | sublime.DRAW_NO_OUTLINE | sublime.PERSISTENT)
            self.show_status(view)

    def show_status(self, view):
        if not view.id() in self.translation_units:
            return;

        selection = view.sel()

        # don't attempt to show status if the user is using multi select
        if len(selection) > 1:
            view.erase_status('clime_error')
            return

        row, col = view.rowcol(selection[0].begin())

        error_status = self.translation_units[view.id()].error_for_line(row)
        if (error_status):
            view.set_status('clime_error', error_status)
        else:
            view.erase_status('clime_error')

def plugin_loaded():
    global shared_settings
    global translation_unit_manager
    global syntax_checker

    shared_settings = Settings()

    if not (clang.cindex.Config.library_path or clang.cindex.Config.library_file):
        if shared_settings.has('libclang_path'):
            clang.cindex.Config.set_library_path(shared_settings.get('libclang_path'))
        elif sublime.platform() == 'osx':
            clang.cindex.Config.set_library_file('/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/libclang.dylib')

    syntax_checker = SyntaxChecker()

class ClimeSyntaxCheckCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        syntax_checker.reparse_translation_unit(self.view)

class ClimeEventListener(sublime_plugin.EventListener):
    def __init__(self):
        self.pendingSyntaxChecks = set()
        self.should_check_syntax = {}

    def on_modified_async(self, view):
        if not view.id() in self.should_check_syntax:
            self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)
        if self.should_check_syntax[view.id()]:
            self.run_syntax_check(view)

    def on_save_async(self, view):
        self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)

    def on_load_async(self, view):
        self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)

    def on_activated_async(self, view):
        if not view.id() in self.should_check_syntax:
            self.should_check_syntax[view.id()] = syntax_checker.create_translation_unit(view)

    def on_selection_modified_async(self, view):
        syntax_checker.show_status(view)

    def run_syntax_check(self, view):
        if view.id() in self.pendingSyntaxChecks or not view.file_name():
            return

        self.pendingSyntaxChecks.add(view.id())
        sublime.set_timeout(lambda: self.run_syntax_check_callback(view), 1000)

    def run_syntax_check_callback(self, view):
        self.pendingSyntaxChecks.discard(view.id())
        view.run_command('clime_syntax_check')

<|endoftext|>


<commit_before># Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file "docs/LICENSE" for copying permission.

import os
import hashlib

import lib.maec.maec11 as maec
from lib.cuckoo.common.abstracts import Report
from lib.cuckoo.common.exceptions import CuckooReportError
from lib.cuckoo.common.utils import datetime_to_iso

class Report(Report):
    """Generates a MAEC 1.1 report."""

    def run(self, results):
        """Writes report.
        @param results: Cuckoo results dict.
        @raise CuckooReportError: if fails to write report.
        """
        self.idMap = {}
        # Save results
        self.results = results
        # Build MAEC doc
        self.addBundle()
        self.addPools()
        self.addAnalysis()
        self.addActions()
        # Write report.
        self.output()

    def addBundle(self):
        """Generates MAEC bundle structure."""
        if self.results["target"]["category"] == "file":
            self.idMap["prefix"] = "maec:%s" % self.results["target"]["file"]["md5"]
        elif self.results["target"]["category"] == "url":
            self.idMap["urlmd5"] = hashlib.md5(self.results["target"]["url"]).hexdigest()
            self.idMap["prefix"] = "maec:%s" % self.idMap["urlmd5"]
        else:
            raise CuckooReportError("Unknown target type")

        # Generate bundle
        self.m = maec.BundleType(
                                id = "%s:bnd:1" % self.idMap["prefix"],
                                schema_version="1.1"
                                )
        # Analyses
        self.analyses = maec.AnalysesType()
        self.m.set_Analyses(self.analyses)
        # Actions
        self.actions = maec.ActionsType()
        self.m.set_Actions(self.actions)
        # Behaviors
        self.behaviors = maec.BehaviorsType()
        self.m.set_Behaviors(self.behaviors)
        # Pools
        self.pools = maec.PoolsType()
        self.m.set_Pools(self.pools)

    def getActionId(self):
        """Get a new Action ID.
        @return: new ID.
        """
        try:
            self.actionId = self.actionId + 1
        except AttributeError:
            self.actionId = 1
        return self.actionId

    def getObjectId(self):
        """Get a new Object ID.
        @requires: new ID.
        """
        try:
            self.objectId = self.objectId + 1
        except AttributeError:
            self.objectId = 1
        return self.objectId

    def getProcessId(self):
        """Get a new Process ID.
        @requires: new ID.
        """
        try:
            self.processId = self.processId + 1
        except AttributeError:
            self.processId = 1
        return self.processId

    def getActImpId(self):
        """Get a new Action Implementation ID.
        @requires: new ID.
        """
        try:
            self.actImpId = self.actImpId + 1
        except AttributeError:
            self.actImpId = 1
        return self.actImpId

    def getApiCallId(self):
        """Get a new API Call ID.
        @requires: new ID.
        """
        try:
            self.apiCallId = self.apiCallId + 1
        except AttributeError:
            self.apiCallId = 1
        return self.apiCallId

    def addActions(self):
        """Adds actions section."""
        # Processes
        for process in self.results["behavior"]["processes"]:
            self.createActionAPI(process)
        # Network
        if "network" in self.results and isinstance(self.results["network"], dict):
            if "udp" in self.results["network"] and isinstance(self.results["network"]["udp"], list):
                for pkt in self.results["network"]["udp"]:
                    self.createActionNet(pkt)
            if "tcp" in self.results["network"] and isinstance(self.results["network"]["tcp"], list):
                for pkt in self.results["network"]["tcp"]:
                    self.createActionNet(pkt)

    def createActionNet(self, packet):
        """Create a network action.
        @return: action.
        """
        act = maec.ActionType(
                              id = "%s:act:%s" % (self.idMap["prefix"], self.getActionId()),
                              )
        act.set_Action_Initiator(maec.Action_InitiatorType(
                                                           type_ = "Process",
                                                           Initiator_Object = maec.ObjectReferenceType(
                                                                                type_ = "Object",
                                                                                object_id = self.idMap["subject"]
                                                                                )
                                                           )
                                 )
        ai = maec.ActionImplementationType(
                                           type_ = "Other",
                                           id = "%s:imp:%s" % (self.idMap["prefix"], self.getActImpId()),
                                           )
        net = maec.Network_Action_AttributesType(
                                                 Internal_Port = packet["sport"],
                                                 External_Port = packet["dport"],
                                                 Internal_IP_Address = packet["src"],
                                                 External_IP_Address = packet["dst"]
                                                 )
        ai.set_Network_Action_Attributes(net)
        act.set_Action_Implementation(ai)
        self.actions.add_Action(act)

    def createActionAPI(self, process):
        """Creates an action object which describes a process.
        @param process: process from cuckoo dict.
        """
        pid = self.getProcessId()
        pos = 1

        for call in process["calls"]:
            act = maec.ActionType(
                                  id = "%s:act:%s" % (self.idMap["prefix"], self.getActionId()),
                                  ordinal_position = pos,
                                  timestamp = call["timestamp"],
                                  successful = call["category"]
                                  )
            try:
                initiator = self.idMap[process["process_name"]]
            except KeyError:
                initiator = self.idMap["subject"]
            act.set_Action_Initiator(maec.Action_InitiatorType(
                                                               type_ = "Process",
                                                               Initiator_Object = maec.ObjectReferenceType(
                                                                                type_ = "Object",
                                                                                object_id = initiator
                                                                                )
                                                               )
                                     )
            ai = maec.ActionImplementationType(
                                               type_ = "API_Call",
                                               id = "%s:imp:%s" % (self.idMap["prefix"], self.getActImpId()),
                                               )
            apicall = maec.APICallType(
                                       id = "%s:api:%s" % (self.idMap["prefix"], self.getApiCallId()),
                                       apifunction_name = call["api"],
                                       ReturnValue = call["return"]
                                       )
            apos = 1
            for arg in call["arguments"]:
                apicall.add_APICall_Parameter(maec.APICall_ParameterType(
                                                                         ordinal_position = apos,
                                                                         Name = arg["name"],
                                                                         Value = arg["value"]
                                                                         )
                                              )
                apos = apos + 1
            ai.set_API_Call(apicall)

            act.set_Action_Implementation(ai)
            self.actions.add_Action(act)
            pos = pos + 1

    def createFileObj(self, file):
        """Creates a File object.
        @param file: file dict from Cuckoo dict.
        @requires: file object.
        """
        obj = maec.ObjectType(
                              id = "%s:obj:%s" % (self.idMap["prefix"], self.getObjectId()),
                              object_name = file["name"],
                              type_ = "File"
                              )
        self.idMap[file["name"]] = obj.id

        fs = maec.File_System_Object_AttributesType()
        fs.set_File_Type(maec.File_TypeType(
                                            type_ = file["type"]
                                            )
                         )
        # Add static analysis if file obj is analysis subject.
        if self.results["target"]["category"] == "file" and file["md5"] == self.results["target"]["file"]["md5"] and self.results["static"]:
            if len(self.results["static"]) > 0:
                pe = maec.PE_Binary_AttributesType(dll_count = self.results["static"]["imported_dll_count"])
                # PE exports
                if len(self.results["static"]["pe_exports"]) > 0:
                    exports = maec.ExportsType()
                    pe.set_Exports(exports)
                    for x in self.results["static"]["pe_exports"]:
                        exp = maec.PEExportType(
                                                Function_Name = x["name"],
                                                Ordinal = x["ordinal"],
                                                Entry_Point = x["address"]
                                                )
                        exports.add_Export(exp)
                # PE Imports
                if len(self.results["static"]["pe_imports"]) > 0:
                    imports = maec.ImportsType()
                    pe.set_Imports(imports)
                    for x in self.results["static"]["pe_imports"]:
                        imp = maec.PEImportType(
                                                File_Name = x["dll"]
                                                )
                        # Imported functions
                        funcs = maec.Imported_FunctionsType()
                        imp.set_Imported_Functions(funcs)
                        for i in x["imports"]:
                            f = maec.Imported_FunctionType(
                                                           Function_Name = i["name"],
                                                           Virtual_Address = i["address"]
                                                           )
                            funcs.add_Imported_Function(f)
                        imports.add_Import(imp)
                # Resources
                if len(self.results["static"]["pe_resources"]) > 0:
                    resources = maec.ResourcesType()
                    pe.set_Resources(resources)
                    for r in self.results["static"]["pe_resources"]:
                        res = maec.PEResourceType(
                                                Name = r["name"]
                                                )
                        resources.add_Resource(res)
                # Sections
                if len(self.results["static"]["pe_sections"]) > 0:
                    sections = maec.SectionsType()
                    pe.set_Sections(sections)
                    for s in self.results["static"]["pe_sections"]:
                        sec = maec.PESectionType(
                                                Virtual_Size = int(s["virtual_size"], 16),
                                                Virtual_Address = s["virtual_address"],
                                                Entropy = s["entropy"],
                                                Section_Name = s["name"]
                                                )
                        sections.add_Section(sec)
                # Version info
                if len(self.results["static"]["pe_versioninfo"]) > 0:
                    version = maec.Version_BlockType()
                    pe.set_Version_Block(version)
                    for k in self.results["static"]["pe_versioninfo"]:
                        if k["name"] == "ProductVersion":
                            version.set_Product_Version_Text(k["value"])
                        if k["name"] == "ProductName":
                            version.set_Product_Name(k["value"])
                        if k["name"] == "FileVersion":
                            version.set_File_Version_Text(k["value"])
                        if k["name"] == "CompanyName":
                            version.set_Company_Name(k["value"])
                        if k["name"] == "OriginalFilename":
                            version.set_Original_File_Name(k["value"])
                fs.set_File_Type_Attributes(maec.File_Type_AttributesType(pe))
        h = maec.HashesType()
        h.add_Hash(maec.HashType(
                                 type_ = "MD5",
                                 Hash_Value = file["md5"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "SHA1",
                                 Hash_Value = file["sha1"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "SHA256",
                                 Hash_Value = file["sha256"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "Other",
                                 other_type = "SHA512",
                                 Hash_Value = file["sha512"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "Other",
                                 other_type = "CRC32",
                                 Hash_Value = file["crc32"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "Other",
                                 other_type = "SSDEEP",
                                 Hash_Value = file["ssdeep"]
                                 ))
        fs.set_Hashes(h)
        obj.set_File_System_Object_Attributes(fs)
        obj.set_Object_Size(maec.Object_SizeType(
                                                 units = "Bytes",
                                                 valueOf_ = file["size"]
                                                 ))
        return obj

    def createSubjectFile(self, file):
        """Create a file subject entity.
        @param file: file as in cuckoo dict.
        @return: subject object.
        """
        subject = maec.SubjectType()
        subject.set_Object_Reference(maec.ObjectReferenceType(
                                                              type_ = "Object",
                                                              object_id = self.idMap[file["name"]]
                                                              )
                                     )
        self.idMap["subject"] = self.idMap[file["name"]]
        return subject

    def createSubjectUrl(self, url):
        """Create a URL subject entity.
        @param url: URL path.
        @return: subject object.
        """
        subject = maec.SubjectType()
        subject.set_Object_Reference(maec.ObjectReferenceType(
                                                              type_ = "URI",
                                                              object_id = self.idMap["urlmd5"]
                                                              )
                                     )
        self.idMap["subject"] = self.idMap["urlmd5"]
        return subject

    def createTools(self):
        """Creates a tools element.
        @return: Tools object.
        """
        tools = maec.Tools_UsedType()
        tool = maec.ToolType(
                             id = "%s:tol:1" % self.idMap["prefix"],
                             Name = "Cuckoo Sandbox",
                             Version = self.results["info"]["version"],
                             Organization = "http://www.cuckoosandbox.org"
                             )
        tools.add_Tool(tool)
        return tools

    def addAnalysis(self):
        """Adds analysis header."""
        analysis = maec.AnalysisType(
                                id = "%s:ana:1" % self.idMap["prefix"],
                                analysis_method = "Dynamic",
                                start_datetime = datetime_to_iso(self.results["info"]["started"]),
                                complete_datetime = datetime_to_iso(self.results["info"]["ended"]),
                                lastupdate_datetime = datetime_to_iso(self.results["info"]["ended"])
                                )
        # Add tool
        analysis.set_Tools_Used(self.createTools())
        # Add subject
        if self.results["target"]["category"] == "file":
            analysis.add_Subject(self.createSubjectFile(self.results["target"]["file"]))
        elif self.results["target"]["category"] == "url":
            analysis.add_Subject(self.createSubjectUrl(self.results["target"]["url"]))

        self.analyses.add_Analysis(analysis)

    def addPools(self):
        """Adds Pools section."""
        objs = self.results["dropped"]
        if self.results["target"]["category"] == "file":
            objs.append(self.results["target"]["file"])
        pool = maec.Object_PoolType()
        for file in objs:
            pool.add_Object(self.createFileObj(file))
        self.pools.set_Object_Pool(pool)

    def output(self):
        """Writes report to disk."""
        try:
            report = open(os.path.join(self.reports_path, "report.maec-1.1.xml"), "w")
            report.write("<?xml version='1.0' ?>\n")
            report.write("<!--\n")
            report.write("Cuckoo Sandbox MAEC 1.1 malware analysis report\n")
            report.write("http://www.cuckoosandbox.org\n")
            report.write("-->\n")
            self.m.export(report, 0, namespace_="", name_="MAEC_Bundle", namespacedef_="xmlns='http://maec.mitre.org/XMLSchema/maec-core-1' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://maec.mitre.org/XMLSchema/maec-core-1 file:MAEC_v1.1.xsd'")
            report.close()
        except (TypeError, IOError) as e:
            raise CuckooReportError("Failed to generate MAEC 1.1 report: %s" % e)
<commit_msg>Renamed<commit_after># Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file "docs/LICENSE" for copying permission.

import os
import hashlib

import lib.maec.maec11 as maec
from lib.cuckoo.common.abstracts import Report
from lib.cuckoo.common.exceptions import CuckooReportError
from lib.cuckoo.common.utils import datetime_to_iso

class ReportMAEC(Report):
    """Generates a MAEC 1.1 report."""

    def run(self, results):
        """Writes report.
        @param results: Cuckoo results dict.
        @raise CuckooReportError: if fails to write report.
        """
        self.idMap = {}
        # Save results
        self.results = results
        # Build MAEC doc
        self.addBundle()
        self.addPools()
        self.addAnalysis()
        self.addActions()
        # Write report.
        self.output()

    def addBundle(self):
        """Generates MAEC bundle structure."""
        if self.results["target"]["category"] == "file":
            self.idMap["prefix"] = "maec:%s" % self.results["target"]["file"]["md5"]
        elif self.results["target"]["category"] == "url":
            self.idMap["urlmd5"] = hashlib.md5(self.results["target"]["url"]).hexdigest()
            self.idMap["prefix"] = "maec:%s" % self.idMap["urlmd5"]
        else:
            raise CuckooReportError("Unknown target type")

        # Generate bundle
        self.m = maec.BundleType(
                                id = "%s:bnd:1" % self.idMap["prefix"],
                                schema_version="1.1"
                                )
        # Analyses
        self.analyses = maec.AnalysesType()
        self.m.set_Analyses(self.analyses)
        # Actions
        self.actions = maec.ActionsType()
        self.m.set_Actions(self.actions)
        # Behaviors
        self.behaviors = maec.BehaviorsType()
        self.m.set_Behaviors(self.behaviors)
        # Pools
        self.pools = maec.PoolsType()
        self.m.set_Pools(self.pools)

    def getActionId(self):
        """Get a new Action ID.
        @return: new ID.
        """
        try:
            self.actionId = self.actionId + 1
        except AttributeError:
            self.actionId = 1
        return self.actionId

    def getObjectId(self):
        """Get a new Object ID.
        @requires: new ID.
        """
        try:
            self.objectId = self.objectId + 1
        except AttributeError:
            self.objectId = 1
        return self.objectId

    def getProcessId(self):
        """Get a new Process ID.
        @requires: new ID.
        """
        try:
            self.processId = self.processId + 1
        except AttributeError:
            self.processId = 1
        return self.processId

    def getActImpId(self):
        """Get a new Action Implementation ID.
        @requires: new ID.
        """
        try:
            self.actImpId = self.actImpId + 1
        except AttributeError:
            self.actImpId = 1
        return self.actImpId

    def getApiCallId(self):
        """Get a new API Call ID.
        @requires: new ID.
        """
        try:
            self.apiCallId = self.apiCallId + 1
        except AttributeError:
            self.apiCallId = 1
        return self.apiCallId

    def addActions(self):
        """Adds actions section."""
        # Processes
        for process in self.results["behavior"]["processes"]:
            self.createActionAPI(process)
        # Network
        if "network" in self.results and isinstance(self.results["network"], dict):
            if "udp" in self.results["network"] and isinstance(self.results["network"]["udp"], list):
                for pkt in self.results["network"]["udp"]:
                    self.createActionNet(pkt)
            if "tcp" in self.results["network"] and isinstance(self.results["network"]["tcp"], list):
                for pkt in self.results["network"]["tcp"]:
                    self.createActionNet(pkt)

    def createActionNet(self, packet):
        """Create a network action.
        @return: action.
        """
        act = maec.ActionType(
                              id = "%s:act:%s" % (self.idMap["prefix"], self.getActionId()),
                              )
        act.set_Action_Initiator(maec.Action_InitiatorType(
                                                           type_ = "Process",
                                                           Initiator_Object = maec.ObjectReferenceType(
                                                                                type_ = "Object",
                                                                                object_id = self.idMap["subject"]
                                                                                )
                                                           )
                                 )
        ai = maec.ActionImplementationType(
                                           type_ = "Other",
                                           id = "%s:imp:%s" % (self.idMap["prefix"], self.getActImpId()),
                                           )
        net = maec.Network_Action_AttributesType(
                                                 Internal_Port = packet["sport"],
                                                 External_Port = packet["dport"],
                                                 Internal_IP_Address = packet["src"],
                                                 External_IP_Address = packet["dst"]
                                                 )
        ai.set_Network_Action_Attributes(net)
        act.set_Action_Implementation(ai)
        self.actions.add_Action(act)

    def createActionAPI(self, process):
        """Creates an action object which describes a process.
        @param process: process from cuckoo dict.
        """
        pid = self.getProcessId()
        pos = 1

        for call in process["calls"]:
            act = maec.ActionType(
                                  id = "%s:act:%s" % (self.idMap["prefix"], self.getActionId()),
                                  ordinal_position = pos,
                                  timestamp = call["timestamp"],
                                  successful = call["category"]
                                  )
            try:
                initiator = self.idMap[process["process_name"]]
            except KeyError:
                initiator = self.idMap["subject"]
            act.set_Action_Initiator(maec.Action_InitiatorType(
                                                               type_ = "Process",
                                                               Initiator_Object = maec.ObjectReferenceType(
                                                                                type_ = "Object",
                                                                                object_id = initiator
                                                                                )
                                                               )
                                     )
            ai = maec.ActionImplementationType(
                                               type_ = "API_Call",
                                               id = "%s:imp:%s" % (self.idMap["prefix"], self.getActImpId()),
                                               )
            apicall = maec.APICallType(
                                       id = "%s:api:%s" % (self.idMap["prefix"], self.getApiCallId()),
                                       apifunction_name = call["api"],
                                       ReturnValue = call["return"]
                                       )
            apos = 1
            for arg in call["arguments"]:
                apicall.add_APICall_Parameter(maec.APICall_ParameterType(
                                                                         ordinal_position = apos,
                                                                         Name = arg["name"],
                                                                         Value = arg["value"]
                                                                         )
                                              )
                apos = apos + 1
            ai.set_API_Call(apicall)

            act.set_Action_Implementation(ai)
            self.actions.add_Action(act)
            pos = pos + 1

    def createFileObj(self, file):
        """Creates a File object.
        @param file: file dict from Cuckoo dict.
        @requires: file object.
        """
        obj = maec.ObjectType(
                              id = "%s:obj:%s" % (self.idMap["prefix"], self.getObjectId()),
                              object_name = file["name"],
                              type_ = "File"
                              )
        self.idMap[file["name"]] = obj.id

        fs = maec.File_System_Object_AttributesType()
        fs.set_File_Type(maec.File_TypeType(
                                            type_ = file["type"]
                                            )
                         )
        # Add static analysis if file obj is analysis subject.
        if self.results["target"]["category"] == "file" and file["md5"] == self.results["target"]["file"]["md5"] and self.results["static"]:
            if len(self.results["static"]) > 0:
                pe = maec.PE_Binary_AttributesType(dll_count = self.results["static"]["imported_dll_count"])
                # PE exports
                if len(self.results["static"]["pe_exports"]) > 0:
                    exports = maec.ExportsType()
                    pe.set_Exports(exports)
                    for x in self.results["static"]["pe_exports"]:
                        exp = maec.PEExportType(
                                                Function_Name = x["name"],
                                                Ordinal = x["ordinal"],
                                                Entry_Point = x["address"]
                                                )
                        exports.add_Export(exp)
                # PE Imports
                if len(self.results["static"]["pe_imports"]) > 0:
                    imports = maec.ImportsType()
                    pe.set_Imports(imports)
                    for x in self.results["static"]["pe_imports"]:
                        imp = maec.PEImportType(
                                                File_Name = x["dll"]
                                                )
                        # Imported functions
                        funcs = maec.Imported_FunctionsType()
                        imp.set_Imported_Functions(funcs)
                        for i in x["imports"]:
                            f = maec.Imported_FunctionType(
                                                           Function_Name = i["name"],
                                                           Virtual_Address = i["address"]
                                                           )
                            funcs.add_Imported_Function(f)
                        imports.add_Import(imp)
                # Resources
                if len(self.results["static"]["pe_resources"]) > 0:
                    resources = maec.ResourcesType()
                    pe.set_Resources(resources)
                    for r in self.results["static"]["pe_resources"]:
                        res = maec.PEResourceType(
                                                Name = r["name"]
                                                )
                        resources.add_Resource(res)
                # Sections
                if len(self.results["static"]["pe_sections"]) > 0:
                    sections = maec.SectionsType()
                    pe.set_Sections(sections)
                    for s in self.results["static"]["pe_sections"]:
                        sec = maec.PESectionType(
                                                Virtual_Size = int(s["virtual_size"], 16),
                                                Virtual_Address = s["virtual_address"],
                                                Entropy = s["entropy"],
                                                Section_Name = s["name"]
                                                )
                        sections.add_Section(sec)
                # Version info
                if len(self.results["static"]["pe_versioninfo"]) > 0:
                    version = maec.Version_BlockType()
                    pe.set_Version_Block(version)
                    for k in self.results["static"]["pe_versioninfo"]:
                        if k["name"] == "ProductVersion":
                            version.set_Product_Version_Text(k["value"])
                        if k["name"] == "ProductName":
                            version.set_Product_Name(k["value"])
                        if k["name"] == "FileVersion":
                            version.set_File_Version_Text(k["value"])
                        if k["name"] == "CompanyName":
                            version.set_Company_Name(k["value"])
                        if k["name"] == "OriginalFilename":
                            version.set_Original_File_Name(k["value"])
                fs.set_File_Type_Attributes(maec.File_Type_AttributesType(pe))
        h = maec.HashesType()
        h.add_Hash(maec.HashType(
                                 type_ = "MD5",
                                 Hash_Value = file["md5"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "SHA1",
                                 Hash_Value = file["sha1"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "SHA256",
                                 Hash_Value = file["sha256"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "Other",
                                 other_type = "SHA512",
                                 Hash_Value = file["sha512"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "Other",
                                 other_type = "CRC32",
                                 Hash_Value = file["crc32"]
                                 ))
        h.add_Hash(maec.HashType(
                                 type_ = "Other",
                                 other_type = "SSDEEP",
                                 Hash_Value = file["ssdeep"]
                                 ))
        fs.set_Hashes(h)
        obj.set_File_System_Object_Attributes(fs)
        obj.set_Object_Size(maec.Object_SizeType(
                                                 units = "Bytes",
                                                 valueOf_ = file["size"]
                                                 ))
        return obj

    def createSubjectFile(self, file):
        """Create a file subject entity.
        @param file: file as in cuckoo dict.
        @return: subject object.
        """
        subject = maec.SubjectType()
        subject.set_Object_Reference(maec.ObjectReferenceType(
                                                              type_ = "Object",
                                                              object_id = self.idMap[file["name"]]
                                                              )
                                     )
        self.idMap["subject"] = self.idMap[file["name"]]
        return subject

    def createSubjectUrl(self, url):
        """Create a URL subject entity.
        @param url: URL path.
        @return: subject object.
        """
        subject = maec.SubjectType()
        subject.set_Object_Reference(maec.ObjectReferenceType(
                                                              type_ = "URI",
                                                              object_id = self.idMap["urlmd5"]
                                                              )
                                     )
        self.idMap["subject"] = self.idMap["urlmd5"]
        return subject

    def createTools(self):
        """Creates a tools element.
        @return: Tools object.
        """
        tools = maec.Tools_UsedType()
        tool = maec.ToolType(
                             id = "%s:tol:1" % self.idMap["prefix"],
                             Name = "Cuckoo Sandbox",
                             Version = self.results["info"]["version"],
                             Organization = "http://www.cuckoosandbox.org"
                             )
        tools.add_Tool(tool)
        return tools

    def addAnalysis(self):
        """Adds analysis header."""
        analysis = maec.AnalysisType(
                                id = "%s:ana:1" % self.idMap["prefix"],
                                analysis_method = "Dynamic",
                                start_datetime = datetime_to_iso(self.results["info"]["started"]),
                                complete_datetime = datetime_to_iso(self.results["info"]["ended"]),
                                lastupdate_datetime = datetime_to_iso(self.results["info"]["ended"])
                                )
        # Add tool
        analysis.set_Tools_Used(self.createTools())
        # Add subject
        if self.results["target"]["category"] == "file":
            analysis.add_Subject(self.createSubjectFile(self.results["target"]["file"]))
        elif self.results["target"]["category"] == "url":
            analysis.add_Subject(self.createSubjectUrl(self.results["target"]["url"]))

        self.analyses.add_Analysis(analysis)

    def addPools(self):
        """Adds Pools section."""
        objs = self.results["dropped"]
        if self.results["target"]["category"] == "file":
            objs.append(self.results["target"]["file"])
        pool = maec.Object_PoolType()
        for file in objs:
            pool.add_Object(self.createFileObj(file))
        self.pools.set_Object_Pool(pool)

    def output(self):
        """Writes report to disk."""
        try:
            report = open(os.path.join(self.reports_path, "report.maec-1.1.xml"), "w")
            report.write("<?xml version='1.0' ?>\n")
            report.write("<!--\n")
            report.write("Cuckoo Sandbox MAEC 1.1 malware analysis report\n")
            report.write("http://www.cuckoosandbox.org\n")
            report.write("-->\n")
            self.m.export(report, 0, namespace_="", name_="MAEC_Bundle", namespacedef_="xmlns='http://maec.mitre.org/XMLSchema/maec-core-1' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://maec.mitre.org/XMLSchema/maec-core-1 file:MAEC_v1.1.xsd'")
            report.close()
        except (TypeError, IOError) as e:
            raise CuckooReportError("Failed to generate MAEC 1.1 report: %s" % e)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   Debian Changes Bot
#   Copyright (C) 2008 Chris Lamb <chris@chris-lamb.co.uk>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as
#   published by the Free Software Foundation, either version 3 of the
#   License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import time
import random
import urllib
import supybot
import threading

from supybot.commands import wrap, many
from supybot import ircdb, log, schedule

from DebianDevelChangesBot.mailparsers import get_message
from DebianDevelChangesBot.datasources import get_datasources, TestingRCBugs, \
    NewQueue, RmQueue, Maintainer
from DebianDevelChangesBot.utils import parse_mail, FifoReader, colourise, \
    rewrite_topic, madison, bug_synopsis, format_email_address, popcon

class DebianDevelChanges(supybot.callbacks.Plugin):
    threaded = True

    def __init__(self, irc):
        self.__parent = super(DebianDevelChanges, self)
        self.__parent.__init__(irc)
        self.irc = irc
        self.topic_lock = threading.Lock()

        fr = FifoReader()
        fifo_loc = '/var/run/debian-devel-changes-bot/fifo'
        fr.start(self._email_callback, fifo_loc)

        self.queued_topics = {}
        self.last_n_messages = []

        # Schedule datasource updates
        for klass, interval, name in get_datasources():
            try:
                schedule.removePeriodicEvent(name)
            except KeyError:
                pass

            def wrapper(klass=klass):
                klass().update()
                self._topic_callback()

            schedule.addPeriodicEvent(wrapper, interval, name, now=False)
            schedule.addEvent(wrapper, time.time() + 1)

    def die(self):
        FifoReader().stop()
        for _, _, name in get_datasources():
            try:
                schedule.removePeriodicEvent(name)
            except KeyError:
                # A newly added event may not exist, ignore exception.
                pass

    def _email_callback(self, fileobj):
        try:
            email = parse_mail(fileobj)
            msg = get_message(email)

            if not msg:
                return

            txt = colourise(msg.for_irc())

            # Simple flood/duplicate detection
            if txt in self.last_n_messages:
                return
            self.last_n_messages.insert(0, txt)
            self.last_n_messages = self.last_n_messages[:20]

            log.info("Processing message")

            for channel in self.irc.state.channels:
                log.info("Processing channel %s", channel)

                package_regex = self.registryValue(
                    'package_regex',
                    channel,
                ) or 'a^' # match nothing by default

                if not re.search(package_regex, msg.package):
                    log.info("Not sending message to this channel package regex does not match")
                    continue

                distribution_regex = self.registryValue(
                    'distribution_regex',
                    channel,
                )

                log.info("Distribution regex %r", distribution_regex)

                if distribution_regex:
                    if not hasattr(msg, 'distribution'):
                        # If this channel has a distribution regex, don't
                        # bother continuing unless the message actually has a
                        # distribution. This filters security messages, etc.
                        log.info("Skipping channel/message as message has no distribution")
                        continue

                    if not re.search(distribution_regex, msg.distribution):
                        # Distribution doesn't match regex; don't send this
                        # message.
                        log.info("Skipping channel/message as message as distribution %r does not match", msg.distribution)
                        continue

                log.info("Sending message to this channel")

                ircmsg = supybot.ircmsgs.privmsg(channel, txt)
                self.irc.queueMsg(ircmsg)

        except:
            log.exception('Uncaught exception')

    def _topic_callback(self):
        self.topic_lock.acquire()

        sections = {
            lambda: len(TestingRCBugs().get_bugs()): 'RC bug count:',
            NewQueue().get_size: 'NEW queue:',
            RmQueue().get_size: 'RM queue:',
        }

        try:
            values = {}
            for callback, prefix in sections.iteritems():
                values[callback] = callback()

            for channel in self.irc.state.channels:
                new_topic = topic = self.irc.state.getTopic(channel)

                for callback, prefix in sections.iteritems():
                    if values[callback]:
                        new_topic = rewrite_topic(new_topic, prefix, values[callback])

                if topic != new_topic:
                    log.info("Queueing change of topic in #%s to '%s'" % (channel, new_topic))
                    self.queued_topics[channel] = new_topic

                    event_name = '%s_topic' % channel
                    try:
                        schedule.removeEvent(event_name)
                    except KeyError:
                        pass
                    schedule.addEvent(lambda channel=channel: self._update_topic(channel),
                        time.time() + 60, event_name)
        finally:
            self.topic_lock.release()

    def _update_topic(self, channel):
        self.topic_lock.acquire()
        try:
            try:
                new_topic = self.queued_topics[channel]
                log.info("Changing topic in #%s to '%s'" % (channel, new_topic))
                self.irc.queueMsg(supybot.ircmsgs.topic(channel, new_topic))
            except KeyError:
                pass
        finally:
            self.topic_lock.release()

    def greeting(self, prefix, irc, msg, args):
        num_bugs = len(TestingRCBugs().get_bugs())
        if type(num_bugs) is int:
            advice = random.choice((
                'Why not go and fix one?',
                'Why not peek at the list and find one?',
                'Stop IRCing and fix one! :]',
                'You realise they don\'t fix themselves, right?',
                'How about fixing yourself some caffeine and then poking at the bug list?',
            ))
            txt = "%s %s! There are currently %d RC bugs in wheezy. %s Try !random." % \
                (prefix, msg.nick, num_bugs, advice)
        else:
            txt = "%s %s!" % (prefix, msg.name)
        irc.reply(txt, prefixNick=False)

    def morning(self, *args):
        self.greeting('Good morning,', *args)
    morning = wrap(morning)
    yawn = wrap(morning)
    wakeywakey = wrap(morning)

    def night(self, *args):
        self.greeting( 'Good night,', *args)
    night = wrap(night)
    nn = wrap(night)
    goodnight = wrap(night)

    def sup(self, *args):
        self.greeting("'sup", *args)
    sup = wrap(sup)
    lo = wrap(sup)

    def rc(self, irc, msg, args):
        num_bugs = len(TestingRCBugs().get_bugs())
        if type(num_bugs) is int:
            irc.reply("There are %d release-critical bugs in the testing distribution. " \
                "See http://bts.turmzimmer.net/details.php?bydist=wheezy&igncontrib=on&ignnonfree=on" % num_bugs)
        else:
            irc.reply("No data at this time.")
    rc = wrap(rc)
    bugs = wrap(rc)

    def randombug(self, irc, msg, args):
        bug = random.choice(list(TestingRCBugs().get_bugs()))
        msg = "Your randomly chosen wheezy RC bug is: %s. Happy bugfixing!" % bug_synopsis(str(bug)).for_irc()
        irc.reply(colourise(msg))
    randombug = wrap(randombug)
    random = wrap(randombug)

    def update(self, irc, msg, args):
        if not ircdb.checkCapability(msg.prefix, 'owner'):
            irc.reply("You are not authorised to run this command.")
            return

        for klass, interval, name in get_datasources():
            klass().update()
            irc.reply("Updated %s." % name)
        self._topic_callback()
    update = wrap(update)

    def madison(self, irc, msg, args, package):
        try:
            lines = madison(package)
            if not lines:
                irc.reply('Did not get a response -- is "%s" a valid package?' % package)
                return

            field_styles = ('package', 'version', 'distribution', 'section')
            for line in lines:
                out = []
                fields = line.strip().split('|', len(field_styles))
                for style, data in zip(field_styles, fields):
                    out.append('[%s]%s' % (style, data))
                irc.reply(colourise('[reset]|'.join(out)), prefixNick=False)
        except Exception, e:
            irc.reply("Error: %s" % e.message)
    madison = wrap(madison, ['text'])

    def bug(self, irc, msg, args, bug_string):
        try:
            msg = bug_synopsis(bug_string)
            if msg:
                irc.reply(colourise(msg.for_irc()), prefixNick=False)
        except ValueError:
            irc.reply('Could not parse bug number')
        except Exception, e:
            irc.reply("Error: %s" % e.message)

    bug = wrap(bug, ['text'])

    def get_pool_url(self, package):
        if package.startswith('lib'):
            return (package[:4], package)
        else:
            return (package[:1], package)

    def _maintainer(self, irc, msg, args, items):
        for package in items:
            info = Maintainer().get_maintainer(package)
            if info:
                display_name = format_email_address("%s <%s>" % (info['name'], info['email']), max_domain=18)

                login = info['email']
                if login.endswith('@debian.org'):
                    login = login.replace('@debian.org', '')

                msg = "[desc]Maintainer for[reset] [package]%s[reset] [desc]is[reset] [by]%s[reset]: " % (package, display_name)
                msg += "[url]http://qa.debian.org/developer.php?login=%s[/url]" % login
            else:
                msg = 'Unknown source package "%s"' % package

            irc.reply(colourise(msg), prefixNick=False)
    maintainer = wrap(_maintainer, [many('anything')])
    maint = wrap(_maintainer, [many('anything')])
    who_maintains = wrap(_maintainer, [many('anything')])

    def _qa(self, irc, msg, args, items):
        for package in items:
            url = "http://packages.qa.debian.org/%s/%s.html" % self.get_pool_url(package)
            msg = "[desc]QA page for[reset] [package]%s[reset]: [url]%s[/url]" % (package, url)
            irc.reply(colourise(msg), prefixNick=False)
    qa = wrap(_qa, [many('anything')])
    overview = wrap(_qa, [many('anything')])
    package = wrap(_qa, [many('anything')])
    pkg = wrap(_qa, [many('anything')])
    srcpkg = wrap(_qa, [many('anything')])

    def _changelog(self, irc, msg, args, items):
        for package in items:
            url = "http://packages.debian.org/changelogs/pool/main/%s/%s/current/changelog" % self.get_pool_url(package)
            msg = "[desc]debian/changelog for[reset] [package]%s[reset]: [url]%s[/url]" % (package, url)
            irc.reply(colourise(msg), prefixNick=False)
    changelog = wrap(_changelog, [many('anything')])
    changes = wrap(_changelog, [many('anything')])

    def _copyright(self, irc, msg, args, items):
        for package in items:
            url = "http://packages.debian.org/changelogs/pool/main/%s/%s/current/copyright" % self.get_pool_url(package)
            msg = "[desc]debian/copyright for[reset] [package]%s[reset]: [url]%s[/url]" % (package, url)
            irc.reply(colourise(msg), prefixNick=False)
    copyright = wrap(_copyright, [many('anything')])

    def _buggraph(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]Bug graph for[reset] [package]%s[reset]: [url]http://people.debian.org/~glandium/bts/%s/%s.png[/url]" % \
                (package, package[0], package)
            irc.reply(colourise(msg), prefixNick=False)
    buggraph = wrap(_buggraph, [many('anything')])
    bug_graph = wrap(_buggraph, [many('anything')])

    def _buildd(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]buildd status for[reset] [package]%s[reset]: [url]http://buildd.debian.org/pkg.cgi?pkg=%s[/url]" % \
                (package, package)
            irc.reply(colourise(msg), prefixNick=False)
    buildd = wrap(_buildd, [many('anything')])

    def _buildde(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]experimental/backports.org buildd status for[reset] [package]%s[reset]: [url]http://experimental.ftbfs.de/build.php?pkg=%s[/url]" % \
                (package, package)
            irc.reply(colourise(msg), prefixNick=False)
    buildde = wrap(_buildde, [many('anything')])
    builddb = wrap(_buildde, [many('anything')])
    experimental = wrap(_buildde, [many('anything')])
    backports = wrap(_buildde, [many('anything')])

    def _popcon(self, irc, msg, args, package):
        try:
            msg = popcon(package)
            if msg:
                irc.reply(colourise(msg.for_irc()), prefixNick=False)
        except Exception, e:
            irc.reply("Error: unable to obtain popcon data for %s" % package)
    popcon = wrap(_popcon, ['text'])

    def _testing(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]Testing migration status for[reset] [package]%s[reset]: [url]http://release.debian.org/migration/testing.pl?package=%s[/url]" % \
                (package, package)
            irc.reply(colourise(msg), prefixNick=False)
    testing = wrap(_testing, [many('anything')])
    migration = wrap(_testing, [many('anything')])

    def _dehs(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]Debian External Health Status for[reset] [package]%s[reset]: [url]http://dehs.alioth.debian.org/report.php?package=%s[/url]" % \
                (package, urllib.quote(package))
            irc.reply(colourise(msg), prefixNick=False)
    dehs = wrap(_dehs, [many('anything')])
    health = wrap(_dehs, [many('anything')])

    def _swirl(self, irc, msg, args):
        swirl = [
            "  [brightred] ,''`.[reset]",
            "  [brightred]: :' :[reset]           [b]Debian GNU/Linux[/b]",
            "  [brightred]`. `' [reset]           [url]http://www.debian.org/[/url]",
            "  [brightred]  `-  [reset]",
        ]
        for line in swirl:
            irc.reply(colourise(line), prefixNick=False)
    swirl = wrap(_swirl)
    debian = wrap(_swirl)

    def _new(self, irc, msg, args):
        line = "[desc]NEW queue is[reset]: [url]%s[/url]. [desc]Current size is:[reset] %d" % \
            ("http://ftp-master.debian.org/new.html", NewQueue().get_size())
        irc.reply(colourise(line))
    new = wrap(_new)
    new_queue = wrap(_new)
    newqueue = wrap(_new)

Class = DebianDevelChanges
<commit_msg>Revert "Add some logging to fix the backports issue."<commit_after># -*- coding: utf-8 -*-
#
#   Debian Changes Bot
#   Copyright (C) 2008 Chris Lamb <chris@chris-lamb.co.uk>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as
#   published by the Free Software Foundation, either version 3 of the
#   License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import time
import random
import urllib
import supybot
import threading

from supybot.commands import wrap, many
from supybot import ircdb, log, schedule

from DebianDevelChangesBot.mailparsers import get_message
from DebianDevelChangesBot.datasources import get_datasources, TestingRCBugs, \
    NewQueue, RmQueue, Maintainer
from DebianDevelChangesBot.utils import parse_mail, FifoReader, colourise, \
    rewrite_topic, madison, bug_synopsis, format_email_address, popcon

class DebianDevelChanges(supybot.callbacks.Plugin):
    threaded = True

    def __init__(self, irc):
        self.__parent = super(DebianDevelChanges, self)
        self.__parent.__init__(irc)
        self.irc = irc
        self.topic_lock = threading.Lock()

        fr = FifoReader()
        fifo_loc = '/var/run/debian-devel-changes-bot/fifo'
        fr.start(self._email_callback, fifo_loc)

        self.queued_topics = {}
        self.last_n_messages = []

        # Schedule datasource updates
        for klass, interval, name in get_datasources():
            try:
                schedule.removePeriodicEvent(name)
            except KeyError:
                pass

            def wrapper(klass=klass):
                klass().update()
                self._topic_callback()

            schedule.addPeriodicEvent(wrapper, interval, name, now=False)
            schedule.addEvent(wrapper, time.time() + 1)

    def die(self):
        FifoReader().stop()
        for _, _, name in get_datasources():
            try:
                schedule.removePeriodicEvent(name)
            except KeyError:
                # A newly added event may not exist, ignore exception.
                pass

    def _email_callback(self, fileobj):
        try:
            email = parse_mail(fileobj)
            msg = get_message(email)

            if not msg:
                return

            txt = colourise(msg.for_irc())

            # Simple flood/duplicate detection
            if txt in self.last_n_messages:
                return
            self.last_n_messages.insert(0, txt)
            self.last_n_messages = self.last_n_messages[:20]

            for channel in self.irc.state.channels:
                package_regex = self.registryValue(
                    'package_regex',
                    channel,
                ) or 'a^' # match nothing by default

                if not re.search(package_regex, msg.package):
                    continue

                distribution_regex = self.registryValue(
                    'distribution_regex',
                    channel,
                )

                if distribution_regex:
                    if not hasattr(msg, 'distribution'):
                        # If this channel has a distribution regex, don't
                        # bother continuing unless the message actually has a
                        # distribution. This filters security messages, etc.
                        continue

                    if not re.search(distribution_regex, msg.distribution):
                        # Distribution doesn't match regex; don't send this
                        # message.
                        continue

                ircmsg = supybot.ircmsgs.privmsg(channel, txt)
                self.irc.queueMsg(ircmsg)

        except:
            log.exception('Uncaught exception')

    def _topic_callback(self):
        self.topic_lock.acquire()

        sections = {
            lambda: len(TestingRCBugs().get_bugs()): 'RC bug count:',
            NewQueue().get_size: 'NEW queue:',
            RmQueue().get_size: 'RM queue:',
        }

        try:
            values = {}
            for callback, prefix in sections.iteritems():
                values[callback] = callback()

            for channel in self.irc.state.channels:
                new_topic = topic = self.irc.state.getTopic(channel)

                for callback, prefix in sections.iteritems():
                    if values[callback]:
                        new_topic = rewrite_topic(new_topic, prefix, values[callback])

                if topic != new_topic:
                    log.info("Queueing change of topic in #%s to '%s'" % (channel, new_topic))
                    self.queued_topics[channel] = new_topic

                    event_name = '%s_topic' % channel
                    try:
                        schedule.removeEvent(event_name)
                    except KeyError:
                        pass
                    schedule.addEvent(lambda channel=channel: self._update_topic(channel),
                        time.time() + 60, event_name)
        finally:
            self.topic_lock.release()

    def _update_topic(self, channel):
        self.topic_lock.acquire()
        try:
            try:
                new_topic = self.queued_topics[channel]
                log.info("Changing topic in #%s to '%s'" % (channel, new_topic))
                self.irc.queueMsg(supybot.ircmsgs.topic(channel, new_topic))
            except KeyError:
                pass
        finally:
            self.topic_lock.release()

    def greeting(self, prefix, irc, msg, args):
        num_bugs = len(TestingRCBugs().get_bugs())
        if type(num_bugs) is int:
            advice = random.choice((
                'Why not go and fix one?',
                'Why not peek at the list and find one?',
                'Stop IRCing and fix one! :]',
                'You realise they don\'t fix themselves, right?',
                'How about fixing yourself some caffeine and then poking at the bug list?',
            ))
            txt = "%s %s! There are currently %d RC bugs in wheezy. %s Try !random." % \
                (prefix, msg.nick, num_bugs, advice)
        else:
            txt = "%s %s!" % (prefix, msg.name)
        irc.reply(txt, prefixNick=False)

    def morning(self, *args):
        self.greeting('Good morning,', *args)
    morning = wrap(morning)
    yawn = wrap(morning)
    wakeywakey = wrap(morning)

    def night(self, *args):
        self.greeting( 'Good night,', *args)
    night = wrap(night)
    nn = wrap(night)
    goodnight = wrap(night)

    def sup(self, *args):
        self.greeting("'sup", *args)
    sup = wrap(sup)
    lo = wrap(sup)

    def rc(self, irc, msg, args):
        num_bugs = len(TestingRCBugs().get_bugs())
        if type(num_bugs) is int:
            irc.reply("There are %d release-critical bugs in the testing distribution. " \
                "See http://bts.turmzimmer.net/details.php?bydist=wheezy&igncontrib=on&ignnonfree=on" % num_bugs)
        else:
            irc.reply("No data at this time.")
    rc = wrap(rc)
    bugs = wrap(rc)

    def randombug(self, irc, msg, args):
        bug = random.choice(list(TestingRCBugs().get_bugs()))
        msg = "Your randomly chosen wheezy RC bug is: %s. Happy bugfixing!" % bug_synopsis(str(bug)).for_irc()
        irc.reply(colourise(msg))
    randombug = wrap(randombug)
    random = wrap(randombug)

    def update(self, irc, msg, args):
        if not ircdb.checkCapability(msg.prefix, 'owner'):
            irc.reply("You are not authorised to run this command.")
            return

        for klass, interval, name in get_datasources():
            klass().update()
            irc.reply("Updated %s." % name)
        self._topic_callback()
    update = wrap(update)

    def madison(self, irc, msg, args, package):
        try:
            lines = madison(package)
            if not lines:
                irc.reply('Did not get a response -- is "%s" a valid package?' % package)
                return

            field_styles = ('package', 'version', 'distribution', 'section')
            for line in lines:
                out = []
                fields = line.strip().split('|', len(field_styles))
                for style, data in zip(field_styles, fields):
                    out.append('[%s]%s' % (style, data))
                irc.reply(colourise('[reset]|'.join(out)), prefixNick=False)
        except Exception, e:
            irc.reply("Error: %s" % e.message)
    madison = wrap(madison, ['text'])

    def bug(self, irc, msg, args, bug_string):
        try:
            msg = bug_synopsis(bug_string)
            if msg:
                irc.reply(colourise(msg.for_irc()), prefixNick=False)
        except ValueError:
            irc.reply('Could not parse bug number')
        except Exception, e:
            irc.reply("Error: %s" % e.message)

    bug = wrap(bug, ['text'])

    def get_pool_url(self, package):
        if package.startswith('lib'):
            return (package[:4], package)
        else:
            return (package[:1], package)

    def _maintainer(self, irc, msg, args, items):
        for package in items:
            info = Maintainer().get_maintainer(package)
            if info:
                display_name = format_email_address("%s <%s>" % (info['name'], info['email']), max_domain=18)

                login = info['email']
                if login.endswith('@debian.org'):
                    login = login.replace('@debian.org', '')

                msg = "[desc]Maintainer for[reset] [package]%s[reset] [desc]is[reset] [by]%s[reset]: " % (package, display_name)
                msg += "[url]http://qa.debian.org/developer.php?login=%s[/url]" % login
            else:
                msg = 'Unknown source package "%s"' % package

            irc.reply(colourise(msg), prefixNick=False)
    maintainer = wrap(_maintainer, [many('anything')])
    maint = wrap(_maintainer, [many('anything')])
    who_maintains = wrap(_maintainer, [many('anything')])

    def _qa(self, irc, msg, args, items):
        for package in items:
            url = "http://packages.qa.debian.org/%s/%s.html" % self.get_pool_url(package)
            msg = "[desc]QA page for[reset] [package]%s[reset]: [url]%s[/url]" % (package, url)
            irc.reply(colourise(msg), prefixNick=False)
    qa = wrap(_qa, [many('anything')])
    overview = wrap(_qa, [many('anything')])
    package = wrap(_qa, [many('anything')])
    pkg = wrap(_qa, [many('anything')])
    srcpkg = wrap(_qa, [many('anything')])

    def _changelog(self, irc, msg, args, items):
        for package in items:
            url = "http://packages.debian.org/changelogs/pool/main/%s/%s/current/changelog" % self.get_pool_url(package)
            msg = "[desc]debian/changelog for[reset] [package]%s[reset]: [url]%s[/url]" % (package, url)
            irc.reply(colourise(msg), prefixNick=False)
    changelog = wrap(_changelog, [many('anything')])
    changes = wrap(_changelog, [many('anything')])

    def _copyright(self, irc, msg, args, items):
        for package in items:
            url = "http://packages.debian.org/changelogs/pool/main/%s/%s/current/copyright" % self.get_pool_url(package)
            msg = "[desc]debian/copyright for[reset] [package]%s[reset]: [url]%s[/url]" % (package, url)
            irc.reply(colourise(msg), prefixNick=False)
    copyright = wrap(_copyright, [many('anything')])

    def _buggraph(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]Bug graph for[reset] [package]%s[reset]: [url]http://people.debian.org/~glandium/bts/%s/%s.png[/url]" % \
                (package, package[0], package)
            irc.reply(colourise(msg), prefixNick=False)
    buggraph = wrap(_buggraph, [many('anything')])
    bug_graph = wrap(_buggraph, [many('anything')])

    def _buildd(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]buildd status for[reset] [package]%s[reset]: [url]http://buildd.debian.org/pkg.cgi?pkg=%s[/url]" % \
                (package, package)
            irc.reply(colourise(msg), prefixNick=False)
    buildd = wrap(_buildd, [many('anything')])

    def _buildde(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]experimental/backports.org buildd status for[reset] [package]%s[reset]: [url]http://experimental.ftbfs.de/build.php?pkg=%s[/url]" % \
                (package, package)
            irc.reply(colourise(msg), prefixNick=False)
    buildde = wrap(_buildde, [many('anything')])
    builddb = wrap(_buildde, [many('anything')])
    experimental = wrap(_buildde, [many('anything')])
    backports = wrap(_buildde, [many('anything')])

    def _popcon(self, irc, msg, args, package):
        try:
            msg = popcon(package)
            if msg:
                irc.reply(colourise(msg.for_irc()), prefixNick=False)
        except Exception, e:
            irc.reply("Error: unable to obtain popcon data for %s" % package)
    popcon = wrap(_popcon, ['text'])

    def _testing(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]Testing migration status for[reset] [package]%s[reset]: [url]http://release.debian.org/migration/testing.pl?package=%s[/url]" % \
                (package, package)
            irc.reply(colourise(msg), prefixNick=False)
    testing = wrap(_testing, [many('anything')])
    migration = wrap(_testing, [many('anything')])

    def _dehs(self, irc, msg, args, items):
        for package in items:
            msg = "[desc]Debian External Health Status for[reset] [package]%s[reset]: [url]http://dehs.alioth.debian.org/report.php?package=%s[/url]" % \
                (package, urllib.quote(package))
            irc.reply(colourise(msg), prefixNick=False)
    dehs = wrap(_dehs, [many('anything')])
    health = wrap(_dehs, [many('anything')])

    def _swirl(self, irc, msg, args):
        swirl = [
            "  [brightred] ,''`.[reset]",
            "  [brightred]: :' :[reset]           [b]Debian GNU/Linux[/b]",
            "  [brightred]`. `' [reset]           [url]http://www.debian.org/[/url]",
            "  [brightred]  `-  [reset]",
        ]
        for line in swirl:
            irc.reply(colourise(line), prefixNick=False)
    swirl = wrap(_swirl)
    debian = wrap(_swirl)

    def _new(self, irc, msg, args):
        line = "[desc]NEW queue is[reset]: [url]%s[/url]. [desc]Current size is:[reset] %d" % \
            ("http://ftp-master.debian.org/new.html", NewQueue().get_size())
        irc.reply(colourise(line))
    new = wrap(_new)
    new_queue = wrap(_new)
    newqueue = wrap(_new)

Class = DebianDevelChanges
<|endoftext|>


<commit_before>"""Command Generator Application (GET)"""
from pysnmp.carrier.asynsock.dispatch import AsynsockDispatcher
from pysnmp.carrier.asynsock.dgram import udp
from pyasn1.codec.ber import encoder, decoder
from pysnmp.proto import api
from time import time

# Protocol version to use
pMod = api.protoModules[api.protoVersion1]

# Build PDU
reqPDU =  pMod.SetRequestPDU()
pMod.apiPDU.setDefaults(reqPDU)
pMod.apiPDU.setVarBinds(
    reqPDU,
    # A list of Var-Binds to SET
    (((1,3,6,1,2,1,1,1,0), pMod.OctetString('New system description')),
     ((1,3,6,1,2,1,1,3,0), pMod.TimeTicks(12)))
    )

# Build message
reqMsg = pMod.Message()
pMod.apiMessage.setDefaults(reqMsg)
pMod.apiMessage.setCommunity(reqMsg, 'public')
pMod.apiMessage.setPDU(reqMsg, reqPDU)

def cbTimerFun(timeNow, startedAt=time()):
    if timeNow - startedAt > 3:
        raise "Request timed out"

def cbRecvFun(transportDispatcher, transportDomain, transportAddress,
              wholeMsg, reqPDU=reqPDU):
    while wholeMsg:
        rspMsg, wholeMsg = decoder.decode(wholeMsg, asn1Spec=pMod.Message())
        rspPDU = pMod.apiMessage.getPDU(rspMsg)
        # Match response to request
        if pMod.apiPDU.getRequestID(reqPDU)==pMod.apiPDU.getRequestID(rspPDU):
            # Check for SNMP errors reported
            errorStatus = pMod.apiPDU.getErrorStatus(rspPDU)
            if errorStatus:
                print errorStatus.prettyOut(errorStatus)
            else:
                for oid, val in pMod.apiPDU.getVarBinds(rspPDU):
                    print oid, val.prettyOut(val)
            transportDispatcher.jobFinished(1)
    return wholeMsg

transportDispatcher = AsynsockDispatcher()
transportDispatcher.registerTransport(
    udp.domainName, udp.UdpSocketTransport().openClientMode()
    )
transportDispatcher.registerRecvCbFun(cbRecvFun)
transportDispatcher.registerTimerCbFun(cbTimerFun)
transportDispatcher.sendMessage(
    encoder.encode(reqMsg), udp.domainName, ('localhost', 161)
    )
transportDispatcher.jobStarted(1)
transportDispatcher.runDispatcher()
transportDispatcher.closeDispatcher()
<commit_msg>prettyOut() -> prettyPrint()<commit_after>"""Command Generator Application (GET)"""
from pysnmp.carrier.asynsock.dispatch import AsynsockDispatcher
from pysnmp.carrier.asynsock.dgram import udp
from pyasn1.codec.ber import encoder, decoder
from pysnmp.proto import api
from time import time

# Protocol version to use
pMod = api.protoModules[api.protoVersion1]

# Build PDU
reqPDU =  pMod.SetRequestPDU()
pMod.apiPDU.setDefaults(reqPDU)
pMod.apiPDU.setVarBinds(
    reqPDU,
    # A list of Var-Binds to SET
    (((1,3,6,1,2,1,1,1,0), pMod.OctetString('New system description')),
     ((1,3,6,1,2,1,1,3,0), pMod.TimeTicks(12)))
    )

# Build message
reqMsg = pMod.Message()
pMod.apiMessage.setDefaults(reqMsg)
pMod.apiMessage.setCommunity(reqMsg, 'public')
pMod.apiMessage.setPDU(reqMsg, reqPDU)

def cbTimerFun(timeNow, startedAt=time()):
    if timeNow - startedAt > 3:
        raise "Request timed out"

def cbRecvFun(transportDispatcher, transportDomain, transportAddress,
              wholeMsg, reqPDU=reqPDU):
    while wholeMsg:
        rspMsg, wholeMsg = decoder.decode(wholeMsg, asn1Spec=pMod.Message())
        rspPDU = pMod.apiMessage.getPDU(rspMsg)
        # Match response to request
        if pMod.apiPDU.getRequestID(reqPDU)==pMod.apiPDU.getRequestID(rspPDU):
            # Check for SNMP errors reported
            errorStatus = pMod.apiPDU.getErrorStatus(rspPDU)
            if errorStatus:
                print errorStatus.prettyPrint()
            else:
                for oid, val in pMod.apiPDU.getVarBinds(rspPDU):
                    print oid.prettyPrint(), val.prettyPrint()
            transportDispatcher.jobFinished(1)
    return wholeMsg

transportDispatcher = AsynsockDispatcher()
transportDispatcher.registerTransport(
    udp.domainName, udp.UdpSocketTransport().openClientMode()
    )
transportDispatcher.registerRecvCbFun(cbRecvFun)
transportDispatcher.registerTimerCbFun(cbTimerFun)
transportDispatcher.sendMessage(
    encoder.encode(reqMsg), udp.domainName, ('localhost', 161)
    )
transportDispatcher.jobStarted(1)
transportDispatcher.runDispatcher()
transportDispatcher.closeDispatcher()
<|endoftext|>


<commit_before># Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the License);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and#
# limitations under the License.

import cloud
import cloud_ferry
from cloudferrylib.base.action import copy_var, rename_info, merge, is_end_iter, get_info_iter, create_reference
from cloudferrylib.os.actions import identity_transporter
from cloudferrylib.scheduler import scheduler
from cloudferrylib.scheduler import namespace
from cloudferrylib.scheduler import cursor
from cloudferrylib.os.image import glance_image
from cloudferrylib.os.storage import cinder_storage
from cloudferrylib.os.network import neutron
from cloudferrylib.os.identity import keystone
from cloudferrylib.os.compute import nova_compute
from cloudferrylib.os.actions import get_info_images
from cloudferrylib.os.actions import transport_instance
from cloudferrylib.os.actions import attach_used_volumes_via_compute
from cloudferrylib.os.actions import cleanup_images
from cloudferrylib.os.actions import copy_g2g
from cloudferrylib.os.actions import convert_image_to_compute
from cloudferrylib.os.actions import convert_image_to_volume
from cloudferrylib.os.actions import convert_compute_to_image
from cloudferrylib.os.actions import convert_compute_to_volume
from cloudferrylib.os.actions import convert_volume_to_image
from cloudferrylib.os.actions import convert_volume_to_compute
from cloudferrylib.os.actions import attach_used_volumes
from cloudferrylib.os.actions import networks_transporter
from cloudferrylib.base.action import create_reference
from cloudferrylib.os.actions import prepare_volumes_data_map
from cloudferrylib.os.actions import transport_ceph_to_ceph_via_ssh
from cloudferrylib.os.actions import get_info_instances
from cloudferrylib.os.actions import prepare_networks
from cloudferrylib.os.actions import map_compute_info
from cloudferrylib.os.actions import start_vm
from cloudferrylib.os.actions import stop_vm
from cloudferrylib.utils import utils as utl
from cloudferrylib.os.actions import transport_compute_resources

class OS2OSFerry(cloud_ferry.CloudFerry):

    def __init__(self, config):
        super(OS2OSFerry, self). __init__(config)
        resources = {'identity': keystone.KeystoneIdentity,
                     'image': glance_image.GlanceImage,
                     'storage': cinder_storage.CinderStorage,
                     'network': neutron.NeutronNetwork,
                     'compute': nova_compute.NovaCompute}
        self.src_cloud = cloud.Cloud(resources, cloud.SRC, config)
        self.dst_cloud = cloud.Cloud(resources, cloud.DST, config)
        self.init = {
            'src_cloud': self.src_cloud,
            'dst_cloud': self.dst_cloud,
            'cfg': self.config
        }

    def migrate(self):
        namespace_scheduler = namespace.Namespace({
            '__init_task__': self.init,
            'info_result': {
                utl.COMPUTE_RESOURCE: {utl.INSTANCES_TYPE: {}}
            }
        })

        act_identity_trans = identity_transporter.IdentityTransporter(self.init)
        act_comp_res_trans = transport_compute_resources.TransportComputeResources(self.init)
        act_get_info_images = get_info_images.GetInfoImages(self.init, cloud='src_cloud')
        act_get_info_inst = get_info_instances.GetInfoInstances(self.init, cloud='src_cloud')
        act_conv_comp_img = convert_compute_to_image.ConvertComputeToImage(self.init, cloud='src_cloud')
        act_conv_image_comp = convert_image_to_compute.ConvertImageToCompute(self.init)
        act_map_com_info = map_compute_info.MapComputeInfo(self.init)
        act_stop_vms = stop_vm.StopVms(self.init, cloud='src_cloud')
        act_deploy_images = copy_g2g.CopyFromGlanceToGlance(self.init)
        act_copy_inst_images = copy_g2g.CopyFromGlanceToGlance(self.init)
        act_net_prep = prepare_networks.PrepareNetworks(self.init, cloud='dst_cloud')
        act_deploy_instances = transport_instance.TransportInstance(self.init)
        act_copy_g2g_vols = copy_g2g.CopyFromGlanceToGlance(self.init)
        act_convert_c_to_v = convert_compute_to_volume.ConvertComputeToVolume(self.init, cloud='src_cloud')
        act_convert_v_to_i = convert_volume_to_image.ConvertVolumeToImage(self.init, cloud='src_cloud')
        act_convert_i_to_v = convert_image_to_volume.ConvertImageToVolume(self.init, cloud='dst_cloud')
        act_convert_v_to_c = convert_volume_to_compute.ConvertVolumeToCompute(self.init, cloud='dst_cloud')
        act_cleanup_images = cleanup_images.CleanupImages(self.init)
        act_attaching = attach_used_volumes_via_compute.AttachVolumesCompute(self.init, cloud='dst_cloud')
        save_result = merge.Merge('info', 'info_result', 'info_result', 'compute', 'instances')

        act_network_trans = networks_transporter.NetworkTransporter(self.init)

        task_ident_trans = act_identity_trans
        task_images_trans = act_get_info_images >> act_deploy_images
        task_compres_trans = act_comp_res_trans

        task_resources_transporting = task_ident_trans >> task_images_trans >> act_network_trans >> task_compres_trans

        task_convert_c_to_v_to_i = act_convert_c_to_v >> act_convert_v_to_i
        task_convert_i_to_v_to_c = act_convert_i_to_v >> act_convert_v_to_c
        task_transport_volumes = task_convert_c_to_v_to_i >> act_copy_g2g_vols >> task_convert_i_to_v_to_c
        task_attaching_volumes = act_attaching
        task_get_inst_info = act_get_info_inst
        init_iteration_instance = copy_var.CopyVar('info', 'info_backup', True) >> \
                                  create_reference.CreateReference('info', 'info_iter')
        get_next_instance = get_info_iter.GetInfoIter()
        rename_info_iter = rename_info.RenameInfo('info_result', 'info')
        is_instances = is_end_iter.IsEndIter()
        act_start_vms = start_vm.StartVms(self.init, cloud='dst_cloud')

        task_cleanup_images = act_cleanup_images

        transport_image = act_conv_comp_img >> act_copy_inst_images >> act_conv_image_comp
        transport_resource_inst = transport_image >> task_transport_volumes
        transport_inst = act_net_prep >> act_map_com_info >> act_deploy_instances

        trans_one_inst = act_stop_vms >> transport_resource_inst >> transport_inst >> \
                         task_attaching_volumes >> act_start_vms

        transport_instances_and_dependency_resources = \
            task_get_inst_info >> \
            init_iteration_instance >> \
            get_next_instance >> \
            trans_one_inst >> \
            save_result >> \
            (is_instances | get_next_instance) >>\
            rename_info_iter >> task_cleanup_images

        process_migration = task_resources_transporting >> transport_instances_and_dependency_resources

        process_migration = cursor.Cursor(process_migration)
        scheduler_migr = scheduler.Scheduler(namespace=namespace_scheduler, cursor=process_migration)
        scheduler_migr.start()<commit_msg>Some structing in os2os.py<commit_after># Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the License);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and#
# limitations under the License.

import cloud
import cloud_ferry
from cloudferrylib.base.action import copy_var, rename_info, merge, is_end_iter, get_info_iter, create_reference
from cloudferrylib.os.actions import identity_transporter
from cloudferrylib.scheduler import scheduler
from cloudferrylib.scheduler import namespace
from cloudferrylib.scheduler import cursor
from cloudferrylib.os.image import glance_image
from cloudferrylib.os.storage import cinder_storage
from cloudferrylib.os.network import neutron
from cloudferrylib.os.identity import keystone
from cloudferrylib.os.compute import nova_compute
from cloudferrylib.os.actions import get_info_images
from cloudferrylib.os.actions import transport_instance
from cloudferrylib.os.actions import attach_used_volumes_via_compute
from cloudferrylib.os.actions import cleanup_images
from cloudferrylib.os.actions import copy_g2g
from cloudferrylib.os.actions import convert_image_to_compute
from cloudferrylib.os.actions import convert_image_to_volume
from cloudferrylib.os.actions import convert_compute_to_image
from cloudferrylib.os.actions import convert_compute_to_volume
from cloudferrylib.os.actions import convert_volume_to_image
from cloudferrylib.os.actions import convert_volume_to_compute
from cloudferrylib.os.actions import attach_used_volumes
from cloudferrylib.os.actions import networks_transporter
from cloudferrylib.base.action import create_reference
from cloudferrylib.os.actions import prepare_volumes_data_map
from cloudferrylib.os.actions import transport_ceph_to_ceph_via_ssh
from cloudferrylib.os.actions import get_info_instances
from cloudferrylib.os.actions import prepare_networks
from cloudferrylib.os.actions import map_compute_info
from cloudferrylib.os.actions import start_vm
from cloudferrylib.os.actions import stop_vm
from cloudferrylib.utils import utils as utl
from cloudferrylib.os.actions import transport_compute_resources

class OS2OSFerry(cloud_ferry.CloudFerry):

    def __init__(self, config):
        super(OS2OSFerry, self). __init__(config)
        resources = {'identity': keystone.KeystoneIdentity,
                     'image': glance_image.GlanceImage,
                     'storage': cinder_storage.CinderStorage,
                     'network': neutron.NeutronNetwork,
                     'compute': nova_compute.NovaCompute}
        self.src_cloud = cloud.Cloud(resources, cloud.SRC, config)
        self.dst_cloud = cloud.Cloud(resources, cloud.DST, config)
        self.init = {
            'src_cloud': self.src_cloud,
            'dst_cloud': self.dst_cloud,
            'cfg': self.config
        }

    def migrate(self):
        namespace_scheduler = namespace.Namespace({
            '__init_task__': self.init,
            'info_result': {
                utl.COMPUTE_RESOURCE: {utl.INSTANCES_TYPE: {}}
            }
        })

        task_resources_transporting = self.transport_resources()
        transport_instances_and_dependency_resources = self.migrate_instances()

        process_migration = task_resources_transporting >> transport_instances_and_dependency_resources

        process_migration = cursor.Cursor(process_migration)
        scheduler_migr = scheduler.Scheduler(namespace=namespace_scheduler, cursor=process_migration)
        scheduler_migr.start()

    def migrate_instances(self):
        name_data = 'info'
        name_result = 'info_result'
        name_backup = 'info_backup'
        name_iter = 'info_iter'
        save_result = self.save_result(name_data, name_result, name_result, 'compute', 'instances')
        trans_one_inst = self.migrate_process_instance()
        init_iteration_instance = self.init_iteration_instance(name_data, name_backup, name_iter)
        act_get_info_inst = get_info_instances.GetInfoInstances(self.init, cloud='src_cloud')
        act_cleanup_images = cleanup_images.CleanupImages(self.init)
        get_next_instance = get_info_iter.GetInfoIter()
        rename_info_iter = rename_info.RenameInfo(name_result, name_data)
        is_instances = is_end_iter.IsEndIter()

        transport_instances_and_dependency_resources = \
            act_get_info_inst >> \
            init_iteration_instance >> \
            get_next_instance >> \
            trans_one_inst >> \
            save_result >> \
            (is_instances | get_next_instance) >>\
            rename_info_iter >> \
            act_cleanup_images
        return transport_instances_and_dependency_resources

    def init_iteration_instance(self, data, name_backup, name_iter):
        init_iteration_instance = copy_var.CopyVar(data, name_backup, True) >>\
                                  create_reference.CreateReference(data, name_iter)
        return init_iteration_instance

    def migration_images(self):
        act_get_info_images = get_info_images.GetInfoImages(self.init, cloud='src_cloud')
        act_deploy_images = copy_g2g.CopyFromGlanceToGlance(self.init)
        return act_get_info_images >> act_deploy_images

    def save_result(self, data1, data2, result, resource_type, resources_name):
        return merge.Merge(data1, data2, result, resource_type, resources_name)

    def transport_volumes_by_instance(self):
        act_copy_g2g_vols = copy_g2g.CopyFromGlanceToGlance(self.init)
        act_convert_c_to_v = convert_compute_to_volume.ConvertComputeToVolume(self.init, cloud='src_cloud')
        act_convert_v_to_i = convert_volume_to_image.ConvertVolumeToImage(self.init, cloud='src_cloud')
        act_convert_i_to_v = convert_image_to_volume.ConvertImageToVolume(self.init, cloud='dst_cloud')
        act_convert_v_to_c = convert_volume_to_compute.ConvertVolumeToCompute(self.init, cloud='dst_cloud')
        task_convert_c_to_v_to_i = act_convert_c_to_v >> act_convert_v_to_i
        task_convert_i_to_v_to_c = act_convert_i_to_v >> act_convert_v_to_c
        return task_convert_c_to_v_to_i >> act_copy_g2g_vols >> task_convert_i_to_v_to_c

    def transport_resources(self):
        task_images_trans = self.migration_images()
        act_identity_trans = identity_transporter.IdentityTransporter(self.init)
        act_comp_res_trans = transport_compute_resources.TransportComputeResources(self.init)
        act_network_trans = networks_transporter.NetworkTransporter(self.init)
        return act_identity_trans >> task_images_trans >> act_network_trans >> act_comp_res_trans

    def migrate_images_by_instances(self):
        act_conv_comp_img = convert_compute_to_image.ConvertComputeToImage(self.init, cloud='src_cloud')
        act_conv_image_comp = convert_image_to_compute.ConvertImageToCompute(self.init)
        act_copy_inst_images = copy_g2g.CopyFromGlanceToGlance(self.init)
        return act_conv_comp_img >> act_copy_inst_images >> act_conv_image_comp

    def migrate_resources_by_instance(self):
        transport_images = self.migrate_images_by_instances()
        task_transport_volumes = self.transport_volumes_by_instance()
        return transport_images >> task_transport_volumes

    def migrate_instance(self):
        act_map_com_info = map_compute_info.MapComputeInfo(self.init)
        act_net_prep = prepare_networks.PrepareNetworks(self.init, cloud='dst_cloud')
        act_deploy_instances = transport_instance.TransportInstance(self.init)
        return act_net_prep >> act_map_com_info >> act_deploy_instances

    def migrate_process_instance(self):
        act_attaching = attach_used_volumes_via_compute.AttachVolumesCompute(self.init, cloud='dst_cloud')
        act_stop_vms = stop_vm.StopVms(self.init, cloud='src_cloud')
        act_start_vms = start_vm.StartVms(self.init, cloud='dst_cloud')
        transport_resource_inst = self.migrate_resources_by_instance()
        transport_inst = self.migrate_instance()
        return act_stop_vms >> transport_resource_inst >> transport_inst >> act_attaching >> act_start_vms
<|endoftext|>