<commit_before>#!/usr/bin/python
# Copyright IBM Corp. 2007
#
# indication_tester.py - Tool for testing indication subscription and
#                        delivery against a CIMOM
# Author: Dan Smith <danms@us.ibm.com>

import sys
from optparse import OptionParser
import BaseHTTPServer
import httplib
import base64
from xml.dom.minidom import parse, parseString

def filter_xml(name, type, ns, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
    <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
      <SIMPLEREQ>
        <IMETHODCALL NAME="CreateInstance">
          <LOCALNAMESPACEPATH>
            <NAMESPACE NAME="root"/>
            <NAMESPACE NAME="PG_InterOp"/>
          </LOCALNAMESPACEPATH>
          <IPARAMVALUE NAME="NewInstance">
              <INSTANCE CLASSNAME="CIM_IndicationFilter">
                <PROPERTY NAME="SystemCreationClassName" TYPE="string">
                  <VALUE>CIM_ComputerSystem</VALUE>
                </PROPERTY>
                <PROPERTY NAME="SystemName" TYPE="string">
                  <VALUE>%s</VALUE>
                </PROPERTY>
                <PROPERTY NAME="CreationClassName" TYPE="string">
                  <VALUE>CIM_IndicationFilter</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Name" TYPE="string">
                  <VALUE>%sFilter</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Query" TYPE="string">
                  <VALUE> SELECT * FROM %s
                  </VALUE>
                </PROPERTY>
                <PROPERTY NAME="QueryLanguage" TYPE="string">
                  <VALUE>WQL</VALUE>
                </PROPERTY>
                <PROPERTY NAME="SourceNamespace" TYPE="string">
                  <VALUE>%s</VALUE>
                </PROPERTY>
              </INSTANCE>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
    </CIM>
    """ % (sysname, name, type, ns)

def handler_xml(name, port, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
        <IMETHODCALL NAME="CreateInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="NewInstance">
              <INSTANCE CLASSNAME="CIM_IndicationHandlerCIMXML">
                <PROPERTY NAME="SystemCreationClassName" TYPE="string">
                  <VALUE>CIM_ComputerSystem</VALUE>
                </PROPERTY>
                <PROPERTY NAME="SystemName" TYPE="string">
                  <VALUE>%s</VALUE>
                </PROPERTY>
                <PROPERTY NAME="CreationClassName" TYPE="string">
                  <VALUE>CIM_IndicationHandlerCIMXML</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Name" TYPE="string">
                  <VALUE>%sHandler</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Destination" TYPE="string">
                  <VALUE>localhost:%i</VALUE>
                </PROPERTY>
              </INSTANCE>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
      </CIM>
      """ % (sysname, name, port)

def subscription_xml(name, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
          <IMETHODCALL NAME="CreateInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="NewInstance">
              <INSTANCE CLASSNAME="CIM_IndicationSubscription">
                <PROPERTY.REFERENCE NAME="Filter"
                                    REFERENCECLASS="CIM_IndicationFilter">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationFilter">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationFilter
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sFilter
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </PROPERTY.REFERENCE>
                <PROPERTY.REFERENCE NAME="Handler"
                                    REFERENCECLASS="CIM_IndicationHandler">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationHandlerCIMXML">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationHandlerCIMXML
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sHandler
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </PROPERTY.REFERENCE>
                <PROPERTY NAME="SubscriptionState" TYPE="uint16">
                  <VALUE> 2 </VALUE>
                </PROPERTY>
              </INSTANCE>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
      </CIM>
      """ % (sysname, name, sysname, name)

def delete_inst_xml(name, type, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
          <IMETHODCALL NAME="DeleteInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="InstanceName">
              <INSTANCENAME CLASSNAME="CIM_Indication%sCIMXML">
                <KEYBINDING NAME="SystemCreationClassName">
                  <KEYVALUE>CIM_ComputerSystem</KEYVALUE>
                </KEYBINDING>
                <KEYBINDING NAME="SystemName">
                  <KEYVALUE>%s</KEYVALUE>
                </KEYBINDING>
                <KEYBINDING NAME="CreationClassName">
                  <KEYVALUE>CIM_Indication%sCIMXML</KEYVALUE>
                </KEYBINDING>
                <KEYBINDING NAME="Name">
                  <KEYVALUE>%s%s</KEYVALUE>
                </KEYBINDING>
              </INSTANCENAME>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
    </CIM>;
    """ % (type, sysname, type, name, type);

def delete_sub_xml(name, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
          <IMETHODCALL NAME="DeleteInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="InstanceName">
              <INSTANCENAME CLASSNAME="CIM_IndicationSubscription">
                <KEYBINDING NAME="Filter">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationFilter">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationFilter
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sFilter
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </KEYBINDING>
                <KEYBINDING NAME="Handler">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationHandlerCIMXML">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationHandlerCIMXML
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sHandler
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </KEYBINDING>
              </INSTANCENAME>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
    </CIM>;
    """ % (sysname, name, sysname, name)

class CIMIndication:
    def __init__(self, xmldata):
        dom = parseString(xmldata)

        instances = dom.getElementsByTagName("INSTANCE")
        attrs = instances[0].attributes.items()
        self.name = attrs[0][1]

    def __str__(self):
        return self.name

class CIMSocketHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    def do_POST(self):
        length = self.headers.getheader('content-length')
        data = self.rfile.read(int(length))

        indication = CIMIndication(data)
        print "Got indication: %s" % indication
        if self.server.print_ind:
            print "%s\n\n" % data

class CIMIndicationSubscription:
    def __init__(self, name, typ, ns, print_ind, sysname):
        self.name = name
        self.type = typ
        self.ns = ns
        self.sysname = sysname

        self.server = BaseHTTPServer.HTTPServer(('', 8000), CIMSocketHandler)
        self.server.print_ind = print_ind
        self.port = 8000

        self.filter_xml = filter_xml(name, typ, ns, sysname)
        self.handler_xml = handler_xml(name, self.port, sysname)
        self.subscription_xml = subscription_xml(name, sysname)

    def __do_cimpost(self, conn, body, method, auth_hdr=None):
        headers = {"CIMOperation" : "MethodCall",
                   "CIMMethod"    : method,
                   "CIMObject"    : "root/PG_Interop",
                   "Content-Type" : "text/cimxml"}

        if auth_hdr:
            headers["Authorization"] = "Basic %s" % auth_hdr

        conn.request("POST", "/cimom", body, headers)
        resp = conn.getresponse()
        if not resp.getheader("content-length"):
            raise Exception("Authentication (or request) Failed!")

        resp.read()

    def subscribe(self, url, cred=None):
        self.conn = httplib.HTTPConnection(url)
        if cred:
            (u, p) = cred
            auth_hdr = base64.b64encode("%s:%s" % (u, p))
        else:
            auth_hdr = None

        self.__do_cimpost(self.conn, self.filter_xml,
                          "CreateInstance", auth_hdr)
        self.__do_cimpost(self.conn, self.handler_xml,
                          "CreateInstance", auth_hdr)
        self.__do_cimpost(self.conn, self.subscription_xml,
                          "CreateInstance", auth_hdr)

    def unsubscribe(self, cred=None):
        if cred:
            (u, p) = cred
            auth_hdr = base64.b64encode("%s:%s" % (u, p))
        else:
            auth_hdr = None

        xml = delete_sub_xml(self.name, self.sysname)
        self.__do_cimpost(self.conn, xml,
                          "DeleteInstance", auth_hdr)
        xml = delete_inst_xml(self.name, "Handler", self.sysname)
        self.__do_cimpost(self.conn, xml,
                          "DeleteInstance", auth_hdr)
        xml = delete_inst_xml(self.name, "Filter", self.sysname)
        self.__do_cimpost(self.conn, xml,
                          "DeleteInstance", auth_hdr)

def dump_xml(name, typ, ns):
    filter_str = filter_xml(name, typ, ns)
    handler_str = handler_xml(name, 8000)
    subscript_str = subscription_xml(name)
    del_filter_str = delete_inst_xml(name, "Filter")
    del_handler_str = delete_inst_xml(name, "Handler")
    del_subscript_str = delete_sub_xml(name)

    print "CreateFilter:\n%s\n" % filter_str
    print "DeleteFilter:\n%s\n" % del_filter_str
    print "CreateHandler:\n%s\n" % handler_str
    print "DeleteHandler:\n%s\n" % del_handler_str
    print "CreateSubscription:\n%s\n" % subscript_str
    print "DeleteSubscription:\n%s\n" % del_subscript_str

def main():
    usage = "usage: %prog [options] provider\nex: %prog CIM_InstModification"
    parser = OptionParser(usage)

    parser.add_option("-u", "--url", dest="url", default="localhost:5988",
                      help="URL of CIMOM to connect to (host:port)")
    parser.add_option("-N", "--ns", dest="ns", default="root/virt",
                      help="Namespace (default is root/virt)")
    parser.add_option("-n", "--name", dest="name", default="Test",
                      help="Name for filter, handler, subscription \
                      (default: Test)")
    parser.add_option("-d", "--dump-xml", dest="dump", default=False,
                      action="store_true",
                      help="Dump the xml that would be used and quit.")
    parser.add_option("-p", "--print-ind", dest="print_ind", default=False,
                      action="store_true",
                      help="Print received indications to stdout.")
    parser.add_option("-U", "--user", dest="username", default=None,
                      help="HTTP Auth username", dest="username")
    parser.add_option("-P", "--pass", dest="password", default=None,
                      help="HTTP Auth password", dest="password")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Fatal: no indication type provided."
        sys.exit(1)

    if options.dump:
        dump_xml(options.name, args[0], options.ns)
        sys.exit(0)

    if options.username:
        auth = (options.username, options.password)
    else:
        auth = None

    if ":" in options.url:
        (sysname, port) = options.url.split(":")
    else:
        sysname = url

    sub = CIMIndicationSubscription(options.name, args[0], options.ns,
                                    options.print_ind, sysname)
    sub.subscribe(options.url, auth)
    print "Watching for %s" % args[0]

    try:
        sub.server.serve_forever()
    except KeyboardInterrupt,e:
        sub.unsubscribe(auth)
        print "Cancelling subscription for %s" % args[0]

if __name__=="__main__":
    sys.exit(main())
<commit_msg>[CU] Fix the --dump-xml option in libcmpiutil<commit_after>#!/usr/bin/python
# Copyright IBM Corp. 2007
#
# indication_tester.py - Tool for testing indication subscription and
#                        delivery against a CIMOM
# Author: Dan Smith <danms@us.ibm.com>

import sys
from optparse import OptionParser
import BaseHTTPServer
import httplib
import base64
from xml.dom.minidom import parse, parseString

def filter_xml(name, type, ns, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
    <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
      <SIMPLEREQ>
        <IMETHODCALL NAME="CreateInstance">
          <LOCALNAMESPACEPATH>
            <NAMESPACE NAME="root"/>
            <NAMESPACE NAME="PG_InterOp"/>
          </LOCALNAMESPACEPATH>
          <IPARAMVALUE NAME="NewInstance">
              <INSTANCE CLASSNAME="CIM_IndicationFilter">
                <PROPERTY NAME="SystemCreationClassName" TYPE="string">
                  <VALUE>CIM_ComputerSystem</VALUE>
                </PROPERTY>
                <PROPERTY NAME="SystemName" TYPE="string">
                  <VALUE>%s</VALUE>
                </PROPERTY>
                <PROPERTY NAME="CreationClassName" TYPE="string">
                  <VALUE>CIM_IndicationFilter</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Name" TYPE="string">
                  <VALUE>%sFilter</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Query" TYPE="string">
                  <VALUE> SELECT * FROM %s
                  </VALUE>
                </PROPERTY>
                <PROPERTY NAME="QueryLanguage" TYPE="string">
                  <VALUE>WQL</VALUE>
                </PROPERTY>
                <PROPERTY NAME="SourceNamespace" TYPE="string">
                  <VALUE>%s</VALUE>
                </PROPERTY>
              </INSTANCE>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
    </CIM>
    """ % (sysname, name, type, ns)

def handler_xml(name, port, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
        <IMETHODCALL NAME="CreateInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="NewInstance">
              <INSTANCE CLASSNAME="CIM_IndicationHandlerCIMXML">
                <PROPERTY NAME="SystemCreationClassName" TYPE="string">
                  <VALUE>CIM_ComputerSystem</VALUE>
                </PROPERTY>
                <PROPERTY NAME="SystemName" TYPE="string">
                  <VALUE>%s</VALUE>
                </PROPERTY>
                <PROPERTY NAME="CreationClassName" TYPE="string">
                  <VALUE>CIM_IndicationHandlerCIMXML</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Name" TYPE="string">
                  <VALUE>%sHandler</VALUE>
                </PROPERTY>
                <PROPERTY NAME="Destination" TYPE="string">
                  <VALUE>localhost:%i</VALUE>
                </PROPERTY>
              </INSTANCE>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
      </CIM>
      """ % (sysname, name, port)

def subscription_xml(name, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
          <IMETHODCALL NAME="CreateInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="NewInstance">
              <INSTANCE CLASSNAME="CIM_IndicationSubscription">
                <PROPERTY.REFERENCE NAME="Filter"
                                    REFERENCECLASS="CIM_IndicationFilter">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationFilter">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationFilter
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sFilter
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </PROPERTY.REFERENCE>
                <PROPERTY.REFERENCE NAME="Handler"
                                    REFERENCECLASS="CIM_IndicationHandler">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationHandlerCIMXML">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationHandlerCIMXML
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sHandler
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </PROPERTY.REFERENCE>
                <PROPERTY NAME="SubscriptionState" TYPE="uint16">
                  <VALUE> 2 </VALUE>
                </PROPERTY>
              </INSTANCE>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
      </CIM>
      """ % (sysname, name, sysname, name)

def delete_inst_xml(name, type, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
          <IMETHODCALL NAME="DeleteInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="InstanceName">
              <INSTANCENAME CLASSNAME="CIM_Indication%sCIMXML">
                <KEYBINDING NAME="SystemCreationClassName">
                  <KEYVALUE>CIM_ComputerSystem</KEYVALUE>
                </KEYBINDING>
                <KEYBINDING NAME="SystemName">
                  <KEYVALUE>%s</KEYVALUE>
                </KEYBINDING>
                <KEYBINDING NAME="CreationClassName">
                  <KEYVALUE>CIM_Indication%sCIMXML</KEYVALUE>
                </KEYBINDING>
                <KEYBINDING NAME="Name">
                  <KEYVALUE>%s%s</KEYVALUE>
                </KEYBINDING>
              </INSTANCENAME>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
    </CIM>;
    """ % (type, sysname, type, name, type);

def delete_sub_xml(name, sysname):
    return """
    <?xml version="1.0" encoding="utf-8"?>
    <CIM CIMVERSION="2.0" DTDVERSION="2.0">
      <MESSAGE ID="4711" PROTOCOLVERSION="1.0">
        <SIMPLEREQ>
          <IMETHODCALL NAME="DeleteInstance">
            <LOCALNAMESPACEPATH>
              <NAMESPACE NAME="root"/>
              <NAMESPACE NAME="PG_InterOp"/>
            </LOCALNAMESPACEPATH>
            <IPARAMVALUE NAME="InstanceName">
              <INSTANCENAME CLASSNAME="CIM_IndicationSubscription">
                <KEYBINDING NAME="Filter">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationFilter">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationFilter
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sFilter
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </KEYBINDING>
                <KEYBINDING NAME="Handler">
                  <VALUE.REFERENCE>
                    <INSTANCENAME CLASSNAME="CIM_IndicationHandlerCIMXML">
                      <KEYBINDING NAME="SystemCreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_ComputerSystem
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="SystemName">
                        <KEYVALUE VALUETYPE="string">
                        %s
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="CreationClassName">
                        <KEYVALUE VALUETYPE="string">
                        CIM_IndicationHandlerCIMXML
                        </KEYVALUE>
                      </KEYBINDING>
                      <KEYBINDING NAME="Name">
                        <KEYVALUE VALUETYPE="string">
                        %sHandler
                        </KEYVALUE>
                      </KEYBINDING>
                    </INSTANCENAME>
                  </VALUE.REFERENCE>
                </KEYBINDING>
              </INSTANCENAME>
            </IPARAMVALUE>
          </IMETHODCALL>
        </SIMPLEREQ>
      </MESSAGE>
    </CIM>;
    """ % (sysname, name, sysname, name)

class CIMIndication:
    def __init__(self, xmldata):
        dom = parseString(xmldata)

        instances = dom.getElementsByTagName("INSTANCE")
        attrs = instances[0].attributes.items()
        self.name = attrs[0][1]

    def __str__(self):
        return self.name

class CIMSocketHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    def do_POST(self):
        length = self.headers.getheader('content-length')
        data = self.rfile.read(int(length))

        indication = CIMIndication(data)
        print "Got indication: %s" % indication
        if self.server.print_ind:
            print "%s\n\n" % data

class CIMIndicationSubscription:
    def __init__(self, name, typ, ns, print_ind, sysname):
        self.name = name
        self.type = typ
        self.ns = ns
        self.sysname = sysname

        self.server = BaseHTTPServer.HTTPServer(('', 8000), CIMSocketHandler)
        self.server.print_ind = print_ind
        self.port = 8000

        self.filter_xml = filter_xml(name, typ, ns, sysname)
        self.handler_xml = handler_xml(name, self.port, sysname)
        self.subscription_xml = subscription_xml(name, sysname)

    def __do_cimpost(self, conn, body, method, auth_hdr=None):
        headers = {"CIMOperation" : "MethodCall",
                   "CIMMethod"    : method,
                   "CIMObject"    : "root/PG_Interop",
                   "Content-Type" : "text/cimxml"}

        if auth_hdr:
            headers["Authorization"] = "Basic %s" % auth_hdr

        conn.request("POST", "/cimom", body, headers)
        resp = conn.getresponse()
        if not resp.getheader("content-length"):
            raise Exception("Authentication (or request) Failed!")

        resp.read()

    def subscribe(self, url, cred=None):
        self.conn = httplib.HTTPConnection(url)
        if cred:
            (u, p) = cred
            auth_hdr = base64.b64encode("%s:%s" % (u, p))
        else:
            auth_hdr = None

        self.__do_cimpost(self.conn, self.filter_xml,
                          "CreateInstance", auth_hdr)
        self.__do_cimpost(self.conn, self.handler_xml,
                          "CreateInstance", auth_hdr)
        self.__do_cimpost(self.conn, self.subscription_xml,
                          "CreateInstance", auth_hdr)

    def unsubscribe(self, cred=None):
        if cred:
            (u, p) = cred
            auth_hdr = base64.b64encode("%s:%s" % (u, p))
        else:
            auth_hdr = None

        xml = delete_sub_xml(self.name, self.sysname)
        self.__do_cimpost(self.conn, xml,
                          "DeleteInstance", auth_hdr)
        xml = delete_inst_xml(self.name, "Handler", self.sysname)
        self.__do_cimpost(self.conn, xml,
                          "DeleteInstance", auth_hdr)
        xml = delete_inst_xml(self.name, "Filter", self.sysname)
        self.__do_cimpost(self.conn, xml,
                          "DeleteInstance", auth_hdr)

def dump_xml(name, typ, ns, sysname):
    filter_str = filter_xml(name, typ, ns, sysname)
    handler_str = handler_xml(name, 8000, sysname)
    subscript_str = subscription_xml(name, sysname)
    del_filter_str = delete_inst_xml(name, "Filter", sysname)
    del_handler_str = delete_inst_xml(name, "Handler", sysname)
    del_subscript_str = delete_sub_xml(name, sysname)

    print "CreateFilter:\n%s\n" % filter_str
    print "DeleteFilter:\n%s\n" % del_filter_str
    print "CreateHandler:\n%s\n" % handler_str
    print "DeleteHandler:\n%s\n" % del_handler_str
    print "CreateSubscription:\n%s\n" % subscript_str
    print "DeleteSubscription:\n%s\n" % del_subscript_str

def main():
    usage = "usage: %prog [options] provider\nex: %prog CIM_InstModification"
    parser = OptionParser(usage)

    parser.add_option("-u", "--url", dest="url", default="localhost:5988",
                      help="URL of CIMOM to connect to (host:port)")
    parser.add_option("-N", "--ns", dest="ns", default="root/virt",
                      help="Namespace (default is root/virt)")
    parser.add_option("-n", "--name", dest="name", default="Test",
                      help="Name for filter, handler, subscription \
                      (default: Test)")
    parser.add_option("-d", "--dump-xml", dest="dump", default=False,
                      action="store_true",
                      help="Dump the xml that would be used and quit.")
    parser.add_option("-p", "--print-ind", dest="print_ind", default=False,
                      action="store_true",
                      help="Print received indications to stdout.")
    parser.add_option("-U", "--user", dest="username", default=None,
                      help="HTTP Auth username", dest="username")
    parser.add_option("-P", "--pass", dest="password", default=None,
                      help="HTTP Auth password", dest="password")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        print "Fatal: no indication type provided."
        sys.exit(1)

    if options.username:
        auth = (options.username, options.password)
    else:
        auth = None

    if ":" in options.url:
        (sysname, port) = options.url.split(":")
    else:
        sysname = url

    if options.dump:
        dump_xml(options.name, args[0], options.ns, sysname)
        sys.exit(0)

    sub = CIMIndicationSubscription(options.name, args[0], options.ns,
                                    options.print_ind, sysname)
    sub.subscribe(options.url, auth)
    print "Watching for %s" % args[0]

    try:
        sub.server.serve_forever()
    except KeyboardInterrupt,e:
        sub.unsubscribe(auth)
        print "Cancelling subscription for %s" % args[0]

if __name__=="__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>import datetime
import logging
import os
import platform
import subprocess

import yaml
from watchmaker.managers import LinuxWorkersManager, WindowsWorkersManager
from watchmaker import SystemFatal as exceptionhandler
from watchmaker as static

class Prepare(object):
    """
    This class is the entry point for watchmaker.  It prepares a system for setup and installation.
    """

    def __init__(self, noreboot=False, s3=False, config_path=None, stream=False, log_path=None):
        """
        :param noreboot: Instances are rebooted after installation.  If this value is set to
        True then the instance will not be rebooted.
        :type noreboot: bool
        :param s3: Should an s3 bucket be used for the installation files.
        :type s3: bool
        :param config_path: Path to YAML configuration file.
        :type config_path: basestring
        :param stream: Enables self.logger to a file.
        :type stream: bool
        :param log_path: Path to logfile for stream self.logger.
        :type log_path: basestring
        """

        self.kwargs = {}
        self.noreboot = noreboot
        self.s3 = s3
        self.system = platform.system()
        self.config_path = config_path
        self.default_config = os.path.join(static.__path__[0], 'config.yaml')
        self.log_path = log_path
        self.config = None
        self.system_params = None
        self.system_drive = None
        self.execution_scripts = None
        self.logger = logging.getLogger()

        if stream and os.path.exists(log_path):
            logging.basicConfig(filename=os.path.join(self.log_path,
                                                      'watchmaker-{0}.log'.format(str(datetime.date.today()))),
                                format='%(levelname)s:\t%(message)s',
                                level=logging.DEBUG)
            self.logger = logging.getLogger()
            self.logger.info('\n\n\n{0}'.format(datetime.datetime.now()))
        elif stream:
            self.logger.error('{0} does not exist'.format(log_path))
        else:
            self.logger.warning('Stream logger is not enabled!')

        self.logger.info('Parameters:  {0}'.format(self.kwargs))
        self.logger.info('System Type: {0}'.format(self.system))

    def _get_config_data(self):
        """
        Private method for reading configuration data for installation.

        :return self.config: Sets the config attribute with the data from the configuration YAML file after validation.
        """

        if self.config_path and not os.path.exists(self.config_path):
            self.logger.warning('User supplied config {0} does not exist.  '
                                'Using the default config.'.format(self.config_path))
            self.config_path = self.default_config
        elif not self.config_path:
            self.logger.warning('User did not supply a config.  Using the default config.')
            self.config_path = self.default_config
        else:
            self.logger.info('User supplied config being used.')

        with open(self.config_path) as f:
            data = f.read()

        if data:
            self.config = yaml.load(data)
        else:
            logging.debug('No data to load.')

    def _linux_paths(self):
        """
        Private method for setting up a linux environment for installation.

        :return self.system_params: Sets the system_params attribute for a linux system.
        """

        params = {}
        params['prepdir'] = os.path.join('{0}'.format(self.system_drive), 'usr', 'tmp', 'systemprep')
        params['readyfile'] = os.path.join('{0}'.format(self.system_drive), 'var', 'run', 'system-is-ready')
        params['logdir'] = os.path.join('{0}'.format(self.system_drive), 'var', 'log')
        params['workingdir'] = os.path.join('{0}'.format(params['prepdir']), 'workingfiles')
        params['restart'] = 'shutdown -r +1 &'
        self.system_params = params

    def _windows_paths(self):
        """
        Private method for setting up a Windows environment for installation.

        :return self.system_params: Sets the system_params attribute for a Windows system.
        """

        params = {}
        params['prepdir'] = os.path.join('{0}'.format(self.system_drive), 'watchmaker')
        params['readyfile'] = os.path.join('{0}'.format(params['prepdir']), 'system-is-ready')
        params['logdir'] = os.path.join('{0}'.format(params['prepdir']), 'Logs')
        params['workingdir'] = os.path.join('{0}'.format(params['prepdir']), 'WorkingFiles')
        params['shutdown_path'] = os.path.join('{0}'.format(os.environ['SYSTEMROOT']), 'system32', 'shutdown.exe')
        params['restart'] = params["shutdown_path"] + " /r /t 30 /d p:2:4 /c " + \
                            "\"watchmaker complete. Rebooting computer.\""
        self.system_params = params

    def _get_system_params(self):
        """
        Private method for handling the setup of the workspace and environment for the appropriate operating system.
        This method also creates the appropriate directories necessary for installation.

        """

        if 'Linux' in self.system:
            self.system_drive = '/'
            self._linux_paths()
        elif 'Windows' in self.system:
            self.system_drive = os.environ['SYSTEMDRIVE']
            self._windows_paths()
        else:
            self.logger.fatal('System, {0}, is not recognized?'.format(self.system))
            exceptionhandler('The scripts do not recognize this system type: {0}'.format(self.system))

        # Create watchmaker directories
        try:
            if not os.path.exists(self.system_params['logdir']):
                os.makedirs(self.system_params['logdir'])
            if not os.path.exists(self.system_params['workingdir']):
                os.makedirs(self.system_params['workingdir'])
        except Exception as exc:
            self.logger.fatal('Could not create a directory in {0}.\n'
                              'Exception: {0}'.format(self.system_params['prepdir'], exc))
            exceptionhandler(exc)

    def _get_scripts_to_execute(self):
        """
        Private method for parsing configuration data and updating configuration with specified parameters from
        user configuration.

        :return self.execution_scripts: Sets attribute with prepared configuration data for the target system.
        """

        self._get_config_data()

        scriptstoexecute = self.config[self.system]
        for item in self.config[self.system]:
            try:
                self.config[self.system][item]['Parameters'].update(self.kwargs)
            except Exception as exc:
                self.logger.fatal('For {0} in {1} the parameters could not be merged'.format(
                    item,
                    self.config_path
                ))
                exceptionhandler(exc)

        self.execution_scripts = scriptstoexecute

    def install_system(self):
        """
        Method for initiating the installation of the prepared system.  After execution the system should be properly
        provisioned.
        """

        self.logger.info('+' * 80)

        self._get_system_params()
        self.logger.info(self.system_params)

        self._get_scripts_to_execute()
        self.logger.info('Got scripts to execute.')

        if 'Linux' in self.system:
            workers_manager = LinuxWorkersManager(self.s3, self.system_params, self.execution_scripts)
        elif 'Windows' in self.system:
            workers_manager = WindowsWorkersManager(self.s3, self.system_params, self.execution_scripts)
        else:
            exceptionhandler('There is no known System!')

        try:
            workers_manager.worker_cadence()
        except Exception as e:
            exceptionhandler('Execution of the workers cadence has failed. {0}'.format(e))

        if self.noreboot:
            self.logger.info('Detected `noreboot` switch. System will not be rebooted.')
        else:
            self.logger.info('Reboot scheduled. System will reboot after the script exits.')
            subprocess.call(self.system_params['restart'], shell=True)

        self.logger.info('-' * 80)
<commit_msg>fixing poorly formed import.<commit_after>import datetime
import logging
import os
import platform
import subprocess

import yaml
from watchmaker.managers import LinuxWorkersManager, WindowsWorkersManager
from watchmaker import SystemFatal as exceptionhandler
from watchmaker import static

class Prepare(object):
    """
    This class is the entry point for watchmaker.  It prepares a system for setup and installation.
    """

    def __init__(self, noreboot=False, s3=False, config_path=None, stream=False, log_path=None):
        """
        :param noreboot: Instances are rebooted after installation.  If this value is set to
        True then the instance will not be rebooted.
        :type noreboot: bool
        :param s3: Should an s3 bucket be used for the installation files.
        :type s3: bool
        :param config_path: Path to YAML configuration file.
        :type config_path: basestring
        :param stream: Enables self.logger to a file.
        :type stream: bool
        :param log_path: Path to logfile for stream self.logger.
        :type log_path: basestring
        """

        self.kwargs = {}
        self.noreboot = noreboot
        self.s3 = s3
        self.system = platform.system()
        self.config_path = config_path
        self.default_config = os.path.join(static.__path__[0], 'config.yaml')
        self.log_path = log_path
        self.config = None
        self.system_params = None
        self.system_drive = None
        self.execution_scripts = None
        self.logger = logging.getLogger()

        if stream and os.path.exists(log_path):
            logging.basicConfig(filename=os.path.join(self.log_path,
                                                      'watchmaker-{0}.log'.format(str(datetime.date.today()))),
                                format='%(levelname)s:\t%(message)s',
                                level=logging.DEBUG)
            self.logger = logging.getLogger()
            self.logger.info('\n\n\n{0}'.format(datetime.datetime.now()))
        elif stream:
            self.logger.error('{0} does not exist'.format(log_path))
        else:
            self.logger.warning('Stream logger is not enabled!')

        self.logger.info('Parameters:  {0}'.format(self.kwargs))
        self.logger.info('System Type: {0}'.format(self.system))

    def _get_config_data(self):
        """
        Private method for reading configuration data for installation.

        :return self.config: Sets the config attribute with the data from the configuration YAML file after validation.
        """

        if self.config_path and not os.path.exists(self.config_path):
            self.logger.warning('User supplied config {0} does not exist.  '
                                'Using the default config.'.format(self.config_path))
            self.config_path = self.default_config
        elif not self.config_path:
            self.logger.warning('User did not supply a config.  Using the default config.')
            self.config_path = self.default_config
        else:
            self.logger.info('User supplied config being used.')

        with open(self.config_path) as f:
            data = f.read()

        if data:
            self.config = yaml.load(data)
        else:
            logging.debug('No data to load.')

    def _linux_paths(self):
        """
        Private method for setting up a linux environment for installation.

        :return self.system_params: Sets the system_params attribute for a linux system.
        """

        params = {}
        params['prepdir'] = os.path.join('{0}'.format(self.system_drive), 'usr', 'tmp', 'systemprep')
        params['readyfile'] = os.path.join('{0}'.format(self.system_drive), 'var', 'run', 'system-is-ready')
        params['logdir'] = os.path.join('{0}'.format(self.system_drive), 'var', 'log')
        params['workingdir'] = os.path.join('{0}'.format(params['prepdir']), 'workingfiles')
        params['restart'] = 'shutdown -r +1 &'
        self.system_params = params

    def _windows_paths(self):
        """
        Private method for setting up a Windows environment for installation.

        :return self.system_params: Sets the system_params attribute for a Windows system.
        """

        params = {}
        params['prepdir'] = os.path.join('{0}'.format(self.system_drive), 'watchmaker')
        params['readyfile'] = os.path.join('{0}'.format(params['prepdir']), 'system-is-ready')
        params['logdir'] = os.path.join('{0}'.format(params['prepdir']), 'Logs')
        params['workingdir'] = os.path.join('{0}'.format(params['prepdir']), 'WorkingFiles')
        params['shutdown_path'] = os.path.join('{0}'.format(os.environ['SYSTEMROOT']), 'system32', 'shutdown.exe')
        params['restart'] = params["shutdown_path"] + " /r /t 30 /d p:2:4 /c " + \
                            "\"watchmaker complete. Rebooting computer.\""
        self.system_params = params

    def _get_system_params(self):
        """
        Private method for handling the setup of the workspace and environment for the appropriate operating system.
        This method also creates the appropriate directories necessary for installation.

        """

        if 'Linux' in self.system:
            self.system_drive = '/'
            self._linux_paths()
        elif 'Windows' in self.system:
            self.system_drive = os.environ['SYSTEMDRIVE']
            self._windows_paths()
        else:
            self.logger.fatal('System, {0}, is not recognized?'.format(self.system))
            exceptionhandler('The scripts do not recognize this system type: {0}'.format(self.system))

        # Create watchmaker directories
        try:
            if not os.path.exists(self.system_params['logdir']):
                os.makedirs(self.system_params['logdir'])
            if not os.path.exists(self.system_params['workingdir']):
                os.makedirs(self.system_params['workingdir'])
        except Exception as exc:
            self.logger.fatal('Could not create a directory in {0}.\n'
                              'Exception: {0}'.format(self.system_params['prepdir'], exc))
            exceptionhandler(exc)

    def _get_scripts_to_execute(self):
        """
        Private method for parsing configuration data and updating configuration with specified parameters from
        user configuration.

        :return self.execution_scripts: Sets attribute with prepared configuration data for the target system.
        """

        self._get_config_data()

        scriptstoexecute = self.config[self.system]
        for item in self.config[self.system]:
            try:
                self.config[self.system][item]['Parameters'].update(self.kwargs)
            except Exception as exc:
                self.logger.fatal('For {0} in {1} the parameters could not be merged'.format(
                    item,
                    self.config_path
                ))
                exceptionhandler(exc)

        self.execution_scripts = scriptstoexecute

    def install_system(self):
        """
        Method for initiating the installation of the prepared system.  After execution the system should be properly
        provisioned.
        """

        self.logger.info('+' * 80)

        self._get_system_params()
        self.logger.info(self.system_params)

        self._get_scripts_to_execute()
        self.logger.info('Got scripts to execute.')

        if 'Linux' in self.system:
            workers_manager = LinuxWorkersManager(self.s3, self.system_params, self.execution_scripts)
        elif 'Windows' in self.system:
            workers_manager = WindowsWorkersManager(self.s3, self.system_params, self.execution_scripts)
        else:
            exceptionhandler('There is no known System!')

        try:
            workers_manager.worker_cadence()
        except Exception as e:
            exceptionhandler('Execution of the workers cadence has failed. {0}'.format(e))

        if self.noreboot:
            self.logger.info('Detected `noreboot` switch. System will not be rebooted.')
        else:
            self.logger.info('Reboot scheduled. System will reboot after the script exits.')
            subprocess.call(self.system_params['restart'], shell=True)

        self.logger.info('-' * 80)
<|endoftext|>


<commit_before>import json
import os
import socket
import sys
import threading
import time

# mesos
import mesos_pb2 as mesos_pb

from twitter.common import log
from twitter.common.concurrent import deadline, defer, Timeout
from twitter.common.exceptions import ExceptionalThread
from twitter.common.quantity import Amount, Time

# thermos
from twitter.mesos.clusters import Cluster
from twitter.mesos.common.http_signaler import HttpSignaler
from twitter.mesos.config import PortResolver
from twitter.mesos.config.schema import MesosJob, MesosTaskInstance
from twitter.mesos.config.thrift import task_instance_from_job
from twitter.thermos.base.path import TaskPath
from twitter.thermos.monitoring.monitor import TaskMonitor

# thrifts
from gen.twitter.mesos.constants import DEFAULT_ENVIRONMENT
from gen.twitter.mesos.ttypes import AssignedTask
from thrift.Thrift import TException
from thrift.TSerialization import deserialize as thrift_deserialize

from .discovery_manager import DiscoveryManager
from .executor_base import ThermosExecutorBase
from .health_checker import HealthCheckerThread
from .kill_manager import KillManager
from .resource_checkpoints import ResourceCheckpointer
from .resource_manager import ResourceManager
from .status_manager import StatusManager
from .task_runner_wrapper import (
  ProductionTaskRunner,
  AngrybirdTaskRunner)

if 'ANGRYBIRD_THERMOS' in os.environ:
  RUNNER_CLASS = AngrybirdTaskRunner
else:
  RUNNER_CLASS = ProductionTaskRunner

def default_exit_action():
  sys.exit(0)

class ThermosExecutorTimer(ExceptionalThread):
  EXECUTOR_TIMEOUT = Amount(10, Time.SECONDS)

  def __init__(self, executor, driver):
    self._executor = executor
    self._driver = driver
    super(ThermosExecutorTimer, self).__init__()
    self.daemon = True

  def run(self):
    self._executor.launched.wait(self.EXECUTOR_TIMEOUT.as_(Time.SECONDS))
    if not self._executor.launched.is_set():
      self._executor.log('Executor timing out on lack of launchTask.')
      self._driver.stop()

class ThermosExecutor(ThermosExecutorBase):
  STOP_WAIT = Amount(5, Time.SECONDS)
  RESOURCE_CHECKPOINT = 'resource_usage.recordio'
  RUNNER_INITIALIZATION_TIMEOUT = Amount(5, Time.MINUTES)

  def __init__(self, runner_class=RUNNER_CLASS, manager_class=StatusManager):
    ThermosExecutorBase.__init__(self)
    self._runner = None
    self._task_id = None
    self._manager = None
    self._runner_class = runner_class
    self._manager_class = manager_class
    self._kill_manager = KillManager()
    # To catch killTasks sent while the runner initialization is occurring
    self._abort_runner = threading.Event()
    self.launched = threading.Event()

  @staticmethod
  def deserialize_assigned_task(task):
    """
      Deserialize task from a launchTask task protocol buffer.
      Returns AssignedTask
    """
    try:
      assigned_task = thrift_deserialize(AssignedTask(), task.data)
    except (EOFError, TException) as e:
      raise ValueError('Could not deserialize task! %s' % e)
    return assigned_task

  @staticmethod
  def deserialize_thermos_task(assigned_task):
    """
      Deserialize MesosTaskInstance from a AssignedTask thrift.
      Returns twitter.mesos.config.schema.MesosTaskInstance and the map of
      ports assigned by the scheduler.
    """
    thermos_task = assigned_task.task.thermosConfig
    if not thermos_task:
      raise ValueError('Task did not have a thermosConfig!')
    try:
      json_blob = json.loads(thermos_task)
    except Exception as e:
      raise ValueError('Could not deserialize thermosConfig JSON! %s' % e)
    # As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
    # or we can be sending a MesosJob.  So handle both possible cases.  Once everyone
    # is using MesosJob, then we can begin to leverage additional information that
    # becomes available such as cluster.
    if 'instance' in json_blob:
      return MesosTaskInstance.json_loads(thermos_task)
    else:
      mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task),
          assigned_task.task.shardId)
      if refs:
        raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, refs)))
      return mti

  @classmethod
  def extract_ensemble(cls, assigned_task, default=Cluster.DEFAULT_ENSEMBLE):
    thermos_task = assigned_task.task.thermosConfig
    if 'instance' in json.loads(thermos_task):
      # Received MesosTaskInstance
      return default
    else:
      job = MesosJob.json_loads(thermos_task)
      try:
        cluster = Cluster.get(job.cluster().get())
        return cluster.zk
      except KeyError:
        return default

  @property
  def runner(self):
    return self._runner

  def _start_runner(self, driver, assigned_task, mesos_task, portmap):
    """
      Commence running a Task.
        - Initialize the TaskRunnerWrapper (create sandbox)
        - Start the TaskRunnerWrapper (fork the Thermos TaskRunner)
        - Set up necessary HealthCheckers
        - Set up DiscoveryManager, if applicable
        - Set up ResourceCheckpointer
        - Set up StatusManager, and attach HealthCheckers

    """

    try:
      deadline(self._runner.initialize, timeout=self.RUNNER_INITIALIZATION_TIMEOUT,
               daemon=True, propagate=True)
    except Timeout:
      msg = 'Timed out waiting for sandbox to initialize!'
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'FAILED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return
    except self._runner.TaskError as e:
      msg = 'Initialization of task runner failed: %s' % e
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'FAILED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    if self._abort_runner.is_set():
      msg = 'Task killed during initialization'
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'KILLED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    try:
      self._runner.start()
    except self._runner.TaskError as e:
      msg = 'Task initialization failed: %s' % e
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'FAILED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    def wait_for_task():
      while not self._runner.is_started():
        log.debug('   - sleeping...')
        time.sleep(Amount(250, Time.MILLISECONDS).as_(Time.SECONDS))

    try:
      log.debug('Waiting for task to start.')
      deadline(wait_for_task, timeout=Amount(1, Time.MINUTES))
    except Timeout:
      msg = 'Timed out waiting for task to start!'
      log.fatal(msg)
      self._runner.lose()
      self.send_update(driver, self._task_id, 'LOST', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    log.debug('Task started.')
    self.send_update(driver, self._task_id, 'RUNNING')

    health_checkers = []

    http_signaler = None
    if portmap.get('health'):
      health_check_config = mesos_task.health_check_config().get()
      http_signaler = HttpSignaler(portmap.get('health'),
          timeout_secs=health_check_config.get('timeout_secs'))
      health_checker = HealthCheckerThread(
          http_signaler.health,
          interval_secs=health_check_config.get('interval_secs'),
          initial_interval_secs=health_check_config.get('initial_interval_secs'),
          max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
      health_checkers.append(health_checker)

    task_path = TaskPath(root=self._runner._checkpoint_root, task_id=self._task_id)
    resource_manager = ResourceManager(
        mesos_task.task().resources(),
        TaskMonitor(task_path, self._task_id),
        self._runner.sandbox.root
    )
    health_checkers.append(resource_manager)

    ResourceCheckpointer(lambda: resource_manager.sample,
        os.path.join(self._runner.artifact_dir, self.RESOURCE_CHECKPOINT),
        recordio=True).start()

    if mesos_task.has_announce():
      discovery_manager = DiscoveryManager(
          mesos_task.role().get(),
          mesos_task.environment().get() if mesos_task.has_environment() else DEFAULT_ENVIRONMENT,
          assigned_task.task.jobName,
          socket.gethostname(),
          mesos_task.announce().primary_port().get(),
          portmap,
          assigned_task.task.shardId)
          # TODO(wickman) Possibly return this code once we've hashed out MESOS-2753
          # ensemble=self.extract_ensemble(assigned_task))
      health_checkers.append(discovery_manager)

    health_checkers.append(self._kill_manager)

    self._manager = self._manager_class(
        self._runner,
        driver,
        self._task_id,
        signaler=http_signaler,
        health_checkers=health_checkers)
    self._manager.start()

  """ Mesos Executor API methods follow """

  @classmethod
  def resolve_ports(cls, mesos_task, portmap):
    """Given a mesos task and the portmap of resolved ports from the scheduler,
       create a fully resolved map of port name => port number for the thermos
       runner and discovery manager."""
    task_portmap = mesos_task.announce().portmap().get() if mesos_task.has_announce() else {}
    task_portmap.update(portmap)
    task_portmap = PortResolver.resolve(task_portmap)

    for name, port in task_portmap.items():
      if not isinstance(port, int):
        log.warning('Task has unmapped port: %s => %s' % (name, port))

    return dict((name, port) for (name, port) in task_portmap.items() if isinstance(port, int))

  def launchTask(self, driver, task):
    """
      Invoked when a task has been launched on this executor (initiated via Scheduler::launchTasks).
      Note that this task can be realized with a thread, a process, or some simple computation,
      however, no other callbacks will be invoked on this executor until this callback has returned.
    """
    self.launched.set()
    self.log('launchTask got task: %s:%s' % (task.name, task.task_id.value))

    if self._runner:
      # XXX(wickman) This looks like it's sending LOST for the wrong task?
      # TODO(wickman) Send LOST immediately for both tasks?
      log.error('Error!  Already running a task! %s' % self._runner)
      self.send_update(driver, self._task_id, 'LOST',
          "Task already running on this executor: %s" % self._task_id)
      return

    self._slave_id = task.slave_id.value
    self._task_id = task.task_id.value

    try:
      assigned_task = self.deserialize_assigned_task(task)
      mesos_task = self.deserialize_thermos_task(assigned_task)
    except Exception as e:
      log.fatal('Could not deserialize AssignedTask: %s' % e)
      self.send_update(driver, self._task_id, 'FAILED', "Could not deserialize task: %s" % e)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    self.send_update(driver, self._task_id, 'STARTING', 'Initializing sandbox.')

    # Fully resolve the portmap
    portmap = self.resolve_ports(mesos_task, assigned_task.assignedPorts)

    # start the process on a separate thread and give the message processing thread back
    # to the driver
    try:
      self._runner = self._runner_class(self._task_id, mesos_task, mesos_task.role().get(), portmap)
    except self._runner_class.TaskError as e:
      self.send_update(driver, self._task_id, 'FAILED', str(e))
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    defer(lambda: self._start_runner(driver, assigned_task, mesos_task, portmap))

  def _signal_kill_manager(self, driver, task_id, reason):
    if self._runner is None:
      log.error('Was asked to kill task but no task running!')
      return
    if task_id.value != self._task_id:
      log.error('Asked to kill a task other than what we are running!')
      return
    if not self._runner.is_initialized():
      log.error('Asked to kill task with incomplete sandbox - aborting runner start')
      self._abort_runner.set()
      return
    self.log('Activating kill manager.')
    self._kill_manager.kill(reason)

  def killTask(self, driver, task_id):
    """
     Invoked when a task running within this executor has been killed (via
     SchedulerDriver::killTask). Note that no status update will be sent on behalf of the executor,
     the executor is responsible for creating a new TaskStatus (i.e., with TASK_KILLED) and invoking
     ExecutorDriver::sendStatusUpdate.
    """
    self.log('killTask got task_id: %s' % task_id)
    self._signal_kill_manager(driver, task_id, "Instructed to kill task.")

  def shutdown(self, driver):
    """
     Invoked when the executor should terminate all of its currently running tasks. Note that after
     Mesos has determined that an executor has terminated any tasks that the executor did not send
     terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED, etc) a TASK_LOST
     status update will be created.

    """
    self.log('shutdown called')
    if self._task_id:
      self._signal_kill_manager(driver, mesos_pb.TaskID(value=self._task_id),
          "Told to shut down executor.")
    self.log('shutdown returned')
<commit_msg>Double runner initialization timeout in Thermos executor<commit_after>import json
import os
import socket
import sys
import threading
import time

# mesos
import mesos_pb2 as mesos_pb

from twitter.common import log
from twitter.common.concurrent import deadline, defer, Timeout
from twitter.common.exceptions import ExceptionalThread
from twitter.common.quantity import Amount, Time

# thermos
from twitter.mesos.clusters import Cluster
from twitter.mesos.common.http_signaler import HttpSignaler
from twitter.mesos.config import PortResolver
from twitter.mesos.config.schema import MesosJob, MesosTaskInstance
from twitter.mesos.config.thrift import task_instance_from_job
from twitter.thermos.base.path import TaskPath
from twitter.thermos.monitoring.monitor import TaskMonitor

# thrifts
from gen.twitter.mesos.constants import DEFAULT_ENVIRONMENT
from gen.twitter.mesos.ttypes import AssignedTask
from thrift.Thrift import TException
from thrift.TSerialization import deserialize as thrift_deserialize

from .discovery_manager import DiscoveryManager
from .executor_base import ThermosExecutorBase
from .health_checker import HealthCheckerThread
from .kill_manager import KillManager
from .resource_checkpoints import ResourceCheckpointer
from .resource_manager import ResourceManager
from .status_manager import StatusManager
from .task_runner_wrapper import (
  ProductionTaskRunner,
  AngrybirdTaskRunner)

if 'ANGRYBIRD_THERMOS' in os.environ:
  RUNNER_CLASS = AngrybirdTaskRunner
else:
  RUNNER_CLASS = ProductionTaskRunner

def default_exit_action():
  sys.exit(0)

class ThermosExecutorTimer(ExceptionalThread):
  EXECUTOR_TIMEOUT = Amount(10, Time.SECONDS)

  def __init__(self, executor, driver):
    self._executor = executor
    self._driver = driver
    super(ThermosExecutorTimer, self).__init__()
    self.daemon = True

  def run(self):
    self._executor.launched.wait(self.EXECUTOR_TIMEOUT.as_(Time.SECONDS))
    if not self._executor.launched.is_set():
      self._executor.log('Executor timing out on lack of launchTask.')
      self._driver.stop()

class ThermosExecutor(ThermosExecutorBase):
  STOP_WAIT = Amount(5, Time.SECONDS)
  RESOURCE_CHECKPOINT = 'resource_usage.recordio'
  RUNNER_INITIALIZATION_TIMEOUT = Amount(10, Time.MINUTES)

  def __init__(self, runner_class=RUNNER_CLASS, manager_class=StatusManager):
    ThermosExecutorBase.__init__(self)
    self._runner = None
    self._task_id = None
    self._manager = None
    self._runner_class = runner_class
    self._manager_class = manager_class
    self._kill_manager = KillManager()
    # To catch killTasks sent while the runner initialization is occurring
    self._abort_runner = threading.Event()
    self.launched = threading.Event()

  @staticmethod
  def deserialize_assigned_task(task):
    """
      Deserialize task from a launchTask task protocol buffer.
      Returns AssignedTask
    """
    try:
      assigned_task = thrift_deserialize(AssignedTask(), task.data)
    except (EOFError, TException) as e:
      raise ValueError('Could not deserialize task! %s' % e)
    return assigned_task

  @staticmethod
  def deserialize_thermos_task(assigned_task):
    """
      Deserialize MesosTaskInstance from a AssignedTask thrift.
      Returns twitter.mesos.config.schema.MesosTaskInstance and the map of
      ports assigned by the scheduler.
    """
    thermos_task = assigned_task.task.thermosConfig
    if not thermos_task:
      raise ValueError('Task did not have a thermosConfig!')
    try:
      json_blob = json.loads(thermos_task)
    except Exception as e:
      raise ValueError('Could not deserialize thermosConfig JSON! %s' % e)
    # As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
    # or we can be sending a MesosJob.  So handle both possible cases.  Once everyone
    # is using MesosJob, then we can begin to leverage additional information that
    # becomes available such as cluster.
    if 'instance' in json_blob:
      return MesosTaskInstance.json_loads(thermos_task)
    else:
      mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task),
          assigned_task.task.shardId)
      if refs:
        raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, refs)))
      return mti

  @classmethod
  def extract_ensemble(cls, assigned_task, default=Cluster.DEFAULT_ENSEMBLE):
    thermos_task = assigned_task.task.thermosConfig
    if 'instance' in json.loads(thermos_task):
      # Received MesosTaskInstance
      return default
    else:
      job = MesosJob.json_loads(thermos_task)
      try:
        cluster = Cluster.get(job.cluster().get())
        return cluster.zk
      except KeyError:
        return default

  @property
  def runner(self):
    return self._runner

  def _start_runner(self, driver, assigned_task, mesos_task, portmap):
    """
      Commence running a Task.
        - Initialize the TaskRunnerWrapper (create sandbox)
        - Start the TaskRunnerWrapper (fork the Thermos TaskRunner)
        - Set up necessary HealthCheckers
        - Set up DiscoveryManager, if applicable
        - Set up ResourceCheckpointer
        - Set up StatusManager, and attach HealthCheckers

    """

    try:
      deadline(self._runner.initialize, timeout=self.RUNNER_INITIALIZATION_TIMEOUT,
               daemon=True, propagate=True)
    except Timeout:
      msg = 'Timed out waiting for sandbox to initialize!'
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'FAILED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return
    except self._runner.TaskError as e:
      msg = 'Initialization of task runner failed: %s' % e
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'FAILED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    if self._abort_runner.is_set():
      msg = 'Task killed during initialization'
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'KILLED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    try:
      self._runner.start()
    except self._runner.TaskError as e:
      msg = 'Task initialization failed: %s' % e
      log.fatal(msg)
      self.send_update(driver, self._task_id, 'FAILED', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    def wait_for_task():
      while not self._runner.is_started():
        log.debug('   - sleeping...')
        time.sleep(Amount(250, Time.MILLISECONDS).as_(Time.SECONDS))

    try:
      log.debug('Waiting for task to start.')
      deadline(wait_for_task, timeout=Amount(1, Time.MINUTES))
    except Timeout:
      msg = 'Timed out waiting for task to start!'
      log.fatal(msg)
      self._runner.lose()
      self.send_update(driver, self._task_id, 'LOST', msg)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    log.debug('Task started.')
    self.send_update(driver, self._task_id, 'RUNNING')

    health_checkers = []

    http_signaler = None
    if portmap.get('health'):
      health_check_config = mesos_task.health_check_config().get()
      http_signaler = HttpSignaler(portmap.get('health'),
          timeout_secs=health_check_config.get('timeout_secs'))
      health_checker = HealthCheckerThread(
          http_signaler.health,
          interval_secs=health_check_config.get('interval_secs'),
          initial_interval_secs=health_check_config.get('initial_interval_secs'),
          max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
      health_checkers.append(health_checker)

    task_path = TaskPath(root=self._runner._checkpoint_root, task_id=self._task_id)
    resource_manager = ResourceManager(
        mesos_task.task().resources(),
        TaskMonitor(task_path, self._task_id),
        self._runner.sandbox.root
    )
    health_checkers.append(resource_manager)

    ResourceCheckpointer(lambda: resource_manager.sample,
        os.path.join(self._runner.artifact_dir, self.RESOURCE_CHECKPOINT),
        recordio=True).start()

    if mesos_task.has_announce():
      discovery_manager = DiscoveryManager(
          mesos_task.role().get(),
          mesos_task.environment().get() if mesos_task.has_environment() else DEFAULT_ENVIRONMENT,
          assigned_task.task.jobName,
          socket.gethostname(),
          mesos_task.announce().primary_port().get(),
          portmap,
          assigned_task.task.shardId)
          # TODO(wickman) Possibly return this code once we've hashed out MESOS-2753
          # ensemble=self.extract_ensemble(assigned_task))
      health_checkers.append(discovery_manager)

    health_checkers.append(self._kill_manager)

    self._manager = self._manager_class(
        self._runner,
        driver,
        self._task_id,
        signaler=http_signaler,
        health_checkers=health_checkers)
    self._manager.start()

  """ Mesos Executor API methods follow """

  @classmethod
  def resolve_ports(cls, mesos_task, portmap):
    """Given a mesos task and the portmap of resolved ports from the scheduler,
       create a fully resolved map of port name => port number for the thermos
       runner and discovery manager."""
    task_portmap = mesos_task.announce().portmap().get() if mesos_task.has_announce() else {}
    task_portmap.update(portmap)
    task_portmap = PortResolver.resolve(task_portmap)

    for name, port in task_portmap.items():
      if not isinstance(port, int):
        log.warning('Task has unmapped port: %s => %s' % (name, port))

    return dict((name, port) for (name, port) in task_portmap.items() if isinstance(port, int))

  def launchTask(self, driver, task):
    """
      Invoked when a task has been launched on this executor (initiated via Scheduler::launchTasks).
      Note that this task can be realized with a thread, a process, or some simple computation,
      however, no other callbacks will be invoked on this executor until this callback has returned.
    """
    self.launched.set()
    self.log('launchTask got task: %s:%s' % (task.name, task.task_id.value))

    if self._runner:
      # XXX(wickman) This looks like it's sending LOST for the wrong task?
      # TODO(wickman) Send LOST immediately for both tasks?
      log.error('Error!  Already running a task! %s' % self._runner)
      self.send_update(driver, self._task_id, 'LOST',
          "Task already running on this executor: %s" % self._task_id)
      return

    self._slave_id = task.slave_id.value
    self._task_id = task.task_id.value

    try:
      assigned_task = self.deserialize_assigned_task(task)
      mesos_task = self.deserialize_thermos_task(assigned_task)
    except Exception as e:
      log.fatal('Could not deserialize AssignedTask: %s' % e)
      self.send_update(driver, self._task_id, 'FAILED', "Could not deserialize task: %s" % e)
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    self.send_update(driver, self._task_id, 'STARTING', 'Initializing sandbox.')

    # Fully resolve the portmap
    portmap = self.resolve_ports(mesos_task, assigned_task.assignedPorts)

    # start the process on a separate thread and give the message processing thread back
    # to the driver
    try:
      self._runner = self._runner_class(self._task_id, mesos_task, mesos_task.role().get(), portmap)
    except self._runner_class.TaskError as e:
      self.send_update(driver, self._task_id, 'FAILED', str(e))
      defer(driver.stop, delay=self.STOP_WAIT)
      return

    defer(lambda: self._start_runner(driver, assigned_task, mesos_task, portmap))

  def _signal_kill_manager(self, driver, task_id, reason):
    if self._runner is None:
      log.error('Was asked to kill task but no task running!')
      return
    if task_id.value != self._task_id:
      log.error('Asked to kill a task other than what we are running!')
      return
    if not self._runner.is_initialized():
      log.error('Asked to kill task with incomplete sandbox - aborting runner start')
      self._abort_runner.set()
      return
    self.log('Activating kill manager.')
    self._kill_manager.kill(reason)

  def killTask(self, driver, task_id):
    """
     Invoked when a task running within this executor has been killed (via
     SchedulerDriver::killTask). Note that no status update will be sent on behalf of the executor,
     the executor is responsible for creating a new TaskStatus (i.e., with TASK_KILLED) and invoking
     ExecutorDriver::sendStatusUpdate.
    """
    self.log('killTask got task_id: %s' % task_id)
    self._signal_kill_manager(driver, task_id, "Instructed to kill task.")

  def shutdown(self, driver):
    """
     Invoked when the executor should terminate all of its currently running tasks. Note that after
     Mesos has determined that an executor has terminated any tasks that the executor did not send
     terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED, etc) a TASK_LOST
     status update will be created.

    """
    self.log('shutdown called')
    if self._task_id:
      self._signal_kill_manager(driver, mesos_pb.TaskID(value=self._task_id),
          "Told to shut down executor.")
    self.log('shutdown returned')
<|endoftext|>


<commit_before><commit_msg>num_doc tem que ser numérico; quando ausente deve ser zero<commit_after><|endoftext|>


<commit_before>import datetime

from third import filetail

from __reader import Reader
from __message import Message

class AzionAnalytics(Reader):
    """Conf:
        - logpath (required): path of log file,
            e.g. /tmp/my.log
        - delimiter (required): character to split log lines,
            e.g. '\t'
        - columns (required): list of columns for each log line,
            e.g. ['date', 'hour', 'message']"""

    def setup(self):
        self.check_conf(['delimiter', 'columns', 'logpath'])
        self.tail = filetail.Tail(self.logpath, max_sleep=1)
        self.client = self.logpath.split('.')[1]
        self.time_format = "%d/%b/%Y:%H:%M:%S"
        self.start_counters()

    def start_counters(self):
        if self.count:
            self.agg_count = {}
            for key, value in self.count.items():
                d = {'value' : 0,
                     'start_time' : 0,
                     'content' : value[0],
                     'interval' : value[1]*60}
                self.agg_count[key] = d

    def check_conf(self, items):
        for item in items:
            if not hasattr(self, item):
                print '%s not defined in conf.yaml.' % item
                exit(-1)

    def dictify_line(self, line):
        return dict(zip(self.columns, line.strip().split(self.delimiter)))

    def get_datetime(self, date_format):
        return datetime.datetime.strptime(date_format, self.time_format)

    def get_minute(self, date):
        return date - datetime.timedelta(0, date.second)

    def get_empty_periods(self, beginning, end, period):
        empty_periods = []
        while beginning + period <= end:
            empty_periods.append(beginning)
            beginning = beginning + period
        return empty_periods

    def store_empty_periods(self, empty_periods):
        for empty_period in empty_periods:
            content = {'count' : 0,
                       'client' : self.client,
                       'time' : empty_period}
            msg = Message(content=content)
            self.store(msg)

    def read(self):
        cur_time = 0
        while True:
            try:
                line = self.tail.nextline()
                line_data = self.dictify_line(line)
                try:
                    cur_time = self.get_datetime(line_data['time_local'][1:21])
                    cur_minute = self.get_minute(cur_time)
                except ValueError:
                    print 'Cannot parse date, skipping line'
                    continue

                for column, metadata in self.agg_count.items():
                    #match occurred
                    if line_data[column] == metadata['content']:

                        #first occurrence
                        if metadata['start_time'] == 0:
                            metadata['start_time'] = cur_minute
                            metadata['value'] = 1
                            continue

                        time_passed = cur_time - metadata['start_time']

                        #in the interval
                        if time_passed.seconds <= metadata['interval']:
                            metadata['value'] += 1
                            continue

                        #closing interval
                        else:
                            time = metadata['start_time'].strftime(self.time_format)
                            content={'count_' + column : metadata['value'],
                                     'client' : self.client,
                                     'time' : time}
                            msg = Message(content=content)
                            self.store(msg)

                            #empty periods
                            next_interval = metadata['start_time'] + \
                                            datetime.timedelta(0, metadata['interval'])
                            empty_periods = self.get_empty_periods(next_interval,
                                                                   cur_minute,
                                                                   datetime.timedelta(0, metadata['interval']))
                            self.store_empty_periods(empty_periods)

                            metadata['value'] = 1
                            metadata['start_time'] = cur_minute

            except Exception, e:
                print 'error reading line: %s' % line
                print e
<commit_msg>refactoring<commit_after>import datetime

from third import filetail

from __reader import Reader
from __message import Message

class AzionAnalytics(Reader):
    """Conf:
        - logpath (required): path of log file,
            e.g. /tmp/my.log
        - delimiter (required): character to split log lines,
            e.g. '\t'
        - columns (required): list of columns for each log line,
            e.g. ['date', 'hour', 'message']"""

    def setup(self):
        self.check_conf(['delimiter', 'columns', 'logpath'])
        self.tail = filetail.Tail(self.logpath, max_sleep=1)
        self.client = self.logpath.split('.')[1]
        self.time_format = "%d/%b/%Y:%H:%M:%S"
        self.start_counters()

    def start_counters(self):
        if self.count:
            self.agg_count = {}
            for key, value in self.count.items():
                d = {'value' : 0,
                     'start_time' : 0,
                     'content' : value[0],
                     'interval' : value[1]*60}
                self.agg_count[key] = d

    def check_conf(self, items):
        for item in items:
            if not hasattr(self, item):
                print '%s not defined in conf.yaml.' % item
                exit(-1)

    def dictify_line(self, line):
        return dict(zip(self.columns, line.strip().split(self.delimiter)))

    def get_datetime(self, date_format):
        return datetime.datetime.strptime(date_format, self.time_format)

    def get_minute(self, date):
        return date - datetime.timedelta(0, date.second)

    def add_interval(self, date, seconds):
        return date + datetime.timedelta(0, seconds)

    def get_empty_periods(self, beginning, end, period):
        empty_periods = []
        while beginning + period <= end:
            empty_periods.append(beginning)
            beginning = beginning + period
        return empty_periods

    def store_empty_periods(self, empty_periods):
        for empty_period in empty_periods:
            content = {'count' : 0,
                       'client' : self.client,
                       'time' : empty_period}
            msg = Message(content=content)
            self.store(msg)

    def read(self):
        cur_time = 0
        while True:
            try:
                line = self.tail.nextline()
                line_data = self.dictify_line(line)
                try:
                    cur_time = self.get_datetime(line_data['time_local'][1:21])
                    cur_minute = self.get_minute(cur_time)
                except ValueError:
                    print 'Cannot parse date, skipping line'
                    continue

                for column, metadata in self.agg_count.items():
                    #match occurred
                    if line_data[column] == metadata['content']:

                        #first occurrence
                        if metadata['start_time'] == 0:
                            metadata['start_time'] = cur_minute
                            metadata['value'] = 1
                            continue

                        time_passed = cur_time - metadata['start_time']

                        #in the interval
                        if time_passed.seconds <= metadata['interval']:
                            metadata['value'] += 1
                            continue

                        #closing interval
                        else:
                            time = metadata['start_time'].strftime(self.time_format)
                            content={'count_' + column : metadata['value'],
                                     'client' : self.client,
                                     'time' : time}
                            msg = Message(content=content)
                            self.store(msg)

                            #empty periods
                            next_interval = self.add_interval(metadata['start_time'],
                                                              metadata['interval'])
                            empty_periods = self.get_empty_periods(next_interval,
                                                                   cur_minute,
                                                                   datetime.timedelta(0, metadata['interval']))
                            self.store_empty_periods(empty_periods)

                            metadata['value'] = 1
                            metadata['start_time'] = cur_minute

            except Exception, e:
                print 'error reading line: %s' % line
                print e
<|endoftext|>