<commit_before>#!/usr/bin/env python

import threading
import time
import random

import sock
import model
import sp_exceptions
import handler

class Agent:
    def __init__(self):
        # whether we're connected to a server yet or not
        self.__connected = False

        # set all variables and important objects to appropriate values for
        # pre-connect state.
        self.__sock = None

        self.world = None
        self.body = None
        self.msg_handler = None
        self.act_handler = None

        self.__parsing = False
        self.__msg_thread = None

        self.__thinking = False
        self.__think_thread = None

    def connect(self, host, port, teamname, version=11):
        """
        Gives us a connection to the server as one player on a team.  This
        immediately connects the agent to the server and starts receiving and
        parsing the information it sends.
        """

        # if already connected, raise an error since user may have wanted to
        # connect again to a different server.
        if self.__connected:
            msg = "Cannot connect while already connected, disconnect first."
            raise sp_exceptions.AgentConnectionStateError(msg)

        # set connected state
        self.__connected = True

        # the pipe through which all of our communication takes place
        self.__sock = sock.Socket(host, port)

        # our models of the world and our body
        self.world = model.WorldModel()
        self.body = model.BodyModel()

        # handles all messages received from the server
        self.msg_handler = handler.MessageHandler(self.world, self.body)

        # handles the sending of actions to the server
        self.act_handler = handler.ActionHandler(self.__sock)

        # set up our threaded message receiving system
        self.__parsing = True # tell thread that we're currently running
        self.__msg_thread = threading.Thread(target=self.__message_loop,
                                           name="message_loop")
        self.__msg_thread.daemon = True # dies when parent thread dies

        # start processing received messages. this will catch the initial server
        # response and all subsequent communication.
        self.__msg_thread.start()

        # send the init message and allow the message handler to handle further
        # responses.
        init_address = self.__sock.address
        init_msg = "(init %s (version %d))"
        self.__sock.send(init_msg % (teamname, version))

        # wait until the socket receives a response from the server and gets its
        # assigned port.
        while self.__sock.address == init_address:
            time.sleep(0.0001)

        # create our thinking thread.  this will perform the actions necessary
        # to play a game of robo-soccer.
        self.__thinking = False
        self.__think_thread = threading.Thread(target=self.__think_loop,
                                             name="think_loop")
        self.__think_thread.daemon = True

    def play(self):
        """
        Kicks off the thread that does the agent's thinking, allowing it to play
        during the game.  Throws an exception if called while the agent is
        already playing.
        """

        # ensure we're connected before doing anything
        if not self.__connected:
            msg = "Must be connected to a server to begin play."
            raise sp_exceptions.AgentConnectionStateError(msg)

        # throw exception if called while thread is already running
        if self.__thinking:
            raise sp_exceptions.AgentAlreadyPlayingError(
                "Agent is already playing.")

        # run the method that sets up the agent's persistant variables
        self.setup_environment()

        # tell the thread that it should be running, then start it
        self.__thinking = True
        self.__think_thread.start()

    def disconnect(self):
        """
        Tell the loop threads to stop and signal the server that we're
        disconnecting, then join the loop threads and destroy all our inner
        methods.

        Since the message loop thread can conceiveably block indefinitely while
        waiting for the server to respond, we only allow it (and the think loop
        for good measure) a short time to finish before simply giving up.

        Once an agent has been disconnected, it is 'dead' and cannot be used
        again.  All of its methods get replaced by a method that raises an
        exception every time it is called.
        """

        # don't do anything if not connected
        if not self.__connected:
            return

        # tell the message loop to terminate
        self.__parsing = False
        self.__thinking = False

        # tell the server that we're quitting
        self.__sock.send("(bye)")

        # tell our threads to join, but only wait breifly for them to do so.
        # don't join them if they haven't been started (this can happen if
        # disconnect is called very quickly after connect).
        if self.__msg_thread.is_alive():
            self.__msg_thread.join(0.1)

        if self.__think_thread.is_alive():
            self.__think_thread.join(0.1)

        # reset all standard variables in this object.  self.__connected gets
        # reset here, along with all other non-user defined internal variables.
        Agent.__init__(self)

    def __message_loop(self):
        """
        Handles messages received from the server.

        This SHOULD NOT be called externally, since it's used as a threaded loop
        internally by this object.  Calling it externally is a BAD THING!
        """

        # loop until we're told to stop
        while self.__parsing:
            # receive message data from the server and pass it along to the
            # world model as-is.  the world model parses it and stores it within
            # itself for perusal at our leisure.
            raw_msg = self.__sock.recv()
            self.msg_handler.handle_message(raw_msg)

    def __think_loop(self):
        """
        Performs world model analysis and sends appropriate commands to the
        server to allow the agent to participate in the current game.

        Like the message loop, this SHOULD NOT be called externally.  Use the
        play method to start play, and the disconnect method to end it.
        """

        # TODO: prevent the think method from being run too often, as it spams
        # the server with messages currently and slows things down ALOT.

        while self.__thinking:
            # performs the actions necessary for the agent to play soccer
            self.think()

            # this is necessary to allow our socket some time to recv messages,
            # as well as allowing us to send them.  this occurs since we use
            # threads, which in python share the same physical process.  if we
            # eat up all the time trying to send messages, we never recv any.
            time.sleep(0.0001)

    def setup_environment(self):
        """
        Called before the think loop starts, this allows the user to store any
        variables/objects they'll want access to across subsequent calls to the
        think method.
        """

        self.moved = False

    def think(self):
        """
        Performs a single step of thinking for our agent.  Gets called on every
        iteration of our think loop.
        """

        # DEBUG
        if not self.__think_thread.is_alive() or not self.__msg_thread.is_alive():
            raise Exception("A thread died.")

        # move to a random field position on first call
        if not self.moved:
            self.act_handler.move(1000, 1000)
            self.moved = True

        # perform random-play strategy
        if self.world.ball is not None and self.world.ball.direction is not None:
            # kick in a random direction if the ball is close enough
            if self.world.ball.distance <= 1:
                self.act_handler.say("K")
                self.act_handler.kick(100, 180 - random.randint(0, 361))
                return
            # dash towards the ball if it's within our field of view
            elif -5 < self.world.ball.direction < 5:
                self.act_handler.dash(100)
                return
            # turn to face the ball
            else:
                self.act_handler.turn(self.world.ball.direction / 2)
                return
        else:
            # search for the ball
            self.act_handler.turn(10 + random.randint(0, 10))
            return

if __name__ == "__main__":
    import sys

    # arg1: team name
    # arg2: number of players to start

    agentlist = []
    for agent in xrange(min(11, int(sys.argv[2]))):
        a = Agent()
        a.connect("localhost", 6000, sys.argv[1])
        a.play()

        agentlist.append(a)

    try:
        while 1:
            time.sleep(0.05)
    except KeyboardInterrupt:
        print "Disconnecting agents..."
        for agent in agentlist:
            agent.disconnect()
        print "Agents disconnected."

<commit_msg>added flag to tell think thread when to bother running (only when new data comes in).  this prevents it from running constantly while still allowing the message handling thread to process data while thinking.<commit_after>#!/usr/bin/env python

import threading
import time
import random

import sock
import model
import sp_exceptions
import handler

class Agent:
    def __init__(self):
        # whether we're connected to a server yet or not
        self.__connected = False

        # set all variables and important objects to appropriate values for
        # pre-connect state.
        self.__sock = None

        self.world = None
        self.body = None
        self.msg_handler = None
        self.act_handler = None

        self.__parsing = False
        self.__msg_thread = None

        self.__thinking = False
        self.__think_thread = None

        self.__should_think_on_data = False

    def connect(self, host, port, teamname, version=11):
        """
        Gives us a connection to the server as one player on a team.  This
        immediately connects the agent to the server and starts receiving and
        parsing the information it sends.
        """

        # if already connected, raise an error since user may have wanted to
        # connect again to a different server.
        if self.__connected:
            msg = "Cannot connect while already connected, disconnect first."
            raise sp_exceptions.AgentConnectionStateError(msg)

        # the pipe through which all of our communication takes place
        self.__sock = sock.Socket(host, port)

        # our models of the world and our body
        self.world = model.WorldModel()
        self.body = model.BodyModel()

        # handles all messages received from the server
        self.msg_handler = handler.MessageHandler(self.world, self.body)

        # handles the sending of actions to the server
        self.act_handler = handler.ActionHandler(self.__sock)

        # set up our threaded message receiving system
        self.__parsing = True # tell thread that we're currently running
        self.__msg_thread = threading.Thread(target=self.__message_loop,
                                           name="message_loop")
        self.__msg_thread.daemon = True # dies when parent thread dies

        # start processing received messages. this will catch the initial server
        # response and all subsequent communication.
        self.__msg_thread.start()

        # send the init message and allow the message handler to handle further
        # responses.
        init_address = self.__sock.address
        init_msg = "(init %s (version %d))"
        self.__sock.send(init_msg % (teamname, version))

        # wait until the socket receives a response from the server and gets its
        # assigned port.
        while self.__sock.address == init_address:
            time.sleep(0.0001)

        # create our thinking thread.  this will perform the actions necessary
        # to play a game of robo-soccer.
        self.__thinking = False
        self.__think_thread = threading.Thread(target=self.__think_loop,
                                             name="think_loop")
        self.__think_thread.daemon = True

        # set connected state.  done last to prevent state inconsistency if
        # something goes wrong beforehand.
        self.__connected = True

    def play(self):
        """
        Kicks off the thread that does the agent's thinking, allowing it to play
        during the game.  Throws an exception if called while the agent is
        already playing.
        """

        # ensure we're connected before doing anything
        if not self.__connected:
            msg = "Must be connected to a server to begin play."
            raise sp_exceptions.AgentConnectionStateError(msg)

        # throw exception if called while thread is already running
        if self.__thinking:
            raise sp_exceptions.AgentAlreadyPlayingError(
                "Agent is already playing.")

        # run the method that sets up the agent's persistant variables
        self.setup_environment()

        # tell the thread that it should be running, then start it
        self.__thinking = True
        self.__should_think_on_data = True
        self.__think_thread.start()

    def disconnect(self):
        """
        Tell the loop threads to stop and signal the server that we're
        disconnecting, then join the loop threads and destroy all our inner
        methods.

        Since the message loop thread can conceiveably block indefinitely while
        waiting for the server to respond, we only allow it (and the think loop
        for good measure) a short time to finish before simply giving up.

        Once an agent has been disconnected, it is 'dead' and cannot be used
        again.  All of its methods get replaced by a method that raises an
        exception every time it is called.
        """

        # don't do anything if not connected
        if not self.__connected:
            return

        # tell the loops to terminate
        self.__parsing = False
        self.__thinking = False

        # tell the server that we're quitting
        self.__sock.send("(bye)")

        # tell our threads to join, but only wait breifly for them to do so.
        # don't join them if they haven't been started (this can happen if
        # disconnect is called very quickly after connect).
        if self.__msg_thread.is_alive():
            self.__msg_thread.join(0.01)

        if self.__think_thread.is_alive():
            self.__think_thread.join(0.01)

        # reset all standard variables in this object.  self.__connected gets
        # reset here, along with all other non-user defined internal variables.
        Agent.__init__(self)

    def __message_loop(self):
        """
        Handles messages received from the server.

        This SHOULD NOT be called externally, since it's used as a threaded loop
        internally by this object.  Calling it externally is a BAD THING!
        """

        # loop until we're told to stop
        while self.__parsing:
            # receive message data from the server and pass it along to the
            # world model as-is.  the world model parses it and stores it within
            # itself for perusal at our leisure.
            raw_msg = self.__sock.recv()
            self.msg_handler.handle_message(raw_msg)

            # flag new data as needing the think loop's attention
            self.__should_think_on_data = True

    def __think_loop(self):
        """
        Performs world model analysis and sends appropriate commands to the
        server to allow the agent to participate in the current game.

        Like the message loop, this SHOULD NOT be called externally.  Use the
        play method to start play, and the disconnect method to end it.
        """

        while self.__thinking:
            # only think if new data has arrived
            if self.__should_think_on_data:
                # flag that data has been processed.  this shouldn't be a race
                # condition, since the only change would be to make it True
                # before changing it to False again, and we're already going to
                # process data, so it doesn't make any difference.
                self.__should_think_on_data = False

                # performs the actions necessary for the agent to play soccer
                self.think()
            else:
                # prevent from burning up all the cpu time while waiting for data
                time.sleep(0.0001)

    def setup_environment(self):
        """
        Called before the think loop starts, this allows the user to store any
        variables/objects they'll want access to across subsequent calls to the
        think method.
        """

        self.moved = False

    def think(self):
        """
        Performs a single step of thinking for our agent.  Gets called on every
        iteration of our think loop.
        """

        # DEBUG:  tells us if a thread dies
        if not self.__think_thread.is_alive() or not self.__msg_thread.is_alive():
            raise Exception("A thread died.")

        # move to a random field position on first call
        if not self.moved:
            self.act_handler.move(1000, 1000)
            self.moved = True

        # perform random-play strategy
        if self.world.ball is not None and self.world.ball.direction is not None:
            # kick in a random direction if the ball is close enough
            if self.world.ball.distance <= 1:
                self.act_handler.say("K")
                self.act_handler.kick(100, 180 - random.randint(0, 361))
                return
            # dash towards the ball if it's within our field of view
            elif -5 < self.world.ball.direction < 5:
                self.act_handler.dash(100)
                return
            # turn to face the ball
            else:
                self.act_handler.turn(self.world.ball.direction / 2)
                return
        else:
            # search for the ball
            self.act_handler.turn(10 + random.randint(0, 10))
            return

if __name__ == "__main__":
    import sys

    # arg1: team name
    # arg2: number of players to start

    agentlist = []
    for agent in xrange(min(11, int(sys.argv[2]))):
        a = Agent()
        a.connect("localhost", 6000, sys.argv[1])
        a.play()

        agentlist.append(a)

    try:
        while 1:
            time.sleep(0.05)
    except KeyboardInterrupt:
        print "Disconnecting agents..."
        for agent in agentlist:
            agent.disconnect()
        print "Agents disconnected."

<|endoftext|>


<commit_before># -*- coding: utf-8
#!/usr/bin/env python

import os
import sys
import re
import time
import getpass
import datetime
import platform
import base64, pickle

import json
import urllib
import httplib
import subprocess

AGENT_VERSION = "0.1"

SECRET = "" # to be injected on download by Cloudly
if(not SECRET): SECRET = raw_input("Enter your secret: ")

API_SERVER = "" # to be injected on download by Cloudly
if(not API_SERVER): API_SERVER = "127.0.0.1:5001"

if(sys.version_info.major==2 and sys.version_info.minor<7):
    print 'Please upgrade to the latest version of python.'
    sys.exit(0)

if(not getpass.getuser()=="root"):
    print "You must be root to run this script."
    sys.exit(0)

def _get_sys_loadavg():

    threshold_values = {
        "5-mins":None,
        "10-mins":None,
        "15-mins":None,
    }
    loadavg=subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    loadavg = re.findall(r"(\d+\.\d{2})", loadavg)

    return loadavg

def _get_sys_uptime():

    uptime = subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    uptime = re.findall("[ 0-9:]up[ ](.*)[,][ 0-9]+user", uptime)[0]

    return uptime

def _get_sys_cpu():

    cpu_info = subprocess.Popen(["ps","axo","pcpu"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

    c=0
    cpu_total = float(0)

    for line in cpu_info.split('\n'):

        line = line.replace(' ','')

        try:
            cpu_process_usage = float(line)
            cpu_total += cpu_process_usage
        except:
            pass

    cpu_usage = {
        'cpu_used':round(cpu_total,2),
        'cpu_free':round(float(100-cpu_total),2)
    }

    return cpu_usage

def _get_sys_cpu_info():

    cpu = 0
    cpu_info = {}

    for line in open('/proc/cpuinfo').readlines():

        element = re.findall(r"^(.+)\s+[:][ ](.*)$", line)

        if(element):

            key = element[0][0].replace('\t','').replace(' ','_')
            value = element[0][1]

            try:
                cpu_info['cpu'+str(cpu)][key] = value
            except:
                cpu_info['cpu'+str(cpu)] = {}
                cpu_info['cpu'+str(cpu)][key] = value

        if(len(line)==1): cpu += 1

    return cpu_info

def _get_memory_usage():

    memory_free = ""
    memory_total = ""

    memory_info = subprocess.Popen(["cat","/proc/meminfo"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

    for element in memory_info.split('\n'):
        if("MemTotal" in element): memory_total = element
        if("MemFree" in element): memory_free = element

    try:
        memory_total = long(memory_total.split(' ')[-2:-1][0]) * 1024
        memory_free = long(memory_free.split(' ')[-2:-1][0]) * 1024
        memory_used = long(memory_total-memory_free)
    except:
        memory_total = -1
        memory_free = -1
        memory_used = -1

    memory_usage = {
        'memory_total': memory_total,
        'memory_free': memory_free,
        'memory_used': memory_used,
        'memory_used_percentage': round(float(memory_used)/float(memory_total)*100,2),
        'swap_total': 0,
        'swap_used': 0,
        'swap_free': 0,
        'swap_used_percentage': 0,
    }

    mem_info = subprocess.Popen(['free',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

    for line in mem_info.split('\n'):

        if('swap' in line.lower()):
            mem_info = re.findall("(\d+)", line)
            memory_usage['swap_total'] = long(mem_info[0]) * 1024
            memory_usage['swap_used'] = long(mem_info[1]) * 1024
            memory_usage['swap_free'] = long(mem_info[2]) * 1024
            try:
                memory_usage['swap_used_percentage'] = round(float(mem_info[1])/float(mem_info[0])*100,2)
            except:
                memory_usage['swap_used_percentage'] = 0

    return memory_usage

def _get_ip_address():

    ifconfig = subprocess.Popen(["ifconfig","-a"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    ip = re.search( r'inet addr:[^\s]+', ifconfig )

    if(not ip):
        ip = re.search( r'inet [^\s]+', ifconfig )

    if not platform.system() == 'Darwin':
        ip = ip.group().split(':')[-1]
    else:
        ip = "127.0.0.1"

    ip = ip.replace(" ","")
    ip = ip.replace("inet","")

    return ip

def _get_sys_cpu_virtualization():

    virtualization_support = False

    cpuinfo = open('/proc/cpuinfo','rt').readlines()
    for line in cpuinfo:

        if("vmx" in line):
            virtualization_support = True
        if("svm" in line):
            virtualization_support = True

    return virtualization_support

def _get_disks_usage():

    proc = subprocess.Popen(['df', '-B 1'], stdout=subprocess.PIPE, close_fds=True)
    df = proc.communicate()[0]

    try:
        volumes = df.split('\n')
        volumes.pop(0)
        volumes.pop()
    except: return {"error":True,"error_description":"_get_disks_usage"}

    regexp = re.compile(r'([0-9]+)')
    previousVolume = None
    volumeCount = 0
    disks_usage = []

    for volume in volumes:

        volume = volume.split(None, 10)

        if len(volume) == 1:
            previousVolume = volume[0]
            continue

        if previousVolume != None:
            volume.insert(0, previousVolume)
            previousVolume = None

        volumeCount = volumeCount + 1

        if regexp.match(volume[1]) == None:
            pass

        else:
            try:
                volume[2] = int(volume[2]) # Used
                volume[3] = int(volume[3]) # Available
            except IndexError:
                pass
            except KeyError:
                pass

            disks_usage.append(volume)

    return disks_usage

def _get_networking_stats():

    proc = subprocess.Popen(['iptables','-L','-vxn'], stdout=subprocess.PIPE, close_fds=True)
    data = proc.communicate()[0]

    inbound_text = ""
    outbound_text = ""
    forward_text = ""

    psc = 0
    for line in data.split('\n'):

        if(psc==0):
            if(line): inbound_text += line+"\n"
        if(psc==1):
            if(line): forward_text += line+"\n"
        if(psc==2):
            if(line): outbound_text += line+"\n"

        if(len(line)==0): psc += 1

    inbound_traffic = {}

    for line in inbound_text.split('\n'):

        if("INPUT" in line):
            input_accept = line.split('policy ACCEPT ')[1]
            input_accept = input_accept.split(' ')
            input_accept_packets = input_accept[0]
            input_accept_bytes = input_accept[2]
            break

    inbound_traffic['input_accept_packets'] = input_accept_packets
    inbound_traffic['input_accept_bytes'] = input_accept_bytes

    input_accept_packets = 0
    input_accept_bytes = 0

    if(len(inbound_text.split('\n'))>3):

        c=0
        for line in inbound_text.split('\n'):

            if(c>3 and c<len(inbound_text.split('\n'))-1):
                input_accept_packets += int(line.split(' ')[0])
                input_accept_bytes += int(line.split(' ')[1])

            c+=1

    if(input_accept_packets>0): inbound_traffic['input_accept_packets'] = input_accept_packets
    if(input_accept_bytes>0): inbound_traffic['input_accept_bytes'] = input_accept_bytes

    outbound_traffic = {}

    for line in outbound_text.split('\n'):

        if("OUTPUT" in line):
            output_accept = line.split('policy ACCEPT ')[1]
            output_accept = output_accept.split(' ')
            output_accept_packets = output_accept[0]
            output_accept_bytes = output_accept[2]
            break

    outbound_traffic['output_accept_packets'] = output_accept_packets
    outbound_traffic['output_accept_bytes'] = output_accept_bytes

    output_accept_packets = 0
    output_accept_bytes = 0

    if(len(outbound_text.split('\n'))>3):

        c=0
        for line in outbound_text.split('\n'):

            if(c>3 and c<len(outbound_text.split('\n'))-1):
                output_accept_packets += int(line.split(' ')[0])
                output_accept_bytes += int(line.split(' ')[1])

            c+=1

    if(output_accept_packets>0): outbound_traffic['output_accept_packets'] = output_accept_packets
    if(output_accept_bytes>0): outbound_traffic['output_accept_bytes'] = output_accept_bytes

    networking = {'input_accept_packets':inbound_traffic['input_accept_packets'],'input_accept_bytes':inbound_traffic['input_accept_bytes'],'output_accept_packets':outbound_traffic['output_accept_packets'],'output_accept_bytes':outbound_traffic['output_accept_bytes'],}
    proc = subprocess.Popen(['iptables','-Z'], stdout=subprocess.PIPE, close_fds=True)

    return networking

def _get_distro():

    distro = "?"
    try:
        for i in platform.linux_distribution():
            distro += i.title() + " "
        distro = distro[:-1]

    except: pass

    return distro

def _get_hostname():

    hostname = subprocess.Popen(["hostname"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    hostname = hostname.replace("\n","")
    return hostname

def _get_processes():

    processes = subprocess.Popen(["ps","a","u","x"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    return processes

# XXX
# class nginx
# class mysql
# class mwah

def get_system_metrics( uuid, secret ):

    print datetime.datetime.now(), 'Collecting system metrics..'

    uuid = uuid
    ip = _get_ip_address()
    distro = _get_distro()
    loadavg = _get_sys_loadavg()
    uptime = _get_sys_uptime()
    cpu_usage = _get_sys_cpu()
    cpu_info = _get_sys_cpu_info()
    cpu_virtualization = _get_sys_cpu_virtualization()
    memory_usage = _get_memory_usage()
    disks_usage = _get_disks_usage()
    processes = _get_processes()
    hostname = _get_hostname()
    networking = _get_networking_stats()

    system_metrics_json = {
        'uuid': uuid,
        'ip': ip,
        'hostname': hostname,
        'distro': distro,
        'secret': secret,
        'loadavg': loadavg,
        'uptime': uptime,
        'cpu_usage': cpu_usage,
        'cpu_info': cpu_info,
        'cpu_virtualization': cpu_virtualization,
        'memory_usage': memory_usage,
        'disks_usage': disks_usage,
        'processes': processes,
        'networking': networking,
        'agent_version': AGENT_VERSION,
    }

    return system_metrics_json

def send_data( secret, api_call, data ):

    params = urllib.urlencode(data)
    headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
    api_url = "http://"+API_SERVER+api_call

    conn = httplib.HTTPConnection(API_SERVER)
    conn.request("POST", api_call, json.dumps(data), headers)
    response = conn.getresponse()
    response_data = response.read()
    conn.close()

    return response_data

def setup_system():

    installer = ""

    proc = subprocess.Popen(['which','yum'], stdout=subprocess.PIPE, close_fds=True)
    proc = proc.communicate()[0]
    if('yum' in proc): installer = proc

    if(not installer):

        proc = subprocess.Popen(['which','apt-get'], stdout=subprocess.PIPE, close_fds=True)
        proc = proc.communicate()[0]
        if('apt-get' in proc): installer = proc

        installer = installer.replace('\n','')

    if(not installer):

        proc = subprocess.Popen(['which','emerge'], stdout=subprocess.PIPE, close_fds=True)
        proc = proc.communicate()[0]
        if('emerge' in proc): installer = proc

        installer = installer.replace('\n','')

    if(not installer):

        proc = subprocess.Popen(['which','zypper'], stdout=subprocess.PIPE, close_fds=True)
        proc = proc.communicate()[0]
        if('zypper' in proc): installer = proc

        installer = installer.replace('\n','')

    # iptables..

    proc = subprocess.Popen(['which','iptables'], stdout=subprocess.PIPE, close_fds=True)
    proc = proc.communicate()[0]

    if(not 'iptables' in proc):

        print 'Installing iptables..'

        if(not installer):
            print 'Please install the iptables and re-run the agent.'
            sys.exit(0)

        if("emerge" in installer):
            os.system(installer+" iptables") # there is no install param in emerge
        else:
            os.system(installer+" install iptables")

    # old versions of python such as python 2.5.1 do not come with json nor they have support for one..

    # XXX improvise for the simplejson on the agent

def main():

    setup_system()
    os.system("clear")

    print "AGENT: v"+AGENT_VERSION
    print "Written By: Jan Paricka"

    HWADDR = subprocess.Popen(["ifconfig","-a"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    UUID = re.search(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', HWADDR, re.I).group()

    api_call = "/v10/activity/"
    activity = {
        'secret': SECRET,
        'agent_version': AGENT_VERSION,
        'uuid': UUID,
        'activity': "Agent v"+AGENT_VERSION+" started."
    }
    send_data(SECRET,api_call,activity)

    while True:

        api_call = "/v10/ping/"
        system_metrics = get_system_metrics(UUID, SECRET)
        api_response = send_data(SECRET,api_call,system_metrics)
        time.sleep(2)

    print "ze end."

if __name__=="__main__":

    main()
<commit_msg>conditional import json on the agent to work the old python issues<commit_after># -*- coding: utf-8
#!/usr/bin/env python

import os
import sys
import re
import time
import getpass
import datetime
import platform
import base64, pickle

import urllib
import httplib
import subprocess

try:
    import json
except: pass

AGENT_VERSION = "0.1"

SECRET = "" # to be injected on download by Cloudly
if(not SECRET): SECRET = raw_input("Enter your secret: ")

API_SERVER = "" # to be injected on download by Cloudly
if(not API_SERVER): API_SERVER = "127.0.0.1:5001"

if(sys.version_info.major==2 and sys.version_info.minor<7):
    print 'Please upgrade to the latest version of python.'
    sys.exit(0)

if(not getpass.getuser()=="root"):
    print "You must be root to run this script."
    sys.exit(0)

def _get_sys_loadavg():

    threshold_values = {
        "5-mins":None,
        "10-mins":None,
        "15-mins":None,
    }
    loadavg=subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    loadavg = re.findall(r"(\d+\.\d{2})", loadavg)

    return loadavg

def _get_sys_uptime():

    uptime = subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    uptime = re.findall("[ 0-9:]up[ ](.*)[,][ 0-9]+user", uptime)[0]

    return uptime

def _get_sys_cpu():

    cpu_info = subprocess.Popen(["ps","axo","pcpu"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

    c=0
    cpu_total = float(0)

    for line in cpu_info.split('\n'):

        line = line.replace(' ','')

        try:
            cpu_process_usage = float(line)
            cpu_total += cpu_process_usage
        except:
            pass

    cpu_usage = {
        'cpu_used':round(cpu_total,2),
        'cpu_free':round(float(100-cpu_total),2)
    }

    return cpu_usage

def _get_sys_cpu_info():

    cpu = 0
    cpu_info = {}

    for line in open('/proc/cpuinfo').readlines():

        element = re.findall(r"^(.+)\s+[:][ ](.*)$", line)

        if(element):

            key = element[0][0].replace('\t','').replace(' ','_')
            value = element[0][1]

            try:
                cpu_info['cpu'+str(cpu)][key] = value
            except:
                cpu_info['cpu'+str(cpu)] = {}
                cpu_info['cpu'+str(cpu)][key] = value

        if(len(line)==1): cpu += 1

    return cpu_info

def _get_memory_usage():

    memory_free = ""
    memory_total = ""

    memory_info = subprocess.Popen(["cat","/proc/meminfo"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

    for element in memory_info.split('\n'):
        if("MemTotal" in element): memory_total = element
        if("MemFree" in element): memory_free = element

    try:
        memory_total = long(memory_total.split(' ')[-2:-1][0]) * 1024
        memory_free = long(memory_free.split(' ')[-2:-1][0]) * 1024
        memory_used = long(memory_total-memory_free)
    except:
        memory_total = -1
        memory_free = -1
        memory_used = -1

    memory_usage = {
        'memory_total': memory_total,
        'memory_free': memory_free,
        'memory_used': memory_used,
        'memory_used_percentage': round(float(memory_used)/float(memory_total)*100,2),
        'swap_total': 0,
        'swap_used': 0,
        'swap_free': 0,
        'swap_used_percentage': 0,
    }

    mem_info = subprocess.Popen(['free',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

    for line in mem_info.split('\n'):

        if('swap' in line.lower()):
            mem_info = re.findall("(\d+)", line)
            memory_usage['swap_total'] = long(mem_info[0]) * 1024
            memory_usage['swap_used'] = long(mem_info[1]) * 1024
            memory_usage['swap_free'] = long(mem_info[2]) * 1024
            try:
                memory_usage['swap_used_percentage'] = round(float(mem_info[1])/float(mem_info[0])*100,2)
            except:
                memory_usage['swap_used_percentage'] = 0

    return memory_usage

def _get_ip_address():

    ifconfig = subprocess.Popen(["ifconfig","-a"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    ip = re.search( r'inet addr:[^\s]+', ifconfig )

    if(not ip):
        ip = re.search( r'inet [^\s]+', ifconfig )

    if not platform.system() == 'Darwin':
        ip = ip.group().split(':')[-1]
    else:
        ip = "127.0.0.1"

    ip = ip.replace(" ","")
    ip = ip.replace("inet","")

    return ip

def _get_sys_cpu_virtualization():

    virtualization_support = False

    cpuinfo = open('/proc/cpuinfo','rt').readlines()
    for line in cpuinfo:

        if("vmx" in line):
            virtualization_support = True
        if("svm" in line):
            virtualization_support = True

    return virtualization_support

def _get_disks_usage():

    proc = subprocess.Popen(['df', '-B 1'], stdout=subprocess.PIPE, close_fds=True)
    df = proc.communicate()[0]

    try:
        volumes = df.split('\n')
        volumes.pop(0)
        volumes.pop()
    except: return {"error":True,"error_description":"_get_disks_usage"}

    regexp = re.compile(r'([0-9]+)')
    previousVolume = None
    volumeCount = 0
    disks_usage = []

    for volume in volumes:

        volume = volume.split(None, 10)

        if len(volume) == 1:
            previousVolume = volume[0]
            continue

        if previousVolume != None:
            volume.insert(0, previousVolume)
            previousVolume = None

        volumeCount = volumeCount + 1

        if regexp.match(volume[1]) == None:
            pass

        else:
            try:
                volume[2] = int(volume[2]) # Used
                volume[3] = int(volume[3]) # Available
            except IndexError:
                pass
            except KeyError:
                pass

            disks_usage.append(volume)

    return disks_usage

def _get_networking_stats():

    proc = subprocess.Popen(['iptables','-L','-vxn'], stdout=subprocess.PIPE, close_fds=True)
    data = proc.communicate()[0]

    inbound_text = ""
    outbound_text = ""
    forward_text = ""

    psc = 0
    for line in data.split('\n'):

        if(psc==0):
            if(line): inbound_text += line+"\n"
        if(psc==1):
            if(line): forward_text += line+"\n"
        if(psc==2):
            if(line): outbound_text += line+"\n"

        if(len(line)==0): psc += 1

    inbound_traffic = {}

    for line in inbound_text.split('\n'):

        if("INPUT" in line):
            input_accept = line.split('policy ACCEPT ')[1]
            input_accept = input_accept.split(' ')
            input_accept_packets = input_accept[0]
            input_accept_bytes = input_accept[2]
            break

    inbound_traffic['input_accept_packets'] = input_accept_packets
    inbound_traffic['input_accept_bytes'] = input_accept_bytes

    input_accept_packets = 0
    input_accept_bytes = 0

    if(len(inbound_text.split('\n'))>3):

        c=0
        for line in inbound_text.split('\n'):

            if(c>3 and c<len(inbound_text.split('\n'))-1):
                input_accept_packets += int(line.split(' ')[0])
                input_accept_bytes += int(line.split(' ')[1])

            c+=1

    if(input_accept_packets>0): inbound_traffic['input_accept_packets'] = input_accept_packets
    if(input_accept_bytes>0): inbound_traffic['input_accept_bytes'] = input_accept_bytes

    outbound_traffic = {}

    for line in outbound_text.split('\n'):

        if("OUTPUT" in line):
            output_accept = line.split('policy ACCEPT ')[1]
            output_accept = output_accept.split(' ')
            output_accept_packets = output_accept[0]
            output_accept_bytes = output_accept[2]
            break

    outbound_traffic['output_accept_packets'] = output_accept_packets
    outbound_traffic['output_accept_bytes'] = output_accept_bytes

    output_accept_packets = 0
    output_accept_bytes = 0

    if(len(outbound_text.split('\n'))>3):

        c=0
        for line in outbound_text.split('\n'):

            if(c>3 and c<len(outbound_text.split('\n'))-1):
                output_accept_packets += int(line.split(' ')[0])
                output_accept_bytes += int(line.split(' ')[1])

            c+=1

    if(output_accept_packets>0): outbound_traffic['output_accept_packets'] = output_accept_packets
    if(output_accept_bytes>0): outbound_traffic['output_accept_bytes'] = output_accept_bytes

    networking = {'input_accept_packets':inbound_traffic['input_accept_packets'],'input_accept_bytes':inbound_traffic['input_accept_bytes'],'output_accept_packets':outbound_traffic['output_accept_packets'],'output_accept_bytes':outbound_traffic['output_accept_bytes'],}
    proc = subprocess.Popen(['iptables','-Z'], stdout=subprocess.PIPE, close_fds=True)

    return networking

def _get_distro():

    distro = "?"
    try:
        for i in platform.linux_distribution():
            distro += i.title() + " "
        distro = distro[:-1]

    except: pass

    return distro

def _get_hostname():

    hostname = subprocess.Popen(["hostname"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    hostname = hostname.replace("\n","")
    return hostname

def _get_processes():

    processes = subprocess.Popen(["ps","a","u","x"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    return processes

# XXX
# class nginx
# class mysql
# class mwah

def get_system_metrics( uuid, secret ):

    print datetime.datetime.now(), 'Collecting system metrics..'

    uuid = uuid
    ip = _get_ip_address()
    distro = _get_distro()
    loadavg = _get_sys_loadavg()
    uptime = _get_sys_uptime()
    cpu_usage = _get_sys_cpu()
    cpu_info = _get_sys_cpu_info()
    cpu_virtualization = _get_sys_cpu_virtualization()
    memory_usage = _get_memory_usage()
    disks_usage = _get_disks_usage()
    processes = _get_processes()
    hostname = _get_hostname()
    networking = _get_networking_stats()

    system_metrics_json = {
        'uuid': uuid,
        'ip': ip,
        'hostname': hostname,
        'distro': distro,
        'secret': secret,
        'loadavg': loadavg,
        'uptime': uptime,
        'cpu_usage': cpu_usage,
        'cpu_info': cpu_info,
        'cpu_virtualization': cpu_virtualization,
        'memory_usage': memory_usage,
        'disks_usage': disks_usage,
        'processes': processes,
        'networking': networking,
        'agent_version': AGENT_VERSION,
    }

    return system_metrics_json

def send_data( secret, api_call, data ):

    params = urllib.urlencode(data)
    headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
    api_url = "http://"+API_SERVER+api_call

    conn = httplib.HTTPConnection(API_SERVER)
    conn.request("POST", api_call, json.dumps(data), headers)
    response = conn.getresponse()
    response_data = response.read()
    conn.close()

    return response_data

def setup_system():

    installer = ""

    proc = subprocess.Popen(['which','yum'], stdout=subprocess.PIPE, close_fds=True)
    proc = proc.communicate()[0]
    if('yum' in proc): installer = proc

    if(not installer):

        proc = subprocess.Popen(['which','apt-get'], stdout=subprocess.PIPE, close_fds=True)
        proc = proc.communicate()[0]
        if('apt-get' in proc): installer = proc

        installer = installer.replace('\n','')

    if(not installer):

        proc = subprocess.Popen(['which','emerge'], stdout=subprocess.PIPE, close_fds=True)
        proc = proc.communicate()[0]
        if('emerge' in proc): installer = proc

        installer = installer.replace('\n','')

    if(not installer):

        proc = subprocess.Popen(['which','zypper'], stdout=subprocess.PIPE, close_fds=True)
        proc = proc.communicate()[0]
        if('zypper' in proc): installer = proc

        installer = installer.replace('\n','')

    # iptables..

    proc = subprocess.Popen(['which','iptables'], stdout=subprocess.PIPE, close_fds=True)
    proc = proc.communicate()[0]

    if(not 'iptables' in proc):

        print 'Installing iptables..'

        if(not installer):
            print 'Please install the iptables and re-run the agent.'
            sys.exit(0)

        if("emerge" in installer):
            os.system(installer+" iptables") # there is no install param in emerge
        else:
            os.system(installer+" install iptables")

    # old versions of python such as python 2.5.1 do not come with json nor they have support for one..

    try:
        import json
    except: pass

def main():

    setup_system()
    os.system("clear")

    print "AGENT: v"+AGENT_VERSION
    print "Written By: Jan Paricka"

    HWADDR = subprocess.Popen(["ifconfig","-a"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    UUID = re.search(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', HWADDR, re.I).group()

    api_call = "/v10/activity/"
    activity = {
        'secret': SECRET,
        'agent_version': AGENT_VERSION,
        'uuid': UUID,
        'activity': "Agent v"+AGENT_VERSION+" started."
    }
    send_data(SECRET,api_call,activity)

    while True:

        api_call = "/v10/ping/"
        system_metrics = get_system_metrics(UUID, SECRET)
        api_response = send_data(SECRET,api_call,system_metrics)
        time.sleep(2)

    print "ze end."

if __name__=="__main__":

    main()
<|endoftext|>


<commit_before># -*- coding: utf-8
#!/usr/bin/env python

import os
import sys
import re
import time
import datetime
import platform
import base64, pickle

import json
import urllib
import httplib
import subprocess

AGENT_VERSION = "0.1"

SECRET = "" # to be injected on download by Cloudly
if(not SECRET): SECRET = raw_input("Enter your secret: ")

API_SERVER = "" # to be injected on download by Cloudly
if(not API_SERVER): API_SERVER = "127.0.0.1:5000"

HWADDR = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
UUID = re.search(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', HWADDR, re.I).group()

# XXX hostname
# XXX distro

def _get_sys_loadavg():

	loadavg=subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	loadavg = re.findall(r"(\d+\.\d{2})", loadavg)

	return loadavg

def _get_sys_uptime():

	uptime = subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	uptime = re.findall("[ 0-9:]up[ ](.*)[,][ 0-9]+user", uptime)[0]

	return uptime

def _get_sys_cpu():

	cpu_info = subprocess.Popen(["ps","e","o","pcpu"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	c=0
	cpu_total = float(0)

	for line in cpu_info.split('\n'):

		line = line.replace(' ','')

		try:
			cpu_process_usage = float(line)
			cpu_total += cpu_process_usage
		except:
			pass

	cpu_usage = {
		'cpu_used':round(cpu_total,2),
		'cpu_free':round(float(100-cpu_total),2)
	}

	return cpu_usage

def _get_sys_cpu_info():

	cpu = 0
	cpu_info = {}

	for line in open('/proc/cpuinfo').readlines():

		element = re.findall(r"^(.+)\s+[:][ ](.*)$", line)

		if(element):

			key = element[0][0].replace('\t','').replace(' ','_')
			value = element[0][1]

			try:
				cpu_info['cpu'+str(cpu)][key] = value
			except:
				cpu_info['cpu'+str(cpu)] = {}
				cpu_info['cpu'+str(cpu)][key] = value

		if(len(line)==1): cpu += 1

	return cpu_info

def _get_memory_usage():

	memory_info = subprocess.Popen(["cat","/proc/meminfo"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	memory_total = ""
	memory_free = ""
	for element in memory_info.split('\n'):
		if("MemTotal" in element): memory_total = element
		if("MemFree" in element): memory_free = element

	try:
		memory_total = long(memory_total.split(' ')[-2:-1][0]) * 1024
		memory_free = long(memory_free.split(' ')[-2:-1][0]) * 1024
		memory_used = long(memory_total-memory_free)
	except:
		memory_total = -1
		memory_free = -1
		memory_used = -1

	memory_usage = {
		'memory_total': memory_total,
		'memory_free': memory_free,
		'memory_used': memory_used,
		'memory_used_percentage': round(float(memory_used)/float(memory_total)*100,2),
	}

	return memory_usage

def _get_ip_address():

	ifconfig = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	ip = re.search( r'inet addr:[^\s]+', ifconfig )
	if(not ip):
		ip = re.search( r'inet [^\s]+', ifconfig )
	if not platform.system() == 'Darwin':
		ip = ip.group().split(':')[-1]
	else:
		ip = "127.0.0.1"
	ip = ip.replace(" ","")
	ip = ip.replace("inet","")
	return ip

def _get_sys_cpu_virtualization():

	virtualization_support = False

	cpuinfo = open('/proc/cpuinfo','rt').readlines()
	for line in cpuinfo:

		if("vmx" in line):
			virtualization_support = True
		if("svm" in line):
			virtualization_support = True

	return virtualization_support

def _get_physical_disks():

	physical_disks = []

	physical_disks_list = subprocess.Popen(["fdisk","-l"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	physical_disks_list_ = physical_disks_list.split('\n')

	physical_disks_list = []
	for line in physical_disks_list_:

		if('Disk' in line):
			disk = line.split(' ')[1].replace(':','')
			try:
				bsize = long(line.split(' ')[-2:-1][0])
				physical_disks_list.append([disk,bsize])
			except: pass

	for hdd in physical_disks_list:

		FNULL = open(os.devnull, 'w')
		disk_info = subprocess.Popen(["hdparm","-I",hdd[0],], stderr=FNULL, stdout=subprocess.PIPE, close_fds=True).communicate()[0]

		model_number = "n/a"
		serial_number = "n/a"
		firmware_revision = "n/a"

		for line in disk_info.split('\n'):

			if("Model Number" in line): model_number = line.split(':')[1].replace(' ','')
			if("Serial Number" in line): serial_number = line.split(':')[1].replace(' ','')
			if("Firmware Revision" in line): firmware_revision = line.split(':')[1].replace(' ','')

		try: model_number.decode('utf-8')
		except UnicodeDecodeError: model_number = "n/a"

		try: serial_number.decode('utf-8')
		except UnicodeDecodeError: serial_number = "n/a"

		try: firmware_revision.decode('utf-8')
		except UnicodeDecodeError: firmware_revision = "n/a"

		disk_info = {
			'device':hdd[0],
			'size':hdd[1],
			'model_number':model_number,
			'serial_number':serial_number,
			'firmware_revision':firmware_revision,
		}
		physical_disks.append(disk_info)

	return physical_disks

def _get_disks_usage():

	proc = subprocess.Popen(['df', '-B 1'], stdout=subprocess.PIPE, close_fds=True)
	df = proc.communicate()[0]

	try:
		volumes = df.split('\n')
		volumes.pop(0)
		volumes.pop()
	except: return {"error":True,"error_description":"_get_disks_usage"}

	regexp = re.compile(r'([0-9]+)')
	previousVolume = None
	volumeCount = 0
	disks_usage = []

	for volume in volumes:

		volume = volume.split(None, 10)

		if len(volume) == 1:
			previousVolume = volume[0]
			continue

		if previousVolume != None:
			volume.insert(0, previousVolume)
			previousVolume = None

		volumeCount = volumeCount + 1

		if regexp.match(volume[1]) == None:
			pass

		else:
			try:
				volume[2] = int(volume[2]) / 1024 / 1000 # Used
				volume[3] = int(volume[3]) / 1024 / 1000 # Available
			except IndexError:
				pass
			except KeyError:
				pass

			disks_usage.append(volume)

	return disks_usage

def _get_hostname():
    return "XXX"

def _get_distro():

    distro = ""
    for i in platform.linux_distribution():
        distro += i.title() + " "

    distro = distro[:-1]

    return distro

def get_system_metrics( secret ):

	print datetime.datetime.now(), SECRET, 'Collecting system metrics..'

	uuid = UUID
	ip = _get_ip_address()
	loadavg = _get_sys_loadavg()
	uptime = _get_sys_uptime()
	cpu_usage = _get_sys_cpu()
	cpu_info = _get_sys_cpu_info()
	cpu_virtualization = _get_sys_cpu_virtualization()
	memory_usage = _get_memory_usage()
	physical_disks = _get_physical_disks()
	disks_usage = _get_disks_usage()
	hostname = _get_hostname()
	distro = _get_distro()

	system_metrics_json = {
		'uuid': uuid,
		'ip': ip,
		'hostname': hostname,
		'distro': distro,
		'secret': secret,
		'loadavg': loadavg,
		'uptime': uptime,
		'cpu_usage': cpu_usage,
		'cpu_info': cpu_info,
		'cpu_virtualization': cpu_virtualization,
		'memory_usage': memory_usage,
		'physical_disks': physical_disks,
		'disks_usage': disks_usage,
		'agent_version': AGENT_VERSION,
	}

	return system_metrics_json

def send_data( secret, api_call, data ):

	params = urllib.urlencode(data)
	headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        api_url = "http://"+API_SERVER+api_call

	conn = httplib.HTTPConnection(API_SERVER)
	conn.request("POST", api_call, json.dumps(data), headers)
	response = conn.getresponse()
	response_data = response.read()
	conn.close()

	return response_data

if __name__=="__main__":

	os.system("clear")

	print "AGENT: v"+AGENT_VERSION
	print "Written By: Jan Paricka"

	api_call = "/v10/activity/"
	activity = {
		'secret': SECRET,
		'agent_version': AGENT_VERSION,
		'uuid': UUID,
		'activity': "Agent v"+AGENT_VERSION+" has started."
	}
	send_data(SECRET,api_call,activity)

	while True:

		api_call = "/v10/ping/"
		system_metrics = get_system_metrics(SECRET)
		api_response = send_data(SECRET,api_call,system_metrics)
		time.sleep(3)

	print "ze end."
<commit_msg>Made hostname to work on the agent.<commit_after># -*- coding: utf-8
#!/usr/bin/env python

import os
import sys
import re
import time
import datetime
import platform
import base64, pickle

import json
import urllib
import httplib
import subprocess

AGENT_VERSION = "0.1"

SECRET = "" # to be injected on download by Cloudly
if(not SECRET): SECRET = raw_input("Enter your secret: ")

API_SERVER = "" # to be injected on download by Cloudly
if(not API_SERVER): API_SERVER = "127.0.0.1:5000"

HWADDR = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
UUID = re.search(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', HWADDR, re.I).group()

# XXX hostname
# XXX distro

def _get_sys_loadavg():

	loadavg=subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	loadavg = re.findall(r"(\d+\.\d{2})", loadavg)

	return loadavg

def _get_sys_uptime():

	uptime = subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	uptime = re.findall("[ 0-9:]up[ ](.*)[,][ 0-9]+user", uptime)[0]

	return uptime

def _get_sys_cpu():

	cpu_info = subprocess.Popen(["ps","e","o","pcpu"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	c=0
	cpu_total = float(0)

	for line in cpu_info.split('\n'):

		line = line.replace(' ','')

		try:
			cpu_process_usage = float(line)
			cpu_total += cpu_process_usage
		except:
			pass

	cpu_usage = {
		'cpu_used':round(cpu_total,2),
		'cpu_free':round(float(100-cpu_total),2)
	}

	return cpu_usage

def _get_sys_cpu_info():

	cpu = 0
	cpu_info = {}

	for line in open('/proc/cpuinfo').readlines():

		element = re.findall(r"^(.+)\s+[:][ ](.*)$", line)

		if(element):

			key = element[0][0].replace('\t','').replace(' ','_')
			value = element[0][1]

			try:
				cpu_info['cpu'+str(cpu)][key] = value
			except:
				cpu_info['cpu'+str(cpu)] = {}
				cpu_info['cpu'+str(cpu)][key] = value

		if(len(line)==1): cpu += 1

	return cpu_info

def _get_memory_usage():

	memory_info = subprocess.Popen(["cat","/proc/meminfo"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	memory_total = ""
	memory_free = ""
	for element in memory_info.split('\n'):
		if("MemTotal" in element): memory_total = element
		if("MemFree" in element): memory_free = element

	try:
		memory_total = long(memory_total.split(' ')[-2:-1][0]) * 1024
		memory_free = long(memory_free.split(' ')[-2:-1][0]) * 1024
		memory_used = long(memory_total-memory_free)
	except:
		memory_total = -1
		memory_free = -1
		memory_used = -1

	memory_usage = {
		'memory_total': memory_total,
		'memory_free': memory_free,
		'memory_used': memory_used,
		'memory_used_percentage': round(float(memory_used)/float(memory_total)*100,2),
	}

	return memory_usage

def _get_ip_address():

	ifconfig = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	ip = re.search( r'inet addr:[^\s]+', ifconfig )
	if(not ip):
		ip = re.search( r'inet [^\s]+', ifconfig )
	if not platform.system() == 'Darwin':
		ip = ip.group().split(':')[-1]
	else:
		ip = "127.0.0.1"
	ip = ip.replace(" ","")
	ip = ip.replace("inet","")
	return ip

def _get_sys_cpu_virtualization():

	virtualization_support = False

	cpuinfo = open('/proc/cpuinfo','rt').readlines()
	for line in cpuinfo:

		if("vmx" in line):
			virtualization_support = True
		if("svm" in line):
			virtualization_support = True

	return virtualization_support

def _get_physical_disks():

	physical_disks = []

	physical_disks_list = subprocess.Popen(["fdisk","-l"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	physical_disks_list_ = physical_disks_list.split('\n')

	physical_disks_list = []
	for line in physical_disks_list_:

		if('Disk' in line):
			disk = line.split(' ')[1].replace(':','')
			try:
				bsize = long(line.split(' ')[-2:-1][0])
				physical_disks_list.append([disk,bsize])
			except: pass

	for hdd in physical_disks_list:

		FNULL = open(os.devnull, 'w')
		disk_info = subprocess.Popen(["hdparm","-I",hdd[0],], stderr=FNULL, stdout=subprocess.PIPE, close_fds=True).communicate()[0]

		model_number = "n/a"
		serial_number = "n/a"
		firmware_revision = "n/a"

		for line in disk_info.split('\n'):

			if("Model Number" in line): model_number = line.split(':')[1].replace(' ','')
			if("Serial Number" in line): serial_number = line.split(':')[1].replace(' ','')
			if("Firmware Revision" in line): firmware_revision = line.split(':')[1].replace(' ','')

		try: model_number.decode('utf-8')
		except UnicodeDecodeError: model_number = "n/a"

		try: serial_number.decode('utf-8')
		except UnicodeDecodeError: serial_number = "n/a"

		try: firmware_revision.decode('utf-8')
		except UnicodeDecodeError: firmware_revision = "n/a"

		disk_info = {
			'device':hdd[0],
			'size':hdd[1],
			'model_number':model_number,
			'serial_number':serial_number,
			'firmware_revision':firmware_revision,
		}
		physical_disks.append(disk_info)

	return physical_disks

def _get_disks_usage():

	proc = subprocess.Popen(['df', '-B 1'], stdout=subprocess.PIPE, close_fds=True)
	df = proc.communicate()[0]

	try:
		volumes = df.split('\n')
		volumes.pop(0)
		volumes.pop()
	except: return {"error":True,"error_description":"_get_disks_usage"}

	regexp = re.compile(r'([0-9]+)')
	previousVolume = None
	volumeCount = 0
	disks_usage = []

	for volume in volumes:

		volume = volume.split(None, 10)

		if len(volume) == 1:
			previousVolume = volume[0]
			continue

		if previousVolume != None:
			volume.insert(0, previousVolume)
			previousVolume = None

		volumeCount = volumeCount + 1

		if regexp.match(volume[1]) == None:
			pass

		else:
			try:
				volume[2] = int(volume[2]) / 1024 / 1000 # Used
				volume[3] = int(volume[3]) / 1024 / 1000 # Available
			except IndexError:
				pass
			except KeyError:
				pass

			disks_usage.append(volume)

	return disks_usage

def _get_hostname():

    hostname = subprocess.Popen(["hostname"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    hostname = hostname.replace("\n","")

    return hostname

def _get_distro():

    distro = ""
    for i in platform.linux_distribution():
        distro += i.title() + " "

    distro = distro[:-1]

    return distro

def get_system_metrics( secret ):

	print datetime.datetime.now(), SECRET, 'Collecting system metrics..'

	uuid = UUID
	ip = _get_ip_address()
	loadavg = _get_sys_loadavg()
	uptime = _get_sys_uptime()
	cpu_usage = _get_sys_cpu()
	cpu_info = _get_sys_cpu_info()
	cpu_virtualization = _get_sys_cpu_virtualization()
	memory_usage = _get_memory_usage()
	physical_disks = _get_physical_disks()
	disks_usage = _get_disks_usage()
	hostname = _get_hostname()
	distro = _get_distro()

	system_metrics_json = {
		'uuid': uuid,
		'ip': ip,
		'hostname': hostname,
		'distro': distro,
		'secret': secret,
		'loadavg': loadavg,
		'uptime': uptime,
		'cpu_usage': cpu_usage,
		'cpu_info': cpu_info,
		'cpu_virtualization': cpu_virtualization,
		'memory_usage': memory_usage,
		'physical_disks': physical_disks,
		'disks_usage': disks_usage,
		'agent_version': AGENT_VERSION,
	}

	return system_metrics_json

def send_data( secret, api_call, data ):

	params = urllib.urlencode(data)
	headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        api_url = "http://"+API_SERVER+api_call

	conn = httplib.HTTPConnection(API_SERVER)
	conn.request("POST", api_call, json.dumps(data), headers)
	response = conn.getresponse()
	response_data = response.read()
	conn.close()

	return response_data

if __name__=="__main__":

	os.system("clear")

	print "AGENT: v"+AGENT_VERSION
	print "Written By: Jan Paricka"

	api_call = "/v10/activity/"
	activity = {
		'secret': SECRET,
		'agent_version': AGENT_VERSION,
		'uuid': UUID,
		'activity': "Agent v"+AGENT_VERSION+" has started."
	}
	send_data(SECRET,api_call,activity)

	while True:

		api_call = "/v10/ping/"
		system_metrics = get_system_metrics(SECRET)
		api_response = send_data(SECRET,api_call,system_metrics)
		time.sleep(3)

	print "ze end."
<|endoftext|>


<commit_before># -*- coding: utf-8
#!/usr/bin/env python

import os
import sys
import re
import time
import datetime
import platform
import base64, pickle

import json
import urllib
import httplib
import subprocess

AGENT_VERSION = "0.1"

SECRET = "" # to be injected on download by Cloudly
if(not SECRET): SECRET = raw_input("Enter your secret: ")

API_SERVER = "" # to be injected on download by Cloudly
if(not API_SERVER): API_SERVER = "127.0.0.1:5000"

HWADDR = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
UUID = re.search(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', HWADDR, re.I).group()

# XXX hostname
# XXX distro

def _get_sys_loadavg():

	loadavg=subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	loadavg = re.findall(r"(\d+\.\d{2})", loadavg)

	return loadavg

def _get_sys_uptime():

	uptime = subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	uptime = re.findall("[ 0-9:]up[ ](.*)[,][ 0-9]+user", uptime)[0]

	return uptime

def _get_sys_cpu():

	cpu_info = subprocess.Popen(["ps","e","o","pcpu"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	c=0
	cpu_total = float(0)

	for line in cpu_info.split('\n'):

		line = line.replace(' ','')

		try:
			cpu_process_usage = float(line)
			cpu_total += cpu_process_usage
		except:
			pass

	cpu_usage = {
		'cpu_used':round(cpu_total,2),
		'cpu_free':round(float(100-cpu_total),2)
	}

	return cpu_usage

def _get_sys_cpu_info():

	cpu = 0
	cpu_info = {}

	for line in open('/proc/cpuinfo').readlines():

		element = re.findall(r"^(.+)\s+[:][ ](.*)$", line)

		if(element):

			key = element[0][0].replace('\t','').replace(' ','_')
			value = element[0][1]

			try:
				cpu_info['cpu'+str(cpu)][key] = value
			except:
				cpu_info['cpu'+str(cpu)] = {}
				cpu_info['cpu'+str(cpu)][key] = value

		if(len(line)==1): cpu += 1

	return cpu_info

def _get_memory_usage():

	memory_info = subprocess.Popen(["cat","/proc/meminfo"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	memory_total = ""
	memory_free = ""
	for element in memory_info.split('\n'):
		if("MemTotal" in element): memory_total = element
		if("MemFree" in element): memory_free = element

	try:
		memory_total = long(memory_total.split(' ')[-2:-1][0]) * 1024
		memory_free = long(memory_free.split(' ')[-2:-1][0]) * 1024
		memory_used = long(memory_total-memory_free)
	except:
		memory_total = -1
		memory_free = -1
		memory_used = -1

	memory_usage = {
		'memory_total': memory_total,
		'memory_free': memory_free,
		'memory_used': memory_used,
		'memory_used_percentage': round(float(memory_used)/float(memory_total)*100,2),
	}

	return memory_usage

def _get_ip_address():

	ifconfig = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	ip = re.search( r'inet addr:[^\s]+', ifconfig )
	if(not ip):
		ip = re.search( r'inet [^\s]+', ifconfig )
	if not platform.system() == 'Darwin':
		ip = ip.group().split(':')[-1]
	else:
		ip = "127.0.0.1"
	ip = ip.replace(" ","")
	ip = ip.replace("inet","")
	return ip

def _get_sys_cpu_virtualization():

	virtualization_support = False

	cpuinfo = open('/proc/cpuinfo','rt').readlines()
	for line in cpuinfo:

		if("vmx" in line):
			virtualization_support = True
		if("svm" in line):
			virtualization_support = True

	return virtualization_support

def _get_physical_disks():

	physical_disks = []

	physical_disks_list = subprocess.Popen(["fdisk","-l"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	physical_disks_list_ = physical_disks_list.split('\n')

	physical_disks_list = []
	for line in physical_disks_list_:

		if('Disk' in line):
			disk = line.split(' ')[1].replace(':','')
			try:
				bsize = long(line.split(' ')[-2:-1][0])
				physical_disks_list.append([disk,bsize])
			except: pass

	for hdd in physical_disks_list:

		FNULL = open(os.devnull, 'w')
		disk_info = subprocess.Popen(["hdparm","-I",hdd[0],], stderr=FNULL, stdout=subprocess.PIPE, close_fds=True).communicate()[0]

		model_number = "n/a"
		serial_number = "n/a"
		firmware_revision = "n/a"

		for line in disk_info.split('\n'):

			if("Model Number" in line): model_number = line.split(':')[1].replace(' ','')
			if("Serial Number" in line): serial_number = line.split(':')[1].replace(' ','')
			if("Firmware Revision" in line): firmware_revision = line.split(':')[1].replace(' ','')

		try: model_number.decode('utf-8')
		except UnicodeDecodeError: model_number = "n/a"

		try: serial_number.decode('utf-8')
		except UnicodeDecodeError: serial_number = "n/a"

		try: firmware_revision.decode('utf-8')
		except UnicodeDecodeError: firmware_revision = "n/a"

		disk_info = {
			'device':hdd[0],
			'size':hdd[1],
			'model_number':model_number,
			'serial_number':serial_number,
			'firmware_revision':firmware_revision,
		}
		physical_disks.append(disk_info)

	return physical_disks

def _get_disks_usage():

	proc = subprocess.Popen(['df', '-B 1'], stdout=subprocess.PIPE, close_fds=True)
	df = proc.communicate()[0]

	try:
		volumes = df.split('\n')
		volumes.pop(0)
		volumes.pop()
	except: return {"error":True,"error_description":"_get_disks_usage"}

	regexp = re.compile(r'([0-9]+)')
	previousVolume = None
	volumeCount = 0
	disks_usage = []

	for volume in volumes:

		volume = volume.split(None, 10)

		if len(volume) == 1:
			previousVolume = volume[0]
			continue

		if previousVolume != None:
			volume.insert(0, previousVolume)
			previousVolume = None

		volumeCount = volumeCount + 1

		if regexp.match(volume[1]) == None:
			pass

		else:
			try:
				volume[2] = int(volume[2]) / 1024 / 1000 # Used
				volume[3] = int(volume[3]) / 1024 / 1000 # Available
			except IndexError:
				pass
			except KeyError:
				pass

			disks_usage.append(volume)

	return disks_usage

def _get_hostname():

    hostname = subprocess.Popen(["hostname"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    hostname = hostname.replace("\n","")

    return hostname

def _get_distro():

    distro = ""
    for i in platform.linux_distribution():
        distro += i.title() + " "

    distro = distro[:-1]

    return distro

def get_system_metrics( secret ):

	print datetime.datetime.now(), SECRET, 'Collecting system metrics..'

	uuid = UUID
	ip = _get_ip_address()
	loadavg = _get_sys_loadavg()
	uptime = _get_sys_uptime()
	cpu_usage = _get_sys_cpu()
	cpu_info = _get_sys_cpu_info()
	cpu_virtualization = _get_sys_cpu_virtualization()
	memory_usage = _get_memory_usage()
	physical_disks = _get_physical_disks()
	disks_usage = _get_disks_usage()
	hostname = _get_hostname()
	distro = _get_distro()

	system_metrics_json = {
		'uuid': uuid,
		'ip': ip,
		'hostname': hostname,
		'distro': distro,
		'secret': secret,
		'loadavg': loadavg,
		'uptime': uptime,
		'cpu_usage': cpu_usage,
		'cpu_info': cpu_info,
		'cpu_virtualization': cpu_virtualization,
		'memory_usage': memory_usage,
		'physical_disks': physical_disks,
		'disks_usage': disks_usage,
		'agent_version': AGENT_VERSION,
	}

	return system_metrics_json

def send_data( secret, api_call, data ):

	params = urllib.urlencode(data)
	headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        api_url = "http://"+API_SERVER+api_call

	conn = httplib.HTTPConnection(API_SERVER)
	conn.request("POST", api_call, json.dumps(data), headers)
	response = conn.getresponse()
	response_data = response.read()
	conn.close()

	return response_data

if __name__=="__main__":

	os.system("clear")

	print "AGENT: v"+AGENT_VERSION
	print "Written By: Jan Paricka"

	# Filing an activity: example
	#api_call = "/v10/activity/"
	#activity = {
	#	'secret': SECRET,
	#	'agent_version': AGENT_VERSION,
	#	'uuid': UUID,
	#	'activity': "Agent v"+AGENT_VERSION+" has started."
	#}
	#send_data(SECRET,api_call,activity)

	while True:

		api_call = "/v10/ping/"
		system_metrics = get_system_metrics(SECRET)
		api_response = send_data(SECRET,api_call,system_metrics)
		time.sleep(3)

	print "ze end."
<commit_msg>cleaned up old todos from the agent since these are already done<commit_after># -*- coding: utf-8
#!/usr/bin/env python

import os
import sys
import re
import time
import datetime
import platform
import base64, pickle

import json
import urllib
import httplib
import subprocess

AGENT_VERSION = "0.1"

SECRET = "" # to be injected on download by Cloudly
if(not SECRET): SECRET = raw_input("Enter your secret: ")

API_SERVER = "" # to be injected on download by Cloudly
if(not API_SERVER): API_SERVER = "127.0.0.1:5000"

HWADDR = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
UUID = re.search(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', HWADDR, re.I).group()

def _get_sys_loadavg():

	loadavg=subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	loadavg = re.findall(r"(\d+\.\d{2})", loadavg)

	return loadavg

def _get_sys_uptime():

	uptime = subprocess.Popen(['uptime',], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	uptime = re.findall("[ 0-9:]up[ ](.*)[,][ 0-9]+user", uptime)[0]

	return uptime

def _get_sys_cpu():

	cpu_info = subprocess.Popen(["ps","e","o","pcpu"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	c=0
	cpu_total = float(0)

	for line in cpu_info.split('\n'):

		line = line.replace(' ','')

		try:
			cpu_process_usage = float(line)
			cpu_total += cpu_process_usage
		except:
			pass

	cpu_usage = {
		'cpu_used':round(cpu_total,2),
		'cpu_free':round(float(100-cpu_total),2)
	}

	return cpu_usage

def _get_sys_cpu_info():

	cpu = 0
	cpu_info = {}

	for line in open('/proc/cpuinfo').readlines():

		element = re.findall(r"^(.+)\s+[:][ ](.*)$", line)

		if(element):

			key = element[0][0].replace('\t','').replace(' ','_')
			value = element[0][1]

			try:
				cpu_info['cpu'+str(cpu)][key] = value
			except:
				cpu_info['cpu'+str(cpu)] = {}
				cpu_info['cpu'+str(cpu)][key] = value

		if(len(line)==1): cpu += 1

	return cpu_info

def _get_memory_usage():

	memory_info = subprocess.Popen(["cat","/proc/meminfo"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]

	memory_total = ""
	memory_free = ""
	for element in memory_info.split('\n'):
		if("MemTotal" in element): memory_total = element
		if("MemFree" in element): memory_free = element

	try:
		memory_total = long(memory_total.split(' ')[-2:-1][0]) * 1024
		memory_free = long(memory_free.split(' ')[-2:-1][0]) * 1024
		memory_used = long(memory_total-memory_free)
	except:
		memory_total = -1
		memory_free = -1
		memory_used = -1

	memory_usage = {
		'memory_total': memory_total,
		'memory_free': memory_free,
		'memory_used': memory_used,
		'memory_used_percentage': round(float(memory_used)/float(memory_total)*100,2),
	}

	return memory_usage

def _get_ip_address():

	ifconfig = subprocess.Popen(["ifconfig","eth0"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	ip = re.search( r'inet addr:[^\s]+', ifconfig )
	if(not ip):
		ip = re.search( r'inet [^\s]+', ifconfig )
	if not platform.system() == 'Darwin':
		ip = ip.group().split(':')[-1]
	else:
		ip = "127.0.0.1"
	ip = ip.replace(" ","")
	ip = ip.replace("inet","")
	return ip

def _get_sys_cpu_virtualization():

	virtualization_support = False

	cpuinfo = open('/proc/cpuinfo','rt').readlines()
	for line in cpuinfo:

		if("vmx" in line):
			virtualization_support = True
		if("svm" in line):
			virtualization_support = True

	return virtualization_support

def _get_physical_disks():

	physical_disks = []

	physical_disks_list = subprocess.Popen(["fdisk","-l"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
	physical_disks_list_ = physical_disks_list.split('\n')

	physical_disks_list = []
	for line in physical_disks_list_:

		if('Disk' in line):
			disk = line.split(' ')[1].replace(':','')
			try:
				bsize = long(line.split(' ')[-2:-1][0])
				physical_disks_list.append([disk,bsize])
			except: pass

	for hdd in physical_disks_list:

		FNULL = open(os.devnull, 'w')
		disk_info = subprocess.Popen(["hdparm","-I",hdd[0],], stderr=FNULL, stdout=subprocess.PIPE, close_fds=True).communicate()[0]

		model_number = "n/a"
		serial_number = "n/a"
		firmware_revision = "n/a"

		for line in disk_info.split('\n'):

			if("Model Number" in line): model_number = line.split(':')[1].replace(' ','')
			if("Serial Number" in line): serial_number = line.split(':')[1].replace(' ','')
			if("Firmware Revision" in line): firmware_revision = line.split(':')[1].replace(' ','')

		try: model_number.decode('utf-8')
		except UnicodeDecodeError: model_number = "n/a"

		try: serial_number.decode('utf-8')
		except UnicodeDecodeError: serial_number = "n/a"

		try: firmware_revision.decode('utf-8')
		except UnicodeDecodeError: firmware_revision = "n/a"

		disk_info = {
			'device':hdd[0],
			'size':hdd[1],
			'model_number':model_number,
			'serial_number':serial_number,
			'firmware_revision':firmware_revision,
		}
		physical_disks.append(disk_info)

	return physical_disks

def _get_disks_usage():

	proc = subprocess.Popen(['df', '-B 1'], stdout=subprocess.PIPE, close_fds=True)
	df = proc.communicate()[0]

	try:
		volumes = df.split('\n')
		volumes.pop(0)
		volumes.pop()
	except: return {"error":True,"error_description":"_get_disks_usage"}

	regexp = re.compile(r'([0-9]+)')
	previousVolume = None
	volumeCount = 0
	disks_usage = []

	for volume in volumes:

		volume = volume.split(None, 10)

		if len(volume) == 1:
			previousVolume = volume[0]
			continue

		if previousVolume != None:
			volume.insert(0, previousVolume)
			previousVolume = None

		volumeCount = volumeCount + 1

		if regexp.match(volume[1]) == None:
			pass

		else:
			try:
				volume[2] = int(volume[2]) / 1024 / 1000 # Used
				volume[3] = int(volume[3]) / 1024 / 1000 # Available
			except IndexError:
				pass
			except KeyError:
				pass

			disks_usage.append(volume)

	return disks_usage

def _get_hostname():

    hostname = subprocess.Popen(["hostname"], stdout=subprocess.PIPE, close_fds=True).communicate()[0]
    hostname = hostname.replace("\n","")

    return hostname

def _get_distro():

    distro = ""
    for i in platform.linux_distribution():
        distro += i.title() + " "

    distro = distro[:-1]

    return distro

def get_system_metrics( secret ):

	print datetime.datetime.now(), SECRET, 'Collecting system metrics..'

	uuid = UUID
	ip = _get_ip_address()
	loadavg = _get_sys_loadavg()
	uptime = _get_sys_uptime()
	cpu_usage = _get_sys_cpu()
	cpu_info = _get_sys_cpu_info()
	cpu_virtualization = _get_sys_cpu_virtualization()
	memory_usage = _get_memory_usage()
	physical_disks = _get_physical_disks()
	disks_usage = _get_disks_usage()
	hostname = _get_hostname()
	distro = _get_distro()

	system_metrics_json = {
		'uuid': uuid,
		'ip': ip,
		'hostname': hostname,
		'distro': distro,
		'secret': secret,
		'loadavg': loadavg,
		'uptime': uptime,
		'cpu_usage': cpu_usage,
		'cpu_info': cpu_info,
		'cpu_virtualization': cpu_virtualization,
		'memory_usage': memory_usage,
		'physical_disks': physical_disks,
		'disks_usage': disks_usage,
		'agent_version': AGENT_VERSION,
	}

	return system_metrics_json

def send_data( secret, api_call, data ):

	params = urllib.urlencode(data)
	headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        api_url = "http://"+API_SERVER+api_call

	conn = httplib.HTTPConnection(API_SERVER)
	conn.request("POST", api_call, json.dumps(data), headers)
	response = conn.getresponse()
	response_data = response.read()
	conn.close()

	return response_data

if __name__=="__main__":

	os.system("clear")

	print "AGENT: v"+AGENT_VERSION
	print "Written By: Jan Paricka"

	# Filing an activity: example
	#api_call = "/v10/activity/"
	#activity = {
	#	'secret': SECRET,
	#	'agent_version': AGENT_VERSION,
	#	'uuid': UUID,
	#	'activity': "Agent v"+AGENT_VERSION+" has started."
	#}
	#send_data(SECRET,api_call,activity)

	while True:

		api_call = "/v10/ping/"
		system_metrics = get_system_metrics(SECRET)
		api_response = send_data(SECRET,api_call,system_metrics)
		time.sleep(3)

	print "ze end."
<|endoftext|>


<commit_before>#!/usr/bin/python
#  Copyright (C) 2012  Alexander Gude - gude@physics.umn.edu
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

from optparse import OptionParser # Command line parsing
from tempfile import mkdtemp # Secure methods of generating random directories
from random import choice
from string import ascii_uppercase,digits,ascii_lowercase
from sys import exit # Cleanly exit program
from subprocess import call # Access external programs
from os import listdir,remove,devnull
from os.path import isfile
from shutil import copy2,rmtree

## Help functions
def listdirwithpath(dir):
    """ Return a list of files with their path """
    return [ dir+'/'+f for f in listdir(dir) ]

## Hadd class
class hadd:
    """ A class to handle hadding of files, and cleanup of output """
    def __init__(self, outfile, infiles, tmpdir, verbose=False, vverbose=False, quite=False, force_overwrite=False, save=False, natonce=20):
        """ Set up the class """
        self.outfile = outfile
        self.infiles = infiles
        self.verbose = verbose
        self.vverbose = vverbose
        self.quite = quite
        self.force_overwrite = force_overwrite
        self.natonce = natonce
        self.save = save
        self.counter = 0
        self.__checkOutFile()
        self.__printInAndOut()
        self.__setTmpDir(tmpdir)

    def run(self):
        """ Combine files by looping over them """
        if not self.quite: print "Combining files"

        i = 0
        infiles = list(self.infiles) # deep copies the list
        while True:
            currentWriteDir = mkdtemp(prefix=str(i)+"_", dir=self.tmpdir) + '/'
            self.__haddMultiple(currentWriteDir, infiles)
            currentReadDir = currentWriteDir
            infiles = listdirwithpath(currentReadDir)
            if len(infiles) == 1:
                if not self.quite: print "Copying final file:",infiles[0],"-->",self.outfile
                copy2(infiles[0], self.outfile)
                break
            else:
                i += 1
        self.__cleanup()

    def __printInAndOut(self):
        """ Print the input files, and output file """
        if not self.quite:
            print "Output file:",out_file
            print "Input files:"
            for f in in_files:
                print "\t",f
            print "Number of files to hadd at once:",self.natonce

    def __checkOutFile(self):
        """ Check if the output file exists, if so exit if we don't force
        overwrite """
        if not self.force_overwrite and isfile(self.outfile):
            print "Output file already exists! File:",self.outfile
            exit(1) # Other error
        elif self.force_overwrite and isfile(self.outfile):
            if not self.quite: print "Output file already exists; it will be overwriten! File:",self.outfile

    def __cleanup(self):
        """ Clean up intermediate files """
        if not self.save:
            rmtree(self.tmpdir)
        exit(0) # Normal exit

    def __haddMultiple(self, writeDir, inFiles):
        """ Given a list of inFiles, and a write dir, hadds the files and saves
        them to the directory """
        while inFiles:
            currentFiles = []
            if len(inFiles) >= self.natonce:
                for i in xrange(self.natonce):
                    currentFiles.append(inFiles.pop())
            else:
                currentFiles = inFiles
                inFiles = []

            outfile = self.__getRandomRootName(writeDir)
            self.__hadd(outfile, currentFiles)

    def __getRandomRootName(self, dir):
        """ Return a random file name """
        chars = ascii_uppercase+digits+ascii_lowercase
        ran = []
        for x in xrange(6):
            ran.append(choice(chars))
        num = self.counter
        self.counter += 1
        return dir + '/input_%i_'%(num) + ''.join(ran) + ".root"

    def __setTmpDir(self,tmpdir):
        """ Set up a temporary directory to hold intermediate files """
        if self.verbose: print "Making temporary directory:",

        if tmpdir == None:
            self.tmpdir = mkdtemp(prefix="ahadd_") + '/'
        else:
            self.tmpdir = mkdtemp(prefix="ahadd_",dir=options.tmp_dir) + '/'

        if self.verbose: print self.tmpdir

    def __hadd(self, outfile, infiles):
        """ Call hadd """
        args = ["hadd", outfile] + infiles
        if self.verbose:
            print "Calling hadd"
            print "\tOutput:",outfile
            print "\tInput:"
            for f in infiles:
                print "\t\t",f

        if self.vverbose:
            return (call(args), outfile)
        else:
            return (call(args, stdout=open(devnull, 'wb')), outfile)

##### START OF CODE
""" Allows command line options to be parsed. Called first to in order to let functions use them.  """

usage = "usage: %prog [Options] outputfile inputfiles"
version = "%prog Version 1.1.0\n\nCopyright (C) 2012 Alexander Gude - gude@physics.umn.edu\nThis is free software.  You may redistribute copies of it under the terms of\nthe GNU General Public License <http://www.gnu.org/licenses/gpl.html>.\nThere is NO WARRANTY, to the extent permitted by law.\n\nWritten by Alexander Gude."
parser = OptionParser(usage=usage,version=version)
parser.add_option("-n", "--n-files-at-once", action="store", type="int", dest="natonce", default=20, help="combine this many files at one time [defualt 4]")
parser.add_option("-t", "--temp-dir", action="store", type="string", dest="tmp_dir", default=None, help="location to store temporary intermediate files")
parser.add_option("-s", "--save-temp", action="store_true", dest="save_tmp", default=False, help="save temporary files, otherwise they are cleaned up when the program exits [default false]")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="print some extra status messages to stdout [default false]")
parser.add_option("-q", "--quite", action="store_true", dest="quite", default=False, help="do not print any status messages to stdout [default false]")
parser.add_option("-V", "--very-verbose", action="store_true", dest="vverbose", default=False, help="print everything, even the output from hadd [default false]")
parser.add_option("-f", "--force--overwrite", action="store_true", dest="force_overwrite", default=False, help="Overwrite the output file if it exists [default false]")

(options, args) = parser.parse_args()

## Check verbosity
if options.quite:
    options.verbose = False
    options.vverbose = False
elif options.vverbose:
    options.verbose = True

## Check that we have at least a few files to work on
in_files = []
out_file = None
if len(args) <= 2:
    print "Not enough arguments on the command line. Exiting."
    exit(2) # Not enough commands
else:
    out_file = args[0]
    in_files = args[1:]

## Check that the number of files to hadd each step is sane
if options.natonce <= 1:
    print "Requested to hadd 1 or fewer files per iteration; this will never converge."
    exit(1) # Other error

## Set up and run hadd
h = hadd(out_file, in_files, options.tmp_dir, options.verbose, options.vverbose, options.quite, options.force_overwrite, options.save_tmp, options.natonce)
h.run()
<commit_msg>v1.1.1<commit_after>#!/usr/bin/python
#  Copyright (C) 2012  Alexander Gude - gude@physics.umn.edu
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

from optparse import OptionParser # Command line parsing
from tempfile import mkdtemp # Secure methods of generating random directories
from random import choice
from string import ascii_uppercase,digits,ascii_lowercase
from sys import exit # Cleanly exit program
from subprocess import call # Access external programs
from os import listdir,remove,devnull
from os.path import isfile
from shutil import copy2,rmtree

## Help functions
def listdirwithpath(dir):
    """ Return a list of files with their path """
    return [ dir+'/'+f for f in listdir(dir) ]

## Hadd class
class hadd:
    """ A class to handle hadding of files, and cleanup of output """
    def __init__(self, outfile, infiles, tmpdir, verbose=False, vverbose=False, quite=False, force_overwrite=False, save=False, natonce=20):
        """ Set up the class """
        self.outfile = outfile
        self.infiles = infiles
        self.verbose = verbose
        self.vverbose = vverbose
        self.quite = quite
        self.force_overwrite = force_overwrite
        self.natonce = natonce
        self.save = save
        self.counter = 0
        self.__checkOutFile()
        self.__printInAndOut()
        self.__setTmpDir(tmpdir)

    def run(self):
        """ Combine files by looping over them """
        if not self.quite: print "Combining files"

        i = 0
        infiles = list(self.infiles) # deep copies the list
        while True:
            currentWriteDir = mkdtemp(prefix=str(i)+"_", dir=self.tmpdir) + '/'
            self.__haddMultiple(currentWriteDir, infiles)
            currentReadDir = currentWriteDir
            infiles = listdirwithpath(currentReadDir)
            if len(infiles) == 1:
                if not self.quite: print "Copying final file:",infiles[0],"-->",self.outfile
                copy2(infiles[0], self.outfile)
                break
            else:
                i += 1
        self.__cleanup()

    def __printInAndOut(self):
        """ Print the input files, and output file """
        if not self.quite:
            print "Output file:",out_file
            print "Input files:"
            for f in in_files:
                print "\t",f
            print "Number of files to hadd at once:",self.natonce

    def __checkOutFile(self):
        """ Check if the output file exists, if so exit if we don't force
        overwrite """
        if not self.force_overwrite and isfile(self.outfile):
            print "Output file already exists! File:",self.outfile
            exit(1) # Other error
        elif self.force_overwrite and isfile(self.outfile):
            if not self.quite: print "Output file already exists; it will be overwriten! File:",self.outfile

    def __cleanup(self):
        """ Clean up intermediate files """
        if not self.save:
            rmtree(self.tmpdir)
        exit(0) # Normal exit

    def __haddMultiple(self, writeDir, inFiles):
        """ Given a list of inFiles, and a write dir, hadds the files and saves
        them to the directory """
        while inFiles:
            currentFiles = []
            if len(inFiles) >= self.natonce:
                for i in xrange(self.natonce):
                    currentFiles.append(inFiles.pop())
            else:
                currentFiles = inFiles
                inFiles = []

            outfile = self.__getRandomRootName(writeDir)
            self.__hadd(outfile, currentFiles)

    def __getRandomRootName(self, dir):
        """ Return a random file name """
        chars = ascii_uppercase+digits+ascii_lowercase
        ran = []
        for x in xrange(6):
            ran.append(choice(chars))
        num = self.counter
        self.counter += 1
        return dir + '/input_%i_'%(num) + ''.join(ran) + ".root"

    def __setTmpDir(self,tmpdir):
        """ Set up a temporary directory to hold intermediate files """
        if self.verbose: print "Making temporary directory:",

        if tmpdir == None:
            self.tmpdir = mkdtemp(prefix="ahadd_") + '/'
        else:
            self.tmpdir = mkdtemp(prefix="ahadd_",dir=options.tmp_dir) + '/'

        if self.verbose: print self.tmpdir

    def __hadd(self, outfile, infiles):
        """ Call hadd """
        args = ["hadd", outfile] + infiles
        if self.verbose:
            print "Calling hadd"
            print "\tOutput:",outfile
            print "\tInput:"
            for f in infiles:
                print "\t\t",f

        if self.vverbose:
            return (call(args), outfile)
        else:
            return (call(args, stdout=open(devnull, 'wb')), outfile)

##### START OF CODE
""" Allows command line options to be parsed. Called first to in order to let functions use them.  """

usage = "usage: %prog [Options] outputfile inputfiles"
version = "%prog Version 1.1.1\n\nCopyright (C) 2012 Alexander Gude - gude@physics.umn.edu\nThis is free software.  You may redistribute copies of it under the terms of\nthe GNU General Public License <http://www.gnu.org/licenses/gpl.html>.\nThere is NO WARRANTY, to the extent permitted by law.\n\nWritten by Alexander Gude."
parser = OptionParser(usage=usage,version=version)
parser.add_option("-n", "--n-files-at-once", action="store", type="int", dest="natonce", default=20, help="combine this many files at one time [defualt 4]")
parser.add_option("-t", "--temp-dir", action="store", type="string", dest="tmp_dir", default=None, help="location to store temporary intermediate files")
parser.add_option("-s", "--save-temp", action="store_true", dest="save_tmp", default=False, help="save temporary files, otherwise they are cleaned up when the program exits [default false]")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="print some extra status messages to stdout [default false]")
parser.add_option("-q", "--quite", action="store_true", dest="quite", default=False, help="do not print any status messages to stdout [default false]")
parser.add_option("-V", "--very-verbose", action="store_true", dest="vverbose", default=False, help="print everything, even the output from hadd [default false]")
parser.add_option("-f", "--force--overwrite", action="store_true", dest="force_overwrite", default=False, help="Overwrite the output file if it exists [default false]")

(options, args) = parser.parse_args()

## Check verbosity
if options.quite:
    options.verbose = False
    options.vverbose = False
elif options.vverbose:
    options.verbose = True

## Check that we have at least a few files to work on
in_files = []
out_file = None
if len(args) <= 2:
    print "Not enough arguments on the command line. Exiting."
    exit(2) # Not enough commands
else:
    out_file = args[0]
    in_files = args[1:]

## Check that the number of files to hadd each step is sane
if options.natonce <= 1:
    print "Requested to hadd 1 or fewer files per iteration; this will never converge."
    exit(1) # Other error

## Set up and run hadd
h = hadd(out_file, in_files, options.tmp_dir, options.verbose, options.vverbose, options.quite, options.force_overwrite, options.save_tmp, options.natonce)
h.run()
<|endoftext|>