<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
traceroute - Get traceroute results with associated geolocation information
for each hop for a specified host from geographically distant source(s).
"""

__author__ = 'Dazzlepod (info@dazzlepod.com)'
__copyright__ = 'Copyright (c) 2013 Dazzlepod'
__version__ = '$Revision: #7 $'

import datetime
import json
import optparse
import os
import re
import sys
import urllib
import urllib2
from multiprocessing import Process
from subprocess import Popen, PIPE

class Traceroute(object):
    """Traceroute instance."""
    def __init__(self, ip_address='8.8.8.8', debug=False):
        super(Traceroute, self).__init__()
        self.ip_address = ip_address
        self.debug = debug
        # cache geocoded IP addresses during the lifetime of this instance
        self.locations = {}

    def traceroute(self):
        """Instead of running the actual traceroute command, we will fetch
        standard traceroute results from several publicly available webpages
        that are listed at traceroute.org.

        For each hop, we will then attach geolocation information to it."""
        self.print_debug("ip_address = %s" % self.ip_address)
        if not os.path.exists('%s.txt' % self.ip_address):
            (status_code, traceroute) = self.get_traceroute_output()
            f = open('%s.txt' % self.ip_address, 'w')
            f.write(traceroute)
            f.close()

        traceroute = open('%s.txt' % self.ip_address, 'r').read()

        # hops = dicts with keys: hop_num, hosts
        hops = self.get_hops(traceroute)

        # hops = dicts with keys: hop_num, hostname, ip_address, rtt
        hops = self.get_formatted_hops(hops)

        # hops = dicts with keys: hop_num, hostname, ip_address, rtt, location
        hops = self.get_geocoded_hops(hops)

        # hops = dicts with keys: hop_num, ip_address, latitude, longitude, rtt
        hops = self.get_stripped_hops(hops)

        return hops

    def get_traceroute_output(self):
        """Fetch traceroute output from a webpage."""

        # Example from traceroute.org
        url = "http://www.net.princeton.edu/cgi-bin/traceroute.pl"
        (status_code, response) = self.urlopen(url, context = {'target': self.ip_address})

        pattern = re.compile(r'
(?P<traceroute>.*?)
', re.DOTALL|re.IGNORECASE)
        traceroute = re.findall(pattern, response)[0].strip()

        return (status_code, traceroute)

    def get_hops(self, traceroute):
        """Get hops from a traceroute output and return the hops in an array
        of dicts each representing hop number and the associated hosts data."""
        hops = []

        lines = traceroute.split('\n')
        for line in lines:
            line = line.strip()
            hop = {}
            if not line: continue
            try:
                hop = re.match(r'^(?P<hop_num>\d+)(?P<hosts>.*?)$', line).groupdict()
            except AttributeError:
                continue
            self.print_debug(hop)
            hops.append(hop)

        return hops

    def get_formatted_hops(self, hops):
        """hosts data from get_hops() is represented in a single string.
        We use this function to better represent the hosts data in a dict."""
        formatted_hops = []

        for hop in hops:
            hop_num = int(hop['hop_num'].strip())
            hosts = hop['hosts'].replace('  ', ' ').strip()

            # Using re.findall(), first we split the hosts, then for each host we store a tuple containing hostname, IP address and the first round-trip time
            # [('<HOSTNAME>', '<IP_ADDRESS>', '<RTT1> ms'), ('<HOSTNAME_N>', '<IP_ADDRESS_N>', '<RTT1_N> ms')]
            hosts = re.findall(r'(?P<hostname>[\w.-]+) \((?P<ip_address>[\d.]+)\) (?P<rtt>\d{1,4}.\d{1,4} ms)', hosts)

            for host in hosts:
                hop_context = {
                    'hop_num': hop_num,
                    'hostname': host[0],
                    'ip_address': host[1],
                    'rtt': host[2],
                }
                self.print_debug(hop_context)
                formatted_hops.append(hop_context)

        return formatted_hops

    def get_geocoded_hops(self, hops):
        """Return hops from get_formatted_hops() with geolocation information
        for each hop."""
        for hop in hops:
            ip_address = hop['ip_address']
            location = None
            if self.locations.has_key(ip_address):
                location = self.locations[ip_address]
            else:
                location = self.get_location(ip_address)
                self.locations[ip_address] = location
            hop['location'] = location

        return hops

    def get_stripped_hops(self, hops):
        """ Return hops from get_geocoded_hops() with each dict containing
        data ready for plotting with e.g. Google Maps JavaScript API."""
        stripped_hops = []

        for hop in hops:
            if hop['location']:
                lat = hop['location']['latitude']
                lon = hop['location']['longitude']
                stripped_hops.append({
                    'hop_num': hop['hop_num'],
                    'ip_address': hop['ip_address'],
                    'latitude': lat,
                    'longitude': lon,
                    'rtt': hop['rtt'],
                })

        return stripped_hops

    def get_location(self, ip_address):
        """Return geolocation information for the specified IP address, e.g.:
            {"ip": "75.126.24.77",
            "hostname": "web365.webfaction.com",
            "isp": "SoftLayer Technologies",
            "organization": "Client Intellect",
            "country": "United States",
            "region": "Texas",
            "city": "Dallas",
            "latitude": 32.9299,
            "longitude": -96.8353}
        """
        location = None
        url = "https://dazzlepod.com/ip/%s.json" % ip_address
        (status_code, json_data) = self.urlopen(url)
        if status_code == 200 and json_data:
            tmp_location = json.loads(json_data)
            if tmp_location.has_key('latitude') and tmp_location.has_key('longitude'):
                location = tmp_location
        return location

    def urlopen(self, url, context = None):
        """Perform HTTP GET/POST on the specified URL and return the resultant
        status code and response.
        """
        status_code = 200

        request = urllib2.Request(url = url)

        if context:
            data = urllib.urlencode(context)
            request.add_data(data)

        response = ''
        try:
            response = urllib2.urlopen(request).read()
        except urllib2.HTTPError, e:
            status_code = e.code
        except urllib2.URLError:
            pass

        try:
            self.urlopen_count += 1
        except AttributeError:
            self.urlopen_count = 1
        self.print_debug("[%d] url = %s, status_code = %d" % (self.urlopen_count, url, status_code))

        return (status_code, response)

    def print_debug(self, msg):
        """Print debug message to standard output."""
        if self.debug:
            print "[DEBUG %s] %s" % (datetime.datetime.now(), msg)

def main():
    usage = """%prog --ip_address=IP_ADDRESS"""
    cmdparser = optparse.OptionParser(usage, version=("traceroute " + __version__))
    cmdparser.add_option("-i", "--ip_address", type="string", default="", help="IP address of destination host")
    cmdparser.add_option("-d", "--debug", action="store_true", default=False, help="Show debug output")

    (options, args) = cmdparser.parse_args()

    if options.ip_address:
        traceroute = Traceroute(ip_address=options.ip_address, debug=options.debug)
        hops = traceroute.traceroute()
        hops = json.dumps(hops, indent=2)
        print hops
    else:
        cmdparser.print_usage()

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>added --tmp_dir to specify download directory and --no_geo to skip geolocation data<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
traceroute - Get traceroute results with associated geolocation information
for each hop for a specified host from geographically distant source(s).
"""

__author__ = 'Dazzlepod (info@dazzlepod.com)'
__copyright__ = 'Copyright (c) 2013 Dazzlepod'
__version__ = '$Revision: #11 $'

import datetime
import json
import optparse
import os
import re
import sys
import urllib
import urllib2
from multiprocessing import Process
from subprocess import Popen, PIPE

class Traceroute(object):
    """Traceroute instance."""
    def __init__(self, ip_address='8.8.8.8', tmp_dir='/tmp', no_geo=False, debug=False):
        super(Traceroute, self).__init__()
        self.ip_address = ip_address
        self.tmp_dir = tmp_dir
        self.no_geo = no_geo
        self.debug = debug
        # cache geocoded IP addresses during the lifetime of this instance
        self.locations = {}

    def traceroute(self):
        """Instead of running the actual traceroute command, we will fetch
        standard traceroute results from several publicly available webpages
        that are listed at traceroute.org. For each hop, we will then attach
        geolocation information to it."""
        self.print_debug("ip_address = %s" % self.ip_address)
        txt = os.path.join(self.tmp_dir, '%s.txt' % self.ip_address)
        if not os.path.exists(txt):
            (status_code, traceroute) = self.get_traceroute_output()
            f = open(txt, 'w')
            f.write(traceroute)
            f.close()

        traceroute = open(txt, 'r').read()

        # hops = dicts with keys: hop_num, hosts
        hops = self.get_hops(traceroute)

        # hops = dicts with keys: hop_num, hostname, ip_address, rtt
        hops = self.get_formatted_hops(hops)

        if not self.no_geo:
            # hops = dicts with keys: hop_num, hostname, ip_address, rtt, latitude, longitude
            hops = self.get_geocoded_hops(hops)

        return hops

    def get_traceroute_output(self):
        """Fetch traceroute output from a webpage."""

        # Example from traceroute.org
        url = "http://www.net.princeton.edu/cgi-bin/traceroute.pl"
        (status_code, response) = self.urlopen(url, context = {'target': self.ip_address})

        pattern = re.compile(r'
(?P<traceroute>.*?)
', re.DOTALL|re.IGNORECASE)
        traceroute = re.findall(pattern, response)[0].strip()

        return (status_code, traceroute)

    def get_hops(self, traceroute):
        """Get hops from a traceroute output and return the hops in an array
        of dicts each representing hop number and the associated hosts data."""
        hops = []

        lines = traceroute.split('\n')
        for line in lines:
            line = line.strip()
            hop = {}
            if not line: continue
            try:
                hop = re.match(r'^(?P<hop_num>\d+)(?P<hosts>.*?)$', line).groupdict()
            except AttributeError:
                continue
            self.print_debug(hop)
            hops.append(hop)

        return hops

    def get_formatted_hops(self, hops):
        """hosts data from get_hops() is represented in a single string.
        We use this function to better represent the hosts data in a dict."""
        formatted_hops = []

        for hop in hops:
            hop_num = int(hop['hop_num'].strip())
            hosts = hop['hosts'].replace('  ', ' ').strip()

            # Using re.findall(), first we split the hosts, then for each host we store a tuple containing hostname, IP address and the first round-trip time
            # [('<HOSTNAME>', '<IP_ADDRESS>', '<RTT1> ms'), ('<HOSTNAME_N>', '<IP_ADDRESS_N>', '<RTT1_N> ms')]
            hosts = re.findall(r'(?P<hostname>[\w.-]+) \((?P<ip_address>[\d.]+)\) (?P<rtt>\d{1,4}.\d{1,4} ms)', hosts)

            for host in hosts:
                hop_context = {
                    'hop_num': hop_num,
                    'hostname': host[0],
                    'ip_address': host[1],
                    'rtt': host[2],
                }
                self.print_debug(hop_context)
                formatted_hops.append(hop_context)

        return formatted_hops

    def get_geocoded_hops(self, hops):
        """Return hops from get_formatted_hops() with geolocation information
        for each hop."""
        geocoded_hops = []

        for hop in hops:
            ip_address = hop['ip_address']

            location = None
            if self.locations.has_key(ip_address):
                location = self.locations[ip_address]
            else:
                location = self.get_location(ip_address)
                self.locations[ip_address] = location

            if location:
                geocoded_hops.append({
                    'hop_num': hop['hop_num'],
                    'hostname': hop['hostname'],
                    'ip_address': hop['ip_address'],
                    'rtt': hop['rtt'],
                    'latitude': location['latitude'],
                    'longitude': location['longitude'],
                })

        return geocoded_hops

    def get_location(self, ip_address):
        """Return geolocation information for the specified IP address, e.g.:
            {"ip": "75.126.24.77",
            "hostname": "web365.webfaction.com",
            "isp": "SoftLayer Technologies",
            "organization": "Client Intellect",
            "country": "United States",
            "region": "Texas",
            "city": "Dallas",
            "latitude": 32.9299,
            "longitude": -96.8353}
        """
        location = None
        url = "https://dazzlepod.com/ip/%s.json" % ip_address
        (status_code, json_data) = self.urlopen(url)
        if status_code == 200 and json_data:
            tmp_location = json.loads(json_data)
            if tmp_location.has_key('latitude') and tmp_location.has_key('longitude'):
                location = tmp_location
        return location

    def urlopen(self, url, context=None):
        """Perform HTTP GET/POST on the specified URL and return the resultant
        status code and response.
        """
        status_code = 200

        request = urllib2.Request(url = url)

        if context:
            data = urllib.urlencode(context)
            request.add_data(data)

        response = ''
        try:
            response = urllib2.urlopen(request).read()
        except urllib2.HTTPError, e:
            status_code = e.code
        except urllib2.URLError:
            pass

        try:
            self.urlopen_count += 1
        except AttributeError:
            self.urlopen_count = 1
        self.print_debug("[%d] url = %s, status_code = %d" % (self.urlopen_count, url, status_code))

        return (status_code, response)

    def print_debug(self, msg):
        """Print debug message to standard output."""
        if self.debug:
            print "[DEBUG %s] %s" % (datetime.datetime.now(), msg)

def main():
    usage = """%prog --ip_address=IP_ADDRESS"""
    cmdparser = optparse.OptionParser(usage, version=("traceroute " + __version__))
    cmdparser.add_option("-i", "--ip_address", type="string", default="8.8.8.8", help="IP address of destination host (default: 8.8.8.8)")
    cmdparser.add_option("-t", "--tmp_dir", type="string", default="/tmp", help="Temporary directory to store downloaded traceroute results (default: /tmp)")
    cmdparser.add_option("-n", "--no_geo", action="store_true", default=False, help="No geolocation data (default: False)")
    cmdparser.add_option("-d", "--debug", action="store_true", default=False, help="Show debug output (default: False)")

    (options, args) = cmdparser.parse_args()

    if options.ip_address:
        traceroute = Traceroute(ip_address=options.ip_address, tmp_dir=options.tmp_dir, no_geo=options.no_geo, debug=options.debug)
        hops = traceroute.traceroute()
        print hops
    else:
        cmdparser.print_usage()

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import json
from dataclasses import asdict
from typing import NoReturn, Union

from flask import Blueprint, jsonify, url_for, abort, flash, redirect, request, \
    render_template
from flask_login import current_user

from pycroft.lib.exc import PycroftLibException
from pycroft.model import session

from pycroft.lib.task import cancel_task, task_type_to_impl, manually_execute_task
from pycroft.model.facilities import Building
from pycroft.model.task import Task, TaskStatus
from web.blueprints import redirect_or_404

from web.blueprints.access import BlueprintAccess
from web.table.table import datetime_format
from web.blueprints.helpers.user import get_user_or_404
from web.blueprints.navigation import BlueprintNavigation
from web.blueprints.task.tables import TaskTable
from web.template_filters import datetime_filter

bp = Blueprint('task', __name__)
access = BlueprintAccess(bp, required_properties=['user_show'])
nav = BlueprintNavigation(bp, "Tasks", blueprint_access=access)

def format_parameters(parameters):
    """Make task parameters human readable by looking up objects behind ids"""

    # Replace building_id by the buildings short name
    if parameters.get("building_id"):
        building = Building.get(parameters["building_id"])

        if building:
            parameters["building"] = building.short_name
            del parameters["building_id"]

    return parameters

def task_row(task: Task):
    task_impl = task_type_to_impl.get(task.type)
    T = TaskTable
    return {
        "id": task.id,
        "user": T.user.value(
            href=url_for('user.user_show', user_id=task.user.id),
            title=task.user.name
        ),
        "name": task_impl.name,
        "type": task.type.name,
        "status": task.status.name,
        "parameters": format_parameters(asdict(task.parameters)),
        "errors": task.errors if task.errors is not None else list(),
        "due": datetime_format(task.due, default='', formatter=datetime_filter),
        "created": task.created.strftime("%Y-%m-%d %H:%M:%S"),
        "creator": T.creator.value(
            href=url_for('user.user_show', user_id=task.creator.id),
            title=task.creator.name
        ),
        'actions': [
            T.actions.single_value(
                href=url_for(
                    '.cancel_user_task',
                    task_id=task.id,
                    redirect=url_for('user.user_show', user_id=task.user.id, _anchor='tasks')
                ),
                title="Abbrechen",
                icon='fa-times',
                btn_class='btn-link'
            ),
            T.actions.single_value(
                href=url_for(
                    '.manually_execute_user_task',
                    task_id=task.id,
                    redirect=url_for('user.user_show', user_id=task.user.id, _anchor='tasks')
                ),
                title="Sofort ausführen",
                icon='fa-fast-forward',
                btn_class='btn-link'
            )
        ] if task.status == TaskStatus.OPEN else None,
    }

@bp.route("/user/<int:user_id>/json")
def json_tasks_for_user(user_id):
    user = get_user_or_404(user_id)

    return jsonify(items=[task_row(task) for task in user.tasks])

@bp.route("/user/json")
def json_user_tasks():
    failed_only = bool(request.args.get("failed_only", False))
    open_only = bool(request.args.get("open_only", False))

    tasks = Task.q.order_by(Task.status.desc(), Task.due.asc())\

    if failed_only:
        tasks = tasks.filter_by(status=TaskStatus.FAILED).all()
    elif open_only:
        tasks = tasks.filter_by(status=TaskStatus.OPEN).all()
    else:
        tasks = tasks.all()

    return jsonify(items=[task_row(task) for task in tasks])

def get_task_or_404(task_id) -> Union[Task, NoReturn]:
    if task := session.session.get(Task, task_id):
        return task
    abort(404)

@bp.route("/<int:task_id>/manually_execute")
@access.require('user_change')
def manually_execute_user_task(task_id: int):
    task = get_task_or_404(task_id)
    try:
        manually_execute_task(task, processor=current_user)
        session.session.commit()
    except Exception as e:
        if not isinstance(e, PycroftLibException):
            import logging
            logging.getLogger('pycroft.web').error(
                "Unexpected error in manual task execution: %s", e,
                exc_info=True
            )
        flash(f"Fehler bei der Ausführung: {e}", 'error')
        session.session.rollback()
    else:
        flash("Aufgabe erfolgreich ausgeführt", 'success')

    return redirect_or_404(request.args.get("redirect"))

@bp.route("/<int:task_id>/cancel")
@access.require('user_change')
def cancel_user_task(task_id):
    task = get_task_or_404(task_id)

    cancel_task(task, current_user)
    session.session.commit()

    flash(u'Aufgabe erfolgreich abgebrochen.', 'success')
    return redirect_or_404(request.args.get("redirect"))

@bp.route("/user")
@nav.navigate("Tasks")
def user_tasks():
    return render_template("task/tasks.html",
                           task_table=TaskTable(
                               data_url=url_for('.json_user_tasks',
                                                open_only=1)),
                           task_failed_table=TaskTable(
                               data_url=url_for('.json_user_tasks',
                                                failed_only=1),
                               sort_order='desc'),
                           page_title="Aufgaben (Nutzer)")
<commit_msg>[cleanup] minor formatting improvements<commit_after>import json
from dataclasses import asdict
from typing import NoReturn, Union

from flask import Blueprint, jsonify, url_for, abort, flash, redirect, request, \
    render_template
from flask_login import current_user

from pycroft.lib.exc import PycroftLibException
from pycroft.model import session

from pycroft.lib.task import cancel_task, task_type_to_impl, manually_execute_task
from pycroft.model.facilities import Building
from pycroft.model.task import Task, TaskStatus
from web.blueprints import redirect_or_404

from web.blueprints.access import BlueprintAccess
from web.table.table import datetime_format
from web.blueprints.helpers.user import get_user_or_404
from web.blueprints.navigation import BlueprintNavigation
from web.blueprints.task.tables import TaskTable
from web.template_filters import datetime_filter

bp = Blueprint('task', __name__)
access = BlueprintAccess(bp, required_properties=['user_show'])
nav = BlueprintNavigation(bp, "Tasks", blueprint_access=access)

def format_parameters(parameters):
    """Make task parameters human readable by looking up objects behind ids"""

    # Replace building_id by the buildings short name
    if bid := parameters.get("building_id"):
        if building := Building.get(bid):
            parameters["building"] = building.short_name
            del parameters["building_id"]
    return parameters

def task_row(task: Task):
    task_impl = task_type_to_impl.get(task.type)
    T = TaskTable
    return {
        "id": task.id,
        "user": T.user.value(
            href=url_for('user.user_show', user_id=task.user.id),
            title=task.user.name
        ),
        "name": task_impl.name,
        "type": task.type.name,
        "status": task.status.name,
        "parameters": format_parameters(asdict(task.parameters)),
        "errors": task.errors if task.errors is not None else list(),
        "due": datetime_format(task.due, default='', formatter=datetime_filter),
        "created": task.created.strftime("%Y-%m-%d %H:%M:%S"),
        "creator": T.creator.value(
            href=url_for('user.user_show', user_id=task.creator.id),
            title=task.creator.name
        ),
        'actions': [
            T.actions.single_value(
                href=url_for(
                    '.cancel_user_task',
                    task_id=task.id,
                    redirect=url_for('user.user_show', user_id=task.user.id, _anchor='tasks')
                ),
                title="Abbrechen",
                icon='fa-times',
                btn_class='btn-link'
            ),
            T.actions.single_value(
                href=url_for(
                    '.manually_execute_user_task',
                    task_id=task.id,
                    redirect=url_for('user.user_show', user_id=task.user.id, _anchor='tasks')
                ),
                title="Sofort ausführen",
                icon='fa-fast-forward',
                btn_class='btn-link'
            )
        ] if task.status == TaskStatus.OPEN else None,
    }

@bp.route("/user/<int:user_id>/json")
def json_tasks_for_user(user_id):
    user = get_user_or_404(user_id)

    return jsonify(items=[task_row(task) for task in user.tasks])

@bp.route("/user/json")
def json_user_tasks():
    failed_only = bool(request.args.get("failed_only", False))
    open_only = bool(request.args.get("open_only", False))

    tasks = Task.q.order_by(Task.status.desc(), Task.due.asc())\

    if failed_only:
        tasks = tasks.filter_by(status=TaskStatus.FAILED).all()
    elif open_only:
        tasks = tasks.filter_by(status=TaskStatus.OPEN).all()
    else:
        tasks = tasks.all()

    return jsonify(items=[task_row(task) for task in tasks])

def get_task_or_404(task_id) -> Union[Task, NoReturn]:
    if task := session.session.get(Task, task_id):
        return task
    abort(404)

@bp.route("/<int:task_id>/manually_execute")
@access.require('user_change')
def manually_execute_user_task(task_id: int):
    task = get_task_or_404(task_id)
    try:
        manually_execute_task(task, processor=current_user)
        session.session.commit()
    except Exception as e:
        if not isinstance(e, PycroftLibException):
            import logging
            logging.getLogger('pycroft.web').error(
                "Unexpected error in manual task execution: %s", e,
                exc_info=True
            )
        flash(f"Fehler bei der Ausführung: {e}", 'error')
        session.session.rollback()
    else:
        flash("Aufgabe erfolgreich ausgeführt", 'success')

    return redirect_or_404(request.args.get("redirect"))

@bp.route("/<int:task_id>/cancel")
@access.require('user_change')
def cancel_user_task(task_id):
    task = get_task_or_404(task_id)

    cancel_task(task, current_user)
    session.session.commit()

    flash(u'Aufgabe erfolgreich abgebrochen.', 'success')
    return redirect_or_404(request.args.get("redirect"))

@bp.route("/user")
@nav.navigate("Tasks")
def user_tasks():
    return render_template(
        "task/tasks.html",
        task_table=TaskTable(data_url=url_for('.json_user_tasks', open_only=1)),
        task_failed_table=TaskTable(data_url=url_for('.json_user_tasks', failed_only=1),
                                    sort_order='desc'),
        page_title="Aufgaben (Nutzer)"
    )
<|endoftext|>


<commit_before><commit_msg>for migrating the old model to the couch DB one for messagelog<commit_after>import hashlib
from django.core.management.base import LabelCommand
from corehq.apps.sms.models import MessageLogOld, MessageLog

class Command(LabelCommand):
    help = "Migrates old django-based sms message log model of 2010 to new couch based model of Nov. 2011."
    args = ""
    label = ""

    def handle(self, *labels, **options):
        old_django_messages = MessageLogOld.objects.all()

        print "Migrating MessageLog"
        for message in old_django_messages:
            new_key = hashlib.md5("%s %s %s %s %s %s" % (
                message.couch_recipient,
                message.domain,
                message.phone_number,
                message.direction,
                message.date,
                message.text
            )).hexdigest()
            try:
                couch_message = MessageLog(_id=new_key,
                                            couch_recipiet=message.couch_recipient,
                                            domain=message.domain,
                                            phone_number=message.phone_number,
                                            direction=message.direction,
                                            date=message.date,
                                            text=message.text)
                couch_message.save()
            except Exception as e:
                print "There was an error migrating MessageLog with text %s to couch_recipient %s in domain %s" % (
                    message.text,
                    message.couch_recipient,
                    message.domain
                )
<|endoftext|>


<commit_before>#!/usr/local/bin/python
"""
File       : historical_analytics.py
Author     : Bjorn Barrefors <bjorn dot peter dot barrefors AT cern dot ch>
Description: Collect historical data for popularity model
"""

# system modules
import os
import sys
import time

# package modules
from UADR.utils.config import get_config
from UADR.utils.utils import pop_db_timestamp_to_timestamp
from UADR.utils.utils import phedex_timestamp_to_timestamp
from UADR.utils.utils import bytes_to_gb
from UADR.utils.utils import timestamp_day
from UADR.utils.utils import timestamp_to_date
from UADR.services.phedex import PhEDExService
from UADR.services.pop_db import PopDBService

class HUA(object):
    """
    HUA or Historical User Analytics collects historical data on user behavior
    """
    def __init__(self, debug=0):
        self.config = get_config()
        self.phedex = PhEDExService(self.config, debug)
        self.pop_db = PopDBService(self.config, debug)
        self.data_path = '%s/data' % (os.environ.get('CUADRNT_ROOT'))

    def get_datasets(self):
        """
        Get all analysis datasets in CMS
        """
        # NOTE: We would like to get all datasets and have pattern for non analysis datasets
        dataset_names = set()
        api = 'blockReplicas'
        params = {'create_since':0, 'group':'AnalysisOps', 'show_dataset':'y'}
        json_data = self.phedex.fetch(api, params)
        datasets = json_data.get('phedex').get('dataset')
        for dataset in datasets:
            dataset_names.add(dataset.get('name'))
        return dataset_names

    def get_accesses(self, dataset_names):
        """
        Get accesses for all analysis datasets
        """
        # NOTE: For now we only use accesses as CPUH was not working for a while in pop db
        dataset_accesses = dict()
        test_dataset_re = re.compile('^/GenericTTbar/HC-.*/GEN-SIM-RECO$')
        yesterday = timestamp_to_date(time.time()-86400)
        api = 'DSStatInTimeWindow'
        params = {'sitename':'summary', 'tstart':'1999-01-01', 'tstop':yesterday}
        json_data = self.pop_db.fetch(api, params)
        for data in json_data.get('data')[0].get('data'):
            timestamp = pop_db_timestamp_to_timestamp(data[0])
            accesses = data[1]
            tmp_accesses[timestamp] = accesses
            dataset_accesses[dataset_name] = dataset_accesses
        return dataset_accesses

    def get_size_gb(self, dataset_names):
        """
        Get dataset size in GB for datasets
        """
        dataset_sizes = dict()
        api = 'data'
        for dataset_name in dataset_names:
            size_gb = 0
            params = {'dataset':dataset_name, 'level':'block', 'create_since':0}
            json_data = self.phedex.fetch(api, params)
            dataset = json_data.get('phedex').get('dataset')[0]
            for block in dataset.get('block'):
                size_gb += bytes_to_gb(block.get('bytes'))
            dataset_sizes[dataset_name] = size_gb
        return dataset_sizes

    def get_creation_date(self, dataset_names):
        """
        Get dataset size in GB for datasets
        """
        dataset_dates = dict()
        api = 'data'
        for dataset_name in dataset_names:
            params = {'dataset':dataset_name, 'level':'block', 'create_since':0}
            json_data = self.phedex.fetch(api, params)
            dataset = json_data.get('phedex').get('dataset')[0]
            dataset_dates[dataset_name] = phedex_timestamp_to_timestamp(dataset.get('time_create'))
        return dataset_dates

    def get_data_tier(self, dataset_names):
        """
        Get data tier of datasets
        """
        dataset_tiers = dict()
        for dataset_name in dataset_names:
            dataset_tiers[dataset_name] = dataset_name.split('/')[-1]
        return dataset_tiers

    def organize_data(self, headers, dataset_names, dataset_accesses, dataset_sizes, dataset_tiers, dataset_dates):
        """
        Organize data into the structure:
            data format: [(header_1, header_2, ...), (data_1, data_2, ...)]
        """
        data = []
        data.append(tuple(headers))
        date_stop = timestamp_day(int(time.time()))
        date_step = 86400
        for dataset_name in dataset_names:
            for timestamp in xrange(dataset_dates[dataset_name], date_stop, date_step):
                try:
                    accesses = dataset_accesses[dataset_name][timestamp]
                except Exception:
                    accesses = 0
                row = (dataset_name, timestamp_to_date(timestamp), accesses, dataset_sizes[dataset_name], dataset_tiers[dataset_name])
                data.append(row)
        return data

    def export(self, data):
        """
        Export data to file using csv format
        data format: [(header_1, header_2, ...), (data_1, data_2, ...)]
        """
        # FIXME: Get file path from config file
        export_file = '%s/historical_analytics.csv' % (self.data_path)
        fs = open(export_file)
        for row in data:
            text_row = ''
            for field in row:
                text_row += '%s,' % (field)
            text_row = text_row[:-1] + '\n'
            fs = open(export_file)
            fs.write(text_row, 'a')

def main():
    """
    Main driver for historical analytics
    """
    headers = ('dataset_name', 'date', 'accesses', 'size', 'data_tier')
    hua = HUA(debug=1)
    dataset_names = hua.get_datasets()
    dataset_accesses = hua.get_accesses(dataset_names)
    dataset_sizes = hua.get_size_gb(dataset_names)
    dataset_dates = hua.get_creation_date(dataset_names)
    dataset_tiers = hua.get_data_tier(dataset_names)
    data = hua.organize_data(headers, dataset_names, dataset_accesses, dataset_sizes, dataset_tiers, dataset_dates)
    hua.export(data)

if __name__ == "__main__":
    main()
    sys.exit(0)
<commit_msg>Changed how to get all datasets<commit_after>#!/usr/local/bin/python
"""
File       : historical_analytics.py
Author     : Bjorn Barrefors <bjorn dot peter dot barrefors AT cern dot ch>
Description: Collect historical data for popularity model
"""

# system modules
import os
import sys
import getopt
import re
import time

# package modules
from UADR.utils.config import get_config
from UADR.utils.utils import pop_db_timestamp_to_timestamp
from UADR.utils.utils import phedex_timestamp_to_timestamp
from UADR.utils.utils import bytes_to_gb
from UADR.utils.utils import timestamp_day
from UADR.utils.utils import timestamp_to_date
from UADR.services.phedex import PhEDExService
from UADR.services.pop_db import PopDBService

class HUA(object):
    """
    HUA or Historical User Analytics collects historical data on user behavior
    """
    def __init__(self, debug=0):
        self.config = get_config()
        self.phedex = PhEDExService(self.config, debug)
        self.pop_db = PopDBService(self.config, debug)
        self.data_path = '%s/data' % (os.environ.get('CUADRNT_ROOT'))

    def get_datasets(self):
        """
        Get all analysis datasets in CMS
        """
        # NOTE: We would like to get all datasets and have pattern for non analysis datasets
        dataset_names = set()
        valid_dataset_re = re.compile('/[0-9a-zA-Z\-_]+/[0-9a-zA-Z\-_]+/[0-9a-zA-Z\-_]+$')
        test_dataset_re = re.compile('^/GenericTTbar/HC-.*/GEN-SIM-RECO$')
        #user_dataset_re = re.compile('^.*/USER$')
        yesterday = timestamp_to_date(time.time()-86400)
        api = 'DSStatInTimeWindow'
        params = {'sitename':'summary', 'tstart':'1999-01-01', 'tstop':yesterday}
        json_data = self.pop_db.fetch(api, params)
        for dataset in json_data.get('DATA'):
            dataset_name = dataset.get('COLLNAME')
            if (test_dataset_re.match(dataset_name) or not valid_dataset_re.match(dataset_name)):
                continue
            dataset_names.add(dataset_name)
        print len(dataset_names)
        return dataset_names

    def get_accesses(self, dataset_names):
        """
        Get accesses for all analysis datasets
        """
        # NOTE: For now we only use accesses as CPUH was not working for a while in pop db
        dataset_accesses = dict()
        api = 'getSingleDSstat'
        for dataset_name in dataset_names:
            tmp_accesses = dict()
            params = {'sitename':'summary', 'aggr':'day', 'orderby':'naccess', 'name':dataset_name}
            json_data = self.pop_db.fetch(api, params)
            for data in json_data.get('data')[0].get('data'):
                timestamp = pop_db_timestamp_to_timestamp(data[0])
                accesses = data[1]
                tmp_accesses[timestamp] = accesses
            dataset_accesses[dataset_name] = dataset_accesses
        return dataset_accesses

    def get_size_gb(self, dataset_names):
        """
        Get dataset size in GB for datasets
        """
        dataset_sizes = dict()
        api = 'data'
        for dataset_name in dataset_names:
            size_gb = 0
            params = {'dataset':dataset_name, 'level':'block', 'create_since':0}
            json_data = self.phedex.fetch(api, params)
            dataset = json_data.get('phedex').get('dataset')[0]
            for block in dataset.get('block'):
                size_gb += bytes_to_gb(block.get('bytes'))
            dataset_sizes[dataset_name] = size_gb
        return dataset_sizes

    def get_creation_date(self, dataset_names):
        """
        Get dataset size in GB for datasets
        """
        dataset_dates = dict()
        api = 'data'
        for dataset_name in dataset_names:
            params = {'dataset':dataset_name, 'level':'block', 'create_since':0}
            json_data = self.phedex.fetch(api, params)
            dataset = json_data.get('phedex').get('dataset')[0]
            dataset_dates[dataset_name] = phedex_timestamp_to_timestamp(dataset.get('time_create'))
        return dataset_dates

    def get_data_tier(self, dataset_names):
        """
        Get data tier of datasets
        """
        dataset_tiers = dict()
        for dataset_name in dataset_names:
            dataset_tiers[dataset_name] = dataset_name.split('/')[-1]
        return dataset_tiers

    def organize_data(self, headers, dataset_names, dataset_accesses, dataset_sizes, dataset_tiers, dataset_dates):
        """
        Organize data into the structure:
            data format: [(header_1, header_2, ...), (data_1, data_2, ...)]
        """
        data = []
        data.append(tuple(headers))
        date_stop = timestamp_day(int(time.time()))
        date_step = 86400
        for dataset_name in dataset_names:
            for timestamp in xrange(dataset_dates[dataset_name], date_stop, date_step):
                try:
                    accesses = dataset_accesses[dataset_name][timestamp]
                except Exception:
                    accesses = 0
                row = (dataset_name, timestamp_to_date(timestamp), accesses, dataset_sizes[dataset_name], dataset_tiers[dataset_name])
                data.append(row)
        return data

    def export(self, data):
        """
        Export data to file using csv format
        data format: [(header_1, header_2, ...), (data_1, data_2, ...)]
        """
        # FIXME: Get file path from config file
        export_file = '%s/historical_analytics.csv' % (self.data_path)
        fs = open(export_file)
        for row in data:
            text_row = ''
            for field in row:
                text_row += '%s,' % (field)
            text_row = text_row[:-1] + '\n'
            fs = open(export_file)
            fs.write(text_row, 'a')

def main(argv):
    """
    Main driver for historical analytics
    """
    debug = 0
    try:
        opts, args = getopt.getopt(argv, 'hd', ['help', 'debug'])
    except getopt.GetoptError:
        print "historical_analytics.py [--debug]"
        sys.exit(2)
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print "historical_analytics.py [--debug]"
            sys.exit()
        elif opt in ('-d', '--debug'):
            debug = 1
    headers = ('dataset_name', 'date', 'accesses', 'size', 'data_tier')
    hua = HUA(debug=debug)
    dataset_names = hua.get_datasets()
    dataset_accesses = hua.get_accesses(dataset_names)
    dataset_sizes = hua.get_size_gb(dataset_names)
    dataset_dates = hua.get_creation_date(dataset_names)
    dataset_tiers = hua.get_data_tier(dataset_names)
    data = hua.organize_data(headers, dataset_names, dataset_accesses, dataset_sizes, dataset_tiers, dataset_dates)
    hua.export(data)

if __name__ == "__main__":
    # TODO: Add support to pass debug parameter
    main(sys.argv[1:])
    sys.exit()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess
import os
import os.path as p
from functools import partial
import platform as stdplatform

def detect_arch():
    arch = stdplatform.architecture()[0]
    # need to be a little more sneaky to check the platform on Windows:
    # http://stackoverflow.com/questions/2208828/detect-64bit-os-windows-in-python
    if host_platform == 'Windows':
        if 'APPVEYOR' in os.environ:
            av_platform = os.environ['PLATFORM']
            if av_platform == 'x86':
                arch = '32bit'
            elif av_platform == 'x64':
                arch = '64bit'
            else:
                print('Was unable to interpret the platform "{}"'.format())
    return arch

host_platform = stdplatform.system()
host_arch = detect_arch()

# define our commands
if host_platform == 'Windows':
    script_dir_name = 'Scripts'
    miniconda_installer_path = 'C:\miniconda.exe'
    miniconda_dir = p.expanduser('C:\Miniconda')
else:
    script_dir_name = 'bin'
    miniconda_installer_path = p.expanduser('~/miniconda.sh')
    miniconda_dir = p.expanduser('~/miniconda')

miniconda_script_dir = p.join(miniconda_dir, script_dir_name)
conda = p.join(miniconda_script_dir, 'conda')
binstar = p.join(miniconda_script_dir, 'binstar')
python = 'python'

# Amazingly, adding these causes conda-build to fail parsing yaml. :-|
#print('running on {} {}'.format(platform, arch))
#print('miniconda_installer_path is {}'.format(miniconda_installer_path))
#print('miniconda will be installed to {}'.format(miniconda_dir))

def url_for_platform_version(platform, py_version, arch):
    version = '3.6.0'
    base_url = 'http://repo.continuum.io/miniconda/Miniconda'
    platform_str = {'Linux': 'Linux',
                    'Darwin': 'MacOSX',
                    'Windows': 'Windows'}
    arch_str = {'64bit': 'x86_64',
                '32bit': 'x86'}
    ext = {'Linux': '.sh',
           'Darwin': '.sh',
           'Windows': '.exe'}

    if py_version == '3':
        base_url = base_url + py_version
    return '-'.join([base_url, version,
                     platform_str[platform],
                     arch_str[arch]]) + ext[platform]

# forward stderr to stdout
co = partial(subprocess.check_output, stderr=subprocess.STDOUT)
check = partial(subprocess.check_call, stderr=subprocess.STDOUT)

def execute(cmd, verbose=False):
    r""" Runs a command, printing the command and it's output to screen.
    """
    if verbose:
        print('> {}'.format(' '.join(cmd)))
    result = co(cmd)
    if verbose:
        print(result)
    return result

def execute_sequence(*cmds, **kwargs):
    r""" Execute a sequence of commands. If any fails, display an error.
    """
    verbose = kwargs.get('verbose', True)
    try:
        for cmd in cmds:
            execute(cmd, verbose)
    except subprocess.CalledProcessError as e:
        print(' -> {}'.format(e.output))
        raise e

def download_file(url, path_to_download):
    import urllib2
    f = urllib2.urlopen(url)
    with open(path_to_download, "wb") as fp:
        fp.write(f.read())

# BINSTAR LOGIN

def login():
    from binstar_client.utils import get_binstar
    return get_binstar()

class LetMeIn:
    def __init__(self, key):
        self.token = key

def login_with_key(key):
    from binstar_client.utils import get_binstar
    return get_binstar(args=LetMeIn(key))

# BINSTAR FILE PURGING

class BinstarFile(object):

    def __init__(self, full_name):
        self.full_name = full_name

    @property
    def user(self):
        return self.full_name.split('/')[0]

    @property
    def name(self):
        return self.full_name.split('/')[1]

    @property
    def basename(self):
        return '/'.join(self.full_name.split('/')[3:])

    @property
    def version(self):
        return self.full_name.split('/')[2]

    @property
    def platform(self):
        return self.full_name.replace('\\', '/').split('/')[3]

    @property
    def configuration(self):
        return self.full_name.replace('\\', '/').split('/')[4].split('-')[2].split('.')[0]

    def __str__(self):
        return self.full_name

    def __repr__(self):
        return self.full_name

    def all_info(self):
        s = ["         user: {}".format(self.user),
             "         name: {}".format(self.name),
             "     basename: {}".format(self.basename),
             "      version: {}".format(self.version),
             "     platform: {}".format(self.platform),
             "configuration: {}".format(self.configuration)]
        return "\n".join(s)

configuration_from_filename = lambda fn: fn.split('-')[-1].split('.')[0]
name_from_filename = lambda fn: fn.split('-')[0]
version_from_filename = lambda fn: fn.split('-')[1]
platform_from_filepath = lambda fp: p.split(p.split(fp)[0])[-1]
version_is_tag = lambda v: '_' not in v

def channels_for_user(b, user):
    return b.list_channels(user).keys()

def files_on_channel(b, user, channel):
    info = b.show_channel(channel, user)
    return [BinstarFile(i['full_name']) for i in info['files']]

def remove_file(b, bfile):
    b.remove_dist(bfile.user, bfile.name, bfile.version, bfile.basename)

def files_to_remove(b, user, channel, filepath):
    platform_ = platform_from_filepath(filepath)
    filename = p.split(filepath)[-1]
    name = name_from_filename(filename)
    version = version_from_filename(filename)
    configuration = configuration_from_filename(filename)
    # find all the files on this channel
    all_files = files_on_channel(b, user, channel)
    # other versions of this exact setup that are not tagged versions should
    # be removed
    return [f for f in all_files if
            f.name == name and
            f.configuration == configuration and
            f.platform == platform_ and
            f.version != version and
            not version_is_tag(f.version)]

def purge_old_files(b, user, channel, filepath):
    to_remove = files_to_remove(b, user, channel, filepath)
    print("Found {} releases to remove".format(len(to_remove)))
    for old_file in to_remove:
        print("Removing '{}'".format(old_file))
        remove_file(b, old_file)

# TRAVICONDA CONVIENIENCE FUNCTIONS

def acquire_miniconda(url, path_to_download):
    print('Downloading miniconda from {} to {}'.format(url, path_to_download))
    download_file(url, path_to_download)

def install_miniconda(path_to_installer, path_to_install):
    print('Installing miniconda to {}'.format(path_to_install))
    if host_platform == 'Windows':
        execute([path_to_installer, '/S', '/D={}'.format(path_to_install)])
    else:
        execute(['chmod', '+x', path_to_installer])
        execute([path_to_installer, '-b', '-p', path_to_install])

def setup_miniconda(python_version, channel=None):
    url = url_for_platform_version(host_platform, python_version, host_arch)
    print('Setting up miniconda from URL {}'.format(url))
    acquire_miniconda(url, miniconda_installer_path)
    install_miniconda(miniconda_installer_path, miniconda_dir)
    cmds = [[conda, 'update', '-q', '--yes', 'conda'],
            [conda, 'install', '-q', '--yes', 'conda-build', 'jinja2',
             'binstar']]
    if channel is not None:
        print("(adding channel '{}' for dependencies)".format(channel))
        cmds.append([conda, 'config', '--add', 'channels', channel])
    else:
        print("No channels have been configured (all dependencies have to be "
              "sourced from anaconda)")
    execute_sequence(*cmds)

def build(path):
    execute_sequence([conda, 'build', '-q', path])

def get_conda_build_path(path):
    from conda_build.metadata import MetaData
    from conda_build.build import bldpkg_path
    return bldpkg_path(MetaData(path))

def binstar_upload(key, user, channel, path):
    try:
        # TODO - could this safely be co? then we would get the binstar error..
        check([binstar, '-t', key, 'upload',
               '--force', '-u', user, '-c', channel, path])
    except subprocess.CalledProcessError as e:
        # mask the binstar key...
        cmd = e.cmd
        cmd[2] = 'BINSTAR_KEY'
        # ...then raise the error
        raise subprocess.CalledProcessError(e.returncode, cmd)

def build_upload_and_purge(path, user=None, key=None):
    print('Building package at path {}'.format(ns.path))
    # actually issue conda build
    build(path)
    if key is None:
        print('No binstar key provided')
    if user is None:
        print('No binstar user provided')
    if user is None or key is None:
        print('-> Unable to upload to binstar')
        return
    print('Have a user ({}) and key - can upload if suitable'.format(user))
    # decide if we should attempt an upload
    if resolve_can_upload_from_travis():
        channel = resolve_channel_from_travis_state()
        print("Fit to upload to channel '{}'".format(channel))
        upload_and_purge(key, user, channel, get_conda_build_path(path))
    else:
        print("Cannot upload to binstar - must be a PR.")

def upload_and_purge(key, user, channel, filepath):
    print('Uploading to {}/{}'.format(user, channel))
    binstar_upload(key, user, channel, filepath)
    b = login_with_key(key)
    if channel != 'main':
        print("Purging old releases from channel '{}'".format(channel))
        purge_old_files(b, user, channel, filepath)
    else:
        print("On main channel - no purging of releases will be done.")

def resolve_can_upload_from_travis():
    is_a_pr = os.environ['TRAVIS_PULL_REQUEST'] != 'false'
    can_upload = not is_a_pr
    print("Can we can upload? : {}".format(can_upload))
    return can_upload

def resolve_channel_from_travis_state():
    branch = os.environ['TRAVIS_BRANCH']
    tag = os.environ['TRAVIS_TAG']
    print('Travis branch is "{}"'.format(branch))
    print('Travis tag found is: "{}"'.format(tag))
    if tag != '' and branch == tag:
        # final release, channel is 'main'
        print("on a tagged release -> upload to 'main'")
        return 'main'
    else:
        print("not on a tag on master - "
              "just upload to the branch name {}".format(branch))
        return branch

def version_from_git_tags():
    return subprocess.check_output(
        ['git', 'describe', '--tags']).strip()[1:].replace('-', '_')

if __name__ == "__main__":
    from argparse import ArgumentParser
    parser = ArgumentParser(
        description=r"""
        Sets up miniconda, builds, and uploads to binstar on Travis CI.
        """)
    parser.add_argument("mode", choices=['setup', 'build', 'version'])
    parser.add_argument("--python", choices=['2', '3'])
    parser.add_argument("-c", "--channel",
                        help="binstar channel to activate "
                             "(setup only, optional)")
    parser.add_argument("--path",
                        help="path to the conda build scripts "
                             "(build only, required)")
    parser.add_argument("-u", "--user",
                        help="binstar user to upload to "
                             "(build only, required to upload)")
    parser.add_argument("-k", "--key", nargs='?', default=None,
                        help="The binstar key for uploading "
                             "(build only, required to upload)")
    ns = parser.parse_args()

    if ns.mode == 'setup':
        setup_miniconda(ns.python, channel=ns.channel)
    elif ns.mode == 'build':
        build_upload_and_purge(ns.path, user=ns.user, key=ns.key)
    else:
        print(version_from_git_tags())
<commit_msg>move to cleaner argparse<commit_after>#!/usr/bin/env python
import subprocess
import os
import os.path as p
from functools import partial
import platform as stdplatform

def detect_arch():
    arch = stdplatform.architecture()[0]
    # need to be a little more sneaky to check the platform on Windows:
    # http://stackoverflow.com/questions/2208828/detect-64bit-os-windows-in-python
    if host_platform == 'Windows':
        if 'APPVEYOR' in os.environ:
            av_platform = os.environ['PLATFORM']
            if av_platform == 'x86':
                arch = '32bit'
            elif av_platform == 'x64':
                arch = '64bit'
            else:
                print('Was unable to interpret the platform "{}"'.format())
    return arch

host_platform = stdplatform.system()
host_arch = detect_arch()

# define our commands
if host_platform == 'Windows':
    script_dir_name = 'Scripts'
    default_installer_path = 'C:\miniconda.exe'
    default_miniconda_dir = p.expanduser('C:\Miniconda')
else:
    script_dir_name = 'bin'
    default_installer_path = p.expanduser('~/miniconda.sh')
    default_miniconda_dir = p.expanduser('~/miniconda')

miniconda_script_dir = p.join(default_miniconda_dir, script_dir_name)
conda = p.join(miniconda_script_dir, 'conda')
binstar = p.join(miniconda_script_dir, 'binstar')
python = 'python'

# Amazingly, adding these causes conda-build to fail parsing yaml. :-|
#print('running on {} {}'.format(platform, arch))
#print('miniconda_installer_path is {}'.format(miniconda_installer_path))
#print('miniconda will be installed to {}'.format(miniconda_dir))

def url_for_platform_version(platform, py_version, arch):
    version = '3.6.0'
    base_url = 'http://repo.continuum.io/miniconda/Miniconda'
    platform_str = {'Linux': 'Linux',
                    'Darwin': 'MacOSX',
                    'Windows': 'Windows'}
    arch_str = {'64bit': 'x86_64',
                '32bit': 'x86'}
    ext = {'Linux': '.sh',
           'Darwin': '.sh',
           'Windows': '.exe'}

    if py_version == '3':
        base_url = base_url + py_version
    return '-'.join([base_url, version,
                     platform_str[platform],
                     arch_str[arch]]) + ext[platform]

# forward stderr to stdout
co = partial(subprocess.check_output, stderr=subprocess.STDOUT)
check = partial(subprocess.check_call, stderr=subprocess.STDOUT)

def execute(cmd, verbose=False):
    r""" Runs a command, printing the command and it's output to screen.
    """
    if verbose:
        print('> {}'.format(' '.join(cmd)))
    result = co(cmd)
    if verbose:
        print(result)
    return result

def execute_sequence(*cmds, **kwargs):
    r""" Execute a sequence of commands. If any fails, display an error.
    """
    verbose = kwargs.get('verbose', True)
    try:
        for cmd in cmds:
            execute(cmd, verbose)
    except subprocess.CalledProcessError as e:
        print(' -> {}'.format(e.output))
        raise e

def download_file(url, path_to_download):
    if p.exists(path_to_download):
        raise ValueError('Cannot download file to {} - '
                         'file exists'.format(path_to_download))
    import urllib2
    f = urllib2.urlopen(url)
    with open(path_to_download, "wb") as fp:
        fp.write(f.read())

# BINSTAR LOGIN

def login():
    from binstar_client.utils import get_binstar
    return get_binstar()

class LetMeIn:
    def __init__(self, key):
        self.token = key

def login_with_key(key):
    from binstar_client.utils import get_binstar
    return get_binstar(args=LetMeIn(key))

# BINSTAR FILE PURGING

class BinstarFile(object):

    def __init__(self, full_name):
        self.full_name = full_name

    @property
    def user(self):
        return self.full_name.split('/')[0]

    @property
    def name(self):
        return self.full_name.split('/')[1]

    @property
    def basename(self):
        return '/'.join(self.full_name.split('/')[3:])

    @property
    def version(self):
        return self.full_name.split('/')[2]

    @property
    def platform(self):
        return self.full_name.replace('\\', '/').split('/')[3]

    @property
    def configuration(self):
        return self.full_name.replace('\\', '/').split('/')[4].split('-')[2].split('.')[0]

    def __str__(self):
        return self.full_name

    def __repr__(self):
        return self.full_name

    def all_info(self):
        s = ["         user: {}".format(self.user),
             "         name: {}".format(self.name),
             "     basename: {}".format(self.basename),
             "      version: {}".format(self.version),
             "     platform: {}".format(self.platform),
             "configuration: {}".format(self.configuration)]
        return "\n".join(s)

configuration_from_filename = lambda fn: fn.split('-')[-1].split('.')[0]
name_from_filename = lambda fn: fn.split('-')[0]
version_from_filename = lambda fn: fn.split('-')[1]
platform_from_filepath = lambda fp: p.split(p.split(fp)[0])[-1]
version_is_tag = lambda v: '_' not in v

def channels_for_user(b, user):
    return b.list_channels(user).keys()

def files_on_channel(b, user, channel):
    info = b.show_channel(channel, user)
    return [BinstarFile(i['full_name']) for i in info['files']]

def remove_file(b, bfile):
    b.remove_dist(bfile.user, bfile.name, bfile.version, bfile.basename)

def files_to_remove(b, user, channel, filepath):
    platform_ = platform_from_filepath(filepath)
    filename = p.split(filepath)[-1]
    name = name_from_filename(filename)
    version = version_from_filename(filename)
    configuration = configuration_from_filename(filename)
    # find all the files on this channel
    all_files = files_on_channel(b, user, channel)
    # other versions of this exact setup that are not tagged versions should
    # be removed
    return [f for f in all_files if
            f.name == name and
            f.configuration == configuration and
            f.platform == platform_ and
            f.version != version and
            not version_is_tag(f.version)]

def purge_old_files(b, user, channel, filepath):
    to_remove = files_to_remove(b, user, channel, filepath)
    print("Found {} releases to remove".format(len(to_remove)))
    for old_file in to_remove:
        print("Removing '{}'".format(old_file))
        remove_file(b, old_file)

# TRAVICONDA CONVIENIENCE FUNCTIONS

def acquire_miniconda(url, path_to_download):
    print('Downloading miniconda from {} to {}'.format(url, path_to_download))
    download_file(url, path_to_download)

def install_miniconda(path_to_installer, path_to_install):
    print('Installing miniconda to {}'.format(path_to_install))
    if host_platform == 'Windows':
        execute([path_to_installer, '/S', '/D={}'.format(path_to_install)])
    else:
        execute(['chmod', '+x', path_to_installer])
        execute([path_to_installer, '-b', '-p', path_to_install])

def setup_miniconda(python_version, installation_path, channel=None):
    url = url_for_platform_version(host_platform, python_version, host_arch)
    print('Setting up miniconda from URL {}'.format(url))
    print("(Installing to '{}')".format(installation_path))
    acquire_miniconda(url, default_installer_path)
    install_miniconda(default_installer_path, installation_path)
    cmds = [[conda, 'update', '-q', '--yes', 'conda'],
            [conda, 'install', '-q', '--yes', 'conda-build', 'jinja2',
             'binstar']]
    if channel is not None:
        print("(adding channel '{}' for dependencies)".format(channel))
        cmds.append([conda, 'config', '--add', 'channels', channel])
    else:
        print("No channels have been configured (all dependencies have to be "
              "sourced from anaconda)")
    execute_sequence(*cmds)

def build(path):
    execute_sequence([conda, 'build', '-q', path])

def get_conda_build_path(path):
    from conda_build.metadata import MetaData
    from conda_build.build import bldpkg_path
    return bldpkg_path(MetaData(path))

def binstar_upload(key, user, channel, path):
    try:
        # TODO - could this safely be co? then we would get the binstar error..
        check([binstar, '-t', key, 'upload',
               '--force', '-u', user, '-c', channel, path])
    except subprocess.CalledProcessError as e:
        # mask the binstar key...
        cmd = e.cmd
        cmd[2] = 'BINSTAR_KEY'
        # ...then raise the error
        raise subprocess.CalledProcessError(e.returncode, cmd)

def build_upload_and_purge(path, user=None, key=None):
    print('Building package at path {}'.format(ns.path))
    # actually issue conda build
    build(path)
    if key is None:
        print('No binstar key provided')
    if user is None:
        print('No binstar user provided')
    if user is None or key is None:
        print('-> Unable to upload to binstar')
        return
    print('Have a user ({}) and key - can upload if suitable'.format(user))
    # decide if we should attempt an upload
    if resolve_can_upload_from_travis():
        channel = resolve_channel_from_travis_state()
        print("Fit to upload to channel '{}'".format(channel))
        upload_and_purge(key, user, channel, get_conda_build_path(path))
    else:
        print("Cannot upload to binstar - must be a PR.")

def upload_and_purge(key, user, channel, filepath):
    print('Uploading to {}/{}'.format(user, channel))
    binstar_upload(key, user, channel, filepath)
    b = login_with_key(key)
    if channel != 'main':
        print("Purging old releases from channel '{}'".format(channel))
        purge_old_files(b, user, channel, filepath)
    else:
        print("On main channel - no purging of releases will be done.")

def resolve_can_upload_from_travis():
    is_a_pr = os.environ['TRAVIS_PULL_REQUEST'] != 'false'
    can_upload = not is_a_pr
    print("Can we can upload? : {}".format(can_upload))
    return can_upload

def resolve_channel_from_travis_state():
    branch = os.environ['TRAVIS_BRANCH']
    tag = os.environ['TRAVIS_TAG']
    print('Travis branch is "{}"'.format(branch))
    print('Travis tag found is: "{}"'.format(tag))
    if tag != '' and branch == tag:
        # final release, channel is 'main'
        print("on a tagged release -> upload to 'main'")
        return 'main'
    else:
        print("not on a tag on master - "
              "just upload to the branch name {}".format(branch))
        return branch

def version_from_git_tags():
    return subprocess.check_output(
        ['git', 'describe', '--tags']).strip()[1:].replace('-', '_')

def setup_cmd(ns):
    print ns
    if ns.path is None:
        path = default_miniconda_dir
    else:
        path = ns.path
    setup_miniconda(ns.python, path, channel=ns.channel)

def build_cmd(ns):
    build_upload_and_purge(ns.path, user=ns.user, key=ns.key)

def upload_cmd(args):
    print('upload being called with args: {}'.format(args))

def version_cmd(_):
    print(version_from_git_tags())

if __name__ == "__main__":
    from argparse import ArgumentParser
    parser = ArgumentParser(
        description=r"""
        Sets up miniconda, builds, and uploads to binstar on Travis CI.
        """)
    subp = parser.add_subparsers()

    sp = subp.add_parser('setup', help='setup a miniconda environment')
    sp.add_argument("python", choices=['2', '3'])
    sp.add_argument('-p', '--path', help='The path to install miniconda to. '
                                         'If not provided defaults to {'
                                         '}'.format(default_miniconda_dir))
    sp.add_argument("-c", "--channel",
                    help="binstar channel to activate")
    sp.set_defaults(func=setup_cmd)

    bp = subp.add_parser('build', help='run a conda build')
    bp.add_argument("path", help="path to the conda build scripts")
    bp.set_defaults(func=build_cmd)

    up = subp.add_parser('upload', help='upload a conda build to binstar')
    up.add_argument('path', help='path to the conda build scripts')
    up.add_argument('user', help='Binstar user(or organisation) to upload to')
    up.add_argument('key', help='Binstar API key to use for uploading')
    up.set_defaults(func=upload_cmd)

    vp = subp.add_parser('version', help='print the version as reported by '
                                         'git')
    vp.set_defaults(func=version_cmd)

    args = parser.parse_args()
    args.func(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""POSIX specific tests.  These are implicitly run by test_psutil.py."""

import datetime
import os
import subprocess
import sys
import time

import psutil

from psutil._compat import PY3, callable
from test_psutil import LINUX, SUNOS, OSX, BSD, PYTHON, POSIX
from test_psutil import (get_test_subprocess, skip_on_access_denied,
                         retry_before_failing, reap_children, sh, unittest,
                         get_kernel_version, wait_for_pid)

def ps(cmd):
    """Expects a ps command with a -o argument and parse the result
    returning only the value of interest.
    """
    if not LINUX:
        cmd = cmd.replace(" --no-headers ", " ")
    if SUNOS:
        cmd = cmd.replace("-o command", "-o comm")
        cmd = cmd.replace("-o start", "-o stime")
    p = subprocess.Popen(cmd, shell=1, stdout=subprocess.PIPE)
    output = p.communicate()[0].strip()
    if PY3:
        output = str(output, sys.stdout.encoding)
    if not LINUX:
        output = output.split('\n')[1].strip()
    try:
        return int(output)
    except ValueError:
        return output

@unittest.skipUnless(POSIX, "not a POSIX system")
class PosixSpecificTestCase(unittest.TestCase):
    """Compare psutil results against 'ps' command line utility."""

    @classmethod
    def setUpClass(cls):
        cls.pid = get_test_subprocess([PYTHON, "-E", "-O"],
                                      stdin=subprocess.PIPE).pid
        wait_for_pid(cls.pid)

    @classmethod
    def tearDownClass(cls):
        reap_children()

    # for ps -o arguments see: http://unixhelp.ed.ac.uk/CGI/man-cgi?ps

    def test_process_parent_pid(self):
        ppid_ps = ps("ps --no-headers -o ppid -p %s" % self.pid)
        ppid_psutil = psutil.Process(self.pid).ppid()
        self.assertEqual(ppid_ps, ppid_psutil)

    def test_process_uid(self):
        uid_ps = ps("ps --no-headers -o uid -p %s" % self.pid)
        uid_psutil = psutil.Process(self.pid).uids().real
        self.assertEqual(uid_ps, uid_psutil)

    def test_process_gid(self):
        gid_ps = ps("ps --no-headers -o rgid -p %s" % self.pid)
        gid_psutil = psutil.Process(self.pid).gids().real
        self.assertEqual(gid_ps, gid_psutil)

    def test_process_username(self):
        username_ps = ps("ps --no-headers -o user -p %s" % self.pid)
        username_psutil = psutil.Process(self.pid).username()
        self.assertEqual(username_ps, username_psutil)

    @skip_on_access_denied()
    @retry_before_failing()
    def test_process_rss_memory(self):
        # give python interpreter some time to properly initialize
        # so that the results are the same
        time.sleep(0.1)
        rss_ps = ps("ps --no-headers -o rss -p %s" % self.pid)
        rss_psutil = psutil.Process(self.pid).memory_info()[0] / 1024
        self.assertEqual(rss_ps, rss_psutil)

    @skip_on_access_denied()
    @retry_before_failing()
    def test_process_vsz_memory(self):
        # give python interpreter some time to properly initialize
        # so that the results are the same
        time.sleep(0.1)
        vsz_ps = ps("ps --no-headers -o vsz -p %s" % self.pid)
        vsz_psutil = psutil.Process(self.pid).memory_info()[1] / 1024
        self.assertEqual(vsz_ps, vsz_psutil)

    def test_process_name(self):
        # use command + arg since "comm" keyword not supported on all platforms
        name_ps = ps("ps --no-headers -o command -p %s" % (
            self.pid)).split(' ')[0]
        # remove path if there is any, from the command
        name_ps = os.path.basename(name_ps).lower()
        name_psutil = psutil.Process(self.pid).name().lower()
        self.assertEqual(name_ps, name_psutil)

    @unittest.skipIf(OSX or BSD,
                     'ps -o start not available')
    def test_process_create_time(self):
        time_ps = ps("ps --no-headers -o start -p %s" % self.pid).split(' ')[0]
        time_psutil = psutil.Process(self.pid).create_time()
        time_psutil_tstamp = datetime.datetime.fromtimestamp(
            time_psutil).strftime("%H:%M:%S")
        # sometimes ps shows the time rounded up instead of down, so we check
        # for both possible values
        round_time_psutil = round(time_psutil)
        round_time_psutil_tstamp = datetime.datetime.fromtimestamp(
            round_time_psutil).strftime("%H:%M:%S")
        self.assertIn(time_ps, [time_psutil_tstamp, round_time_psutil_tstamp])

    def test_process_exe(self):
        ps_pathname = ps("ps --no-headers -o command -p %s" %
                         self.pid).split(' ')[0]
        psutil_pathname = psutil.Process(self.pid).exe()
        try:
            self.assertEqual(ps_pathname, psutil_pathname)
        except AssertionError:
            # certain platforms such as BSD are more accurate returning:
            # "/usr/local/bin/python2.7"
            # ...instead of:
            # "/usr/local/bin/python"
            # We do not want to consider this difference in accuracy
            # an error.
            adjusted_ps_pathname = ps_pathname[:len(ps_pathname)]
            self.assertEqual(ps_pathname, adjusted_ps_pathname)

    def test_process_cmdline(self):
        ps_cmdline = ps("ps --no-headers -o command -p %s" % self.pid)
        psutil_cmdline = " ".join(psutil.Process(self.pid).cmdline())
        if SUNOS:
            # ps on Solaris only shows the first part of the cmdline
            psutil_cmdline = psutil_cmdline.split(" ")[0]
        self.assertEqual(ps_cmdline, psutil_cmdline)

    @retry_before_failing()
    def test_pids(self):
        # Note: this test might fail if the OS is starting/killing
        # other processes in the meantime
        if SUNOS:
            cmd = ["ps", "ax"]
        else:
            cmd = ["ps", "ax", "-o", "pid"]
        p = get_test_subprocess(cmd, stdout=subprocess.PIPE)
        output = p.communicate()[0].strip()
        if PY3:
            output = str(output, sys.stdout.encoding)
        pids_ps = []
        for line in output.split('\n')[1:]:
            if line:
                pid = int(line.split()[0].strip())
                pids_ps.append(pid)
        # remove ps subprocess pid which is supposed to be dead in meantime
        pids_ps.remove(p.pid)
        pids_psutil = psutil.pids()
        pids_ps.sort()
        pids_psutil.sort()

        # on OSX ps doesn't show pid 0
        if OSX and 0 not in pids_ps:
            pids_ps.insert(0, 0)

        if pids_ps != pids_psutil:
            difference = [x for x in pids_psutil if x not in pids_ps] + \
                         [x for x in pids_ps if x not in pids_psutil]
            self.fail("difference: " + str(difference))

    # for some reason ifconfig -a does not report differente interfaces
    # psutil does
    @unittest.skipIf(SUNOS, "test not reliable on SUNOS")
    def test_nic_names(self):
        p = subprocess.Popen("ifconfig -a", shell=1, stdout=subprocess.PIPE)
        output = p.communicate()[0].strip()
        if PY3:
            output = str(output, sys.stdout.encoding)
        for nic in psutil.net_io_counters(pernic=True).keys():
            for line in output.split():
                if line.startswith(nic):
                    break
            else:
                self.fail("couldn't find %s nic in 'ifconfig -a' output" % nic)

    @retry_before_failing()
    def test_users(self):
        out = sh("who")
        lines = out.split('\n')
        users = [x.split()[0] for x in lines]
        self.assertEqual(len(users), len(psutil.users()))
        terminals = [x.split()[1] for x in lines]
        for u in psutil.users():
            self.assertTrue(u.name in users, u.name)
            self.assertTrue(u.terminal in terminals, u.terminal)

    def test_fds_open(self):
        # Note: this fails from time to time; I'm keen on thinking
        # it doesn't mean something is broken
        def call(p, attr):
            args = ()
            attr = getattr(p, name, None)
            if attr is not None and callable(attr):
                if name == 'rlimit':
                    args = (psutil.RLIMIT_NOFILE,)
                attr(*args)
            else:
                attr

        p = psutil.Process(os.getpid())
        failures = []
        ignored_names = ['terminate', 'kill', 'suspend', 'resume', 'nice',
                         'send_signal', 'wait', 'children', 'as_dict']
        if LINUX and get_kernel_version() < (2, 6, 36):
            ignored_names.append('rlimit')
        if LINUX and get_kernel_version() < (2, 6, 23):
            ignored_names.append('num_ctx_switches')
        for name in dir(psutil.Process):
            if (name.startswith('_') or name in ignored_names):
                continue
            else:
                try:
                    num1 = p.num_fds()
                    for x in range(2):
                        call(p, name)
                    num2 = p.num_fds()
                except psutil.AccessDenied:
                    pass
                else:
                    if abs(num2 - num1) > 1:
                        fail = "failure while processing Process.%s method " \
                               "(before=%s, after=%s)" % (name, num1, num2)
                        failures.append(fail)
        if failures:
            self.fail('\n' + '\n'.join(failures))

def main():
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.makeSuite(PosixSpecificTestCase))
    result = unittest.TextTestRunner(verbosity=2).run(test_suite)
    return result.wasSuccessful()

if __name__ == '__main__':
    if not main():
        sys.exit(1)
<commit_msg>better msg failure<commit_after>#!/usr/bin/env python

# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""POSIX specific tests.  These are implicitly run by test_psutil.py."""

import datetime
import os
import subprocess
import sys
import time

import psutil

from psutil._compat import PY3, callable
from test_psutil import LINUX, SUNOS, OSX, BSD, PYTHON, POSIX
from test_psutil import (get_test_subprocess, skip_on_access_denied,
                         retry_before_failing, reap_children, sh, unittest,
                         get_kernel_version, wait_for_pid)

def ps(cmd):
    """Expects a ps command with a -o argument and parse the result
    returning only the value of interest.
    """
    if not LINUX:
        cmd = cmd.replace(" --no-headers ", " ")
    if SUNOS:
        cmd = cmd.replace("-o command", "-o comm")
        cmd = cmd.replace("-o start", "-o stime")
    p = subprocess.Popen(cmd, shell=1, stdout=subprocess.PIPE)
    output = p.communicate()[0].strip()
    if PY3:
        output = str(output, sys.stdout.encoding)
    if not LINUX:
        output = output.split('\n')[1].strip()
    try:
        return int(output)
    except ValueError:
        return output

@unittest.skipUnless(POSIX, "not a POSIX system")
class PosixSpecificTestCase(unittest.TestCase):
    """Compare psutil results against 'ps' command line utility."""

    @classmethod
    def setUpClass(cls):
        cls.pid = get_test_subprocess([PYTHON, "-E", "-O"],
                                      stdin=subprocess.PIPE).pid
        wait_for_pid(cls.pid)

    @classmethod
    def tearDownClass(cls):
        reap_children()

    # for ps -o arguments see: http://unixhelp.ed.ac.uk/CGI/man-cgi?ps

    def test_process_parent_pid(self):
        ppid_ps = ps("ps --no-headers -o ppid -p %s" % self.pid)
        ppid_psutil = psutil.Process(self.pid).ppid()
        self.assertEqual(ppid_ps, ppid_psutil)

    def test_process_uid(self):
        uid_ps = ps("ps --no-headers -o uid -p %s" % self.pid)
        uid_psutil = psutil.Process(self.pid).uids().real
        self.assertEqual(uid_ps, uid_psutil)

    def test_process_gid(self):
        gid_ps = ps("ps --no-headers -o rgid -p %s" % self.pid)
        gid_psutil = psutil.Process(self.pid).gids().real
        self.assertEqual(gid_ps, gid_psutil)

    def test_process_username(self):
        username_ps = ps("ps --no-headers -o user -p %s" % self.pid)
        username_psutil = psutil.Process(self.pid).username()
        self.assertEqual(username_ps, username_psutil)

    @skip_on_access_denied()
    @retry_before_failing()
    def test_process_rss_memory(self):
        # give python interpreter some time to properly initialize
        # so that the results are the same
        time.sleep(0.1)
        rss_ps = ps("ps --no-headers -o rss -p %s" % self.pid)
        rss_psutil = psutil.Process(self.pid).memory_info()[0] / 1024
        self.assertEqual(rss_ps, rss_psutil)

    @skip_on_access_denied()
    @retry_before_failing()
    def test_process_vsz_memory(self):
        # give python interpreter some time to properly initialize
        # so that the results are the same
        time.sleep(0.1)
        vsz_ps = ps("ps --no-headers -o vsz -p %s" % self.pid)
        vsz_psutil = psutil.Process(self.pid).memory_info()[1] / 1024
        self.assertEqual(vsz_ps, vsz_psutil)

    def test_process_name(self):
        # use command + arg since "comm" keyword not supported on all platforms
        name_ps = ps("ps --no-headers -o command -p %s" % (
            self.pid)).split(' ')[0]
        # remove path if there is any, from the command
        name_ps = os.path.basename(name_ps).lower()
        name_psutil = psutil.Process(self.pid).name().lower()
        self.assertEqual(name_ps, name_psutil)

    @unittest.skipIf(OSX or BSD,
                     'ps -o start not available')
    def test_process_create_time(self):
        time_ps = ps("ps --no-headers -o start -p %s" % self.pid).split(' ')[0]
        time_psutil = psutil.Process(self.pid).create_time()
        time_psutil_tstamp = datetime.datetime.fromtimestamp(
            time_psutil).strftime("%H:%M:%S")
        # sometimes ps shows the time rounded up instead of down, so we check
        # for both possible values
        round_time_psutil = round(time_psutil)
        round_time_psutil_tstamp = datetime.datetime.fromtimestamp(
            round_time_psutil).strftime("%H:%M:%S")
        self.assertIn(time_ps, [time_psutil_tstamp, round_time_psutil_tstamp])

    def test_process_exe(self):
        ps_pathname = ps("ps --no-headers -o command -p %s" %
                         self.pid).split(' ')[0]
        psutil_pathname = psutil.Process(self.pid).exe()
        try:
            self.assertEqual(ps_pathname, psutil_pathname)
        except AssertionError:
            # certain platforms such as BSD are more accurate returning:
            # "/usr/local/bin/python2.7"
            # ...instead of:
            # "/usr/local/bin/python"
            # We do not want to consider this difference in accuracy
            # an error.
            adjusted_ps_pathname = ps_pathname[:len(ps_pathname)]
            self.assertEqual(ps_pathname, adjusted_ps_pathname)

    def test_process_cmdline(self):
        ps_cmdline = ps("ps --no-headers -o command -p %s" % self.pid)
        psutil_cmdline = " ".join(psutil.Process(self.pid).cmdline())
        if SUNOS:
            # ps on Solaris only shows the first part of the cmdline
            psutil_cmdline = psutil_cmdline.split(" ")[0]
        self.assertEqual(ps_cmdline, psutil_cmdline)

    @retry_before_failing()
    def test_pids(self):
        # Note: this test might fail if the OS is starting/killing
        # other processes in the meantime
        if SUNOS:
            cmd = ["ps", "ax"]
        else:
            cmd = ["ps", "ax", "-o", "pid"]
        p = get_test_subprocess(cmd, stdout=subprocess.PIPE)
        output = p.communicate()[0].strip()
        if PY3:
            output = str(output, sys.stdout.encoding)
        pids_ps = []
        for line in output.split('\n')[1:]:
            if line:
                pid = int(line.split()[0].strip())
                pids_ps.append(pid)
        # remove ps subprocess pid which is supposed to be dead in meantime
        pids_ps.remove(p.pid)
        pids_psutil = psutil.pids()
        pids_ps.sort()
        pids_psutil.sort()

        # on OSX ps doesn't show pid 0
        if OSX and 0 not in pids_ps:
            pids_ps.insert(0, 0)

        if pids_ps != pids_psutil:
            difference = [x for x in pids_psutil if x not in pids_ps] + \
                         [x for x in pids_ps if x not in pids_psutil]
            self.fail("difference: " + str(difference))

    # for some reason ifconfig -a does not report differente interfaces
    # psutil does
    @unittest.skipIf(SUNOS, "test not reliable on SUNOS")
    def test_nic_names(self):
        p = subprocess.Popen("ifconfig -a", shell=1, stdout=subprocess.PIPE)
        output = p.communicate()[0].strip()
        if PY3:
            output = str(output, sys.stdout.encoding)
        for nic in psutil.net_io_counters(pernic=True).keys():
            for line in output.split():
                if line.startswith(nic):
                    break
            else:
                self.fail(
                    "couldn't find %s nic in 'ifconfig -a' output\n%s" % (
                        nic, output))

    @retry_before_failing()
    def test_users(self):
        out = sh("who")
        lines = out.split('\n')
        users = [x.split()[0] for x in lines]
        self.assertEqual(len(users), len(psutil.users()))
        terminals = [x.split()[1] for x in lines]
        for u in psutil.users():
            self.assertTrue(u.name in users, u.name)
            self.assertTrue(u.terminal in terminals, u.terminal)

    def test_fds_open(self):
        # Note: this fails from time to time; I'm keen on thinking
        # it doesn't mean something is broken
        def call(p, attr):
            args = ()
            attr = getattr(p, name, None)
            if attr is not None and callable(attr):
                if name == 'rlimit':
                    args = (psutil.RLIMIT_NOFILE,)
                attr(*args)
            else:
                attr

        p = psutil.Process(os.getpid())
        failures = []
        ignored_names = ['terminate', 'kill', 'suspend', 'resume', 'nice',
                         'send_signal', 'wait', 'children', 'as_dict']
        if LINUX and get_kernel_version() < (2, 6, 36):
            ignored_names.append('rlimit')
        if LINUX and get_kernel_version() < (2, 6, 23):
            ignored_names.append('num_ctx_switches')
        for name in dir(psutil.Process):
            if (name.startswith('_') or name in ignored_names):
                continue
            else:
                try:
                    num1 = p.num_fds()
                    for x in range(2):
                        call(p, name)
                    num2 = p.num_fds()
                except psutil.AccessDenied:
                    pass
                else:
                    if abs(num2 - num1) > 1:
                        fail = "failure while processing Process.%s method " \
                               "(before=%s, after=%s)" % (name, num1, num2)
                        failures.append(fail)
        if failures:
            self.fail('\n' + '\n'.join(failures))

def main():
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.makeSuite(PosixSpecificTestCase))
    result = unittest.TextTestRunner(verbosity=2).run(test_suite)
    return result.wasSuccessful()

if __name__ == '__main__':
    if not main():
        sys.exit(1)
<|endoftext|>


<commit_before>import buildTopology
#import Mpls_snmp
from SNMP_utilization_src import *

#####################################
#find method are for private use
#get method are for public call
#####################################

class Manager:

    def __init__(self, anchorIp, cs):
        #main
        self.anchorIp = anchorIp
        self.communityString = cs

        #topology
        self.topologyMatrix = None
        self.listInfo = None

        #utilization

        #tunnel
        #key search are ip
        self.confTunnelsDictionary = {}
        self.lspTableDictionary = {}

    ###############TOPOLOGY
    def findTopology(self):
        """find topology and save list of routers and matrix"""
        self.listInfo, self.topologyMatrix = buildTopology.getTopology(self.anchorIp)

    def getListIP(self):
        """return list of ip if not present call function to populate data"""
        listIp = []
        if self.listInfo is None:
            #unallocated memory call find function
            self.findTopology()
        for i in range(0, len(self.listInfo)):
            listIp.append(self.listInfo[i]['routerId'])
        return listIp

    def getTopology(self):
        """return topology if not present call function to populate data"""
        if self.topologyMatrix is None:
            #unallocated memory call find function
            self.findTopology()
        return self.topologyMatrix

    ###############UTILIZATION
    def findUtilization(self, addr):
        """if the output is None some error occurred"""
        index = None
        for k in range(0, len(self.listInfo)):
            if self.listInfo[k]['routerId'] == addr:
                index = k
        if index is None:
            return None
        #print(index)
        a = []
        a.append(addr)
        r = getRouterInfo.get_routers_list(a, self.communityString)
        ifs = r[0].get_interfaces()
        for i in range(0, len(ifs)):
            ip = ifs[i].get_address_if()
            for j in range(0, self.listInfo[index]['nRoutes']):
                if ip == self.listInfo[index][str(j) + '_ip']:
                    #match
                    #add new info
                    self.listInfo[index][str(j) + '_name'] = ifs[i].get_name()
                    self.listInfo[index][str(j) + '_id'] = ifs[i].get_id()
                    self.listInfo[index][str(j) + '_speed'] = ifs[i].get_if_speed()
                    self.listInfo[index][str(j) + '_utilization'] = ifs[i].get_in_out_utilization()

    #def getUtilization():
    #def getAllUtilization():

    ###############TUNNEL
    #def findTunnel(ip, communityString):
        #t = TeTunnels(ip, communityString)
        #t.start()

        #self.confTunnelsDictionary[ip] = t.getConfTunnels()
        #self.lspTableDictionary[ip] = t.getLspTable()

        ##For each tunnel configuration:
        ##for name in confTunnels.keys():
        ##    print name, confTunnels[name].getAttributeDict()

        ##For each LSP instance
        ##for name in Lsp.keys():
        ##    print name, Lsp[name].getAttributeDict()

    #def getTunnel(ip, communityString):
        #if (self.confTunnelsDictionary.has_key(ip) or self.lspTableDictionary.has_key(ip)) is False:
            #findTunnel(ip, communityString)
        #return self.confTunnelsDictionary[ip], self.lspTableDictionary[ip]

    ###############GRAPH
    #def getGraph():<commit_msg>working utilization TODO output in standard form<commit_after>import buildTopology
from Mpls_snmp import *
from SNMP_utilization_src import *

#####################################
#find method are for private use
#get method are for public call
#####################################

class Manager:

    def __init__(self, anchorIp, cs):
        #main
        self.anchorIp = anchorIp
        self.communityString = cs

        #topology
        self.topologyMatrix = None
        self.listInfo = None

        #utilization

        #tunnel
        #key search are ip
        self.confTunnelsDictionary = {}
        self.lspTableDictionary = {}

    ###############UTILITY
    def returnIndex(self, addr):
        for k in range(0, len(self.listInfo)):
            if self.listInfo[k]['routerId'] == addr:
                return k
        return None

    ###############TOPOLOGY
    def findTopology(self):
        """find topology and save list of routers and matrix"""
        self.listInfo, self.topologyMatrix = buildTopology.getTopology(self.anchorIp)

    def getListIP(self):
        """return list of ip if not present call function to populate data"""
        listIp = []
        if self.listInfo is None:
            #unallocated memory call find function
            self.findTopology()
        for i in range(0, len(self.listInfo)):
            listIp.append(self.listInfo[i]['routerId'])
        return listIp

    def getTopology(self):
        """return topology if not present call function to populate data"""
        if self.topologyMatrix is None:
            #unallocated memory call find function
            self.findTopology()
        return self.topologyMatrix

    ###############UTILIZATION
    def findUtilization(self, addr):
        """if the output is None some error occurred"""
        index = self.returnIndex(addr)
        if index is None:
            return None

        a = []
        a.append(addr)
        r = getRouterInfo.get_routers_list(a, self.communityString)
        ifs = r[0].get_interfaces()
        for i in range(0, len(ifs)):
            ip = ifs[i].get_address_if()
            for j in range(0, self.listInfo[index]['nRoutes']):
                if ip == self.listInfo[index][str(j) + '_ip']:
                    #match
                    #add new info
                    self.listInfo[index][str(j) + '_name'] = ifs[i].get_name()
                    self.listInfo[index][str(j) + '_id'] = ifs[i].get_id()
                    self.listInfo[index][str(j) + '_speed'] = ifs[i].get_if_speed()
                    self.listInfo[index][str(j) + '_utilization'] = ifs[i].get_in_out_utilization()

    def getUtilization(self, addr):
        """return utilization for the current ip if return None some error occurred"""
        if self.listInfo is None:
            self.findTopology()
            res = self.findUtilization(addr)
            if res is None:
                return None

        index = self.returnIndex(addr)
        if index is None:
            return None

        if ('0_name') not in list(listInfo[index].keys()):
            #it's only needed to test if one name is present
            res = self.findUtilization(addr)
            if res is None:
                return None
        #return utilization list
    #def getAllUtilization():

    ###############TUNNEL
    #def findTunnel(self, ip):
        #t = TeTunnels.TeTunnels(ip, self.communityString)
        #t.start()

        #self.confTunnelsDictionary[ip] = t.getConfTunnels()
        #self.lspTableDictionary[ip] = t.getLspTable()

        ##For each tunnel configuration:
        ##for name in confTunnels.keys():
        ##    print name, confTunnels[name].getAttributeDict()

        ##For each LSP instance
        ##for name in Lsp.keys():
        ##    print name, Lsp[name].getAttributeDict()

    #def getTunnel(ip, communityString):
        #if (ip in self.confTunnelsDictionary or ip in self.lspTableDictionary) is False:
            #findTunnel(ip, communityString)
        #return self.confTunnelsDictionary[ip], self.lspTableDictionary[ip]

    ###############GRAPH
    #def getGraph():<|endoftext|>