<commit_before>import argparse

import numpy as np
import pandas as pd

from django.core.management import BaseCommand

from cl.people_db.import_judges.populate_state_judges import make_state_judge
from cl.people_db.import_judges.populate_fjc_judges import make_federal_judge

class Command(BaseCommand):
    help = 'Import data from the SCDB Case Centered CSV.'

    def valid_actions(self, s):
        if s.lower() not in self.VALID_ACTIONS:
            raise argparse.ArgumentTypeError(
                "Unable to parse action. Valid actions are: %s" % (
                    ', '.join(self.VALID_ACTIONS.keys())
                )
            )

        return self.VALID_ACTIONS[s]

    def ensure_input_file(self):
        if not self.options['input_file']:
            raise argparse.ArgumentTypeError(
                "--input_file is a required argument for this action."
            )

    def add_arguments(self, parser):
        parser.add_argument(
            '--debug',
            action='store_true',
            default=False,
            help="Don't change the data."
        )
        parser.add_argument(
            '--action',
            type=self.valid_actions,
            required=True,
            help="The action you wish to take. Valid choices are: %s" % (
                ', '.join(self.VALID_ACTIONS.keys())
            )
        )
        parser.add_argument(
            '--input_file',
            help='The input file required for certain operations.'
        )

    def handle(self, *args, **options):
        self.debug = options['debug']
        self.options = options

        # Run the requested method.
        self.options['action'](self)

    def import_fjc_judges(self):
        self.ensure_input_file()
        textfields = ['firstname', 'midname', 'lastname', 'gender',
                      'Place of Birth (City)', 'Place of Birth (State)',
                      'Place of Death (City)', 'Place of Death (State)']
        df = pd.read_excel(self.options['input_file'], 0)
        for x in textfields:
            df[x] = df[x].replace(np.nan, '', regex=True)

        for i, row in df.iterrows():
            make_federal_judge(dict(row), testing=self.debug)

    def import_state_judges(self):
        self.ensure_input_file()
        textfields = ['firstname', 'midname', 'lastname', 'gender', 'howended']
        df = pd.read_excel(self.options['input_file'], 0)
        for x in textfields:
            df[x] = df[x].replace(np.nan, '', regex=True)
        for i, row in df.iterrows():
            make_state_judge(dict(row), testing=self.debug)

    VALID_ACTIONS = {
        'import-fjc-judges': import_fjc_judges,
        'import-state-judges': import_state_judges,
    }

<commit_msg>Corrects help string for judge importer.<commit_after>import argparse

import numpy as np
import pandas as pd

from django.core.management import BaseCommand

from cl.people_db.import_judges.populate_state_judges import make_state_judge
from cl.people_db.import_judges.populate_fjc_judges import make_federal_judge

class Command(BaseCommand):
    help = 'Import judge data from various files.'

    def valid_actions(self, s):
        if s.lower() not in self.VALID_ACTIONS:
            raise argparse.ArgumentTypeError(
                "Unable to parse action. Valid actions are: %s" % (
                    ', '.join(self.VALID_ACTIONS.keys())
                )
            )

        return self.VALID_ACTIONS[s]

    def ensure_input_file(self):
        if not self.options['input_file']:
            raise argparse.ArgumentTypeError(
                "--input_file is a required argument for this action."
            )

    def add_arguments(self, parser):
        parser.add_argument(
            '--debug',
            action='store_true',
            default=False,
            help="Don't change the data."
        )
        parser.add_argument(
            '--action',
            type=self.valid_actions,
            required=True,
            help="The action you wish to take. Valid choices are: %s" % (
                ', '.join(self.VALID_ACTIONS.keys())
            )
        )
        parser.add_argument(
            '--input_file',
            help='The input file required for certain operations.'
        )

    def handle(self, *args, **options):
        self.debug = options['debug']
        self.options = options

        # Run the requested method.
        self.options['action'](self)

    def import_fjc_judges(self):
        self.ensure_input_file()
        textfields = ['firstname', 'midname', 'lastname', 'gender',
                      'Place of Birth (City)', 'Place of Birth (State)',
                      'Place of Death (City)', 'Place of Death (State)']
        df = pd.read_excel(self.options['input_file'], 0)
        for x in textfields:
            df[x] = df[x].replace(np.nan, '', regex=True)

        for i, row in df.iterrows():
            make_federal_judge(dict(row), testing=self.debug)

    def import_state_judges(self):
        self.ensure_input_file()
        textfields = ['firstname', 'midname', 'lastname', 'gender', 'howended']
        df = pd.read_excel(self.options['input_file'], 0)
        for x in textfields:
            df[x] = df[x].replace(np.nan, '', regex=True)
        for i, row in df.iterrows():
            make_state_judge(dict(row), testing=self.debug)

    VALID_ACTIONS = {
        'import-fjc-judges': import_fjc_judges,
        'import-state-judges': import_state_judges,
    }

<|endoftext|>


<commit_before>from enum import Enum
from typing import List, Union
import logging
import math

try:
    from flask_babel import _
except ModuleNotFoundError:
    pass

class VehicleType(Enum):
    CAR = 1
    TRUCK_UPTO_4 = 2
    PICKUP_UPTO_4 = 3
    TRUCK_4_TO_10 = 4
    TRUCK_12_TO_16 = 5
    TRUCK_16_TO_34 = 6
    TRUCK_ABOVE_34 = 7
    MOTORCYCLE_UPTO_50 = 8
    MOTORCYCLE_50_TO_250 = 9
    MOTORCYCLE_250_TO_500 = 10
    BUS = 11
    TAXI = 12
    WORK = 13
    TRACTOR = 14
    BIKE = 15
    TRAIN = 16
    OTHER_AND_UNKNOWN = 17
    MINIBUS = 18
    MOTORCYCLE_ABOVE_500 = 19
    ELECTRIC_SCOOTER = 21
    MOBILITY_SCOOTER = 22
    ELECTRIC_BIKE = 23
    TRUCK_3_5_TO_10 = 24
    TRUCK_10_TO_12 = 25

    def get_categories(self) -> List[int]:
        res = []
        for t in list(VehicleCategory):
            if self in t.value:
                res.append(t)
        return res

    def get_english_display_name(self):
        english_vehicle_type_display_names = {
            VehicleType.CAR: "private car",
            VehicleType.TRUCK_UPTO_4: "truck upto 4 tons",
            VehicleType.PICKUP_UPTO_4: "pickup upto 4 tons",
            VehicleType.TRUCK_4_TO_10: "truck 4 to 10 tons",
            VehicleType.TRUCK_12_TO_16: "truck 12 to 16 tons",
            VehicleType.TRUCK_16_TO_34: "truck 16 to 34 tons",
            VehicleType.TRUCK_ABOVE_34: "truck above 34 tons",
            VehicleType.MOTORCYCLE_UPTO_50: "motorcycle upto 50 cc",
            VehicleType.MOTORCYCLE_50_TO_250: "motorcycle 50 to 250 cc",
            VehicleType.MOTORCYCLE_250_TO_500: "motorcycle 250 to 500 cc",
            VehicleType.BUS: "bus",
            VehicleType.TAXI: "taxi",
            VehicleType.WORK: "work vehicle",
            VehicleType.TRACTOR: "tractor",
            VehicleType.BIKE: "bike",
            VehicleType.TRAIN: "train",
            VehicleType.OTHER_AND_UNKNOWN: "other and unknown",
            VehicleType.MINIBUS: "minibus",
            VehicleType.MOTORCYCLE_ABOVE_500: "motorcycle above 500 cc",
            VehicleType.ELECTRIC_SCOOTER: "electric scooter",
            VehicleType.MOBILITY_SCOOTER: "mobility scooter",
            VehicleType.ELECTRIC_BIKE: "electric bike",
            VehicleType.TRUCK_3_5_TO_10: "truck 3.5 to 10 tons",
            VehicleType.TRUCK_10_TO_12: "truck 10 to 12 tons",
        }
        try:
            return english_vehicle_type_display_names[self]
        except (KeyError, TypeError):
            logging.exception(f"VehicleType.get_display_name: {self}: no display string defined")
            return "no display name defined"

    @staticmethod
    def to_type_code(db_val: Union[float, int]) -> int:
        """Values read from DB may arrive as float, and empty values come as nan"""
        if isinstance(db_val, float):
            if math.isnan(db_val):
                return VehicleType.OTHER_AND_UNKNOWN.value
            else:
                return int(db_val)
        elif isinstance(db_val, int):
            return db_val
        else:
            logging.error(
                f"VehicleType.fo_type_code: unknown value: {db_val}({type(db_val)})"
                ". returning OTHER_AND_UNKNOWN"
            )
            return VehicleType.OTHER_AND_UNKNOWN.value

VT = VehicleType

class VehicleCategory(Enum):
    PROFESSIONAL_DRIVER = 1
    PRIVATE_DRIVER = 2
    LIGHT_ELECTRIC = 3
    CAR = 4
    LARGE = 5
    MOTORCYCLE = 6
    BICYCLE_AND_SMALL_MOTOR = 7
    OTHER = 8

    def get_codes(self) -> List[int]:
        """returns VehicleType codes of category"""
        category_vehicle_types = {
            VehicleCategory.PROFESSIONAL_DRIVER: [
                VehicleType.TRUCK_UPTO_4,
                VehicleType.PICKUP_UPTO_4,
                VehicleType.TRUCK_4_TO_10,
                VehicleType.TRUCK_12_TO_16,
                VehicleType.TRUCK_16_TO_34,
                VehicleType.TRUCK_ABOVE_34,
                VehicleType.BUS,
                VehicleType.TAXI,
                VehicleType.WORK,
                VehicleType.TRACTOR,
                VehicleType.MINIBUS,
                VehicleType.TRUCK_3_5_TO_10,
                VehicleType.TRUCK_10_TO_12,
            ],
            VehicleCategory.PRIVATE_DRIVER: [
                VehicleType.CAR,
                VehicleType.MOTORCYCLE_UPTO_50,
                VehicleType.MOTORCYCLE_50_TO_250,
                VehicleType.MOTORCYCLE_250_TO_500,
                VehicleType.MOTORCYCLE_ABOVE_500,
            ],
            VehicleCategory.LIGHT_ELECTRIC: [
                VehicleType.ELECTRIC_SCOOTER,
                VehicleType.MOBILITY_SCOOTER,
                VehicleType.ELECTRIC_BIKE,
            ],
            VehicleCategory.CAR: [VehicleType.CAR, VehicleType.TAXI],
            VehicleCategory.LARGE: [
                VehicleType.TRUCK_UPTO_4,
                VehicleType.PICKUP_UPTO_4,
                VehicleType.TRUCK_4_TO_10,
                VehicleType.TRUCK_12_TO_16,
                VehicleType.TRUCK_16_TO_34,
                VehicleType.TRUCK_ABOVE_34,
                VehicleType.BUS,
                VehicleType.WORK,
                VehicleType.TRACTOR,
                VehicleType.MINIBUS,
                VehicleType.TRUCK_3_5_TO_10,
                VehicleType.TRUCK_10_TO_12,
            ],
            VehicleCategory.MOTORCYCLE: [
                VehicleType.MOTORCYCLE_UPTO_50,
                VehicleType.MOTORCYCLE_50_TO_250,
                VehicleType.MOTORCYCLE_250_TO_500,
                VehicleType.MOTORCYCLE_ABOVE_500,
            ],
            VehicleCategory.BICYCLE_AND_SMALL_MOTOR: [
                VehicleType.BIKE,
                VehicleType.ELECTRIC_SCOOTER,
                VehicleType.ELECTRIC_BIKE,
            ],
            VehicleCategory.OTHER: [
                VehicleType.BIKE,
                VehicleType.TRAIN,
                VehicleType.OTHER_AND_UNKNOWN,
            ],
        }
        return list(map(lambda x: x.value, category_vehicle_types[self]))

    def contains(self, vt_code: int) -> bool:
        # noinspection PyTypeChecker
        if not isinstance(int, vt_code):
            logging.warning(f"VehicleCategory.contains: {vt_code}:{type(vt_code)}: not int")
            return False
        return vt_code in self.get_codes()

    def get_english_display_name(self):
        english_vehicle_type_display_names = {
            VehicleCategory.PROFESSIONAL_DRIVER: "professional driver",
            VehicleCategory.PRIVATE_DRIVER: "private driver",
            VehicleCategory.LIGHT_ELECTRIC: "light electric vehicles",
            VehicleCategory.CAR: "private car",
            VehicleCategory.LARGE: "large vehicle",
            VehicleCategory.MOTORCYCLE: "motorcycle",
            VehicleCategory.BICYCLE_AND_SMALL_MOTOR: "bicycle and small motor vehicles",
            VehicleCategory.OTHER: "other vehicle",
        }
        try:
            return english_vehicle_type_display_names[self]
        except (KeyError, TypeError):
            logging.exception(f"VehicleType.get_display_name: {self}: no display string defined")
            return "no display name defined"

try:
    _("professional driver")
    _("private driver")
    _("light electric vehicles")
    _("private car")
    _("large vehicle")
    _("motorcycle")
    _("bicycle and small motor vehicles")
    _("other vehicle")
except: # pylint: disable=bare-except
    pass<commit_msg>fix airflow error<commit_after>from enum import Enum
from typing import List, Union
import logging
import math

try:
    from flask_babel import _
except ModuleNotFoundError:
    def _(str):
        return str

class VehicleType(Enum):
    CAR = 1
    TRUCK_UPTO_4 = 2
    PICKUP_UPTO_4 = 3
    TRUCK_4_TO_10 = 4
    TRUCK_12_TO_16 = 5
    TRUCK_16_TO_34 = 6
    TRUCK_ABOVE_34 = 7
    MOTORCYCLE_UPTO_50 = 8
    MOTORCYCLE_50_TO_250 = 9
    MOTORCYCLE_250_TO_500 = 10
    BUS = 11
    TAXI = 12
    WORK = 13
    TRACTOR = 14
    BIKE = 15
    TRAIN = 16
    OTHER_AND_UNKNOWN = 17
    MINIBUS = 18
    MOTORCYCLE_ABOVE_500 = 19
    ELECTRIC_SCOOTER = 21
    MOBILITY_SCOOTER = 22
    ELECTRIC_BIKE = 23
    TRUCK_3_5_TO_10 = 24
    TRUCK_10_TO_12 = 25

    def get_categories(self) -> List[int]:
        res = []
        for t in list(VehicleCategory):
            if self in t.value:
                res.append(t)
        return res

    def get_english_display_name(self):
        english_vehicle_type_display_names = {
            VehicleType.CAR: "private car",
            VehicleType.TRUCK_UPTO_4: "truck upto 4 tons",
            VehicleType.PICKUP_UPTO_4: "pickup upto 4 tons",
            VehicleType.TRUCK_4_TO_10: "truck 4 to 10 tons",
            VehicleType.TRUCK_12_TO_16: "truck 12 to 16 tons",
            VehicleType.TRUCK_16_TO_34: "truck 16 to 34 tons",
            VehicleType.TRUCK_ABOVE_34: "truck above 34 tons",
            VehicleType.MOTORCYCLE_UPTO_50: "motorcycle upto 50 cc",
            VehicleType.MOTORCYCLE_50_TO_250: "motorcycle 50 to 250 cc",
            VehicleType.MOTORCYCLE_250_TO_500: "motorcycle 250 to 500 cc",
            VehicleType.BUS: "bus",
            VehicleType.TAXI: "taxi",
            VehicleType.WORK: "work vehicle",
            VehicleType.TRACTOR: "tractor",
            VehicleType.BIKE: "bike",
            VehicleType.TRAIN: "train",
            VehicleType.OTHER_AND_UNKNOWN: "other and unknown",
            VehicleType.MINIBUS: "minibus",
            VehicleType.MOTORCYCLE_ABOVE_500: "motorcycle above 500 cc",
            VehicleType.ELECTRIC_SCOOTER: "electric scooter",
            VehicleType.MOBILITY_SCOOTER: "mobility scooter",
            VehicleType.ELECTRIC_BIKE: "electric bike",
            VehicleType.TRUCK_3_5_TO_10: "truck 3.5 to 10 tons",
            VehicleType.TRUCK_10_TO_12: "truck 10 to 12 tons",
        }
        try:
            return english_vehicle_type_display_names[self]
        except (KeyError, TypeError):
            logging.exception(f"VehicleType.get_display_name: {self}: no display string defined")
            return "no display name defined"

    @staticmethod
    def to_type_code(db_val: Union[float, int]) -> int:
        """Values read from DB may arrive as float, and empty values come as nan"""
        if isinstance(db_val, float):
            if math.isnan(db_val):
                return VehicleType.OTHER_AND_UNKNOWN.value
            else:
                return int(db_val)
        elif isinstance(db_val, int):
            return db_val
        else:
            logging.error(
                f"VehicleType.fo_type_code: unknown value: {db_val}({type(db_val)})"
                ". returning OTHER_AND_UNKNOWN"
            )
            return VehicleType.OTHER_AND_UNKNOWN.value

VT = VehicleType

class VehicleCategory(Enum):
    PROFESSIONAL_DRIVER = 1
    PRIVATE_DRIVER = 2
    LIGHT_ELECTRIC = 3
    CAR = 4
    LARGE = 5
    MOTORCYCLE = 6
    BICYCLE_AND_SMALL_MOTOR = 7
    OTHER = 8

    def get_codes(self) -> List[int]:
        """returns VehicleType codes of category"""
        category_vehicle_types = {
            VehicleCategory.PROFESSIONAL_DRIVER: [
                VehicleType.TRUCK_UPTO_4,
                VehicleType.PICKUP_UPTO_4,
                VehicleType.TRUCK_4_TO_10,
                VehicleType.TRUCK_12_TO_16,
                VehicleType.TRUCK_16_TO_34,
                VehicleType.TRUCK_ABOVE_34,
                VehicleType.BUS,
                VehicleType.TAXI,
                VehicleType.WORK,
                VehicleType.TRACTOR,
                VehicleType.MINIBUS,
                VehicleType.TRUCK_3_5_TO_10,
                VehicleType.TRUCK_10_TO_12,
            ],
            VehicleCategory.PRIVATE_DRIVER: [
                VehicleType.CAR,
                VehicleType.MOTORCYCLE_UPTO_50,
                VehicleType.MOTORCYCLE_50_TO_250,
                VehicleType.MOTORCYCLE_250_TO_500,
                VehicleType.MOTORCYCLE_ABOVE_500,
            ],
            VehicleCategory.LIGHT_ELECTRIC: [
                VehicleType.ELECTRIC_SCOOTER,
                VehicleType.MOBILITY_SCOOTER,
                VehicleType.ELECTRIC_BIKE,
            ],
            VehicleCategory.CAR: [VehicleType.CAR, VehicleType.TAXI],
            VehicleCategory.LARGE: [
                VehicleType.TRUCK_UPTO_4,
                VehicleType.PICKUP_UPTO_4,
                VehicleType.TRUCK_4_TO_10,
                VehicleType.TRUCK_12_TO_16,
                VehicleType.TRUCK_16_TO_34,
                VehicleType.TRUCK_ABOVE_34,
                VehicleType.BUS,
                VehicleType.WORK,
                VehicleType.TRACTOR,
                VehicleType.MINIBUS,
                VehicleType.TRUCK_3_5_TO_10,
                VehicleType.TRUCK_10_TO_12,
            ],
            VehicleCategory.MOTORCYCLE: [
                VehicleType.MOTORCYCLE_UPTO_50,
                VehicleType.MOTORCYCLE_50_TO_250,
                VehicleType.MOTORCYCLE_250_TO_500,
                VehicleType.MOTORCYCLE_ABOVE_500,
            ],
            VehicleCategory.BICYCLE_AND_SMALL_MOTOR: [
                VehicleType.BIKE,
                VehicleType.ELECTRIC_SCOOTER,
                VehicleType.ELECTRIC_BIKE,
            ],
            VehicleCategory.OTHER: [
                VehicleType.BIKE,
                VehicleType.TRAIN,
                VehicleType.OTHER_AND_UNKNOWN,
            ],
        }
        return list(map(lambda x: x.value, category_vehicle_types[self]))

    def contains(self, vt_code: int) -> bool:
        # noinspection PyTypeChecker
        if not isinstance(int, vt_code):
            logging.warning(f"VehicleCategory.contains: {vt_code}:{type(vt_code)}: not int")
            return False
        return vt_code in self.get_codes()

    def get_english_display_name(self):
        english_vehicle_type_display_names = {
            VehicleCategory.PROFESSIONAL_DRIVER: "professional driver",
            VehicleCategory.PRIVATE_DRIVER: "private driver",
            VehicleCategory.LIGHT_ELECTRIC: "light electric vehicles",
            VehicleCategory.CAR: "private car",
            VehicleCategory.LARGE: "large vehicle",
            VehicleCategory.MOTORCYCLE: "motorcycle",
            VehicleCategory.BICYCLE_AND_SMALL_MOTOR: "bicycle and small motor vehicles",
            VehicleCategory.OTHER: "other vehicle",
        }
        try:
            return english_vehicle_type_display_names[self]
        except (KeyError, TypeError):
            logging.exception(f"VehicleType.get_display_name: {self}: no display string defined")
            return "no display name defined"

_("professional driver")
_("private driver")
_("light electric vehicles")
_("private car")
_("large vehicle")
_("motorcycle")
_("bicycle and small motor vehicles")
_("other vehicle")
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
url.py - Jenni Bitly Module
Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham.
Licensed under the Eiffel Forum License 2.

More info:
 * Jenni: https://github.com/myano/jenni/
 * Phenny: http://inamidst.com/phenny/

This module will record all URLs to bitly via an api key and account.
It also automatically displays the "title" of any URL pasted into the channel.
"""

import re, urllib2
from htmlentitydefs import name2codepoint
import web

# Place a file in your ~/jenni/ folder named, bitly.txt
# and inside this file place your API key followed by a ','
# and then your username. For example, the only line in that
# file should look like this:
# R_d67798xkjc87sdx6x8c7kjc87,myusername

# this variable is to determine when to use bitly. If the URL is more
# than this length, it'll display a bitly URL instead. To disable bit.ly, put None
# even if it's set to None, triggering .bitly command will still work!
BITLY_TRIGGER_LEN = 65
EXCLUSION_CHAR = "!"
IGNORE = ["git.io"]

# do not edit below this line unless you know what you're doing
bitly_loaded = 0

try:
    file = open("bitly.txt", "r")
    key = file.read()
    key = key.split(",")
    bitly_api_key = str(key[0].lstrip().rstrip())
    bitly_user = str(key[1].lstrip().rstrip())
    file.close()
    bitly_loaded = 1
except:
    print "ERROR: No bitly.txt found."

url_finder = re.compile(r'(%s?(http|https|ftp)(://\S+))' % (EXCLUSION_CHAR))
r_entity = re.compile(r'&[A-Za-z0-9#]+;')
INVALID_WEBSITE = 0x01

def find_title(url):
    """
    This finds the title when provided with a string of a URL.
    """
    uri = url

    for item in IGNORE:
        if item in url:
            return

    if not re.search('^((https?)|(ftp))://', uri):
        uri = 'http://' + uri

    if "twitter.com" in uri:
        uri = uri.replace('#!', '?_escaped_fragment_=')

    redirects = 0
    ## follow re-directs, if someone pastes a bitly of a tinyurl, etc..
    while True:
        req = urllib2.Request(uri, headers={'Accept':'text/html'})
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0')
        u = urllib2.urlopen(req)
        info = u.info()
        u.close()

        if not isinstance(info, list):
            status = '200'
        else:
            status = str(info[1])
            info = info[0]
        if status.startswith('3'):
            uri = urlparse.urljoin(uri, info['Location'])
        else: break

        redirects += 1
        if redirects >= 50:
            return "Too many re-directs."

    try: mtype = info['content-type']
    except:
        return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return

    u = urllib2.urlopen(req)
    bytes = u.read(262144)
    u.close()
    content = bytes
    regex = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
    content = regex.sub(r'<\1title>',content)
    regex = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
    content = regex.sub('',content)
    start = content.find('<title>')
    if start == -1: return
    end = content.find('</title>', start)
    if end == -1: return
    content = content[start+7:end]
    content = content.strip('\n').rstrip().lstrip()
    title = content

    if len(title) > 200:
        title = title[:200] + '[...]'

    def e(m):
        entity = m.group()
        if entity.startswith('&#x'):
            cp = int(entity[3:-1],16)
            return unichr(cp).encode('utf-8')
        elif entity.startswith('&#'):
            cp = int(entity[2:-1])
            return unichr(cp).encode('utf-8')
        else:
            char = name2codepoint[entity[1:-1]]
            return unichr(char).encode('utf-8')

    title = r_entity.sub(e, title)

    if title:
        try: title.decode('utf-8')
        except:
            try: title = title.decode('iso-8859-1').encode('utf-8')
            except: title = title.decode('cp1252').encode('utf-8')
        else: pass
    else: title = 'None'

    title = title.replace('\n', '')
    title = title.replace('\r', '')

    def remove_spaces(x):
        if "  " in x:
            x = x.replace("  ", " ")
            return remove_spaces(x)
        else:
            return x

    title = remove_spaces (title)

    if title:
        return title

def short(text):
    """
    This function creates a bitly url for each url in the provided "text" string.
    The return type is a list.
    """

    if not bitly_loaded: return [ ]
    bitlys = [ ]
    try:
        a = re.findall(url_finder, text)
        k = len(a)
        i = 0
        while i < k:
            b = str(a[i][0])
            if not b.startswith("http://bit.ly") or not b.startswith("http://j.mp/"):
                # check to see if the url is valid
                try: c = web.head(b)
                except: return [[None, None]]

                url = "http://api.j.mp/v3/shorten?login=%s&apiKey=%s&longUrl=%s&format=txt" % (bitly_user, bitly_api_key, urllib2.quote(b))
                shorter = web.get(url)
                shorter.strip()
                bitlys.append([b, shorter])
            i += 1
        return bitlys
    except:
        return

def generateBitLy (jenni, input):
    if not bitly_loaded: return
    bitly = short(input)
    idx = 7
    for b in bitly:
        displayBitLy(jenni, b[0], b[1])
generateBitLy.commands = ['bitly']
generateBitLy.priority = 'high'

def displayBitLy (jenni, url, shorten):
    if url is None or shorten is None: return
    u = getTLD(url)
    jenni.say('%s  -  %s' % (u, shorten))

def getTLD (url):
    idx = 7
    if url.startswith('https://'): idx = 8
    elif url.startswith('ftp://'): idx = 6
    u = url[idx:]
    f = u.find('/')
    if f == -1: u = url
    else: u = url[0:idx] + u[0:f]
    return u

def doUseBitLy (url):
    return bitly_loaded and BITLY_TRIGGER_LEN is not None and len(url) > BITLY_TRIGGER_LEN

def get_results(text):
    a = re.findall(url_finder, text)
    k = len(a)
    i = 0
    display = [ ]
    while i < k:
        url = str(a[i][0])
        if not url.startswith(EXCLUSION_CHAR):
            try:
                page_title = find_title(url)
            except:
                page_title = None # if it can't access the site fail silently
            if bitly_loaded: # and (page_title is not None or page_title == INVALID_WEBSITE):
                bitly = short(url)
                bitly = bitly[0][1]
            else: bitly = url
            display.append([page_title, url, bitly])
        i += 1
    return display

def show_title_auto (jenni, input):
    if (input.startswith('.title ') or input.startswith('.bitly ') or input.startswith('.dftba') or re.match('.*(youtube.com/watch\S*v=|youtu.be/)([\w]+.*)', input)):
        return
    if len(re.findall("\([\d]+\sfiles\sin\s[\d]+\sdirs\)", input)) == 1: return
    try:
        results = get_results(input)
    except: return
    if results is None: return

    k = 1
    for r in results:
        if k > 3: break
        k += 1

        useBitLy = doUseBitLy(r[1])
        if r[0] is None:
            if useBitLy: displayBitLy(jenni, r[1], r[2])
            continue
        if useBitLy: r[1] = r[2]
        else: r[1] = getTLD(r[1])
        jenni.say('[ %s ] - %s' % (r[0], r[1]))
show_title_auto.rule = '.*(%s?(http|https)(://\S+)).*' % (EXCLUSION_CHAR)
show_title_auto.priority = 'high'

def show_title_demand (jenni, input):
    try:
        results = get_results(input)
    except: return
    if results is None: return

    for r in results:
        if r[0] is None: continue
        if doUseBitLy(r[1]): r[1] = r[2]
        else: r[1] = getTLD(r[1])
        jenni.say('[ %s ] - %s' % (r[0], r[1]))
show_title_demand.commands = ['title']
show_title_demand.priority = 'high'

def ytinfo(jenni, input):
    #Right now, this uses a parsing script from rscript.org. Eventually, I'd
    #like to use the YouTube API directly.

    #Grab info from rscript
    uri = 'http://rscript.org/lookup.php?type=youtubeinfo&id=' + input.group(2)
    redirects = 0
    while True:
        req = urllib2.Request(uri, headers={'Accept':'text/html'})
        req.add_header('User-Agent', 'OpenAnything/1.0 +http://diveintopython.org/')
        u = urllib2.urlopen(req)
        info = u.info()
        u.close()
        # info = web.head(uri)
        if not isinstance(info, list):
            status = '200'
        else:
            status = str(info[1])
            info = info[0]
        if status.startswith('3'):
            uri = urlparse.urljoin(uri, info['Location'])
        else: break
        redirects += 1
        if redirects >= 50:
            return "Too many re-directs."
    try: mtype = info['content-type']
    except:
        return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return
    u = urllib2.urlopen(req)
    bytes = u.read(262144)
    u.close()

    #Parse rscript info.
    rtitle = re.search('(TITLE: )(.*)', bytes)
    title = rtitle.group(2)

    author = re.search('(AUTHOR: )(\S*) (20\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d).*', bytes)
    uploader = author.group(2)
    year = author.group(3)
    month = author.group(4)
    day = author.group(5)
    hour = author.group(6)
    minute = author.group(7)
    uploaded = day + '/' + month + '/' + year + ', ' + hour + ':' + minute

    duration = int(re.search('(DURATION: )(.*)', bytes).group(2))
    if duration < 1: length = 'LIVE'
    else:
        hours = duration / (60 * 60)
        minutes = duration / 60 - (hours * 60)
        seconds = duration % 60

        length = ''
        if hours:
            length = str(hours) + 'hourse'
            if minutes:
                length = length + ' '
        if minutes:
            length = length + str(minutes) + 'mins'
            if seconds:
                length = length + ' '
        if seconds: length = length + str(seconds) + 'secs'

    views = str('{:20,d}'.format(int(re.search('(VIEWS: )(.*)', bytes).group(2)))).lstrip(' ')
    comments = str('{:20,d}'.format(int(re.search('(COMMENTS: )(.*)', bytes).group(2)))).lstrip(' ')
    #Favorite, like, dislike
    favorite = re.search('(FAVORITE: )([\d,]+) ([\d,]+) ([\d,]+)', bytes)
    likes = favorite.group(3)
    dislikes = favorite.group(4)

    message = '[YouTube] Title: ' + title + ' | Uploader: ' + uploader + \
              ' | Uploaded: ' + uploaded + ' | Length: ' + length + \
              ' | Views: ' + views + ' | Comments: ' + comments + ' | Likes: '\
              + likes + ' | Dislikes: ' + dislikes

    jenni.say(message)
ytinfo.rule = '.*(youtube.com/watch\S*v=|youtu.be/)([\w-]+).*'

if __name__ == '__main__':
    print __doc__.strip()
<commit_msg>Last fix, I think.<commit_after>#!/usr/bin/env python
"""
url.py - Jenni Bitly Module
Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham.
Licensed under the Eiffel Forum License 2.

More info:
 * Jenni: https://github.com/myano/jenni/
 * Phenny: http://inamidst.com/phenny/

This module will record all URLs to bitly via an api key and account.
It also automatically displays the "title" of any URL pasted into the channel.
"""

import re, urllib2
from htmlentitydefs import name2codepoint
import web

# Place a file in your ~/jenni/ folder named, bitly.txt
# and inside this file place your API key followed by a ','
# and then your username. For example, the only line in that
# file should look like this:
# R_d67798xkjc87sdx6x8c7kjc87,myusername

# this variable is to determine when to use bitly. If the URL is more
# than this length, it'll display a bitly URL instead. To disable bit.ly, put None
# even if it's set to None, triggering .bitly command will still work!
BITLY_TRIGGER_LEN = 65
EXCLUSION_CHAR = "!"
IGNORE = ["git.io"]

# do not edit below this line unless you know what you're doing
bitly_loaded = 0

try:
    file = open("bitly.txt", "r")
    key = file.read()
    key = key.split(",")
    bitly_api_key = str(key[0].lstrip().rstrip())
    bitly_user = str(key[1].lstrip().rstrip())
    file.close()
    bitly_loaded = 1
except:
    print "ERROR: No bitly.txt found."

url_finder = re.compile(r'(%s?(http|https|ftp)(://\S+))' % (EXCLUSION_CHAR))
r_entity = re.compile(r'&[A-Za-z0-9#]+;')
INVALID_WEBSITE = 0x01

def find_title(url):
    """
    This finds the title when provided with a string of a URL.
    """
    uri = url

    for item in IGNORE:
        if item in url:
            return

    if not re.search('^((https?)|(ftp))://', uri):
        uri = 'http://' + uri

    if "twitter.com" in uri:
        uri = uri.replace('#!', '?_escaped_fragment_=')

    redirects = 0
    ## follow re-directs, if someone pastes a bitly of a tinyurl, etc..
    while True:
        req = urllib2.Request(uri, headers={'Accept':'text/html'})
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0')
        u = urllib2.urlopen(req)
        info = u.info()
        u.close()

        if not isinstance(info, list):
            status = '200'
        else:
            status = str(info[1])
            info = info[0]
        if status.startswith('3'):
            uri = urlparse.urljoin(uri, info['Location'])
        else: break

        redirects += 1
        if redirects >= 50:
            return "Too many re-directs."

    try: mtype = info['content-type']
    except:
        return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return

    u = urllib2.urlopen(req)
    bytes = u.read(262144)
    u.close()
    content = bytes
    regex = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
    content = regex.sub(r'<\1title>',content)
    regex = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
    content = regex.sub('',content)
    start = content.find('<title>')
    if start == -1: return
    end = content.find('</title>', start)
    if end == -1: return
    content = content[start+7:end]
    content = content.strip('\n').rstrip().lstrip()
    title = content

    if len(title) > 200:
        title = title[:200] + '[...]'

    def e(m):
        entity = m.group()
        if entity.startswith('&#x'):
            cp = int(entity[3:-1],16)
            return unichr(cp).encode('utf-8')
        elif entity.startswith('&#'):
            cp = int(entity[2:-1])
            return unichr(cp).encode('utf-8')
        else:
            char = name2codepoint[entity[1:-1]]
            return unichr(char).encode('utf-8')

    title = r_entity.sub(e, title)

    if title:
        try: title.decode('utf-8')
        except:
            try: title = title.decode('iso-8859-1').encode('utf-8')
            except: title = title.decode('cp1252').encode('utf-8')
        else: pass
    else: title = 'None'

    title = title.replace('\n', '')
    title = title.replace('\r', '')

    def remove_spaces(x):
        if "  " in x:
            x = x.replace("  ", " ")
            return remove_spaces(x)
        else:
            return x

    title = remove_spaces (title)

    if title:
        return title

def short(text):
    """
    This function creates a bitly url for each url in the provided "text" string.
    The return type is a list.
    """

    if not bitly_loaded: return [ ]
    bitlys = [ ]
    try:
        a = re.findall(url_finder, text)
        k = len(a)
        i = 0
        while i < k:
            b = str(a[i][0])
            if not b.startswith("http://bit.ly") or not b.startswith("http://j.mp/"):
                # check to see if the url is valid
                try: c = web.head(b)
                except: return [[None, None]]

                url = "http://api.j.mp/v3/shorten?login=%s&apiKey=%s&longUrl=%s&format=txt" % (bitly_user, bitly_api_key, urllib2.quote(b))
                shorter = web.get(url)
                shorter.strip()
                bitlys.append([b, shorter])
            i += 1
        return bitlys
    except:
        return

def generateBitLy (jenni, input):
    if not bitly_loaded: return
    bitly = short(input)
    idx = 7
    for b in bitly:
        displayBitLy(jenni, b[0], b[1])
generateBitLy.commands = ['bitly']
generateBitLy.priority = 'high'

def displayBitLy (jenni, url, shorten):
    if url is None or shorten is None: return
    u = getTLD(url)
    jenni.say('%s  -  %s' % (u, shorten))

def getTLD (url):
    idx = 7
    if url.startswith('https://'): idx = 8
    elif url.startswith('ftp://'): idx = 6
    u = url[idx:]
    f = u.find('/')
    if f == -1: u = url
    else: u = url[0:idx] + u[0:f]
    return u

def doUseBitLy (url):
    return bitly_loaded and BITLY_TRIGGER_LEN is not None and len(url) > BITLY_TRIGGER_LEN

def get_results(text):
    a = re.findall(url_finder, text)
    k = len(a)
    i = 0
    display = [ ]
    while i < k:
        url = str(a[i][0])
        if not url.startswith(EXCLUSION_CHAR):
            try:
                page_title = find_title(url)
            except:
                page_title = None # if it can't access the site fail silently
            if bitly_loaded: # and (page_title is not None or page_title == INVALID_WEBSITE):
                bitly = short(url)
                bitly = bitly[0][1]
            else: bitly = url
            display.append([page_title, url, bitly])
        i += 1
    return display

def show_title_auto (jenni, input):
    if (input.startswith('.title ') or input.startswith('.bitly ') or input.startswith('.dftba') or re.match('.*(youtube.com/watch\S*v=|youtu.be/)([\w]+.*)', input)):
        return
    if len(re.findall("\([\d]+\sfiles\sin\s[\d]+\sdirs\)", input)) == 1: return
    try:
        results = get_results(input)
    except: return
    if results is None: return

    k = 1
    for r in results:
        if k > 3: break
        k += 1

        useBitLy = doUseBitLy(r[1])
        if r[0] is None:
            if useBitLy: displayBitLy(jenni, r[1], r[2])
            continue
        if useBitLy: r[1] = r[2]
        else: r[1] = getTLD(r[1])
        jenni.say('[ %s ] - %s' % (r[0], r[1]))
show_title_auto.rule = '.*(%s?(http|https)(://\S+)).*' % (EXCLUSION_CHAR)
show_title_auto.priority = 'high'

def show_title_demand (jenni, input):
    try:
        results = get_results(input)
    except: return
    if results is None: return

    for r in results:
        if r[0] is None: continue
        if doUseBitLy(r[1]): r[1] = r[2]
        else: r[1] = getTLD(r[1])
        jenni.say('[ %s ] - %s' % (r[0], r[1]))
show_title_demand.commands = ['title']
show_title_demand.priority = 'high'

def ytinfo(jenni, input):
    #Right now, this uses a parsing script from rscript.org. Eventually, I'd
    #like to use the YouTube API directly.

    #Grab info from rscript
    uri = 'http://rscript.org/lookup.php?type=youtubeinfo&id=' + input.group(2)
    redirects = 0
    while True:
        req = urllib2.Request(uri, headers={'Accept':'text/html'})
        req.add_header('User-Agent', 'OpenAnything/1.0 +http://diveintopython.org/')
        u = urllib2.urlopen(req)
        info = u.info()
        u.close()
        # info = web.head(uri)
        if not isinstance(info, list):
            status = '200'
        else:
            status = str(info[1])
            info = info[0]
        if status.startswith('3'):
            uri = urlparse.urljoin(uri, info['Location'])
        else: break
        redirects += 1
        if redirects >= 50:
            return "Too many re-directs."
    try: mtype = info['content-type']
    except:
        return
    if not (('/html' in mtype) or ('/xhtml' in mtype)):
        return
    u = urllib2.urlopen(req)
    bytes = u.read(262144)
    u.close()

    #Parse rscript info.
    rtitle = re.search('(TITLE: )(.*)', bytes)
    title = rtitle.group(2)

    author = re.search('(AUTHOR: )(\S*) (20\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d).*', bytes)
    uploader = author.group(2)
    year = author.group(3)
    month = author.group(4)
    day = author.group(5)
    hour = author.group(6)
    minute = author.group(7)
    uploaded = day + '/' + month + '/' + year + ', ' + hour + ':' + minute

    duration = int(re.search('(DURATION: )(.*)', bytes).group(2))
    if duration < 1: length = 'LIVE'
    else:
        hours = duration / (60 * 60)
        minutes = duration / 60 - (hours * 60)
        seconds = duration % 60

        length = ''
        if hours:
            length = str(hours) + 'hours'
            if minutes or seconds:
                length = length + ' '
        if minutes:
            length = length + str(minutes) + 'mins'
            if seconds:
                length = length + ' '
        if seconds: length = length + str(seconds) + 'secs'

    views = str('{:20,d}'.format(int(re.search('(VIEWS: )(.*)', bytes).group(2)))).lstrip(' ')
    comments = str('{:20,d}'.format(int(re.search('(COMMENTS: )(.*)', bytes).group(2)))).lstrip(' ')
    #Favorite, like, dislike
    favorite = re.search('(FAVORITE: )([\d,]+) ([\d,]+) ([\d,]+)', bytes)
    likes = favorite.group(3)
    dislikes = favorite.group(4)

    message = '[YouTube] Title: ' + title + ' | Uploader: ' + uploader + \
              ' | Uploaded: ' + uploaded + ' | Length: ' + length + \
              ' | Views: ' + views + ' | Comments: ' + comments + ' | Likes: '\
              + likes + ' | Dislikes: ' + dislikes

    jenni.say(message)
ytinfo.rule = '.*(youtube.com/watch\S*v=|youtu.be/)([\w-]+).*'

if __name__ == '__main__':
    print __doc__.strip()
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from anvil import date
from anvil import log as logging
from anvil import shell as sh
from anvil import utils

import json

from contextlib import contextmanager

LOG = logging.getLogger(__name__)

class PhaseRecorder(object):
    def __init__(self, fn):
        self.fn = fn

    @contextmanager
    def mark(self, phasename):
        contents = dict()
        contents['name'] = phasename
        contents['when'] = date.rcf8222date()
        yield phasename
        LOG.debug("Marking the completion of phase %r in file %r", phasename, self.fn)
        lines = utils.joinlinesep([json.dumps(contents), ''])
        sh.append_file(self.fn, lines)

    def has_ran(self, phasename):
        phases = self.list_phases()
        if phasename in phases:
            return True
        return False

    def list_phases(self):
        phases = set()
        if not sh.isfile(self.fn):
            return phases
        line_num = 1
        for line in sh.load_file(self.fn).splitlines():
            line = line.strip()
            if line:
                data = json.loads(line)
                if not isdict(data):
                    raise TypeError("Unknown phase entry in %s on line %s" % (self.fn, line_num))
                if 'name' in data:
                    phases.add(data['name'])
            line_num += 1
        return phases
<commit_msg>Update how phases are tracked<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from anvil import date
from anvil import log as logging
from anvil import shell as sh
from anvil import utils

import json

from contextlib import contextmanager

LOG = logging.getLogger(__name__)

class PhaseRecorder(object):
    def __init__(self, fn):
        self.fn = fn

    @contextmanager
    def mark(self, phasename):
        contents = dict()
        contents['name'] = phasename
        contents['when'] = date.rcf8222date()
        yield phasename
        LOG.debug("Marking the completion of phase %r in file %r", phasename, self.fn)
        lines = [json.dumps(contents), '']
        sh.append_file(self.fn, utils.joinlinesep(*lines))

    def has_ran(self, phasename):
        phases = self.list_phases()
        if phasename in phases:
            return True
        return False

    def list_phases(self):
        phases = set()
        if not sh.isfile(self.fn):
            return phases
        line_num = 1
        for line in sh.load_file(self.fn).splitlines():
            line = line.strip()
            if line:
                data = json.loads(line)
                if not isdict(data):
                    raise TypeError("Unknown phase entry in %s on line %s" % (self.fn, line_num))
                if 'name' in data:
                    phases.add(data['name'])
            line_num += 1
        return phases
<|endoftext|>


<commit_before># Copyright 2016, DELLEMC, Inc.
"""
This is a module that contains the tool for python to run a list of commands
and generate report for results of running.
"""

import os
import subprocess
import sys

try:
    from ParallelTasks import ParallelTasks
    import common
except ImportError as import_err:
    print import_err
    sys.exit(1)

class BuildResult(object):
    """
    Complete output from the running of a build command on a given host.
    Contains the command that was supposed to be run, whether it was present ot not,
    the return code, and the standard output and standard error text.
    """
    def __init__(self, command, present, return_code=None, stdout=None, stderr=None):
        self._command = command
        self._present = present

        self._return_code = return_code
        self._stdout = stdout
        self._stderr = stderr

    @property
    def command(self):
        return self._command

    @command.setter
    def command(self, command):
        self._command = command

    @property
    def present(self):
        return self._present

    @present.setter
    def present(self, present):
        self._present = present

    @property
    def return_code(self):
        return self._return_code

    @return_code.setter
    def return_code(self, return_code):
        self._return_code = return_code

    @property
    def stdout(self):
        return self._stdout

    @stdout.setter
    def stdout(self, stdout):
        self._stdout = stdout

    @property
    def stderr(self):
        return self._stderr

    @stderr.setter
    def stderr(self, stderr):
        self._stderr = stderr

    def generate_detailed_report(self):
        """
        Generate reports with details: return code, stdout, stderr
        """
        detailed = []
        if self._present is True:
            detailed.append(self._command)
            if self._return_code != 0:
                detailed.append("  ERROR: EXIT {0}".format(self._return_code))
                if self._stdout is not None and self._stdout != "":
                    detailed.append(self._stdout)
                if self._stderr is not None and self._stderr != "":
                    detailed.append(self._stderr)

        return detailed

    def generate_summary_report(self):
        """
        Generate report with the result of running:
        Good or ERROR: EXIT or Not Present
        """
        summary = []
        short_command = os.path.basename(self._command)
        if self._present is True:
            if self._return_code is not None:
                if self._return_code == 0:
                    status = "GOOD"
                else:
                    status = "ERROR: EXIT {0}".format(self._return_code)
            else:
                status = "RETURN CODE IS NONE"
        else:
            status = " Not Present"
        summary.append("    {0}: {1}".format(short_command, status))
        return summary

    @staticmethod
    def summarize_errors(results):
        errors = 0
        for result in results:
            if result.return_code != 0:
                errors += 1
        return errors

class BuildCommand(object):
    """
    A module for build command.
    """
    def __init__(self, name, directory, arguments=None, use_sudo=False, sudo_creds=None):
        """
        _name: the command name that was supposed to be run
        _directory: the directory under which to run command
        _use_sudo: whether the command was run with sudo previleges
        _sudo_creds: the environment variable name of sudo credentials.
                     For example: SUDO_CREDS=username:password
        _arguments: the arguments of the command
        """
        self._name = name
        self._directory = directory
        self._use_sudo = use_sudo
        self._sudo_creds = sudo_creds
        self._arguments = arguments

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, name):
        self._name = name

    @property
    def directory(self):
        return self._directory

    @directory.setter
    def directory(self, directory):
        self._directory = directory

    @property
    def use_sudo(self):
        return self._use_sudo

    @use_sudo.setter
    def use_sudo(self, use_sudo):
        self._use_sudo = use_sudo

    @property
    def sudo_creds(self):
        return self._sudo_creds

    @sudo_creds.setter
    def sudo_creds(self, sudo_creds):
        self._sudo_creds = sudo_creds

    def to_string(self):
        cmd_args = []
        if self._use_sudo:
            if self._sudo_creds is None:
                raise ValueError("sudo credential missing from commands {0}".format(name))
            (username, password) = common.parse_credential_variable(self._sudo_creds)
            cmd_args += ["echo"]
            cmd_args += [password]
            cmd_args += ["|sudo -S"]

        cmd_args += [self._name]

        if self._arguments:
            cmd_args += self._arguments

        command_str = " ".join(cmd_args)
        return command_str

    def run(self):
        try:
            command = self.to_string()
            print "Execute command: {0} under {1}".format(command, self._directory)
            proc = subprocess.Popen(command,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    cwd=self._directory,
                                    shell=True)
            (out, err) = proc.communicate()
        except Exception, ex:
            # this is a terrible failure, not just process exit != 0
            return BuildResult(self._name,
                               present=True,
                               return_code=1,
                               stderr=ex)

        result = BuildResult(self._name,
                             present=True,
                             return_code=proc.returncode,
                             stdout=out,
                             stderr=err)
        return result

    def is_executable(self):
        exe_file = os.path.join(self._directory, self._name)
        if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
            return True

        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
                return True

        return False

class Builder(ParallelTasks):
    """
    Run a list of command under a directory.

    This class is intended for use with ParallelTasks, and each commands list may be done
    in a separate process.

    """
    def add_task(self, data, name):
        """
        Add a task to task queue
        :param data: A dictonary which should contain:
                     commands: A list of comamnd instances. It's required.
                     env_file: A property file which contains environment variables. It's optional
        :param name: The name of the task. The key by which the job results will be returned.
        :return: None
        """
        if data is None:
            raise ValueError("Task parameter data not present")

        if 'commands' not in data:
            raise ValueError("commands key missing from data: {0}".format(data))

        super(Builder, self).add_task(data, name)

    @staticmethod
    def initail_environment(env_file):
        print "start to export environment file {0}".format(env_file)
        if not os.path.isfile(env_file):
            raise RuntimeError("Failed to initial environment due to the file {0} doesn't exist"
                               .format(env_file))
        props = common.parse_property_file(env_file)
        if props:
            for item in props.items():
                key = item[0]
                value = item[1]
                os.environ[key] = value

    def do_one_task(self, name, data, results):
        """
        Perform the actual work.
        name and data will come from the values passed in to add_task()
        :param results: a list of instances of BuildResult
        :return: None
        """
        try:
            if 'command' not in results:
                results['command'] = []

            if 'env_file' in data and data['env_file'] is not None:
                self.initail_environment(data['env_file'])

            for command in data['commands']:
                if not command.is_executable():
                    build_result = BuildResult(command, present=False)
                build_result = command.run()
                if build_result is not None:
                    results['command'].append(build_result)

        except Exception, e:
            raise RuntimeError("Failed to do task {0} due to {1}".format(name, e))

    def summarize_results(self):
        """
        :return: True if the number of errors found is 0
                 False if the number of errors found is greater than 0
        """
        results = self.get_results()
        key_list = results.keys()

        for name in sorted(key_list):
            if 'command' in results[name]:
                build_results = results[name]['command']
                errors = BuildResult.summarize_errors(build_results)
                if errors > 0:
                    return False
        return True

    def generate_detailed_report(self):
        """
        Generate reports with details: return code, stdout, stderr
        """
        all_detailed = []
        results = self.get_results()
        key_list = results.keys()
        for name in sorted(key_list):
            if 'command' in results[name]:
                task_detailed = []
                task_detailed.append("=== Results for {0} ===".format(name))
                for result in results[name]['command']:
                    detailed = result.generate_detailed_report()
                    task_detailed.extend(detailed)

                all_detailed.extend(task_detailed)
        return all_detailed

    def generate_summary_report(self):
        """
        Generate report with the result of running:
        Good or ERROR: EXIT or Not Present
        """
        all_summary = []

        results = self.get_results()
        key_list = results.keys()

        for name in sorted(key_list):
            if 'command' in results[name]:
                task_summary = []
                task_summary.append("{0}:".format(name))
                build_results = results[name]['command']
                errors = BuildResult.summarize_errors(build_results)
                if errors > 0:
                    task_summary.append("    Number of falied commands: {0}".format(errors))

                for result in build_results:
                    summary = result.generate_summary_report()
                    task_summary.extend(summary)

                all_summary.extend(task_summary)

        return all_summary

<commit_msg>[skip ci]Fix: on-imagebuilder failed to get version from environment<commit_after># Copyright 2016, DELLEMC, Inc.
"""
This is a module that contains the tool for python to run a list of commands
and generate report for results of running.
"""

import os
import subprocess
import sys

try:
    from ParallelTasks import ParallelTasks
    import common
except ImportError as import_err:
    print import_err
    sys.exit(1)

class BuildResult(object):
    """
    Complete output from the running of a build command on a given host.
    Contains the command that was supposed to be run, whether it was present ot not,
    the return code, and the standard output and standard error text.
    """
    def __init__(self, command, present, return_code=None, stdout=None, stderr=None):
        self._command = command
        self._present = present

        self._return_code = return_code
        self._stdout = stdout
        self._stderr = stderr

    @property
    def command(self):
        return self._command

    @command.setter
    def command(self, command):
        self._command = command

    @property
    def present(self):
        return self._present

    @present.setter
    def present(self, present):
        self._present = present

    @property
    def return_code(self):
        return self._return_code

    @return_code.setter
    def return_code(self, return_code):
        self._return_code = return_code

    @property
    def stdout(self):
        return self._stdout

    @stdout.setter
    def stdout(self, stdout):
        self._stdout = stdout

    @property
    def stderr(self):
        return self._stderr

    @stderr.setter
    def stderr(self, stderr):
        self._stderr = stderr

    def generate_detailed_report(self):
        """
        Generate reports with details: return code, stdout, stderr
        """
        detailed = []
        if self._present is True:
            detailed.append(self._command)
            if self._return_code != 0:
                detailed.append("  ERROR: EXIT {0}".format(self._return_code))
                if self._stdout is not None and self._stdout != "":
                    detailed.append(self._stdout)
                if self._stderr is not None and self._stderr != "":
                    detailed.append(self._stderr)

        return detailed

    def generate_summary_report(self):
        """
        Generate report with the result of running:
        Good or ERROR: EXIT or Not Present
        """
        summary = []
        short_command = os.path.basename(self._command)
        if self._present is True:
            if self._return_code is not None:
                if self._return_code == 0:
                    status = "GOOD"
                else:
                    status = "ERROR: EXIT {0}".format(self._return_code)
            else:
                status = "RETURN CODE IS NONE"
        else:
            status = " Not Present"
        summary.append("    {0}: {1}".format(short_command, status))
        return summary

    @staticmethod
    def summarize_errors(results):
        errors = 0
        for result in results:
            if result.return_code != 0:
                errors += 1
        return errors

class BuildCommand(object):
    """
    A module for build command.
    """
    def __init__(self, name, directory, arguments=None, use_sudo=False, sudo_creds=None):
        """
        _name: the command name that was supposed to be run
        _directory: the directory under which to run command
        _use_sudo: whether the command was run with sudo previleges
        _sudo_creds: the environment variable name of sudo credentials.
                     For example: SUDO_CREDS=username:password
        _arguments: the arguments of the command
        """
        self._name = name
        self._directory = directory
        self._use_sudo = use_sudo
        self._sudo_creds = sudo_creds
        self._arguments = arguments

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, name):
        self._name = name

    @property
    def directory(self):
        return self._directory

    @directory.setter
    def directory(self, directory):
        self._directory = directory

    @property
    def use_sudo(self):
        return self._use_sudo

    @use_sudo.setter
    def use_sudo(self, use_sudo):
        self._use_sudo = use_sudo

    @property
    def sudo_creds(self):
        return self._sudo_creds

    @sudo_creds.setter
    def sudo_creds(self, sudo_creds):
        self._sudo_creds = sudo_creds

    @staticmethod
    def import_environment_file(env_file):
        print "start to import environment file {0}".format(env_file)
        if not os.path.isfile(env_file):
            raise RuntimeError("Failed to import environment file due to the file {0} doesn't exist"
                               .format(env_file))
        props = common.parse_property_file(env_file)
        if props:
            my_env = os.environ.copy()
            my_env_str = ""
            for item in props.items():
                key = item[0]
                value = item[1]
                my_env[key] = value
                my_env_str += "{key}={value} ".format(key=key, value=value)
        return my_env, my_env_str

    def to_string(self, my_env_str):
        cmd_args = []
        if self._use_sudo:
            if self._sudo_creds is None:
                raise ValueError("sudo credential missing from commands {0}".format(name))
            (username, password) = common.parse_credential_variable(self._sudo_creds)
            cmd_args += ["echo"]
            cmd_args += [password]
            cmd_args += ["|sudo -S"]
            if my_env_str is not None and my_env_str != "":
                cmd_args += ["env "]
                cmd_args += [my_env_str]

        cmd_args += [self._name]

        if self._arguments:
            cmd_args += self._arguments

        command_str = " ".join(cmd_args)
        return command_str

    def run(self, env_file = None):
        try:
            my_env = os.environ.copy()
            my_env_str = ""
            if env_file is not None:
                my_env, my_env_str = self.import_environment_file(env_file)

            command = self.to_string(my_env_str)
            print "Execute command: {0} under {1}".format(command, self._directory)
            proc = subprocess.Popen(command,
                                    stderr=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    cwd=self._directory,
                                    env=my_env,
                                    shell=True)
            (out, err) = proc.communicate()
        except Exception, ex:
            # this is a terrible failure, not just process exit != 0
            return BuildResult(self._name,
                               present=True,
                               return_code=1,
                               stderr=ex)

        result = BuildResult(self._name,
                             present=True,
                             return_code=proc.returncode,
                             stdout=out,
                             stderr=err)
        return result

    def is_executable(self):
        exe_file = os.path.join(self._directory, self._name)
        if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
            return True

        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
                return True

        return False

class Builder(ParallelTasks):
    """
    Run a list of command under a directory.

    This class is intended for use with ParallelTasks, and each commands list may be done
    in a separate process.

    """
    def add_task(self, data, name):
        """
        Add a task to task queue
        :param data: A dictonary which should contain:
                     commands: A list of comamnd instances. It's required.
                     env_file: A property file which contains environment variables. It's optional
        :param name: The name of the task. The key by which the job results will be returned.
        :return: None
        """
        if data is None:
            raise ValueError("Task parameter data not present")

        if 'commands' not in data:
            raise ValueError("commands key missing from data: {0}".format(data))

        super(Builder, self).add_task(data, name)

    def do_one_task(self, name, data, results):
        """
        Perform the actual work.
        name and data will come from the values passed in to add_task()
        :param results: a list of instances of BuildResult
        :return: None
        """
        try:
            if 'command' not in results:
                results['command'] = []

            my_env_file = None
            if 'env_file' in data and data['env_file'] is not None:
                my_env_file = data['env_file']

            for command in data['commands']:
                if not command.is_executable():
                    build_result = BuildResult(command, present=False)
                build_result = command.run(env_file=my_env_file)
                if build_result is not None:
                    results['command'].append(build_result)

        except Exception, e:
            raise RuntimeError("Failed to do task {0} due to {1}".format(name, e))

    def summarize_results(self):
        """
        :return: True if the number of errors found is 0
                 False if the number of errors found is greater than 0
        """
        results = self.get_results()
        key_list = results.keys()

        for name in sorted(key_list):
            if 'command' in results[name]:
                build_results = results[name]['command']
                errors = BuildResult.summarize_errors(build_results)
                if errors > 0:
                    return False
        return True

    def generate_detailed_report(self):
        """
        Generate reports with details: return code, stdout, stderr
        """
        all_detailed = []
        results = self.get_results()
        key_list = results.keys()
        for name in sorted(key_list):
            if 'command' in results[name]:
                task_detailed = []
                task_detailed.append("=== Results for {0} ===".format(name))
                for result in results[name]['command']:
                    detailed = result.generate_detailed_report()
                    task_detailed.extend(detailed)

                all_detailed.extend(task_detailed)
        return all_detailed

    def generate_summary_report(self):
        """
        Generate report with the result of running:
        Good or ERROR: EXIT or Not Present
        """
        all_summary = []

        results = self.get_results()
        key_list = results.keys()

        for name in sorted(key_list):
            if 'command' in results[name]:
                task_summary = []
                task_summary.append("{0}:".format(name))
                build_results = results[name]['command']
                errors = BuildResult.summarize_errors(build_results)
                if errors > 0:
                    task_summary.append("    Number of falied commands: {0}".format(errors))

                for result in build_results:
                    summary = result.generate_summary_report()
                    task_summary.extend(summary)

                all_summary.extend(task_summary)

        return all_summary

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Retrieve passing and failing WebKit revision numbers from canaries.

From each canary,
- the last WebKit revision number for which all the tests have passed,
- the last WebKit revision number for which the tests were run, and
- the names of failing layout tests
are retrieved and printed.
"""

import json
import optparse
import sys
import urllib2

_DEFAULT_BUILDERS = [
  "Webkit Win",
  "Webkit Vista",
  "Webkit Win7",
  "Webkit Win (dbg)(1)",
  "Webkit Win (dbg)(2)",
  "Webkit Mac10.5",
  "Webkit Mac10.6",
  "Webkit Mac10.5 (dbg)(1)",
  "Webkit Mac10.5 (dbg)(2)",
  "Webkit Mac10.6 (dbg)",
  "Webkit Linux",
  "Webkit Linux 64",
  "Webkit Linux (dbg)(1)",
  "Webkit Linux (dbg)(2)",
]
_DEFAULT_MAX_BUILDS = 10
_TEST_PREFIX = "&tests="
_TEST_SUFFIX = '">'
_WEBKIT_TESTS = "webkit_tests"

class _BuildResult(object):
  """Build result for a builder.

  Holds builder name, the last passing revision, the last run revision, and
  a list of names of failing tests. Revision nubmer 0 is used to represent
  that the revision doesn't exist.
  """
  def __init__(self, builder, last_passing_revision, last_run_revision,
               failing_tests):
    """Constructs build results."""
    self.builder = builder
    self.last_passing_revision = last_passing_revision
    self.last_run_revision = last_run_revision
    self.failing_tests = failing_tests

def _BuilderUrlFor(builder, max_builds):
  """Constructs the URL for a builder to retrieve the last results."""
  return ("http://build.chromium.org/p/chromium.webkit/json/builders/" +
          urllib2.quote(builder) + "/builds?as_text=1&" +
          '&'.join(["select=%d" % -i for i in range(1, 1 + max_builds)]))

def _ExtractFailingTests(build):
  """Extracts failing test names from a build result entry JSON object."""
  for step in build["steps"]:
    if step["name"] == _WEBKIT_TESTS:
      for text in step["text"]:
        prefix = text.find(_TEST_PREFIX)
        suffix = text.find(_TEST_SUFFIX)
        if prefix != -1 and suffix != -1:
          return sorted(text[prefix + len(_TEST_PREFIX): suffix].split(","))

def _RetrieveBuildResult(builder, max_builds):
  """Retrieves build results for a builder.

  Checks the last passing revision, the last run revision, and failing tests
  for the last builds of a builder.

  Args:
      builder: Builder name.
      max_builds: Maximum number of builds to check.

  Returns:
      _BuildResult instance.
  """
  last_run_revision = 0
  failing_tests = []
  succeeded = False
  builds_json = urllib2.urlopen(_BuilderUrlFor(builder, max_builds))
  if not builds_json:
    return _BuildResult(builder, 0, 0, failing_tests)
  builds = [(int(value["number"]), value) for unused_key, value
            in json.loads(''.join(builds_json)).items()
            if value.has_key("number")]
  builds.sort()
  builds.reverse()
  for unused_key, build in builds:
    if not build.has_key("text"):
      continue
    if len(build["text"]) < 2:
      continue
    if not build.has_key("sourceStamp"):
      continue
    if build["text"][1] == "successful":
      succeeded = True
    elif not failing_tests and _WEBKIT_TESTS in build["text"][1:]:
      failing_tests = _ExtractFailingTests(build)
    revision = 0
    if build["sourceStamp"]["branch"] == "trunk":
      revision = int(build["sourceStamp"]["changes"][-1]["revision"])
    if revision and not last_run_revision:
      last_run_revision = revision
    if not succeeded or not revision:
      continue
    return _BuildResult(builder, revision, last_run_revision, failing_tests)
  return _BuildResult(builder, 0, last_run_revision, failing_tests)

def _PrintPassingRevisions(results):
  """Prints passing revisions and the range of such revisions.

  Args:
      results: A list of build results.
  """
  print "**** Passing revisions *****"
  minimum_revision = sys.maxint
  maximum_revision = 0
  for result in results:
    if result.last_passing_revision:
      minimum_revision = min(minimum_revision, result.last_passing_revision)
      maximum_revision = max(maximum_revision, result.last_passing_revision)
      print 'The last passing run was at r%d on "%s"' % (
          result.last_passing_revision, result.builder)
    else:
      print 'No passing runs on "%s"' % result.builder
  if maximum_revision:
    print "Passing revision range: r%d - r%d" % (
          minimum_revision, maximum_revision)

def _PrintFailingRevisions(results):
  """Prints failing revisions and the failing tests.

  Args:
      results: A list of build results.
  """
  print "**** Failing revisions *****"
  for result in results:
    if result.last_run_revision and result.failing_tests:
      print ('The last run was at r%d on "%s" and the following %d tests'
             ' failed' % (result.last_run_revision, result.builder,
                          len(result.failing_tests)))
      for test in result.failing_tests:
        print "  " + test

def _ParseOptions():
  """Parses command-line options."""
  parser = optparse.OptionParser(usage="%prog [options] [builders]")
  parser.add_option("-m", "--max_builds", type="int",
                    default=_DEFAULT_MAX_BUILDS,
                    help="maximum number of builds to check for each builder")
  return parser.parse_args()

def _Main():
  """The main function."""
  options, builders = _ParseOptions()
  if not builders:
    builders = _DEFAULT_BUILDERS
  print "Checking the last %d builds for:" % options.max_builds
  sys.stdout.flush()
  results = []
  for builder in builders:
    print '"%s"' % builder
    sys.stdout.flush()
    results.append(_RetrieveBuildResult(builder, options.max_builds))
  _PrintFailingRevisions(results)
  _PrintPassingRevisions(results)

if __name__ == "__main__":
  _Main()
<commit_msg>Check the canary builds up to the last-rolled WebKit revision.<commit_after>#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Retrieve passing and failing WebKit revision numbers from canaries.

From each canary,
- the last WebKit revision number for which all the tests have passed,
- the last WebKit revision number for which the tests were run, and
- the names of failing layout tests
are retrieved and printed.
"""

import json
import optparse
import re
import sys
import urllib2

_WEBKIT_REVISION_IN_DEPS_RE = re.compile(r'"webkit_revision"\s*:\s*"(\d+)"')
_DEPS_FILE_URL = "http://src.chromium.org/viewvc/chrome/trunk/src/DEPS"
_DEFAULT_BUILDERS = [
  "Webkit Win",
  "Webkit Vista",
  "Webkit Win7",
  "Webkit Win (dbg)(1)",
  "Webkit Win (dbg)(2)",
  "Webkit Mac10.5",
  "Webkit Mac10.6",
  "Webkit Mac10.5 (dbg)(1)",
  "Webkit Mac10.5 (dbg)(2)",
  "Webkit Mac10.6 (dbg)",
  "Webkit Linux",
  "Webkit Linux 64",
  "Webkit Linux (dbg)(1)",
  "Webkit Linux (dbg)(2)",
]
_DEFAULT_MAX_BUILDS = 10
_TEST_PREFIX = "&tests="
_TEST_SUFFIX = '">'
_WEBKIT_TESTS = "webkit_tests"

def _OpenUrl(url):
  """Opens a URL.

  Returns:
      A file-like object in case of success, an empty list otherwise.
  """
  try:
    return urllib2.urlopen(url)
  except urllib2.URLError, url_error:
    message = ""
    # Surprisingly, urllib2.URLError has different attributes based on the
    # kinds of errors -- "code" for HTTP-level errors, "reason" for others.
    if hasattr(url_error, "code"):
      message = "Status code: %d" % url_error.code
    if hasattr(url_error, "reason"):
      message = url_error.reason
    print >>sys.stderr, "Failed to open %s: %s" % (url, message)
  return []

def _WebkitRevisionInDeps():
  """Returns the WebKit revision specified in DEPS file.

  Returns:
      Revision number as int. -1 in case of error.
  """
  for line in _OpenUrl(_DEPS_FILE_URL):
    match = _WEBKIT_REVISION_IN_DEPS_RE.search(line)
    if match:
      return int(match.group(1))
  return -1

class _BuildResult(object):
  """Build result for a builder.

  Holds builder name, the last passing revision, the last run revision, and
  a list of names of failing tests. Revision nubmer 0 is used to represent
  that the revision doesn't exist.
  """
  def __init__(self, builder, last_passing_revision, last_run_revision,
               failing_tests):
    """Constructs build results."""
    self.builder = builder
    self.last_passing_revision = last_passing_revision
    self.last_run_revision = last_run_revision
    self.failing_tests = failing_tests

def _BuilderUrlFor(builder, max_builds):
  """Constructs the URL for a builder to retrieve the last results."""
  url = ("http://build.chromium.org/p/chromium.webkit/json/builders/%s/builds" %
         urllib2.quote(builder))
  if max_builds == -1:
    return url + "/_all?as_text=1"
  return (url + "?as_text=1&" +
          '&'.join(["select=%d" % -i for i in range(1, 1 + max_builds)]))

def _ExtractFailingTests(build):
  """Extracts failing test names from a build result entry JSON object."""
  for step in build["steps"]:
    if step["name"] == _WEBKIT_TESTS:
      for text in step["text"]:
        prefix = text.find(_TEST_PREFIX)
        suffix = text.find(_TEST_SUFFIX)
        if prefix != -1 and suffix != -1:
          return sorted(text[prefix + len(_TEST_PREFIX): suffix].split(","))

def _RetrieveBuildResult(builder, max_builds, oldest_revision_to_check):
  """Retrieves build results for a builder.

  Checks the last passing revision, the last run revision, and failing tests
  for the last builds of a builder.

  Args:
      builder: Builder name.
      max_builds: Maximum number of builds to check.
      oldest_revision_to_check: Oldest WebKit revision to check.

  Returns:
      _BuildResult instance.
  """
  last_run_revision = 0
  failing_tests = []
  succeeded = False
  builds_json = _OpenUrl(_BuilderUrlFor(builder, max_builds))
  if not builds_json:
    return _BuildResult(builder, 0, 0, failing_tests)
  builds = [(int(value["number"]), value) for unused_key, value
            in json.loads(''.join(builds_json)).items()
            if value.has_key("number")]
  builds.sort()
  builds.reverse()
  for unused_key, build in builds:
    if not build.has_key("text"):
      continue
    if len(build["text"]) < 2:
      continue
    if not build.has_key("sourceStamp"):
      continue
    if build["text"][1] == "successful":
      succeeded = True
    elif not failing_tests and _WEBKIT_TESTS in build["text"][1:]:
      failing_tests = _ExtractFailingTests(build)
    revision = 0
    if build["sourceStamp"]["branch"] == "trunk":
      revision = int(build["sourceStamp"]["changes"][-1]["revision"])
    if revision and not last_run_revision:
      last_run_revision = revision
    if revision and revision < oldest_revision_to_check:
      break
    if not succeeded or not revision:
      continue
    return _BuildResult(builder, revision, last_run_revision, failing_tests)
  return _BuildResult(builder, 0, last_run_revision, failing_tests)

def _PrintPassingRevisions(results):
  """Prints passing revisions and the range of such revisions.

  Args:
      results: A list of build results.
  """
  print "**** Passing revisions *****"
  min_passing_revision = sys.maxint
  max_passing_revision = 0
  for result in results:
    if result.last_passing_revision:
      min_passing_revision = min(min_passing_revision,
                                 result.last_passing_revision)
      max_passing_revision = max(max_passing_revision,
                                 result.last_passing_revision)
      print 'The last passing run was at r%d on "%s"' % (
          result.last_passing_revision, result.builder)
    else:
      print 'No passing runs on "%s"' % result.builder
  if max_passing_revision:
    print "Passing revision range: r%d - r%d" % (
          min_passing_revision, max_passing_revision)

def _PrintFailingRevisions(results):
  """Prints failing revisions and the failing tests.

  Args:
      results: A list of build results.
  """
  print "**** Failing revisions *****"
  for result in results:
    if result.last_run_revision and result.failing_tests:
      print ('The last run was at r%d on "%s" and the following %d tests'
             ' failed' % (result.last_run_revision, result.builder,
                          len(result.failing_tests)))
      for test in result.failing_tests:
        print "  " + test

def _ParseOptions():
  """Parses command-line options."""
  parser = optparse.OptionParser(usage="%prog [options] [builders]")
  parser.add_option("-m", "--max_builds", type="int",
                    default=-1,
                    help="Maximum number of builds to check for each builder."
                         " Defaults to all builds for which record is"
                         " available. Checking is ended either when the maximum"
                         " number is reached, the remaining builds are older"
                         " than the DEPS WebKit revision, or a passing"
                         " revision is found.")
  return parser.parse_args()

def _Main():
  """The main function."""
  options, builders = _ParseOptions()
  if not builders:
    builders = _DEFAULT_BUILDERS
  oldest_revision_to_check = _WebkitRevisionInDeps()
  if options.max_builds == -1 and oldest_revision_to_check == -1:
    options.max_builds = _DEFAULT_MAX_BUILDS
  if options.max_builds != -1:
    print "Maxium number of builds to check: %d" % options.max_builds
  if oldest_revision_to_check != -1:
    print "Oldest revision to check: %d" % oldest_revision_to_check
  sys.stdout.flush()
  results = []
  for builder in builders:
    print '"%s"' % builder
    sys.stdout.flush()
    results.append(_RetrieveBuildResult(
        builder, options.max_builds, oldest_revision_to_check))
  _PrintFailingRevisions(results)
  _PrintPassingRevisions(results)

if __name__ == "__main__":
  _Main()
<|endoftext|>


<commit_before>import logging
import os

import dateutil.parser
from gi.repository import GObject

from .service import Service, ServiceError
from ..instrument.sample import Sample
from ..utils.errorvalue import ErrorValue

logger=logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class SampleStoreError(ServiceError):
    pass

class SampleStore(Service):
    __gsignals__={'list-changed':(GObject.SignalFlags.RUN_FIRST, None, ()),
                  'active-changed':(GObject.SignalFlags.RUN_FIRST, None, ()),
                  }

    def __init__(self, *args, **kwargs):
        Service.__init__(self, *args, **kwargs)
        self._list = []
        self._active = None

    def _load_state(self, dictionary):
        Service._load_state(self, dictionary)
        if isinstance(dictionary['list'], list):
            self._list = [Sample.fromdict(sampledict)
                          for sampledict in dictionary['list']]
        else:
            self._list=[Sample.fromdict(sampledict) for sampledict in dictionary['list'].values()]
        self._active = dictionary['active']
        try:
            with open(os.path.join(self.instrument.config['path']['directories']['config'], 'samples.conf'), 'rt', encoding='utf-8') as f:
                for l in f:
                    try:
                        rhs = l.split('=', 1)[1].strip()
                    except IndexError:
                        pass
                    if l.startswith('['):
                        sample = {}
                    elif l.startswith('title ='):
                        sample['title'] = rhs
                    elif l.startswith('positionx ='):
                        sample['positionx'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('positionxerror ='):
                        sample['positionx'].err = float(rhs)
                    elif l.startswith('positiony ='):
                        sample['positiony'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('positionyerror ='):
                        sample['positiony'].err = float(rhs)
                    elif l.startswith('transmission ='):
                        sample['transmission'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('transmissionerror ='):
                        sample['transmission'].err = float(rhs)
                    elif l.startswith('thickness ='):
                        sample['thickness'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('thicknesserror ='):
                        sample['thickness'].err = float(rhs)
                    elif l.startswith('distminus ='):
                        sample['distminus'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('distminuserror ='):
                        sample['distminus'].err = float(rhs)
                    elif l.startswith('preparedby ='):
                        sample['preparedby'] = rhs
                    elif l.startswith('preparetime ='):
                        sample['preparetime'] = dateutil.parser.parse(rhs)
                    elif l.startswith('description ='):
                        sample['description'] = rhs
                    elif l.startswith('category ='):
                        sample['category'] = rhs
                    elif l.startswith('situation ='):
                        sample['situation'] = rhs
                    else:
                        if sample['title'] in self:
                            self.remove(sample['title'])
                        self.add(Sample(**sample))
        except IOError:
            pass
        self.emit('list-changed')

    def _save_state(self):
        dic = Service._save_state(self)
        dic['active'] = self._active
        dic['list'] = {x.title:x.todict() for x in self._list}
        return dic

    def add(self, sample):
        if not [s for s in self._list if s.title == sample.title]:
            self._list.append(sample)
            self._list=sorted(self._list, key=lambda x:x.title)
            self.emit('list-changed')
        else:
            return False
        #        if self._active is None:
        #            self._active = sample.title
        #            self.emit('active-changed')
        return True

    def remove(self, sample):
        if isinstance(sample, Sample):
            sample=sample.title
        if not [s for s in self._list if s.title==sample]:
            raise KeyError('Unknown sample with title %s'%sample)
        self._list = [s for s in self._list if s.title != sample]
        if sample == self._active:
            try:
                self._active = self._list[0]
            except IndexError:
                self._active = None
            self.emit('active-changed')
        self.emit('list-changed')

    def set_active(self, sample):
        """sample: string or None"""
        assert (isinstance(sample, str) or (sample is None))
        if sample is None:
            self._active = None
            self.emit('active-changed')
            return
        if isinstance(sample, Sample):
            sample = sample.title
        if [s for s in self._list if s.title == sample]:
            self._active = sample
            self.emit('active-changed')
        else:
            raise SampleStoreError('No sample %s defined.' % sample)

    def get_active(self):
        if self._active is None:
            return None
        else:
            try:
                return [x for x in self._list if x.title == self._active][0]
            except IndexError:
                self._active = None
                return None

    def get_active_name(self):
        if self._active is None:
            return None
        else:
            return self.get_active().title

    def __iter__(self):
        for l in self._list:
            yield l

    def get_sample(self, title):
        sample=[s for s in self._list if s.title==title]
        if not sample:
            raise KeyError('Unknown sample:',title)
        assert(len(sample)==1)
        return sample[0]

    def set_sample(self, title, sample):
        self._list=sorted([s for s in self._list if s.title !=title]+[sample], key=lambda x:x.title)
        self.emit('list-changed')

    def __contains__(self, item):
        return bool([s for s in self._list if s.title == item])
<commit_msg>Saving the current sample correctly (only the title, not the full Sample class, which is not JSON-serializable)<commit_after>import logging
import os

import dateutil.parser
from gi.repository import GObject

from .service import Service, ServiceError
from ..instrument.sample import Sample
from ..utils.errorvalue import ErrorValue

logger=logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class SampleStoreError(ServiceError):
    pass

class SampleStore(Service):
    __gsignals__={'list-changed':(GObject.SignalFlags.RUN_FIRST, None, ()),
                  'active-changed':(GObject.SignalFlags.RUN_FIRST, None, ()),
                  }

    def __init__(self, *args, **kwargs):
        Service.__init__(self, *args, **kwargs)
        self._list = []
        self._active = None

    def _load_state(self, dictionary):
        Service._load_state(self, dictionary)
        if isinstance(dictionary['list'], list):
            self._list = [Sample.fromdict(sampledict)
                          for sampledict in dictionary['list']]
        else:
            self._list=[Sample.fromdict(sampledict) for sampledict in dictionary['list'].values()]
        self._active = dictionary['active']
        try:
            with open(os.path.join(self.instrument.config['path']['directories']['config'], 'samples.conf'), 'rt', encoding='utf-8') as f:
                for l in f:
                    try:
                        rhs = l.split('=', 1)[1].strip()
                    except IndexError:
                        pass
                    if l.startswith('['):
                        sample = {}
                    elif l.startswith('title ='):
                        sample['title'] = rhs
                    elif l.startswith('positionx ='):
                        sample['positionx'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('positionxerror ='):
                        sample['positionx'].err = float(rhs)
                    elif l.startswith('positiony ='):
                        sample['positiony'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('positionyerror ='):
                        sample['positiony'].err = float(rhs)
                    elif l.startswith('transmission ='):
                        sample['transmission'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('transmissionerror ='):
                        sample['transmission'].err = float(rhs)
                    elif l.startswith('thickness ='):
                        sample['thickness'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('thicknesserror ='):
                        sample['thickness'].err = float(rhs)
                    elif l.startswith('distminus ='):
                        sample['distminus'] = ErrorValue(float(rhs), 0)
                    elif l.startswith('distminuserror ='):
                        sample['distminus'].err = float(rhs)
                    elif l.startswith('preparedby ='):
                        sample['preparedby'] = rhs
                    elif l.startswith('preparetime ='):
                        sample['preparetime'] = dateutil.parser.parse(rhs)
                    elif l.startswith('description ='):
                        sample['description'] = rhs
                    elif l.startswith('category ='):
                        sample['category'] = rhs
                    elif l.startswith('situation ='):
                        sample['situation'] = rhs
                    else:
                        if sample['title'] in self:
                            self.remove(sample['title'])
                        self.add(Sample(**sample))
        except IOError:
            pass
        self.emit('list-changed')

    def _save_state(self):
        dic = Service._save_state(self)
        dic['active'] = self._active.title
        dic['list'] = {x.title:x.todict() for x in self._list}
        return dic

    def add(self, sample):
        if not [s for s in self._list if s.title == sample.title]:
            self._list.append(sample)
            self._list=sorted(self._list, key=lambda x:x.title)
            self.emit('list-changed')
        else:
            return False
        #        if self._active is None:
        #            self._active = sample.title
        #            self.emit('active-changed')
        return True

    def remove(self, sample):
        if isinstance(sample, Sample):
            sample=sample.title
        if not [s for s in self._list if s.title==sample]:
            raise KeyError('Unknown sample with title %s'%sample)
        self._list = [s for s in self._list if s.title != sample]
        if sample == self._active:
            try:
                self._active = self._list[0]
            except IndexError:
                self._active = None
            self.emit('active-changed')
        self.emit('list-changed')

    def set_active(self, sample):
        """sample: string or None"""
        assert (isinstance(sample, str) or (sample is None))
        if sample is None:
            self._active = None
            self.emit('active-changed')
            return
        if isinstance(sample, Sample):
            sample = sample.title
        if [s for s in self._list if s.title == sample]:
            self._active = sample
            self.emit('active-changed')
        else:
            raise SampleStoreError('No sample %s defined.' % sample)

    def get_active(self):
        if self._active is None:
            return None
        else:
            try:
                return [x for x in self._list if x.title == self._active][0]
            except IndexError:
                self._active = None
                return None

    def get_active_name(self):
        if self._active is None:
            return None
        else:
            return self.get_active().title

    def __iter__(self):
        for l in self._list:
            yield l

    def get_sample(self, title):
        sample=[s for s in self._list if s.title==title]
        if not sample:
            raise KeyError('Unknown sample:',title)
        assert(len(sample)==1)
        return sample[0]

    def set_sample(self, title, sample):
        self._list=sorted([s for s in self._list if s.title !=title]+[sample], key=lambda x:x.title)
        self.emit('list-changed')

    def __contains__(self, item):
        return bool([s for s in self._list if s.title == item])
<|endoftext|>