<commit_before>#!/usr/bin/env python

from datetime import date
import cv2
import numpy as np
import os
import random
from directory import Directory
import xml.etree.ElementTree as xml

import common as com
from ibeis_image import IBEIS_Image

class IBEIS_Data(object):

    def __init__(ibsd, dataset_path, **kwargs):
        com._kwargs(kwargs, 'object_min_width',    32)
        com._kwargs(kwargs, 'object_min_height',   32)
        com._kwargs(kwargs, 'mine_negatives',      True)
        com._kwargs(kwargs, 'mine_width_min',      50)
        com._kwargs(kwargs, 'mine_width_max',      400)
        com._kwargs(kwargs, 'mine_height_min',     50)
        com._kwargs(kwargs, 'mine_height_max',     400)
        com._kwargs(kwargs, 'mine_max_attempts',   100)
        com._kwargs(kwargs, 'mine_max_keep',       10)
        com._kwargs(kwargs, 'mine_overlap_margin', 0.25)
        com._kwargs(kwargs, 'mine_exclude_categories', [])

        ibsd.dataset_path = dataset_path
        ibsd.absolute_dataset_path = os.path.realpath(dataset_path)

        direct = Directory(os.path.join(ibsd.dataset_path, "Annotations") , include_file_extensions=["xml"])
        ibsd.images = []
        files = direct.files()
        print "Loading Database"
        for i, filename in enumerate(files):
            if len(files) > 10:
                if i % (len(files) / 10) == 0:
                    print "%0.2f" %(float(i) / len(files))
            ibsd.images.append(IBEIS_Image(filename, ibsd.absolute_dataset_path, **kwargs))
        print "    ...Loaded"

        ibsd.categories_images = []
        ibsd.categories_rois = []

        for image in ibsd.images:
            temp = image.categories(unique=False)
            ibsd.categories_rois += temp
            ibsd.categories_images += set(temp)

            if len(image.objects) == 0:
                ibsd.categories_images += ["IGNORED"]

        ibsd.distribution_images = com.histogram(ibsd.categories_images)
        ibsd.distribution_rois = com.histogram(ibsd.categories_rois)
        ibsd.rois = sum(ibsd.distribution_rois.values())

        ibsd.categories = sorted(set(ibsd.categories_images))

    def __str__(ibsd):
        return "<IBEIS Data Object | %s | %d images | %d categories | %d rois>" \
            %(ibsd.absolute_dataset_path, len(ibsd.images), len(ibsd.categories), ibsd.rois)

    def __repr__(ibsd):
        return "<IBEIS Data Object | %s>" % (ibsd.absolute_dataset_path)

    def __len__(ibsd):
        return len(ibsd.images)

    def __getitem__(ibsd, key):
        if isinstance(key, str):
            for image in ibsd.images:
                if key in image.filename:
                    return image
            return None
        else:
            return ibsd.images[key]

    def print_distribution(ibsd):
        def _print_line(category, spacing, images, rois):
            images = str(images)
            rois = str(rois)
            print "%s%s\t%s" %(category + " " * (spacing - len(category)), images, rois)

        _max = max([ len(category) for category in ibsd.distribution_rois.keys() + ['TOTAL', 'CATEGORY'] ]) + 3

        _print_line("CATEGORY", _max, "IMGs", "ROIs")
        if "IGNORED" in ibsd.distribution_images:
            _print_line("IGNORED", _max, ibsd.distribution_images["IGNORED"], "")

        for category in sorted(ibsd.distribution_rois):
            _print_line(category, _max, ibsd.distribution_images[category], ibsd.distribution_rois[category])

        _print_line("TOTAL", _max, len(ibsd.images), ibsd.rois)

    def dataset(ibsd, positive_category, neg_exclude_categories=[], max_rois_pos=None, max_rois_neg=None):
        def _parse_dataset_file(category, _type):
            filepath = os.path.join(ibsd.dataset_path, "ImageSets", "Main", category + "_" + _type + ".txt")
            _dict = {}
            try:
                _file = open(filepath)
                for line in _file:
                    line = line.strip().split(" ")
                    line = [line[0], line[-1]]
                    _dict[line[0]] = True
            except IOError as e:
                print "<", e, ">", filepath

            return _dict

        positives = []
        negatives = []
        validation = []
        test = []

        train_values = _parse_dataset_file(positive_category, "train")
        val_values = _parse_dataset_file(positive_category, "val")
        test_values = _parse_dataset_file(positive_category, "test")

        pos_rois = 0
        neg_rois = 0
        for image in ibsd.images:
            filename = image.filename
            _train = train_values.get(image.filename[:-4], False)
            _val = val_values.get(image.filename[:-4], False)
            _test = test_values.get(image.filename[:-4], False)

            temp = image.categories(unique=False)
            flag = False

            if _train:
                for val in temp:
                    if val == positive_category:
                        flag = True
                        pos_rois += 1
                    elif val not in neg_exclude_categories:
                        neg_rois += 1

                if flag:
                    positives.append(image)
                elif val not in neg_exclude_categories:
                    negatives.append(image)

            if _val:
                validation.append(image)

            if _test:
                test.append(image)

        # Setup auto normalize variables for equal positives and negatives
        if max_rois_pos == 'auto' or max_rois_pos == -1:
            max_rois_pos = neg_rois

        if max_rois_neg == 'auto' or max_rois_neg == -1:
            max_rois_neg = pos_rois

        # Remove positives to target, not gauranteed to give target, but 'close'.
        if max_rois_pos is not None and len(positives) > 0:
            pos_density = float(pos_rois) / len(positives)
            target_num = int(max_rois_pos / pos_density)
            print "Normalizing Positives, Target:", target_num

            # Remove images to match target
            while len(positives) > target_num:
                positives.pop( com.randInt(0, len(positives) - 1) )

            # Recalculate rois left
            pos_rois = 0
            for image in positives:
                temp = image.categories(unique=False)
                for val in temp:
                    if val in positive_category:
                        pos_rois += 1

        # Remove positives to target, not gauranteed to give target, but 'close'.
        if max_rois_neg is not None and len(negatives) > 0:
            neg_density = float(neg_rois) / len(negatives)
            target_num = int(max_rois_neg / neg_density)
            print "Normalizing Negatives, Target:", target_num

            # Remove images to match target
            while len(negatives) > target_num:
                negatives.pop( com.randInt(0, len(negatives) - 1) )

            # Recalculate rois left
            neg_rois = 0
            for image in negatives:
                temp = image.categories(unique=False)
                for val in temp:
                    if val not in positive_category:
                        neg_rois += 1

        print("%s\t%s\t%s\t%s\t%s" %("       ", "Pos", "Neg", "Val", "Test"))
        print("%s\t%s\t%s\t%s\t%s" %("Images:", len(positives), len(negatives), len(validation), len(test)))
        print("%s\t%s\t%s\t%s\t%s" %("ROIs:  ", pos_rois, neg_rois, "", ""))

        return (positives, pos_rois), (negatives, neg_rois), validation, test

    def export_yaml(ibsd):
        os.system("mkdir " + os.path.join(ibsd.dataset_path, "AnnotationsYAML"))

        for image in ibsd.images:
            _filename = image.filename.split(".")[0] + ".yml"

            template_an = open("/Users/bluemellophone/code/python-modules/pypascalxml/template_annotation.yml", 'r')
            template_ob = open("/Users/bluemellophone/code/python-modules/pypascalxml/template_object.yml", 'r')
            template_pt = open("/Users/bluemellophone/code/python-modules/pypascalxml/template_part.yml", 'r')
            template_an = ''.join(template_an.readlines())
            template_ob = ''.join(template_ob.readlines())
            template_pt = ''.join(template_pt.readlines())

            template_an = template_an.replace('_^_FOLDER_^_', image.folder)
            template_an = template_an.replace('_^_FILENAME_^_', image.filename)
            template_an = template_an.replace('_^_DARABASE_NAME_^_', image.source_database)
            template_an = template_an.replace('_^_DATABASE_YEAR_^_', image.source_annotation)
            template_an = template_an.replace('_^_SOURCE_^_', image.source_image)
            template_an = template_an.replace('_^_WIDTH_^_', str(image.width))
            template_an = template_an.replace('_^_HEIGHT_^_', str(image.height))
            template_an = template_an.replace('_^_CHANNELS_^_', str(image.depth))
            template_an = template_an.replace('_^_SEGMENTED_^_', str(int(image.segmented)))

            objects = []
            for _object in image.objects:
                if _object.name == "MINED":
                    continue

                temp_ob = template_ob[:]

                temp_ob = temp_ob.replace('_^_NAME_^_', _object.name)
                temp_ob = temp_ob.replace('_^_POSE_^_', _object.pose)
                temp_ob = temp_ob.replace('_^_TRUNCATED_^_', str(int(_object.truncated)))
                temp_ob = temp_ob.replace('_^_DIFFICULT_^_', str(int(_object.difficult)))
                temp_ob = temp_ob.replace('_^_XMIN_^_', str(_object.xmin))
                temp_ob = temp_ob.replace('_^_YMIN_^_', str(_object.ymin))
                temp_ob = temp_ob.replace('_^_XMAX_^_', str(_object.xmax))
                temp_ob = temp_ob.replace('_^_YMAX_^_', str(_object.ymax))

                parts = []
                for  part in _object.parts:
                    temp_pt = template_pt[:]

                    temp_pt = temp_pt.replace('_^_NAME_^_', part.name)
                    temp_pt = temp_pt.replace('_^_XMIN_^_', str(part.xmin))
                    temp_pt = temp_pt.replace('_^_YMIN_^_', str(part.ymin))
                    temp_pt = temp_pt.replace('_^_XMAX_^_', str(part.xmax))
                    temp_pt = temp_pt.replace('_^_YMAX_^_', str(part.ymax))

                    parts.append(temp_pt)

                if len(parts) > 0:
                    parts =  "\n\t\t\tpart:" + "".join(parts)
                else:
                    parts = ""

                temp_ob = temp_ob.replace('_^_PART_MULTIPLE_OPTIONAL_^_', parts)

                objects.append(temp_ob)

            if len(objects) > 0:
                objects =  "\n\tobject:" + "".join(objects)
            else:
                objects = ""

            template_an = template_an.replace('_^_OBJECT_MULTIPLE_^_', objects)

            output = open(os.path.join(ibsd.dataset_path, "AnnotationsYAML", _filename), 'w')
            output.write(template_an)
            output.close()

if __name__ == "__main__":

    information = {
        'mine_negatives':   True,
        'mine_max_keep':    1,
        'mine_exclude_categories': ['zebra_grevys', 'zebra_plains'],
    }

    # dataset = IBEIS_Data('test/', **information)

    dataset = IBEIS_Data('/Users/bluemellophone/Desktop/BACKUP/IBEIS2014/', **information)
    print dataset
    print

    # dataset.export_yaml()

    # Access specific information about the dataset
    print "Categories:", dataset.categories
    print "Number of images:", len(dataset)

    print
    dataset.print_distribution()
    print

    # Access specific image from dataset using filename or index
    print dataset['2014_000002']
    print dataset['_000002'] #partial also works (takes first match)
    cont = True
    while cont:
        # Show the detection regions by drawing them on the source image
        print "Enter something to continue, empty to get new image"
        cont = dataset[com.randInt(0, len(dataset) - 1)].show()

    # Get all images using a specific positive set
    (pos, pos_rois), (neg, neg_rois), val, test = dataset.dataset('zebra_grevys')

    # Get a specific number of images (-1 for auto normalize to what the other gives)
    # (pos, pos_rois), (neg, neg_rois), val, test = dataset.dataset('zebra_grevys', max_rois_neg=-1)

    print "\nPositives:"
    for _pos in pos:
        print _pos.image_path()
        print _pos.bounding_boxes(parts=True)

    print "\nNegatives:"
    for _neg in neg:
        print _neg.image_path()
        print _neg.bounding_boxes(parts=True)

    print "\nValidation:"
    for _val in val:
        print _val.image_path()
        print _val.bounding_boxes(parts=True)

    print "\nTest:"
    for _test in test:
        print _test.image_path()
        print _test.bounding_boxes(parts=True)
<commit_msg>Removed local dataset calls<commit_after>#!/usr/bin/env python

from datetime import date
import cv2
import numpy as np
import os
import random
from directory import Directory
import xml.etree.ElementTree as xml

import common as com
from ibeis_image import IBEIS_Image

class IBEIS_Data(object):

    def __init__(ibsd, dataset_path, **kwargs):
        com._kwargs(kwargs, 'object_min_width',    32)
        com._kwargs(kwargs, 'object_min_height',   32)
        com._kwargs(kwargs, 'mine_negatives',      True)
        com._kwargs(kwargs, 'mine_width_min',      50)
        com._kwargs(kwargs, 'mine_width_max',      400)
        com._kwargs(kwargs, 'mine_height_min',     50)
        com._kwargs(kwargs, 'mine_height_max',     400)
        com._kwargs(kwargs, 'mine_max_attempts',   100)
        com._kwargs(kwargs, 'mine_max_keep',       10)
        com._kwargs(kwargs, 'mine_overlap_margin', 0.25)
        com._kwargs(kwargs, 'mine_exclude_categories', [])

        ibsd.dataset_path = dataset_path
        ibsd.absolute_dataset_path = os.path.realpath(dataset_path)

        direct = Directory(os.path.join(ibsd.dataset_path, "Annotations") , include_file_extensions=["xml"])
        ibsd.images = []
        files = direct.files()
        print "Loading Database"
        for i, filename in enumerate(files):
            if len(files) > 10:
                if i % (len(files) / 10) == 0:
                    print "%0.2f" %(float(i) / len(files))
            ibsd.images.append(IBEIS_Image(filename, ibsd.absolute_dataset_path, **kwargs))
        print "    ...Loaded"

        ibsd.categories_images = []
        ibsd.categories_rois = []

        for image in ibsd.images:
            temp = image.categories(unique=False)
            ibsd.categories_rois += temp
            ibsd.categories_images += set(temp)

            if len(image.objects) == 0:
                ibsd.categories_images += ["IGNORED"]

        ibsd.distribution_images = com.histogram(ibsd.categories_images)
        ibsd.distribution_rois = com.histogram(ibsd.categories_rois)
        ibsd.rois = sum(ibsd.distribution_rois.values())

        ibsd.categories = sorted(set(ibsd.categories_images))

    def __str__(ibsd):
        return "<IBEIS Data Object | %s | %d images | %d categories | %d rois>" \
            %(ibsd.absolute_dataset_path, len(ibsd.images), len(ibsd.categories), ibsd.rois)

    def __repr__(ibsd):
        return "<IBEIS Data Object | %s>" % (ibsd.absolute_dataset_path)

    def __len__(ibsd):
        return len(ibsd.images)

    def __getitem__(ibsd, key):
        if isinstance(key, str):
            for image in ibsd.images:
                if key in image.filename:
                    return image
            return None
        else:
            return ibsd.images[key]

    def print_distribution(ibsd):
        def _print_line(category, spacing, images, rois):
            images = str(images)
            rois = str(rois)
            print "%s%s\t%s" %(category + " " * (spacing - len(category)), images, rois)

        _max = max([ len(category) for category in ibsd.distribution_rois.keys() + ['TOTAL', 'CATEGORY'] ]) + 3

        _print_line("CATEGORY", _max, "IMGs", "ROIs")
        if "IGNORED" in ibsd.distribution_images:
            _print_line("IGNORED", _max, ibsd.distribution_images["IGNORED"], "")

        for category in sorted(ibsd.distribution_rois):
            _print_line(category, _max, ibsd.distribution_images[category], ibsd.distribution_rois[category])

        _print_line("TOTAL", _max, len(ibsd.images), ibsd.rois)

    def dataset(ibsd, positive_category, neg_exclude_categories=[], max_rois_pos=None, max_rois_neg=None):
        def _parse_dataset_file(category, _type):
            filepath = os.path.join(ibsd.dataset_path, "ImageSets", "Main", category + "_" + _type + ".txt")
            _dict = {}
            try:
                _file = open(filepath)
                for line in _file:
                    line = line.strip().split(" ")
                    line = [line[0], line[-1]]
                    _dict[line[0]] = True
            except IOError as e:
                print "<", e, ">", filepath

            return _dict

        positives = []
        negatives = []
        validation = []
        test = []

        train_values = _parse_dataset_file(positive_category, "train")
        val_values = _parse_dataset_file(positive_category, "val")
        test_values = _parse_dataset_file(positive_category, "test")

        pos_rois = 0
        neg_rois = 0
        for image in ibsd.images:
            filename = image.filename
            _train = train_values.get(image.filename[:-4], False)
            _val = val_values.get(image.filename[:-4], False)
            _test = test_values.get(image.filename[:-4], False)

            temp = image.categories(unique=False)
            flag = False

            if _train:
                for val in temp:
                    if val == positive_category:
                        flag = True
                        pos_rois += 1
                    elif val not in neg_exclude_categories:
                        neg_rois += 1

                if flag:
                    positives.append(image)
                elif val not in neg_exclude_categories:
                    negatives.append(image)

            if _val:
                validation.append(image)

            if _test:
                test.append(image)

        # Setup auto normalize variables for equal positives and negatives
        if max_rois_pos == 'auto' or max_rois_pos == -1:
            max_rois_pos = neg_rois

        if max_rois_neg == 'auto' or max_rois_neg == -1:
            max_rois_neg = pos_rois

        # Remove positives to target, not gauranteed to give target, but 'close'.
        if max_rois_pos is not None and len(positives) > 0:
            pos_density = float(pos_rois) / len(positives)
            target_num = int(max_rois_pos / pos_density)
            print "Normalizing Positives, Target:", target_num

            # Remove images to match target
            while len(positives) > target_num:
                positives.pop( com.randInt(0, len(positives) - 1) )

            # Recalculate rois left
            pos_rois = 0
            for image in positives:
                temp = image.categories(unique=False)
                for val in temp:
                    if val in positive_category:
                        pos_rois += 1

        # Remove positives to target, not gauranteed to give target, but 'close'.
        if max_rois_neg is not None and len(negatives) > 0:
            neg_density = float(neg_rois) / len(negatives)
            target_num = int(max_rois_neg / neg_density)
            print "Normalizing Negatives, Target:", target_num

            # Remove images to match target
            while len(negatives) > target_num:
                negatives.pop( com.randInt(0, len(negatives) - 1) )

            # Recalculate rois left
            neg_rois = 0
            for image in negatives:
                temp = image.categories(unique=False)
                for val in temp:
                    if val not in positive_category:
                        neg_rois += 1

        print("%s\t%s\t%s\t%s\t%s" %("       ", "Pos", "Neg", "Val", "Test"))
        print("%s\t%s\t%s\t%s\t%s" %("Images:", len(positives), len(negatives), len(validation), len(test)))
        print("%s\t%s\t%s\t%s\t%s" %("ROIs:  ", pos_rois, neg_rois, "", ""))

        return (positives, pos_rois), (negatives, neg_rois), validation, test

    def export_yaml(ibsd):
        os.system("mkdir " + os.path.join(ibsd.dataset_path, "AnnotationsYAML"))

        for image in ibsd.images:
            _filename = image.filename.split(".")[0] + ".yml"

            template_an = open("/Users/bluemellophone/code/python-modules/pypascalxml/template_annotation.yml", 'r')
            template_ob = open("/Users/bluemellophone/code/python-modules/pypascalxml/template_object.yml", 'r')
            template_pt = open("/Users/bluemellophone/code/python-modules/pypascalxml/template_part.yml", 'r')
            template_an = ''.join(template_an.readlines())
            template_ob = ''.join(template_ob.readlines())
            template_pt = ''.join(template_pt.readlines())

            template_an = template_an.replace('_^_FOLDER_^_', image.folder)
            template_an = template_an.replace('_^_FILENAME_^_', image.filename)
            template_an = template_an.replace('_^_DARABASE_NAME_^_', image.source_database)
            template_an = template_an.replace('_^_DATABASE_YEAR_^_', image.source_annotation)
            template_an = template_an.replace('_^_SOURCE_^_', image.source_image)
            template_an = template_an.replace('_^_WIDTH_^_', str(image.width))
            template_an = template_an.replace('_^_HEIGHT_^_', str(image.height))
            template_an = template_an.replace('_^_CHANNELS_^_', str(image.depth))
            template_an = template_an.replace('_^_SEGMENTED_^_', str(int(image.segmented)))

            objects = []
            for _object in image.objects:
                if _object.name == "MINED":
                    continue

                temp_ob = template_ob[:]

                temp_ob = temp_ob.replace('_^_NAME_^_', _object.name)
                temp_ob = temp_ob.replace('_^_POSE_^_', _object.pose)
                temp_ob = temp_ob.replace('_^_TRUNCATED_^_', str(int(_object.truncated)))
                temp_ob = temp_ob.replace('_^_DIFFICULT_^_', str(int(_object.difficult)))
                temp_ob = temp_ob.replace('_^_XMIN_^_', str(_object.xmin))
                temp_ob = temp_ob.replace('_^_YMIN_^_', str(_object.ymin))
                temp_ob = temp_ob.replace('_^_XMAX_^_', str(_object.xmax))
                temp_ob = temp_ob.replace('_^_YMAX_^_', str(_object.ymax))

                parts = []
                for  part in _object.parts:
                    temp_pt = template_pt[:]

                    temp_pt = temp_pt.replace('_^_NAME_^_', part.name)
                    temp_pt = temp_pt.replace('_^_XMIN_^_', str(part.xmin))
                    temp_pt = temp_pt.replace('_^_YMIN_^_', str(part.ymin))
                    temp_pt = temp_pt.replace('_^_XMAX_^_', str(part.xmax))
                    temp_pt = temp_pt.replace('_^_YMAX_^_', str(part.ymax))

                    parts.append(temp_pt)

                if len(parts) > 0:
                    parts =  "\n\t\t\tpart:" + "".join(parts)
                else:
                    parts = ""

                temp_ob = temp_ob.replace('_^_PART_MULTIPLE_OPTIONAL_^_', parts)

                objects.append(temp_ob)

            if len(objects) > 0:
                objects =  "\n\tobject:" + "".join(objects)
            else:
                objects = ""

            template_an = template_an.replace('_^_OBJECT_MULTIPLE_^_', objects)

            output = open(os.path.join(ibsd.dataset_path, "AnnotationsYAML", _filename), 'w')
            output.write(template_an)
            output.close()

if __name__ == "__main__":

    information = {
        'mine_negatives':   True,
        'mine_max_keep':    1,
        'mine_exclude_categories': ['zebra_grevys', 'zebra_plains'],
    }

    dataset = IBEIS_Data('test/', **information)
    print dataset
    print

    dataset.export_yaml()

    # Access specific information about the dataset
    print "Categories:", dataset.categories
    print "Number of images:", len(dataset)

    print
    dataset.print_distribution()
    print

    # Access specific image from dataset using filename or index
    print dataset['2014_000002']
    print dataset['_000002'] #partial also works (takes first match)
    cont = True
    while cont:
        # Show the detection regions by drawing them on the source image
        print "Enter something to continue, empty to get new image"
        cont = dataset[com.randInt(0, len(dataset) - 1)].show()

    # Get all images using a specific positive set
    (pos, pos_rois), (neg, neg_rois), val, test = dataset.dataset('zebra_grevys')

    # Get a specific number of images (-1 for auto normalize to what the other gives)
    # (pos, pos_rois), (neg, neg_rois), val, test = dataset.dataset('zebra_grevys', max_rois_neg=-1)

    print "\nPositives:"
    for _pos in pos:
        print _pos.image_path()
        print _pos.bounding_boxes(parts=True)

    print "\nNegatives:"
    for _neg in neg:
        print _neg.image_path()
        print _neg.bounding_boxes(parts=True)

    print "\nValidation:"
    for _val in val:
        print _val.image_path()
        print _val.bounding_boxes(parts=True)

    print "\nTest:"
    for _test in test:
        print _test.image_path()
        print _test.bounding_boxes(parts=True)
<|endoftext|>


<commit_before>"""Submit and register one or more test jobs."""

import sys
import os
import argparse

from autocms.core import load_configuration
from autocms.submit import perform_test_submission

def main():
    """Call run_submission with arguments from the command line."""
    parser = argparse.ArgumentParser(description='Submit one or more jobs.')
    parser.add_argument('testname', help='test directory')
    parser.add_argument('-n', '--num_jobs', type=int, default=1,
                        help='number of jobs to submit (max)')
    parser.add_argument('-c', '--configfile', type=str,
                        default='autocms.cfg',
                        help='AutoCMS configuration file name')
    args = parser.parse_args()
    config = load_configuration(args.configfile)
    perform_test_submission(args.num_jobs, args.testname, config)
    return 0

if __name__ == '__main__':
    exit_status = main()
    sys.exit(exit_status)
<commit_msg>remove unneeded import<commit_after>"""Submit and register one or more test jobs."""

import sys
import argparse

from autocms.core import load_configuration
from autocms.submit import perform_test_submission

def main():
    """Call run_submission with arguments from the command line."""
    parser = argparse.ArgumentParser(description='Submit one or more jobs.')
    parser.add_argument('testname', help='test directory')
    parser.add_argument('-n', '--num_jobs', type=int, default=1,
                        help='number of jobs to submit (max)')
    parser.add_argument('-c', '--configfile', type=str,
                        default='autocms.cfg',
                        help='AutoCMS configuration file name')
    args = parser.parse_args()
    config = load_configuration(args.configfile)
    perform_test_submission(args.num_jobs, args.testname, config)
    return 0

if __name__ == '__main__':
    status = main()
    sys.exit(status)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import division, print_function

from functools import partial
import os
import sys

import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin

from .features import Features, DEFAULT_VARFRAC
from .utils import (izip, positive_int, portion, nonnegative_float,
                    confirm_outfile)

_do_nothing = lambda *a, **k: None

################################################################################
### Handling blank descriptors

DEFAULT_BLANK_HANDLER = 'fill'
DEFAULT_BLANK_THRESH = 1000

class FillBlanks(BaseEstimator, TransformerMixin):
    '''
    Fills in any almost-blank SIFT descriptors with a random one.

    copy: if False, change the input in-place. By default, operates on a copy.

    blank_thresh: the threshold for considering a descriptor to be blank.
        Any descriptors whose sum is less than this threshold are replaced.
    '''

    def __init__(self, copy=True, blank_thresh=DEFAULT_BLANK_THRESH):
        self.copy = copy
        self.blank_thresh = blank_thresh

    def fit(self, X, y=None):
        "Does nothing, since this transform doesn't require fitting."
        pass

    def transform(self, X, y=None, copy=None):
        copy = copy if copy is not None else self.copy
        n_f, dim = X.shape
        if copy:
            X = X.copy()

        mag = 3 * (2 * self.blank_thresh / dim)

        blank_idx = np.sum(X, axis=1) < self.blank_thresh
        X[blank_idx, :] = mag * np.random.rand(blank_idx.sum(), dim)
        return X

class ZeroBlanks(BaseEstimator, TransformerMixin):
    '''
    Zeroes out any almost-blank SIFT descriptors.

    copy: if False, change the input in-place. By default, operates on a copy.

    blank_thresh: the threshold for considering a descriptor to be blank.
        Any descriptors whose sum is less than this threshold are replaced.
    '''

    def __init__(self, copy=True, blank_thresh=DEFAULT_BLANK_THRESH):
        self.copy = copy
        self.blank_thresh = blank_thresh

    def fit(self, X, y=None):
        "Does nothing, since this transform doesn't require fitting."
        pass

    def transform(self, X, y=None, copy=None):
        copy = copy if copy is not None else self.copy
        if copy:
            X = X.copy()
        X[np.sum(X, axis=1) < self.blank_thresh, :] = 0
        return X

BLANK_HANDLERS = {
    'fill': FillBlanks,
    'zero': ZeroBlanks,
    'drop': None,
}

def handle_blanks(features, blank_thresh=DEFAULT_BLANK_THRESH,
                            blank_handler=DEFAULT_BLANK_HANDLER,
                            inplace=False):
    '''Handles any SIFT descriptors that are blank, or nearly blank.'''

    if blank_handler not in BLANK_HANDLERS:
        msg = "unknown blank handler {!r}, expected one of {}".format(
            blank_handler, ", ".join(map(repr, BLANK_HANDLERS)))
        raise ValueError(msg)

    if blank_handler == 'drop':
        # TODO handle this more efficiently
        feats = [
            f[np.sum(f, axis=1) >= blank_thresh, :] for f in features.features
        ]

        args = dict((k, features.data[k]) for k in f._extra_names)
        args['categories'] = features.categories
        args['names'] = features.names

        if inplace:
            features.__init__(feats, **args)
            return
        else:
            return Features(feats, **args)

    handler = BLANK_HANDLERS[blank_handler](blank_thresh=blank_thresh)
    r = features._apply_transform(handler, fit_first=True, inplace=inplace)

    if not inplace:
        return r

################################################################################
### Add spatial information

def add_spatial_info(features, add_x=True, add_y=True, inplace=False):
    if not add_x and not add_y:
        return features

    indices = []
    if add_x:
        indices.append(0)
    if add_y:
        indices.append(1)

    all_spatial = []
    for feat, frame in izip(features, frames):
        spatial = frame[:, indices].astype(feat.dtype)
        spatial /= spatial.max(axis=0)
        all_spatial.append(spatial)

    new_feats = np.hstack((features._features, np.vstack(all_spatial)))
    if inplace:
        features._features = new_feats
        features._refresh_features()
    else:
        return Features(
            new_feats, n_pts=features._n_pts,
            categories=features.categories, names=features.names,
            **dict((k, features.data[k]) for k in features._extra_names))

################################################################################
### Wrapper for general processing

def process_image_features(features, verbose=False, inplace=False,
        blank_thresh=DEFAULT_BLANK_THRESH, blank_handler=DEFAULT_BLANK_HANDLER,
        do_pca=True, pca=None,
            pca_k=None, pca_varfrac=DEFAULT_VARFRAC, pca_random=False,
            pca_whiten=False,
        add_x=True, add_y=True,
        normalize_feats=True, scaler=None,
        ret_pca=False, ret_scaler=False):
    # TODO: use sklearn.Pipeline instead?
    pr = partial(print, file=sys.stderr) if verbose else _do_nothing

    if blank_handler not in (None, "none"):
        pr("Handling blanks...")
        ret = handle_blanks(features, blank_thresh=blank_thresh,
                            blank_handler=blank_handler, inplace=inplace)
        if not inplace:
            features = ret

    if do_pca:
        pr("Running PCA...")
        old_dim = features.dim

        ret = features.pca(
            pca=pca, ret_pca=True, k=pca_k, varfrac=pca_varfrac,
            randomize=pca_random, whiten=pca_whiten, inplace=inplace)
        if inplace:
            pca = ret
        else:
            features, pca = ret

        new_dim = features.dim
        pr("Reduced dimensionality from {} to {}.".format(old_dim, new_dim))

    if add_x or add_y:
        pr("Adding spatial info...")
        ret = add_spatial_info(features, add_x, add_y, inplace=inplace)
        if not inplace:
            features = ret

    if normalize_feats:
        pr("Normalizing features...")
        ret = features.standardize(scaler=scaler, ret_scaler=True,
                                   inplace=inplace)
        if inplace:
            scaler = ret
        else:
            features, scaler = ret

    if not ret_pca and not ret_scaler:
        return features
    ret = [features]
    if ret_pca:
        ret.append(pca)
    if ret_scaler:
        ret.append(scaler)
    return ret

def parse_args(args=None):
    import argparse

    # helper for boolean flags
    # based on http://stackoverflow.com/a/9236426/344821
    class ActionNoYes(argparse.Action):
        def __init__(self, opt_name, off_name=None, dest=None,
                     default=True, required=False, help=None):

            if off_name is None:
                off_name = 'no-' + opt_name
            self.off_name = '--' + off_name

            if dest is None:
                dest = opt_name.replace('-', '_')

            super(ActionNoYes, self).__init__(
                    ['--' + opt_name, '--' + off_name],
                    dest, nargs=0, const=None,
                    default=default, required=required, help=help)

        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, self.dest, option_string != self.off_name)

    _def = "; default %(default)s."
    parser = argparse.ArgumentParser(
        description="Processes raw PHOW features before running the SDM.")

    blanks = parser.add_argument_group("Blank handling")
    blanks.add_argument('--blank-threshold', dest='blank_thresh',
        default=DEFAULT_BLANK_THRESH, type=nonnegative_float,
        help="Consider descriptors with sum < BLANK_THRESH to be blank" + _def)
    blanks.add_argument('--blank-handler',
        choices=list(BLANK_HANDLERS) + ['none'], default=DEFAULT_BLANK_HANDLER,
        help="What to do with blanks" + _def)

    pca = parser.add_argument_group("PCA options")
    pca._add_action(ActionNoYes('do-pca', 'no-pca', default=True,
        help="Whether to run PCA; does by default."))
    dim = pca.add_mutually_exclusive_group()
    dim.add_argument('--pca-k', type=positive_int, default=None,
        help="An explicit dimensionality to reduce the features to.")
    dim.add_argument('--pca-varfrac', default=DEFAULT_VARFRAC,
        type=portion, metavar='FRAC',
        help="The fraction of variance to maintain in the PCA" + _def)
    pca._add_action(ActionNoYes('pca-random', default=False,
        help="Whether to use a randomized PCA implementation; default don't."))
    pca._add_action(ActionNoYes('pca-whiten', default=False,
        help="Whether to do whitening in the PCA, removing linear correlations "
             "between dimensions; default don't."))

    spa = parser.add_argument_group('Spatial information')
    spa._add_action(ActionNoYes('add-x', default=True,
        help="Append normalized x coord of patches; does by default."))
    spa._add_action(ActionNoYes('add-y', default=True,
        help="Append normalized y coord of patches; does by default."))

    std = parser.add_argument_group('Standardization')
    std._add_action(ActionNoYes('normalize-feats', default=True,
        help="Normalize features to mean 0, variance 1 at the end (default)."))

    parser._add_action(ActionNoYes('verbose', 'quiet', default=True,
        help="Print out info along the way (the default)."))

    parser.add_argument('load_file',
        help="Load features from this file (output of extract_features).")
    parser.add_argument('save_file', help="Save into this file.")

    args = parser.parse_args(args)
    load = args.load_file
    save = args.save_file
    del args.load_file, args.save_file
    return args, load, save

def main():
    args, load_file, save_file = parse_args()
    confirm_outfile(save_file)

    pr = partial(print, file=sys.stderr) if args.verbose else _do_nothing

    pr("Loading features from '{}'...".format(load_file))
    kwargs = {'load_attrs': True, 'features_dtype': np.float32}
    if os.path.isdir(load_file):
        orig, attrs = Features.load_from_perbag(load_file, **kwargs)
    else:
        orig, attrs = Features.load_from_hdf5(load_file, **kwargs)

    new, pca, scaler = process_image_features(
        orig, ret_pca=True, ret_scaler=True, **vars(args))

    if pca is not None:
        attrs['pca_mean'] = pca.mean_
        attrs['pca_components'] = pca.components_
    if scaler is not None:
        attrs['scaler_mean'] = scaler.mean_
        attrs['scaler_std'] = scaler.std_

    pr("Saving features to '{}'...".format(save_file))
    new.save_as_hdf5(save_file, process_args=repr(vars(args)), **attrs)

if __name__ == '__main__':
    main()
<commit_msg>fix add_spatial_info<commit_after>#!/usr/bin/env python
from __future__ import division, print_function

from functools import partial
import os
import sys

import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin

from .features import Features, DEFAULT_VARFRAC
from .utils import positive_int, portion, nonnegative_float, confirm_outfile

_do_nothing = lambda *a, **k: None

################################################################################
### Handling blank descriptors

DEFAULT_BLANK_HANDLER = 'fill'
DEFAULT_BLANK_THRESH = 1000

class FillBlanks(BaseEstimator, TransformerMixin):
    '''
    Fills in any almost-blank SIFT descriptors with a random one.

    copy: if False, change the input in-place. By default, operates on a copy.

    blank_thresh: the threshold for considering a descriptor to be blank.
        Any descriptors whose sum is less than this threshold are replaced.
    '''

    def __init__(self, copy=True, blank_thresh=DEFAULT_BLANK_THRESH):
        self.copy = copy
        self.blank_thresh = blank_thresh

    def fit(self, X, y=None):
        "Does nothing, since this transform doesn't require fitting."
        pass

    def transform(self, X, y=None, copy=None):
        copy = copy if copy is not None else self.copy
        n_f, dim = X.shape
        if copy:
            X = X.copy()

        mag = 3 * (2 * self.blank_thresh / dim)

        blank_idx = np.sum(X, axis=1) < self.blank_thresh
        X[blank_idx, :] = mag * np.random.rand(blank_idx.sum(), dim)
        return X

class ZeroBlanks(BaseEstimator, TransformerMixin):
    '''
    Zeroes out any almost-blank SIFT descriptors.

    copy: if False, change the input in-place. By default, operates on a copy.

    blank_thresh: the threshold for considering a descriptor to be blank.
        Any descriptors whose sum is less than this threshold are replaced.
    '''

    def __init__(self, copy=True, blank_thresh=DEFAULT_BLANK_THRESH):
        self.copy = copy
        self.blank_thresh = blank_thresh

    def fit(self, X, y=None):
        "Does nothing, since this transform doesn't require fitting."
        pass

    def transform(self, X, y=None, copy=None):
        copy = copy if copy is not None else self.copy
        if copy:
            X = X.copy()
        X[np.sum(X, axis=1) < self.blank_thresh, :] = 0
        return X

BLANK_HANDLERS = {
    'fill': FillBlanks,
    'zero': ZeroBlanks,
    'drop': None,
}

def handle_blanks(features, blank_thresh=DEFAULT_BLANK_THRESH,
                            blank_handler=DEFAULT_BLANK_HANDLER,
                            inplace=False):
    '''Handles any SIFT descriptors that are blank, or nearly blank.'''

    if blank_handler not in BLANK_HANDLERS:
        msg = "unknown blank handler {!r}, expected one of {}".format(
            blank_handler, ", ".join(map(repr, BLANK_HANDLERS)))
        raise ValueError(msg)

    if blank_handler == 'drop':
        # TODO handle this more efficiently
        feats = [
            f[np.sum(f, axis=1) >= blank_thresh, :] for f in features.features
        ]

        args = dict((k, features.data[k]) for k in f._extra_names)
        args['categories'] = features.categories
        args['names'] = features.names

        if inplace:
            features.__init__(feats, **args)
            return
        else:
            return Features(feats, **args)

    handler = BLANK_HANDLERS[blank_handler](blank_thresh=blank_thresh)
    r = features._apply_transform(handler, fit_first=True, inplace=inplace)

    if not inplace:
        return r

################################################################################
### Add spatial information

def add_spatial_info(features, add_x=True, add_y=True, inplace=False,
                     dtype=None):
    if not add_x and not add_y:
        return features

    indices = []
    if add_x:
        indices.append(0)
    if add_y:
        indices.append(1)

    if dtype is None:
        dtype = features.dtype
        if dtype.kind != 'f':
            dtype = np.float32

    spatial = np.asarray(np.vstack(features.frames)[:, indices], dtype=dtype)
    spatial /= spatial.max(axis=0)

    new_feats = np.hstack((features._features, spatial))
    if inplace:
        features._features = new_feats
        features._refresh_features()
    else:
        return Features(
            new_feats, n_pts=features._n_pts,
            categories=features.categories, names=features.names,
            **dict((k, features.data[k]) for k in features._extra_names))

################################################################################
### Wrapper for general processing

def process_image_features(features, verbose=False, inplace=False,
        blank_thresh=DEFAULT_BLANK_THRESH, blank_handler=DEFAULT_BLANK_HANDLER,
        do_pca=True, pca=None,
            pca_k=None, pca_varfrac=DEFAULT_VARFRAC, pca_random=False,
            pca_whiten=False,
        add_x=True, add_y=True,
        normalize_feats=True, scaler=None,
        ret_pca=False, ret_scaler=False):
    # TODO: use sklearn.Pipeline instead?
    pr = partial(print, file=sys.stderr) if verbose else _do_nothing

    if blank_handler not in (None, "none"):
        pr("Handling blanks...")
        ret = handle_blanks(features, blank_thresh=blank_thresh,
                            blank_handler=blank_handler, inplace=inplace)
        if not inplace:
            features = ret

    if do_pca:
        pr("Running PCA...")
        old_dim = features.dim

        ret = features.pca(
            pca=pca, ret_pca=True, k=pca_k, varfrac=pca_varfrac,
            randomize=pca_random, whiten=pca_whiten, inplace=inplace)
        if inplace:
            pca = ret
        else:
            features, pca = ret

        new_dim = features.dim
        pr("Reduced dimensionality from {} to {}.".format(old_dim, new_dim))

    if add_x or add_y:
        pr("Adding spatial info...")
        ret = add_spatial_info(features, add_x, add_y, inplace=inplace)
        if not inplace:
            features = ret

    if normalize_feats:
        pr("Normalizing features...")
        ret = features.standardize(scaler=scaler, ret_scaler=True,
                                   inplace=inplace)
        if inplace:
            scaler = ret
        else:
            features, scaler = ret

    if not ret_pca and not ret_scaler:
        return features
    ret = [features]
    if ret_pca:
        ret.append(pca)
    if ret_scaler:
        ret.append(scaler)
    return ret

def parse_args(args=None):
    import argparse

    # helper for boolean flags
    # based on http://stackoverflow.com/a/9236426/344821
    class ActionNoYes(argparse.Action):
        def __init__(self, opt_name, off_name=None, dest=None,
                     default=True, required=False, help=None):

            if off_name is None:
                off_name = 'no-' + opt_name
            self.off_name = '--' + off_name

            if dest is None:
                dest = opt_name.replace('-', '_')

            super(ActionNoYes, self).__init__(
                    ['--' + opt_name, '--' + off_name],
                    dest, nargs=0, const=None,
                    default=default, required=required, help=help)

        def __call__(self, parser, namespace, values, option_string=None):
            setattr(namespace, self.dest, option_string != self.off_name)

    _def = "; default %(default)s."
    parser = argparse.ArgumentParser(
        description="Processes raw PHOW features before running the SDM.")

    blanks = parser.add_argument_group("Blank handling")
    blanks.add_argument('--blank-threshold', dest='blank_thresh',
        default=DEFAULT_BLANK_THRESH, type=nonnegative_float,
        help="Consider descriptors with sum < BLANK_THRESH to be blank" + _def)
    blanks.add_argument('--blank-handler',
        choices=list(BLANK_HANDLERS) + ['none'], default=DEFAULT_BLANK_HANDLER,
        help="What to do with blanks" + _def)

    pca = parser.add_argument_group("PCA options")
    pca._add_action(ActionNoYes('do-pca', 'no-pca', default=True,
        help="Whether to run PCA; does by default."))
    dim = pca.add_mutually_exclusive_group()
    dim.add_argument('--pca-k', type=positive_int, default=None,
        help="An explicit dimensionality to reduce the features to.")
    dim.add_argument('--pca-varfrac', default=DEFAULT_VARFRAC,
        type=portion, metavar='FRAC',
        help="The fraction of variance to maintain in the PCA" + _def)
    pca._add_action(ActionNoYes('pca-random', default=False,
        help="Whether to use a randomized PCA implementation; default don't."))
    pca._add_action(ActionNoYes('pca-whiten', default=False,
        help="Whether to do whitening in the PCA, removing linear correlations "
             "between dimensions; default don't."))

    spa = parser.add_argument_group('Spatial information')
    spa._add_action(ActionNoYes('add-x', default=True,
        help="Append normalized x coord of patches; does by default."))
    spa._add_action(ActionNoYes('add-y', default=True,
        help="Append normalized y coord of patches; does by default."))

    std = parser.add_argument_group('Standardization')
    std._add_action(ActionNoYes('normalize-feats', default=True,
        help="Normalize features to mean 0, variance 1 at the end (default)."))

    parser._add_action(ActionNoYes('verbose', 'quiet', default=True,
        help="Print out info along the way (the default)."))

    parser.add_argument('load_file',
        help="Load features from this file (output of extract_features).")
    parser.add_argument('save_file', help="Save into this file.")

    args = parser.parse_args(args)
    load = args.load_file
    save = args.save_file
    del args.load_file, args.save_file
    return args, load, save

def main():
    args, load_file, save_file = parse_args()
    confirm_outfile(save_file)

    pr = partial(print, file=sys.stderr) if args.verbose else _do_nothing

    pr("Loading features from '{}'...".format(load_file))
    kwargs = {'load_attrs': True, 'features_dtype': np.float32}
    if os.path.isdir(load_file):
        orig, attrs = Features.load_from_perbag(load_file, **kwargs)
    else:
        orig, attrs = Features.load_from_hdf5(load_file, **kwargs)

    new, pca, scaler = process_image_features(
        orig, ret_pca=True, ret_scaler=True, **vars(args))

    if pca is not None:
        attrs['pca_mean'] = pca.mean_
        attrs['pca_components'] = pca.components_
    if scaler is not None:
        attrs['scaler_mean'] = scaler.mean_
        attrs['scaler_std'] = scaler.std_

    pr("Saving features to '{}'...".format(save_file))
    new.save_as_hdf5(save_file, process_args=repr(vars(args)), **attrs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
    スクリプト概要
    ~~~~~~~~~~

    systemdでtimerを作成し、定期的に実行するPython
    1,Redmineから期限切れ、期限切れそうなチケットを取得する
    2,チケットの期限によって分類
    3,各チャンネルに期限切れ、期限が切れそうなチケットの情報を通知する。

"""

from redminelib import Redmine
from datetime import timedelta, datetime
from slackclient import SlackClient

from haro.plugins.redmine_models import ProjectChannel

from slackbot_settings import REDMINE_URL, API_KEY
from slackbot.bot import listen_to, respond_to

from db import Session

@respond_to('^redmine\s+test_daik$')
def get_ticket_information():
    """Redmineのチケット情報とチケットと結びついているSlackチャンネルを取得

    """
    redmine = Redmine(REDMINE_URL, key=API_KEY)
    # すべてのチケットを取得
    issues = redmine.issue.all(sort='subject:desc')

    projects_past_due_date = {}
    projects_close_to_due_date = {}
    for issue in issues:
        # due_date属性とdue_dateがnoneの場合は除外
        if not getattr(issue, 'due_date', None):
            continue
        proj_id = issue.project.id
        # issueのデータをSlack通知用にformatする。
        issue_display = display_issue(issue)
        # issueの期限が過ぎていた場合
       if check_past_due_date(issue.due_date):
           # proj_idをkeyにして値にformatしたIssue情報の値を入れる。
           # 辞書のkeyと値の例:{proj_id: ['- 2017-03-31 23872: サーバーセキュリティーの基準を作ろう(@takanory)'], xxx:['- xxxxxxx']}
            if proj_id not in projects_past_due_date.keys():
                projects_past_due_date[proj_id] = [issue_display]
            else:
                projects_past_due_date[proj_id].append(issue_display)
        # issueの期限が1週間以内の場合
        elif check_close_to_due_date(issue.due_date):
            if proj_id not in projects_close_to_due_date.keys():
                 projects_close_to_due_date[proj_id] = [issue_display]
            else:
                projects_close_to_due_date[proj_id].append(issue_display)
        else:
            continue

    # 各プロジェクトのチケット通知をSlackチャンネルに送る。
    send_ticket_info_to_channels(projects_past_due_date, 0)
    send_ticket_info_to_channels(projects_close_to_due_date, 1)

def display_issue(issue):
    """issueの詳細をSlackに表示用にフォーマットする。

    　 :param issue: redmineのissue
    """

    return '- ' + str(issue.due_date) + ' ' + str(issue.id) + ': ' + str(issue.subject) + '(@' + str(issue.author) + ')'

def check_past_due_date(due_date):
    """期限切れたチケットをチェックするFunction
    期限が切れていたらTrueを返す。それ以外はFalse

    :param due_date: 各チケットのdue_date
    """
    today = datetime.now()
    if datetime.combine(due_date, datetime.min.time()) < today:
        return True
    else:
        return False

def check_close_to_due_date(due_date):
    """期限が1週間以内に切れそうなチケットをチェックするFunction
    期限が切れていたらTrueを返す。それ以外はFalse

    :param due_date: 各チケットのdue_date
    """
    today = datetime.now()
    if datetime.combine(due_date, datetime.min.time()) < today - timedelta(7):
        return True
    else:
        return False

def send_ticket_info_to_channels(projects, type):
    """チケット情報を各Slackチャンネルごとに通知する。

        :param projects: 期限が切れたプロジェクト、期限が切れそうなプロジェクトのdict
        :param type: int type=0 -> 期限切れ type=1 ->期限切れそうなチケット
    """
    s = Session()
    sc = SlackClient(SLACK_API_TOKEN)
    for project in projects.keys():
        # 各プロジェクトのproj_roomを獲得する。
        proj_room = s.query(ProjectChannel).filter(ProjectChannel.project_id == project).one_or_none()

        # api_call()を使用し、すべてのSlackチャンネルに期限が切れたチケット、期限が切れそうな通知をチケットまとめて送る
        if proj_room:
            # 1つのredmineプロジェクトが複数のslackチャンネルに関連付けられているケースに対応
            channels = proj_room.channels
            # プロジェクトごとのチケット数カウントを取得
            issue_count = len(projects[project])
            if not type: # 期限切れチケット
                message = '期限が切れたチケットは' + str(issue_count) + ' 件です\n'
            else: # 期限切れそうなチケット
                message = 'もうすぐ期限が切れそうなチケットは' + str(issue_count) + ' 件です\n'
                message += '\n'.join(projects[project])
            for channel in channels:
                sc.api_call(
                    "chat.postMessage",
                    channel=channel,
                    text= message
                )
<commit_msg>レビューを途中まで修正、単独のスクリプトとして動くように変更<commit_after>"""
スクリプト概要
~~~~~~~~~~
systemdでtimerを作成し、定期的に実行するPython
1,Redmineから期限切れ、期限切れそうなチケットを取得する
2,チケットの期限によって分類
3,各チャンネルに期限切れ、期限が切れそうなチケットの情報を通知する。
"""

from redminelib import Redmine
from datetime import timedelta, date
from slackclient import SlackClient
import argparse
from textwrap import dedent
from configparser import ConfigParser, NoSectionError

from haro.plugins.redmine_models import ProjectChannel

from slackbot_settings import REDMINE_URL, API_KEY, SLACK_API_TOKEN
from configparser import ConfigParser, NoSectionError

from db import init_dbsession
from slackbot_settings import (
    SQLALCHEMY_URL,
    SQLALCHEMY_ECHO,
    SQLALCHEMY_POOL_SIZE,
)

from db import Session

# @respond_to('^redmine\s+test_daik$')
def get_ticket_information():
    """Redmineのチケット情報とチケットと結びついているSlackチャンネルを取得
    """
    redmine = Redmine(REDMINE_URL, key=API_KEY)
    # すべてのチケットを取得
    issues = redmine.issue.all(sort='subject:desc')

    projects_past_due_date = {}
    projects_close_to_due_date = {}
    for issue in issues:
        # due_date属性とdue_dateがnoneの場合は除外
        if not getattr(issue, 'due_date', None):
            continue
        proj_id = issue.project.id
        # issueのデータをSlack通知用にformatする。
        issue_display = display_issue(issue)
        # issueの期限が過ぎていた場合
        if is_past_due_date(issue.due_date):
           # proj_idをkeyにして値にformatしたIssue情報の値を入れる。
           # 辞書のkeyと値の例:{proj_id: ['- 2017-03-31 23872: サーバーセキュリティーの基準を作ろう(@takanory)'], xxx:['- xxxxxxx']}
            if proj_id not in projects_past_due_date.keys():
                projects_past_due_date[proj_id] = [issue_display]
            else:
                projects_past_due_date[proj_id].append(issue_display)
        # issueの期限が1週間以内の場合
        elif is_close_to_due_date(issue.due_date):
            if proj_id not in projects_close_to_due_date.keys():
                 projects_close_to_due_date[proj_id] = [issue_display]
            else:
                 projects_close_to_due_date[proj_id].append(issue_display)
        else:
            continue

    # 各プロジェクトのチケット通知をSlackチャンネルに送る。
    send_ticket_info_to_channels(projects_past_due_date, 0)
    print(projects_past_due_date)
    send_ticket_info_to_channels(projects_close_to_due_date, 1)

def display_issue(issue):
    """issueの詳細をSlackに表示用にフォーマットする。

    フォーマット例:
    - 2017-03-31 23872: サーバーセキュリティーの基準を作ろう(@takanory)
    :param issue: redmineのissue
    """

    return '{} {} {}: {} (@{})'.format('-', issue.due_date, issue.id, issue.subject, issue.author)

def is_past_due_date(due_date):
    """期限切れたチケットをチェックするFunction
    期限が切れていたらTrueを返す。それ以外はFalse

    :param due_date: 各チケットのdue_date
    """
    today = date.today()

    return due_date < today

def is_close_to_due_date(due_date):
    """期限が1週間以内に切れそうなチケットをチェックするFunction
    期限が切れていたらTrueを返す。それ以外はFalse

    :param due_date: 各チケットのdue_date
    """
    today = date.today()

    return due_date < today - timedelta(7)

def send_ticket_info_to_channels(projects, type):
    """チケット情報を各Slackチャンネルごとに通知する。

        :param projects: 期限が切れたプロジェクト、期限が切れそうなプロジェクトのdict
        :param type: int type=0 -> 期限切れ type=1 ->期限切れそうなチケット
    """
    s = Session()
    sc = SlackClient(SLACK_API_TOKEN)
    for project in projects.keys():
        # 各プロジェクトのproj_roomを獲得する。
        proj_room = s.query(ProjectChannel).filter(ProjectChannel.project_id == project).one_or_none()
        # for testing only
        issue_count = len(projects[project])
        if not type:  # 期限切れチケット
            message = '期限が切れたチケットは' + str(issue_count) + ' 件です\n'
        else:  # 期限切れそうなチケット
            message = 'もうすぐ期限が切れそうなチケットは' + str(issue_count) + ' 件です\n'
        message += '\n'.join(projects[project])
        sc.api_call(
            "chat.postMessage",
            channel="CBEL53NPM",
            text=message
        )
        # api_call()を使用し、すべてのSlackチャンネルに期限が切れたチケット、期限が切れそうな通知をチケットまとめて送る
        if proj_room:
            # 1つのredmineプロジェクトが複数のslackチャンネルに関連付けられているケースに対応
            channels = proj_room.channels
            # プロジェクトごとのチケット数カウントを取得
            issue_count = len(projects[project])
            if not type: # 期限切れチケット
                message = '期限が切れたチケットは' + str(issue_count) + ' 件です\n'
            else: # 期限切れそうなチケット
                message = 'もうすぐ期限が切れそうなチケットは' + str(issue_count) + ' 件です\n'
                message += '\n'.join(projects[project])
            # for channel in channels:
            #     sc.api_call(
            #         "chat.postMessage",
            #         channel=channel,
            #         text= message
            #     )

def get_argparser():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=dedent('''\
            説明:
            haroの設定ファイルを読み込んだ後にslackbotを起動します'''))

    parser.add_argument('-c', '--config',
                        type=argparse.FileType('r'),
                        default='alembic/conf.ini',
                        help='ini形式のファイルをファイルパスで指定します')

    return parser

def main():
    """設定ファイルをparseして、slackbotを起動します

    1. configparserで設定ファイルを読み込む
    2. 設定ファイルに `alembic` セクションが設定されているかチェック
    3. 設定ファイルの情報でDB周りの設定を初期化
    4. slackbotの処理を開始
    """
    parser = get_argparser()
    args = parser.parse_args()
    conf = ConfigParser()
    conf.read_file(args.config)
    # 環境変数で指定したいため ini ファイルでなくここで追記
    conf["alembic"]['sqlalchemy.url'] = SQLALCHEMY_URL
    conf["alembic"]['sqlalchemy.echo'] = SQLALCHEMY_ECHO
    # if SQLALCHEMY_POOL_SIZE:
    #     conf["alembic"]['sqlalchemy.pool_size'] = '20' #SQLALCHEMY_POOL_SIZE
    if not conf.has_section('alembic'):
        raise NoSectionError('alembic')

    init_dbsession(conf['alembic'])
    get_ticket_information()

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>from datetime import datetime
from django.core.management.base import LabelCommand, CommandError
from corehq.form_processor.utils import should_use_sql_backend
from corehq.util.dates import iso_string_to_datetime
from couchforms.models import XFormInstance
from fluff.management.commands.ptop_reindexer_fluff import ReindexEventHandler
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider

class Command(LabelCommand):

    def handle_label(self, domain, **options):
        if should_use_sql_backend(domain):
            raise CommandError(u'It looks like {} has already been migrated.'.format(domain))

        _do_couch_to_sql_migration(domain)

def _do_couch_to_sql_migration(domain):
    # (optional) collect some information about the domain's cases and forms for cross-checking
    _process_main_forms(domain)
    _copy_unprocessed_forms(domain)
    # (optional) compare the information collected to the information at the beginning

def _process_main_forms(domain):
    last_received_on = datetime.min
    # process main forms (including cases and ledgers)
    for change in _get_main_form_iterator(domain).iter_all_changes():
        form = change.get_document()
        form_received = iso_string_to_datetime(form['received_on'])
        assert last_received_on <= form_received
        last_received_on = form_received
        print 'processing form {}: {}'.format(form['_id'], form_received)
        _migrate_form_and_associated_models(domain, form)

def _get_main_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=['XFormInstance'],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)),
    )

def _migrate_form_and_associated_models(domain, form):
    sql_form = _migrate_form_and_attachments(domain, form)
    _migrate_submission_properties(domain, form, sql_form)
    case_stock_result = _process_cases_and_ledgers(domain, sql_form)
    _save_migrated_models(sql_form, case_stock_result)

def _migrate_form_and_attachments(domain, form):
    """
    see form_processor.parsers.form._create_new_xform for what this should do
    """
    # convert couch form to SQL form
    # adjust datetimes (if doing tzmigration)
    # migrate all attachments
    pass

def _migrate_submission_properties(domain, couch_form_json, sql_form):
    """
    See SubmissionPost._set_submission_properties for what this should do
    """
    pass

def _process_cases_and_ledgers(domain, sql_form):
    """
    See SubmissionPost.process_xforms_for_cases for what this should do
    """
    pass

def _save_migrated_models(sql_form, case_stock_result):
    """
    See SubmissionPost.save_processed_models for ~what this should do.
    However, note that that function does some things that this one shouldn't,
    e.g. process ownership cleanliness flags.
    """

def _copy_unprocessed_forms(domain):
    # copy unprocessed forms
    for change in _get_unprocessed_form_iterator(domain).iter_all_changes():
        form = change.get_document()
        print 'copying unprocessed {} {}: {}'.format(form['doc_type'], form['_id'], form['received_on'])
        # save updated models

def _get_unprocessed_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=[
            'XFormArchived',
            'XFormError',
            'XFormDeprecated',
            'XFormDuplicate',
            # todo: need to figure out which of these we plan on supporting
            'XFormInstance-Deleted',
            'HQSubmission',
            'SubmissionErrorLog',
        ],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({} unprocessed forms)'.format(domain)),
    )
<commit_msg>start filling in migration functions<commit_after>from datetime import datetime
from django.core.management.base import LabelCommand, CommandError
from casexml.apps.case.xform import get_all_extensions_to_close, CaseProcessingResult
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.models import XFormInstanceSQL
from corehq.form_processor.submission_post import CaseStockProcessingResult
from corehq.form_processor.utils import should_use_sql_backend, adjust_datetimes
from corehq.form_processor.utils.general import set_local_domain_sql_backend_override
from corehq.util.dates import iso_string_to_datetime
from couchforms.models import XFormInstance
from fluff.management.commands.ptop_reindexer_fluff import ReindexEventHandler
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider

class Command(LabelCommand):

    def handle_label(self, domain, **options):
        if should_use_sql_backend(domain):
            raise CommandError(u'It looks like {} has already been migrated.'.format(domain))

        _do_couch_to_sql_migration(domain)

def _do_couch_to_sql_migration(domain):
    # (optional) collect some information about the domain's cases and forms for cross-checking
    set_local_domain_sql_backend_override(domain)
    assert should_use_sql_backend(domain)
    _process_main_forms(domain)
    _copy_unprocessed_forms(domain)
    # (optional) compare the information collected to the information at the beginning

def _process_main_forms(domain):
    last_received_on = datetime.min
    # process main forms (including cases and ledgers)
    for change in _get_main_form_iterator(domain).iter_all_changes():
        form = change.get_document()
        wrapped_form = XFormInstance.wrap(form)
        form_received = iso_string_to_datetime(form['received_on'])
        assert last_received_on <= form_received
        last_received_on = form_received
        print 'processing form {}: {}'.format(form['_id'], form_received)
        _migrate_form_and_associated_models(domain, wrapped_form)

def _get_main_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=['XFormInstance'],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)),
    )

def _migrate_form_and_associated_models(domain, couch_form):
    sql_form = _migrate_form_and_attachments(domain, couch_form)
    case_stock_result = _get_case_and_ledger_updates(domain, sql_form)
    _save_migrated_models(sql_form, case_stock_result)

def _migrate_form_and_attachments(domain, couch_form):
    """
    This copies the couch form into a new sql form but does not save it.

    See form_processor.parsers.form._create_new_xform
    and SubmissionPost._set_submission_properties for what this should do.
    """
    interface = FormProcessorInterface(domain)

    form_data = couch_form.form
    # todo: timezone migration if we want here
    # adjust_datetimes(form_data)
    sql_form = interface.new_xform(form_data)
    assert isinstance(sql_form, XFormInstanceSQL)
    sql_form.domain = domain

    # todo: attachments.
    # note that if these are in the blobdb then we likely don't need to move them,
    # just need to bring the references across
    # interface.store_attachments(xform, attachments)

    # submission properties
    sql_form.auth_context = couch_form.auth_context
    sql_form.submit_ip = couch_form.submit_ip

    # todo: this property appears missing from sql forms - do we need it?
    # sql_form.path = couch_form.path

    sql_form.openrosa_headers = couch_form.openrosa_headers
    sql_form.last_sync_token = couch_form.last_sync_token
    sql_form.received_on = couch_form.received_on
    sql_form.date_header = couch_form.date_header
    sql_form.app_id = couch_form.app_id
    sql_form.build_id = couch_form.build_id
    # export_tag intentionally removed
    # sql_form.export_tag = ["domain", "xmlns"]
    sql_form.partial_submission = couch_form.partial_submission
    return sql_form

def _get_case_and_ledger_updates(domain, sql_form):
    """
    Get a CaseStockProcessingResult with the appropriate cases and ledgers to
    be saved.

    See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent
    section of the form-processing code.
    """
    from casexml.apps.case.xform import get_and_check_xform_domain
    from corehq.apps.commtrack.processing import process_stock

    interface = FormProcessorInterface(domain)

    get_and_check_xform_domain(sql_form)
    xforms = [sql_form]
    # todo: I think this can be changed to lock=False
    with interface.casedb_cache(domain=domain, lock=True, deleted_ok=True, xforms=xforms) as case_db:
        touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms)
        extensions_to_close = get_all_extensions_to_close(domain, touched_cases.values())
        case_result = CaseProcessingResult(
            domain,
            [update.case for update in touched_cases.values()],
            [],  # ignore dirtiness_flags,
            extensions_to_close
        )
        # todo: is this necessary?
        for case in case_result.cases:
            case_db.mark_changed(case)

        stock_result = process_stock(xforms, case_db)
        cases = case_db.get_cases_for_saving(sql_form.received_on)
        stock_result.populate_models()

    return CaseStockProcessingResult(
        case_result=case_result,
        case_models=cases,
        stock_result=stock_result,
    )

def _save_migrated_models(sql_form, case_stock_result):
    """
    See SubmissionPost.save_processed_models for ~what this should do.
    However, note that that function does some things that this one shouldn't,
    e.g. process ownership cleanliness flags.
    """

def _copy_unprocessed_forms(domain):
    # copy unprocessed forms
    for change in _get_unprocessed_form_iterator(domain).iter_all_changes():
        form = change.get_document()
        print 'copying unprocessed {} {}: {}'.format(form['doc_type'], form['_id'], form['received_on'])
        # save updated models

def _get_unprocessed_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=[
            'XFormArchived',
            'XFormError',
            'XFormDeprecated',
            'XFormDuplicate',
            # todo: need to figure out which of these we plan on supporting
            'XFormInstance-Deleted',
            'HQSubmission',
            'SubmissionErrorLog',
        ],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({} unprocessed forms)'.format(domain)),
    )
<|endoftext|>


<commit_before>import datetime
import logging

from corehq.apps.es import CaseSearchES
from django.core.management import BaseCommand
from casexml.apps.case.mock import CaseStructure, CaseFactory
from casexml.apps.case.xform import get_case_updates
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors

logger = logging.getLogger('enikshay_2b_cbnaat_fix')

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'domain',
            help="The domain to migrate."
        )
        parser.add_argument(
            "log_path",
            help="Path to write the log to"
        )
        parser.add_argument(
            '--commit',
            action='store_true',
            help="actually modifies the cases. Without this flag, it's a dry run."
        )

    @staticmethod
    def _get_result_recorded_form(test):
        """get last form that set result_recorded to yes"""
        for action in reversed(test.actions):
            for update in get_case_updates(action.form):
                if update.id == test.case_id:
                    case_properties = update.get_update_action().dynamic_properties
                    if case_properties.get('result_recorded') == 'yes':
                        return action.form.form_data

    @staticmethod
    def _get_path(path, form_data):
        block = form_data
        while path:
            block = block.get(path[0], {})
            path = path[1:]
        return block

    def handle(self, domain, log_path, **options):
        commit = options['commit']
        factory = CaseFactory(domain)

        logger.info("Starting {} migration on {} at {}".format(
            "real" if commit else "fake", domain, datetime.datetime.utcnow()
        ))

        cases = (CaseSearchES()
                 .domain(domain)
                 .case_type("test")
                 .case_property_query("test_type_value", "cbnaat", "must")
                 .values_list('case_id', flat=True))

        with open(log_path, "w") as f:
            for test in CaseAccessors(domain=domain).iter_cases(cases):
                updated_by_migration = test.get_case_property('updated_by_migration')
                if ((updated_by_migration == 'enikshay_2b_case_properties' or
                     updated_by_migration == 'enikshay_2b_reason_for_test_fix')
                        and test.get_case_property('result_recorded') == 'yes'):

                    drug_resistance_list = ''
                    drug_sensitive_list = ''
                    resistance_display = None

                    form_data = self._get_result_recorded_form(test)
                    sample_a = self._get_path(
                        'update_test_result cbnaat ql_sample_a sample_a_rif_resistance_result'.split(),
                        form_data,
                    )
                    sample_b = self._get_path(
                        'update_test_result cbnaat ql_sample_b sample_b_rif_resistance_result'.split(),
                        form_data,
                    )
                    if sample_a == 'detected' or sample_b == 'detected':
                        detected = 'TB Detected'
                        drug_resistance_list = 'r'
                        resistance_display = 'R: Res'
                    elif sample_a == 'not_detected' or sample_b == 'not_detected':
                        detected = 'TB Not Detected'
                        drug_sensitive_list = 'r'
                    else:
                        detected = ''

                    result_summary_display = '\n'.join(filter(None, [
                        detected,
                        resistance_display,
                    ]))

                    case_id = test.case_id
                    f.write(case_id + "\n")
                    logger.info(case_id)

                    case_structure = CaseStructure(
                        case_id=case_id,
                        walk_related=False,
                        attrs={
                            "create": False,
                            "update": {
                                "updated_by_migration": "enikshay_2b_cbnaat_fix",
                                "drug_resistance_list": drug_resistance_list,
                                "drug_sensitive_list": drug_sensitive_list,
                                "result_summary_display": result_summary_display,
                            },
                        },
                    )

                    if commit:
                        factory.create_or_update_case(case_structure)
        logger.info("Migration finished at {}".format(datetime.datetime.utcnow()))
<commit_msg>update get_result_recorded_form<commit_after>import datetime
import logging

from corehq.apps.es import CaseSearchES
from django.core.management import BaseCommand
from casexml.apps.case.mock import CaseStructure, CaseFactory
from casexml.apps.case.xform import get_case_updates
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors

logger = logging.getLogger('enikshay_2b_cbnaat_fix')

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'domain',
            help="The domain to migrate."
        )
        parser.add_argument(
            "log_path",
            help="Path to write the log to"
        )
        parser.add_argument(
            '--commit',
            action='store_true',
            help="actually modifies the cases. Without this flag, it's a dry run."
        )

    @staticmethod
    def _get_result_recorded_form(test):
        """get last form that set result_recorded to yes"""
        for action in reversed(test.actions):
            for update in get_case_updates(action.form):
                if (
                    update.id == test.case_id
                    and update.get_update_action()
                    and update.get_update_action().dynamic_properties.get('result_recorded') == 'yes'
                ):
                    return action.form.form_data

    @staticmethod
    def _get_path(path, form_data):
        block = form_data
        while path:
            block = block.get(path[0], {})
            path = path[1:]
        return block

    def handle(self, domain, log_path, **options):
        commit = options['commit']
        factory = CaseFactory(domain)

        logger.info("Starting {} migration on {} at {}".format(
            "real" if commit else "fake", domain, datetime.datetime.utcnow()
        ))

        cases = (CaseSearchES()
                 .domain(domain)
                 .case_type("test")
                 .case_property_query("test_type_value", "cbnaat", "must")
                 .values_list('case_id', flat=True))

        with open(log_path, "w") as f:
            for test in CaseAccessors(domain=domain).iter_cases(cases):
                updated_by_migration = test.get_case_property('updated_by_migration')
                if ((updated_by_migration == 'enikshay_2b_case_properties' or
                     updated_by_migration == 'enikshay_2b_reason_for_test_fix')
                        and test.get_case_property('result_recorded') == 'yes'):

                    drug_resistance_list = ''
                    drug_sensitive_list = ''
                    resistance_display = None

                    form_data = self._get_result_recorded_form(test)
                    sample_a = self._get_path(
                        'update_test_result cbnaat ql_sample_a sample_a_rif_resistance_result'.split(),
                        form_data,
                    )
                    sample_b = self._get_path(
                        'update_test_result cbnaat ql_sample_b sample_b_rif_resistance_result'.split(),
                        form_data,
                    )
                    if sample_a == 'detected' or sample_b == 'detected':
                        detected = 'TB Detected'
                        drug_resistance_list = 'r'
                        resistance_display = 'R: Res'
                    elif sample_a == 'not_detected' or sample_b == 'not_detected':
                        detected = 'TB Not Detected'
                        drug_sensitive_list = 'r'
                    else:
                        detected = ''

                    result_summary_display = '\n'.join(filter(None, [
                        detected,
                        resistance_display,
                    ]))

                    case_id = test.case_id
                    f.write(case_id + "\n")
                    logger.info(case_id)

                    case_structure = CaseStructure(
                        case_id=case_id,
                        walk_related=False,
                        attrs={
                            "create": False,
                            "update": {
                                "updated_by_migration": "enikshay_2b_cbnaat_fix",
                                "drug_resistance_list": drug_resistance_list,
                                "drug_sensitive_list": drug_sensitive_list,
                                "result_summary_display": result_summary_display,
                            },
                        },
                    )

                    if commit:
                        factory.create_or_update_case(case_structure)
        logger.info("Migration finished at {}".format(datetime.datetime.utcnow()))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import paramiko  # pylint: disable=import-error
import os
import sys
import re
import time
import socket

SOURCE_DIR = "/data/openpilot_source/"
TEST_DIR = "/data/openpilot/"

def run_on_phone(test_cmd):

  eon_ip = os.environ.get('eon_ip', None)
  if eon_ip is None:
    raise Exception("'eon_ip' not set")

  ssh = paramiko.SSHClient()
  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

  key_file = open(os.path.join(os.path.dirname(__file__), "id_rsa"))
  key = paramiko.RSAKey.from_private_key(key_file)

  print("SSH to phone at {}".format(eon_ip))

  # try connecting for one minute
  t_start = time.time()
  while True:
    try:
      ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
    except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
      print("Connection failed")
      if time.time() - t_start > 60:
        raise
    else:
      break
    time.sleep(1)

  branch = os.environ['GIT_BRANCH']
  commit = os.environ.get('GIT_COMMIT', branch)

  conn = ssh.invoke_shell()

  # pass in all environment variables prefixed with 'CI_'
  for k, v in os.environ.items():
    if k.startswith("CI_") or k in ["GIT_BRANCH", "GIT_COMMIT"]:
      conn.send(f"export {k}='{v}'\n")
  conn.send("export CI=1\n")

  # set up environment
  conn.send(f"cd {SOURCE_DIR}\n")
  conn.send("git reset --hard\n")
  conn.send("git fetch origin\n")
  conn.send("find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \\;\n")
  conn.send(f"git reset --hard {commit}\n")
  conn.send(f"git checkout {commit}\n")
  conn.send("git clean -xdf\n")
  conn.send("git submodule update --init\n")
  conn.send("git submodule foreach --recursive git reset --hard\n")
  conn.send("git submodule foreach --recursive git clean -xdf\n")
  conn.send('echo "git took $SECONDS seconds"\n')

  conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")

  # run the test
  conn.send(test_cmd + "\n")

  # get the result and print it back out
  conn.send('echo "RESULT:" $?\n')
  conn.send("exit\n")

  dat = b""
  conn.settimeout(240)

  while True:
    try:
      recvd = conn.recv(4096)
    except socket.timeout:
      print("connection to phone timed out")
      sys.exit(1)

    if len(recvd) == 0:
      break

    dat += recvd
    sys.stdout.buffer.write(recvd)
    sys.stdout.flush()

  return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
  sys.exit(return_code)

if __name__ == "__main__":
  run_on_phone(sys.argv[1])
<commit_msg>clear scons cache dirs older than a day in CI<commit_after>#!/usr/bin/env python3
import paramiko  # pylint: disable=import-error
import os
import sys
import re
import time
import socket

SOURCE_DIR = "/data/openpilot_source/"
TEST_DIR = "/data/openpilot/"

def run_on_phone(test_cmd):

  eon_ip = os.environ.get('eon_ip', None)
  if eon_ip is None:
    raise Exception("'eon_ip' not set")

  ssh = paramiko.SSHClient()
  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

  key_file = open(os.path.join(os.path.dirname(__file__), "id_rsa"))
  key = paramiko.RSAKey.from_private_key(key_file)

  print("SSH to phone at {}".format(eon_ip))

  # try connecting for one minute
  t_start = time.time()
  while True:
    try:
      ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
    except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
      print("Connection failed")
      if time.time() - t_start > 60:
        raise
    else:
      break
    time.sleep(1)

  branch = os.environ['GIT_BRANCH']
  commit = os.environ.get('GIT_COMMIT', branch)

  conn = ssh.invoke_shell()

  # pass in all environment variables prefixed with 'CI_'
  for k, v in os.environ.items():
    if k.startswith("CI_") or k in ["GIT_BRANCH", "GIT_COMMIT"]:
      conn.send(f"export {k}='{v}'\n")
  conn.send("export CI=1\n")

  # clear scons cache dirs that haven't been written to in one day
  conn.send("cd /tmp && find -name 'scons_cache_*' -type d -maxdepth 1 -mtime 1 -exec rm -rf '{}' \\;\n")

  # set up environment
  conn.send(f"cd {SOURCE_DIR}\n")
  conn.send("git reset --hard\n")
  conn.send("git fetch origin\n")
  conn.send("find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \\;\n")
  conn.send(f"git reset --hard {commit}\n")
  conn.send(f"git checkout {commit}\n")
  conn.send("git clean -xdf\n")
  conn.send("git submodule update --init\n")
  conn.send("git submodule foreach --recursive git reset --hard\n")
  conn.send("git submodule foreach --recursive git clean -xdf\n")
  conn.send('echo "git took $SECONDS seconds"\n')

  conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")

  # run the test
  conn.send(test_cmd + "\n")

  # get the result and print it back out
  conn.send('echo "RESULT:" $?\n')
  conn.send("exit\n")

  dat = b""
  conn.settimeout(240)

  while True:
    try:
      recvd = conn.recv(4096)
    except socket.timeout:
      print("connection to phone timed out")
      sys.exit(1)

    if len(recvd) == 0:
      break

    dat += recvd
    sys.stdout.buffer.write(recvd)
    sys.stdout.flush()

  return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
  sys.exit(return_code)

if __name__ == "__main__":
  run_on_phone(sys.argv[1])
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import logging

from osc_lib.command import command
from osc_lib import utils

class Service(command.ShowOne):
    """Retrieve build information."""

    log = logging.getLogger(__name__ + ".Service")

    def get_parser(self, prog_name):
        parser = super(Service, self).get_parser(prog_name)
        return parser

    def take_action(self, parsed_args):
        self.log.debug("take_action(%s)", parsed_args)

        senlin_client = self.app.client_manager.clustering
        queries = {}
        result = senlin_client.get_service(**queries)

        formatters = {}
        columns = ['Binary', 'Host', 'Status', 'State', 'Updated_at',
                   'Disabled Reason']
        return columns, utils.get_dict_properties(result, columns,
                                                  formatters=formatters)
<commit_msg>Revise service list help info<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import logging

from osc_lib.command import command
from osc_lib import utils

class Service(command.ShowOne):
    """Show a list of all running services."""

    log = logging.getLogger(__name__ + ".Service")

    def get_parser(self, prog_name):
        parser = super(Service, self).get_parser(prog_name)
        return parser

    def take_action(self, parsed_args):
        self.log.debug("take_action(%s)", parsed_args)

        senlin_client = self.app.client_manager.clustering
        queries = {}
        result = senlin_client.get_service(**queries)

        formatters = {}
        columns = ['Binary', 'Host', 'Status', 'State', 'Updated_at',
                   'Disabled Reason']
        return columns, utils.get_dict_properties(result, columns,
                                                  formatters=formatters)
<|endoftext|>