<commit_before>import subprocess
from subprocess import check_call as call, PIPE
import unittest
import time

from jbrowse_selenium import JBrowseTest

class AbstractVolvoxBiodbTest( JBrowseTest ):

    data_dir = 'sample_data/json/volvox'

    def setUp( self ):
        call( "rm -rf sample_data/json/volvox/", shell=True )
        call( "bin/prepare-refseqs.pl --fasta docs/tutorial/data_files/volvox.fa --out sample_data/json/volvox/", shell=True )
        call( "bin/biodb-to-json.pl --conf docs/tutorial/conf_files/volvox.json --out sample_data/json/volvox/", shell=True )
        call( "bin/wig-to-json.pl --out sample_data/json/volvox/ --wig docs/tutorial/data_files/volvox_microarray.wig", shell=True )
        call( "bin/add-track-json.pl sample_data/raw/volvox/volvox_microarray.bw.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/add-track-json.pl sample_data/raw/volvox/volvox-sorted.bam.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/add-track-json.pl sample_data/raw/volvox/volvox-sorted.bam.coverage.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/add-track-json.pl docs/tutorial/data_files/volvox.vcf.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/generate-names.pl --dir sample_data/json/volvox/", shell=True )
        super( AbstractVolvoxBiodbTest, self ).setUp()

    def test_volvox( self ):

        # select "ctgA from the dropdown
        self.select_refseq( 'ctgA' )

        # check a good browser title
        assert "ctgA" in self.browser.title, "browser title is actually %s" % self.browser.title

        # do a test where we search for a certain gene using the search box
        self.search_f15()

        self.assert_no_js_errors()

        # test scrolling, make sure we get no js errors
        self.scroll()

        # test context menus
        self.context_menus()

        # test dragging in and displaying the wiggle track
        self.wiggle()

        # test sequence track display
        self.sequence()

        # test that the frame usage track claims to have links to NCBI
        self.turn_on_track( 'HTMLFeatures - mRNAs' )
        self.do_typed_query('ctgA:2,381..21,220')
        self.assert_element("//div[@title='search at NCBI']")
        self.turn_off_track( 'HTMLFeatures - mRNAs' )

        # test combination tracks
        self.combination()

        # test search tracks
        self.search_track()

        # test bigwig
        self.bigwig()

        # test data export
        self.export()

        # test bam
        self.bam()

        # test vcf
        self.vcf()

        # test CanvasFeatures tracks
        self.canvasfeatures()

        #self.browser.close()

    def canvasfeatures( self ):

        # turn on CanvasFeatures tracks and make sure they are created
        self.do_typed_query( 'Apple1' )
        self.turn_on_track('CanvasFeatures - Protein-coding genes')

        self.assert_elements("//div[@id='track_Genes']//canvas")
        self.assert_elements("//div[@id='track_CDS']//canvas")
        self.assert_no_js_errors()

        self.turn_off_track('CanvasFeatures - mixed')

        # test left-clicking on CanvasFeatures track
        self.do_typed_query( 'ctgA:1049..9000' )
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'details')]")
        canvas = self.assert_element("//div[@id='track_Genes']/canvas")
        canvas.click()
        self.assert_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'details')]")
        self.close_dialog("EDEN details")
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'snippet')]")

        # test Canvas-features context menu functionality
        # right-click one of them
        self.actionchains() \
            .context_click(canvas) \
            .perform()

        self.menu_item_click("Popup with content snippet from string (feature EDEN)")
        self.assert_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'snippet')]")
        self.close_dialog('snippet')

        # turn off canvasFeatures tracks so they're not cluttering everything up
        self.turn_off_track('CanvasFeatures - Protein-coding genes')

    def vcf( self ):
        self.do_typed_query('ctgA:18918..19070')
        self.turn_on_track('VCF - volvox-sorted')
        self.turn_on_track('VCF - additional')

        self.assert_elements("//div[@id='track_volvox_sorted_vcf']//canvas")
        self.assert_elements("//div[@id='track_volvox_vcf_test']//canvas")

        self.turn_off_track('VCF - volvox-sorted')
        self.turn_off_track('VCF - additional')

    def bam( self ):
        self.do_typed_query('ctgA:18918..19070')
        self.turn_on_track('volvox-sorted.bam')
        self.turn_on_track('volvox-sorted SNPs/Coverage')

        self.assert_elements("//div[@id='track_volvox_sorted_bam']//canvas")
        self.assert_elements("//div[@id='track_volvox_sorted_bam_coverage']//canvas")

        self.turn_off_track('volvox-sorted.bam')
        self.turn_off_track('volvox-sorted SNPs/Coverage')

    def export( self ):
        self.do_typed_query('ctgA')

        self.turn_on_track( 'BigWig XY - volvox_microarray' )
        trackname = 'volvox_microarray.bw_xyplot'
        self.export_track( trackname, 'Visible region','GFF3','View')
        self.close_dialog('export')
        self.export_track( trackname, 'Whole', 'bedGraph', 'Save' )
        self.export_track( trackname, 'Whole', 'Wiggle', 'Save' )
        self.turn_off_track( 'BigWig XY - volvox_microarray' )

        self.turn_on_track( 'HTMLFeatures - Example Features' )
        trackname = 'ExampleFeatures'
        self.export_track( trackname, 'Visible region','GFF3','View')
        self.close_dialog('export')
        self.export_track( trackname, 'Visible region','BED','Save')
        self.turn_off_track( 'HTMLFeatures - Example Features' )

        self.turn_on_track( 'CanvasFeatures - transcripts' )
        trackname = 'Transcript'
        self.export_track( trackname, 'Visible region', 'GFF3', 'View')
        self.close_dialog('export')
        self.export_track( trackname, 'Visible region', 'BED', 'Save')
        self.turn_off_track( 'CanvasFeatures - transcripts' )

        self.do_typed_query('ctgA:8379..31627')
        self.export_track( 'DNA', 'Visible region','FASTA','View')
        self.assert_no_js_errors()
        self.close_dialog('export')
        self.assert_no_js_errors()

    def bigwig( self ):
        self.turn_on_track('BigWig XY - volvox_microarray')
        self.assert_elements("//div[@id='track_volvox_microarray.bw_xyplot']//canvas")
        self.assert_no_js_errors()
        self.turn_on_track('BigWig Density - volvox_microarray')
        self.assert_elements("//div[@id='track_volvox_microarray.bw_density']//canvas")
        self.assert_no_js_errors()
        self.turn_off_track('BigWig XY - volvox_microarray')
        self.turn_off_track('BigWig Density - volvox_microarray')

    def sequence( self ):
        self.do_typed_query( '0..80' )
        sequence_div_xpath_templ = "/html//div[contains(@class,'sequence')]//*[contains(.,'%s')]"
        sequence_div_xpath_1 = sequence_div_xpath_templ % 'aacaACGG'
        self.assert_element( sequence_div_xpath_1)
        self.turn_off_track( 'Reference sequence' )
        self.assert_no_element( sequence_div_xpath_1 )
        self.turn_on_track( 'Reference sequence' )
        self.assert_element( sequence_div_xpath_1 )
        self.do_typed_query( '1..20000')
        self.assert_no_element( sequence_div_xpath_1 )
        self.do_typed_query( 'ctgA:19961..20047')
        self.assert_element( sequence_div_xpath_templ % 'ccgcgtgtagtc' )

    def context_menus( self ):
        self.turn_on_track( 'HTMLFeatures - Features with menus' )
        self.do_typed_query( '20147..35574' )

        # check that there is no dialog open
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title,'snippet')]")

        # get the example alignments features
        feature_elements = self.assert_elements("//div[@id='track_malformed_alignments']//div[contains(@class,'plus-feature4')]")

        # right-click one of them
        self.actionchains() \
            .context_click(feature_elements[int(len(feature_elements)/2)]) \
            .perform()

        self.menu_item_click( 'Open popup' )

        # check that the proper HTML snippet popped up in the dialog
        self.assert_element("//div[contains(@class,'dijitDialog')]//span[@class='amazingTestSnippet']")

        self.close_dialog('Random XHR')

        # check that the dialog closed
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title,'Random XHR')]")
        self.turn_off_track( 'HTMLFeatures - Features with menus' )

    def wiggle( self ):

        self.turn_on_track( 'microarray' )

        # see that we have an image track png in the DOM now
        imagetrack_xpath =  "//div[contains(@class,'track')]//img[@class='image-track']"
        imagetrack_png = self.assert_element( imagetrack_xpath )

        self.turn_off_track( 'microarray' )
        # check that imagetrack png is not still in the DOM after the
        # track is turned off
        self.assert_no_element( imagetrack_xpath )

    def search_f15( self ):

        # check that a f15 feature label is not yet in the DOM
        xpath = "//div[contains(@class,'feature-label')]//*[contains(.,'f15')]"
        # check that f15 is not already in the DOM at load time
        self.assert_no_element( xpath )

        self.do_typed_query( "f15" )

        # test that f15 appeared in the DOM (TODO: check that it's
        # actually centered in the window), and that the protein-coding
        # genes track is now selected
        assert self.assert_element(xpath).text =='f15'

        self.turn_off_track('HTMLFeatures - Example Features')

    def search_track( self ):
        self.assert_element("#dropdownbutton_file").click()
        self.assert_element("#dijit_MenuItem_0").click()
        text = "aaaccc"
        box = self.assert_element("#dijit_form_TextBox_0")
        for i in range( len(text) ):
            box.send_keys( text[i] )
        self.assert_element("#dijit_form_Button_3_label").click()
        self.assert_elements("//div[@id='track_search_track_0']//canvas")
        self.turn_off_track("Search reference sequence for")

    def combination( self ):
        self.assert_element("#dropdownbutton_file").click()
        self.assert_element("#menubar_combotrack_text").click()

        self.turn_on_track("HTMLFeatures - mRNAs")
        mRNA = self.assert_element("#label_ReadingFrame")
        combo_track = self.assert_element("#track_combination_track0")
        self.actionchains().move_to_element(mRNA).click_and_hold().move_to_element(combo_track).move_by_offset(4, 60).release().perform()
        self.turn_off_track("HTMLFeatures - mRNAs")

class VolvoxBiodbTest( AbstractVolvoxBiodbTest, unittest.TestCase ):
    pass

if __name__ == '__main__':
    import unittest
    unittest.main()

<commit_msg>moved tests around, changed element css to match<commit_after>import subprocess
from subprocess import check_call as call, PIPE
import unittest
import time

from jbrowse_selenium import JBrowseTest

class AbstractVolvoxBiodbTest( JBrowseTest ):

    data_dir = 'sample_data/json/volvox'

    def setUp( self ):
        call( "rm -rf sample_data/json/volvox/", shell=True )
        call( "bin/prepare-refseqs.pl --fasta docs/tutorial/data_files/volvox.fa --out sample_data/json/volvox/", shell=True )
        call( "bin/biodb-to-json.pl --conf docs/tutorial/conf_files/volvox.json --out sample_data/json/volvox/", shell=True )
        call( "bin/wig-to-json.pl --out sample_data/json/volvox/ --wig docs/tutorial/data_files/volvox_microarray.wig", shell=True )
        call( "bin/add-track-json.pl sample_data/raw/volvox/volvox_microarray.bw.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/add-track-json.pl sample_data/raw/volvox/volvox-sorted.bam.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/add-track-json.pl sample_data/raw/volvox/volvox-sorted.bam.coverage.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/add-track-json.pl docs/tutorial/data_files/volvox.vcf.conf sample_data/json/volvox/trackList.json", shell=True )
        call( "bin/generate-names.pl --dir sample_data/json/volvox/", shell=True )
        super( AbstractVolvoxBiodbTest, self ).setUp()

    def test_volvox( self ):

        # select "ctgA from the dropdown
        self.select_refseq( 'ctgA' )

        # check a good browser title
        assert "ctgA" in self.browser.title, "browser title is actually %s" % self.browser.title

        # do a test where we search for a certain gene using the search box
        self.search_f15()

        self.assert_no_js_errors()

        # test scrolling, make sure we get no js errors
        self.scroll()

        # test context menus
        self.context_menus()

        # test dragging in and displaying the wiggle track
        self.wiggle()

        # test sequence track display
        self.sequence()

        # test that the frame usage track claims to have links to NCBI
        self.turn_on_track( 'HTMLFeatures - mRNAs' )
        self.do_typed_query('ctgA:2,381..21,220')
        self.assert_element("//div[@title='search at NCBI']")
        self.turn_off_track( 'HTMLFeatures - mRNAs' )

        # test bigwig
        self.bigwig()

        # test data export
        self.export()

        # test bam
        self.bam()

        # test vcf
        self.vcf()

        # test CanvasFeatures tracks
        self.canvasfeatures()

        # test search tracks
        self.search_track()

        # test combination tracks
        self.combination()

        #self.browser.close()

    def canvasfeatures( self ):

        # turn on CanvasFeatures tracks and make sure they are created
        self.do_typed_query( 'Apple1' )
        self.turn_on_track('CanvasFeatures - Protein-coding genes')

        self.assert_elements("//div[@id='track_Genes']//canvas")
        self.assert_elements("//div[@id='track_CDS']//canvas")
        self.assert_no_js_errors()

        self.turn_off_track('CanvasFeatures - mixed')

        # test left-clicking on CanvasFeatures track
        self.do_typed_query( 'ctgA:1049..9000' )
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'details')]")
        canvas = self.assert_element("//div[@id='track_Genes']/canvas")
        canvas.click()
        self.assert_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'details')]")
        self.close_dialog("EDEN details")
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'snippet')]")

        # test Canvas-features context menu functionality
        # right-click one of them
        self.actionchains() \
            .context_click(canvas) \
            .perform()

        self.menu_item_click("Popup with content snippet from string (feature EDEN)")
        self.assert_element("//div[@class='dijitDialogTitleBar'][contains(@title, 'snippet')]")
        self.close_dialog('snippet')

        # turn off canvasFeatures tracks so they're not cluttering everything up
        self.turn_off_track('CanvasFeatures - Protein-coding genes')

    def vcf( self ):
        self.do_typed_query('ctgA:18918..19070')
        self.turn_on_track('VCF - volvox-sorted')
        self.turn_on_track('VCF - additional')

        self.assert_elements("//div[@id='track_volvox_sorted_vcf']//canvas")
        self.assert_elements("//div[@id='track_volvox_vcf_test']//canvas")

        self.turn_off_track('VCF - volvox-sorted')
        self.turn_off_track('VCF - additional')

    def bam( self ):
        self.do_typed_query('ctgA:18918..19070')
        self.turn_on_track('volvox-sorted.bam')
        self.turn_on_track('volvox-sorted SNPs/Coverage')

        self.assert_elements("//div[@id='track_volvox_sorted_bam']//canvas")
        self.assert_elements("//div[@id='track_volvox_sorted_bam_coverage']//canvas")

        self.turn_off_track('volvox-sorted.bam')
        self.turn_off_track('volvox-sorted SNPs/Coverage')

    def export( self ):
        self.do_typed_query('ctgA')

        self.turn_on_track( 'BigWig XY - volvox_microarray' )
        trackname = 'volvox_microarray.bw_xyplot'
        self.export_track( trackname, 'Visible region','GFF3','View')
        self.close_dialog('export')
        self.export_track( trackname, 'Whole', 'bedGraph', 'Save' )
        self.export_track( trackname, 'Whole', 'Wiggle', 'Save' )
        self.turn_off_track( 'BigWig XY - volvox_microarray' )

        self.turn_on_track( 'HTMLFeatures - Example Features' )
        trackname = 'ExampleFeatures'
        self.export_track( trackname, 'Visible region','GFF3','View')
        self.close_dialog('export')
        self.export_track( trackname, 'Visible region','BED','Save')
        self.turn_off_track( 'HTMLFeatures - Example Features' )

        self.turn_on_track( 'CanvasFeatures - transcripts' )
        trackname = 'Transcript'
        self.export_track( trackname, 'Visible region', 'GFF3', 'View')
        self.close_dialog('export')
        self.export_track( trackname, 'Visible region', 'BED', 'Save')
        self.turn_off_track( 'CanvasFeatures - transcripts' )

        self.do_typed_query('ctgA:8379..31627')
        self.export_track( 'DNA', 'Visible region','FASTA','View')
        self.assert_no_js_errors()
        self.close_dialog('export')
        self.assert_no_js_errors()

    def bigwig( self ):
        self.turn_on_track('BigWig XY - volvox_microarray')
        self.assert_elements("//div[@id='track_volvox_microarray.bw_xyplot']//canvas")
        self.assert_no_js_errors()
        self.turn_on_track('BigWig Density - volvox_microarray')
        self.assert_elements("//div[@id='track_volvox_microarray.bw_density']//canvas")
        self.assert_no_js_errors()
        self.turn_off_track('BigWig XY - volvox_microarray')
        self.turn_off_track('BigWig Density - volvox_microarray')

    def sequence( self ):
        self.do_typed_query( '0..80' )
        sequence_div_xpath_templ = "/html//div[contains(@class,'sequence')]//*[contains(.,'%s')]"
        sequence_div_xpath_1 = sequence_div_xpath_templ % 'aacaACGG'
        self.assert_element( sequence_div_xpath_1)
        self.turn_off_track( 'Reference sequence' )
        self.assert_no_element( sequence_div_xpath_1 )
        self.turn_on_track( 'Reference sequence' )
        self.assert_element( sequence_div_xpath_1 )
        self.do_typed_query( '1..20000')
        self.assert_no_element( sequence_div_xpath_1 )
        self.do_typed_query( 'ctgA:19961..20047')
        self.assert_element( sequence_div_xpath_templ % 'ccgcgtgtagtc' )

    def context_menus( self ):
        self.turn_on_track( 'HTMLFeatures - Features with menus' )
        self.do_typed_query( '20147..35574' )

        # check that there is no dialog open
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title,'snippet')]")

        # get the example alignments features
        feature_elements = self.assert_elements("//div[@id='track_malformed_alignments']//div[contains(@class,'plus-feature4')]")

        # right-click one of them
        self.actionchains() \
            .context_click(feature_elements[int(len(feature_elements)/2)]) \
            .perform()

        self.menu_item_click( 'Open popup' )

        # check that the proper HTML snippet popped up in the dialog
        self.assert_element("//div[contains(@class,'dijitDialog')]//span[@class='amazingTestSnippet']")

        self.close_dialog('Random XHR')

        # check that the dialog closed
        self.assert_no_element("//div[@class='dijitDialogTitleBar'][contains(@title,'Random XHR')]")
        self.turn_off_track( 'HTMLFeatures - Features with menus' )

    def wiggle( self ):

        self.turn_on_track( 'microarray' )

        # see that we have an image track png in the DOM now
        imagetrack_xpath =  "//div[contains(@class,'track')]//img[@class='image-track']"
        imagetrack_png = self.assert_element( imagetrack_xpath )

        self.turn_off_track( 'microarray' )
        # check that imagetrack png is not still in the DOM after the
        # track is turned off
        self.assert_no_element( imagetrack_xpath )

    def search_f15( self ):

        # check that a f15 feature label is not yet in the DOM
        xpath = "//div[contains(@class,'feature-label')]//*[contains(.,'f15')]"
        # check that f15 is not already in the DOM at load time
        self.assert_no_element( xpath )

        self.do_typed_query( "f15" )

        # test that f15 appeared in the DOM (TODO: check that it's
        # actually centered in the window), and that the protein-coding
        # genes track is now selected
        assert self.assert_element(xpath).text =='f15'

        self.turn_off_track('HTMLFeatures - Example Features')

    def search_track( self ):
        self.assert_element("#dropdownbutton_file").click()
        self.assert_element("#dijit_MenuItem_0").click()
        text = "aaaccc"
        box = self.assert_element("#dijit_form_TextBox_4")
        for i in range( len(text) ):
            box.send_keys( text[i] )
        self.assert_element("#dijit_form_Button_36_label").click()
        self.assert_elements("//div[@id='track_search_track_0']//canvas")
        self.turn_off_track("Search reference sequence for")

    def combination( self ):
        self.assert_element("#dropdownbutton_file").click()
        self.assert_element("#menubar_combotrack_text").click()

        self.turn_on_track("HTMLFeatures - mRNAs")
        mRNA = self.assert_element("#label_ReadingFrame")
        combo_track = self.assert_element("#track_combination_track0")
        self.actionchains().move_to_element(mRNA).click_and_hold().move_to_element(combo_track).move_by_offset(4, 60).release().perform()
        self.turn_off_track("HTMLFeatures - mRNAs")

class VolvoxBiodbTest( AbstractVolvoxBiodbTest, unittest.TestCase ):
    pass

if __name__ == '__main__':
    import unittest
    unittest.main()

<|endoftext|>


<commit_before>import matplotlib
import platform
from os import environ
# set only if it's not MacOS
if environ.get('CI') or platform.system() == 'Darwin':
    matplotlib.rcParams['backend'] = 'agg'
else:
    matplotlib.rcParams['backend'] = 'TkAgg'

import seaborn as sns
sns.set(style="whitegrid", color_codes=True, font_scale=0.8,
        rc={'lines.linewidth': 1.0, 'backend': matplotlib.rcParams['backend']})
palette = sns.color_palette("Blues_d")
palette.reverse()
sns.set_palette(palette)

import numpy as np
import pandas as pd
import warnings
from functools import partial
from rl.util import *

warnings.filterwarnings("ignore", module="matplotlib")

STATS_COLS = [
    'performance_score',
    'mean_rewards_per_epi_stats_mean',
    'mean_rewards_stats_mean',
    'epi_stats_mean',
    'solved_ratio_of_sessions',
    'num_of_sessions',
    'max_total_rewards_stats_mean',
    't_stats_mean',
    'trial_id'
]
EXPERIMENT_GRID_Y_COLS = [
    'performance_score',
    'mean_rewards_stats_mean',
    'max_total_rewards_stats_mean'
]

class Grapher(object):

    '''
    Grapher object that belongs to a Session
    to draw graphs from its data
    '''

    def __init__(self, session):
        if not args.plot_graph or environ.get('CI'):
            return
        import matplotlib.pyplot as plt
        plt.rcParams['toolbar'] = 'None'  # mute matplotlib toolbar
        self.plt = plt
        self.session = session
        self.graph_filename = self.session.graph_filename
        self.subgraphs = {}
        self.figure = self.plt.figure(facecolor='white', figsize=(8, 9))
        self.figure.suptitle(wrap_text(self.session.session_id))
        self.init_figure()

    def init_figure(self):
        # graph 1
        ax1 = self.figure.add_subplot(
            311,
            frame_on=False,
            title="\n\ntotal rewards per episode",
            ylabel='total rewards')
        p1, = ax1.plot([], [])
        self.subgraphs['total rewards'] = (ax1, p1)

        ax1e = ax1.twinx()
        ax1e.set_ylabel('exploration rate').set_color('r')
        ax1e.set_frame_on(False)
        ax1e.grid(False)
        p1e, = ax1e.plot([], [], 'r')
        self.subgraphs['e'] = (ax1e, p1e)

        # graph 2
        ax2 = self.figure.add_subplot(
            312,
            frame_on=False,
            title='mean rewards over last 100 episodes',
            ylabel='mean rewards')
        p2, = ax2.plot([], [], 'g')
        self.subgraphs['mean rewards'] = (ax2, p2)

        # graph 3
        ax3 = self.figure.add_subplot(
            313,
            frame_on=False,
            title='loss over time, episode',
            ylabel='loss')
        p3, = ax3.plot([], [])
        self.subgraphs['loss'] = (ax3, p3)

        self.plt.tight_layout()  # auto-fix spacing
        self.plt.ion()  # for live plot

    def plot(self):
        '''do live plotting'''
        if not args.plot_graph or environ.get('CI'):
            return
        sys_vars = self.session.sys_vars
        ax1, p1 = self.subgraphs['total rewards']
        p1.set_ydata(
            sys_vars['total_rewards_history'])
        p1.set_xdata(np.arange(len(p1.get_ydata())))
        ax1.relim()
        ax1.autoscale_view(tight=True, scalex=True, scaley=True)

        ax1e, p1e = self.subgraphs['e']
        p1e.set_ydata(
            sys_vars['explore_history'])
        p1e.set_xdata(np.arange(len(p1e.get_ydata())))
        ax1e.relim()
        ax1e.autoscale_view(tight=True, scalex=True, scaley=True)

        ax2, p2 = self.subgraphs['mean rewards']
        p2.set_ydata(
            sys_vars['mean_rewards_history'])
        p2.set_xdata(np.arange(len(p2.get_ydata())))
        ax2.relim()
        ax2.autoscale_view(tight=True, scalex=True, scaley=True)

        ax3, p3 = self.subgraphs['loss']
        p3.set_ydata(sys_vars['loss'])
        p3.set_xdata(np.arange(len(p3.get_ydata())))
        ax3.relim()
        ax3.autoscale_view(tight=True, scalex=True, scaley=True)

        self.plt.draw()
        self.plt.pause(0.01)
        self.save()

    def save(self):
        '''save graph to filename'''
        self.figure.savefig(self.graph_filename)

    def clear(self):
        if not args.plot_graph or environ.get('CI'):
            return
        self.plt.close()
        del_self_attr(self)

def basic_stats(array):
    '''generate the basic stats for a numerical array'''
    if not len(array):
        return None
    return {
        'min': np.min(array).astype(float),
        'max': np.max(array).astype(float),
        'mean': np.mean(array).astype(float),
        'std': np.std(array).astype(float),
    }

def compose_data(trial):
    '''
    compose raw data from an trial object
    into useful summary and full metrics for analysis
    '''
    sys_vars_array = trial.data['sys_vars_array']

    # collect all data from sys_vars_array
    solved_sys_vars_array = list(filter(
        lambda sv: sv['solved'], sys_vars_array))
    mean_rewards_array = np.array(list(map(
        lambda sv: sv['mean_rewards'], sys_vars_array)))
    max_total_rewards_array = np.array(list(map(
        lambda sv: np.max(sv['total_rewards_history']), sys_vars_array)))
    epi_array = np.array(list(map(lambda sv: sv['epi'], sys_vars_array)))
    mean_rewards_per_epi_array = np.divide(mean_rewards_array, epi_array)
    t_array = np.array(list(map(lambda sv: sv['t'], sys_vars_array)))
    time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        sys_vars_array)))
    solved_epi_array = np.array(list(map(
        lambda sv: sv['epi'], solved_sys_vars_array)))
    solved_t_array = np.array(list(map(
        lambda sv: sv['t'], solved_sys_vars_array)))
    solved_time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        solved_sys_vars_array)))

    # compose sys_vars stats
    stats = {
        'num_of_sessions': len(sys_vars_array),
        'solved_num_of_sessions': len(solved_sys_vars_array),
        'solved_ratio_of_sessions': float(len(
            solved_sys_vars_array)) / trial.times,
        'mean_rewards_stats': basic_stats(mean_rewards_array),
        'mean_rewards_per_epi_stats': basic_stats(
            mean_rewards_per_epi_array),
        'max_total_rewards_stats': basic_stats(max_total_rewards_array),
        'epi_stats': basic_stats(epi_array),
        't_stats': basic_stats(t_array),
        'time_taken_stats': basic_stats(time_taken_array),
        'solved_epi_stats': basic_stats(solved_epi_array),
        'solved_t_stats': basic_stats(solved_t_array),
        'solved_time_taken_stats': basic_stats(solved_time_taken_array),
    }
    stats.update({
        'performance_score': stats[
            'mean_rewards_per_epi_stats']['mean'] * (stats[
                'solved_ratio_of_sessions'] ** 2)
    })

    # summary metrics
    metrics = {
        'performance_score': stats['performance_score'],
        'mean_rewards_per_epi_stats_mean': stats[
            'mean_rewards_per_epi_stats']['mean'],
        'mean_rewards_stats_mean': stats['mean_rewards_stats']['mean'],
        'epi_stats_mean': stats['epi_stats']['mean'],
        'solved_ratio_of_sessions': stats['solved_ratio_of_sessions'],
        'max_total_rewards_stats_mean': stats[
            'max_total_rewards_stats']['mean'],
        't_stats_mean': stats['t_stats']['mean'],
    }

    # param variables for independent vars of trials
    param_variables = {
        pv: trial.experiment_spec['param'][pv] for
        pv in trial.param_variables}
    param_variables = flat_cast_dict(param_variables)

    trial.data['metrics'].update(metrics)
    trial.data['param_variables'] = param_variables
    trial.data['stats'] = stats
    return trial.data

# plot the experiment data from data_df
# X are columns with name starting with 'variable_'
# Y cols are defined below
def plot_experiment(data_df, trial_id):
    if len(data_df) < 2:  # no multi selection
        return
    experiment_id = parse_experiment_id(trial_id)
    X_cols = list(filter(lambda c: c.startswith('variable_'), data_df.columns))
    for x in X_cols:
        for y in EXPERIMENT_GRID_Y_COLS:
            df_plot = sns.swarmplot(data=data_df, x=x, y=y,
                                    hue='solved_ratio_of_sessions')
            fig = df_plot.get_figure()
            fig.suptitle(wrap_text(experiment_id))
            filename = './data/{}/{}_analysis_{}_vs_{}.png'.format(
                experiment_id, experiment_id, x, y)
            fig.savefig(filename)
            fig.clear()

    fig = sns.PairGrid(
        data_df, x_vars=X_cols, y_vars=EXPERIMENT_GRID_Y_COLS,
        hue='solved_ratio_of_sessions')
    fig.map(partial(sns.swarmplot, size=3))
    fig.fig.suptitle(wrap_text(experiment_id))
    fig.add_legend()
    filename = './data/{0}/{0}_analysis.png'.format(
        experiment_id)
    fig.savefig(filename)
    fig.fig.clear()
    sns.plt.close()

def analyze_data(experiment_data_or_experiment_id):
    '''
    get all the data from all trials.run()
    or read from all data files matching the prefix of trial_id
    e.g. usage without running:
    experiment_id = 'DevCartPole-v0_DQN_LinearMemoryWithForgetting_BoltzmannPolicy_2017-01-15_142810'
    analyze_data(experiment_id)
    '''
    if isinstance(experiment_data_or_experiment_id, str):
        experiment_data = load_data_array_from_experiment_id(
            experiment_data_or_experiment_id)
    else:
        experiment_data = experiment_data_or_experiment_id

    stats_array, param_variables_array = [], []
    for data in experiment_data:
        stats = flatten_dict(data['stats'])
        stats.update({'trial_id': data['trial_id']})
        stats_array.append(stats)
        param_variables = data['param_variables']
        param_variables_array.append(param_variables)

    raw_stats_df = pd.DataFrame.from_dict(stats_array)
    stats_df = raw_stats_df[STATS_COLS]

    param_variables_df = pd.DataFrame.from_dict(param_variables_array)
    param_variables_df.columns = [
        'variable_'+c for c in param_variables_df.columns]

    data_df = pd.concat([stats_df, param_variables_df], axis=1)
    for c in data_df.columns:
        if data_df[c].dtype == object:  # guard
            data_df[c] = data_df[c].astype('category')

    data_df.sort_values(
        ['performance_score'],
        inplace=True, ascending=False)

    trial_id = experiment_data[0]['trial_id']
    save_experiment_data(data_df, trial_id)
    plot_experiment(data_df, trial_id)
    return data_df
<commit_msg>add epi plot to analysis, resize bigger<commit_after>import matplotlib
import platform
from os import environ
# set only if it's not MacOS
if environ.get('CI') or platform.system() == 'Darwin':
    matplotlib.rcParams['backend'] = 'agg'
else:
    matplotlib.rcParams['backend'] = 'TkAgg'

import seaborn as sns
sns.set(style="whitegrid", color_codes=True, font_scale=1.0,
        rc={'lines.linewidth': 1.0, 'backend': matplotlib.rcParams['backend']})
palette = sns.color_palette("Blues_d")
palette.reverse()
sns.set_palette(palette)

import numpy as np
import pandas as pd
import warnings
from functools import partial
from rl.util import *

warnings.filterwarnings("ignore", module="matplotlib")

STATS_COLS = [
    'performance_score',
    'mean_rewards_per_epi_stats_mean',
    'mean_rewards_stats_mean',
    'epi_stats_mean',
    'solved_ratio_of_sessions',
    'num_of_sessions',
    'max_total_rewards_stats_mean',
    't_stats_mean',
    'trial_id'
]
EXPERIMENT_GRID_Y_COLS = [
    'performance_score',
    'mean_rewards_stats_mean',
    'max_total_rewards_stats_mean',
    'epi_stats_mean'
]

class Grapher(object):

    '''
    Grapher object that belongs to a Session
    to draw graphs from its data
    '''

    def __init__(self, session):
        if not args.plot_graph or environ.get('CI'):
            return
        import matplotlib.pyplot as plt
        plt.rcParams['toolbar'] = 'None'  # mute matplotlib toolbar
        self.plt = plt
        self.session = session
        self.graph_filename = self.session.graph_filename
        self.subgraphs = {}
        self.figure = self.plt.figure(facecolor='white', figsize=(8, 9))
        self.figure.suptitle(wrap_text(self.session.session_id))
        self.init_figure()

    def init_figure(self):
        # graph 1
        ax1 = self.figure.add_subplot(
            311,
            frame_on=False,
            title="\n\ntotal rewards per episode",
            ylabel='total rewards')
        p1, = ax1.plot([], [])
        self.subgraphs['total rewards'] = (ax1, p1)

        ax1e = ax1.twinx()
        ax1e.set_ylabel('exploration rate').set_color('r')
        ax1e.set_frame_on(False)
        ax1e.grid(False)
        p1e, = ax1e.plot([], [], 'r')
        self.subgraphs['e'] = (ax1e, p1e)

        # graph 2
        ax2 = self.figure.add_subplot(
            312,
            frame_on=False,
            title='mean rewards over last 100 episodes',
            ylabel='mean rewards')
        p2, = ax2.plot([], [], 'g')
        self.subgraphs['mean rewards'] = (ax2, p2)

        # graph 3
        ax3 = self.figure.add_subplot(
            313,
            frame_on=False,
            title='loss over time, episode',
            ylabel='loss')
        p3, = ax3.plot([], [])
        self.subgraphs['loss'] = (ax3, p3)

        self.plt.tight_layout()  # auto-fix spacing
        self.plt.ion()  # for live plot

    def plot(self):
        '''do live plotting'''
        if not args.plot_graph or environ.get('CI'):
            return
        sys_vars = self.session.sys_vars
        ax1, p1 = self.subgraphs['total rewards']
        p1.set_ydata(
            sys_vars['total_rewards_history'])
        p1.set_xdata(np.arange(len(p1.get_ydata())))
        ax1.relim()
        ax1.autoscale_view(tight=True, scalex=True, scaley=True)

        ax1e, p1e = self.subgraphs['e']
        p1e.set_ydata(
            sys_vars['explore_history'])
        p1e.set_xdata(np.arange(len(p1e.get_ydata())))
        ax1e.relim()
        ax1e.autoscale_view(tight=True, scalex=True, scaley=True)

        ax2, p2 = self.subgraphs['mean rewards']
        p2.set_ydata(
            sys_vars['mean_rewards_history'])
        p2.set_xdata(np.arange(len(p2.get_ydata())))
        ax2.relim()
        ax2.autoscale_view(tight=True, scalex=True, scaley=True)

        ax3, p3 = self.subgraphs['loss']
        p3.set_ydata(sys_vars['loss'])
        p3.set_xdata(np.arange(len(p3.get_ydata())))
        ax3.relim()
        ax3.autoscale_view(tight=True, scalex=True, scaley=True)

        self.plt.draw()
        self.plt.pause(0.01)
        self.save()

    def save(self):
        '''save graph to filename'''
        self.figure.savefig(self.graph_filename)

    def clear(self):
        if not args.plot_graph or environ.get('CI'):
            return
        self.plt.close()
        del_self_attr(self)

def basic_stats(array):
    '''generate the basic stats for a numerical array'''
    if not len(array):
        return None
    return {
        'min': np.min(array).astype(float),
        'max': np.max(array).astype(float),
        'mean': np.mean(array).astype(float),
        'std': np.std(array).astype(float),
    }

def compose_data(trial):
    '''
    compose raw data from an trial object
    into useful summary and full metrics for analysis
    '''
    sys_vars_array = trial.data['sys_vars_array']

    # collect all data from sys_vars_array
    solved_sys_vars_array = list(filter(
        lambda sv: sv['solved'], sys_vars_array))
    mean_rewards_array = np.array(list(map(
        lambda sv: sv['mean_rewards'], sys_vars_array)))
    max_total_rewards_array = np.array(list(map(
        lambda sv: np.max(sv['total_rewards_history']), sys_vars_array)))
    epi_array = np.array(list(map(lambda sv: sv['epi'], sys_vars_array)))
    mean_rewards_per_epi_array = np.divide(mean_rewards_array, epi_array)
    t_array = np.array(list(map(lambda sv: sv['t'], sys_vars_array)))
    time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        sys_vars_array)))
    solved_epi_array = np.array(list(map(
        lambda sv: sv['epi'], solved_sys_vars_array)))
    solved_t_array = np.array(list(map(
        lambda sv: sv['t'], solved_sys_vars_array)))
    solved_time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        solved_sys_vars_array)))

    # compose sys_vars stats
    stats = {
        'num_of_sessions': len(sys_vars_array),
        'solved_num_of_sessions': len(solved_sys_vars_array),
        'solved_ratio_of_sessions': float(len(
            solved_sys_vars_array)) / trial.times,
        'mean_rewards_stats': basic_stats(mean_rewards_array),
        'mean_rewards_per_epi_stats': basic_stats(
            mean_rewards_per_epi_array),
        'max_total_rewards_stats': basic_stats(max_total_rewards_array),
        'epi_stats': basic_stats(epi_array),
        't_stats': basic_stats(t_array),
        'time_taken_stats': basic_stats(time_taken_array),
        'solved_epi_stats': basic_stats(solved_epi_array),
        'solved_t_stats': basic_stats(solved_t_array),
        'solved_time_taken_stats': basic_stats(solved_time_taken_array),
    }
    stats.update({
        'performance_score': stats[
            'mean_rewards_per_epi_stats']['mean'] * (stats[
                'solved_ratio_of_sessions'] ** 2)
    })

    # summary metrics
    metrics = {
        'performance_score': stats['performance_score'],
        'mean_rewards_per_epi_stats_mean': stats[
            'mean_rewards_per_epi_stats']['mean'],
        'mean_rewards_stats_mean': stats['mean_rewards_stats']['mean'],
        'epi_stats_mean': stats['epi_stats']['mean'],
        'solved_ratio_of_sessions': stats['solved_ratio_of_sessions'],
        'max_total_rewards_stats_mean': stats[
            'max_total_rewards_stats']['mean'],
        't_stats_mean': stats['t_stats']['mean'],
    }

    # param variables for independent vars of trials
    param_variables = {
        pv: trial.experiment_spec['param'][pv] for
        pv in trial.param_variables}
    param_variables = flat_cast_dict(param_variables)

    trial.data['metrics'].update(metrics)
    trial.data['param_variables'] = param_variables
    trial.data['stats'] = stats
    return trial.data

# plot the experiment data from data_df
# X are columns with name starting with 'variable_'
# Y cols are defined below
def plot_experiment(data_df, trial_id):
    if len(data_df) < 2:  # no multi selection
        return
    experiment_id = parse_experiment_id(trial_id)
    X_cols = list(filter(lambda c: c.startswith('variable_'), data_df.columns))
    for x in X_cols:
        for y in EXPERIMENT_GRID_Y_COLS:
            df_plot = sns.swarmplot(data=data_df, x=x, y=y,
                                    hue='solved_ratio_of_sessions')
            fig = df_plot.get_figure()
            fig.suptitle(wrap_text(experiment_id))
            filename = './data/{}/{}_analysis_{}_vs_{}.png'.format(
                experiment_id, experiment_id, x, y)
            fig.savefig(filename)
            fig.clear()

    fig = sns.PairGrid(
        data_df, x_vars=X_cols, y_vars=EXPERIMENT_GRID_Y_COLS,
        hue='solved_ratio_of_sessions', size=3)
    fig.map(partial(sns.swarmplot, size=3))
    fig.fig.suptitle(wrap_text(experiment_id))
    fig.add_legend()
    filename = './data/{0}/{0}_analysis.png'.format(
        experiment_id)
    fig.savefig(filename)
    fig.fig.clear()
    sns.plt.close()

def analyze_data(experiment_data_or_experiment_id):
    '''
    get all the data from all trials.run()
    or read from all data files matching the prefix of trial_id
    e.g. usage without running:
    experiment_id = 'DevCartPole-v0_DQN_LinearMemoryWithForgetting_BoltzmannPolicy_2017-01-15_142810'
    analyze_data(experiment_id)
    '''
    if isinstance(experiment_data_or_experiment_id, str):
        experiment_data = load_data_array_from_experiment_id(
            experiment_data_or_experiment_id)
    else:
        experiment_data = experiment_data_or_experiment_id

    stats_array, param_variables_array = [], []
    for data in experiment_data:
        stats = flatten_dict(data['stats'])
        stats.update({'trial_id': data['trial_id']})
        stats_array.append(stats)
        param_variables = data['param_variables']
        param_variables_array.append(param_variables)

    raw_stats_df = pd.DataFrame.from_dict(stats_array)
    stats_df = raw_stats_df[STATS_COLS]

    param_variables_df = pd.DataFrame.from_dict(param_variables_array)
    param_variables_df.columns = [
        'variable_'+c for c in param_variables_df.columns]

    data_df = pd.concat([stats_df, param_variables_df], axis=1)
    for c in data_df.columns:
        if data_df[c].dtype == object:  # guard
            data_df[c] = data_df[c].astype('category')

    data_df.sort_values(
        ['performance_score'],
        inplace=True, ascending=False)

    trial_id = experiment_data[0]['trial_id']
    save_experiment_data(data_df, trial_id)
    plot_experiment(data_df, trial_id)
    return data_df
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# See LICENSE.txt for licensing terms

import logging
import sys

logging.basicConfig(
    format='[%(levelname)s] %(filename)s:%(lineno)d %(message)s',
    level=logging.WARNING)

log = logging.getLogger('rst2pdf')

def nodeid(node):
    """Given a node, tries to return a way to see where it was in the
    source text"""
    fname='UNKNOWN'
    line='UNKNOWN'
    try:
        if node.line: line=str(node.line)
    except:
        pass
    try:
        if node.source: fname=str(node.source)
    except:
        pass
    return 'near line %s in file %s'%(line,fname)<commit_msg>do not use basicConfig for logging<commit_after># -*- coding: utf-8 -*-
# See LICENSE.txt for licensing terms

import logging
import sys

log = logging.getLogger('rst2pdf')
_fmt = logging.Formatter('[%(levelname)s] %(filename)s:%(lineno)d %(message)s')
_hdlr = logging.StreamHandler()
_hdlr.setFormatter(_fmt)
log.addHandler(_hdlr)
log.setLevel(logging.WARNING)

def nodeid(node):
    """Given a node, tries to return a way to see where it was in the
    source text"""
    fname='UNKNOWN'
    line='UNKNOWN'
    try:
        if node.line: line=str(node.line)
    except:
        pass
    try:
        if node.source: fname=str(node.source)
    except:
        pass
    return 'near line %s in file %s'%(line,fname)
<|endoftext|>


<commit_before>"""

Cluster correlations from correlations.py based on hierarchical linkage.

correlations.py outputs a tuple of

[object 1, object 2, pearson correlation, p value]

Here, we use 1-pearson correlation as our distance

"""
import json
import os
import sys
import math
import argparse

import numpy as np
import scipy.cluster.hierarchy as sch

def parse_text_file(tf, pearsoncol=2, insep="\t"):
    """
    Parse a text file and return an n-choose-2 array of the elements. The array returned has the distance from the first
    element to all other elements, and then the second element to n-1 elements (all but the first), and then the
    third element to n-2 elements (all but the first & second) and so on.
    :param tf: Text file with [a, b, pearson correlation, p-value] (e.g. the output from correlations.py)
    :type tf: str
    :param pearsoncol: the zero indexed column of the pearson correlation score
    :type numcols: int
    :param insep: Input separator. Default = tab
    :type insep: str
    :return: n-choose-2 array of the data and the list of the keys in order
    :rtype: np.array, list
    """

    data = {}
    ks = set()
    with open(tf, 'r') as fin:
        for l in fin:
            p=l.strip().split(insep)
            if len(p) <= pearsoncol:
                sys.stderr.write(f"ERROR: {l} does not have enough entries for {pearsoncol} to be the score. Skipped\n")
                continue
            ks.add(p[0])
            ks.add(p[1])
            if p[0] not in data:
                data[p[0]]={}
            if p[1] not in data:
                data[p[1]] = {}
            val = float(p[pearsoncol])
            if val > 1:
                val = 1
            data[p[0]][p[1]] = val
            data[p[1]][p[0]] = val

    allkeys = list(ks)
    allkeys.sort()
    # our new array will have (len(allkeys)) choose 2 elements
    nal = math.comb(len(allkeys), 2)
    nct = np.empty(nal)
    p = 0
    for i in range(len(allkeys)):
        for j in range(i + 1, len(allkeys)):
            if allkeys[j] not in  data[allkeys[i]]:
                data[allkeys[i]][allkeys[j]] = 0
            nct[p] = 1 - data[allkeys[i]][allkeys[j]]
            p += 1
    return nct, allkeys

def generate_clusters(matrix, idlist, jsonout, noclust=100, print_singles=False):

    L = sch.linkage(matrix, method='average')

    outputdata = []
    for ele in range(noclust + 1):
        threshold = ele / noclust
        ind = sch.fcluster(L, threshold, 'distance')
        uniqs, counts = np.unique(ind, return_counts=True)
        freqs = {}
        for idx, u in enumerate(uniqs):
            freqs[u] = counts[idx]

        clusters = {}
        for idx, j in enumerate(ind):
            if freqs[j] == 1 and not print_singles:
                continue
            if j not in clusters:
                clusters[j] = []
            clusters[j].append(idlist[idx])

        # res = [i idlist[x] for x in ind]
        # res = [x for x in ind]
        # out.write(f"{{cluster_id : {i}, largest_cluster : {max(counts)}, num_clusters: {uniqs.shape[0]}, clusters: {res}}},\n")
        singles = np.isin(counts, [1]).sum()
        outputdata.append({
            "cluster_id": ele,
            "threshold": threshold,
            "largest_cluster": max(counts),
            "num_clusters": uniqs.shape[0],
            "num_singleton_clusters": singles,
            "clusters": clusters
        })
        print(f"Threshold: {threshold}\tNumber of clusters: {uniqs.shape[0]}\tLargest cluster{max(counts)}. See the output file ({jsonout}) for more details")

    with open(jsonout, 'w') as out:
        json.dump(outputdata, out)

def generate_a_cluster(matrix, idlist, jsonout, threshold=0.05, print_singles=False):
    """
    Generate a single cluster given a threshold
    """

    L = sch.linkage(matrix, method='average')

    ind = sch.fcluster(L, threshold, 'distance')
    uniqs, counts = np.unique(ind, return_counts=True)
    freqs = {}
    for idx, u in enumerate(uniqs):
        freqs[u] = counts[idx]

    clusters = {}
    for idx, j in enumerate(ind):
        if freqs[j] == 1 and not print_singles:
            continue
        if j not in clusters:
            clusters[j] = []
        clusters[j].append(idlist[idx])

    singles = np.isin(counts, [1]).sum()
    outputdata = {
        "threshold": threshold,
        "largest_cluster": max(counts),
        "num_clusters": uniqs.shape[0],
        "num_singleton_clusters": singles,
        "clusters": clusters
    }
    print(f"{threshold}\t{uniqs.shape[0]}\t{max(counts)}")

    with open(jsonout, 'w') as out:
        json.dump(outputdata, out)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Cluster genes based on Pearson correlation")
    parser.add_argument('-f', '--file', help='file with [a, b, distance]', required=True)
    parser.add_argument('-o', '--output', help='clusters output file name. We print them out in json format', required=True)
    parser.add_argument('-p', '--pearsoncol', help='0 indexed column in input file with the pearson score. Default = 2', type=int, default=2)
    parser.add_argument('-s', '--separator', help='Input separator. Default = tab', default="\t", type=str)
    parser.add_argument('--singles', help='print clusters with one element in them', action='store_true')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-t', '--threshold', help='clustering threshold to print a single cluster', type=float)
    group.add_argument('-n', '--numclust', help='number of clusters to print with a range of thresholds (default=100)', type=int, default=100)

    args = parser.parse_args()

    matrix, idlist = parse_text_file(args.file, args.pearsoncol, args.separator)
    if args.threshold:
        generate_a_cluster(matrix, idlist, args.output, args.threshold, args.singles)
    elif args.numclust:
        generate_clusters(matrix, idlist, args.output, args.numclust, args.singles)
<commit_msg>Converting np numbers to ints<commit_after>"""

Cluster correlations from correlations.py based on hierarchical linkage.

correlations.py outputs a tuple of

[object 1, object 2, pearson correlation, p value]

Here, we use 1-pearson correlation as our distance

"""
import json
import os
import sys
import math
import argparse

import numpy as np
import scipy.cluster.hierarchy as sch

def parse_text_file(tf, pearsoncol=2, insep="\t"):
    """
    Parse a text file and return an n-choose-2 array of the elements. The array returned has the distance from the first
    element to all other elements, and then the second element to n-1 elements (all but the first), and then the
    third element to n-2 elements (all but the first & second) and so on.
    :param tf: Text file with [a, b, pearson correlation, p-value] (e.g. the output from correlations.py)
    :type tf: str
    :param pearsoncol: the zero indexed column of the pearson correlation score
    :type numcols: int
    :param insep: Input separator. Default = tab
    :type insep: str
    :return: n-choose-2 array of the data and the list of the keys in order
    :rtype: np.array, list
    """

    data = {}
    ks = set()
    with open(tf, 'r') as fin:
        for l in fin:
            p=l.strip().split(insep)
            if len(p) <= pearsoncol:
                sys.stderr.write(f"ERROR: {l} does not have enough entries for {pearsoncol} to be the score. Skipped\n")
                continue
            ks.add(p[0])
            ks.add(p[1])
            if p[0] not in data:
                data[p[0]]={}
            if p[1] not in data:
                data[p[1]] = {}
            val = float(p[pearsoncol])
            if val > 1:
                val = 1
            data[p[0]][p[1]] = val
            data[p[1]][p[0]] = val

    allkeys = list(ks)
    allkeys.sort()
    # our new array will have (len(allkeys)) choose 2 elements
    nal = math.comb(len(allkeys), 2)
    nct = np.empty(nal)
    p = 0
    for i in range(len(allkeys)):
        for j in range(i + 1, len(allkeys)):
            if allkeys[j] not in  data[allkeys[i]]:
                data[allkeys[i]][allkeys[j]] = 0
            nct[p] = 1 - data[allkeys[i]][allkeys[j]]
            p += 1
    return nct, allkeys

def generate_clusters(matrix, idlist, jsonout, noclust=100, print_singles=False):

    L = sch.linkage(matrix, method='average')

    outputdata = []
    for ele in range(noclust + 1):
        threshold = ele / noclust
        ind = sch.fcluster(L, threshold, 'distance')
        uniqs, counts = np.unique(ind, return_counts=True)
        freqs = {}
        for idx, u in enumerate(uniqs):
            freqs[u] = counts[idx]

        clusters = {}
        for idx, j in enumerate(ind):
            if freqs[j] == 1 and not print_singles:
                continue
            if j not in clusters:
                clusters[j] = []
            clusters[j].append(idlist[idx])

        # res = [i idlist[x] for x in ind]
        # res = [x for x in ind]
        # out.write(f"{{cluster_id : {i}, largest_cluster : {max(counts)}, num_clusters: {uniqs.shape[0]}, clusters: {res}}},\n")
        singles = np.isin(counts, [1]).sum()
        outputdata.append({
            "cluster_id": ele,
            "threshold": threshold,
            "largest_cluster": max(counts),
            "num_clusters": uniqs.shape[0],
            "num_singleton_clusters": singles,
            "clusters": clusters
        })
        print(f"Threshold: {threshold}\tNumber of clusters: {uniqs.shape[0]}\tLargest cluster{max(counts)}. See the output file ({jsonout}) for more details")

    with open(jsonout, 'w') as out:
        json.dump(outputdata, out)

def generate_a_cluster(matrix, idlist, jsonout, threshold=0.05, print_singles=False):
    """
    Generate a single cluster given a threshold
    """

    L = sch.linkage(matrix, method='average')

    ind = sch.fcluster(L, threshold, 'distance')
    uniqs, counts = np.unique(ind, return_counts=True)
    freqs = {}
    for idx, u in enumerate(uniqs):
        freqs[u] = counts[idx]

    clusters = {}
    for idx, j in enumerate(ind):
        if freqs[j] == 1 and not print_singles:
            continue
        if j not in clusters:
            clusters[j] = []
        clusters[j].append(idlist[idx])

    singles = int(np.isin(counts, [1]).sum())
    outputdata = {
        "threshold": threshold,
        "largest_cluster": max(counts),
        "num_clusters": int(uniqs.shape[0]),
        "num_singleton_clusters": singles,
        "clusters": clusters
    }
    print(f"Threshold: {threshold}\tNumber of clusters: {uniqs.shape[0]}\tLargest cluster: {max(counts)}")

    with open(jsonout, 'w') as out:
        json.dump(outputdata, out)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Cluster genes based on Pearson correlation")
    parser.add_argument('-f', '--file', help='file with [a, b, distance]', required=True)
    parser.add_argument('-o', '--output', help='clusters output file name. We print them out in json format', required=True)
    parser.add_argument('-p', '--pearsoncol', help='0 indexed column in input file with the pearson score. Default = 2', type=int, default=2)
    parser.add_argument('-s', '--separator', help='Input separator. Default = tab', default="\t", type=str)
    parser.add_argument('--singles', help='print clusters with one element in them', action='store_true')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-t', '--threshold', help='clustering threshold to print a single cluster', type=float)
    group.add_argument('-n', '--numclust', help='number of clusters to print with a range of thresholds (default=100)', type=int, default=100)

    args = parser.parse_args()

    matrix, idlist = parse_text_file(args.file, args.pearsoncol, args.separator)
    if args.threshold:
        generate_a_cluster(matrix, idlist, args.output, args.threshold, args.singles)
    elif args.numclust:
        generate_clusters(matrix, idlist, args.output, args.numclust, args.singles)
<|endoftext|>


<commit_before>import os
import sys
from setuptools import setup, find_packages

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.md')).read()
CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()

requires = [
    'ott.utils',
    'ott.geocoder',
    'ott.data[postgresql]',

    'gtfsdb',
    'simplejson',
    'python-dateutil',

    'pyramid < 1.8',
    'pyramid_tm',
    'pyramid_exclog',
    'waitress',
]

extras_require = dict(
    dev=[],
)

#
# eggs that you need if you're running a version of python lower than 2.7
#
if sys.version_info[:2] < (2, 7):
    requires.extend(['argparse>=1.2.1', 'unittest2>=0.5.1'])

setup(
    name='ott.otp_client',
    version='0.1.0',
    description='Open Transit Tools - OTT Database',
    long_description=README + '\n\n' + CHANGES,
    classifiers=[
        "Programming Language :: Python",
        "Topic :: Internet :: WWW/HTTP",
        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
    ],
    author="Open Transit Tools",
    author_email="info@opentransittools.org",
    dependency_links=[
        'git+https://github.com/OpenTransitTools/utils.git#egg=ott.utils-0.1.0',
        'git+https://github.com/OpenTransitTools/data.git#egg=ott.data-0.1.0',
        'git+https://github.com/OpenTransitTools/geocoder.git#egg=ott.geocoder-0.1.0'
    ],
    license="Mozilla-derived (http://opentransittools.com)",
    url='http://opentransittools.com',
    keywords='ott, otp, gtfs, gtfsdb, data, database, services, transit',
    packages=find_packages(),
    include_package_data=True,
    zip_safe=False,
    install_requires=requires,
    extras_require=extras_require,
    tests_require=requires,
    test_suite="ott.otp_client.tests",
    entry_points="""\
        [paste.app_factory]
        main = ott.otp_client.pyramid.app:main

        [console_scripts]
        parse_otp_json = ott.otp_client.otp_to_ott:main
        trip_planner = ott.otp_client.trip_planner:main
        ti = ott.otp_client.transit_index.base:main
    """,
)
<commit_msg>remove gtfsdb ... it pulls in very old pypi version ... need ot update<commit_after>import os
import sys
from setuptools import setup, find_packages

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.md')).read()
CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()

requires = [
    'ott.utils',
    'ott.geocoder',
    'ott.data[postgresql]',

    'simplejson',
    'python-dateutil',

    'pyramid < 1.8',
    'pyramid_tm',
    'pyramid_exclog',
    'waitress',
]

extras_require = dict(
    dev=[],
)

#
# eggs that you need if you're running a version of python lower than 2.7
#
if sys.version_info[:2] < (2, 7):
    requires.extend(['argparse>=1.2.1', 'unittest2>=0.5.1'])

setup(
    name='ott.otp_client',
    version='0.1.0',
    description='Open Transit Tools - OTT Database',
    long_description=README + '\n\n' + CHANGES,
    classifiers=[
        "Programming Language :: Python",
        "Topic :: Internet :: WWW/HTTP",
        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
    ],
    author="Open Transit Tools",
    author_email="info@opentransittools.org",
    dependency_links=[
        'git+https://github.com/OpenTransitTools/utils.git#egg=ott.utils-0.1.0',
        'git+https://github.com/OpenTransitTools/data.git#egg=ott.data-0.1.0',
        'git+https://github.com/OpenTransitTools/geocoder.git#egg=ott.geocoder-0.1.0'
    ],
    license="Mozilla-derived (http://opentransittools.com)",
    url='http://opentransittools.com',
    keywords='ott, otp, gtfs, gtfsdb, data, database, services, transit',
    packages=find_packages(),
    include_package_data=True,
    zip_safe=False,
    install_requires=requires,
    extras_require=extras_require,
    tests_require=requires,
    test_suite="ott.otp_client.tests",
    entry_points="""\
        [paste.app_factory]
        main = ott.otp_client.pyramid.app:main

        [console_scripts]
        parse_otp_json = ott.otp_client.otp_to_ott:main
        trip_planner = ott.otp_client.trip_planner:main
        ti = ott.otp_client.transit_index.base:main
    """,
)
<|endoftext|>


<commit_before>'''
Defines all the possible activity types.

##BOILERPLATE_COPYRIGHT
##BOILERPLATE_COPYRIGHT_END
'''

__activities = [
    'new_post',
    'edit_post',
    'top_level_post',
    'new_user',
    'edit_user',
    'edit_self'
]

activity_table = dict((v,k) for k,v in enumerate(__activities, start=1))
<commit_msg>alphanumeric in case any squirley DB doesn't like non-alpha in strings<commit_after>'''
Defines all the possible activity types.

##BOILERPLATE_COPYRIGHT
##BOILERPLATE_COPYRIGHT_END
'''

__activities = [
    'new_post',
    'edit_post',
    'top_level_post',
    'new_user',
    'edit_user',
    'edit_self'
]

activity_table = dict((v,k) for k,v in \
    enumerate(__activities, start=ord('a')))
<|endoftext|>


<commit_before>#!/usr/bin/python

import argparse
import re
import time
from datetime import datetime

import requests
import urllib3
import yaml
from diskcache import Cache

urllib3.disable_warnings()

try:
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urljoin

OOO_PROJECTS = [
    'openstack/puppet-tripleo', 'openstack/python-tripleoclient',
    'openstack/tripleo-upgrade', 'openstack/tripleo-quickstart-extras',
    'openstack/tripleo-common', 'openstack/tripleo-ci',
    'openstack/tripleo-quickstart', 'openstack/tripleo-heat-templates',
    'openstack/tripleo-ansible', 'openstack/tripleo-validations',
    'rdo-infra/ansible-role-tripleo-ci-reproducer',
    'containers/libpod', 'ceph/ceph-ansible',
    'openstack/ansible-collections-openstack', 'openstack/heat',
    'rdoinfo', 'nfvinfo'
]

INTERNAL_OOO_PROJECTS = [
    'openstack/tripleo-ci-internal-jobs',
    'rhos-release',
    'openstack-tempest',
    'openstack-tripleo-common',
    'openstack-tripleo-heat-templates-compat',
    'openstack-tripleo-heat-templates',
    'openstack-tripleo-image-elements',
    'openstack-tripleo-puppet-elements',
    'openstack-tripleo-ui',
    'openstack-tripleo-validations',
    'openstack-tripleo',
    'openstack/tripleo-ci',
    'rhos-ops/openstack-tripleo-heat-templates'
]

TIMESTAMP_PATTERN = '%Y-%m-%dT%H:%M:%S'
TIMESTAMP_PATTERN2 = '%Y-%m-%d %H:%M:%S'

JOBS_FOR_ARA = []
#    'tripleo-ci-centos-7-standalone',
#    'tripleo-ci-centos-7-containers-multinode',
#    'tripleo-ci-centos-7-undercloud-containers',
#    'tripleo-ci-centos-7-scenario001-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario002-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario003-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario004-multinode-oooq-container',
# ]

ARA_JSONS = [
    '/logs/ara.oooq.root.json',
    '/logs/ara.oooq.oc.json',
    '/logs/ara.json'
]

TASK_DURATION_TRESHOLD = 10
CACHE_EXPIRE_SECS = 604800  # a week
CACHE_SIZE = int(2e9)  # 2GB

cache = Cache('/tmp/ruck_rover_cache', size_limit=CACHE_SIZE)
cache.expire()

# Convert datetime to timestamp

def to_ts(d, seconds=False, pattern=TIMESTAMP_PATTERN):
    return datetime.strptime(
        d, pattern).strftime('%s') + ('' if seconds else "000000000")

def to_seconds(duration):
    x = time.strptime(duration, '%H:%M:%S')
    return datetime.timedelta(
        hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()

def get(url, json_view, query=None, timeout=20):
    query = query or {}
    try:
        response = requests.get(url,
                                params=query,
                                timeout=timeout,
                                verify=False)
        if response and response.ok:
            if json_view:
                return response.json()
            else:
                response.encoding = 'utf-8'
                return response.text
    except Exception:
        return None

    return None

def get_builds_info(url, query, pages, offset):
    builds = []
    for p in range(pages):
        if p > 0:
            query['skip'] = offset + (p * 50)
            # let's not abuse ZUUL API and sleep betwen requests
            time.sleep(2)
        else:
            query['skip'] = offset
        builds_api = url + "builds"
        response = get(builds_api, True, query)
        if response is not None:
            builds += response
    return builds

def get_file_from_build(build, file_relative_path, json_view):
    if 'log_url' in build and build['log_url']:
        if build['log_url'].endswith("/html/"):
            build['log_url'] = build['log_url'].replace('html/', '')
        if build['log_url'].endswith("/cover/"):
            build['log_url'] = build['log_url'].replace('cover/', '')
        file_path = urljoin(build['log_url'], file_relative_path)

        if file_path not in cache:
            resp = get(file_path, json_view)

            if resp is not None:
                if json_view:
                    cache.add(
                        file_path, yaml.safe_load(resp),
                        expire=CACHE_EXPIRE_SECS)
                else:
                    cache.add(file_path, resp)
            # Add negative cache
            else:
                cache[file_path] = None

        return cache[file_path]

def add_inventory_info(build, json_view=False):
    try:
        inventory = get_file_from_build(build,
                                        "zuul-info/inventory.yaml",
                                        json_view)
        inventory = yaml.safe_load(inventory)
        hosts = inventory['all']['hosts']
        host = hosts[list(hosts.keys())[0]]
        if 'nodepool' in host:
            nodepool = host['nodepool']
            build['cloud'] = nodepool['cloud']
            build['region'] = nodepool['region']
            build['provider'] = nodepool['provider']

    except Exception:
        pass

def add_sova_info(build, json_view=False):
    try:
        failures_file = get_file_from_build(build,
                                            "logs/failures_file",
                                            json_view)
        lines = failures_file.split('\n')
        reason = lines[0]
        tag = lines[1].split("Reason: ")[1]
        # Cut long reasons and remove quotes
        build['sova_reason'] = reason[:200].replace('"', '')
        build['sova_tag'] = tag.replace('"', '')
    except Exception:
        pass

# rdoinfo and nfvinfo job meta re which package is updated
def add_rdopkg_change(build, json_view=False):
    rdo_file = "logs/tested_pkgs_updates.txt.gz"
    rdo_project = ["rdoinfo", "nfvinfo"]
    rpm_change_file = ""

    if build['type'] == "rdo" and build['project'] in rdo_project:
        file = rdo_file
        try:
            rpm_change_file = get_file_from_build(build,
                                                  file,
                                                  json_view)
            rpm_change_file = rpm_change_file.splitlines()
        except Exception:
            pass
        if rpm_change_file:
            build['dep_change'] = " ".join(rpm_change_file)

def add_container_prep_time(build):
    if build['log_url'] is not None:
        job_terms = ['featureset', 'oooq', 'multinode']
        skip_terms = ['update', 'upgrade', 'rocky']
        skip_branch = ['stable/rocky', 'stable/queens']

        if any(x in build['job_name'] for x in skip_terms):
            return
        elif any(x in build['branch'] for x in skip_branch):
            return
        elif any(x in build['job_name'] for x in job_terms):
            file_relative_path = ("logs/undercloud/home/zuul/"
                                  "install-undercloud.log.txt.gz")
            resp_data = get_file_from_build(
                build, file_relative_path,
                json_view=False)
        elif 'standalone' in build['job_name']:
            file_relative_path = ("logs/undercloud/home/zuul/"
                                  "standalone_deploy.log.txt.gz")

            resp_data = get_file_from_build(
                build, file_relative_path,
                json_view=False)
        else:
            return

        if resp_data is None:
            return

        # This is the old regular expression, commented here because it's hard
        # to remember
        # container_prep_line = (r'(\d{4}-\d{2}-\d{2}\s(?:[01]\d|2[0-3]):'
        #                        r'(?:[0-5]\d):(?:[0-5]\d))\s\|.*(tripleo-'
        #                        r'container-image-prepare.log).*\n^.*(\d{4}-'
        #                        r'\d{2}-\d{2}\s(?:[01]\d|2[0-3]):(?:[0-5]\d):'
        #                        r'(?:[0-5]\d))\s\|\schanged')
        # For example:
        # 2019-09-23 09:57:04 | TASK [tripleo-container-image-prepare :
        # Run tripleo-container-image-prepare logged to: /var/log/tripleo-
        # container-image-prepare.log] ***\n
        # 2019-09-23 10:15:25 | changed: [undercloud]

        # This is the previous regular expression.
        # Example:
        # 2019-09-30 08:51:23 | tripleo-container-image-prepare :
        # Run tripleo-container-image-prepare logged to:
        # /var/log/tripleo-container-image-prepare.log  4335.64s

        # container_prep_line = (r'\d{4}-\d{2}-\d{2}\s(?:[01]\d|2[0-3])'
        #                        r':(?:[0-5]'
        #                        r'\d):(?:[0-5]\d)\s\|.*tripleo-container-image'
        #                        r'-prepare.log\s*([1-9]*\.?[1-9]*)s')

        # The regex below matches the new Warning log and the previous one
        # where the regex commented above also match

        container_prep_line = (r'Run.*tripleo-container-image-prepare.log'
                               r'\s*\-?\s*([1-9]*\.?[1-9]*)s')

        match = re.findall(container_prep_line, resp_data, re.MULTILINE)

        if len(match) > 0:
            build['container_prep_time_u'] = float(match[0])

def fix_task_name(task_name):
    return re.sub(r'/tmp/tripleo-modify-image.*', '/tmp/tripleo-modify-image',
                  task_name).replace(',', '_')

def print_influx_ara_tasks(build, ara_json_file):
    if build['job_name'] not in JOBS_FOR_ARA:
        return
    try:
        tasks = get_file_from_build(build, ara_json_file, json_view=True)
        if tasks is None:
            return
        for task in tasks:
            duration = to_seconds(task['Duration'])
            if duration > TASK_DURATION_TRESHOLD:
                print("build-task,task_name={},logs_path={},json_path={}"
                      " duration={},job_result=\"{}\",job_branch=\"{}\" {}".
                      format(
                          fix_task_name(task['Name'].replace(' ', '\\ ')),
                          build['log_url'], ara_json_file, duration,
                          build['result'], build['branch'],
                          to_ts(
                              task['Time Start'],
                              seconds=False,
                              pattern=TIMESTAMP_PATTERN2)))

    except Exception:
        pass

def influx(build):

    add_inventory_info(build)
    # add_container_prep_time(build)
    add_sova_info(build)
    add_rdopkg_change(build)

    if build['end_time'] is None:
        build['end_time'] = datetime.datetime.fromtimestamp(
            time.time()).strftime(TIMESTAMP_PATTERN)

    if build['start_time'] is None:
        build['start_time'] = build['end_time']
    duration = build.get('duration', 0)
    if duration is None:
        duration = 0
    # Get the nodename
    return ('build,'
            'type=%s,'
            'pipeline=%s,'
            'branch=%s,'
            'project=%s,'
            'job_name=%s,'
            'voting=%s,'
            'change=%s,'
            'patchset=%s,'
            'passed=%s,'
            'cloud=%s,'
            'region=%s,'
            'provider=%s,'
            'result="%s"'
            ' '
            'result="%s",'
            'result_num=%s,'
            'log_url="%s",'
            'log_link="%s",'
            'duration=%s,'
            'start=%s,'
            'end=%s,'
            'cloud="%s",'
            'region="%s",'
            'provider="%s",'
            'sova_reason="%s",'
            'sova_tag="%s",'
            'dep_change="%s",'
            'container_prep_time_u=0'
            ' '
            '%s' %
            (build['type'],
             build['pipeline'], 'none' if not build['branch']
             else build['branch'], build['project'], build['job_name'],
             build['voting'], build['change'], build['patchset'], 'True'
             if build['result'] == 'SUCCESS' else 'False',
             build.get('cloud', 'null'), build.get('region', 'null'),
             build.get('provider', 'null'), build['result'], build['result'], 1
             if build['result'] == 'SUCCESS' else 0, build['log_url'],
             "{}".format(
                 build['log_url'], build['job_name']),
             duration,
             to_ts(build['start_time'], seconds=True),
             to_ts(build['end_time'], seconds=True), build.get(
                 'cloud', 'null'), build.get('region', 'null'),
             build.get('provider', 'null'),
             build.get('sova_reason', ''),
             build.get('sova_tag', ''),
             build.get('dep_change', ''),
             to_ts(build['end_time'])))

def print_influx(build_type, builds):
    if builds:
        for build in builds:
            build['type'] = build_type
            for ara_json in ARA_JSONS:
                print_influx_ara_tasks(build, ara_json)
            print(influx(build))

def main():

    parser = argparse.ArgumentParser(
        description="Retrieve as influxdb zuul builds")

    parser.add_argument(
        '--url',
        default="http://zuul.openstack.org/api/",
        help="(default: %(default)s)")
    parser.add_argument(
        '--type', default="upstream", help="(default: %(default)s)")
    parser.add_argument(
        '--pages', type=int, default=1, help="(default: %(default)s)")
    parser.add_argument(
        '--offset', type=int, default=0, help="(default: %(default)s)")
    args = parser.parse_args()

    if args.type == 'internal':
        report_projects = INTERNAL_OOO_PROJECTS
    else:
        report_projects = OOO_PROJECTS

    for project in report_projects:
        print_influx(
            args.type,
            get_builds_info(
                url=args.url,
                query={'project': project},
                pages=args.pages,
                offset=args.offset))

if __name__ == '__main__':
    main()
<commit_msg>Fix error in zuul jobs build file<commit_after>#!/usr/bin/python

import argparse
import re
import time
from datetime import datetime

import requests
import urllib3
import yaml
from diskcache import Cache

urllib3.disable_warnings()

try:
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urljoin

OOO_PROJECTS = [
    'openstack/puppet-tripleo', 'openstack/python-tripleoclient',
    'openstack/tripleo-upgrade', 'openstack/tripleo-quickstart-extras',
    'openstack/tripleo-common', 'openstack/tripleo-ci',
    'openstack/tripleo-quickstart', 'openstack/tripleo-heat-templates',
    'openstack/tripleo-ansible', 'openstack/tripleo-validations',
    'rdo-infra/ansible-role-tripleo-ci-reproducer',
    'containers/libpod', 'ceph/ceph-ansible',
    'openstack/ansible-collections-openstack', 'openstack/heat',
    'rdoinfo', 'nfvinfo'
]

INTERNAL_OOO_PROJECTS = [
    'openstack/tripleo-ci-internal-jobs',
    'rhos-release',
    'openstack-tempest',
    'openstack-tripleo-common',
    'openstack-tripleo-heat-templates-compat',
    'openstack-tripleo-heat-templates',
    'openstack-tripleo-image-elements',
    'openstack-tripleo-puppet-elements',
    'openstack-tripleo-ui',
    'openstack-tripleo-validations',
    'openstack-tripleo',
    'openstack/tripleo-ci',
    'rhos-ops/openstack-tripleo-heat-templates'
]

TIMESTAMP_PATTERN = '%Y-%m-%dT%H:%M:%S'
TIMESTAMP_PATTERN2 = '%Y-%m-%d %H:%M:%S'

JOBS_FOR_ARA = []
#    'tripleo-ci-centos-7-standalone',
#    'tripleo-ci-centos-7-containers-multinode',
#    'tripleo-ci-centos-7-undercloud-containers',
#    'tripleo-ci-centos-7-scenario001-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario002-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario003-multinode-oooq-container',
#    'tripleo-ci-centos-7-scenario004-multinode-oooq-container',
# ]

ARA_JSONS = [
    '/logs/ara.oooq.root.json',
    '/logs/ara.oooq.oc.json',
    '/logs/ara.json'
]

TASK_DURATION_TRESHOLD = 10
CACHE_EXPIRE_SECS = 604800  # a week
CACHE_SIZE = int(2e9)  # 2GB

cache = Cache('/tmp/ruck_rover_cache', size_limit=CACHE_SIZE)
cache.expire()

# Convert datetime to timestamp

def to_ts(d, seconds=False, pattern=TIMESTAMP_PATTERN):
    return datetime.strptime(
        d, pattern).strftime('%s') + ('' if seconds else "000000000")

def to_seconds(duration):
    x = time.strptime(duration, '%H:%M:%S')
    return datetime.timedelta(
        hours=x.tm_hour, minutes=x.tm_min, seconds=x.tm_sec).total_seconds()

def get(url, json_view, query=None, timeout=20):
    query = query or {}
    try:
        response = requests.get(url,
                                params=query,
                                timeout=timeout,
                                verify=False)
        if response and response.ok:
            if json_view:
                return response.json()
            else:
                response.encoding = 'utf-8'
                return response.text
    except Exception:
        return None

    return None

def get_builds_info(url, query, pages, offset):
    builds = []
    for p in range(pages):
        if p > 0:
            query['skip'] = offset + (p * 50)
            # let's not abuse ZUUL API and sleep betwen requests
            time.sleep(2)
        else:
            query['skip'] = offset
        builds_api = url + "builds"
        response = get(builds_api, True, query)
        if response is not None:
            builds += response
    return builds

def get_file_from_build(build, file_relative_path, json_view):
    if 'log_url' in build and build['log_url']:
        if build['log_url'].endswith("/html/"):
            build['log_url'] = build['log_url'].replace('html/', '')
        if build['log_url'].endswith("/cover/"):
            build['log_url'] = build['log_url'].replace('cover/', '')
        file_path = urljoin(build['log_url'], file_relative_path)

        if file_path not in cache:
            resp = get(file_path, json_view)

            if resp is not None:
                if json_view:
                    cache.add(
                        file_path, yaml.safe_load(resp),
                        expire=CACHE_EXPIRE_SECS)
                else:
                    cache.add(file_path, resp)
            # Add negative cache
            else:
                cache[file_path] = None

        return cache[file_path]

def add_inventory_info(build, json_view=False):
    try:
        inventory = get_file_from_build(build,
                                        "zuul-info/inventory.yaml",
                                        json_view)
        inventory = yaml.safe_load(inventory)
        hosts = inventory['all']['hosts']
        host = hosts[list(hosts.keys())[0]]
        if 'nodepool' in host:
            nodepool = host['nodepool']
            build['cloud'] = nodepool['cloud']
            build['region'] = nodepool['region']
            build['provider'] = nodepool['provider']

    except Exception:
        pass

def add_sova_info(build, json_view=False):
    try:
        failures_file = get_file_from_build(build,
                                            "logs/failures_file",
                                            json_view)
        lines = failures_file.split('\n')
        reason = lines[0]
        tag = lines[1].split("Reason: ")[1]
        # Cut long reasons and remove quotes
        build['sova_reason'] = reason[:200].replace('"', '')
        build['sova_tag'] = tag.replace('"', '')
    except Exception:
        pass

# rdoinfo and nfvinfo job meta re which package is updated
def add_rdopkg_change(build, json_view=False):
    rdo_file = "logs/tested_pkgs_updates.txt.gz"
    rdo_project = ["rdoinfo", "nfvinfo"]
    rpm_change_file = ""

    if build['type'] == "rdo" and build['project'] in rdo_project:
        file = rdo_file
        try:
            rpm_change_file = get_file_from_build(build,
                                                  file,
                                                  json_view)
            rpm_change_file = rpm_change_file.splitlines()
        except Exception:
            pass
        if rpm_change_file:
            build['dep_change'] = " ".join(rpm_change_file)

def add_container_prep_time(build):
    if build['log_url'] is not None:
        job_terms = ['featureset', 'oooq', 'multinode']
        skip_terms = ['update', 'upgrade', 'rocky']
        skip_branch = ['stable/rocky', 'stable/queens']

        if any(x in build['job_name'] for x in skip_terms):
            return
        elif any(x in build['branch'] for x in skip_branch):
            return
        elif any(x in build['job_name'] for x in job_terms):
            file_relative_path = ("logs/undercloud/home/zuul/"
                                  "install-undercloud.log.txt.gz")
            resp_data = get_file_from_build(
                build, file_relative_path,
                json_view=False)
        elif 'standalone' in build['job_name']:
            file_relative_path = ("logs/undercloud/home/zuul/"
                                  "standalone_deploy.log.txt.gz")

            resp_data = get_file_from_build(
                build, file_relative_path,
                json_view=False)
        else:
            return

        if resp_data is None:
            return

        # This is the old regular expression, commented here because it's hard
        # to remember
        # container_prep_line = (r'(\d{4}-\d{2}-\d{2}\s(?:[01]\d|2[0-3]):'
        #                        r'(?:[0-5]\d):(?:[0-5]\d))\s\|.*(tripleo-'
        #                        r'container-image-prepare.log).*\n^.*(\d{4}-'
        #                        r'\d{2}-\d{2}\s(?:[01]\d|2[0-3]):(?:[0-5]\d):'
        #                        r'(?:[0-5]\d))\s\|\schanged')
        # For example:
        # 2019-09-23 09:57:04 | TASK [tripleo-container-image-prepare :
        # Run tripleo-container-image-prepare logged to: /var/log/tripleo-
        # container-image-prepare.log] ***\n
        # 2019-09-23 10:15:25 | changed: [undercloud]

        # This is the previous regular expression.
        # Example:
        # 2019-09-30 08:51:23 | tripleo-container-image-prepare :
        # Run tripleo-container-image-prepare logged to:
        # /var/log/tripleo-container-image-prepare.log  4335.64s

        # container_prep_line = (r'\d{4}-\d{2}-\d{2}\s(?:[01]\d|2[0-3])'
        #                        r':(?:[0-5]'
        #                        r'\d):(?:[0-5]\d)\s\|.*tripleo-container-image'
        #                        r'-prepare.log\s*([1-9]*\.?[1-9]*)s')

        # The regex below matches the new Warning log and the previous one
        # where the regex commented above also match

        container_prep_line = (r'Run.*tripleo-container-image-prepare.log'
                               r'\s*\-?\s*([1-9]*\.?[1-9]*)s')

        match = re.findall(container_prep_line, resp_data, re.MULTILINE)

        if len(match) > 0:
            build['container_prep_time_u'] = float(match[0])

def fix_task_name(task_name):
    return re.sub(r'/tmp/tripleo-modify-image.*', '/tmp/tripleo-modify-image',
                  task_name).replace(',', '_')

def print_influx_ara_tasks(build, ara_json_file):
    if build['job_name'] not in JOBS_FOR_ARA:
        return
    try:
        tasks = get_file_from_build(build, ara_json_file, json_view=True)
        if tasks is None:
            return
        for task in tasks:
            duration = to_seconds(task['Duration'])
            if duration > TASK_DURATION_TRESHOLD:
                print("build-task,task_name={},logs_path={},json_path={}"
                      " duration={},job_result=\"{}\",job_branch=\"{}\" {}".
                      format(
                          fix_task_name(task['Name'].replace(' ', '\\ ')),
                          build['log_url'], ara_json_file, duration,
                          build['result'], build['branch'],
                          to_ts(
                              task['Time Start'],
                              seconds=False,
                              pattern=TIMESTAMP_PATTERN2)))

    except Exception:
        pass

def influx(build):

    add_inventory_info(build)
    # add_container_prep_time(build)
    add_sova_info(build)
    add_rdopkg_change(build)

    if build['end_time'] is None:
        build['end_time'] = datetime.fromtimestamp(
            time.time()).strftime(TIMESTAMP_PATTERN)

    if build['start_time'] is None:
        build['start_time'] = build['end_time']
    duration = build.get('duration', 0)
    if duration is None:
        duration = 0
    # Get the nodename
    return ('build,'
            'type=%s,'
            'pipeline=%s,'
            'branch=%s,'
            'project=%s,'
            'job_name=%s,'
            'voting=%s,'
            'change=%s,'
            'patchset=%s,'
            'passed=%s,'
            'cloud=%s,'
            'region=%s,'
            'provider=%s,'
            'result="%s"'
            ' '
            'result="%s",'
            'result_num=%s,'
            'log_url="%s",'
            'log_link="%s",'
            'duration=%s,'
            'start=%s,'
            'end=%s,'
            'cloud="%s",'
            'region="%s",'
            'provider="%s",'
            'sova_reason="%s",'
            'sova_tag="%s",'
            'dep_change="%s",'
            'container_prep_time_u=0'
            ' '
            '%s' %
            (build['type'],
             build['pipeline'], 'none' if not build['branch']
             else build['branch'], build['project'], build['job_name'],
             build['voting'], build['change'], build['patchset'], 'True'
             if build['result'] == 'SUCCESS' else 'False',
             build.get('cloud', 'null'), build.get('region', 'null'),
             build.get('provider', 'null'), build['result'], build['result'], 1
             if build['result'] == 'SUCCESS' else 0, build['log_url'],
             "{}".format(
                 build['log_url'], build['job_name']),
             duration,
             to_ts(build['start_time'], seconds=True),
             to_ts(build['end_time'], seconds=True), build.get(
                 'cloud', 'null'), build.get('region', 'null'),
             build.get('provider', 'null'),
             build.get('sova_reason', ''),
             build.get('sova_tag', ''),
             build.get('dep_change', ''),
             to_ts(build['end_time'])))

def print_influx(build_type, builds):
    if builds:
        for build in builds:
            build['type'] = build_type
            for ara_json in ARA_JSONS:
                print_influx_ara_tasks(build, ara_json)
            print(influx(build))

def main():

    parser = argparse.ArgumentParser(
        description="Retrieve as influxdb zuul builds")

    parser.add_argument(
        '--url',
        default="http://zuul.openstack.org/api/",
        help="(default: %(default)s)")
    parser.add_argument(
        '--type', default="upstream", help="(default: %(default)s)")
    parser.add_argument(
        '--pages', type=int, default=1, help="(default: %(default)s)")
    parser.add_argument(
        '--offset', type=int, default=0, help="(default: %(default)s)")
    args = parser.parse_args()

    if args.type == 'internal':
        report_projects = INTERNAL_OOO_PROJECTS
    else:
        report_projects = OOO_PROJECTS

    for project in report_projects:
        print_influx(
            args.type,
            get_builds_info(
                url=args.url,
                query={'project': project},
                pages=args.pages,
                offset=args.offset))

if __name__ == '__main__':
    main()
<|endoftext|>