<commit_before>#!/usr/bin/env python
'''
Create a series of PCoA plots where the marker size varies by relative
abundance of a particular OTU

@author: Shareef M Dabdoub
'''
from __future__ import division
from util import parse_map_file
import argparse
from collections import OrderedDict, defaultdict
import json
import sys
import pylab as p

def otu_biom_entry_num(ID, biom, entry_type='rows'):
    for i, entry in enumerate(biom[entry_type]):
        if entry['id'] == ID:
            return i

def rel_abundance(otuID, sampleID, biom):
    otuRow = otu_biom_entry_num(otuID, biom)
    sCol = otu_biom_entry_num(sampleID, biom, 'columns')
    otuAbundance = 0
    sampleAbundance = 0

    for row, col, amt in biom['data']:
        if col == sCol:
            sampleAbundance += amt
        if col == sCol and row == otuRow:
            otuAbundance = amt

    if sampleAbundance == 0:
        return 0

    return otuAbundance / sampleAbundance * 15000

def parse_unifrac(unifracFN):
    """
    Parses the unifrac results file into a dictionary

    :@type unifracFN: str
    :@param unifracFN: The path to the unifrac results file
    :@rtype: dict
    :@return: A dictionary with keys: 'pcd' (principle coordinates data) which
              is a dictionary of the data keyed by sample ID,
              'eigvals' (eigenvalues), and 'varexp' (variation explained)
    """
    with open(unifracFN) as uF:
        unifrac = {'pcd':{}, 'eigvals':[], 'varexp':[]}

        lines = uF.readlines()[1:]
        for line in lines:
            if line == '\n': break
            line = line.split('\t')
            unifrac['pcd'][line[0]] = line[1:]

        unifrac['eigvals'] = lines[-2].split('\t')
        unifrac['varexp'] = lines[-1].split('\t')

def link_samples_to_categories(imap, category_idx):
    """
    Creates a dictionary of category types with all the associated Sample IDs

    :@type imap: dict
    :@param imap: The mapping data that lists category information
    :@type category_idx: int
    :@param category_idx: The index in the mapping data of the user-specified
                          category
    :@rtype: OrderedDict
    :@return: A dictionary of category types mapped to a list of corresponding
              sample IDs
    """
    cat_types = defaultdict(list)
    for row in imap.values():
        cat_types[row[category_idx]].append(row[0])
    return OrderedDict(sorted(cat_types.items(), key=lambda t: t[0]))

def rstyle(ax):
    """
    Styles axes to appear like ggplot2
    Must be called after all plot and axis manipulation operations have been
    carried out (needs to know final tick spacing)
    messymind.net/2012/07/making-matplotlib-look-like-ggplot/
    """
    # set the style of the major and minor grid lines, filled blocks
    ax.grid(True, 'major', color='w', linestyle='-', linewidth=1.4)
    ax.grid(True, 'minor', color='0.92', linestyle='-', linewidth=0.7)
    ax.patch.set_facecolor('0.85')
    ax.set_axisbelow(True)

    # set minor tick spacing to 1/2 of the major ticks
    ax.xaxis.set_minor_locator(p.MultipleLocator((p.plt.xticks()[0][1] - p.plt.xticks()[0][0]) / 2.0))
    ax.yaxis.set_minor_locator(p.MultipleLocator((p.plt.yticks()[0][1] - p.plt.yticks()[0][0]) / 2.0))

    # remove axis border
    for child in ax.get_children():
        if isinstance(child, p.matplotlib.spines.Spine):
            child.set_alpha(0)

    # restyle the tick lines
    for line in ax.get_xticklines() + ax.get_yticklines():
        line.set_markersize(5)
        line.set_color("gray")
        line.set_markeredgewidth(1.4)

    # remove the minor tick lines
    for line in ax.xaxis.get_ticklines(minor=True) + ax.yaxis.get_ticklines(minor=True):
        line.set_markersize(0)

    # only show bottom left ticks, pointing out of axis
    p.rcParams['xtick.direction'] = 'out'
    p.rcParams['ytick.direction'] = 'out'
    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_ticks_position('left')

    if ax.legend_ <> None:
        lg = ax.legend_
        lg.get_frame().set_linewidth(0)
        lg.get_frame().set_alpha(0.5)

def plot_PCoA(rh, rd, hh, hd, otu_name):
    p.matplotlib.rc('axes', edgecolor='black')
    p.matplotlib.rc('axes', facecolor='grey')
    fig = p.plt.figure()
    fig.set_figheight(8)
    fig.set_figwidth(10)

    p.scatter(rd['pc1'], rd['pc2'], rd['size'], color='#90EE90', marker='s')
    p.scatter(rd['zpc1'], rd['zpc2'], s=20, edgecolor='#90EE90', facecolor='none', marker='s')
    p.scatter(rh['pc1'], rh['pc2'], rh['size'], color='#FFA07A', marker='^')
    p.scatter(rh['zpc1'], rh['zpc2'], s=20, edgecolor='#FFA07A', facecolor='none', marker='^')
    p.scatter(hh['pc1'], hh['pc2'], hh['size'], color='green', marker='o')
    p.scatter(hh['zpc1'], hh['zpc2'], s=20, edgecolor='green', facecolor='none', marker='o')
    p.scatter(hd['pc1'], hd['pc2'], hd['size'], color='red', marker='p')
    p.scatter(hd['zpc1'], hd['zpc2'], s=20, edgecolor='red', facecolor='none', marker='p')

    # set plot labels
    rhr = p.Rectangle((0, 0), 1, 1, fc='#90EE90')
    rdr = p.Rectangle((0, 0), 1, 1, fc='#FFA07A')
    hhr = p.Rectangle((0, 0), 1, 1, fc='green')
    hdr = p.Rectangle((0, 0), 1, 1, fc='red')
    p.legend((rhr, rdr, hhr, hdr), ('Rat Health', 'Rat Disease', 'Human Health', 'Human Disease'), loc='upper left')
    p.title(otu_name, style='italic')
    p.xlabel('PC2 ({:.2f}%)'.format(float(unifrac[-1].split('\t')[2])))
    p.ylabel('PC1 ({:.2f}%)'.format(float(unifrac[-1].split('\t')[1])))
    p.hlines(0, -0.4, 0.4)
    p.vlines(0, -0.3, 0.3)
    p.xlim(-0.4, 0.4)
    p.ylim(-0.3, 0.3)

    fig.savefig('./figures/' + '_'.join(otu_name.split()) + '.png', facecolor='gray', edgecolor='none')

def handle_program_options():
    parser = argparse.ArgumentParser(description="Create a series of Principle\
                                     Coordinate plots for each OTU in an \
                                     input list where the plot points are \
                                     varied in size by the relative abundance \
                                     of the OTU (relative to either Sample or\
                                     the total contribution of the OTU to the \
                                     data set.")
    parser.add_argument('-i', '--otu_table', required=True,
                        help="The biom-format file with OTU-Sample abundance \
                              data.")
    parser.add_argument('-u', '--unifrac', required=True,
                        help='Principle coordinates analysis file. \
                              Eg. unweighted_unifrac_pc.txt')
    parser.add_argument('-d', '--names_colors_ids_fn', required=True,
                        help='The name of an input data file containing three\
                              items: Line 1: a tab-separated list of display\
                              names for the different types in the specified \
                              mapping category (--mapping_category), Line 2:\
                              a matching tab-separated list of hexadecimal\
                              colors for each of the category types, Line 4:\
                              blank, Lines 5-end: a tab-separated pair \
                              specifying OTU ID and OTU Name. Each entry will\
                              get a separate PCoA plot under a file with the \
                              name of the OTU.')
    parser.add_argument('-m', '--mapping', required=True,
                        help="The mapping file specifying group information \
                              for each sample.")
    parser.add_argument('-o', '--output_dir', default='.',
                        help="The directory to output the PCoA plots to.")
    parser.add_argument('-c', '--map_category',
                        help="Any mapping category, such as treatment type, \
                              that will be used to group the data in the \
                              output plots. For example, one category \
                              with three types will result in three different\
                              point sets in the final output.")
    parser.add_argument('-n', '--normalization', default='sample',
                        choices=['sample','otu'],
                        help="Specifies whether OTU abundance is \
                        normalized column-wise (per-sample) or row-wise \
                        (per-OTU).")
#    parser.add_argument('-v', '--verbose', action='store_true')

    return parser.parse_args()

def main():
    args = handle_program_options()

    with open(args.otu_table) as bF:
        biom = json.loads(bF.readline())

    unifrac = parse_unifrac(args.unifrac)

    otus = {}
    with open(args.names_colors_ids_fn, 'rU') as nciF:
        category_names = nciF.readline().split('\t')
        category_colors = nciF.readline().split('\t')
        for line in nciF.readlines()[2:]:
            line = line.split()
            otus[line[0]] = ' '.join(line[1:])

    imap = parse_map_file(args.mapping)
    with open(args.mapping) as mF:
        header = mF.readline().split('\t')
        try:
            category_idx = header.index(args.map_category)
        except ValueError:
            msg = "Error: Specified mapping category '{}' not found"
            sys.exit(msg.format(args.map_category))
    category_ids = link_samples_to_categories(imap, category_idx)

    # plot samples based on relative abundance of some OTU ID
    for otuID in otus:
        cat_data = {cat:{'pc1':[], 'pc2':[], 'size':[], 'zpc1':[], 'zpc2':[]}
                    for cat in category_ids}

        for e in pcd:
            if 'RM' in e[0]:
                category = rh
            elif 'RS' in e[0]:
                category = rd
            elif 'HH' in e[0]:
                category = hh
            elif 'HD' in e[0]:
                category = hd

            size = rel_abundance(otuID, e[0], biom)
            if size > 0:
                category['pc1'].append(e[1])
                category['pc2'].append(e[2])
                category['size'].append(size)
            else:
                category['zpc1'].append(e[1])
                category['zpc2'].append(e[2])

        plot_PCoA(rh, rd, hh, hd, otus[otuID])
<commit_msg>Completes major functionality<commit_after>#!/usr/bin/env python
'''
Create a series of PCoA plots where the marker size varies by relative
abundance of a particular OTU

@author: Shareef M Dabdoub
'''
from __future__ import division
from util import parse_map_file
import argparse
from collections import OrderedDict, defaultdict
import json
import sys
import pylab as p

def otu_biom_entry_num(ID, biom, entry_type='rows'):
    for i, entry in enumerate(biom[entry_type]):
        if entry['id'] == ID:
            return i

def rel_abundance(otuID, sampleID, biom):
    otuRow = otu_biom_entry_num(otuID, biom)
    sCol = otu_biom_entry_num(sampleID, biom, 'columns')
    otuAbundance = 0
    sampleAbundance = 0

    for row, col, amt in biom['data']:
        if col == sCol:
            sampleAbundance += amt
        if col == sCol and row == otuRow:
            otuAbundance = amt

    if sampleAbundance == 0:
        return 0

    return otuAbundance / sampleAbundance * 15000

def parse_unifrac(unifracFN):
    """
    Parses the unifrac results file into a dictionary

    :@type unifracFN: str
    :@param unifracFN: The path to the unifrac results file
    :@rtype: dict
    :@return: A dictionary with keys: 'pcd' (principle coordinates data) which
              is a dictionary of the data keyed by sample ID,
              'eigvals' (eigenvalues), and 'varexp' (variation explained)
    """
    with open(unifracFN) as uF:
        unifrac = {'pcd':{}, 'eigvals':[], 'varexp':[]}

        lines = uF.readlines()[1:]
        for line in lines:
            if line == '\n': break
            line = line.split('\t')
            unifrac['pcd'][line[0]] = line[1:]

        unifrac['eigvals'] = lines[-2].split('\t')
        unifrac['varexp'] = lines[-1].split('\t')

def link_samples_to_categories(imap, category_idx):
    """
    Creates a dictionary of category types with all the associated Sample IDs

    :@type imap: dict
    :@param imap: The mapping data that lists category information
    :@type category_idx: int
    :@param category_idx: The index in the mapping data of the user-specified
                          category
    :@rtype: OrderedDict
    :@return: A dictionary of category types mapped to a list of corresponding
              sample IDs
    """
    cat_types = defaultdict(list)
    for row in imap.values():
        cat_types[row[category_idx]].append(row[0])
    return OrderedDict(sorted(cat_types.items(), key=lambda t: t[0]))

def rstyle(ax):
    """
    Styles axes to appear like ggplot2
    Must be called after all plot and axis manipulation operations have been
    carried out (needs to know final tick spacing)
    messymind.net/2012/07/making-matplotlib-look-like-ggplot/
    """
    # set the style of the major and minor grid lines, filled blocks
    ax.grid(True, 'major', color='w', linestyle='-', linewidth=1.4)
    ax.grid(True, 'minor', color='0.92', linestyle='-', linewidth=0.7)
    ax.patch.set_facecolor('0.85')
    ax.set_axisbelow(True)

    # set minor tick spacing to 1/2 of the major ticks
    ax.xaxis.set_minor_locator(p.MultipleLocator((p.plt.xticks()[0][1] - p.plt.xticks()[0][0]) / 2.0))
    ax.yaxis.set_minor_locator(p.MultipleLocator((p.plt.yticks()[0][1] - p.plt.yticks()[0][0]) / 2.0))

    # remove axis border
    for child in ax.get_children():
        if isinstance(child, p.matplotlib.spines.Spine):
            child.set_alpha(0)

    # restyle the tick lines
    for line in ax.get_xticklines() + ax.get_yticklines():
        line.set_markersize(5)
        line.set_color("gray")
        line.set_markeredgewidth(1.4)

    # remove the minor tick lines
    for line in ax.xaxis.get_ticklines(minor=True) + ax.yaxis.get_ticklines(minor=True):
        line.set_markersize(0)

    # only show bottom left ticks, pointing out of axis
    p.rcParams['xtick.direction'] = 'out'
    p.rcParams['ytick.direction'] = 'out'
    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_ticks_position('left')

    if ax.legend_ <> None:
        lg = ax.legend_
        lg.get_frame().set_linewidth(0)
        lg.get_frame().set_alpha(0.5)

def plot_PCoA(cat_data, otu_name, unifrac, names, colors):
    p.matplotlib.rc('axes', edgecolor='black')
    p.matplotlib.rc('axes', facecolor='grey')
    fig = p.plt.figure()
    fig.set_figheight(8)
    fig.set_figwidth(10)
    legend = []

    for i,cat in enumerate(cat_data):
        p.scatter(cat['pc1'], cat['pc2'], cat['size'], color=colors[i],
                  marker='s')
        p.scatter(cat['zpc1'], cat['zpc2'], s=20, edgecolor=colors[i],
                  facecolor='none', marker='s')
        legend.append(p.Rectangle((0, 0), 1, 1, fc=colors[i]))

    p.legend(legend, names, loc='upper left')
    p.title(otu_name, style='italic')
    p.xlabel('PC2 ({:.2f}%)'.format(float(unifrac[-1].split('\t')[2])))
    p.ylabel('PC1 ({:.2f}%)'.format(float(unifrac[-1].split('\t')[1])))

    fig.savefig('./figures/' + '_'.join(otu_name.split()) + '.png',
                facecolor='gray', edgecolor='none')

def handle_program_options():
    parser = argparse.ArgumentParser(description="Create a series of Principle\
                                     Coordinate plots for each OTU in an \
                                     input list where the plot points are \
                                     varied in size by the relative abundance \
                                     of the OTU (relative to either Sample or\
                                     the total contribution of the OTU to the \
                                     data set.")
    parser.add_argument('-i', '--otu_table', required=True,
                        help="The biom-format file with OTU-Sample abundance \
                              data.")
    parser.add_argument('-u', '--unifrac', required=True,
                        help='Principle coordinates analysis file. \
                              Eg. unweighted_unifrac_pc.txt')
    parser.add_argument('-d', '--names_colors_ids_fn', required=True,
                        help='The name of an input data file containing three\
                              items: Line 1: a tab-separated list of display\
                              names for the different types in the specified \
                              mapping category (--mapping_category), Line 2:\
                              a matching tab-separated list of hexadecimal\
                              colors for each of the category types, Line 4:\
                              blank, Lines 5-end: a tab-separated pair \
                              specifying OTU ID and OTU Name. Each entry will\
                              get a separate PCoA plot under a file with the \
                              name of the OTU.')
    parser.add_argument('-m', '--mapping', required=True,
                        help="The mapping file specifying group information \
                              for each sample.")
    parser.add_argument('-o', '--output_dir', default='.',
                        help="The directory to output the PCoA plots to.")
    parser.add_argument('-c', '--map_category',
                        help="Any mapping category, such as treatment type, \
                              that will be used to group the data in the \
                              output plots. For example, one category \
                              with three types will result in three different\
                              point sets in the final output.")
    parser.add_argument('-n', '--normalization', default='sample',
                        choices=['sample','otu'],
                        help="Specifies whether OTU abundance is \
                        normalized column-wise (per-sample) or row-wise \
                        (per-OTU).")
#    parser.add_argument('-v', '--verbose', action='store_true')

    return parser.parse_args()

def main():
    args = handle_program_options()

    with open(args.otu_table) as bF:
        biom = json.loads(bF.readline())

    unifrac = parse_unifrac(args.unifrac)

    otus = {}
    with open(args.names_colors_ids_fn, 'rU') as nciF:
        category_names = nciF.readline().split('\t')
        category_colors = nciF.readline().split('\t')
        for line in nciF.readlines()[2:]:
            line = line.split()
            otus[line[0]] = ' '.join(line[1:])

    imap = parse_map_file(args.mapping)
    with open(args.mapping) as mF:
        header = mF.readline().split('\t')
        try:
            category_idx = header.index(args.map_category)
        except ValueError:
            msg = "Error: Specified mapping category '{}' not found"
            sys.exit(msg.format(args.map_category))
    category_ids = link_samples_to_categories(imap, category_idx)

    # plot samples based on relative abundance of some OTU ID
    for otuID in otus:
        cat_data = {cat:{'pc1':[], 'pc2':[], 'size':[], 'zpc1':[], 'zpc2':[]}
                    for cat in category_ids}

        for e in unifrac['pcd']:
            category = cat_data[imap[e[0]][category_idx]]
            size = rel_abundance(otuID, e[0], biom)
            if size > 0:
                category['pc1'].append(e[1])
                category['pc2'].append(e[2])
                category['size'].append(size)
            else:
                category['zpc1'].append(e[1])
                category['zpc2'].append(e[2])

        plot_PCoA(cat_data, otus[otuID], unifrac, category_names, category_colors)
<|endoftext|>


<commit_before><commit_msg>integrate end-to-end stories with yaml training data (#6118)<commit_after><|endoftext|>


<commit_before><commit_msg>Fix ensembl ftp export command<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python

import socket
import PiVisConstants

def getOwnIp():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(('5.255.255.255', 1))
    IP = s.getsockname()[0]
    s.close()
    return IP

class PiVisServer:
    def __init__(self, scheduler, portNo):
        self.portNo = portNo
        self.host = getOwnIp()
        self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connections = []
        self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.serverSocket.bind(('', self.portNo))
        self.__setUpServiceListener__()
        scheduler.registerRunnable(self.run)

    def __setUpServiceListener__(self):
        self.serviceListenerSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.serviceListenerSocket.bind(('224.1.1.1', PiVisConstants.DISCOVER_SERVICE))
        self.serviceListenerSocket.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_IF,
											  socket.inet_aton(self.host))
        self.serviceListenerSocket.setsockopt(socket.SOL_IP,
                                              socket.IP_ADD_MEMBERSHIP,
                                              socket.inet_aton('224.1.1.1') +
                                              socket.inet_aton(self.host))
    def __handleNewConnections__(self):
        try:
            self.serverSocket.settimeout(0.001)
            self.serverSocket.listen(1)
            (connection, address) = self.serverSocket.accept()
        except socket.timeout:
            pass
        except:
            raise
        else:
            print("PiVisionNwM::waitForConnection: Connected to by " + str(address))
            self.connections.append((connection, address))

    def __handleServiceDiscoveryRequests__(self):
        self.serviceListenerSocket.settimeout(0.001)
        try:
            request = self.serviceListenerSocket.recvfrom(4096)
        except socket.timeout:
            pass
        except:
            raise
        else:
            print("PiVisServer::__handleServiceDiscoveryRequests__: New service request for " + str(request[0]) + " received from " + str(request[1]))
            reqString = str(request[0], 'utf-8')
            if reqString.startswith(PiVisConstants.SERVICE_DISCOVER_REQUEST_HEADER):
                splitReqString = str.split(reqString, '_')
                if int(splitReqString[2]) == self.portNo:
                    response = bytearray()
                    response.extend(map(ord, self.host))
                    responseSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                    responseSocket.sendto(response, request[1])
                    responseSocket.close()
                else:
                	print("Discarded request. Wrong service")
            else:
            	print("Discared request. Wrong header")
            self.serviceListenerSocket.close()
            self.__setUpServiceListener__()

    def run(self):
        self.__handleNewConnections__()
        self.__handleServiceDiscoveryRequests__()

    def send(self, data):
        for connection in self.connections:
            try:
                connection[0].sendall(data)
            except Exception as e:
                print("Disconnecting connection due to: " + str(e))
                self.connections.remove(connection)
<commit_msg>Hopefully allows for multiple servers on same host<commit_after>#!/usr/bin/python

import socket
import PiVisConstants

def getOwnIp():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(('5.255.255.255', 1))
    IP = s.getsockname()[0]
    s.close()
    return IP

class PiVisServer:
    def __init__(self, scheduler, portNo):
        self.portNo = portNo
        self.host = getOwnIp()
        self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connections = []
        self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.serverSocket.bind(('', self.portNo))
        self.__setUpServiceListener__()
        scheduler.registerRunnable(self.run)

    def __setUpServiceListener__(self):
        self.serviceListenerSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.serviceListenerSocket.bind(('224.1.1.1', PiVisConstants.DISCOVER_SERVICE))
        self.serviceListenerSocket.setsockopt(socket.SOL_IP, socket.SO_REUSEADDR, 1)
        self.serviceListenerSocket.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_IF,
											  socket.inet_aton(self.host))
        self.serviceListenerSocket.setsockopt(socket.SOL_IP,
                                              socket.IP_ADD_MEMBERSHIP,
                                              socket.inet_aton('224.1.1.1') +
                                              socket.inet_aton(self.host))
    def __handleNewConnections__(self):
        try:
            self.serverSocket.settimeout(0.001)
            self.serverSocket.listen(1)
            (connection, address) = self.serverSocket.accept()
        except socket.timeout:
            pass
        except:
            raise
        else:
            print("PiVisionNwM::waitForConnection: Connected to by " + str(address))
            self.connections.append((connection, address))

    def __handleServiceDiscoveryRequests__(self):
        self.serviceListenerSocket.settimeout(0.001)
        try:
            request = self.serviceListenerSocket.recvfrom(4096)
        except socket.timeout:
            pass
        except:
            raise
        else:
            print("PiVisServer::__handleServiceDiscoveryRequests__: New service request for " + str(request[0]) + " received from " + str(request[1]))
            reqString = str(request[0], 'utf-8')
            if reqString.startswith(PiVisConstants.SERVICE_DISCOVER_REQUEST_HEADER):
                splitReqString = str.split(reqString, '_')
                if int(splitReqString[2]) == self.portNo:
                    response = bytearray()
                    response.extend(map(ord, self.host))
                    responseSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                    responseSocket.sendto(response, request[1])
                    responseSocket.close()
                else:
                    print("Discarded request. Wrong service")
            else:
                print("Discared request. Wrong header")
            self.serviceListenerSocket.close()
            self.__setUpServiceListener__()

    def run(self):
        self.__handleNewConnections__()
        self.__handleServiceDiscoveryRequests__()

    def send(self, data):
        for connection in self.connections:
            try:
                connection[0].sendall(data)
            except Exception as e:
                print("Disconnecting connection due to: " + str(e))
                self.connections.remove(connection)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import logging
import os
import re
import shutil
import zipfile
from abc import ABCMeta, abstractmethod
from tempfile import mkdtemp
from textwrap import dedent

import six
from six import PY3

from virtualenv.info import IS_WIN
from virtualenv.util import ConfigParser
from virtualenv.util.path import Path

@six.add_metaclass(ABCMeta)
class PipInstall(object):
    def __init__(self, wheel, creator, image_folder):
        self._wheel = wheel
        self._creator = creator
        self._image_dir = image_folder
        self._extracted = False
        self.__dist_info = None
        self._console_entry_points = None

    @abstractmethod
    def _sync(self, src, dst):
        raise NotImplementedError

    def install(self):
        self._extracted = True
        # sync image
        site_package = self._creator.site_packages[0]
        for filename in self._image_dir.iterdir():
            into = site_package / filename.name
            logging.debug("link %s of %s", filename, into)
            if into.exists():
                if into.is_dir() and not into.is_symlink():
                    shutil.rmtree(str(into))
                else:
                    into.unlink()
            self._sync(filename, into)
        # generate console executables
        consoles = set()
        bin_dir = self._creator.bin_dir
        for name, module in self._console_scripts.items():
            consoles.update(self._create_console_entry_point(name, module, bin_dir))
        logging.debug("generated console scripts %s", " ".join(i.name for i in consoles))

    def build_image(self):
        # 1. first extract the wheel
        logging.debug("build install image to %s of %s", self._image_dir, self._wheel.name)
        with zipfile.ZipFile(str(self._wheel)) as zip_ref:
            zip_ref.extractall(str(self._image_dir))
            self._extracted = True
        # 2. now add additional files not present in the package
        new_files = self._generate_new_files()
        # 3. finally fix the records file
        self._fix_records(new_files)

    def _records_text(self, files):
        record_data = "\n".join(
            "{},,".format(os.path.relpath(six.ensure_text(str(rec)), six.ensure_text(str(self._image_dir))))
            for rec in files
        )
        return record_data

    def _generate_new_files(self):
        new_files = set()
        installer = self._dist_info / "INSTALLER"
        installer.write_text("pip\n")
        new_files.add(installer)
        # inject a no-op root element, as workaround for bug added
        # by https://github.com/pypa/pip/commit/c7ae06c79#r35523722
        marker = self._image_dir / "{}.virtualenv".format(self._dist_info.name)
        marker.write_text("")
        new_files.add(marker)
        folder = mkdtemp()
        try:
            to_folder = Path(folder)
            rel = os.path.relpath(
                six.ensure_text(str(self._creator.bin_dir)), six.ensure_text(str(self._creator.site_packages[0]))
            )
            for name, module in self._console_scripts.items():
                new_files.update(
                    Path(os.path.normpath(six.ensure_text(str(self._image_dir / rel / i.name))))
                    for i in self._create_console_entry_point(name, module, to_folder)
                )
        finally:
            shutil.rmtree(folder, ignore_errors=True)
        return new_files

    @property
    def _dist_info(self):
        if self._extracted is False:
            return None  # pragma: no cover
        if self.__dist_info is None:
            for filename in self._image_dir.iterdir():
                if filename.suffix == ".dist-info":
                    self.__dist_info = filename
                    break
            else:
                raise RuntimeError("no dist info")  # pragma: no cover
        return self.__dist_info

    @abstractmethod
    def _fix_records(self, extra_record_data):
        raise NotImplementedError

    @property
    def _console_scripts(self):
        if self._extracted is False:
            return None  # pragma: no cover
        if self._console_entry_points is None:
            self._console_entry_points = {}
            entry_points = self._dist_info / "entry_points.txt"
            if entry_points.exists():
                parser = ConfigParser.ConfigParser()
                with entry_points.open() as file_handler:
                    reader = getattr(parser, "read_file" if PY3 else "readfp")
                    reader(file_handler)
                if "console_scripts" in parser.sections():
                    for name, value in parser.items("console_scripts"):
                        match = re.match(r"(.*?)-?\d\.?\d*", name)
                        if match:
                            name = match.groups(1)[0]
                        self._console_entry_points[name] = value
        return self._console_entry_points

    def _create_console_entry_point(self, name, value, to_folder):
        result = []
        if IS_WIN:
            # windows doesn't support simple script files, so fallback to more complicated exe generator
            from distlib.scripts import ScriptMaker

            maker = ScriptMaker(None, str(to_folder))
            maker.clobber = True  # overwrite
            maker.variants = {"", "X", "X.Y"}  # create all variants
            maker.set_mode = True  # ensure they are executable
            maker.executable = str(self._creator.exe)
            specification = "{} = {}".format(name, value)
            new_files = maker.make(specification)
            result.extend(Path(i) for i in new_files)
        else:
            module, func = value.split(":")
            content = (
                dedent(
                    """
            #!{0}
            # -*- coding: utf-8 -*-
            import re
            import sys

            from {1} import {2}

            if __name__ == "__main__":
                sys.argv[0] = re.sub(r"(-script.pyw?|.exe)?$", "", sys.argv[0])
                sys.exit({2}())
            """
                )
                .lstrip()
                .format(self._creator.exe, module, func)
            )

            version = self._creator.interpreter.version_info
            for new_name in (
                name,
                "{}{}".format(name, version.major),
                "{}-{}.{}".format(name, version.major, version.minor),
            ):
                exe = to_folder / new_name
                exe.write_text(content, encoding="utf-8")
                exe.chmod(0o755)
                result.append(exe)
        return result

    def clear(self):
        if self._image_dir.exists():
            shutil.rmtree(six.ensure_text(str(self._image_dir)))

    def has_image(self):
        return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None
<commit_msg>use S_IXUSR | S_IXGRP | S_IXOTH instead of 0755 for chmod exe<commit_after>from __future__ import absolute_import, unicode_literals

import logging
import os
import re
import shutil
import zipfile
from abc import ABCMeta, abstractmethod
from stat import S_IXGRP, S_IXOTH, S_IXUSR
from tempfile import mkdtemp
from textwrap import dedent

import six
from six import PY3

from virtualenv.info import IS_WIN
from virtualenv.util import ConfigParser
from virtualenv.util.path import Path

@six.add_metaclass(ABCMeta)
class PipInstall(object):
    def __init__(self, wheel, creator, image_folder):
        self._wheel = wheel
        self._creator = creator
        self._image_dir = image_folder
        self._extracted = False
        self.__dist_info = None
        self._console_entry_points = None

    @abstractmethod
    def _sync(self, src, dst):
        raise NotImplementedError

    def install(self):
        self._extracted = True
        # sync image
        site_package = self._creator.site_packages[0]
        for filename in self._image_dir.iterdir():
            into = site_package / filename.name
            logging.debug("link %s of %s", filename, into)
            if into.exists():
                if into.is_dir() and not into.is_symlink():
                    shutil.rmtree(str(into))
                else:
                    into.unlink()
            self._sync(filename, into)
        # generate console executables
        consoles = set()
        bin_dir = self._creator.bin_dir
        for name, module in self._console_scripts.items():
            consoles.update(self._create_console_entry_point(name, module, bin_dir))
        logging.debug("generated console scripts %s", " ".join(i.name for i in consoles))

    def build_image(self):
        # 1. first extract the wheel
        logging.debug("build install image to %s of %s", self._image_dir, self._wheel.name)
        with zipfile.ZipFile(str(self._wheel)) as zip_ref:
            zip_ref.extractall(str(self._image_dir))
            self._extracted = True
        # 2. now add additional files not present in the package
        new_files = self._generate_new_files()
        # 3. finally fix the records file
        self._fix_records(new_files)

    def _records_text(self, files):
        record_data = "\n".join(
            "{},,".format(os.path.relpath(six.ensure_text(str(rec)), six.ensure_text(str(self._image_dir))))
            for rec in files
        )
        return record_data

    def _generate_new_files(self):
        new_files = set()
        installer = self._dist_info / "INSTALLER"
        installer.write_text("pip\n")
        new_files.add(installer)
        # inject a no-op root element, as workaround for bug added
        # by https://github.com/pypa/pip/commit/c7ae06c79#r35523722
        marker = self._image_dir / "{}.virtualenv".format(self._dist_info.name)
        marker.write_text("")
        new_files.add(marker)
        folder = mkdtemp()
        try:
            to_folder = Path(folder)
            rel = os.path.relpath(
                six.ensure_text(str(self._creator.bin_dir)), six.ensure_text(str(self._creator.site_packages[0]))
            )
            for name, module in self._console_scripts.items():
                new_files.update(
                    Path(os.path.normpath(six.ensure_text(str(self._image_dir / rel / i.name))))
                    for i in self._create_console_entry_point(name, module, to_folder)
                )
        finally:
            shutil.rmtree(folder, ignore_errors=True)
        return new_files

    @property
    def _dist_info(self):
        if self._extracted is False:
            return None  # pragma: no cover
        if self.__dist_info is None:
            for filename in self._image_dir.iterdir():
                if filename.suffix == ".dist-info":
                    self.__dist_info = filename
                    break
            else:
                raise RuntimeError("no dist info")  # pragma: no cover
        return self.__dist_info

    @abstractmethod
    def _fix_records(self, extra_record_data):
        raise NotImplementedError

    @property
    def _console_scripts(self):
        if self._extracted is False:
            return None  # pragma: no cover
        if self._console_entry_points is None:
            self._console_entry_points = {}
            entry_points = self._dist_info / "entry_points.txt"
            if entry_points.exists():
                parser = ConfigParser.ConfigParser()
                with entry_points.open() as file_handler:
                    reader = getattr(parser, "read_file" if PY3 else "readfp")
                    reader(file_handler)
                if "console_scripts" in parser.sections():
                    for name, value in parser.items("console_scripts"):
                        match = re.match(r"(.*?)-?\d\.?\d*", name)
                        if match:
                            name = match.groups(1)[0]
                        self._console_entry_points[name] = value
        return self._console_entry_points

    def _create_console_entry_point(self, name, value, to_folder):
        result = []
        if IS_WIN:
            # windows doesn't support simple script files, so fallback to more complicated exe generator
            from distlib.scripts import ScriptMaker

            maker = ScriptMaker(None, str(to_folder))
            maker.clobber = True  # overwrite
            maker.variants = {"", "X", "X.Y"}  # create all variants
            maker.set_mode = True  # ensure they are executable
            maker.executable = str(self._creator.exe)
            specification = "{} = {}".format(name, value)
            new_files = maker.make(specification)
            result.extend(Path(i) for i in new_files)
        else:
            module, func = value.split(":")
            content = (
                dedent(
                    """
            #!{0}
            # -*- coding: utf-8 -*-
            import re
            import sys

            from {1} import {2}

            if __name__ == "__main__":
                sys.argv[0] = re.sub(r"(-script.pyw?|.exe)?$", "", sys.argv[0])
                sys.exit({2}())
            """
                )
                .lstrip()
                .format(self._creator.exe, module, func)
            )

            version = self._creator.interpreter.version_info
            for new_name in (
                name,
                "{}{}".format(name, version.major),
                "{}-{}.{}".format(name, version.major, version.minor),
            ):
                exe = to_folder / new_name
                exe.write_text(content, encoding="utf-8")
                exe.chmod(exe.stat() | S_IXUSR | S_IXGRP | S_IXOTH)
                result.append(exe)
        return result

    def clear(self):
        if self._image_dir.exists():
            shutil.rmtree(six.ensure_text(str(self._image_dir)))

    def has_image(self):
        return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None
<|endoftext|>


<commit_before># Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import math
import sys

from telemetry.timeline import model as model_module
from telemetry.value import improvement_direction
from telemetry.value import list_of_scalar_values
from telemetry.value import scalar
from telemetry.web_perf.metrics import timeline_based_metric

TOPLEVEL_GL_CATEGORY = 'gpu_toplevel'
TOPLEVEL_SERVICE_CATEGORY = 'disabled-by-default-gpu.service'
TOPLEVEL_DEVICE_CATEGORY = 'disabled-by-default-gpu.device'

SERVICE_FRAME_END_MARKER = (TOPLEVEL_SERVICE_CATEGORY, 'SwapBuffer')
DEVICE_FRAME_END_MARKER = (TOPLEVEL_DEVICE_CATEGORY, 'SwapBuffer')

TRACKED_GL_CONTEXT_NAME = {'RenderCompositor': 'render_compositor',
                           'BrowserCompositor': 'browser_compositor',
                           'Compositor': 'browser_compositor'}

class GPUTimelineListOfValues(list_of_scalar_values.ListOfScalarValues):
  def __init__(self, page, name, values):
    super(GPUTimelineListOfValues, self).__init__(
        page, name, 'ms', values,
        improvement_direction=improvement_direction.DOWN)

class GPUTimelineValue(scalar.ScalarValue):
  def __init__(self, page, name, value):
    super(GPUTimelineValue, self).__init__(
      page, name, 'ms', value,
      improvement_direction=improvement_direction.DOWN)

def _CalculateFrameTimes(events_per_frame, event_data_func):
  """Given a list of events per frame and a function to extract event time data,
     returns a list of frame times."""
  times_per_frame = []
  for event_list in events_per_frame:
    event_times = [event_data_func(event) for event in event_list]
    times_per_frame.append(sum(event_times))
  return times_per_frame

def _CPUFrameTimes(events_per_frame):
  """Given a list of events per frame, returns a list of CPU frame times."""
  # CPU event frames are calculated using the event thread duration.
  # Some platforms do not support thread_duration, convert those to 0.
  return _CalculateFrameTimes(events_per_frame,
                              lambda event: event.thread_duration or 0)

def _GPUFrameTimes(events_per_frame):
  """Given a list of events per frame, returns a list of GPU frame times."""
  # GPU event frames are asynchronous slices which use the event duration.
  return _CalculateFrameTimes(events_per_frame,
                              lambda event: event.duration)

def TimelineName(name, source_type, value_type):
  """Constructs the standard name given in the timeline.

  Args:
    name: The name of the timeline, for example "total", or "render_compositor".
    source_type: One of "cpu", "gpu" or None. None is only used for total times.
    value_type: the type of value. For example "mean", "stddev"...etc.
  """
  if source_type:
    return '%s_%s_%s_time' % (name, value_type, source_type)
  else:
    return '%s_%s_time' % (name, value_type)

class GPUTimelineMetric(timeline_based_metric.TimelineBasedMetric):
  """Computes GPU based metrics."""

  def __init__(self):
    super(GPUTimelineMetric, self).__init__()

  def AddResults(self, model, _, interaction_records, results):
    self.VerifyNonOverlappedRecords(interaction_records)
    service_times = self._CalculateGPUTimelineData(model)
    for value_item, durations in service_times.iteritems():
      count = len(durations)
      avg = 0.0
      stddev = 0.0
      maximum = 0.0
      if count:
        avg = sum(durations) / count
        stddev = math.sqrt(sum((d - avg) ** 2 for d in durations) / count)
        maximum = max(durations)

      name, src = value_item

      if src:
        frame_times_name = '%s_%s_frame_times' % (name, src)
      else:
        frame_times_name = '%s_frame_times' % (name)

      if durations:
        results.AddValue(GPUTimelineListOfValues(results.current_page,
                                                 frame_times_name,
                                                 durations))

      results.AddValue(GPUTimelineValue(results.current_page,
                                        TimelineName(name, src, 'max'),
                                        maximum))
      results.AddValue(GPUTimelineValue(results.current_page,
                                        TimelineName(name, src, 'mean'),
                                        avg))
      results.AddValue(GPUTimelineValue(results.current_page,
                                        TimelineName(name, src, 'stddev'),
                                        stddev))

  def _CalculateGPUTimelineData(self, model):
    """Uses the model and calculates the times for various values for each
       frame. The return value will be a dictionary of the following format:
         {
           (EVENT_NAME1, SRC1_TYPE): [FRAME0_TIME, FRAME1_TIME...etc.],
           (EVENT_NAME2, SRC2_TYPE): [FRAME0_TIME, FRAME1_TIME...etc.],
         }

       Events:
         swap - The time in milliseconds between each swap marker.
         total - The amount of time spent in the renderer thread.
         TRACKED_NAMES: Using the TRACKED_GL_CONTEXT_NAME dict, we
                        include the traces per frame for the
                        tracked name.
       Source Types:
         None - This will only be valid for the "swap" event.
         cpu - For an event, the "cpu" source type signifies time spent on the
               gpu thread using the CPU. This uses the "gpu.service" markers.
         gpu - For an event, the "gpu" source type signifies time spent on the
               gpu thread using the GPU. This uses the "gpu.device" markers.
    """
    all_service_events = []
    current_service_frame_end = sys.maxint
    current_service_events = []

    all_device_events = []
    current_device_frame_end = sys.maxint
    current_device_events = []

    tracked_events = {}
    tracked_events.update(
        dict([((value, 'cpu'), [])
              for value in TRACKED_GL_CONTEXT_NAME.itervalues()]))
    tracked_events.update(
        dict([((value, 'gpu'), [])
              for value in TRACKED_GL_CONTEXT_NAME.itervalues()]))

    # These will track traces within the current frame.
    current_tracked_service_events = collections.defaultdict(list)
    current_tracked_device_events = collections.defaultdict(list)

    event_iter = model.IterAllEvents(
        event_type_predicate=model_module.IsSliceOrAsyncSlice)
    for event in event_iter:
      # Look for frame end markers
      if (event.category, event.name) == SERVICE_FRAME_END_MARKER:
        current_service_frame_end = event.end
      elif (event.category, event.name) == DEVICE_FRAME_END_MARKER:
        current_device_frame_end = event.end

      # Track all other toplevel gl category markers
      elif event.args.get('gl_category', None) == TOPLEVEL_GL_CATEGORY:
        base_name = event.name
        dash_index = base_name.rfind('-')
        if dash_index != -1:
          base_name = base_name[:dash_index]
        tracked_name = TRACKED_GL_CONTEXT_NAME.get(base_name, None)

        if event.category == TOPLEVEL_SERVICE_CATEGORY:
          # Check if frame has ended.
          if event.start >= current_service_frame_end:
            if current_service_events:
              all_service_events.append(current_service_events)
              for value in TRACKED_GL_CONTEXT_NAME.itervalues():
                tracked_events[(value, 'cpu')].append(
                    current_tracked_service_events[value])
            current_service_events = []
            current_service_frame_end = sys.maxint
            current_tracked_service_events.clear()

          current_service_events.append(event)
          if tracked_name:
            current_tracked_service_events[tracked_name].append(event)

        elif event.category == TOPLEVEL_DEVICE_CATEGORY:
          # Check if frame has ended.
          if event.start >= current_device_frame_end:
            if current_device_events:
              all_device_events.append(current_device_events)
              for value in TRACKED_GL_CONTEXT_NAME.itervalues():
                tracked_events[(value, 'gpu')].append(
                    current_tracked_device_events[value])
            current_device_events = []
            current_device_frame_end = sys.maxint
            current_tracked_device_events.clear()

          current_device_events.append(event)
          if tracked_name:
            current_tracked_device_events[tracked_name].append(event)

    # Append Data for Last Frame.
    if current_service_events:
      all_service_events.append(current_service_events)
      for value in TRACKED_GL_CONTEXT_NAME.itervalues():
        tracked_events[(value, 'cpu')].append(
            current_tracked_service_events[value])
    if current_device_events:
      all_device_events.append(current_device_events)
      for value in TRACKED_GL_CONTEXT_NAME.itervalues():
        tracked_events[(value, 'gpu')].append(
            current_tracked_device_events[value])

    # Calculate Mean Frame Time for the CPU side.
    frame_times = []
    if all_service_events:
      prev_frame_end = all_service_events[0][0].start
      for event_list in all_service_events:
        last_service_event_in_frame = event_list[-1]
        frame_times.append(last_service_event_in_frame.end - prev_frame_end)
        prev_frame_end = last_service_event_in_frame.end

    # Create the timeline data dictionary for service side traces.
    total_frame_value = ('swap', None)
    cpu_frame_value = ('total', 'cpu')
    gpu_frame_value = ('total', 'gpu')
    timeline_data = {}
    timeline_data[total_frame_value] = frame_times
    timeline_data[cpu_frame_value] = _CPUFrameTimes(all_service_events)
    for value in TRACKED_GL_CONTEXT_NAME.itervalues():
      cpu_value = (value, 'cpu')
      timeline_data[cpu_value] = _CPUFrameTimes(tracked_events[cpu_value])

    # Add in GPU side traces if it was supported (IE. device traces exist).
    if all_device_events:
      timeline_data[gpu_frame_value] = _GPUFrameTimes(all_device_events)
      for value in TRACKED_GL_CONTEXT_NAME.itervalues():
        gpu_value = (value, 'gpu')
        tracked_gpu_event = tracked_events[gpu_value]
        timeline_data[gpu_value] = _GPUFrameTimes(tracked_gpu_event)

    return timeline_data
<commit_msg>[Telemetry] Change gpu_timeline to add the the interaction record label directly<commit_after># Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import math
import sys

from telemetry.timeline import model as model_module
from telemetry.value import improvement_direction
from telemetry.value import list_of_scalar_values
from telemetry.value import scalar
from telemetry.web_perf.metrics import timeline_based_metric

TOPLEVEL_GL_CATEGORY = 'gpu_toplevel'
TOPLEVEL_SERVICE_CATEGORY = 'disabled-by-default-gpu.service'
TOPLEVEL_DEVICE_CATEGORY = 'disabled-by-default-gpu.device'

SERVICE_FRAME_END_MARKER = (TOPLEVEL_SERVICE_CATEGORY, 'SwapBuffer')
DEVICE_FRAME_END_MARKER = (TOPLEVEL_DEVICE_CATEGORY, 'SwapBuffer')

TRACKED_GL_CONTEXT_NAME = {'RenderCompositor': 'render_compositor',
                           'BrowserCompositor': 'browser_compositor',
                           'Compositor': 'browser_compositor'}

def _CalculateFrameTimes(events_per_frame, event_data_func):
  """Given a list of events per frame and a function to extract event time data,
     returns a list of frame times."""
  times_per_frame = []
  for event_list in events_per_frame:
    event_times = [event_data_func(event) for event in event_list]
    times_per_frame.append(sum(event_times))
  return times_per_frame

def _CPUFrameTimes(events_per_frame):
  """Given a list of events per frame, returns a list of CPU frame times."""
  # CPU event frames are calculated using the event thread duration.
  # Some platforms do not support thread_duration, convert those to 0.
  return _CalculateFrameTimes(events_per_frame,
                              lambda event: event.thread_duration or 0)

def _GPUFrameTimes(events_per_frame):
  """Given a list of events per frame, returns a list of GPU frame times."""
  # GPU event frames are asynchronous slices which use the event duration.
  return _CalculateFrameTimes(events_per_frame,
                              lambda event: event.duration)

def TimelineName(name, source_type, value_type):
  """Constructs the standard name given in the timeline.

  Args:
    name: The name of the timeline, for example "total", or "render_compositor".
    source_type: One of "cpu", "gpu" or None. None is only used for total times.
    value_type: the type of value. For example "mean", "stddev"...etc.
  """
  if source_type:
    return '%s_%s_%s_time' % (name, value_type, source_type)
  else:
    return '%s_%s_time' % (name, value_type)

class GPUTimelineMetric(timeline_based_metric.TimelineBasedMetric):
  """Computes GPU based metrics."""

  def __init__(self):
    super(GPUTimelineMetric, self).__init__()

  def AddResults(self, model, _, interaction_records, results):
    self.VerifyNonOverlappedRecords(interaction_records)
    service_times = self._CalculateGPUTimelineData(model)
    for value_item, durations in service_times.iteritems():
      count = len(durations)
      avg = 0.0
      stddev = 0.0
      maximum = 0.0
      if count:
        avg = sum(durations) / count
        stddev = math.sqrt(sum((d - avg) ** 2 for d in durations) / count)
        maximum = max(durations)

      name, src = value_item

      if src:
        frame_times_name = '%s_%s_frame_times' % (name, src)
      else:
        frame_times_name = '%s_frame_times' % (name)

      if durations:
        results.AddValue(list_of_scalar_values.ListOfScalarValues(
            results.current_page, frame_times_name, 'ms', durations,
            tir_label=interaction_records[0].label,
            improvement_direction=improvement_direction.DOWN))

      results.AddValue(scalar.ScalarValue(
          results.current_page, TimelineName(name, src, 'max'), 'ms', maximum,
          tir_label=interaction_records[0].label,
          improvement_direction=improvement_direction.DOWN))
      results.AddValue(scalar.ScalarValue(
          results.current_page, TimelineName(name, src, 'mean'), 'ms', avg,
          tir_label=interaction_records[0].label,
          improvement_direction=improvement_direction.DOWN))
      results.AddValue(scalar.ScalarValue(
          results.current_page, TimelineName(name, src, 'stddev'), 'ms', stddev,
          tir_label=interaction_records[0].label,
          improvement_direction=improvement_direction.DOWN))

  def _CalculateGPUTimelineData(self, model):
    """Uses the model and calculates the times for various values for each
       frame. The return value will be a dictionary of the following format:
         {
           (EVENT_NAME1, SRC1_TYPE): [FRAME0_TIME, FRAME1_TIME...etc.],
           (EVENT_NAME2, SRC2_TYPE): [FRAME0_TIME, FRAME1_TIME...etc.],
         }

       Events:
         swap - The time in milliseconds between each swap marker.
         total - The amount of time spent in the renderer thread.
         TRACKED_NAMES: Using the TRACKED_GL_CONTEXT_NAME dict, we
                        include the traces per frame for the
                        tracked name.
       Source Types:
         None - This will only be valid for the "swap" event.
         cpu - For an event, the "cpu" source type signifies time spent on the
               gpu thread using the CPU. This uses the "gpu.service" markers.
         gpu - For an event, the "gpu" source type signifies time spent on the
               gpu thread using the GPU. This uses the "gpu.device" markers.
    """
    all_service_events = []
    current_service_frame_end = sys.maxint
    current_service_events = []

    all_device_events = []
    current_device_frame_end = sys.maxint
    current_device_events = []

    tracked_events = {}
    tracked_events.update(
        dict([((value, 'cpu'), [])
              for value in TRACKED_GL_CONTEXT_NAME.itervalues()]))
    tracked_events.update(
        dict([((value, 'gpu'), [])
              for value in TRACKED_GL_CONTEXT_NAME.itervalues()]))

    # These will track traces within the current frame.
    current_tracked_service_events = collections.defaultdict(list)
    current_tracked_device_events = collections.defaultdict(list)

    event_iter = model.IterAllEvents(
        event_type_predicate=model_module.IsSliceOrAsyncSlice)
    for event in event_iter:
      # Look for frame end markers
      if (event.category, event.name) == SERVICE_FRAME_END_MARKER:
        current_service_frame_end = event.end
      elif (event.category, event.name) == DEVICE_FRAME_END_MARKER:
        current_device_frame_end = event.end

      # Track all other toplevel gl category markers
      elif event.args.get('gl_category', None) == TOPLEVEL_GL_CATEGORY:
        base_name = event.name
        dash_index = base_name.rfind('-')
        if dash_index != -1:
          base_name = base_name[:dash_index]
        tracked_name = TRACKED_GL_CONTEXT_NAME.get(base_name, None)

        if event.category == TOPLEVEL_SERVICE_CATEGORY:
          # Check if frame has ended.
          if event.start >= current_service_frame_end:
            if current_service_events:
              all_service_events.append(current_service_events)
              for value in TRACKED_GL_CONTEXT_NAME.itervalues():
                tracked_events[(value, 'cpu')].append(
                    current_tracked_service_events[value])
            current_service_events = []
            current_service_frame_end = sys.maxint
            current_tracked_service_events.clear()

          current_service_events.append(event)
          if tracked_name:
            current_tracked_service_events[tracked_name].append(event)

        elif event.category == TOPLEVEL_DEVICE_CATEGORY:
          # Check if frame has ended.
          if event.start >= current_device_frame_end:
            if current_device_events:
              all_device_events.append(current_device_events)
              for value in TRACKED_GL_CONTEXT_NAME.itervalues():
                tracked_events[(value, 'gpu')].append(
                    current_tracked_device_events[value])
            current_device_events = []
            current_device_frame_end = sys.maxint
            current_tracked_device_events.clear()

          current_device_events.append(event)
          if tracked_name:
            current_tracked_device_events[tracked_name].append(event)

    # Append Data for Last Frame.
    if current_service_events:
      all_service_events.append(current_service_events)
      for value in TRACKED_GL_CONTEXT_NAME.itervalues():
        tracked_events[(value, 'cpu')].append(
            current_tracked_service_events[value])
    if current_device_events:
      all_device_events.append(current_device_events)
      for value in TRACKED_GL_CONTEXT_NAME.itervalues():
        tracked_events[(value, 'gpu')].append(
            current_tracked_device_events[value])

    # Calculate Mean Frame Time for the CPU side.
    frame_times = []
    if all_service_events:
      prev_frame_end = all_service_events[0][0].start
      for event_list in all_service_events:
        last_service_event_in_frame = event_list[-1]
        frame_times.append(last_service_event_in_frame.end - prev_frame_end)
        prev_frame_end = last_service_event_in_frame.end

    # Create the timeline data dictionary for service side traces.
    total_frame_value = ('swap', None)
    cpu_frame_value = ('total', 'cpu')
    gpu_frame_value = ('total', 'gpu')
    timeline_data = {}
    timeline_data[total_frame_value] = frame_times
    timeline_data[cpu_frame_value] = _CPUFrameTimes(all_service_events)
    for value in TRACKED_GL_CONTEXT_NAME.itervalues():
      cpu_value = (value, 'cpu')
      timeline_data[cpu_value] = _CPUFrameTimes(tracked_events[cpu_value])

    # Add in GPU side traces if it was supported (IE. device traces exist).
    if all_device_events:
      timeline_data[gpu_frame_value] = _GPUFrameTimes(all_device_events)
      for value in TRACKED_GL_CONTEXT_NAME.itervalues():
        gpu_value = (value, 'gpu')
        tracked_gpu_event = tracked_events[gpu_value]
        timeline_data[gpu_value] = _GPUFrameTimes(tracked_gpu_event)

    return timeline_data
<|endoftext|>


<commit_before><commit_msg>Libwhich: which for libraries. (#22461)<commit_after><|endoftext|>


<commit_before><commit_msg>Bump py-salib (#25403)<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

""" PulseBroker

    :copyright: (c) 2012 by Mozilla
    :license: MPLv2

    Assumes Python v2.6+

    Usage
        -c --config         Configuration file (json format)
                            default: None
        -r --redis          Redis server connection string
                            default: localhost:6379
           --redisdb        Redis database ID
                            default: 8
        -p --pulse          Pulse server connection string
                            default: None (i.e. Mozilla's Pulse)
        -t --topic          Pulse topic string
                            default: #
        -d --debug          Turn on debug logging
                            default: False
        -l --logpath        Path where the log file output is written
                            default: None
        -b --background     Fork to a daemon process
                            default: False

    Sample Configuration file

        { 'redis': 'localhost:6379',
          'debug': True,
          'logpath': '.'
        }

    Authors:
        bear    Mike Taylor <bear@mozilla.com>
"""

import os
import json
import time

from Queue import Empty
from multiprocessing import Process, Queue, get_logger

import zmq

from releng import initOptions, initLogs, dbRedis
from releng.constants import ID_PULSE_WORKER

from mozillapulse import consumers

appInfo    = 'bear@mozilla.com|briar-patch'
log        = get_logger()
eventQueue = Queue()

SERVER_CHECK_INTERVAL = 120  # how often, in minutes, to check for new servers
PING_FAIL_MAX         = 1    # how many pings can fail before server is marked inactive
PING_INTERVAL         = 120  # ping servers every 2 minutes
MSG_TIMEOUT           = 120  # 2 minutes until a pending message is considered expired

def OfflineTest(options):
    log.info('Starting Offline message testing')

    hArchive = open(options.testfile, 'r+')

    for msg in hArchive:
        job = json.loads(msg)
        pushJob(job)

    hArchive.close()

def cbMessage(data, message):
    """ cbMessage
    Parses the incoming pulse event and create a "job" that will be sent
    to a job processing server via ZeroMQ router.

    The job is placed into an event queue for async processing.
    """
    message.ack()

    routingKey = data['_meta']['routing_key']
    msgType    = routingKey.split('.')[0]
    payload    = data['payload']

    job              = {}
    job['master']    = data['_meta']['master_name']
    job['pulse_key'] = routingKey
    job['time']      = data['_meta']['sent']
    job['id']        = data['_meta']['message_id']
    job['pulse']     = data

    if msgType == 'build':
        if 'build' in payload:
            job['slave'] = payload['build']['slave']
            job['event'] = 'build'

            pushJob(job)

    elif msgType == 'slave':
        if 'slave' in payload:
            job['slave'] = payload['slave']['name']
            job['event'] = 'slave connect'
        else:
            job['slave'] = payload['slavename']
            job['event'] = 'slave disconnect'

        pushJob(job)

    elif msgType == 'change':
        job['event'] = 'source'
        pushJob(job)

class zmqService(object):
    def __init__(self, serverID, router, db, events):
        self.router   = router
        self.db       = db
        self.events   = events
        self.payload  = None
        self.expires  = None
        self.sequence = 0
        self.errors   = 0
        self.alive    = True
        self.lastPing = time.time()

        self.id      = serverID
        self.address = self.id.replace('%s:' % ID_PULSE_WORKER, '')
        if ':' not in self.address:
            self.address = '%s:5555' % self.address
        self.address = 'tcp://%s' % self.address

        log.debug('connecting to server %s' % self.address)

        self.router.connect(self.address)
        time.sleep(0.1)

    def isAvailable(self):
        return self.alive and self.payload is None

    def reply(self, reply):
        if options.debug:
            log.debug('reply %s' % self.id)

        sequenceReply = reply.pop(0)

        if options.debug:
            log.debug('recv %s [%s]' % (self.id, reply[0]))

        self.errors   = 0
        self.alive    = True
        self.lastPing = time.time()
        self.expires  = self.lastPing + MSG_TIMEOUT

        if int(sequenceReply) == self.sequence:
            self.payload = None
        else:
            log.error('reply received out of sequence')

    def request(self, msg):
        if options.debug:
            log.debug('request %s' % self.id)

        if self.isAvailable():
            self.sequence += 1
            self.payload   = [self.id, str(self.sequence), 'job', msg]
            self.expires   = time.time() + MSG_TIMEOUT

            if options.debug:
                log.debug('send %s %d chars [%s]' % (self.id, len(msg), msg[:42]))

            self.router.send_multipart(self.payload)

            return True
        else:
            return False

    def heartbeat(self):
        if self.payload is not None and time.time() > self.expires:
            if self.payload[2] == 'ping':
                log.warning('server %s has failed to respond to %d ping requests' % (self.id, self.errors))
                if self.errors >= PING_FAIL_MAX:
                    log.error('removing %s from server list' % self.id)
                    db.sadd('%s:inactive' % ID_PULSE_WORKER, self.id)
                else:
                    self.ping(force=True)
            else:
                log.warning('server %s has expired request: %s' % (self.id, ','.join(self.payload)))
                self.ping()

        if time.time() - self.lastPing > PING_INTERVAL:
            self.ping()

    def ping(self, force=False):
        if options.debug:
            log.debug('ping %s' % self.id)

        if force or self.isAvailable():
            self.sequence += 1
            self.payload   = [self.id, str(self.sequence), 'ping']
            self.lastPing  = time.time()
            self.expires   = self.lastPing + MSG_TIMEOUT
            self.errors   += 1
            self.alive     = False

            self.router.send_multipart(self.payload)
        else:
            log.warning('ping requested for offline service [%s]' % self.id)

def discoverServers(servers, db, events, router):
    for serverID in db.lrange(ID_PULSE_WORKER, 0, -1):
        if db.sismember('%s:inactive' % ID_PULSE_WORKER, serverID):
            log.warning('server %s found in inactive list, disconnecting' % serverID)
            if serverID in servers:
                servers[serverID] = None
                del servers[serverID]
        else:
            if serverID not in servers:
                log.debug('server %s is new, adding to connect queue' % serverID)
                servers[serverID] = zmqService(serverID, router, db, events)

def handleZMQ(options, events, db):
    """ handleZMQ
    Primary event loop for everything ZeroMQ related

    All payloads to be sent onward arrive via the event queue.

    Currently it is a very simple implementation of the Freelance
    pattern with no server heartbeat checks.

    The incoming events are structured as a list that always
    begins with the event type.

        Job:            ('job',  "{'payload': 'sample'}")
        Heartbeat:      ('ping',)

    The structure of the message sent between nodes is:

        [destination, sequence, control, payload]

    all items are sent as strings.
    """
    log.info('starting')

    servers       = {}
    lastDiscovery = time.time()

    context = zmq.Context()
    router  = context.socket(zmq.ROUTER)
    poller  = zmq.Poller()
    poller.register(router, zmq.POLLIN)

    while True:
        available = False
        for serverID in servers:
            if servers[serverID].isAvailable():
                available = True
                break

        try:
            event = events.get(False)
        except Empty:
            event = None

        if event is not None:
            if available:
                eventType = event[0]

                if eventType == 'exit':
                    log.info('exit command received, terminating')
                    break

                if eventType == 'ping':
                    ping(servers, event[1])

                elif eventType == 'job':
                    handled = False
                    for serverID in servers:
                        if servers[serverID].isAvailable() and servers[serverID].request(event[1]):
                            handled = True
                            break
                    if not handled:
                        log.error('no active servers to handle request')
                        # TODO - push archived item to redis

                else:
                    log.warning('unknown event [%s]' % eventType)
            else:
                log.error('no active servers to handle request')

        try:
            items = dict(poller.poll(100))
        except:
            break

        if router in items:
            reply    = router.recv_multipart()
            serverID = reply.pop(0)
            servers[serverID].reply(reply)
        else:
            for serverID in servers:
                servers[serverID].heartbeat()

        if time.time() > lastDiscovery:
            discoverServers(servers, db, events, router)
            lastDiscovery = time.time() + SERVER_CHECK_INTERVAL

    log.info('done')

def pushJob(job):
    s = json.dumps(job)
    eventQueue.put(('job', s))

_defaultOptions = { 'config':      ('-c', '--config',     None,             'Configuration file'),
                    'debug':       ('-d', '--debug',      True,             'Enable Debug', 'b'),
                    'appinfo':     ('-a', '--appinfo',    appInfo,          'Mozilla Pulse app string'),
                    'background':  ('-b', '--background', False,            'daemonize ourselves', 'b'),
                    'logpath':     ('-l', '--logpath',    None,             'Path where log file is to be written'),
                    'redis':       ('-r', '--redis',      'localhost:6379', 'Redis connection string'),
                    'redisdb':     ('',   '--redisdb',    '8',              'Redis database'),
                    'pulse':       ('-p', '--pulse',      None,             'Pulse connection string'),
                    'topic':       ('-t', '--topic',      '#',              'Mozilla Pulse Topic filter string'),
                    'testfile':    ('',   '--testfile',   None,             'Offline testing, uses named file instead of Pulse server'),
                  }

if __name__ == '__main__':
    options = initOptions(params=_defaultOptions)
    initLogs(options)

    log.info('Starting')

    log.info('Connecting to datastore')
    db = dbRedis(options)

    log.info('Creating ZeroMQ handler')
    Process(name='zmq', target=handleZMQ, args=(options, eventQueue, db)).start()

    if options.testfile:
        OfflineTest(options)
    else:
        try:
            log.info('Connecting to Mozilla Pulse with topic "%s"' % options.topic)
            pulse = consumers.BuildConsumer(applabel=options.appinfo)
            pulse.configure(topic=options.topic, callback=cbMessage)

            log.debug('Starting pulse.listen()')
            pulse.listen()
        except:
            log.error('Pulse Exception', exc_info=True)
            eventQueue.put(('exit',)

<commit_msg>remove 'b' from default options dictionary to sync up with initOptions() change and fix typo that removed final paren<commit_after>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

""" PulseBroker

    :copyright: (c) 2012 by Mozilla
    :license: MPLv2

    Assumes Python v2.6+

    Usage
        -c --config         Configuration file (json format)
                            default: None
        -r --redis          Redis server connection string
                            default: localhost:6379
           --redisdb        Redis database ID
                            default: 8
        -p --pulse          Pulse server connection string
                            default: None (i.e. Mozilla's Pulse)
        -t --topic          Pulse topic string
                            default: #
        -d --debug          Turn on debug logging
                            default: False
        -l --logpath        Path where the log file output is written
                            default: None
        -b --background     Fork to a daemon process
                            default: False

    Sample Configuration file

        { 'redis': 'localhost:6379',
          'debug': True,
          'logpath': '.'
        }

    Authors:
        bear    Mike Taylor <bear@mozilla.com>
"""

import os
import json
import time

from Queue import Empty
from multiprocessing import Process, Queue, get_logger

import zmq

from releng import initOptions, initLogs, dbRedis
from releng.constants import ID_PULSE_WORKER

from mozillapulse import consumers

appInfo    = 'bear@mozilla.com|briar-patch'
log        = get_logger()
eventQueue = Queue()

SERVER_CHECK_INTERVAL = 120  # how often, in minutes, to check for new servers
PING_FAIL_MAX         = 1    # how many pings can fail before server is marked inactive
PING_INTERVAL         = 120  # ping servers every 2 minutes
MSG_TIMEOUT           = 120  # 2 minutes until a pending message is considered expired

def OfflineTest(options):
    log.info('Starting Offline message testing')

    hArchive = open(options.testfile, 'r+')

    for msg in hArchive:
        job = json.loads(msg)
        pushJob(job)

    hArchive.close()

def cbMessage(data, message):
    """ cbMessage
    Parses the incoming pulse event and create a "job" that will be sent
    to a job processing server via ZeroMQ router.

    The job is placed into an event queue for async processing.
    """
    message.ack()

    routingKey = data['_meta']['routing_key']
    msgType    = routingKey.split('.')[0]
    payload    = data['payload']

    job              = {}
    job['master']    = data['_meta']['master_name']
    job['pulse_key'] = routingKey
    job['time']      = data['_meta']['sent']
    job['id']        = data['_meta']['message_id']
    job['pulse']     = data

    if msgType == 'build':
        if 'build' in payload:
            job['slave'] = payload['build']['slave']
            job['event'] = 'build'

            pushJob(job)

    elif msgType == 'slave':
        if 'slave' in payload:
            job['slave'] = payload['slave']['name']
            job['event'] = 'slave connect'
        else:
            job['slave'] = payload['slavename']
            job['event'] = 'slave disconnect'

        pushJob(job)

    elif msgType == 'change':
        job['event'] = 'source'
        pushJob(job)

class zmqService(object):
    def __init__(self, serverID, router, db, events):
        self.router   = router
        self.db       = db
        self.events   = events
        self.payload  = None
        self.expires  = None
        self.sequence = 0
        self.errors   = 0
        self.alive    = True
        self.lastPing = time.time()

        self.id      = serverID
        self.address = self.id.replace('%s:' % ID_PULSE_WORKER, '')
        if ':' not in self.address:
            self.address = '%s:5555' % self.address
        self.address = 'tcp://%s' % self.address

        log.debug('connecting to server %s' % self.address)

        self.router.connect(self.address)
        time.sleep(0.1)

    def isAvailable(self):
        return self.alive and self.payload is None

    def reply(self, reply):
        if options.debug:
            log.debug('reply %s' % self.id)

        sequenceReply = reply.pop(0)

        if options.debug:
            log.debug('recv %s [%s]' % (self.id, reply[0]))

        self.errors   = 0
        self.alive    = True
        self.lastPing = time.time()
        self.expires  = self.lastPing + MSG_TIMEOUT

        if int(sequenceReply) == self.sequence:
            self.payload = None
        else:
            log.error('reply received out of sequence')

    def request(self, msg):
        if options.debug:
            log.debug('request %s' % self.id)

        if self.isAvailable():
            self.sequence += 1
            self.payload   = [self.id, str(self.sequence), 'job', msg]
            self.expires   = time.time() + MSG_TIMEOUT

            if options.debug:
                log.debug('send %s %d chars [%s]' % (self.id, len(msg), msg[:42]))

            self.router.send_multipart(self.payload)

            return True
        else:
            return False

    def heartbeat(self):
        if self.payload is not None and time.time() > self.expires:
            if self.payload[2] == 'ping':
                log.warning('server %s has failed to respond to %d ping requests' % (self.id, self.errors))
                if self.errors >= PING_FAIL_MAX:
                    log.error('removing %s from server list' % self.id)
                    db.sadd('%s:inactive' % ID_PULSE_WORKER, self.id)
                else:
                    self.ping(force=True)
            else:
                log.warning('server %s has expired request: %s' % (self.id, ','.join(self.payload)))
                self.ping()

        if time.time() - self.lastPing > PING_INTERVAL:
            self.ping()

    def ping(self, force=False):
        if options.debug:
            log.debug('ping %s' % self.id)

        if force or self.isAvailable():
            self.sequence += 1
            self.payload   = [self.id, str(self.sequence), 'ping']
            self.lastPing  = time.time()
            self.expires   = self.lastPing + MSG_TIMEOUT
            self.errors   += 1
            self.alive     = False

            self.router.send_multipart(self.payload)
        else:
            log.warning('ping requested for offline service [%s]' % self.id)

def discoverServers(servers, db, events, router):
    for serverID in db.lrange(ID_PULSE_WORKER, 0, -1):
        if db.sismember('%s:inactive' % ID_PULSE_WORKER, serverID):
            log.warning('server %s found in inactive list, disconnecting' % serverID)
            if serverID in servers:
                servers[serverID] = None
                del servers[serverID]
        else:
            if serverID not in servers:
                log.debug('server %s is new, adding to connect queue' % serverID)
                servers[serverID] = zmqService(serverID, router, db, events)

def handleZMQ(options, events, db):
    """ handleZMQ
    Primary event loop for everything ZeroMQ related

    All payloads to be sent onward arrive via the event queue.

    Currently it is a very simple implementation of the Freelance
    pattern with no server heartbeat checks.

    The incoming events are structured as a list that always
    begins with the event type.

        Job:            ('job',  "{'payload': 'sample'}")
        Heartbeat:      ('ping',)

    The structure of the message sent between nodes is:

        [destination, sequence, control, payload]

    all items are sent as strings.
    """
    log.info('starting')

    servers       = {}
    lastDiscovery = time.time()

    context = zmq.Context()
    router  = context.socket(zmq.ROUTER)
    poller  = zmq.Poller()
    poller.register(router, zmq.POLLIN)

    while True:
        available = False
        for serverID in servers:
            if servers[serverID].isAvailable():
                available = True
                break

        try:
            event = events.get(False)
        except Empty:
            event = None

        if event is not None:
            if available:
                eventType = event[0]

                if eventType == 'exit':
                    log.info('exit command received, terminating')
                    break

                if eventType == 'ping':
                    ping(servers, event[1])

                elif eventType == 'job':
                    handled = False
                    for serverID in servers:
                        if servers[serverID].isAvailable() and servers[serverID].request(event[1]):
                            handled = True
                            break
                    if not handled:
                        log.error('no active servers to handle request')
                        # TODO - push archived item to redis

                else:
                    log.warning('unknown event [%s]' % eventType)
            else:
                log.error('no active servers to handle request')

        try:
            items = dict(poller.poll(100))
        except:
            break

        if router in items:
            reply    = router.recv_multipart()
            serverID = reply.pop(0)
            servers[serverID].reply(reply)
        else:
            for serverID in servers:
                servers[serverID].heartbeat()

        if time.time() > lastDiscovery:
            discoverServers(servers, db, events, router)
            lastDiscovery = time.time() + SERVER_CHECK_INTERVAL

    log.info('done')

def pushJob(job):
    s = json.dumps(job)
    eventQueue.put(('job', s))

_defaultOptions = { 'config':      ('-c', '--config',     None,             'Configuration file'),
                    'debug':       ('-d', '--debug',      True,             'Enable Debug'),
                    'appinfo':     ('-a', '--appinfo',    appInfo,          'Mozilla Pulse app string'),
                    'background':  ('-b', '--background', False,            'daemonize ourselves'),
                    'logpath':     ('-l', '--logpath',    None,             'Path where log file is to be written'),
                    'redis':       ('-r', '--redis',      'localhost:6379', 'Redis connection string'),
                    'redisdb':     ('',   '--redisdb',    '8',              'Redis database'),
                    'pulse':       ('-p', '--pulse',      None,             'Pulse connection string'),
                    'topic':       ('-t', '--topic',      '#',              'Mozilla Pulse Topic filter string'),
                    'testfile':    ('',   '--testfile',   None,             'Offline testing, uses named file instead of Pulse server'),
                  }

if __name__ == '__main__':
    options = initOptions(params=_defaultOptions)
    initLogs(options)

    log.info('Starting')

    log.info('Connecting to datastore')
    db = dbRedis(options)

    log.info('Creating ZeroMQ handler')
    Process(name='zmq', target=handleZMQ, args=(options, eventQueue, db)).start()

    if options.testfile:
        OfflineTest(options)
    else:
        try:
            log.info('Connecting to Mozilla Pulse with topic "%s"' % options.topic)
            pulse = consumers.BuildConsumer(applabel=options.appinfo)
            pulse.configure(topic=options.topic, callback=cbMessage)

            log.debug('Starting pulse.listen()')
            pulse.listen()
        except:
            log.error('Pulse Exception', exc_info=True)
            eventQueue.put(('exit',))
<|endoftext|>


<commit_before>#!/usr/bin/python

import glob
import re
import os
from html.parser import HTMLParser

stopparsing = False

html = {}
css = {}

class MyHTMLParser(HTMLParser):

    def __init__(self, afile=None, cssdict=None, **kwds):
        super(MyHTMLParser, self).__init__(**kwds)
        self.afile = afile
        self.cssclasses = cssdict
        self.cssfound = {}

    def handle_starttag(self, tag, attrs):

        if tag == 'link':

            for attr in attrs:

                if (attr[0] == 'href'):

                    extension = os.path.splitext(attr[1])[1]
                    if (extension == '.css'):

                        cssfile = os.path.abspath((userdir + attr[1]))

                        self.cssfound[cssfile] = {}

                        csshandle = open(cssfile, "r")
                        csscontent = csshandle.readlines()
                        csslinenum = 0
                        for line in csscontent:

                            words = line.split()
                            for word in words:
                                if word.startswith(".") or word.startswith("#"):
                                    self.cssfound[cssfile][word] = csslinenum

                            csslinenum = csslinenum + 1

        if self.cssfound:
            css[self.afile] = self.cssfound

        return css

    def handle_endtag(self, tag):
        if tag == 'head':
            stopparsing = True

userdir = input("Enter a directory to search\n")
userexts = input("Enter a comma separated list of file extensions to search in e.g php,html\n")

exts = userexts.split(",")

for ext in exts:
    print ("Searching file extension... ", ext)

    files = glob.glob(userdir + "*." + ext)
    print("number of files searched:", len(files))
    for file in files:

        parser = MyHTMLParser(file, css, strict=False)

        filecss = {}

        handle = open(file, "r")
        content = handle.readlines()
        linenum = 0
        cssfound = False

        for line in content:

            if not stopparsing:
                parser.feed(line)

            classes = re.findall(r'class=\"(.+?)\"', line)
            if classes:
                cssfound = True
                for htmlclass in classes:
                    htmlclass = "." + htmlclass
                    filecss[htmlclass] = linenum

            ids = re.findall(r'id=\"(.+?)\"', line)
            if ids:
                cssfound = True
                for htmlid in ids:
                    htmlid = "#" + htmlid
                    filecss[htmlid] = linenum

            linenum = linenum + 1

        if cssfound:
            html[file] = filecss

print("html list is ", set(html.keys()))
print("css list is ", set(css.keys()))

print(html)
print("dict of files with linked css files and classes/line numbers therein", css)

for fileid,cssitems in css.items():
    print("    The file is",fileid)
    # loop through the html file ids and classes here
    for k,v in cssitems.items():
        print("        linked css file is .",k, sep='')
        for c,l in v.items():
            print("            css is ",c, " found on line ", l, sep='')

htmlfiles = set(html.keys())
cssfiles = set(css.keys())
combinedfiles = cssfiles.intersection(htmlfiles)
print("in both", combinedfiles)<commit_msg>Comparing classes and ids used in html and external css and producing lists of used/unused css for each html file.<commit_after>#!/usr/bin/python

import glob
import re
import os
from html.parser import HTMLParser

stopparsing = False

html = {}
css = {}

class MyHTMLParser(HTMLParser):

    def __init__(self, afile=None, cssdict=None, **kwds):
        super(MyHTMLParser, self).__init__(**kwds)
        self.afile = afile
        self.cssclasses = cssdict
        self.cssfound = {}

    def handle_starttag(self, tag, attrs):

        if tag == 'link':

            for attr in attrs:

                if (attr[0] == 'href'):

                    extension = os.path.splitext(attr[1])[1]
                    if (extension == '.css'):

                        cssfile = os.path.abspath((userdir + attr[1]))

                        self.cssfound[cssfile] = {}

                        csshandle = open(cssfile, "r")
                        csscontent = csshandle.readlines()
                        csslinenum = 0
                        for line in csscontent:

                            words = line.split()
                            for word in words:
                                if word.startswith(".") or word.startswith("#"):
                                    self.cssfound[cssfile][word] = csslinenum

                            csslinenum = csslinenum + 1

        if self.cssfound:
            css[self.afile] = self.cssfound

        return css

    def handle_endtag(self, tag):
        if tag == 'head':
            stopparsing = True

userdir = input("Enter a directory to search\n")
userexts = input("Enter a comma separated list of file extensions to search in e.g php,html\n")

exts = userexts.split(",")

for ext in exts:
    print ("Searching file extension... ", ext)

    files = glob.glob(userdir + "*." + ext)
    print("number of files searched:", len(files))
    for file in files:

        parser = MyHTMLParser(file, css, strict=False)

        filecss = {}

        handle = open(file, "r")
        content = handle.readlines()
        linenum = 0
        cssfound = False

        for line in content:

            if not stopparsing:
                parser.feed(line)

            classes = re.findall(r'class=\"(.+?)\"', line)
            if classes:
                cssfound = True
                for htmlclass in classes:
                    htmlclass = "." + htmlclass
                    filecss[htmlclass] = linenum

            ids = re.findall(r'id=\"(.+?)\"', line)
            if ids:
                cssfound = True
                for htmlid in ids:
                    htmlid = "#" + htmlid
                    filecss[htmlid] = linenum

            linenum = linenum + 1

        if cssfound:
            html[file] = filecss

#print("html list is ", set(html.keys()))
#print("css list is ", set(css.keys()))

# print(html) # dict of files with html classes and/or ids
# print("dict of files with linked css files and classes/line numbers therein", css)

# for fileid,cssitems in css.items():
#     print("    The file is",fileid)
#     # loop through the html file ids and classes here?
#     for k,v in cssitems.items():
#         print("        linked css file is .",k, sep='')
#         for c,l in v.items():
#             print("            css is ",c, " found on line ", l, sep='')

for file in html:

    print("processing", file)
    used = []
    unused = []

    if file in css:
        #print("base file is", file)
        for c,s in css[file].items():
            #print("    css file linked to is", c)

            for i,l in s.items():
                #print("        css class/id found :", i, "on line", l)
                if i in html[file]:
                    used.append(i)
                    #print("            ", html[file])
                else:
                    unused.append(i)
                    #print("            css not used in base file")
    if used:
        print("used css:")
        for l,u in enumerate(used):
            print("    ", u, "on line", l)
    if unused:
        print("unused css:")
        for l,u in enumerate(unused):
            print("    ", u, "on line", l)

# htmlfiles = set(html.keys())
# cssfiles = set(css.keys())
# combinedfiles = cssfiles.intersection(htmlfiles)
# print("in both", combinedfiles)
#now we can compare classes/ids using combinedfiles indices as css/html dict indices?<|endoftext|>