<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.agenda.signals
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Signals for the agenda app.

    :copyright: (c) 2011–2013 by the OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

from django.dispatch import receiver
from django import forms
from django.utils.translation import ugettext_lazy, ugettext_noop, ugettext as _
from django.template.loader import render_to_string

from openslides.config.signals import config_signal
from openslides.config.api import config, ConfigVariable, ConfigPage

from openslides.projector.signals import projector_overlays
from openslides.projector.projector import Overlay
from openslides.projector.api import (get_active_slide, get_slide_from_sid,
                                      clear_projector_cache)

from .models import Speaker, Item

# TODO: Reinsert the datepicker scripts in the template

@receiver(config_signal, dispatch_uid='setup_agenda_config_page')
def setup_agenda_config_page(sender, **kwargs):
    """
    Agenda config variables.
    """
    # TODO: Insert validator for the format or use other field carefully.
    agenda_start_event_date_time = ConfigVariable(
        name='agenda_start_event_date_time',
        default_value='',
        form_field=forms.CharField(
            widget=forms.DateTimeInput(format='%d.%m.%Y %H:%M'),
            required=False,
            label=_('Begin of event'),
            help_text=_('Input format: DD.MM.YYYY HH:MM')))

    agenda_show_last_speakers = ConfigVariable(
        name='agenda_show_last_speakers',
        default_value=1,
        form_field=forms.IntegerField(
            min_value=0,
            label=_('Number of last speakers to be shown on the projector')))

    extra_stylefiles = ['styles/timepicker.css', 'styles/jquery-ui/jquery-ui.custom.min.css']
    extra_javascript = ['javascript/jquery-ui.custom.min.js',
                        'javascript/jquery-ui-timepicker-addon.min.js',
                        'javascript/jquery-ui-sliderAccess.min.js',
                        'javascript/agenda-config-datepicker.js']

    return ConfigPage(title=ugettext_noop('Agenda'),
                      url='agenda',
                      required_permission='config.can_manage',
                      weight=20,
                      variables=(agenda_start_event_date_time, agenda_show_last_speakers),
                      extra_context={'extra_stylefiles': extra_stylefiles,
                                     'extra_javascript': extra_javascript})

@receiver(projector_overlays, dispatch_uid="agenda_list_of_speakers")
def agenda_list_of_speakers(sender, **kwargs):
    """
    Receiver for the list of speaker overlay.
    """
    name = 'agenda_speaker'

    def get_widget_html():
        """
        Returns the the html-code to show in the overly-widget.
        """
        return render_to_string('agenda/overlay_speaker_widget.html')

    def get_projector_html():
        """
        Returns an html-code to show on the projector.
        """
        slide = get_slide_from_sid(get_active_slide(only_sid=True), element=True)
        if not isinstance(slide, Item):
            # Only show list of speakers overlay on agenda items
            return None
        if config['presentation_argument'] == 'show_list_of_speakers':
            # Do not show list of speakers overlay on the list of speakers slide
            return None
        clear_projector_cache()
        list_of_speakers = slide.get_list_of_speakers(
            old_speakers_count=config['agenda_show_last_speakers'],
            coming_speakers_count=5)
        context = {
            'list_of_speakers': list_of_speakers,
            'closed': slide.speaker_list_closed,
        }
        return render_to_string('agenda/overlay_speaker_projector.html', context)

    return Overlay(name, get_widget_html, get_projector_html)
<commit_msg>Validator for agenda_start_time aded. PEP8 issue 'missing whitespace after ','' resolved<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.agenda.signals
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Signals for the agenda app.

    :copyright: (c) 2011–2013 by the OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

from datetime import datetime

from django.dispatch import receiver
from django import forms
from django.utils.translation import ugettext_lazy, ugettext_noop, ugettext as _
from django.template.loader import render_to_string
from django.core.exceptions import ValidationError

from openslides.config.signals import config_signal
from openslides.config.api import config, ConfigVariable, ConfigPage

from openslides.projector.signals import projector_overlays
from openslides.projector.projector import Overlay
from openslides.projector.api import (get_active_slide, get_slide_from_sid,
                                      clear_projector_cache)

from .models import Speaker, Item

def validate_start_time(value):
    try:
        datetime.strptime(value, '%d.%m.%Y %H:%M')
    except ValueError:
        raise ValidationError(_('Invalid input.'))

# TODO: Reinsert the datepicker scripts in the template

@receiver(config_signal, dispatch_uid='setup_agenda_config_page')
def setup_agenda_config_page(sender, **kwargs):
    """
    Agenda config variables.
    """
    # TODO: Insert validator for the format or use other field carefully.
    agenda_start_event_date_time = ConfigVariable(
        name='agenda_start_event_date_time',
        default_value='',
        form_field=forms.CharField(
            validators=[validate_start_time, ],
            widget=forms.DateTimeInput(format='%d.%m.%Y %H:%M'),
            required=False,
            label=_('Begin of event'),
            help_text=_('Input format: DD.MM.YYYY HH:MM')))

    agenda_show_last_speakers = ConfigVariable(
        name='agenda_show_last_speakers',
        default_value=1,
        form_field=forms.IntegerField(
            min_value=0,
            label=_('Number of last speakers to be shown on the projector')))

    extra_stylefiles = ['styles/timepicker.css', 'styles/jquery-ui/jquery-ui.custom.min.css']
    extra_javascript = ['javascript/jquery-ui.custom.min.js',
                        'javascript/jquery-ui-timepicker-addon.min.js',
                        'javascript/jquery-ui-sliderAccess.min.js',
                        'javascript/agenda-config-datepicker.js']

    return ConfigPage(title=ugettext_noop('Agenda'),
                      url='agenda',
                      required_permission='config.can_manage',
                      weight=20,
                      variables=(agenda_start_event_date_time, agenda_show_last_speakers),
                      extra_context={'extra_stylefiles': extra_stylefiles,
                                     'extra_javascript': extra_javascript})

@receiver(projector_overlays, dispatch_uid="agenda_list_of_speakers")
def agenda_list_of_speakers(sender, **kwargs):
    """
    Receiver for the list of speaker overlay.
    """
    name = 'agenda_speaker'

    def get_widget_html():
        """
        Returns the the html-code to show in the overly-widget.
        """
        return render_to_string('agenda/overlay_speaker_widget.html')

    def get_projector_html():
        """
        Returns an html-code to show on the projector.
        """
        slide = get_slide_from_sid(get_active_slide(only_sid=True), element=True)
        if not isinstance(slide, Item):
            # Only show list of speakers overlay on agenda items
            return None
        if config['presentation_argument'] == 'show_list_of_speakers':
            # Do not show list of speakers overlay on the list of speakers slide
            return None
        clear_projector_cache()
        list_of_speakers = slide.get_list_of_speakers(
            old_speakers_count=config['agenda_show_last_speakers'],
            coming_speakers_count=5)
        context = {
            'list_of_speakers': list_of_speakers,
            'closed': slide.speaker_list_closed,
        }
        return render_to_string('agenda/overlay_speaker_projector.html', context)

    return Overlay(name, get_widget_html, get_projector_html)
<|endoftext|>


<commit_before>from xml.dom import minidom
from types import *
import datetime
import csv
import os
import re
import sqlite3
import copy
import unicodedata

class XMLPatent(object):
    """
    Base class for handling parsing of xml fields
    """

    def __init__(self, xmlstring):
        """
        [xmlstring]: string containing xml document. Should be provided by
        parse.py
        """
        # lowercase all tags
        self.xmlstring = re.sub(r"<[/]?[A-Za-z-]+?[ >]", lambda x: x.group().lower(), xmlstring)
        # store the minidom parsed xml doc
        self.xmldoc = minidom.parseString(self.xmlstring)

        # country, patent, kind, date_grant
        self.country, self.patent, self.kind, self.date_grant = self.__tagNme__(xmldoc, ["publication-reference", ["country", "doc-number", "kind", "date"]])
        # pat_type
        self.pat_type = self.__tagNme__(xmldoc, ["application-reference"], iHTML=False)[0].attributes["appl-type"].value
        # date_app, country_app, patent_app
        self.date_app, self.country_app, self.patent_app = self.__tagNme__(xmldoc, ["application-reference", ["date", "country", "doc-number"]])
        # code_app
        self.code_app = self.__tagNme__(xmldoc, ["us-application-series-code"])
        # clm_num
        self.clm_num = self.__tagNme__(xmldoc, ["number-of-claims"])
        # classes
        self.classes = [[x[:3].replace(' ',''), x[3:].replace(' ','')] for x in self.__tagNme__(xmldoc, ["classification-national", ["main-classification", "further-classification"]], idx=1, listType=True)]
        # abstract
        self.abstract = self.__allHTML__(xmldoc, ["abstract", "p"])
        # invention_title
        self.invention_title = self.__allHTML__(xmldoc, ["invention-title"])
        # asg_list
        self.asg_list = self.__asg_detail__(self.__tagNme__(xmldoc, ["assignees", "assignee"], iHTML=False))
        # cit_list
        self.cit_list = self.__cit_detail__(self.__tagNme__(xmldoc, ["references-cited", "citation"], iHTML=False))
        # rel_list
        self.rel_list = self.__rel_detail__(self.__tagNme__(xmldoc, ["us-related-documents"], iHTML=False))
        # inv_list
        self.inv_list = self.__tagSplit__(xmldoc, ["parties", "applicant"], [["addressbook", ["last-name", "first-name"]], ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], [["nationality", "residence"], "country"]], blank=True)
        # law_list
        self.law_list = self.__tagSplit__(xmldoc, ["parties", "agents", "agent"], [["addressbook", ["last-name", "first-name", "country", "orgname"]]], blank=True)

    """
    These are all methods from the old XMLPatent class. I'm keeping them
    in here to assist in the carry-over of parsing the xml fields
    """
    def __allHTML__(self, xmldoc, tagList):
        """
        Replaces the html tags for everything returned by __tagName__(taglist)
        with nothing
        """
        for x in self.__tagNme__(xmldoc, tagList, iHTML=False):
            return re.sub(r"<[/]?%s( .*?)?>" % (tagList[-1]), "", x.toxml())
        return ""

    def __innerHTML__(self, dom_element):
        """
        Gets the text out of all child nodes for a certain
        dom_element
        """
        #if blank return nothing as well!
        if dom_element == '':
            return ''
        else:
            rc = ""
            for node in dom_element.childNodes:
                if node.nodeType == node.TEXT_NODE:
                    rc = rc + node.data
            return rc

    def __tagSplit__(self, xmldoc, xmlList, tagList, baseList=[], idx=0, blank=False, iHTML=True, debug=False):
        """
        For each item in the list of xml tags returned by tagName, we treat each tag's contents
        as its own xml doc and search it for a list of xml tags. Everyything is appended together
        and returned
        """
        d_list = []
        for x in self.__tagNme__(xmldoc, tagList=xmlList, iHTML=False):
            record = copy.copy(baseList)
            for y in tagList:
                if debug:
                    print "-------------------"
                    print x.toxml()
                    print y
                record.extend(self.__tagNme__(x, tagList=y, blank=blank, iHTML=iHTML, debug=debug, idx=idx))
            d_list.append(record)
        return d_list

    def __tagNme__(self, xmldoc, tagList, idx=0, listType=False, blank=False, iHTML=True, debug=False):
        """
        takes in the xml.minidom parsed xml document [xmldoc] and a list of tags [tagList]
        [tagList] may sometimes be a list of lists?
        We iterate the taglist, searching another level deeper into the xml doc with each
        iteration. In tagList, index 0 is the first level of the xmldoc, index 1 is the list
        of all tags we look for at the level right under all the tags we found at index 0,
        and so on.
        For each item, we then iterate through all xmldocs we passed in (usually one?)
        and append the tag we wanted to an xmllist.
        We have empty strings if the tag was not found, and always return a flat list.
        **Returns:
        if the iHTML flag is set, we do the innerHTML method on the xmllist, otherwise
        we just return the xmllist.
        """
        xmldoc = [xmldoc]
        for i,x in enumerate(tagList):
            if type(x) is not ListType:
                x = [x]
            xmlnext = []
            for y in x:
                for z in xmldoc:
                    if z != '':
                        if i==0 and idx!=0:
                            if len(z.getElementsByTagName(y))>0:
                                xmlnext.append(z.getElementsByTagName(y)[idx-1])
                            else:
                                xmlnext.append('')
                        else:
                            blFlag = False
                            for za in z.getElementsByTagName(y):
                                blFlag = True
                                xmlnext.append(za)
                            if blFlag==False and blank==True:
                                xmlnext.append('')
                    else:
                        xmlnext.append('')
            xmldoc = xmlnext

        if debug:
            if len(xmldoc)==1 and iHTML:
                print self.__innerHTML__(xmldoc[0])
            elif iHTML:
                print [self.__innerHTML__(x) for x in xmldoc]
            else:
                print xmldoc

        if len(xmldoc)==1 and iHTML:
            if listType:
                return [self.__innerHTML__(xmldoc[0])]
            else:
                return self.__innerHTML__(xmldoc[0])
        elif iHTML:
            return [self.__innerHTML__(x) for x in xmldoc]
        else:
            return xmldoc

    def __asg_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            record = []
            if len(x.getElementsByTagName("first-name"))>0:
                record = [1]
                record.extend(self.__tagNme__(x, [["last-name", "first-name"]]))
            else:
                record = [0]
                record.extend(self.__tagNme__(x, [["orgname", "role"]]))
            record.extend(self.__tagNme__(x, ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], blank=True))
            record.extend(self.__tagNme__(x, [["nationality", "residence"], "country"], blank=True))
            d_list.append(record)
        return d_list

    def __cit_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            #this means patcit is part of the XML
            record = [self.__tagNme__(x, ["category"])]
            if len(x.getElementsByTagName("patcit"))>0:
                record.extend(self.__tagNme__(x, ["patcit", ["country", "doc-number", "date", "kind", "name"]], blank=True))
                record.extend([""])
            elif len(x.getElementsByTagName("othercit"))>0:
                record.extend(["", "", "", "", ""])
                record.extend([self.__allHTML__(x, ["othercit"])])
                #probably should grab date information
            else:
                print x.toxml()
            d_list.append(record)
        return d_list

    def __rel_detail__(self, xmldoc, debug=False):
        d_list = []
        for x in xmldoc:
            for y in ["continuation-in-part", "continuation", "division", "reissue"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["relation", "child-doc"], [[["doc-number", "country", "kind"]]], baseList=[y, -1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True, idx=1))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-grant-document"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-pct-document"],  [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
            for y in ["related-publication", "us-provisional-application"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["document-id"], [[["doc-number", "country", "kind"]]], baseList=[y, 0], blank=True))
            if debug:
                print "-------------------"
                for x in d_list:
                    print x
        return d_list

    def __repr__(self):
        return \
"""country = %s, patent = %s, pat_type = %s,
date_grant = %s, date_app = %s,
         abstract = %s
  invention_title = %s
len(classes, asg, cit, ret, inv, law) = %s""" % (self.country, self.patent, self.pat_type, self.date_grant, self.date_app,
       self.abstract[:50], self.invention_title[:50],
       str([len(self.classes),  len(self.asg_list), len(self.cit_list),
            len(self.rel_list), len(self.inv_list), len(self.law_list)]))
<commit_msg>add build_table methods<commit_after>from xml.dom import minidom
from types import *
import datetime
import csv
import os
import re
import sqlite3
import copy
import unicodedata

class XMLPatent(object):
    """
    Base class for handling parsing of xml fields
    """

    def __init__(self, xmlstring):
        """
        [xmlstring]: string containing xml document. Should be provided by
        parse.py
        """
        # lowercase all tags
        self.xmlstring = re.sub(r"<[/]?[A-Za-z-]+?[ >]", lambda x: x.group().lower(), xmlstring)
        # store the minidom parsed xml doc
        self.xmldoc = minidom.parseString(self.xmlstring)

        # country, patent, kind, date_grant
        self.country, self.patent, self.kind, self.date_grant = self.__tagNme__(xmldoc, ["publication-reference", ["country", "doc-number", "kind", "date"]])
        # pat_type
        self.pat_type = self.__tagNme__(xmldoc, ["application-reference"], iHTML=False)[0].attributes["appl-type"].value
        # date_app, country_app, patent_app
        self.date_app, self.country_app, self.patent_app = self.__tagNme__(xmldoc, ["application-reference", ["date", "country", "doc-number"]])
        # code_app
        self.code_app = self.__tagNme__(xmldoc, ["us-application-series-code"])
        # clm_num
        self.clm_num = self.__tagNme__(xmldoc, ["number-of-claims"])
        # classes
        self.classes = [[x[:3].replace(' ',''), x[3:].replace(' ','')] for x in self.__tagNme__(xmldoc, ["classification-national", ["main-classification", "further-classification"]], idx=1, listType=True)]
        # abstract
        self.abstract = self.__allHTML__(xmldoc, ["abstract", "p"])
        # invention_title
        self.invention_title = self.__allHTML__(xmldoc, ["invention-title"])
        # asg_list
        self.asg_list = self.__asg_detail__(self.__tagNme__(xmldoc, ["assignees", "assignee"], iHTML=False))
        # cit_list
        self.cit_list = self.__cit_detail__(self.__tagNme__(xmldoc, ["references-cited", "citation"], iHTML=False))
        # rel_list
        self.rel_list = self.__rel_detail__(self.__tagNme__(xmldoc, ["us-related-documents"], iHTML=False))
        # inv_list
        self.inv_list = self.__tagSplit__(xmldoc, ["parties", "applicant"], [["addressbook", ["last-name", "first-name"]], ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], [["nationality", "residence"], "country"]], blank=True)
        # law_list
        self.law_list = self.__tagSplit__(xmldoc, ["parties", "agents", "agent"], [["addressbook", ["last-name", "first-name", "country", "orgname"]]], blank=True)

        """
        Callback structure to handle the build_table specifics
        """
        self.tablecallbacks = {
            'assignee': self.build_assignee,
            'citation': self.build_citation,
            'class': self.build_class,
            'inventor': self.build_inventor,
            'patent': self.build_patent,
            'patdesc': self.build_patdesc,
            'lawyer': self.build_lawyer,
            'sciref': self.build_sciref,
            'usreldoc': self.build_usreldoc
            }

    def build_table(self, tablename):
        """
        Return list of requisite fields for inserting this patent into [tablename]
        """
        return self.tablecallbacks[tablename]()

    def build_assignee(self):
        ack = []
        for i,y in enumerate(self.asg_list):
            if not y[0]:
                ack.extend([[self, y[2], y[1], y[4], y[5], y[7], y[8], i]])
            else:
                ack.extend([[self, "00", "%s, %s" % (y[2], y[1]), y[4], y[5], y[6], y[7], y[8], i]])
        return ack

    def build_citation(self):
        ack = []
        for i,y in enumerate([x for x in self.cit_list if x[1] != ""]):
            ack.extend([[self, y[3], y[5], y[4], y[1], y[2], y[0], i]])
        return ack

    def build_class(self):
        ack = []
        for i,y in enumerate(self.classes):
            ack.extend([[self, (i==0)*1, y[0], y[1]]])
        return ack

    def build_inventor(self):
        ack = []
        for i,y in enumerate(self.inv_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], y[4], y[5], y[6], y[8], i]])
        return ack

    def build_self(self):
        return [[self, self.kind, self.clm_num, self.code_app, self_app, self.date_grant, self.date_grant[:4], self.date_app, self.date_app[:4], self.pat_type]]

    def build_patdesc(self):
        return [[self, self.abstract, self.invention_title]]

    def build_lawyer(self):
        ack = []
        for i,y in enumerate(self.law_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], i]])
        return ack

    def build_sciref(self):
        ack = []
        for i,y in enumerate([y for y in self.cit_list if y[1] == ""]):
            ack.extend([[self, y[-1], i]])
        return ack

    def build_usreldoc(self):
        ack = []
        for i,y in enumerate(self.rel_list):
            if y[1] == 1:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], y[5], y[6]]])
            else:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], "", ""]])
        return ack

    """
    These are all methods from the old XMLPatent class. I'm keeping them
    in here to assist in the carry-over of parsing the xml fields
    """
    def __allHTML__(self, xmldoc, tagList):
        """
        Replaces the html tags for everything returned by __tagName__(taglist)
        with nothing
        """
        for x in self.__tagNme__(xmldoc, tagList, iHTML=False):
            return re.sub(r"<[/]?%s( .*?)?>" % (tagList[-1]), "", x.toxml())
        return ""

    def __innerHTML__(self, dom_element):
        """
        Gets the text out of all child nodes for a certain
        dom_element
        """
        #if blank return nothing as well!
        if dom_element == '':
            return ''
        else:
            rc = ""
            for node in dom_element.childNodes:
                if node.nodeType == node.TEXT_NODE:
                    rc = rc + node.data
            return rc

    def __tagSplit__(self, xmldoc, xmlList, tagList, baseList=[], idx=0, blank=False, iHTML=True, debug=False):
        """
        For each item in the list of xml tags returned by tagName, we treat each tag's contents
        as its own xml doc and search it for a list of xml tags. Everyything is appended together
        and returned
        """
        d_list = []
        for x in self.__tagNme__(xmldoc, tagList=xmlList, iHTML=False):
            record = copy.copy(baseList)
            for y in tagList:
                if debug:
                    print "-------------------"
                    print x.toxml()
                    print y
                record.extend(self.__tagNme__(x, tagList=y, blank=blank, iHTML=iHTML, debug=debug, idx=idx))
            d_list.append(record)
        return d_list

    def __tagNme__(self, xmldoc, tagList, idx=0, listType=False, blank=False, iHTML=True, debug=False):
        """
        takes in the xml.minidom parsed xml document [xmldoc] and a list of tags [tagList]
        [tagList] may sometimes be a list of lists?
        We iterate the taglist, searching another level deeper into the xml doc with each
        iteration. In tagList, index 0 is the first level of the xmldoc, index 1 is the list
        of all tags we look for at the level right under all the tags we found at index 0,
        and so on.
        For each item, we then iterate through all xmldocs we passed in (usually one?)
        and append the tag we wanted to an xmllist.
        We have empty strings if the tag was not found, and always return a flat list.
        **Returns:
        if the iHTML flag is set, we do the innerHTML method on the xmllist, otherwise
        we just return the xmllist.
        """
        xmldoc = [xmldoc]
        for i,x in enumerate(tagList):
            if type(x) is not ListType:
                x = [x]
            xmlnext = []
            for y in x:
                for z in xmldoc:
                    if z != '':
                        if i==0 and idx!=0:
                            if len(z.getElementsByTagName(y))>0:
                                xmlnext.append(z.getElementsByTagName(y)[idx-1])
                            else:
                                xmlnext.append('')
                        else:
                            blFlag = False
                            for za in z.getElementsByTagName(y):
                                blFlag = True
                                xmlnext.append(za)
                            if blFlag==False and blank==True:
                                xmlnext.append('')
                    else:
                        xmlnext.append('')
            xmldoc = xmlnext

        if debug:
            if len(xmldoc)==1 and iHTML:
                print self.__innerHTML__(xmldoc[0])
            elif iHTML:
                print [self.__innerHTML__(x) for x in xmldoc]
            else:
                print xmldoc

        if len(xmldoc)==1 and iHTML:
            if listType:
                return [self.__innerHTML__(xmldoc[0])]
            else:
                return self.__innerHTML__(xmldoc[0])
        elif iHTML:
            return [self.__innerHTML__(x) for x in xmldoc]
        else:
            return xmldoc

    def __asg_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            record = []
            if len(x.getElementsByTagName("first-name"))>0:
                record = [1]
                record.extend(self.__tagNme__(x, [["last-name", "first-name"]]))
            else:
                record = [0]
                record.extend(self.__tagNme__(x, [["orgname", "role"]]))
            record.extend(self.__tagNme__(x, ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], blank=True))
            record.extend(self.__tagNme__(x, [["nationality", "residence"], "country"], blank=True))
            d_list.append(record)
        return d_list

    def __cit_detail__(self, xmldoc):
        d_list = []
        for x in xmldoc:
            #this means patcit is part of the XML
            record = [self.__tagNme__(x, ["category"])]
            if len(x.getElementsByTagName("patcit"))>0:
                record.extend(self.__tagNme__(x, ["patcit", ["country", "doc-number", "date", "kind", "name"]], blank=True))
                record.extend([""])
            elif len(x.getElementsByTagName("othercit"))>0:
                record.extend(["", "", "", "", ""])
                record.extend([self.__allHTML__(x, ["othercit"])])
                #probably should grab date information
            else:
                print x.toxml()
            d_list.append(record)
        return d_list

    def __rel_detail__(self, xmldoc, debug=False):
        d_list = []
        for x in xmldoc:
            for y in ["continuation-in-part", "continuation", "division", "reissue"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["relation", "child-doc"], [[["doc-number", "country", "kind"]]], baseList=[y, -1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True, idx=1))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-grant-document"], [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
                    d_list.extend(self.__tagSplit__(x, ["relation", "parent-doc", "parent-pct-document"],  [[["doc-number", "country", "kind", "date", "parent-status"]]], baseList=[y, 1], blank=True))
            for y in ["related-publication", "us-provisional-application"]:
                if len(x.getElementsByTagName(y))>0:
                    d_list.extend(self.__tagSplit__(x, ["document-id"], [[["doc-number", "country", "kind"]]], baseList=[y, 0], blank=True))
            if debug:
                print "-------------------"
                for x in d_list:
                    print x
        return d_list

    def __repr__(self):
        return \
"""country = %s, patent = %s, pat_type = %s,
date_grant = %s, date_app = %s,
         abstract = %s
  invention_title = %s
len(classes, asg, cit, ret, inv, law) = %s""" % (self.country, self.patent, self.pat_type, self.date_grant, self.date_app,
       self.abstract[:50], self.invention_title[:50],
       str([len(self.classes),  len(self.asg_list), len(self.cit_list),
            len(self.rel_list), len(self.inv_list), len(self.law_list)]))
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# File: prefetch.py
# Author: Yuxin Wu <ppwwyyxx@gmail.com>

import multiprocessing as mp
from threading import Thread
import itertools
from six.moves import range, zip
from six.moves.queue import Queue
import uuid
import os

from .base import ProxyDataFlow
from ..utils.concurrency import *
from ..utils.serialize import loads, dumps
from ..utils import logger, change_env

__all__ = ['PrefetchData', 'BlockParallel']
try:
    import zmq
except ImportError:
    logger.warn("Error in 'import zmq'. PrefetchDataZMQ won't be available.")
else:
    __all__.extend(['PrefetchDataZMQ', 'PrefetchOnGPUs'])

class PrefetchProcess(mp.Process):
    def __init__(self, ds, queue, reset_after_spawn=True):
        """
        :param ds: ds to take data from
        :param queue: output queue to put results in
        """
        super(PrefetchProcess, self).__init__()
        self.ds = ds
        self.queue = queue
        self.reset_after_spawn = reset_after_spawn

    def run(self):
        if self.reset_after_spawn:
            # reset all ds so each process will produce different data
            self.ds.reset_state()
        while True:
            for dp in self.ds.get_data():
                self.queue.put(dp)

class PrefetchData(ProxyDataFlow):
    """
    Prefetch data from a `DataFlow` using multiprocessing
    """
    def __init__(self, ds, nr_prefetch, nr_proc=1):
        """
        :param ds: a `DataFlow` instance.
        :param nr_prefetch: size of the queue to hold prefetched datapoints.
        :param nr_proc: number of processes to use. When larger than 1, order
            of data points will be random.
        """
        super(PrefetchData, self).__init__(ds)
        try:
            self._size = ds.size()
        except NotImplementedError:
            self._size = -1
        self.nr_proc = nr_proc
        self.nr_prefetch = nr_prefetch
        self.queue = mp.Queue(self.nr_prefetch)
        self.procs = [PrefetchProcess(self.ds, self.queue)
                      for _ in range(self.nr_proc)]
        ensure_proc_terminate(self.procs)
        for x in self.procs:
            x.start()

    def get_data(self):
        for k in itertools.count():
            if self._size > 0 and k >= self._size:
                break
            dp = self.queue.get()
            yield dp

    def reset_state(self):
        # do nothing. all ds are reset once and only once in spawned processes
        pass

def BlockParallel(ds, queue_size):
    """
    Insert `BlockParallel` in dataflow pipeline to block parallelism on ds

    :param ds: a `DataFlow`
    :param queue_size: size of the queue used
    """
    return PrefetchData(ds, queue_size, 1)

class PrefetchProcessZMQ(mp.Process):
    def __init__(self, ds, conn_name):
        """
        :param ds: a `DataFlow` instance.
        :param conn_name: the name of the IPC connection
        """
        super(PrefetchProcessZMQ, self).__init__()
        self.ds = ds
        self.conn_name = conn_name

    def run(self):
        self.ds.reset_state()
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PUSH)
        self.socket.set_hwm(1)
        self.socket.connect(self.conn_name)
        while True:
            for dp in self.ds.get_data():
                self.socket.send(dumps(dp), copy=False)

class PrefetchDataZMQ(ProxyDataFlow):
    """ Work the same as `PrefetchData`, but faster. """
    def __init__(self, ds, nr_proc=1, pipedir=None):
        """
        :param ds: a `DataFlow` instance.
        :param nr_proc: number of processes to use. When larger than 1, order
            of datapoints will be random.
        :param pipedir: a local directory where the pipes would be. Useful if you're running on non-local FS such as NFS.
        """
        super(PrefetchDataZMQ, self).__init__(ds)
        try:
            self._size = ds.size()
        except NotImplementedError:
            self._size = -1
        self.nr_proc = nr_proc

        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PULL)

        if pipedir is None:
            pipedir = os.environ.get('TENSORPACK_PIPEDIR', '.')
        assert os.path.isdir(pipedir), pipedir
        self.pipename = "ipc://{}/dataflow-pipe-".format(pipedir.rstrip('/')) + str(uuid.uuid1())[:6]
        self.socket.set_hwm(5)  # a little bit faster than default, don't know why
        self.socket.bind(self.pipename)

        self.procs = [PrefetchProcessZMQ(self.ds, self.pipename)
                      for _ in range(self.nr_proc)]
        self.start_processes()
        # __del__ not guranteed to get called at exit
        import atexit
        atexit.register(lambda x: x.__del__(), self)

    def start_processes(self):
        start_proc_mask_signal(self.procs)

    def get_data(self):
        for k in itertools.count():
            if self._size > 0 and k >= self._size:
                break
            dp = loads(self.socket.recv(copy=False))
            yield dp

    def reset_state(self):
        # do nothing. all ds are reset once and only once in spawned processes
        pass

    def __del__(self):
        # on exit, logger may not be functional anymore
        try:
            logger.info("Prefetch process exiting...")
        except:
            pass
        if not self.context.closed:
            self.context.destroy(0)
        for x in self.procs:
            x.terminate()
        try:
            logger.info("Prefetch process exited.")
        except:
            pass

class PrefetchOnGPUs(PrefetchDataZMQ):
    """ Prefetch with each process having a specific CUDA_VISIBLE_DEVICES"""
    def __init__(self, ds, gpus, pipedir=None):
        super(PrefetchOnGPUs, self).__init__(ds, len(gpus), pipedir)
        self.gpus = gpus

    def start_processes(self):
        with mask_sigint():
            for gpu, proc in zip(self.gpus, self.procs):
                with change_gpu(gpu):
                    proc.start()

<commit_msg>bug fix in prefecthongpus<commit_after># -*- coding: UTF-8 -*-
# File: prefetch.py
# Author: Yuxin Wu <ppwwyyxx@gmail.com>

import multiprocessing as mp
from threading import Thread
import itertools
from six.moves import range, zip
from six.moves.queue import Queue
import uuid
import os

from .base import ProxyDataFlow
from ..utils.concurrency import *
from ..utils.serialize import loads, dumps
from ..utils import logger
from ..utils.gpu import change_gpu

__all__ = ['PrefetchData', 'BlockParallel']
try:
    import zmq
except ImportError:
    logger.warn("Error in 'import zmq'. PrefetchDataZMQ won't be available.")
else:
    __all__.extend(['PrefetchDataZMQ', 'PrefetchOnGPUs'])

class PrefetchProcess(mp.Process):
    def __init__(self, ds, queue, reset_after_spawn=True):
        """
        :param ds: ds to take data from
        :param queue: output queue to put results in
        """
        super(PrefetchProcess, self).__init__()
        self.ds = ds
        self.queue = queue
        self.reset_after_spawn = reset_after_spawn

    def run(self):
        if self.reset_after_spawn:
            # reset all ds so each process will produce different data
            self.ds.reset_state()
        while True:
            for dp in self.ds.get_data():
                self.queue.put(dp)

class PrefetchData(ProxyDataFlow):
    """
    Prefetch data from a `DataFlow` using multiprocessing
    """
    def __init__(self, ds, nr_prefetch, nr_proc=1):
        """
        :param ds: a `DataFlow` instance.
        :param nr_prefetch: size of the queue to hold prefetched datapoints.
        :param nr_proc: number of processes to use. When larger than 1, order
            of data points will be random.
        """
        super(PrefetchData, self).__init__(ds)
        try:
            self._size = ds.size()
        except NotImplementedError:
            self._size = -1
        self.nr_proc = nr_proc
        self.nr_prefetch = nr_prefetch
        self.queue = mp.Queue(self.nr_prefetch)
        self.procs = [PrefetchProcess(self.ds, self.queue)
                      for _ in range(self.nr_proc)]
        ensure_proc_terminate(self.procs)
        for x in self.procs:
            x.start()

    def get_data(self):
        for k in itertools.count():
            if self._size > 0 and k >= self._size:
                break
            dp = self.queue.get()
            yield dp

    def reset_state(self):
        # do nothing. all ds are reset once and only once in spawned processes
        pass

def BlockParallel(ds, queue_size):
    """
    Insert `BlockParallel` in dataflow pipeline to block parallelism on ds

    :param ds: a `DataFlow`
    :param queue_size: size of the queue used
    """
    return PrefetchData(ds, queue_size, 1)

class PrefetchProcessZMQ(mp.Process):
    def __init__(self, ds, conn_name):
        """
        :param ds: a `DataFlow` instance.
        :param conn_name: the name of the IPC connection
        """
        super(PrefetchProcessZMQ, self).__init__()
        self.ds = ds
        self.conn_name = conn_name

    def run(self):
        self.ds.reset_state()
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PUSH)
        self.socket.set_hwm(1)
        self.socket.connect(self.conn_name)
        while True:
            for dp in self.ds.get_data():
                self.socket.send(dumps(dp), copy=False)

class PrefetchDataZMQ(ProxyDataFlow):
    """ Work the same as `PrefetchData`, but faster. """
    def __init__(self, ds, nr_proc=1, pipedir=None):
        """
        :param ds: a `DataFlow` instance.
        :param nr_proc: number of processes to use. When larger than 1, order
            of datapoints will be random.
        :param pipedir: a local directory where the pipes would be. Useful if you're running on non-local FS such as NFS.
        """
        super(PrefetchDataZMQ, self).__init__(ds)
        try:
            self._size = ds.size()
        except NotImplementedError:
            self._size = -1
        self.nr_proc = nr_proc

        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PULL)

        if pipedir is None:
            pipedir = os.environ.get('TENSORPACK_PIPEDIR', '.')
        assert os.path.isdir(pipedir), pipedir
        self.pipename = "ipc://{}/dataflow-pipe-".format(pipedir.rstrip('/')) + str(uuid.uuid1())[:6]
        self.socket.set_hwm(5)  # a little bit faster than default, don't know why
        self.socket.bind(self.pipename)

        self.procs = [PrefetchProcessZMQ(self.ds, self.pipename)
                      for _ in range(self.nr_proc)]
        self.start_processes()
        # __del__ not guranteed to get called at exit
        import atexit
        atexit.register(lambda x: x.__del__(), self)

    def start_processes(self):
        start_proc_mask_signal(self.procs)

    def get_data(self):
        for k in itertools.count():
            if self._size > 0 and k >= self._size:
                break
            dp = loads(self.socket.recv(copy=False))
            yield dp

    def reset_state(self):
        # do nothing. all ds are reset once and only once in spawned processes
        pass

    def __del__(self):
        # on exit, logger may not be functional anymore
        try:
            logger.info("Prefetch process exiting...")
        except:
            pass
        if not self.context.closed:
            self.context.destroy(0)
        for x in self.procs:
            x.terminate()
        try:
            logger.info("Prefetch process exited.")
        except:
            pass

class PrefetchOnGPUs(PrefetchDataZMQ):
    """ Prefetch with each process having a specific CUDA_VISIBLE_DEVICES"""
    def __init__(self, ds, gpus, pipedir=None):
        self.gpus = gpus
        super(PrefetchOnGPUs, self).__init__(ds, len(gpus), pipedir)

    def start_processes(self):
        with mask_sigint():
            for gpu, proc in zip(self.gpus, self.procs):
                with change_gpu(gpu):
                    proc.start()

<|endoftext|>


<commit_before>import collections
import datetime
import json
import logging
import re
import subprocess
import tempfile
from urllib.parse import urljoin

import pytz
from atomicwrites import atomic_write
from django.conf import settings
from django.template.loader import render_to_string

from promgen import models, util
from promgen.celery import app as celery

logger = logging.getLogger(__name__)

def check_rules(rules):
    with tempfile.NamedTemporaryFile(mode='w', encoding='utf8') as fp:
        logger.debug('Rendering to %s', fp.name)
        # Normally we wouldn't bother saving a copy to a variable here and would
        # leave it in the fp.write() call, but saving a copy in the variable
        # means we can see the rendered output in a Sentry stacktrace
        rendered = render_rules(rules)
        fp.write(rendered)
        fp.flush()

        try:
            subprocess.check_output([
                settings.PROMGEN['rule_writer']['promtool_path'],
                'check-rules',
                fp.name
            ], stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            raise Exception(rendered + e.output.decode('utf8'))

def render_rules(rules=None):
    if rules is None:
        rules = models.Rule.objects.filter(enabled=True)
    return render_to_string('promgen/prometheus.rule', {'rules': rules})

def render_urls():
    urls = collections.defaultdict(list)

    for url in models.URL.objects.all():
        urls[(
            url.project.name, url.project.service.name, url.project.service.shard.name,
        )].append(url.url)

    data = [{'labels': {'project': k[0], 'service': k[1], '__shard': k[2]}, 'targets': v} for k, v in urls.items()]
    return json.dumps(data, indent=2, sort_keys=True)

@celery.task
def write_urls(path=None, reload=True):
    if path is None:
        path = settings.PROMGEN['url_writer']['path']
    with atomic_write(path, overwrite=True) as fp:
        fp.write(render_urls())
    if reload:
        reload_prometheus()

def render_config(service=None, project=None):
    data = []
    for exporter in models.Exporter.objects.all():
        if not exporter.project.farm:
            continue
        if service and exporter.project.service.name != service.name:
            continue
        if project and exporter.project.name != project.name:
            continue

        labels = {
            '__shard': exporter.project.service.shard.name,
            'service': exporter.project.service.name,
            'project': exporter.project.name,
            'farm': exporter.project.farm.name,
            '__farm_source': exporter.project.farm.source,
            'job': exporter.job,
        }
        if exporter.path:
            labels['__metrics_path__'] = exporter.path

        hosts = []
        for host in models.Host.objects.filter(farm=exporter.project.farm):
            hosts.append('{}:{}'.format(host.name, exporter.port))

        data.append({
            'labels': labels,
            'targets': hosts,
        })
    return json.dumps(data, indent=2, sort_keys=True)

@celery.task
def write_config(path=None, reload=True):
    if path is None:
        path = settings.PROMGEN['config_writer']['path']
    with atomic_write(path, overwrite=True) as fp:
        fp.write(render_config())
    if reload:
        reload_prometheus()

@celery.task
def write_rules(path=None, reload=True):
    if path is None:
        path = settings.PROMGEN['rule_writer']['path']
    with atomic_write(path, overwrite=True) as fp:
        fp.write(render_rules())
    if reload:
        reload_prometheus()

@celery.task
def reload_prometheus():
    from promgen.signals import post_reload
    target = urljoin(settings.PROMGEN['prometheus']['url'], '/-/reload')
    response = util.post(target)
    post_reload.send(response)

def import_rules(config, default_service=None):
    # Attemps to match the pattern name="value" for Prometheus labels and annotations
    RULE_MATCH = re.compile('((?P<key>\w+)\s*=\s*\"(?P<value>.*?)\")')
    counters = collections.defaultdict(int)

    def parse_prom(text):
        if not text:
            return {}
        converted = {}
        for match, key, value in RULE_MATCH.findall(text.strip().strip('{}')):
            converted[key] = value
        return converted

    tokens = {}
    rules = []
    for line in config.split('\n'):
        line = line.strip()
        if not line:
            continue
        if line.startswith('#'):
            continue

        keyword, data = line.split(' ', 1)

        if keyword != 'ALERT':
            tokens[keyword] = data
            continue

        if keyword == 'ALERT' and 'ALERT' not in tokens:
            tokens[keyword] = data
            continue

        rules.append(tokens)
        # Start building our next rule
        tokens = {keyword: data}
    # Make sure we keep our last token after parsing all lines
    rules.append(tokens)

    for tokens in rules:
        labels = parse_prom(tokens.get('LABELS'))
        annotations = parse_prom(tokens.get('ANNOTATIONS'))

        if default_service:
            service = default_service
        else:
            try:
                service = models.Service.objects.get(name=labels.get('service', 'Default'))
            except models.Service.DoesNotExist:
                service = models.Service.default()

        rule, created = models.Rule.objects.get_or_create(
            name=tokens['ALERT'],
            defaults={
                'clause': tokens['IF'],
                'duration': tokens['FOR'],
                'service': service,
            }
        )

        if created:
            counters['Rules'] += 1
            for k, v in labels.items():
                models.RuleLabel.objects.create(name=k, value=v, rule=rule)
                counters['Labels'] += 1
            for k, v in annotations.items():
                models.RuleAnnotation.objects.create(name=k, value=v, rule=rule)
                counters['Annotations'] += 1

    return dict(counters)

def import_config(config):
    counters = collections.defaultdict(list)
    for entry in config:
        shard, created = models.Shard.objects.get_or_create(
            name=entry['labels'].get('__shard', 'Default')
        )
        if created:
            counters['Shard'].append(shard)

        service, created = models.Service.objects.get_or_create(
            name=entry['labels']['service'],
            defaults={'shard': shard}
        )
        if created:
            counters['Service'].append(service)

        farm, created = models.Farm.objects.get_or_create(
            name=entry['labels']['farm'],
            defaults={'source': entry['labels'].get('__farm_source', 'pmc')}
        )
        if created:
            counters['Farm'].append(farm)

        project, created = models.Project.objects.get_or_create(
            name=entry['labels']['project'],
            service=service,
            defaults={'farm': farm}
        )
        if created:
            counters['Project'].append(project)

        if not project.farm:
            project.farm = farm
            project.save()

        for target in entry['targets']:
            target, port = target.split(':')
            host, created = models.Host.objects.get_or_create(
                name=target,
                farm_id=farm.id,
            )

            if created:
                counters['Host'].append(host)

            exporter, created = models.Exporter.objects.get_or_create(
                job=entry['labels']['job'],
                port=port,
                project=project,
                path=entry['labels'].get('__metrics_path__', '')
            )

            if created:
                counters['Exporter'].append(exporter)

    return counters

def mute(duration, labels):
    '''
    Post a silence message to Alert Manager
    Duration should be sent in a format like 1m 2h 1d etc
    '''
    start = datetime.datetime.now(datetime.timezone.utc)

    if duration.lower().endswith('m'):
        end = start + datetime.timedelta(minutes=int(duration[:-1]))
    elif duration.lower().endswith('h'):
        end = start + datetime.timedelta(hours=int(duration[:-1]))
    elif duration.lower().endswith('d'):
        end = start + datetime.timedelta(days=int(duration[:-1]))
    else:
        raise Exception('Unknown time modifier')

    data = {
        'comment': 'Promgen Mute',
        'createdBy': 'Promgen',
        'matchers': [{'name': name, 'value': value} for name, value in labels.items()],
        'endsAt': end.strftime('%Y-%m-%dT%H:%M:%S.000Z')
    }

    logger.debug('Sending silence for %s %s', end, data)
    url = urljoin(settings.PROMGEN['alertmanager']['url'], '/api/v1/silences')
    util.post(url, json=data).raise_for_status()

def mute_fromto(start, stop, labels):
    '''
    Post a silence message to Alert Manager
    Duration should be sent in a format like 2017-01-01 09:00
    '''
    local_timezone = pytz.timezone(settings.PROMGEN.get('timezone', 'UTC'))
    mute_start = datetime.datetime.strptime(start, '%Y-%m-%d %H:%M').replace(tzinfo=local_timezone)
    mute_stop = datetime.datetime.strptime(stop, '%Y-%m-%d %H:%M').replace(tzinfo=local_timezone)

    data = {
        'comment': 'Promgen Mute',
        'createdBy': 'Promgen',
        'matchers': [{'name': name, 'value': value} for name, value in labels.items()],
        'startsAt': mute_start.astimezone(pytz.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
        'endsAt': mute_stop.astimezone(pytz.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
    }

    logger.debug('Sending silence for %s - %s %s', start, stop, data)
    url = urljoin(settings.PROMGEN['alertmanager']['url'], '/api/v1/silences')
    util.post(url, json=data).raise_for_status()
<commit_msg>Replace farm when importing<commit_after>import collections
import datetime
import json
import logging
import re
import subprocess
import tempfile
from urllib.parse import urljoin

import pytz
from atomicwrites import atomic_write
from django.conf import settings
from django.template.loader import render_to_string

from promgen import models, util
from promgen.celery import app as celery

logger = logging.getLogger(__name__)

def check_rules(rules):
    with tempfile.NamedTemporaryFile(mode='w', encoding='utf8') as fp:
        logger.debug('Rendering to %s', fp.name)
        # Normally we wouldn't bother saving a copy to a variable here and would
        # leave it in the fp.write() call, but saving a copy in the variable
        # means we can see the rendered output in a Sentry stacktrace
        rendered = render_rules(rules)
        fp.write(rendered)
        fp.flush()

        try:
            subprocess.check_output([
                settings.PROMGEN['rule_writer']['promtool_path'],
                'check-rules',
                fp.name
            ], stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            raise Exception(rendered + e.output.decode('utf8'))

def render_rules(rules=None):
    if rules is None:
        rules = models.Rule.objects.filter(enabled=True)
    return render_to_string('promgen/prometheus.rule', {'rules': rules})

def render_urls():
    urls = collections.defaultdict(list)

    for url in models.URL.objects.all():
        urls[(
            url.project.name, url.project.service.name, url.project.service.shard.name,
        )].append(url.url)

    data = [{'labels': {'project': k[0], 'service': k[1], '__shard': k[2]}, 'targets': v} for k, v in urls.items()]
    return json.dumps(data, indent=2, sort_keys=True)

@celery.task
def write_urls(path=None, reload=True):
    if path is None:
        path = settings.PROMGEN['url_writer']['path']
    with atomic_write(path, overwrite=True) as fp:
        fp.write(render_urls())
    if reload:
        reload_prometheus()

def render_config(service=None, project=None):
    data = []
    for exporter in models.Exporter.objects.all():
        if not exporter.project.farm:
            continue
        if service and exporter.project.service.name != service.name:
            continue
        if project and exporter.project.name != project.name:
            continue

        labels = {
            '__shard': exporter.project.service.shard.name,
            'service': exporter.project.service.name,
            'project': exporter.project.name,
            'farm': exporter.project.farm.name,
            '__farm_source': exporter.project.farm.source,
            'job': exporter.job,
        }
        if exporter.path:
            labels['__metrics_path__'] = exporter.path

        hosts = []
        for host in models.Host.objects.filter(farm=exporter.project.farm):
            hosts.append('{}:{}'.format(host.name, exporter.port))

        data.append({
            'labels': labels,
            'targets': hosts,
        })
    return json.dumps(data, indent=2, sort_keys=True)

@celery.task
def write_config(path=None, reload=True):
    if path is None:
        path = settings.PROMGEN['config_writer']['path']
    with atomic_write(path, overwrite=True) as fp:
        fp.write(render_config())
    if reload:
        reload_prometheus()

@celery.task
def write_rules(path=None, reload=True):
    if path is None:
        path = settings.PROMGEN['rule_writer']['path']
    with atomic_write(path, overwrite=True) as fp:
        fp.write(render_rules())
    if reload:
        reload_prometheus()

@celery.task
def reload_prometheus():
    from promgen.signals import post_reload
    target = urljoin(settings.PROMGEN['prometheus']['url'], '/-/reload')
    response = util.post(target)
    post_reload.send(response)

def import_rules(config, default_service=None):
    # Attemps to match the pattern name="value" for Prometheus labels and annotations
    RULE_MATCH = re.compile('((?P<key>\w+)\s*=\s*\"(?P<value>.*?)\")')
    counters = collections.defaultdict(int)

    def parse_prom(text):
        if not text:
            return {}
        converted = {}
        for match, key, value in RULE_MATCH.findall(text.strip().strip('{}')):
            converted[key] = value
        return converted

    tokens = {}
    rules = []
    for line in config.split('\n'):
        line = line.strip()
        if not line:
            continue
        if line.startswith('#'):
            continue

        keyword, data = line.split(' ', 1)

        if keyword != 'ALERT':
            tokens[keyword] = data
            continue

        if keyword == 'ALERT' and 'ALERT' not in tokens:
            tokens[keyword] = data
            continue

        rules.append(tokens)
        # Start building our next rule
        tokens = {keyword: data}
    # Make sure we keep our last token after parsing all lines
    rules.append(tokens)

    for tokens in rules:
        labels = parse_prom(tokens.get('LABELS'))
        annotations = parse_prom(tokens.get('ANNOTATIONS'))

        if default_service:
            service = default_service
        else:
            try:
                service = models.Service.objects.get(name=labels.get('service', 'Default'))
            except models.Service.DoesNotExist:
                service = models.Service.default()

        rule, created = models.Rule.objects.get_or_create(
            name=tokens['ALERT'],
            defaults={
                'clause': tokens['IF'],
                'duration': tokens['FOR'],
                'service': service,
            }
        )

        if created:
            counters['Rules'] += 1
            for k, v in labels.items():
                models.RuleLabel.objects.create(name=k, value=v, rule=rule)
                counters['Labels'] += 1
            for k, v in annotations.items():
                models.RuleAnnotation.objects.create(name=k, value=v, rule=rule)
                counters['Annotations'] += 1

    return dict(counters)

def import_config(config):
    counters = collections.defaultdict(list)
    for entry in config:
        shard, created = models.Shard.objects.get_or_create(
            name=entry['labels'].get('__shard', 'Default')
        )
        if created:
            logger.debug('Created shard %s', shard)
            counters['Shard'].append(shard)

        service, created = models.Service.objects.get_or_create(
            name=entry['labels']['service'],
            defaults={'shard': shard}
        )
        if created:
            logger.debug('Created service %s', service)
            counters['Service'].append(service)

        farm, created = models.Farm.objects.get_or_create(
            name=entry['labels']['farm'],
            defaults={'source': entry['labels'].get('__farm_source', 'pmc')}
        )
        if created:
            logger.debug('Created farm %s', farm)
            counters['Farm'].append(farm)

        project, created = models.Project.objects.get_or_create(
            name=entry['labels']['project'],
            service=service,
            defaults={'farm': farm}
        )
        if created:
            logger.debug('Created project %s', project)
            counters['Project'].append(project)
        elif project.farm != farm:
            logger.debug('Linking farm [%s] with [%s]', farm, project)
            project.farm = farm
            project.save()

        for target in entry['targets']:
            target, port = target.split(':')
            host, created = models.Host.objects.get_or_create(
                name=target,
                farm_id=farm.id,
            )

            if created:
                logger.debug('Created host %s', host)
                counters['Host'].append(host)

            exporter, created = models.Exporter.objects.get_or_create(
                job=entry['labels']['job'],
                port=port,
                project=project,
                path=entry['labels'].get('__metrics_path__', '')
            )

            if created:
                logger.debug('Created exporter %s', exporter)
                counters['Exporter'].append(exporter)

    return counters

def mute(duration, labels):
    '''
    Post a silence message to Alert Manager
    Duration should be sent in a format like 1m 2h 1d etc
    '''
    start = datetime.datetime.now(datetime.timezone.utc)

    if duration.lower().endswith('m'):
        end = start + datetime.timedelta(minutes=int(duration[:-1]))
    elif duration.lower().endswith('h'):
        end = start + datetime.timedelta(hours=int(duration[:-1]))
    elif duration.lower().endswith('d'):
        end = start + datetime.timedelta(days=int(duration[:-1]))
    else:
        raise Exception('Unknown time modifier')

    data = {
        'comment': 'Promgen Mute',
        'createdBy': 'Promgen',
        'matchers': [{'name': name, 'value': value} for name, value in labels.items()],
        'endsAt': end.strftime('%Y-%m-%dT%H:%M:%S.000Z')
    }

    logger.debug('Sending silence for %s %s', end, data)
    url = urljoin(settings.PROMGEN['alertmanager']['url'], '/api/v1/silences')
    util.post(url, json=data).raise_for_status()

def mute_fromto(start, stop, labels):
    '''
    Post a silence message to Alert Manager
    Duration should be sent in a format like 2017-01-01 09:00
    '''
    local_timezone = pytz.timezone(settings.PROMGEN.get('timezone', 'UTC'))
    mute_start = datetime.datetime.strptime(start, '%Y-%m-%d %H:%M').replace(tzinfo=local_timezone)
    mute_stop = datetime.datetime.strptime(stop, '%Y-%m-%d %H:%M').replace(tzinfo=local_timezone)

    data = {
        'comment': 'Promgen Mute',
        'createdBy': 'Promgen',
        'matchers': [{'name': name, 'value': value} for name, value in labels.items()],
        'startsAt': mute_start.astimezone(pytz.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z'),
        'endsAt': mute_stop.astimezone(pytz.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
    }

    logger.debug('Sending silence for %s - %s %s', start, stop, data)
    url = urljoin(settings.PROMGEN['alertmanager']['url'], '/api/v1/silences')
    util.post(url, json=data).raise_for_status()
<|endoftext|>


<commit_before>import os
import shutil
from .h5diff import H5Diff
import unittest
from westpa.cli.tools.w_pdist import entry_point

class Test_W_PDIST(unittest.TestCase):
    '''Class to test w_pdist works to generate a file and that it is the same as the sample pdist.h5 file.'''

    test_name = 'W_PDIST'

    def test_run_w_pdist(self):
        '''Testing if w_pdist runs as expected and the pdist.h5 file looks good.'''

        ref_dir = os.path.join(os.path.dirname(__file__), 'refs')
        shutil.copy2(os.path.join(ref_dir, 'west.cfg'), './')
        shutil.copy2(os.path.join(ref_dir, 'west.h5'), './')

        entry_point()

        assert os.path.isfile('./pdist.h5'), "The pdist.h5 file was not generated."
        diff = H5Diff(os.path.join(ref_dir, 'pdist_ref.h5'), './pdist.h5')
        diff.check()

        # clean up
        os.remove('west.h5')
        os.remove('pdist.h5')
<commit_msg>final working test_w_pdist.py using entry_points and mock.patch<commit_after>import os
import shutil
import unittest
from unittest import mock

from .h5diff import H5Diff
from westpa.cli.tools.w_pdist import entry_point
import argparse

class Test_W_PDIST(unittest.TestCase):
    '''Class to test w_pdist works to generate a file and that it is the same as the sample pdist.h5 file.'''

    test_name = 'W_PDIST'

    def test_run_w_pdist(self):
        '''Testing if w_pdist runs as expected and the pdist.h5 file looks good.'''

        ref_dir = os.path.join(os.path.dirname(__file__), 'refs')
        shutil.copy2(os.path.join(ref_dir, 'west.cfg'), './')
        shutil.copy2(os.path.join(ref_dir, 'west.h5'), './')

        with mock.patch(
            target='argparse.ArgumentParser.parse_args',
            return_value=argparse.Namespace(
                verbosity='0',
                rcfile='west.cfg',
                max_queue_length=None,
                we_h5filename='west.h5',
                construct_dataset=None,
                dsspecs=None,
                group_name='pcoord',
                first_iter=None,
                last_iter=None,
                override_iter_start=True,
                override_iter_stop=True,
                default_iter_step=None,
                bins='100',
                output='pdist.h5',
                ignore_out_of_range=False,
                compress=False,
            ),
        ):
            entry_point()

        assert os.path.isfile('./pdist.h5'), "The pdist.h5 file was not generated."
        diff = H5Diff(os.path.join(ref_dir, 'pdist_ref.h5'), './pdist.h5')
        diff.check()

        # clean up
        os.remove('west.h5')
        os.remove('pdist.h5')
        os.remove('west.cfg')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from django.templatetags.l10n import localize

# https://github.com/azavea/django-queryset-csv/blob/
# master/djqscsv/djqscsv.py#L123
def sanitize_unicode_record(record):

    def _sanitize_value(value):
        if isinstance(val, unicode):
            return value.encode("utf-8")
        else:
            return localize(value)

    obj = {}
    for key, val in record.iteritems():
        if val:
            obj[_sanitize_value(key)] = _sanitize_value(val)

    return obj
<commit_msg>Don't localize CSV values<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

# https://github.com/azavea/django-queryset-csv/blob/
# master/djqscsv/djqscsv.py#L123
def sanitize_unicode_record(record):

    def _sanitize_value(value):
        # make sure every text value is of type 'str', coercing unicode
        if isinstance(value, unicode):
            return value.encode("utf-8")
        elif isinstance(value, str):
            return value
        else:
            return str(value).encode("utf-8")

    obj = {}
    for key, val in record.iteritems():
        if val:
            obj[_sanitize_value(key)] = _sanitize_value(val)

    return obj
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# This script executes all tests in `test/`.

import os
import sys
import argparse
import subprocess
import re
import shlex
from pathlib import Path
from copy import copy
import itertools

cbold = "\x1b[1m"
cpass = "\x1b[32m"
ctime = "\x1b[33m"
cfail = "\x1b[31m"
creset = "\x1b[0m"

crate_dir = os.path.dirname(__file__) + "/.."
test_dir = crate_dir + "/test"

# Parse arguments.
parser = argparse.ArgumentParser(description="Execute all regression tests.")
parser.add_argument("--crate", metavar="DIR", default=crate_dir, help="Root directory of the moore crate")
parser.add_argument("--debug", action="store_true", help="Use debug builds of local crate")
parser.add_argument("--release", action="store_true", help="Use release builds of local crate")
parser.add_argument("--prefix", metavar="PREFIX", help="Use binaries installed at this prefix")
parser.add_argument("--tests", metavar="DIR", default=test_dir, help="Directory where tests are located")
parser.add_argument("-v", "--verbose", action="store_true", help="Print stdout/stderr of failing tests")
parser.add_argument("-c", "--commands", action="store_true", help="Print commands used to execute tests")
parser.add_argument("-n", "--no-build", action="store_true", help="Don't rebuild the project")
parser.add_argument("--check", metavar="FILE", nargs=2, help="Check first file against directives in second file")
parser.add_argument("--check-stdin", metavar="FILE", nargs=1, help="Check stdin against directives in file")
parser.add_argument("TEST", nargs="*", help="Specific test cases to run; all if omitted")
args = parser.parse_args()

# A class to encapsulate the check directives in a file.
class CheckFile:
    regex_dir = re.compile(r'^\s*//\s*(CHECK[^:]*):\s+(.+)$')
    ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')

    def __init__(self, checks, input):
        # Collect the directives in the file.
        self.dirs = list()
        for line in checks.splitlines():
            m = self.regex_dir.match(line)
            if m:
                self.dirs.append((m.group(1), m.group(2)))

        # Split input into lines.
        self.input = input.splitlines()

    def execute(self):
        self.failed = list()
        self.state = iter(self.input)

        # Execute the directives in order.
        for d in self.dirs:
            try:
                self.state = self.execute_directive(d, copy(self.state))
            except Exception as e:
                try:
                    line = next(copy(self.state)).strip()
                except StopIteration:
                    line = "<end of file>"
                self.failed.append((d, e.__str__(), line))

        # Concatenate the failures into information messages.
        info = ""
        for f in self.failed:
            info += "error: {}: {}\n  {}. Scanning from:\n  {}\n".format(f[0][0], f[0][1], f[1], f[2])

        # Package a return value.
        return (len(self.failed) == 0, info)

    def execute_directive(self, directive, state):
        dirname = directive[0].replace("CHECK-ERR", "CHECK")
        if dirname == "CHECK":
            for line in state:
                line = line.split("//")[0].strip()
                line = self.ansi_escape.sub("", line)
                if line == directive[1]:
                    return state
            raise Exception("No matching line found")
        else:
            raise Exception("Unknown directive `{}`".format(directive[0]))

# Handle the special case where we are just supposed to check a file.
check = None
if args.check:
    with open(args.check[0]) as f:
        content = f.read()
    check = (args.check[1], content)
if args.check_stdin:
    content = sys.stdin.read()
    check = (args.check_stdin, content)
if check is not None:
    with open(check[0]) as f:
        checks = f.read()
    passed, info = CheckFile(checks, check[1]).execute()
    if passed:
        sys.exit(0)
    else:
        sys.stdout.write(info)
        sys.exit(1)

# Establish some working directories.
crate_dir = Path(os.path.realpath(args.crate))
test_dir = Path(os.path.realpath(args.tests))

# Build the binaries if requested.
prefix = None
try:
    if args.debug or args.release:
        # Build
        if not args.no_build:
            cmd = ["cargo", "build", "--bins"]
            if args.release:
                cmd += ["--release"]
            subprocess.run(
                cmd,
                stdin=subprocess.DEVNULL,
                cwd=crate_dir.__str__(),
                check=True,
            )

        # Extract build directory
        # cargo metadata --format-version 1 | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p'
        metadata = subprocess.run(
            ["cargo", "metadata", "--format-version", "1"],
            stdin=subprocess.DEVNULL,
            stdout=subprocess.PIPE,
            cwd=crate_dir.__str__(),
            check=True,
            universal_newlines=True,
        ).stdout
        prefix = re.search(r'"target_directory":"([^"]*)"', metadata).group(1)
        if args.debug:
            prefix += "/debug"
        if args.release:
            prefix += "/release"
        prefix = os.path.realpath(prefix)+"/"
except Exception as e:
    sys.stderr.write("{}\n".format(e))
    sys.exit(1)

prefix = prefix or args.prefix or ""
if args.verbose:
    sys.stdout.write("# crate:   {}\n".format(crate_dir))
    sys.stdout.write("# test:    {}\n".format(test_dir))
    sys.stdout.write("# prefix:  {}\n".format(prefix))

# A class to encapsulate the execution and checking of a single test.
class TestCase(object):
    regex_ignore = re.compile(r'^\s*//\s*IGNORE\b', flags=re.MULTILINE)
    regex_fail   = re.compile(r'^\s*//\s*FAIL\b', flags=re.MULTILINE)
    regex_run    = re.compile(r'^\s*//\s*RUN:\s+(.+)$', flags=re.MULTILINE)

    def __init__(self, name, path):
        self.name = name
        self.path = Path(path)
        self.dir = self.path.parent

        # Load the contents of the test file.
        with open(path.__str__()) as f:
            self.content = f.read()

        # Check for a `IGNORE` directive.
        self.ignore = self.regex_ignore.search(self.content) is not None

        # Check for a `FAIL` directive.
        self.should_fail = self.regex_fail.search(self.content) is not None

        # Check for a `RUN` directive.
        self.run = self.regex_run.search(self.content)
        if self.run:
            self.run = self.run.group(1)
        else:
            self.run = "moore --syntax %s"

        # Execution results.
        self.timeout = False
        self.failed = False
        self.info = ""
        self.stdout = ""
        self.stderr = ""

        # Process the run command.
        self.cmd = shlex.split(self.run)
        if self.cmd[0] == "moore":
            self.cmd[0] = "{}{}".format(prefix, self.cmd[0])
        self.cmd = [path if x == "%s" else x for x in self.cmd]
        cmd = list()
        for x in self.cmd:
            if "*" in x.__str__():
                self.info += "Arg: `{}` is a glob pattern\n".format(x)
                cmd += self.dir.glob(x)
            elif (self.dir/x).exists():
                self.info += "Arg: `{}` is a path\n".format(x)
                cmd.append(self.dir/x)
            else:
                self.info += "Arg: `{}` is a plain argument\n".format(x)
                cmd.append(x)
        self.cmd = list([x.__str__() for x in cmd])

    def launch(self):
        if self.ignore:
            return
        try:
            self.info += "Command: {}\n".format(self.cmd)
            self.proc = subprocess.Popen(
                [x.__str__() for x in self.cmd],
                universal_newlines=True,
                stdin=subprocess.DEVNULL,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                cwd=crate_dir.__str__(),
            )
        except Exception as e:
            self.info += "Exception: {}\n".format(e)
            self.failed = True

    def finish(self):
        if self.ignore or self.failed:
            return
        try:
            self.stdout, self.stderr = self.proc.communicate(timeout=10)
        except subprocess.TimeoutExpired as e:
            self.proc.kill()
            self.stdout, self.stderr = self.proc.communicate()
            self.timeout = True
            self.failed = True
            self.info += "Timeout"
            return

        # Check the return code.
        self.failed = (self.proc.returncode != 0)

        # Check if we were supposed to fail, but didn't.
        if self.failed != self.should_fail:
            self.info += "Failed: {}\nShould Fail: {}\n".format(self.failed, self.should_fail)
            self.failed = True
            return
        else:
            self.failed = False

        # Perform the file checks.
        passed, info = CheckFile(self.content, self.stdout + self.stderr).execute()
        if not passed:
            self.failed = True
            self.info += "File checks failed:\n\n"
            self.info += info

# Collect the tests.
if args.TEST:
    tests = [TestCase(p, Path(os.path.realpath(p))) for p in args.TEST]
else:
    suffices = ["sv", "v", "vhd"]
    third_party = test_dir/"third-party"
    globs = [[p for p in test_dir.glob("**/*."+suffix) if third_party not in p.parents] for suffix in suffices]
    tests = [TestCase(p.relative_to(test_dir), p) for p in sorted(itertools.chain(*globs))]
sys.stdout.write("running {} tests\n".format(len(tests)))

# Assemble the test arrays.
num_parallel = 16
heads = tests + [None]*num_parallel
tails = [None]*num_parallel + tests

# Execute the tests and output test results.
ignored = list()
failed = list()
for head, tail in zip(heads, tails):
    if head:
        head.launch()
    if tail:
        sys.stdout.write("test {} ...".format(tail.name))
        sys.stdout.flush()
        tail.finish()
        if tail.ignore:
            ignored.append(tail)
            sys.stdout.write(" ignored\n")
            continue
        if tail.timeout:
            sys.stdout.write(" timeout,")
        if tail.failed:
            failed.append(tail)
            sys.stdout.write(" {}FAILED{}\n".format(cbold+cfail, creset))
            if args.verbose:
                sys.stdout.write("\n=== INFO ===\n")
                sys.stdout.write(tail.info)
                sys.stdout.write("\n=== STDERR ===\n")
                sys.stdout.write(tail.stderr)
                sys.stdout.write("\n=== STDOUT ===\n")
                sys.stdout.write(tail.stdout)
                sys.stdout.write("\n")
        else:
            sys.stdout.write(" {}passed{}\n".format(cpass, creset))
        if args.commands:
            sys.stdout.write("# {}\n".format(" ".join([x.__str__() for x in tail.cmd])))

# Output summary.
sys.stdout.write("\n")
if failed:
    sys.stdout.write("failures:\n")
    for test in failed:
        sys.stdout.write("    {}\n".format(test.name))
    sys.stdout.write("\n")
sys.stdout.write("test result: {}. {} passed, {} failed, {} ignored\n".format(
    "{}PASSED{}".format(cbold+cpass, creset) if not failed else "{}FAILED{}".format(cbold+cfail, creset),
    len(tests) - len(failed) - len(ignored),
    len(failed),
    len(ignored)
))

sys.exit(1 if failed else 0)
<commit_msg>scripts: Make test.py run on python 3.4<commit_after>#!/usr/bin/env python3
# This script executes all tests in `test/`.

import os
import sys
import argparse
import subprocess
import re
import shlex
from pathlib import Path
from copy import copy
import itertools

cbold = "\x1b[1m"
cpass = "\x1b[32m"
ctime = "\x1b[33m"
cfail = "\x1b[31m"
creset = "\x1b[0m"

crate_dir = os.path.dirname(__file__) + "/.."
test_dir = crate_dir + "/test"

# Parse arguments.
parser = argparse.ArgumentParser(description="Execute all regression tests.")
parser.add_argument("--crate", metavar="DIR", default=crate_dir, help="Root directory of the moore crate")
parser.add_argument("--debug", action="store_true", help="Use debug builds of local crate")
parser.add_argument("--release", action="store_true", help="Use release builds of local crate")
parser.add_argument("--prefix", metavar="PREFIX", help="Use binaries installed at this prefix")
parser.add_argument("--tests", metavar="DIR", default=test_dir, help="Directory where tests are located")
parser.add_argument("-v", "--verbose", action="store_true", help="Print stdout/stderr of failing tests")
parser.add_argument("-c", "--commands", action="store_true", help="Print commands used to execute tests")
parser.add_argument("-n", "--no-build", action="store_true", help="Don't rebuild the project")
parser.add_argument("--check", metavar="FILE", nargs=2, help="Check first file against directives in second file")
parser.add_argument("--check-stdin", metavar="FILE", nargs=1, help="Check stdin against directives in file")
parser.add_argument("TEST", nargs="*", help="Specific test cases to run; all if omitted")
args = parser.parse_args()

# A class to encapsulate the check directives in a file.
class CheckFile:
    regex_dir = re.compile(r'^\s*//\s*(CHECK[^:]*):\s+(.+)$')
    ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')

    def __init__(self, checks, input):
        # Collect the directives in the file.
        self.dirs = list()
        for line in checks.splitlines():
            m = self.regex_dir.match(line)
            if m:
                self.dirs.append((m.group(1), m.group(2)))

        # Split input into lines.
        self.input = input.splitlines()

    def execute(self):
        self.failed = list()
        self.state = iter(self.input)

        # Execute the directives in order.
        for d in self.dirs:
            try:
                self.state = self.execute_directive(d, copy(self.state))
            except Exception as e:
                try:
                    line = next(copy(self.state)).strip()
                except StopIteration:
                    line = "<end of file>"
                self.failed.append((d, e.__str__(), line))

        # Concatenate the failures into information messages.
        info = ""
        for f in self.failed:
            info += "error: {}: {}\n  {}. Scanning from:\n  {}\n".format(f[0][0], f[0][1], f[1], f[2])

        # Package a return value.
        return (len(self.failed) == 0, info)

    def execute_directive(self, directive, state):
        dirname = directive[0].replace("CHECK-ERR", "CHECK")
        if dirname == "CHECK":
            for line in state:
                line = line.split("//")[0].strip()
                line = self.ansi_escape.sub("", line)
                if line == directive[1]:
                    return state
            raise Exception("No matching line found")
        else:
            raise Exception("Unknown directive `{}`".format(directive[0]))

# Handle the special case where we are just supposed to check a file.
check = None
if args.check:
    with open(args.check[0]) as f:
        content = f.read()
    check = (args.check[1], content)
if args.check_stdin:
    content = sys.stdin.read()
    check = (args.check_stdin, content)
if check is not None:
    with open(check[0]) as f:
        checks = f.read()
    passed, info = CheckFile(checks, check[1]).execute()
    if passed:
        sys.exit(0)
    else:
        sys.stdout.write(info)
        sys.exit(1)

# Establish some working directories.
crate_dir = Path(os.path.realpath(args.crate))
test_dir = Path(os.path.realpath(args.tests))

# Build the binaries if requested.
prefix = None
try:
    if args.debug or args.release:
        # Build
        if not args.no_build:
            cmd = ["cargo", "build", "--bins"]
            if args.release:
                cmd += ["--release"]
            subprocess.check_call(
                cmd,
                stdin=subprocess.DEVNULL,
                cwd=crate_dir.__str__(),
            )

        # Extract build directory
        # cargo metadata --format-version 1 | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p'
        metadata = subprocess.check_output(
            ["cargo", "metadata", "--format-version", "1"],
            stdin=subprocess.DEVNULL,
            cwd=crate_dir.__str__(),
            universal_newlines=True,
        )
        prefix = re.search(r'"target_directory":"([^"]*)"', metadata).group(1)
        if args.debug:
            prefix += "/debug"
        if args.release:
            prefix += "/release"
        prefix = os.path.realpath(prefix)+"/"
except Exception as e:
    sys.stderr.write("{}\n".format(e))
    sys.exit(1)

prefix = prefix or args.prefix or ""
if args.verbose:
    sys.stdout.write("# crate:   {}\n".format(crate_dir))
    sys.stdout.write("# test:    {}\n".format(test_dir))
    sys.stdout.write("# prefix:  {}\n".format(prefix))

# A class to encapsulate the execution and checking of a single test.
class TestCase(object):
    regex_ignore = re.compile(r'^\s*//\s*IGNORE\b', flags=re.MULTILINE)
    regex_fail   = re.compile(r'^\s*//\s*FAIL\b', flags=re.MULTILINE)
    regex_run    = re.compile(r'^\s*//\s*RUN:\s+(.+)$', flags=re.MULTILINE)

    def __init__(self, name, path):
        self.name = name
        self.path = Path(path)
        self.dir = self.path.parent

        # Load the contents of the test file.
        with open(path.__str__()) as f:
            self.content = f.read()

        # Check for a `IGNORE` directive.
        self.ignore = self.regex_ignore.search(self.content) is not None

        # Check for a `FAIL` directive.
        self.should_fail = self.regex_fail.search(self.content) is not None

        # Check for a `RUN` directive.
        self.run = self.regex_run.search(self.content)
        if self.run:
            self.run = self.run.group(1)
        else:
            self.run = "moore --syntax %s"

        # Execution results.
        self.timeout = False
        self.failed = False
        self.info = ""
        self.stdout = ""
        self.stderr = ""

        # Process the run command.
        self.cmd = shlex.split(self.run)
        if self.cmd[0] == "moore":
            self.cmd[0] = "{}{}".format(prefix, self.cmd[0])
        self.cmd = [path if x == "%s" else x for x in self.cmd]
        cmd = list()
        for x in self.cmd:
            if "*" in x.__str__():
                self.info += "Arg: `{}` is a glob pattern\n".format(x)
                cmd += self.dir.glob(x)
            elif (self.dir/x).exists():
                self.info += "Arg: `{}` is a path\n".format(x)
                cmd.append(self.dir/x)
            else:
                self.info += "Arg: `{}` is a plain argument\n".format(x)
                cmd.append(x)
        self.cmd = list([x.__str__() for x in cmd])

    def launch(self):
        if self.ignore:
            return
        try:
            self.info += "Command: {}\n".format(self.cmd)
            self.proc = subprocess.Popen(
                [x.__str__() for x in self.cmd],
                universal_newlines=True,
                stdin=subprocess.DEVNULL,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                cwd=crate_dir.__str__(),
            )
        except Exception as e:
            self.info += "Exception: {}\n".format(e)
            self.failed = True

    def finish(self):
        if self.ignore or self.failed:
            return
        try:
            self.stdout, self.stderr = self.proc.communicate(timeout=10)
        except subprocess.TimeoutExpired as e:
            self.proc.kill()
            self.stdout, self.stderr = self.proc.communicate()
            self.timeout = True
            self.failed = True
            self.info += "Timeout"
            return

        # Check the return code.
        self.failed = (self.proc.returncode != 0)

        # Check if we were supposed to fail, but didn't.
        if self.failed != self.should_fail:
            self.info += "Failed: {}\nShould Fail: {}\n".format(self.failed, self.should_fail)
            self.failed = True
            return
        else:
            self.failed = False

        # Perform the file checks.
        passed, info = CheckFile(self.content, self.stdout + self.stderr).execute()
        if not passed:
            self.failed = True
            self.info += "File checks failed:\n\n"
            self.info += info

# Collect the tests.
if args.TEST:
    tests = [TestCase(p, Path(os.path.realpath(p))) for p in args.TEST]
else:
    suffices = ["sv", "v", "vhd"]
    third_party = test_dir/"third-party"
    globs = [[p for p in test_dir.glob("**/*."+suffix) if third_party not in p.parents] for suffix in suffices]
    tests = [TestCase(p.relative_to(test_dir), p) for p in sorted(itertools.chain(*globs))]
sys.stdout.write("running {} tests\n".format(len(tests)))

# Assemble the test arrays.
num_parallel = 16
heads = tests + [None]*num_parallel
tails = [None]*num_parallel + tests

# Execute the tests and output test results.
ignored = list()
failed = list()
for head, tail in zip(heads, tails):
    if head:
        head.launch()
    if tail:
        sys.stdout.write("test {} ...".format(tail.name))
        sys.stdout.flush()
        tail.finish()
        if tail.ignore:
            ignored.append(tail)
            sys.stdout.write(" ignored\n")
            continue
        if tail.timeout:
            sys.stdout.write(" timeout,")
        if tail.failed:
            failed.append(tail)
            sys.stdout.write(" {}FAILED{}\n".format(cbold+cfail, creset))
            if args.verbose:
                sys.stdout.write("\n=== INFO ===\n")
                sys.stdout.write(tail.info)
                sys.stdout.write("\n=== STDERR ===\n")
                sys.stdout.write(tail.stderr)
                sys.stdout.write("\n=== STDOUT ===\n")
                sys.stdout.write(tail.stdout)
                sys.stdout.write("\n")
        else:
            sys.stdout.write(" {}passed{}\n".format(cpass, creset))
        if args.commands:
            sys.stdout.write("# {}\n".format(" ".join([x.__str__() for x in tail.cmd])))

# Output summary.
sys.stdout.write("\n")
if failed:
    sys.stdout.write("failures:\n")
    for test in failed:
        sys.stdout.write("    {}\n".format(test.name))
    sys.stdout.write("\n")
sys.stdout.write("test result: {}. {} passed, {} failed, {} ignored\n".format(
    "{}PASSED{}".format(cbold+cpass, creset) if not failed else "{}FAILED{}".format(cbold+cfail, creset),
    len(tests) - len(failed) - len(ignored),
    len(failed),
    len(ignored)
))

sys.exit(1 if failed else 0)
<|endoftext|>