<commit_before>'''
Created on 05 nov. 2015

@author: arxit
'''

import os

from qgis.core import *
from qgis.gui import *
from PyQt4.QtGui import QFileDialog, QMessageBox, QProgressBar
from PyQt4.QtCore import *

import PagLuxembourg.main
from PagLuxembourg.widgets.data_checker.data_checker import *

from importer import *

class ImportGML(Importer):
    '''
    Main class for the import data widget
    '''

    data_checker = DataChecker()

    def __init__(self, filename):
        '''
        Constructor

        :param filename: The GML filename
        :type filename: str, QString
        '''
        self.filename = filename

    def runImport(self):
        '''
        Import a GML file
        '''

        # Open GML
        file = QFile(self.filename)
        file.open(QIODevice.ReadOnly)
        gmlcontent = file.readAll()

        # Guess GML schema
        gmlschema = QgsGmlSchema()
        gmlschema.guessSchema(gmlcontent)

        # XSD Schema
        xsdschema = PagLuxembourg.main.xsd_schema

        unknowntypes = list()

        # Check schema structure table and datatypes
        layer_structure_errors = list()

        # Progress bar + message
        progressMessageBar = PagLuxembourg.main.qgis_interface.messageBar().createMessage(QCoreApplication.translate('ImportData','Importing GML'))
        progress = QProgressBar()
        progress.setMaximum(len(gmlschema.typeNames()))
        progress.setAlignment(Qt.AlignLeft|Qt.AlignVCenter)
        progressMessageBar.layout().addWidget(progress)
        PagLuxembourg.main.qgis_interface.messageBar().pushWidget(progressMessageBar, QgsMessageBar.INFO)

        # Define imported extent
        imported_extent = None

        # Loop GML types
        for gmltype in gmlschema.typeNames():
            xsdtype = xsdschema.getType(gmltype)
            if xsdtype is None:
                unknowntypes.append(gmltype)
                continue

            # Progression message
            progressMessageBar.setText(QCoreApplication.translate('ImportData','Importing {}').format(gmltype))

            gmllayer = QgsVectorLayer('{}|layername={}'.format(self.filename,gmltype), gmltype, "ogr")

            # Check schema structure table and datatypes
            warn_errors, fatal_errors = self.data_checker.checkLayerStructure(gmllayer, xsdtype)
            layer_structure_errors = layer_structure_errors + fatal_errors

            if len(fatal_errors) == 0:
                extent = self._importGmlLayer(gmllayer, xsdtype)

                if extent is not None:
                    if imported_extent is None:
                        imported_extent = extent
                    else:
                        imported_extent.combineExtentWith(extent)

            progress.setValue(progress.value() + 1)

        PagLuxembourg.main.qgis_interface.messageBar().clearWidgets()

        # Success message
        if len(layer_structure_errors) == 0:
            PagLuxembourg.main.qgis_interface.messageBar().pushSuccess(QCoreApplication.translate('ImportData','Success'),
                                                                       QCoreApplication.translate('ImportData','Importation was successful'))
            PagLuxembourg.main.qgis_interface.mapCanvas().setExtent(imported_extent)
        else:
            PagLuxembourg.main.qgis_interface.messageBar().pushWarning(QCoreApplication.translate('ImportData','Warning'),
                                                                       QCoreApplication.translate('ImportData','Some errors encountered during importation'))

            self.dlg = ErrorSummaryDialog(layer_structure_errors, list())
            self.dlg.show()

    def _importGmlLayer(self, gml_layer, xsdtype):
        '''
        Import a GML layer

        :param gml_layer: The GML layer to import
        :type gml_layer: QgsVectorLayer

        :param xsdtype: XSD schema type
        :type xsdtype: PAGType
        '''

        xsd_layer = PagLuxembourg.main.current_project.getLayer(xsdtype)

        if xsd_layer is None:
            return

        imported_extent = self._importLayer(gml_layer,
                                            xsd_layer,
                                            self._getFieldMap(gml_layer, xsd_layer, xsdtype))

        return imported_extent

    def _getFieldMap(self, source_layer, destination_layer, xsdtype):
        '''
        Get the field index map between the source layer and destination layer

        :param source_layer: The source layer to import
        :type source_layer: QgsVectorLayer

        :param destination_layer: The destination layer to write to
        :type destination_layer: QgsVectorLayer
        '''

        mapping = LayerMapping()

        source_fields = source_layer.dataProvider().fields()
        destination_fields = destination_layer.dataProvider().fields()

        for source_field in source_fields:
            if source_field.name() == xsdtype.geometry_fieldname:
                continue

            if source_field.name() == 'gml_id':
                continue

            destination_field_index = destination_fields.fieldNameIndex(source_field.name())

            if destination_field_index == -1:
                continue

            mapping.addFieldMapping(source_fields.fieldNameIndex(source_field.name()),
                                    destination_field_index,
                                    None)

        return mapping<commit_msg>Correction bug import GML<commit_after>'''
Created on 05 nov. 2015

@author: arxit
'''

import os

from qgis.core import *
from qgis.gui import *
from PyQt4.QtGui import QFileDialog, QMessageBox, QProgressBar
from PyQt4.QtCore import *

import PagLuxembourg.main
from PagLuxembourg.widgets.data_checker.data_checker import *

from importer import *

class ImportGML(Importer):
    '''
    Main class for the import data widget
    '''

    data_checker = DataChecker()

    def __init__(self, filename):
        '''
        Constructor

        :param filename: The GML filename
        :type filename: str, QString
        '''
        self.filename = filename

    def runImport(self):
        '''
        Import a GML file
        '''

        # Open GML
        file = QFile(self.filename)
        file.open(QIODevice.ReadOnly)
        gmlcontent = file.readAll()

        # Guess GML schema
        gmlschema = QgsGmlSchema()
        gmlschema.guessSchema(gmlcontent)

        # XSD Schema
        xsdschema = PagLuxembourg.main.xsd_schema

        unknowntypes = list()

        # Check schema structure table and datatypes
        layer_structure_errors = list()

        # Progress bar + message
        progressMessageBar = PagLuxembourg.main.qgis_interface.messageBar().createMessage(QCoreApplication.translate('ImportData','Importing GML'))
        progress = QProgressBar()
        progress.setMaximum(len(gmlschema.typeNames()))
        progress.setAlignment(Qt.AlignLeft|Qt.AlignVCenter)
        progressMessageBar.layout().addWidget(progress)
        PagLuxembourg.main.qgis_interface.messageBar().pushWidget(progressMessageBar, QgsMessageBar.INFO)

        # Define imported extent
        imported_extent = None

        # Loop GML types
        for gmltype in gmlschema.typeNames():
            xsdtype = xsdschema.getType(gmltype)
            if xsdtype is None:
                unknowntypes.append(gmltype)
                continue

            # Progression message
            progressMessageBar.setText(QCoreApplication.translate('ImportData','Importing {}').format(gmltype))

            gmllayer = QgsVectorLayer('{}|layername={}'.format(self.filename,gmltype), gmltype, "ogr")

            # Check schema structure table and datatypes
            warn_errors, fatal_errors = self.data_checker.checkLayerStructure(gmllayer, xsdtype)
            layer_structure_errors = layer_structure_errors + fatal_errors

            if len(fatal_errors) == 0:
                extent = self._importGmlLayer(gmllayer, xsdtype)

                if extent is not None:
                    if imported_extent is None:
                        imported_extent = extent
                    else:
                        imported_extent.combineExtentWith(extent)

            progress.setValue(progress.value() + 1)

        PagLuxembourg.main.qgis_interface.messageBar().clearWidgets()

        # Success message
        if len(layer_structure_errors) == 0:
            PagLuxembourg.main.qgis_interface.messageBar().pushSuccess(QCoreApplication.translate('ImportData','Success'),
                                                                       QCoreApplication.translate('ImportData','Importation was successful'))
            PagLuxembourg.main.qgis_interface.mapCanvas().setExtent(imported_extent)
        else:
            PagLuxembourg.main.qgis_interface.messageBar().pushWarning(QCoreApplication.translate('ImportData','Warning'),
                                                                       QCoreApplication.translate('ImportData','Some errors encountered during importation'))

            self.dlg = ErrorSummaryDialog(layer_structure_errors, list())
            self.dlg.show()

    def _importGmlLayer(self, gml_layer, xsdtype):
        '''
        Import a GML layer

        :param gml_layer: The GML layer to import
        :type gml_layer: QgsVectorLayer

        :param xsdtype: XSD schema type
        :type xsdtype: PAGType
        '''

        xsd_layer = PagLuxembourg.main.current_project.getLayer(xsdtype)

        if xsd_layer is None:
            return

        imported_extent = self._importLayer(gml_layer,
                                            xsd_layer,
                                            self._getFieldMap(gml_layer, xsd_layer, xsdtype))

        return imported_extent

    def _getFieldMap(self, source_layer, destination_layer, xsdtype):
        '''
        Get the field index map between the source layer and destination layer

        :param source_layer: The source layer to import
        :type source_layer: QgsVectorLayer

        :param destination_layer: The destination layer to write to
        :type destination_layer: QgsVectorLayer
        '''

        mapping = LayerMapping()

        source_fields = source_layer.dataProvider().fields()
        destination_fields = destination_layer.dataProvider().fields()

        for source_field in source_fields:
            if source_field.name() == xsdtype.geometry_fieldname:
                continue

            if source_field.name() == 'gml_id':
                continue

            destination_field_index = destination_fields.fieldNameIndex(source_field.name())

            if destination_field_index == -1:
                continue

            mapping.addFieldMapping(source_fields.fieldNameIndex(source_field.name()),
                                    destination_field_index,
                                    None,
                                    True)

        return mapping<|endoftext|>


<commit_before>
import os
import sys
import re
from pprint import pprint
import datetime
import dateutil
from dateutil import parser
from glob import glob
import calendar
from collections import namedtuple
import argparse
import textwrap
import codecs, locale

sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)

argparser = argparse.ArgumentParser(description='Help with scheduling')
argparser.add_argument('inputs', metavar='inputs', type=str, nargs='+',
                       help='input files or input strings')
argparser.add_argument('--start', dest='start', default=datetime.date.today(),
                       help='start date')
argparser.add_argument('--output_html', dest='output_html', action='store_true',
                       default=False, help='ascii codes or html')
argparser.add_argument('--no_align', dest='align', action='store_false',
                       default=True, help='align work if large vacation gaps are detected')
args = argparser.parse_args()

class DateSpan:
    begin = None
    duration = None

    def __init__(self, begin, end):
        if type(begin) == datetime.datetime:
            begin = begin.date()
        self.begin = begin
        if type(end) == int:
            end = dateutil.relativedelta.relativedelta(days=+end)
        self.end = end

    def __repr__(self):
        return '[{},{})'.format(self.begin, (self.begin + self.end))

    def overlaps(self, other):
        max_start = max(self.begin, other.begin)
        min_end = min(self.begin + self.end, other.begin + other.end)
        return (min_end - max_start).days > 0

Holiday = namedtuple('Holiday', ['name', 'span'])

class Entry:
    name = None
    short_name = None
    length = None

    def __init__(self, name, length, short_name = name):
        self.name = name.strip()
        self.length = length
        self.short_name = short_name

    def __repr__(self):
        return '{} - {}'.format(self.name, self.length)

def to_span(input_string):
    if '-' in input_string:
        l, r = input_string.split('-', 1)
        try:
            l = parser.parse(l)
            r = parser.parse(r)
        except ValueError, ex:
            print >> sys.stderr, 'WARNING: could not parse holiday line "', hol, ':', line, '"\n', ex
            raise
        return DateSpan(l, dateutil.relativedelta.relativedelta(r, l) + dateutil.relativedelta.relativedelta(days=+1))
    else:
        try:
            begin = parser.parse(input_string)
        except ValueError, ex:
            print >> sys.stderr, 'WARNING: could not parse holiday line "', hol, ':', line, '"\n', ex
            raise
        return DateSpan(begin, dateutil.relativedelta.relativedelta(days=+1))

# get holidays
holiday_files = [i for i in args.inputs if i.endswith('.HOL')]
holidays = []
for hol in holiday_files:
    for line in open(hol, 'rb').readlines()[1:]:
        line = line.strip()
        if len(line) == 0:
            continue
        parts = line.rsplit(',')
        if len(parts) != 2:
            print >> sys.stderr, 'WARNING: could not parse holiday line "', hol, ':', line, '"'
            continue
        try:
            span = to_span(parts[-1])
            holidays.append(Holiday(name=parts[0], span=span))
        except ValueError, ex:
            continue

# for entry in holidays:
    # print entry

# Grab all command line args that don't end with .HOL
input_args = [i for i in args.inputs if not i.endswith('.HOL')]
text_input = []
for argv in input_args:
    if os.path.exists(argv):
        text_input.append(open(argv, 'rb').read())
    else:
        text_input.append(argv)

text_input = '\n\n'.join(text_input)
text_input = [i.strip() for i in re.split(r'(?m)^\s*$\s*', text_input)]

duration_regex = re.compile(r'(?P<length>\d+)\s+(?P<kind>(weeks?|days?))')
tasks = []
for ti in text_input:
    try:
        desc, duration = ti.rsplit(':', 1)
    except ValueError:
        desc = ti
        duration = '1 day'
    if desc.startswith('--'):
        desc = desc[2:]
    m = duration_regex.match(duration.strip())
    if not m:
        print >> sys.stderr, 'WARNING: could not parse task line; exiting'
        print >> sys.stderr, ti
        print >> sys.stderr, duration
        sys.exit(1)
    parts = m.groupdict()
    parts['length'] = int(parts['length'])
    if parts['kind'].startswith('week'):
        parts['length'] = parts['length'] * 5
        parts['kind'] = 'days'
    try:
        short_name, name = desc.split('--', 1)
    except ValueError:
        name = desc
        short_name = ''
    tasks.append(Entry(name, parts['length'], short_name))

tasks = [t for t in tasks if len(t.name)]
# pprint(tasks)

# today = datetime.date.today()
# one_day = DateSpan(today + dateutil.relativedelta.relativedelta(days=+0), 8)
# two_day = DateSpan(today + dateutil.relativedelta.relativedelta(days=+7), 7)
# print one_day, two_day
# print one_day.overlaps(two_day)

def next_weekday(now):
    while True:
        now = now + dateutil.relativedelta.relativedelta(days=+1)
        if now.weekday() < 5:
            return now

def next_non_masked_weekday(now):
    while True:
        now = now + dateutil.relativedelta.relativedelta(days=+1)
        masks = [hol for hol in holidays if hol.span.overlaps(DateSpan(now, 1))]
        if now.weekday() < 5 and not any(masks):
            return now

if type(args.start) == str:
    args.start = parser.parse(args.start, fuzzy=True).date()
now = args.start
if now.weekday() >= 5:
    now = next_weekday(now)
for task in tasks:
    begin = None
    end = None
    day = 0
    num_masks = 0
    while day < task.length:
        while True:
            masks = [hol for hol in holidays if hol.span.overlaps(DateSpan(now, 1))]
            if any(masks):
                # print 'task was masked', task, masks[0]
                now = next_weekday(now)
                if begin != None:
                    num_masks += 1
            else:
                break
        # print 'scheduled ', task, 'on', now
        if not begin:
            begin = now
        now = next_weekday(now)
        end = now

        day += 1

        # Handle alignment
        if args.align and (num_masks > task.length):
            # find first non-masked Monday and restart
            orig = begin
            begin = next_non_masked_weekday(begin)
            while begin.weekday() != 0:
                begin = next_non_masked_weekday(begin)
            day = 0
            now = begin
            num_masks = 0

    task.begin = begin
    task.end = end

blue = '\033[94m'
reset = '\033[0m'
if args.output_html:
    blue = ''
    reset = ''

for task in tasks:
    half_open_adjustment = dateutil.relativedelta.relativedelta(days=+1)
    cals = []

    cal = calendar.TextCalendar(calendar.SUNDAY)
    text = cal.formatmonth(task.begin.year, task.begin.month)
    regex = re.compile(r'(\b|^){0}(\b|$)'.format(task.begin.day))
    m = regex.search(text)
    text = text[:m.start()] + blue + text[m.start():]

    if (task.begin.year == task.end.year) and (task.begin.month == task.end.month):
        regex = re.compile(r'(\b|^|{1}){0}(\b|$|{1})'.format((task.end - half_open_adjustment).day, re.escape(blue)))
        m = regex.search(text)
        if not m:
            print text
            print regex.pattern
            sys.exit(1)
        text = text[:m.end()] + reset + text[m.end():]
        cals.append(text)
    else:
        text = text[:-1] + reset + text[-1:]
        cals.append(text)

        month = task.begin + dateutil.relativedelta.relativedelta(months=+1)
        while (month < task.end) and (month.month != task.end.month):
            text = cal.formatmonth(month.year, month.month)
            regex = re.compile(r'(\b|^)1(\b|$)')
            m = regex.search(text)
            text = text[:m.start()] + blue + text[m.start():-1] + reset + text[-1:]
            month = month + dateutil.relativedelta.relativedelta(months=+1)
            cals.append(text)
        if month.month == task.end.month:
            text = cal.formatmonth(month.year, month.month)
            regex = re.compile(r'(\b|^)1(\b|$)')
            m = regex.search(text)
            text = text[:m.start()] + blue + text[m.start():]

            regex = re.compile(r'(\b|^){0}(\b|$)'.format((task.end - half_open_adjustment).day))
            m = regex.search(text)
            text = text[:m.end()] + reset + text[m.end():]
            cals.append(text)

    task.cals = ''.join(cals)

html_header = '''
<!DOCTYPE html>

    <meta charset="UTF-8">
    <title>Schedule</title>

'''
if args.output_html:
    print html_header

table = []
for task in tasks:
    name = ['']
    regex = re.compile(r'^(\s*)(\d+\.|-|\*) .*')
    for line in task.name.split('\n'):
        m = regex.search(line)
        if m:
            indent = ' '*(len(m.group(1)) + len(m.group(2)) + 1)
            name += textwrap.wrap(line.strip(), 60, initial_indent=' '*len(m.group(1)), subsequent_indent=indent, break_on_hyphens=True, break_long_words=False)
        else:
            name += textwrap.wrap(line, 60, break_on_hyphens=True, break_long_words=False)
    cals = task.cals.split('\n')

    lines = max(len(name), len(cals))
    cal_width = max(*[len(re.sub('{0}|{1}'.format(re.escape(blue), re.escape(reset)), '', l)) for l in cals])
    cal_format = '{{0: <{0}}}'.format(cal_width)

    date_range = '{0} [{1},{2})'.format(task.short_name, task.begin, task.end)
    print '{0: ^80}'.format(date_range)
    print u'\u2500' * 80
    is_blue = False
    for i in range(lines):
        if i < len(cals):
            b = ''
            if is_blue:
                b = blue
            formatted_cal = b + cal_format.format(cals[i])
            print formatted_cal,
            actual_len = len(formatted_cal.replace(blue, '').replace(reset, ''))
            if actual_len < cal_width:
                print ' ' * (cal_width - actual_len - 1),
            if formatted_cal.find(blue) != -1 and formatted_cal.find(reset) != -1:
                print u'  {0}\u2502{1} '.format(blue, reset),
            elif formatted_cal.find(blue) == -1 and formatted_cal.find(reset) != -1:
                print u'  {0}\u2502{1} '.format(blue, reset),
            elif formatted_cal.find(blue) != -1 and formatted_cal.find(reset) == -1:
                print u'  \u2502 ',
            else:
                print u'  \u2502 ',

            if formatted_cal.find(blue) != -1:
                is_blue = True
            if formatted_cal.find(reset) != -1:
                is_blue = False
        else:
            print cal_format.format(''),
            print u'  \u2502 ',

        if i < len(name):
            print '{0}{1}'.format(reset, name[i]),

        print '{0}'.format(reset)
    print '{0}\n'.format(reset)

html_footer = '''

'''
if args.output_html:
    print html_footer

<commit_msg>First pass at trying to handle unexpected formatting. Not great, but better than nothing.<commit_after>
import os
import sys
import re
from pprint import pprint
import datetime
import dateutil
from dateutil import parser
from glob import glob
import calendar
from collections import namedtuple
import argparse
import textwrap
import codecs, locale

sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)

argparser = argparse.ArgumentParser(description='Help with scheduling')
argparser.add_argument('inputs', metavar='inputs', type=str, nargs='+',
                       help='input files or input strings')
argparser.add_argument('--start', dest='start', default=datetime.date.today(),
                       help='start date')
argparser.add_argument('--output_html', dest='output_html', action='store_true',
                       default=False, help='ascii codes or html')
argparser.add_argument('--no_align', dest='align', action='store_false',
                       default=True, help='align work if large vacation gaps are detected')
args = argparser.parse_args()

class DateSpan:
    begin = None
    duration = None

    def __init__(self, begin, end):
        if type(begin) == datetime.datetime:
            begin = begin.date()
        self.begin = begin
        if type(end) == int:
            end = dateutil.relativedelta.relativedelta(days=+end)
        self.end = end

    def __repr__(self):
        return '[{},{})'.format(self.begin, (self.begin + self.end))

    def overlaps(self, other):
        max_start = max(self.begin, other.begin)
        min_end = min(self.begin + self.end, other.begin + other.end)
        return (min_end - max_start).days > 0

Holiday = namedtuple('Holiday', ['name', 'span'])

class Entry:
    name = None
    short_name = None
    length = None

    def __init__(self, name, length, short_name = name):
        self.name = name.strip()
        self.length = length
        self.short_name = short_name

    def __repr__(self):
        return '{} - {}'.format(self.name, self.length)

def to_span(input_string):
    if '-' in input_string:
        l, r = input_string.split('-', 1)
        try:
            l = parser.parse(l)
            r = parser.parse(r)
        except ValueError, ex:
            print >> sys.stderr, 'WARNING: could not parse holiday line "', hol, ':', line, '"\n', ex
            raise
        return DateSpan(l, dateutil.relativedelta.relativedelta(r, l) + dateutil.relativedelta.relativedelta(days=+1))
    else:
        try:
            begin = parser.parse(input_string)
        except ValueError, ex:
            print >> sys.stderr, 'WARNING: could not parse holiday line "', hol, ':', line, '"\n', ex
            raise
        return DateSpan(begin, dateutil.relativedelta.relativedelta(days=+1))

# get holidays
holiday_files = [i for i in args.inputs if i.endswith('.HOL')]
holidays = []
for hol in holiday_files:
    for line in open(hol, 'rb').readlines()[1:]:
        line = line.strip()
        if len(line) == 0:
            continue
        parts = line.rsplit(',')
        if len(parts) != 2:
            print >> sys.stderr, 'WARNING: could not parse holiday line "', hol, ':', line, '"'
            continue
        try:
            span = to_span(parts[-1])
            holidays.append(Holiday(name=parts[0], span=span))
        except ValueError, ex:
            continue

# for entry in holidays:
    # print entry

# Grab all command line args that don't end with .HOL
input_args = [i for i in args.inputs if not i.endswith('.HOL')]
text_input = []
for argv in input_args:
    if os.path.exists(argv):
        text_input.append(open(argv, 'rb').read())
    else:
        text_input.append(argv)

text_input = '\n\n'.join(text_input)
text_input = [i.strip() for i in re.split(r'(?m)^\s*$\s*', text_input)]

duration_regex = re.compile(r'(?P<length>\d+)\s+(?P<kind>(weeks?|days?))')
tasks = []
for ti in text_input:
    if ti.strip() == '~!~':
        break
    try:
        desc, duration = ti.rsplit(':', 1)
    except ValueError:
        desc = ti
        duration = '1 day'
    if desc.startswith('--'):
        desc = desc[2:]
    m = duration_regex.match(duration.strip())
    if not m:
        print >> sys.stderr, 'WARNING: could not parse task line; exiting'
        print >> sys.stderr, ti
        print >> sys.stderr, duration
        sys.exit(1)
    parts = m.groupdict()
    parts['length'] = int(parts['length'])
    if parts['kind'].startswith('week'):
        parts['length'] = parts['length'] * 5
        parts['kind'] = 'days'
    try:
        short_name, name = desc.split('--', 1)
    except ValueError:
        name = desc
        short_name = ''
    tasks.append(Entry(name, parts['length'], short_name))

tasks = [t for t in tasks if len(t.name)]
# pprint(tasks)

# today = datetime.date.today()
# one_day = DateSpan(today + dateutil.relativedelta.relativedelta(days=+0), 8)
# two_day = DateSpan(today + dateutil.relativedelta.relativedelta(days=+7), 7)
# print one_day, two_day
# print one_day.overlaps(two_day)

def next_weekday(now):
    while True:
        now = now + dateutil.relativedelta.relativedelta(days=+1)
        if now.weekday() < 5:
            return now

def next_non_masked_weekday(now):
    while True:
        now = now + dateutil.relativedelta.relativedelta(days=+1)
        masks = [hol for hol in holidays if hol.span.overlaps(DateSpan(now, 1))]
        if now.weekday() < 5 and not any(masks):
            return now

if type(args.start) == str:
    args.start = parser.parse(args.start, fuzzy=True).date()
now = args.start
if now.weekday() >= 5:
    now = next_weekday(now)
for task in tasks:
    begin = None
    end = None
    day = 0
    num_masks = 0
    while day < task.length:
        while True:
            masks = [hol for hol in holidays if hol.span.overlaps(DateSpan(now, 1))]
            if any(masks):
                # print 'task was masked', task, masks[0]
                now = next_weekday(now)
                if begin != None:
                    num_masks += 1
            else:
                break
        # print 'scheduled ', task, 'on', now
        if not begin:
            begin = now
        now = next_weekday(now)
        end = now

        day += 1

        # Handle alignment
        if args.align and (num_masks > task.length):
            # find first non-masked Monday and restart
            orig = begin
            begin = next_non_masked_weekday(begin)
            while begin.weekday() != 0:
                begin = next_non_masked_weekday(begin)
            day = 0
            now = begin
            num_masks = 0

    task.begin = begin
    task.end = end

blue = '\033[94m'
reset = '\033[0m'
if args.output_html:
    blue = ''
    reset = ''

def bail_if_none(m, text):
    if not m:
        raise RuntimeError('Error processing "{0}"'.format(text))
    return m

for task in tasks:
    half_open_adjustment = dateutil.relativedelta.relativedelta(days=+1)
    cals = []

    cal = calendar.TextCalendar(calendar.SUNDAY)
    text = cal.formatmonth(task.begin.year, task.begin.month)
    regex = re.compile(r'(\b|^){0}(\b|$)'.format(task.begin.day))
    m = bail_if_none(regex.search(text), text)
    text = text[:m.start()] + blue + text[m.start():]

    if (task.begin.year == task.end.year) and (task.begin.month == task.end.month):
        regex = re.compile(r'(\b|^|{1}){0}(\b|$|{1})'.format((task.end - half_open_adjustment).day, re.escape(blue)))
        m = regex.search(text)
        if not m:
            print text
            print regex.pattern
            sys.exit(1)
        text = text[:m.end()] + reset + text[m.end():]
        cals.append(text)
    else:
        text = text[:-1] + reset + text[-1:]
        cals.append(text)

        month = task.begin + dateutil.relativedelta.relativedelta(months=+1)
        while (month < task.end) and (month.month != task.end.month):
            text = cal.formatmonth(month.year, month.month)
            regex = re.compile(r'(\b|^)1(\b|$)')
            m = bail_if_none(regex.search(text), text)
            text = text[:m.start()] + blue + text[m.start():-1] + reset + text[-1:]
            month = month + dateutil.relativedelta.relativedelta(months=+1)
            cals.append(text)
        if month.month == task.end.month:
            text = cal.formatmonth(month.year, month.month)
            regex = re.compile(r'(\b|^)1(\b|$)')
            m = bail_if_none(regex.search(text), text)
            text = text[:m.start()] + blue + text[m.start():]

            regex = re.compile(r'(\b|^){0}(\b|$)'.format((task.end - half_open_adjustment).day))
            m = bail_if_none(regex.search(text), text)
            text = text[:m.end()] + reset + text[m.end():]
            cals.append(text)

    task.cals = ''.join(cals)

html_header = '''
<!DOCTYPE html>

    <meta charset="UTF-8">
    <title>Schedule</title>

'''
if args.output_html:
    print html_header

table = []
for task in tasks:
    name = ['']
    regex = re.compile(r'^(\s*)(\d+\.|-|\*) .*')
    for line in task.name.split('\n'):
        m = regex.search(line)
        if m:
            indent = ' '*(len(m.group(1)) + len(m.group(2)) + 1)
            name += textwrap.wrap(line.strip(), 60, initial_indent=' '*len(m.group(1)), subsequent_indent=indent, break_on_hyphens=True, break_long_words=False)
        else:
            name += textwrap.wrap(line, 60, break_on_hyphens=True, break_long_words=False)
    cals = task.cals.split('\n')

    lines = max(len(name), len(cals))
    cal_width = max(*[len(re.sub('{0}|{1}'.format(re.escape(blue), re.escape(reset)), '', l)) for l in cals])
    cal_format = '{{0: <{0}}}'.format(cal_width)

    date_range = '{0} [{1},{2})'.format(task.short_name, task.begin, task.end)
    print '{0: ^80}'.format(date_range)
    print u'\u2500' * 80
    is_blue = False
    for i in range(lines):
        if i < len(cals):
            b = ''
            if is_blue:
                b = blue
            formatted_cal = b + cal_format.format(cals[i])
            print formatted_cal,
            actual_len = len(formatted_cal.replace(blue, '').replace(reset, ''))
            if actual_len < cal_width:
                print ' ' * (cal_width - actual_len - 1),
            if formatted_cal.find(blue) != -1 and formatted_cal.find(reset) != -1:
                print u'  {0}\u2502{1} '.format(blue, reset),
            elif formatted_cal.find(blue) == -1 and formatted_cal.find(reset) != -1:
                print u'  {0}\u2502{1} '.format(blue, reset),
            elif formatted_cal.find(blue) != -1 and formatted_cal.find(reset) == -1:
                print u'  \u2502 ',
            else:
                print u'  \u2502 ',

            if formatted_cal.find(blue) != -1:
                is_blue = True
            if formatted_cal.find(reset) != -1:
                is_blue = False
        else:
            print cal_format.format(''),
            print u'  \u2502 ',

        if i < len(name):
            print '{0}{1}'.format(reset, name[i]),

        print '{0}'.format(reset)
    print '{0}\n'.format(reset)

html_footer = '''

'''
if args.output_html:
    print html_footer

<|endoftext|>


<commit_before># Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Javadoc rule.

def _impl(ctx):
    zip_output = ctx.outputs.zip

    transitive_jar_set = depset()
    source_jars = depset()
    for l in ctx.attr.libs:
        source_jars += l.java.source_jars
        transitive_jar_set += l.java.transitive_deps

    transitive_jar_paths = [j.path for j in transitive_jar_set.to_list()]
    dir = ctx.outputs.zip.path + ".dir"
    source = ctx.outputs.zip.path + ".source"
    external_docs = ["http://docs.oracle.com/javase/8/docs/api"] + ctx.attr.external_docs
    cmd = [
        "TZ=UTC",
        "export TZ",
        "rm -rf %s" % source,
        "mkdir %s" % source,
        " && ".join(["unzip -qud %s %s" % (source, j.path) for j in source_jars.to_list()]),
        "rm -rf %s" % dir,
        "mkdir %s" % dir,
        " ".join([
            "%s/bin/javadoc" % ctx.attr._jdk[java_common.JavaRuntimeInfo].java_home,
            "-Xdoclint:-missing",
            "-protected",
            "-encoding UTF-8",
            "-charset UTF-8",
            "-notimestamp",
            "-quiet",
            "-windowtitle '%s'" % ctx.attr.title,
            " ".join(["-link %s" % url for url in external_docs]),
            "-sourcepath %s" % source,
            "-subpackages ",
            ":".join(ctx.attr.pkgs),
            " -classpath ",
            ":".join(transitive_jar_paths),
            "-d %s" % dir,
        ]),
        "find %s -exec touch -t 198001010000 '{}' ';'" % dir,
        "(cd %s && zip -Xqr ../%s *)" % (dir, ctx.outputs.zip.basename),
    ]
    ctx.actions.run_shell(
        inputs = transitive_jar_set.to_list() + source_jars.to_list() + ctx.files._jdk,
        outputs = [zip_output],
        command = " && ".join(cmd),
    )

java_doc = rule(
    attrs = {
        "external_docs": attr.string_list(),
        "libs": attr.label_list(allow_files = False),
        "pkgs": attr.string_list(),
        "title": attr.string(),
        "_jdk": attr.label(
            default = Label("@bazel_tools//tools/jdk:current_java_runtime"),
            allow_files = True,
            providers = [java_common.JavaRuntimeInfo],
        ),
    },
    outputs = {"zip": "%{name}.zip"},
    implementation = _impl,
)
<commit_msg>javadoc.bzl: Avoid using deprecated depset union<commit_after># Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Javadoc rule.

def _impl(ctx):
    zip_output = ctx.outputs.zip

    transitive_jars = depset(transitive = [l.java.transitive_deps for l in ctx.attr.libs])
    source_jars = depset(transitive = [l.java.source_jars for l in ctx.attr.libs])

    transitive_jar_paths = [j.path for j in transitive_jars.to_list()]
    dir = ctx.outputs.zip.path + ".dir"
    source = ctx.outputs.zip.path + ".source"
    external_docs = ["http://docs.oracle.com/javase/8/docs/api"] + ctx.attr.external_docs
    cmd = [
        "TZ=UTC",
        "export TZ",
        "rm -rf %s" % source,
        "mkdir %s" % source,
        " && ".join(["unzip -qud %s %s" % (source, j.path) for j in source_jars.to_list()]),
        "rm -rf %s" % dir,
        "mkdir %s" % dir,
        " ".join([
            "%s/bin/javadoc" % ctx.attr._jdk[java_common.JavaRuntimeInfo].java_home,
            "-Xdoclint:-missing",
            "-protected",
            "-encoding UTF-8",
            "-charset UTF-8",
            "-notimestamp",
            "-quiet",
            "-windowtitle '%s'" % ctx.attr.title,
            " ".join(["-link %s" % url for url in external_docs]),
            "-sourcepath %s" % source,
            "-subpackages ",
            ":".join(ctx.attr.pkgs),
            " -classpath ",
            ":".join(transitive_jar_paths),
            "-d %s" % dir,
        ]),
        "find %s -exec touch -t 198001010000 '{}' ';'" % dir,
        "(cd %s && zip -Xqr ../%s *)" % (dir, ctx.outputs.zip.basename),
    ]
    ctx.actions.run_shell(
        inputs = transitive_jars.to_list() + source_jars.to_list() + ctx.files._jdk,
        outputs = [zip_output],
        command = " && ".join(cmd),
    )

java_doc = rule(
    attrs = {
        "external_docs": attr.string_list(),
        "libs": attr.label_list(allow_files = False),
        "pkgs": attr.string_list(),
        "title": attr.string(),
        "_jdk": attr.label(
            default = Label("@bazel_tools//tools/jdk:current_java_runtime"),
            allow_files = True,
            providers = [java_common.JavaRuntimeInfo],
        ),
    },
    outputs = {"zip": "%{name}.zip"},
    implementation = _impl,
)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2009, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Simple Code Hygiene tool meant to be run as a presubmit test or standalone

Usage:
./code_hygiene.py <file1> <file2> ...

For the styleguides see:
http://code.google.com/p/soc/wiki/PythonStyleGuide
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
"""

import os
import re
import signal
import subprocess
import sys
import time

# ======================================================================
VERBOSE = 0
LIMIT = 5
TIMEOUT = 10

# These program are likely not available/working on windows

# http://tidy.sourceforge.net/
HTML_CHECKER = ['tidy', '-errors']

# From depot_tools
# to see a list of all filters run: 'depot_tools/cpplint.py --filter='
CPP_CHECKER = ['cpplint.py', '--filter=-build/header_guard']

# From depot_tools (currently not uses -- too many false positives
# to see a list of all filters run: 'depot_tools/cpplint.py --filter='
C_CHECKER = ['cpplint.py', '--filter=-build/header_guard']

# http://pychecker.sourceforge.net/
PYTHON_CHECKER = ['pychecker']

RE_IGNORE = re.compile(r'@IGNORE_LINES_FOR_CODE_HYGIENE\[([0-9]+)\]')

# ======================================================================
def Debug(s):
  if VERBOSE:
    print s

def RunCommand(cmd):
  """Run a shell command given as an argv style vector."""
  Debug(str(cmd))

  start = time.time()
  p = subprocess.Popen(cmd,
                       bufsize=1000*1000,
                       stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)
  while p.poll() is None:
    time.sleep(1)
    now = time.time()
    if now - start > TIMEOUT:
      Debug('Error: timeout')
      os.kill(p.pid, signal.SIGKILL)
      os.waitpid(-1, os.WNOHANG)
      return -666, "", ""
  stdout = p.stdout.read()
  stderr = p.stderr.read()
  retcode = p.wait()

  return retcode, stdout, stderr

# ======================================================================
class ExternalChecker(object):
  """Base Class"""
  def __init__(self, commandline, use_stderr=False):
    self._commandline = commandline
    self._use_stderr = use_stderr

  def FindProblems(self, filename, unused_data):
    try:
      retcode, stdout, stderr = RunCommand(self._commandline + [filename])
    except Exception, err:
      print 'cannot excuted of command %s failed reason: %s ' % (
          str(self._commandline), str(err))
      return []
    if retcode == 0:
      return []
    if retcode == 666:
      print 'command %s timed out' % str(self._commandline)
      return []
    if self._use_stderr:
      return [stderr]
    else:
      return [stdout]

  def FileFilter(self, unused_filename):
    raise NotImplementedError()

class TidyChecker(ExternalChecker):
  """Invoke tidy tool on html files."""
  def __init__(self):
    ExternalChecker.__init__(self, HTML_CHECKER, use_stderr=True)
    return

  def FileFilter(self, filename):
    return filename.endswith('.html')

class PyChecker(ExternalChecker):
  """Invoke pychecker tool on python files."""
  def __init__(self):
    ExternalChecker.__init__(self, PYTHON_CHECKER)
    return

  def FileFilter(self, filename):
    return filename.endswith('.py')

class CppLintChecker(ExternalChecker):
  """Invoke Google's cpplint tool on c++ files."""
  def __init__(self):
    ExternalChecker.__init__(self, CPP_CHECKER, use_stderr=True)
    return

  def FileFilter(self, filename):
    return filename.endswith('.cc') or filename.endswith('.h')

# ======================================================================
class GenericRegexChecker(object):
  """Base Class"""
  def __init__(self, content_regex, line_regex=None, analyze_match=False):
    self._content_regex = re.compile(content_regex, re.MULTILINE)
    if line_regex:
      self._line_regex = re.compile(line_regex)
    else:
      self._line_regex = re.compile(content_regex)
    self._analyze_match = analyze_match
    return

  def FindProblems(self, unused_filename, data):
    # speedy early out
    if not self._content_regex.search(data):
      return []

    lines = data.split('\n')
    problem = []
    ignore_line_count = 0
    for no, line in enumerate(lines):
      if ignore_line_count > 0:
        ignore_line_count -= 1
        continue
      match = RE_IGNORE.search(line)
      if match:
        ignore_line_count = int(match.group(1))
        continue
      match = self._line_regex.search(line)
      if not match:
        continue
      if self._analyze_match:
        if not self.IsProblemMatch(match):
          continue
      problem.append(self._RenderItem(no, line))
    return problem

  def _RenderItem(self, no, line):
      return 'line %d: [%s]' % (no, repr(line))

  def FileFilter(self, unused_filename):
    raise NotImplementedError()

class OpenCurlyChecker(GenericRegexChecker):
  """Checks for orphan opening curly braces"""

  def __init__(self):
    GenericRegexChecker.__init__(self, r'^ *{ *$')
    return

  def FileFilter(self, filename):
    return (filename.endswith('.h') or
            filename.endswith('.cc') or
            filename.endswith('.c'))

class TrailingWhiteSpaceChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r' $')
    return

  def FileFilter(self, unused_filename):
    # check all files
    return 1

class CarriageReturnChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'\r')
    return

  def FileFilter(self, filename):
    return (filename.endswith('.py') or
            filename.endswith('.h') or
            filename.endswith('.cc') or
            filename.endswith('.c'))

class CppCommentChecker(GenericRegexChecker):

  def __init__(self):
    # we tolerate http:// etc in comments
    GenericRegexChecker.__init__(self, r'(^|[^:])//')
    return

  def FileFilter(self, filename):
    return filename.endswith('.c')

class TabsChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'\t')
    return

  def FileFilter(self, filename):
    return ("akefile" not in filename
        and not filename.endswith('.patch')
        and not filename.endswith('.valerr')
        and not filename.endswith('.dis'))

class FixmeChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'FIXME[:]')
    return

  def FileFilter(self, filename):
    return 1

class ExternChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'^ *extern\s+(.*)$', analyze_match=True)
    return

  def IsProblemMatch(self, match):
    extra = match.group(1).strip()
    # allow 'extern "C" {'
    # but do not allow plain 'extern "C"'
    if extra == '"C" {':
      return False
    return True

  def FileFilter(self, filename):
    return filename.endswith('.c') or filename.endswith('.cc')

class RewriteChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'[@][@]REWRITE')
    return

  def FileFilter(self, unused_filename):
    return 1

VALID_INCLUDE_PREFIX = [
    'native_client/',
    'third_party/',
    'gtest/',
    'gen/native_client/',
    ]

class IncludeChecker(object):
  """Enforce some base 'include' rules. Cpplint does more of these."""
  def __init__(self):
    pass

  def FindProblems(self, unused_filename, data):
    lines = data.split('\n')
    problem = []
    seen_code = False
    for no, line in enumerate(lines):
      if line.startswith('}'):
        # this test is our hacked indicator signaling that no includes
        # directives should follow after we have seen actual code
        seen_code = True
        continue

      if not line.startswith('#include'):
        continue
      if seen_code:
        problem.append('line %d: [%s]' % (no, repr(line)))
      token = line.split()
      path = token[1][1:]
      if '..' in path:
        problem.append('line %d: [%s]' % (no, repr(line)))
      if not token[1].startswith('<'):
        for prefix in VALID_INCLUDE_PREFIX:
          if path.startswith(prefix):
            break
        else:
          # no prefix matched
          problem.append('line %d: [%s]' % (no, repr(line)))

    return problem

  def FileFilter(self, filename):
    return filename.endswith('.c') or filename.endswith('.cc')

class LineLengthChecker(object):
  def __init__(self):
    pass

  def FindProblems(self, unused_filename, data):
    lines = data.split('\n')
    problem = []
    for no, line in enumerate(lines):
      if len(line) <= 80:
        continue
      line = line.strip()
      # we allow comments and as it so happens pre processor directives
      # to be longer
      if line.startswith('//') or line.startswith('/*') or line.startswith('#'):
        continue

      problem.append('line %d: [%s]' % (no + 1, repr(line)))
    return problem

  def FileFilter(self, filename):
    return 1

# ======================================================================
def IsBinary(data):
  return '\x00' in data

def FindExemptions(data):
  match = re.search(r'@EXEMPTION\[([^\]]*)\]', data)
  if match:
    return match.group(1).split(',')
  return []

# ======================================================================
CHECKS = [# fatal checks
          (1, 'tabs', TabsChecker()),
          (1, 'trailing_space', TrailingWhiteSpaceChecker()),
          (1, 'cpp_comment', CppCommentChecker()),
          (1, 'fixme', FixmeChecker()),
          (1, 'include', IncludeChecker()),
          (1, 'extern', ExternChecker()),
          # Non fatal checks
          (0, 'open_curly', OpenCurlyChecker()),
          (0, 'line_length', LineLengthChecker()),
          (0, 'carriage_return', CarriageReturnChecker()),
          (0, 'rewrite', RewriteChecker()),
          (0, 'tidy', TidyChecker()),
          (0, 'pychecker', PyChecker()),
          (0, 'cpplint', CppLintChecker()),
          ]

def EmitStatus(filename, status, details=[]):
  print '%s: %s' % (filename, status)
  for no, line in details[:LIMIT]:
    print '  line %d: [%s]' % (no, repr(line))
  return

def CheckFile(filename, report):
    errors = {}
    warnings = {}

    data = open(filename).read()

    if IsBinary(data):
      EmitStatus(filename, 'binary')
      return errors, warnings

    exemptions = FindExemptions(data)

    for fatal, name, check in CHECKS:
      Debug('trying %s' % name)

      if not check.FileFilter(filename):
        Debug('skipping')
        continue

      if name in exemptions:
        Debug('exempt')
        continue
      problems = check.FindProblems(filename, data)
      if problems:
        info =  '%s: %s' % (filename, name)
        items = problems[:LIMIT]
        if report:
          print info
          for i in items:
            print i
        if fatal:
          errors[info] = items
          break
        else:
          warnings[info] = items
    return errors, warnings

def HasGypFileCorrespondingToSconsFile(sconsfile, list):
  scons_root = os.path.dirname(sconsfile)
  for name in list:
    if name.startswith(scons_root) and name.endswith('.gyp'):
      return True
  return False

def main(argv):
  num_error = 0
  for filename in argv:
    if os.path.isdir(filename):
      EmitStatus(filename, 'directory')
      continue

    if '-' in os.path.basename(filename):
      print 'hyphen in:', filename
      num_error += 1
      continue

    # Verify that gyp files are updated together with scons files.
    # We probably won't have any gyp files for the untrusted code.
    if filename.endswith('.scons') and -1 == filename.find('untrusted'):
      if not HasGypFileCorrespondingToSconsFile(filename, argv):
        print 'please update the gyp file relevant for ', filename
        num_error += 1

    errors, warnings = CheckFile(filename, True)
    if errors:
      num_error += 1
  return num_error

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<commit_msg>Exempted golden files from the line length check. Review URL: http://codereview.chromium.org/195017<commit_after>#!/usr/bin/python
# Copyright 2009, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Simple Code Hygiene tool meant to be run as a presubmit test or standalone

Usage:
./code_hygiene.py <file1> <file2> ...

For the styleguides see:
http://code.google.com/p/soc/wiki/PythonStyleGuide
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
"""

import os
import re
import signal
import subprocess
import sys
import time

# ======================================================================
VERBOSE = 0
LIMIT = 5
TIMEOUT = 10

# These program are likely not available/working on windows

# http://tidy.sourceforge.net/
HTML_CHECKER = ['tidy', '-errors']

# From depot_tools
# to see a list of all filters run: 'depot_tools/cpplint.py --filter='
CPP_CHECKER = ['cpplint.py', '--filter=-build/header_guard']

# From depot_tools (currently not uses -- too many false positives
# to see a list of all filters run: 'depot_tools/cpplint.py --filter='
C_CHECKER = ['cpplint.py', '--filter=-build/header_guard']

# http://pychecker.sourceforge.net/
PYTHON_CHECKER = ['pychecker']

RE_IGNORE = re.compile(r'@IGNORE_LINES_FOR_CODE_HYGIENE\[([0-9]+)\]')

# ======================================================================
def Debug(s):
  if VERBOSE:
    print s

def RunCommand(cmd):
  """Run a shell command given as an argv style vector."""
  Debug(str(cmd))

  start = time.time()
  p = subprocess.Popen(cmd,
                       bufsize=1000*1000,
                       stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)
  while p.poll() is None:
    time.sleep(1)
    now = time.time()
    if now - start > TIMEOUT:
      Debug('Error: timeout')
      os.kill(p.pid, signal.SIGKILL)
      os.waitpid(-1, os.WNOHANG)
      return -666, "", ""
  stdout = p.stdout.read()
  stderr = p.stderr.read()
  retcode = p.wait()

  return retcode, stdout, stderr

# ======================================================================
class ExternalChecker(object):
  """Base Class"""
  def __init__(self, commandline, use_stderr=False):
    self._commandline = commandline
    self._use_stderr = use_stderr

  def FindProblems(self, filename, unused_data):
    try:
      retcode, stdout, stderr = RunCommand(self._commandline + [filename])
    except Exception, err:
      print 'cannot excuted of command %s failed reason: %s ' % (
          str(self._commandline), str(err))
      return []
    if retcode == 0:
      return []
    if retcode == 666:
      print 'command %s timed out' % str(self._commandline)
      return []
    if self._use_stderr:
      return [stderr]
    else:
      return [stdout]

  def FileFilter(self, unused_filename):
    raise NotImplementedError()

class TidyChecker(ExternalChecker):
  """Invoke tidy tool on html files."""
  def __init__(self):
    ExternalChecker.__init__(self, HTML_CHECKER, use_stderr=True)
    return

  def FileFilter(self, filename):
    return filename.endswith('.html')

class PyChecker(ExternalChecker):
  """Invoke pychecker tool on python files."""
  def __init__(self):
    ExternalChecker.__init__(self, PYTHON_CHECKER)
    return

  def FileFilter(self, filename):
    return filename.endswith('.py')

class CppLintChecker(ExternalChecker):
  """Invoke Google's cpplint tool on c++ files."""
  def __init__(self):
    ExternalChecker.__init__(self, CPP_CHECKER, use_stderr=True)
    return

  def FileFilter(self, filename):
    return filename.endswith('.cc') or filename.endswith('.h')

# ======================================================================
class GenericRegexChecker(object):
  """Base Class"""
  def __init__(self, content_regex, line_regex=None, analyze_match=False):
    self._content_regex = re.compile(content_regex, re.MULTILINE)
    if line_regex:
      self._line_regex = re.compile(line_regex)
    else:
      self._line_regex = re.compile(content_regex)
    self._analyze_match = analyze_match
    return

  def FindProblems(self, unused_filename, data):
    # speedy early out
    if not self._content_regex.search(data):
      return []

    lines = data.split('\n')
    problem = []
    ignore_line_count = 0
    for no, line in enumerate(lines):
      if ignore_line_count > 0:
        ignore_line_count -= 1
        continue
      match = RE_IGNORE.search(line)
      if match:
        ignore_line_count = int(match.group(1))
        continue
      match = self._line_regex.search(line)
      if not match:
        continue
      if self._analyze_match:
        if not self.IsProblemMatch(match):
          continue
      problem.append(self._RenderItem(no, line))
    return problem

  def _RenderItem(self, no, line):
      return 'line %d: [%s]' % (no, repr(line))

  def FileFilter(self, unused_filename):
    raise NotImplementedError()

class OpenCurlyChecker(GenericRegexChecker):
  """Checks for orphan opening curly braces"""

  def __init__(self):
    GenericRegexChecker.__init__(self, r'^ *{ *$')
    return

  def FileFilter(self, filename):
    return (filename.endswith('.h') or
            filename.endswith('.cc') or
            filename.endswith('.c'))

class TrailingWhiteSpaceChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r' $')
    return

  def FileFilter(self, unused_filename):
    # check all files
    return 1

class CarriageReturnChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'\r')
    return

  def FileFilter(self, filename):
    return (filename.endswith('.py') or
            filename.endswith('.h') or
            filename.endswith('.cc') or
            filename.endswith('.c'))

class CppCommentChecker(GenericRegexChecker):

  def __init__(self):
    # we tolerate http:// etc in comments
    GenericRegexChecker.__init__(self, r'(^|[^:])//')
    return

  def FileFilter(self, filename):
    return filename.endswith('.c')

class TabsChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'\t')
    return

  def FileFilter(self, filename):
    return ("akefile" not in filename
        and not filename.endswith('.patch')
        and not filename.endswith('.valerr')
        and not filename.endswith('.dis'))

class FixmeChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'FIXME[:]')
    return

  def FileFilter(self, filename):
    return 1

class ExternChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'^ *extern\s+(.*)$', analyze_match=True)
    return

  def IsProblemMatch(self, match):
    extra = match.group(1).strip()
    # allow 'extern "C" {'
    # but do not allow plain 'extern "C"'
    if extra == '"C" {':
      return False
    return True

  def FileFilter(self, filename):
    return filename.endswith('.c') or filename.endswith('.cc')

class RewriteChecker(GenericRegexChecker):

  def __init__(self):
    GenericRegexChecker.__init__(self, r'[@][@]REWRITE')
    return

  def FileFilter(self, unused_filename):
    return 1

VALID_INCLUDE_PREFIX = [
    'native_client/',
    'third_party/',
    'gtest/',
    'gen/native_client/',
    ]

class IncludeChecker(object):
  """Enforce some base 'include' rules. Cpplint does more of these."""
  def __init__(self):
    pass

  def FindProblems(self, unused_filename, data):
    lines = data.split('\n')
    problem = []
    seen_code = False
    for no, line in enumerate(lines):
      if line.startswith('}'):
        # this test is our hacked indicator signaling that no includes
        # directives should follow after we have seen actual code
        seen_code = True
        continue

      if not line.startswith('#include'):
        continue
      if seen_code:
        problem.append('line %d: [%s]' % (no, repr(line)))
      token = line.split()
      path = token[1][1:]
      if '..' in path:
        problem.append('line %d: [%s]' % (no, repr(line)))
      if not token[1].startswith('<'):
        for prefix in VALID_INCLUDE_PREFIX:
          if path.startswith(prefix):
            break
        else:
          # no prefix matched
          problem.append('line %d: [%s]' % (no, repr(line)))

    return problem

  def FileFilter(self, filename):
    return filename.endswith('.c') or filename.endswith('.cc')

class LineLengthChecker(object):
  def __init__(self):
    pass

  def FindProblems(self, filename, data):
    # Exempt golden files used in tests from line-length limits.
    if filename.endswith('.val') or filename.endswith('.valerr'):
      return

    lines = data.split('\n')
    problem = []
    for no, line in enumerate(lines):
      if len(line) <= 80:
        continue
      line = line.strip()
      # we allow comments and as it so happens pre processor directives
      # to be longer
      if line.startswith('//') or line.startswith('/*') or line.startswith('#'):
        continue

      problem.append('line %d: [%s]' % (no + 1, repr(line)))
    return problem

  def FileFilter(self, filename):
    return 1

# ======================================================================
def IsBinary(data):
  return '\x00' in data

def FindExemptions(data):
  match = re.search(r'@EXEMPTION\[([^\]]*)\]', data)
  if match:
    return match.group(1).split(',')
  return []

# ======================================================================
CHECKS = [# fatal checks
          (1, 'tabs', TabsChecker()),
          (1, 'trailing_space', TrailingWhiteSpaceChecker()),
          (1, 'cpp_comment', CppCommentChecker()),
          (1, 'fixme', FixmeChecker()),
          (1, 'include', IncludeChecker()),
          (1, 'extern', ExternChecker()),
          # Non fatal checks
          (0, 'open_curly', OpenCurlyChecker()),
          (0, 'line_length', LineLengthChecker()),
          (0, 'carriage_return', CarriageReturnChecker()),
          (0, 'rewrite', RewriteChecker()),
          (0, 'tidy', TidyChecker()),
          (0, 'pychecker', PyChecker()),
          (0, 'cpplint', CppLintChecker()),
          ]

def EmitStatus(filename, status, details=[]):
  print '%s: %s' % (filename, status)
  for no, line in details[:LIMIT]:
    print '  line %d: [%s]' % (no, repr(line))
  return

def CheckFile(filename, report):
    errors = {}
    warnings = {}

    data = open(filename).read()

    if IsBinary(data):
      EmitStatus(filename, 'binary')
      return errors, warnings

    exemptions = FindExemptions(data)

    for fatal, name, check in CHECKS:
      Debug('trying %s' % name)

      if not check.FileFilter(filename):
        Debug('skipping')
        continue

      if name in exemptions:
        Debug('exempt')
        continue
      problems = check.FindProblems(filename, data)
      if problems:
        info =  '%s: %s' % (filename, name)
        items = problems[:LIMIT]
        if report:
          print info
          for i in items:
            print i
        if fatal:
          errors[info] = items
          break
        else:
          warnings[info] = items
    return errors, warnings

def HasGypFileCorrespondingToSconsFile(sconsfile, list):
  scons_root = os.path.dirname(sconsfile)
  for name in list:
    if name.startswith(scons_root) and name.endswith('.gyp'):
      return True
  return False

def main(argv):
  num_error = 0
  for filename in argv:
    if os.path.isdir(filename):
      EmitStatus(filename, 'directory')
      continue

    if '-' in os.path.basename(filename):
      print 'hyphen in:', filename
      num_error += 1
      continue

    # Verify that gyp files are updated together with scons files.
    # We probably won't have any gyp files for the untrusted code.
    if filename.endswith('.scons') and -1 == filename.find('untrusted'):
      if not HasGypFileCorrespondingToSconsFile(filename, argv):
        print 'please update the gyp file relevant for ', filename
        num_error += 1

    errors, warnings = CheckFile(filename, True)
    if errors:
      num_error += 1
  return num_error

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>"""Module for managing the versions of applications' source
repositories managed by Dusty itself."""

import os
import logging
from contextlib import contextmanager
import urlparse

import git

from .config import get_config_value
from . import constants
from .log import log_to_client
from .path import parent_dir

@contextmanager
def git_error_handling():
    try:
        yield
    except git.exc.GitCommandError:
        log_to_client('ERROR: Git command failed. If you are trying to access a remote repository '
                      'over SSH (e.g. GitHub), please make sure you have added your SSH key to the '
                      'SSH agent using: ssh-add <SSH key filepath>.  If you specified to clone the repo '
                      'using HTTP, the repo must be public; private repo access is only supported with '
                      'ssh')
        raise

class Repo(object):
    def __init__(self, remote_path):
        # remote path is either of actual remote (github.com/gamechanger/dusty) or
        # other local repo (/repos/dusty) format
        self.remote_path = remote_path

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.remote_path == other.remote_path
        return False

    def __hash__(self):
        return hash(self.remote_path)

    @classmethod
    def resolve(cls, all_known_repos, name):
        """We require the list of all remote repo paths to be passed in
        to this because otherwise we would need to import the spec assembler
        in this module, which would give us circular imports."""
        match = None
        for repo in all_known_repos:
            if repo.remote_path == name: # user passed in a full name
                return repo
            if name == repo.short_name:
                if match is None:
                    match = repo
                else:
                    raise RuntimeError('Short repo name {} is ambiguous. It matches both {} and {}'.format(name,
                                                                                match.remote_path,
                                                                                repo.remote_path))
        if match is None:
            raise RuntimeError('Short repo name {} does not match any known repos'.format(name))
        return match

    @property
    def is_local_repo(self):
        return self.remote_path.startswith('/') or self.remote_path.startswith('file:///')

    @property
    def is_http_repo(self):
        return self.remote_path.startswith('http')

    @property
    def short_name(self):
        short_name = self.remote_path.split('/')[-1]
        if short_name.endswith('.git'):
            short_name = short_name[:-4]
        return short_name

    @property
    def managed_path(self):
        return os.path.join(constants.REPOS_DIR, self.rel_path)

    @property
    def is_overridden(self):
        return self.remote_path in get_config_value(constants.CONFIG_REPO_OVERRIDES_KEY)

    @property
    def override_path(self):
        repo_overrides = get_config_value(constants.CONFIG_REPO_OVERRIDES_KEY)
        return repo_overrides.get(self.remote_path)

    @property
    def local_path(self):
        return self.override_path if self.is_overridden else self.managed_path

    @property
    def vm_path(self):
        return os.path.join(constants.VM_REPOS_DIR, self.rel_path)

    @property
    def rel_path(self):
        parsed = urlparse.urlparse(self.remote_path.lstrip('/'))
        if parsed.hostname:
            local_folder = parsed.hostname + parsed.path
        else:
            local_folder = parsed.path
        if local_folder.endswith('.git'):
            local_folder = local_folder[:-4]
        return local_folder.lstrip('/')

    def assemble_remote_path(self):
        if self.is_local_repo:
            if self.remote_path.startswith('file:///'):
                return self.remote_path
            else:
                return 'file:///{}'.format(self.remote_path)
        elif self.is_http_repo:
            if self.remote_path.endswith('.git'):
                return self.remote_path
            else:
                return self.remote_path + '.git'
        else:
            if self.remote_path.startswith('ssh://'):
                return self.remote_path
            else:
                return 'ssh://{}@{}'.format(constants.GIT_USER, self.remote_path)

    def ensure_local_repo(self):
        """Given a Dusty repo object, clone the remote into Dusty's local repos
        directory if it does not already exist."""
        if os.path.exists(self.managed_path):
            logging.debug('Repo {} already exists'.format(self.remote_path))
            return

        logging.info('Initiating clone of local repo {}'.format(self.remote_path))

        repo_path_parent = parent_dir(self.managed_path)
        if not os.path.exists(repo_path_parent):
            os.makedirs(repo_path_parent)
        with git_error_handling():
            git.Repo.clone_from(self.assemble_remote_path(), self.managed_path)

    def get_latest_commit(self):
        repo = git.Repo(self.managed_path)
        for ref in repo.refs:
            if ref.path == 'refs/remotes/origin/master':
                return ref.commit
        return repo.remote().fetch()[0].commit

    def local_is_up_to_date(self):
        self.ensure_local_repo()
        repo = git.Repo(self.managed_path)
        local_commit = repo.commit()
        latest_commit = self.get_latest_commit()
        return local_commit.hexsha == latest_commit.hexsha

    def update_local_repo(self, force=False):
        """Given a remote path (e.g. github.com/gamechanger/gclib), pull the latest
        commits from master to bring the local copy up to date."""
        self.ensure_local_repo()

        logging.info('Updating local repo {}'.format(self.remote_path))

        managed_repo = git.Repo(self.managed_path)
        with git_error_handling():
            managed_repo.remote().pull('master')
            log_to_client('Updated managed copy of {}'.format(self.remote_path))
        if not self.local_is_up_to_date():
            if force:
                with git_error_handling():
                    managed_repo.git.reset('--hard', 'origin/master')
            else:
                log_to_client('WARNING: couldn\'t update {} because of local conflicts. '
                              'A container may have modified files in the repos\'s directory. '
                              'Your code generally shouldn\'t be manipulating the contents of your repo folder - '
                              'please fix this and run `dusty up`'.format(self.managed_path))

    def update_local_repo_async(self, task_queue, force=False):
        self.ensure_local_repo()
        task_queue.enqueue_task(self.update_local_repo, force=force)
<commit_msg>Explain why we need a function for async repo update<commit_after>"""Module for managing the versions of applications' source
repositories managed by Dusty itself."""

import os
import logging
from contextlib import contextmanager
import urlparse

import git

from .config import get_config_value
from . import constants
from .log import log_to_client
from .path import parent_dir

@contextmanager
def git_error_handling():
    try:
        yield
    except git.exc.GitCommandError:
        log_to_client('ERROR: Git command failed. If you are trying to access a remote repository '
                      'over SSH (e.g. GitHub), please make sure you have added your SSH key to the '
                      'SSH agent using: ssh-add <SSH key filepath>.  If you specified to clone the repo '
                      'using HTTP, the repo must be public; private repo access is only supported with '
                      'ssh')
        raise

class Repo(object):
    def __init__(self, remote_path):
        # remote path is either of actual remote (github.com/gamechanger/dusty) or
        # other local repo (/repos/dusty) format
        self.remote_path = remote_path

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.remote_path == other.remote_path
        return False

    def __hash__(self):
        return hash(self.remote_path)

    @classmethod
    def resolve(cls, all_known_repos, name):
        """We require the list of all remote repo paths to be passed in
        to this because otherwise we would need to import the spec assembler
        in this module, which would give us circular imports."""
        match = None
        for repo in all_known_repos:
            if repo.remote_path == name: # user passed in a full name
                return repo
            if name == repo.short_name:
                if match is None:
                    match = repo
                else:
                    raise RuntimeError('Short repo name {} is ambiguous. It matches both {} and {}'.format(name,
                                                                                match.remote_path,
                                                                                repo.remote_path))
        if match is None:
            raise RuntimeError('Short repo name {} does not match any known repos'.format(name))
        return match

    @property
    def is_local_repo(self):
        return self.remote_path.startswith('/') or self.remote_path.startswith('file:///')

    @property
    def is_http_repo(self):
        return self.remote_path.startswith('http')

    @property
    def short_name(self):
        short_name = self.remote_path.split('/')[-1]
        if short_name.endswith('.git'):
            short_name = short_name[:-4]
        return short_name

    @property
    def managed_path(self):
        return os.path.join(constants.REPOS_DIR, self.rel_path)

    @property
    def is_overridden(self):
        return self.remote_path in get_config_value(constants.CONFIG_REPO_OVERRIDES_KEY)

    @property
    def override_path(self):
        repo_overrides = get_config_value(constants.CONFIG_REPO_OVERRIDES_KEY)
        return repo_overrides.get(self.remote_path)

    @property
    def local_path(self):
        return self.override_path if self.is_overridden else self.managed_path

    @property
    def vm_path(self):
        return os.path.join(constants.VM_REPOS_DIR, self.rel_path)

    @property
    def rel_path(self):
        parsed = urlparse.urlparse(self.remote_path.lstrip('/'))
        if parsed.hostname:
            local_folder = parsed.hostname + parsed.path
        else:
            local_folder = parsed.path
        if local_folder.endswith('.git'):
            local_folder = local_folder[:-4]
        return local_folder.lstrip('/')

    def assemble_remote_path(self):
        if self.is_local_repo:
            if self.remote_path.startswith('file:///'):
                return self.remote_path
            else:
                return 'file:///{}'.format(self.remote_path)
        elif self.is_http_repo:
            if self.remote_path.endswith('.git'):
                return self.remote_path
            else:
                return self.remote_path + '.git'
        else:
            if self.remote_path.startswith('ssh://'):
                return self.remote_path
            else:
                return 'ssh://{}@{}'.format(constants.GIT_USER, self.remote_path)

    def ensure_local_repo(self):
        """Given a Dusty repo object, clone the remote into Dusty's local repos
        directory if it does not already exist."""
        if os.path.exists(self.managed_path):
            logging.debug('Repo {} already exists'.format(self.remote_path))
            return

        logging.info('Initiating clone of local repo {}'.format(self.remote_path))

        repo_path_parent = parent_dir(self.managed_path)
        if not os.path.exists(repo_path_parent):
            os.makedirs(repo_path_parent)
        with git_error_handling():
            git.Repo.clone_from(self.assemble_remote_path(), self.managed_path)

    def get_latest_commit(self):
        repo = git.Repo(self.managed_path)
        for ref in repo.refs:
            if ref.path == 'refs/remotes/origin/master':
                return ref.commit
        return repo.remote().fetch()[0].commit

    def local_is_up_to_date(self):
        self.ensure_local_repo()
        repo = git.Repo(self.managed_path)
        local_commit = repo.commit()
        latest_commit = self.get_latest_commit()
        return local_commit.hexsha == latest_commit.hexsha

    def update_local_repo(self, force=False):
        """Given a remote path (e.g. github.com/gamechanger/gclib), pull the latest
        commits from master to bring the local copy up to date."""
        self.ensure_local_repo()

        logging.info('Updating local repo {}'.format(self.remote_path))

        managed_repo = git.Repo(self.managed_path)
        with git_error_handling():
            managed_repo.remote().pull('master')
            log_to_client('Updated managed copy of {}'.format(self.remote_path))
        if not self.local_is_up_to_date():
            if force:
                with git_error_handling():
                    managed_repo.git.reset('--hard', 'origin/master')
            else:
                log_to_client('WARNING: couldn\'t update {} because of local conflicts. '
                              'A container may have modified files in the repos\'s directory. '
                              'Your code generally shouldn\'t be manipulating the contents of your repo folder - '
                              'please fix this and run `dusty up`'.format(self.managed_path))

    def update_local_repo_async(self, task_queue, force=False):
        """Local repo updating suitable for asynchronous, parallel execution.
        We still need to run `ensure_local_repo` synchronously because it
        does a bunch of non-threadsafe filesystem operations."""
        self.ensure_local_repo()
        task_queue.enqueue_task(self.update_local_repo, force=force)
<|endoftext|>


<commit_before># This file is part of Cockpit.
#
# Copyright (C) 2022 Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import asyncio

class Endpoint:
    router = None

    def do_channel_control(self, command, message):
        raise NotImplementedError

    def do_channel_data(self, channel, data):
        raise NotImplementedError

class ChannelError(Exception):
    def __init__(self, problem, **kwargs):
        super().__init__(f'ChannelError {problem}')
        self.kwargs = dict(kwargs, problem=problem)

class Channel(Endpoint):
    payload = None
    restrictions = ()

    @staticmethod
    def create_match_rule(channel):
        assert channel.payload is not None, f'{channel} declares no payload'
        return dict(channel.restrictions, payload=channel.payload)

    @staticmethod
    def create_match_rules(channels):
        rules = [(Channel.create_match_rule(cls), cls) for cls in channels]
        rules.sort(key=lambda rule: len(rule[0]), reverse=True)  # more restrictive rules match first
        return rules

    channel = None

    # input
    def do_control(self, command, message):
        # Break the various different kinds of control messages out into the
        # things that our subclass may be interested in handling.  We drop the
        # 'message' field for handlers that don't need it.
        if command == 'open':
            self.channel = message['channel']
            self.do_open(message)
        elif command == 'ready':
            self.do_ready()
        elif command == 'done':
            self.do_done()
        elif command == 'close':
            self.do_close()
        elif command == 'ping':
            self.do_ping(message)
        elif command == 'pong':
            self.do_pong(message)

    def do_channel_control(self, command, message):
        # Catch errors and turn them into close messages
        try:
            self.do_control(command, message)
        except ChannelError as exc:
            self.close(**exc.kwargs)

    # At least this one really ought to be implemented...
    def do_open(self, options):
        raise NotImplementedError

    # ... but many subclasses may reasonably want to ignore some of these.
    def do_ready(self):
        pass

    def do_done(self):
        pass

    def do_close(self):
        pass

    def do_pong(self, message):
        pass

    # 'reasonable' default, overridden in AsyncChannel for flow control
    def do_ping(self, message):
        self.send_pong(message)

    def do_channel_data(self, channel, data):
        # Catch errors and turn them into close messages
        try:
            self.do_data(data)
        except ChannelError as exc:
            self.close(**exc.kwargs)

    def do_data(self, _data):
        # By default, channels can't receive data.
        self.close()

    # output
    def ready(self):
        self.send_control(command='ready')

    def done(self):
        self.send_control(command='done')

    def close(self, **kwargs):
        self.send_control(command='close', **kwargs)

    def send_data(self, message):
        self.router.send_data(self.channel, message)

    def send_message(self, **kwargs):
        self.router.send_message(self.channel, **kwargs)

    def send_control(self, command, **kwargs):
        self.router.send_control(channel=self.channel, command=command, **kwargs)

    def send_pong(self, message):
        message['command'] = 'pong'
        self.router.send_message('', **message)

class AsyncChannel(Channel):
    '''A subclass for async/await-style implementation of channels, with flow control

    This subclass provides asynchronous `read()` and `write()` calls for
    subclasses, with familiar semantics.  `write()` doesn't buffer, so the
    `done()` method on the base channel class can be used in a way similar to
    `shutdown()`.  The subclass must provide a async `run()` function, which
    will be spawned as a task.

    On the receiving side, the channel will respond to flow control pings to
    indicate that it has received the data, but only after it has been consumed
    by `read()`.

    On the sending side, write() will block if the channel backs up.
    '''

    # Values borrowed from C implementation
    CHANNEL_FLOW_PING = 16 * 1024
    CHANNEL_FLOW_WINDOW = 2 * 1024 * 1024

    loop = None

    # Receive-side flow control: intermix pings and data in the queue and reply
    # to pings as we dequeue them.  This is a buffer: since we need to handle
    # do_data() without blocking, we have no choice.
    receive_queue = None

    # Send-side flow control: no buffers here, just bookkeeping.
    out_sequence = 0
    out_window = CHANNEL_FLOW_WINDOW
    write_waiter = None

    async def run(self, options):
        raise NotImplementedError

    async def read(self):
        while not isinstance(item := await self.receive_queue.get(), bytes):
            self.send_pong(item)
        return item

    async def write(self, data):
        if self.flow_control:
            assert len(data) <= AsyncChannel.CHANNEL_FLOW_WINDOW
            self.out_sequence += len(data)

            while self.out_window < self.out_sequence:
                self.write_waiter = asyncio.get_running_loop().create_future()
                await self.write_waiter

        self.send_data(data)

    def do_pong(self, message):
        self.out_window = message['sequence'] + AsyncChannel.CHANNEL_FLOW_WINDOW
        if self.out_sequence <= self.out_window and self.write_waiter is not None:
            self.write_waiter.set_result(None)
            self.write_waiter = None

    def do_open(self, options):
        self.receive_queue = asyncio.Queue()
        self.flow_control = options.get('flow-control') is True
        asyncio.create_task(self.run(options), name='f{self.__class__.__name__}.run({options})')

    def do_done(self):
        self.receive_queue.put_nowait(b'')

    def do_close(self):
        # we might have already sent EOF for done, but two EOFs won't hurt anyone
        self.receive_queue.put_nowait(b'')

    def do_ping(self, message):
        self.receive_queue.put_nowait(message)

    def do_data(self, data):
        if not isinstance(data, bytes):
            # this will persist past this callback, so make sure we take our
            # own copy, in case this was a memoryview into a bytearray.
            data = bytes(data)

        self.receive_queue.put_nowait(data)
<commit_msg>python: support ChannelError from AsyncChannel.run()<commit_after># This file is part of Cockpit.
#
# Copyright (C) 2022 Red Hat, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import asyncio

class Endpoint:
    router = None

    def do_channel_control(self, command, message):
        raise NotImplementedError

    def do_channel_data(self, channel, data):
        raise NotImplementedError

class ChannelError(Exception):
    def __init__(self, problem, **kwargs):
        super().__init__(f'ChannelError {problem}')
        self.kwargs = dict(kwargs, problem=problem)

class Channel(Endpoint):
    payload = None
    restrictions = ()

    @staticmethod
    def create_match_rule(channel):
        assert channel.payload is not None, f'{channel} declares no payload'
        return dict(channel.restrictions, payload=channel.payload)

    @staticmethod
    def create_match_rules(channels):
        rules = [(Channel.create_match_rule(cls), cls) for cls in channels]
        rules.sort(key=lambda rule: len(rule[0]), reverse=True)  # more restrictive rules match first
        return rules

    channel = None

    # input
    def do_control(self, command, message):
        # Break the various different kinds of control messages out into the
        # things that our subclass may be interested in handling.  We drop the
        # 'message' field for handlers that don't need it.
        if command == 'open':
            self.channel = message['channel']
            self.do_open(message)
        elif command == 'ready':
            self.do_ready()
        elif command == 'done':
            self.do_done()
        elif command == 'close':
            self.do_close()
        elif command == 'ping':
            self.do_ping(message)
        elif command == 'pong':
            self.do_pong(message)

    def do_channel_control(self, command, message):
        # Catch errors and turn them into close messages
        try:
            self.do_control(command, message)
        except ChannelError as exc:
            self.close(**exc.kwargs)

    # At least this one really ought to be implemented...
    def do_open(self, options):
        raise NotImplementedError

    # ... but many subclasses may reasonably want to ignore some of these.
    def do_ready(self):
        pass

    def do_done(self):
        pass

    def do_close(self):
        pass

    def do_pong(self, message):
        pass

    # 'reasonable' default, overridden in AsyncChannel for flow control
    def do_ping(self, message):
        self.send_pong(message)

    def do_channel_data(self, channel, data):
        # Catch errors and turn them into close messages
        try:
            self.do_data(data)
        except ChannelError as exc:
            self.close(**exc.kwargs)

    def do_data(self, _data):
        # By default, channels can't receive data.
        self.close()

    # output
    def ready(self):
        self.send_control(command='ready')

    def done(self):
        self.send_control(command='done')

    def close(self, **kwargs):
        self.send_control(command='close', **kwargs)

    def send_data(self, message):
        self.router.send_data(self.channel, message)

    def send_message(self, **kwargs):
        self.router.send_message(self.channel, **kwargs)

    def send_control(self, command, **kwargs):
        self.router.send_control(channel=self.channel, command=command, **kwargs)

    def send_pong(self, message):
        message['command'] = 'pong'
        self.router.send_message('', **message)

class AsyncChannel(Channel):
    '''A subclass for async/await-style implementation of channels, with flow control

    This subclass provides asynchronous `read()` and `write()` calls for
    subclasses, with familiar semantics.  `write()` doesn't buffer, so the
    `done()` method on the base channel class can be used in a way similar to
    `shutdown()`.  The subclass must provide a async `run()` function, which
    will be spawned as a task.

    On the receiving side, the channel will respond to flow control pings to
    indicate that it has received the data, but only after it has been consumed
    by `read()`.

    On the sending side, write() will block if the channel backs up.
    '''

    # Values borrowed from C implementation
    CHANNEL_FLOW_PING = 16 * 1024
    CHANNEL_FLOW_WINDOW = 2 * 1024 * 1024

    loop = None

    # Receive-side flow control: intermix pings and data in the queue and reply
    # to pings as we dequeue them.  This is a buffer: since we need to handle
    # do_data() without blocking, we have no choice.
    receive_queue = None

    # Send-side flow control: no buffers here, just bookkeeping.
    out_sequence = 0
    out_window = CHANNEL_FLOW_WINDOW
    write_waiter = None

    async def run(self, options):
        raise NotImplementedError

    async def run_wrapper(self, options):
        try:
            await self.run(options)
            self.close()
        except ChannelError as exc:
            self.close(**exc.kwargs)

    async def read(self):
        while not isinstance(item := await self.receive_queue.get(), bytes):
            self.send_pong(item)
        return item

    async def write(self, data):
        if self.flow_control:
            assert len(data) <= AsyncChannel.CHANNEL_FLOW_WINDOW
            self.out_sequence += len(data)

            while self.out_window < self.out_sequence:
                self.write_waiter = asyncio.get_running_loop().create_future()
                await self.write_waiter

        self.send_data(data)

    def do_pong(self, message):
        self.out_window = message['sequence'] + AsyncChannel.CHANNEL_FLOW_WINDOW
        if self.out_sequence <= self.out_window and self.write_waiter is not None:
            self.write_waiter.set_result(None)
            self.write_waiter = None

    def do_open(self, options):
        self.receive_queue = asyncio.Queue()
        self.flow_control = options.get('flow-control') is True
        asyncio.create_task(self.run_wrapper(options), name='f{self.__class__.__name__}.run_wrapper({options})')

    def do_done(self):
        self.receive_queue.put_nowait(b'')

    def do_close(self):
        # we might have already sent EOF for done, but two EOFs won't hurt anyone
        self.receive_queue.put_nowait(b'')

    def do_ping(self, message):
        self.receive_queue.put_nowait(message)

    def do_data(self, data):
        if not isinstance(data, bytes):
            # this will persist past this callback, so make sure we take our
            # own copy, in case this was a memoryview into a bytearray.
            data = bytes(data)

        self.receive_queue.put_nowait(data)
<|endoftext|>