<commit_before>'''Harvester of National Oceanic and Atmosphere Administration's National Oceanographic Data Center

Example API call: http://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.nodc:00001339;view=xml

NODC provides an OAI-PMH interface at
http://data.nodc.noaa.gov/cgi-bin/oai-pmh, but it does not contain the
contributors key mandated by the SHARE-schema spec.  Therefore, we
have to use the XML interface, which contains much more metadata.
However, the XML interface is not searchable, so we still use the
OAI-PMH interface for queries. Fun!

'''

from __future__ import unicode_literals

import time
import logging
from datetime import date, timedelta

from lxml import etree
from nameparser import HumanName
from requests import exceptions

from scrapi import requests
from scrapi import settings
from scrapi.base import XMLHarvester
from scrapi.base.helpers import compose, single_result, date_formatter, language_codes, coerce_to_list, xml_text_only_list, xml_text_only
from scrapi.linter.document import RawDocument
from scrapi.util import copy_to_unicode

logger = logging.getLogger(__name__)

def filter_to_publishers(parties):
    '''Reduce list of ResponsibleParty elements to just publishers'''
    return filter_responsible_parties(parties, ['publisher'])

def filter_to_contributors(parties):
    '''Reduce list of ResponsibleParty elements to just contributors'''
    return filter_responsible_parties(parties, ['resourceProvider'])

def filter_responsible_parties(parties, roles):
    '''Return list of ResponsibleParties whose role is in roles param'''
    return [party for party in parties if party.xpath('./gmd:role/gmd:CI_RoleCode/node()', namespaces=party.nsmap)[0] in roles]

def parse_contributors(parties):
    '''Turn list of gmd:CI_ResponsibleParty elements into
    SHARE-compliant contributors list
    '''

    # The NODC schema doesn't explicitly distinguish between
    # organizations and individuals.  Every party has an
    # organizationName tag, individuals have an individualName tag as
    # well.  The email address is tied to the party, so we can
    contributors = []
    for party in parties:
        contributor = {}
        individual = extract_individual(party)
        organization = extract_organization(party)
        if individual:
            contributor = individual
            if organization:
                contributor['affiliation'] = [organization]
        elif organization:
            contributor = organization
        else:
            raise NotImplementedError

        # the email address is tied to the party, so wait until we know
        # who the contributor is before extracting
        email = party.xpath('./gmd:contactInfo/gmd:CI_Contact/gmd:address/gmd:CI_Address/gmd:electronicMailAddress', namespaces=party.nsmap)
        if len(email) > 0:
            contributor['email'] = xml_text_only(email[0])
        contributors.append(contributor)

    return contributors

def extract_individual(party):
    '''Return SHARE-compliant person object from a CI_ResponsibleParty element'''
    individual = party.xpath('./gmd:individualName/gmx:Anchor', namespaces=party.nsmap)
    if len(individual) > 0:
        name = HumanName(individual[0].text)
        ns_prefix = party.nsmap['xlink']
        sameAs = individual[0].get('{' + ns_prefix + '}href')
        return {
            'name': individual[0].text,
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'sameAs': [sameAs],
        }

def extract_organization(party):
    '''Return SHARE-compliant organization object from a CI_ResponsibleParty element'''
    organization = party.xpath('./gmd:organisationName/gmx:Anchor', namespaces=party.nsmap)
    if len(organization) > 0:
        ns_prefix = party.nsmap['xlink']
        sameAs = organization[0].get('{' + ns_prefix + '}href')
        return {
            'name': organization[0].text,
            'sameAs': [sameAs],
        }

def filter_keywords(keyword_groups):
    '''Filter out the NODC org name from the keywords to reduce noise
    and flatten nested lists'''
    keywords = []
    for keyword_group in keyword_groups:
        keyword_type = keyword_group.xpath('./gmd:type/gmd:MD_KeywordTypeCode/node()', namespaces=keyword_group.nsmap)
        if len(keyword_type) > 0 and keyword_type[0] != 'datacenter':
            keywords.extend(keyword_group.xpath('./gmd:keyword/gmx:Anchor/node()', namespaces=keyword_group.nsmap))
    return keywords

class NODCHarvester(XMLHarvester):
    short_name = 'noaa_nodc'
    long_name = 'National Oceanographic Data Center'
    url = 'https://www.nodc.noaa.gov/'

    DEFAULT_ENCODING = 'UTF-8'
    record_encoding = None

    canonical_base_url = 'http://data.nodc.noaa.gov/cgi-bin/iso?id={}'

    namespaces = {
        'gco': 'http://www.isotc211.org/2005/gco',
        'gmd': 'http://www.isotc211.org/2005/gmd',
        'gmi': 'http://www.isotc211.org/2005/gmi',
        'gml': 'http://www.opengis.net/gml/3.2',
        'gmx': 'http://www.isotc211.org/2005/gmx',
        'gsr': 'http://www.isotc211.org/2005/gsr',
        'gss': 'http://www.isotc211.org/2005/gss',
        'gts': 'http://www.isotc211.org/2005/gts',
        'srv': 'http://www.isotc211.org/2005/srv',
        'xlink': 'http://www.w3.org/1999/xlink',
        'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
        # xsi:schemaLocation="http://www.ngdc.noaa.gov/metadata/published/xsd/schema.xsd",
    }

    id_stanza = './gmd:identificationInfo/gmd:MD_DataIdentification/'
    cite_stanza = id_stanza + 'gmd:citation/gmd:CI_Citation/'
    schema = {
        'title': (cite_stanza + 'gmd:title', compose(xml_text_only, single_result)),
        'description': (id_stanza + 'gmd:abstract', compose(xml_text_only, single_result)),
        'contributors': (cite_stanza + 'gmd:citedResponsibleParty/gmd:CI_ResponsibleParty', compose(parse_contributors, filter_to_contributors)),
        'uris': {
            'canonicalUri': (
                './gmd:fileIdentifier',
                compose(lambda x: 'http://data.nodc.noaa.gov/cgi-bin/iso?id={}'.format(x), xml_text_only, single_result)
            ),
        },
        'publisher': (
            cite_stanza + 'gmd:citedResponsibleParty/gmd:CI_ResponsibleParty',
            compose(extract_organization, single_result, filter_to_publishers),
        ),
        'providerUpdatedDateTime': ('./gmd:dateStamp/gco:DateTime/node()', compose(date_formatter, single_result)),
        'languages': ('./gmd:language/gmd:LanguageCode', compose(language_codes, xml_text_only_list, coerce_to_list)),
        'subjects': (id_stanza + 'gmd:descriptiveKeywords/gmd:MD_Keywords', lambda x: filter_keywords(x)),
    }

    def harvest(self, start_date=None, end_date=None):
        ''' First, get a list of all recently updated study urls,
        then get the xml one by one and save it into a list
        of docs including other information '''

        start_date = (start_date or date.today() - timedelta(settings.DAYS_BACK)).isoformat()
        end_date = (end_date or date.today()).isoformat()
        start_date += 'T00:00:00Z'
        end_date += 'T00:00:00Z'

        search_base_url = 'http://data.nodc.noaa.gov/cgi-bin/oai-pmh?verb=ListRecords'
        search_url_end = '&metadataPrefix=oai_dc&from=' + start_date + '&until=' + end_date
        search_url = search_base_url + search_url_end

        oai_ns = {
            'oai_dc': 'http://www.openarchives.org/OAI/2.0/',
            'dc': 'http://purl.org/dc/elements/1.1/',
        }

        dataset_ids = []
        while True:

            record_list = requests.get(search_url)
            record_list_xml = etree.XML(record_list.content)
            if record_list_xml.xpath('./oai_dc:error', namespaces=oai_ns):
                break

            for dataset in record_list_xml.xpath('./oai_dc:ListRecords/oai_dc:record', namespaces=oai_ns):
                dataset_ids.append(dataset.xpath('./oai_dc:header/oai_dc:identifier/node()', namespaces=oai_ns)[0])

            token = record_list_xml.xpath('./oai_dc:ListRecords/oai_dc:resumptionToken/node()', namespaces=oai_ns)
            if not token:
                break

            search_url = search_base_url + '&resumptionToken=' + token[0]

        # grab each of those urls for full content
        logger.info('There are {} urls to harvest - be patient...'.format(len(dataset_ids)))
        count = 0
        xml_list = []
        xml_base_url = self.canonical_base_url + '&view=xml'
        for dataset_id in dataset_ids:
            try:
                item_url = str(xml_base_url).format(dataset_id)
                content = requests.get(item_url)
            except exceptions.ConnectionError as e:
                logger.info('Connection error: {}, wait a bit...'.format(e))
                time.sleep(30)
                continue
            doc = etree.XML(content.content)

            record = etree.tostring(doc, encoding=self.DEFAULT_ENCODING)
            xml_list.append(RawDocument({
                'doc': record,
                'source': self.short_name,
                'docID': copy_to_unicode(dataset_id),
                'filetype': 'xml',
            }))
            count += 1
            if count % 100 == 0:
                logger.info('You\'ve requested {} studies, keep going!'.format(count))

        return xml_list
<commit_msg>wrap schema property to close over self<commit_after>'''Harvester of National Oceanic and Atmosphere Administration's National Oceanographic Data Center

Example API call: http://data.nodc.noaa.gov/cgi-bin/iso?id=gov.noaa.nodc:00001339;view=xml

NODC provides an OAI-PMH interface at
http://data.nodc.noaa.gov/cgi-bin/oai-pmh, but it does not contain the
contributors key mandated by the SHARE-schema spec.  Therefore, we
have to use the XML interface, which contains much more metadata.
However, the XML interface is not searchable, so we still use the
OAI-PMH interface for queries. Fun!

'''

from __future__ import unicode_literals

import time
import logging
from datetime import date, timedelta

from lxml import etree
from nameparser import HumanName
from requests import exceptions

from scrapi import requests
from scrapi import settings
from scrapi.base import XMLHarvester
from scrapi.base.helpers import compose, single_result, date_formatter, language_codes, coerce_to_list, xml_text_only_list, xml_text_only
from scrapi.linter.document import RawDocument
from scrapi.util import copy_to_unicode

logger = logging.getLogger(__name__)

def filter_to_publishers(parties):
    '''Reduce list of ResponsibleParty elements to just publishers'''
    return filter_responsible_parties(parties, ['publisher'])

def filter_to_contributors(parties):
    '''Reduce list of ResponsibleParty elements to just contributors'''
    return filter_responsible_parties(parties, ['resourceProvider'])

def filter_responsible_parties(parties, roles):
    '''Return list of ResponsibleParties whose role is in roles param'''
    return [party for party in parties if party.xpath('./gmd:role/gmd:CI_RoleCode/node()', namespaces=party.nsmap)[0] in roles]

def parse_contributors(parties):
    '''Turn list of gmd:CI_ResponsibleParty elements into
    SHARE-compliant contributors list
    '''

    # The NODC schema doesn't explicitly distinguish between
    # organizations and individuals.  Every party has an
    # organizationName tag, individuals have an individualName tag as
    # well.  The email address is tied to the party, so we can
    contributors = []
    for party in parties:
        contributor = {}
        individual = extract_individual(party)
        organization = extract_organization(party)
        if individual:
            contributor = individual
            if organization:
                contributor['affiliation'] = [organization]
        elif organization:
            contributor = organization
        else:
            raise NotImplementedError

        # the email address is tied to the party, so wait until we know
        # who the contributor is before extracting
        email = party.xpath('./gmd:contactInfo/gmd:CI_Contact/gmd:address/gmd:CI_Address/gmd:electronicMailAddress', namespaces=party.nsmap)
        if len(email) > 0:
            contributor['email'] = xml_text_only(email[0])
        contributors.append(contributor)

    return contributors

def extract_individual(party):
    '''Return SHARE-compliant person object from a CI_ResponsibleParty element'''
    individual = party.xpath('./gmd:individualName/gmx:Anchor', namespaces=party.nsmap)
    if len(individual) > 0:
        name = HumanName(individual[0].text)
        ns_prefix = party.nsmap['xlink']
        sameAs = individual[0].get('{' + ns_prefix + '}href')
        return {
            'name': individual[0].text,
            'givenName': name.first,
            'additionalName': name.middle,
            'familyName': name.last,
            'sameAs': [sameAs],
        }

def extract_organization(party):
    '''Return SHARE-compliant organization object from a CI_ResponsibleParty element'''
    organization = party.xpath('./gmd:organisationName/gmx:Anchor', namespaces=party.nsmap)
    if len(organization) > 0:
        ns_prefix = party.nsmap['xlink']
        sameAs = organization[0].get('{' + ns_prefix + '}href')
        return {
            'name': organization[0].text,
            'sameAs': [sameAs],
        }

def filter_keywords(keyword_groups):
    '''Filter out the NODC org name from the keywords to reduce noise
    and flatten nested lists'''
    keywords = []
    for keyword_group in keyword_groups:
        keyword_type = keyword_group.xpath('./gmd:type/gmd:MD_KeywordTypeCode/node()', namespaces=keyword_group.nsmap)
        if len(keyword_type) > 0 and keyword_type[0] != 'datacenter':
            keywords.extend(keyword_group.xpath('./gmd:keyword/gmx:Anchor/node()', namespaces=keyword_group.nsmap))
    return keywords

class NODCHarvester(XMLHarvester):
    short_name = 'noaa_nodc'
    long_name = 'National Oceanographic Data Center'
    url = 'https://www.nodc.noaa.gov/'

    DEFAULT_ENCODING = 'UTF-8'
    record_encoding = None

    canonical_base_url = 'http://data.nodc.noaa.gov/cgi-bin/iso?id={}'

    namespaces = {
        'gco': 'http://www.isotc211.org/2005/gco',
        'gmd': 'http://www.isotc211.org/2005/gmd',
        'gmi': 'http://www.isotc211.org/2005/gmi',
        'gml': 'http://www.opengis.net/gml/3.2',
        'gmx': 'http://www.isotc211.org/2005/gmx',
        'gsr': 'http://www.isotc211.org/2005/gsr',
        'gss': 'http://www.isotc211.org/2005/gss',
        'gts': 'http://www.isotc211.org/2005/gts',
        'srv': 'http://www.isotc211.org/2005/srv',
        'xlink': 'http://www.w3.org/1999/xlink',
        'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
        # xsi:schemaLocation="http://www.ngdc.noaa.gov/metadata/published/xsd/schema.xsd",
    }

    @property
    def schema(self):
        id_stanza = './gmd:identificationInfo/gmd:MD_DataIdentification/'
        cite_stanza = id_stanza + 'gmd:citation/gmd:CI_Citation/'
        return {
            'title': (cite_stanza + 'gmd:title', compose(xml_text_only, single_result)),
            'description': (id_stanza + 'gmd:abstract', compose(xml_text_only, single_result)),
            'contributors': (cite_stanza + 'gmd:citedResponsibleParty/gmd:CI_ResponsibleParty', compose(parse_contributors, filter_to_contributors)),
            'uris': {
                'canonicalUri': (
                    './gmd:fileIdentifier',
                    compose(lambda x: str(self.canonical_base_url).format(x), xml_text_only, single_result)
                ),
            },
            'publisher': (
                cite_stanza + 'gmd:citedResponsibleParty/gmd:CI_ResponsibleParty',
                compose(extract_organization, single_result, filter_to_publishers),
            ),
            'providerUpdatedDateTime': ('./gmd:dateStamp/gco:DateTime/node()', compose(date_formatter, single_result)),
            'languages': ('./gmd:language/gmd:LanguageCode', compose(language_codes, xml_text_only_list, coerce_to_list)),
            'subjects': (id_stanza + 'gmd:descriptiveKeywords/gmd:MD_Keywords', lambda x: filter_keywords(x)),
        }

    def harvest(self, start_date=None, end_date=None):
        ''' First, get a list of all recently updated study urls,
        then get the xml one by one and save it into a list
        of docs including other information '''

        start_date = (start_date or date.today() - timedelta(settings.DAYS_BACK)).isoformat()
        end_date = (end_date or date.today()).isoformat()
        start_date += 'T00:00:00Z'
        end_date += 'T00:00:00Z'

        search_base_url = 'http://data.nodc.noaa.gov/cgi-bin/oai-pmh?verb=ListRecords'
        search_url_end = '&metadataPrefix=oai_dc&from=' + start_date + '&until=' + end_date
        search_url = search_base_url + search_url_end

        oai_ns = {
            'oai_dc': 'http://www.openarchives.org/OAI/2.0/',
            'dc': 'http://purl.org/dc/elements/1.1/',
        }

        dataset_ids = []
        while True:

            record_list = requests.get(search_url)
            record_list_xml = etree.XML(record_list.content)
            if record_list_xml.xpath('./oai_dc:error', namespaces=oai_ns):
                break

            for dataset in record_list_xml.xpath('./oai_dc:ListRecords/oai_dc:record', namespaces=oai_ns):
                dataset_ids.append(dataset.xpath('./oai_dc:header/oai_dc:identifier/node()', namespaces=oai_ns)[0])

            token = record_list_xml.xpath('./oai_dc:ListRecords/oai_dc:resumptionToken/node()', namespaces=oai_ns)
            if not token:
                break

            search_url = search_base_url + '&resumptionToken=' + token[0]

        # grab each of those urls for full content
        logger.info('There are {} urls to harvest - be patient...'.format(len(dataset_ids)))
        count = 0
        xml_list = []
        xml_base_url = self.canonical_base_url + '&view=xml'
        for dataset_id in dataset_ids:
            try:
                item_url = str(xml_base_url).format(dataset_id)
                content = requests.get(item_url)
            except exceptions.ConnectionError as e:
                logger.info('Connection error: {}, wait a bit...'.format(e))
                time.sleep(30)
                continue
            doc = etree.XML(content.content)

            record = etree.tostring(doc, encoding=self.DEFAULT_ENCODING)
            xml_list.append(RawDocument({
                'doc': record,
                'source': self.short_name,
                'docID': copy_to_unicode(dataset_id),
                'filetype': 'xml',
            }))
            count += 1
            if count % 100 == 0:
                logger.info('You\'ve requested {} studies, keep going!'.format(count))

        return xml_list
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Ion utility for generating interfaces from object definitions (and vice versa).

__author__ = 'Adam R. Smith'
__license__ = 'Apache 2.0'

import datetime
import fnmatch
import inspect
import os
import re
import sys

import yaml
import hashlib
import argparse

from pyon.service.service import BaseService
from pyon.util.containers import named_any

# Do not remove any of the imports below this comment!!!!!!
from pyon.core.object import IonYamlLoader, service_name_from_file_name
from pyon.util import yaml_ordered_dict

class IonServiceDefinitionError(Exception):
    pass

templates = {
      'file':
'''#!/usr/bin/env python
#
# File generated on {when_generated}
#

from zope.interface import Interface, implements

from collections import OrderedDict, defaultdict

from pyon.service.service import BaseService

{classes}
'''
    , 'class':
'''class I{name}(Interface):
{classdocstring}
{methods}
'''
'''class Base{name}(BaseService):
    implements(I{name})
{classdocstring}
{servicename}
{dependencies}
{classmethods}
'''
    , 'clssdocstr':
'    """\n\
    {classdocstr}\n\
    """'
    , 'svcname':
'    name = \'{name}\''
    , 'depends':
'    dependencies = {namelist}'
    , 'method':
'''
    def {name}({args}):
        {methoddocstring}
        # Return Value
        # ------------
        # {outargs}
        pass
'''
    , 'arg': '{name}={val}'
    , 'methdocstr':
'"""\n\
        {methoddocstr}\n\
        """'
}

def build_args_str(_def, include_self=True):
    # Handle case where method has no parameters
    args = []
    if include_self: args.append('self')

    for key,val in (_def or {}).iteritems():
        if isinstance(val, basestring):
            val = "'%s'" % (val)
        elif isinstance(val, datetime.datetime):
            # TODO: generate the datetime code
            val = "'%s'" % (val)
        # For collections, default to an empty collection of the same base type
        elif isinstance(val, list):
            val = []
        elif isinstance(val, dict):
            val = {}
        args.append(templates['arg'].format(name=key, val=val))

    args_str = ', '.join(args)
    return args_str

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--force', action='store_true', help='Do not do MD5 comparisons, always generate new files')
    parser.add_argument('-d', '--dryrun', action='store_true', help='Do not generate new files, just print status and exit with 1 if changes need to be made')
    opts = parser.parse_args()

    if os.getcwd().endswith('scripts'):
        sys.exit('This script needs to be run from the pyon root.')

    service_dir, interface_dir = 'obj/services', 'interface'
    if not os.path.exists(interface_dir):
        os.makedirs(interface_dir)

    # Clear old generated files
    files = os.listdir(interface_dir)
    for file in fnmatch.filter(files, '*.pyc'):
    #for file in fnmatch.filter(files, '*.py') + fnmatch.filter(files, '*.pyc'):
        os.unlink(os.path.join(interface_dir, file))

    open(os.path.join(interface_dir, '__init__.py'), 'w').close()

    # Load data yaml files in case services define interfaces
    # in terms of common data objects
    yaml_file_re = re.compile('(obj)/(.*)[.](yml)')
    data_dir = 'obj/data'
    entag = u'!enum'
    yaml.add_constructor(entag, lambda loader, node: {})
    for root, dirs, files in os.walk(data_dir):
        for filename in fnmatch.filter(files, '*.yml'):
            yaml_file = os.path.join(root, filename)
            file_match = yaml_file_re.match(yaml_file)
            if file_match is None: continue

            yaml_text = open(yaml_file, 'r').read()
            defs = yaml.load_all(yaml_text, Loader=IonYamlLoader)
            for def_set in defs:
                for name,_def in def_set.iteritems():
                    tag = u'!%s' % (name)
                    yaml.add_constructor(tag, lambda loader, node: {})
                    xtag = u'!Extends_%s' % (name)
                    yaml.add_constructor(xtag, lambda loader, node: {})

    svc_signatures = {}
    sigfile = os.path.join('interface', '.svc_signatures.yml')
    if os.path.exists(sigfile):
        with open(sigfile, 'r') as f:
            cnts = f.read()
            svc_signatures = yaml.load(cnts)

    count = 0

    currtime = str(datetime.datetime.today())
    # Generate the new definitions, for now giving each
    # yaml file its own python service
    for root, dirs, files in os.walk(service_dir):
        for filename in fnmatch.filter(files, '*.yml'):
            yaml_file = os.path.join(root, filename)
            file_match = yaml_file_re.match(yaml_file)
            if '.svc_signatures' in filename: continue
            if file_match is None: continue

            file_path = file_match.group(2)
            interface_base, interface_name = os.path.dirname(file_path), os.path.basename(file_path)
            interface_file = os.path.join('interface', interface_base, 'i%s.py' % interface_name)

            parent_dir = os.path.dirname(interface_file)
            if not os.path.exists(parent_dir):
                os.makedirs(parent_dir)
                parent = parent_dir
                while True:
                    # Add __init__.py files to parent dirs as necessary
                    curdir = os.path.split(os.path.abspath(parent))[1]
                    if curdir == 'services':
                        break
                    else:
                        parent = os.path.split(os.path.abspath(parent))[0]

                        pkg_file = os.path.join(parent, '__init__.py')
                        if not os.path.exists(pkg_file):
                            open(pkg_file, 'w').close()

            pkg_file = os.path.join(parent_dir, '__init__.py')
            if not os.path.exists(pkg_file):
                open(pkg_file, 'w').close()

            with open(yaml_file, 'r') as f:
                yaml_text = f.read()
                m = hashlib.md5()
                m.update(yaml_text)
                cur_md5 = m.hexdigest()

                if yaml_file in svc_signatures and not opts.force:
                    if cur_md5 == svc_signatures[yaml_file]:
                        print "Skipping   %40s (md5 signature match)" % interface_name
                        continue

                if opts.dryrun:
                    count += 1
                    print "Changed    %40s (needs update)" % interface_name
                    continue

                # update signature set
                svc_signatures[yaml_file] = cur_md5
                print 'Generating %40s -> %s' % (interface_name, interface_file)

            defs = yaml.load_all(yaml_text)
            for def_set in defs:
                # Handle object definitions first; make dummy constructors so tags will parse
                if 'obj' in def_set:
                    for obj_name in def_set['obj']:
                        tag = u'!%s' % (obj_name)
                        yaml.add_constructor(tag, lambda loader, node: {})
                    continue

                service_name = def_set.get('name', None)
                class_docstring = def_set.get('docstring', "class docstring")
                dependencies = def_set.get('dependencies', None)
                methods, class_methods = [], []

                # It seems that despite the get with default arg, there still can be None resulting (YAML?)
                meth_list = def_set.get('methods', {}) or {}
                for op_name,op_def in meth_list.iteritems():
                    if not op_def: continue
                    def_docstring, def_in, def_out = op_def.get('docstring', "method docstring"), op_def.get('in', None), op_def.get('out', None)

                    args_str, class_args_str = build_args_str(def_in, False), build_args_str(def_in, True)
                    docstring_str = templates['methdocstr'].format(methoddocstr=def_docstring)
                    outargs_str = '\n        # '.join(yaml.dump(def_out).split('\n'))

                    methods.append(templates['method'].format(name=op_name, args=args_str, methoddocstring=docstring_str, outargs=outargs_str))
                    class_methods.append(templates['method'].format(name=op_name, args=class_args_str, methoddocstring=docstring_str, outargs=outargs_str))

                if service_name is None:
                    raise IonServiceDefinitionError("Service definition file %s does not define name attribute" % yaml_file)
                service_name_str = templates['svcname'].format(name=service_name)
                class_docstring_str = templates['clssdocstr'].format(classdocstr=class_docstring)
                dependencies_str = templates['depends'].format(namelist=dependencies)
                methods_str = ''.join(methods) or '    pass\n'
                classmethods_str = ''.join(class_methods)
                class_name = service_name_from_file_name(interface_name)
                _class = templates['class'].format(name=class_name, classdocstring=class_docstring_str, servicename=service_name_str, dependencies=dependencies_str,
                                                       methods=methods_str, classmethods=classmethods_str)

                interface_contents = templates['file'].format(classes=_class, when_generated=currtime)
                open(interface_file, 'w').write(interface_contents)

                count+=1

    # write current svc_signatures
    if count > 0 and not opts.dryrun:
        print "Writing signature file to ", sigfile
        with open(sigfile, 'w') as f:
            f.write(yaml.dump(svc_signatures))

        # Load interface base classes
        load_mods("interface/services", True)
        base_subtypes = find_subtypes(BaseService)
        # Load impl classes
        load_mods("ion", False)

    # Generate validation report
    validation_results = "Report generated on " + currtime + "\n"
    load_mods("interface/services", True)
    base_subtypes = find_subtypes(BaseService)
    load_mods("ion", False)
    for base_subtype in base_subtypes:
        base_subtype_name = base_subtype.__module__ + "." + base_subtype.__name__
        compare_methods = {}
        for method_tuple in inspect.getmembers(base_subtype, inspect.ismethod):
            method_name = method_tuple[0]
            method = method_tuple[1]
            # Ignore private methods
            if method_name.startswith("_"):
                continue
            # Ignore methods not implemented in the class
            if method_name not in base_subtype.__dict__:
                continue
            compare_methods[method_name] = method

        # Find implementing subtypes of each base interface
        impl_subtypes = find_subtypes(base_subtype)
        if len(impl_subtypes) == 0:
            validation_results += "\nBase service: %s \n" % base_subtype_name
            validation_results += "  No impl subtypes found\n"
        for impl_subtype in find_subtypes(base_subtype):
            impl_subtype_name = impl_subtype.__module__ + "." + impl_subtype.__name__

            # Compare parameters
            added_class_names = False
            found_error = False
            for key in compare_methods:
                if key not in impl_subtype.__dict__:
                    if not added_class_names:
                        added_class_names = True
                        validation_results += "\nBase service: %s\n" % base_subtype_name
                        validation_results += "Impl subtype: %s\n" % impl_subtype_name
                    validation_results += "  Method '%s' not implemented" % key
                else:
                    base_params = inspect.getargspec(compare_methods[key])
                    impl_params = inspect.getargspec(impl_subtype.__dict__[key])

                    if base_params != impl_params:
                        if not added_class_names:
                            added_class_names = True
                            validation_results += "\nBase service: %s\n" % base_subtype_name
                            validation_results += "Impl subtype: %s\n" % impl_subtype_name
                        validation_results +=  "  Method '%s' implementation is out of sync\n" % key
                        validation_results +=  "    Base: %s\n" % str(base_params)
                        validation_results +=  "    Impl: %s\n" % str(impl_params)

            if found_error == False:
                validation_results += "\nBase service: %s\n" % base_subtype_name
                validation_results += "Impl subtype: %s\n" % impl_subtype_name
                validation_results += "  OK\n"

    reportfile = os.path.join('interface', 'validation_report.txt')
    try:
        os.unlink(reportfile)
    except:
        pass
    print "Writing validation report to '" + reportfile + "'"
    with open(reportfile, 'w') as f:
        f.write(validation_results)

    exitcode = 0
    if count > 0:
        exitcode = 1

    sys.exit(exitcode)

def load_mods(path, interfaces):
    import pkgutil
    import string
    mod_prefix = string.replace(path, "/", ".")

    for mod_imp, mod_name, is_pkg in pkgutil.iter_modules([path]):
        if is_pkg:
            load_mods(path+"/"+mod_name, interfaces)
        else:
            mod_qual = "%s.%s" % (mod_prefix, mod_name)
            try:
                named_any(mod_qual)
            except Exception, ex:
                print "Import module '%s' failed: %s" % (mod_qual, ex)
                if not interfaces:
                    print "Make sure that you have defined an __init__.py in your directory and that you have imported the correct base type"

def find_subtypes(clz):
    res = []
    for cls in clz.__subclasses__():
        assert hasattr(cls,'name'), 'Service class must define name value. Service class in error: %s' % cls
        res.append(cls)
    return res

if __name__ == '__main__':
    main()
<commit_msg>add examples to inteface validation<commit_after>#!/usr/bin/env python

# Ion utility for generating interfaces from object definitions (and vice versa).

__author__ = 'Adam R. Smith'
__license__ = 'Apache 2.0'

import datetime
import fnmatch
import inspect
import os
import re
import sys

import yaml
import hashlib
import argparse

from pyon.service.service import BaseService
from pyon.util.containers import named_any

# Do not remove any of the imports below this comment!!!!!!
from pyon.core.object import IonYamlLoader, service_name_from_file_name
from pyon.util import yaml_ordered_dict

class IonServiceDefinitionError(Exception):
    pass

templates = {
      'file':
'''#!/usr/bin/env python
#
# File generated on {when_generated}
#

from zope.interface import Interface, implements

from collections import OrderedDict, defaultdict

from pyon.service.service import BaseService

{classes}
'''
    , 'class':
'''class I{name}(Interface):
{classdocstring}
{methods}
'''
'''class Base{name}(BaseService):
    implements(I{name})
{classdocstring}
{servicename}
{dependencies}
{classmethods}
'''
    , 'clssdocstr':
'    """\n\
    {classdocstr}\n\
    """'
    , 'svcname':
'    name = \'{name}\''
    , 'depends':
'    dependencies = {namelist}'
    , 'method':
'''
    def {name}({args}):
        {methoddocstring}
        # Return Value
        # ------------
        # {outargs}
        pass
'''
    , 'arg': '{name}={val}'
    , 'methdocstr':
'"""\n\
        {methoddocstr}\n\
        """'
}

def build_args_str(_def, include_self=True):
    # Handle case where method has no parameters
    args = []
    if include_self: args.append('self')

    for key,val in (_def or {}).iteritems():
        if isinstance(val, basestring):
            val = "'%s'" % (val)
        elif isinstance(val, datetime.datetime):
            # TODO: generate the datetime code
            val = "'%s'" % (val)
        # For collections, default to an empty collection of the same base type
        elif isinstance(val, list):
            val = []
        elif isinstance(val, dict):
            val = {}
        args.append(templates['arg'].format(name=key, val=val))

    args_str = ', '.join(args)
    return args_str

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--force', action='store_true', help='Do not do MD5 comparisons, always generate new files')
    parser.add_argument('-d', '--dryrun', action='store_true', help='Do not generate new files, just print status and exit with 1 if changes need to be made')
    opts = parser.parse_args()

    if os.getcwd().endswith('scripts'):
        sys.exit('This script needs to be run from the pyon root.')

    service_dir, interface_dir = 'obj/services', 'interface'
    if not os.path.exists(interface_dir):
        os.makedirs(interface_dir)

    # Clear old generated files
    files = os.listdir(interface_dir)
    for file in fnmatch.filter(files, '*.pyc'):
    #for file in fnmatch.filter(files, '*.py') + fnmatch.filter(files, '*.pyc'):
        os.unlink(os.path.join(interface_dir, file))

    open(os.path.join(interface_dir, '__init__.py'), 'w').close()

    # Load data yaml files in case services define interfaces
    # in terms of common data objects
    yaml_file_re = re.compile('(obj)/(.*)[.](yml)')
    data_dir = 'obj/data'
    entag = u'!enum'
    yaml.add_constructor(entag, lambda loader, node: {})
    for root, dirs, files in os.walk(data_dir):
        for filename in fnmatch.filter(files, '*.yml'):
            yaml_file = os.path.join(root, filename)
            file_match = yaml_file_re.match(yaml_file)
            if file_match is None: continue

            yaml_text = open(yaml_file, 'r').read()
            defs = yaml.load_all(yaml_text, Loader=IonYamlLoader)
            for def_set in defs:
                for name,_def in def_set.iteritems():
                    tag = u'!%s' % (name)
                    yaml.add_constructor(tag, lambda loader, node: {})
                    xtag = u'!Extends_%s' % (name)
                    yaml.add_constructor(xtag, lambda loader, node: {})

    svc_signatures = {}
    sigfile = os.path.join('interface', '.svc_signatures.yml')
    if os.path.exists(sigfile):
        with open(sigfile, 'r') as f:
            cnts = f.read()
            svc_signatures = yaml.load(cnts)

    count = 0

    currtime = str(datetime.datetime.today())
    # Generate the new definitions, for now giving each
    # yaml file its own python service
    for root, dirs, files in os.walk(service_dir):
        for filename in fnmatch.filter(files, '*.yml'):
            yaml_file = os.path.join(root, filename)
            file_match = yaml_file_re.match(yaml_file)
            if '.svc_signatures' in filename: continue
            if file_match is None: continue

            file_path = file_match.group(2)
            interface_base, interface_name = os.path.dirname(file_path), os.path.basename(file_path)
            interface_file = os.path.join('interface', interface_base, 'i%s.py' % interface_name)

            parent_dir = os.path.dirname(interface_file)
            if not os.path.exists(parent_dir):
                os.makedirs(parent_dir)
                parent = parent_dir
                while True:
                    # Add __init__.py files to parent dirs as necessary
                    curdir = os.path.split(os.path.abspath(parent))[1]
                    if curdir == 'services':
                        break
                    else:
                        parent = os.path.split(os.path.abspath(parent))[0]

                        pkg_file = os.path.join(parent, '__init__.py')
                        if not os.path.exists(pkg_file):
                            open(pkg_file, 'w').close()

            pkg_file = os.path.join(parent_dir, '__init__.py')
            if not os.path.exists(pkg_file):
                open(pkg_file, 'w').close()

            with open(yaml_file, 'r') as f:
                yaml_text = f.read()
                m = hashlib.md5()
                m.update(yaml_text)
                cur_md5 = m.hexdigest()

                if yaml_file in svc_signatures and not opts.force:
                    if cur_md5 == svc_signatures[yaml_file]:
                        print "Skipping   %40s (md5 signature match)" % interface_name
                        continue

                if opts.dryrun:
                    count += 1
                    print "Changed    %40s (needs update)" % interface_name
                    continue

                # update signature set
                svc_signatures[yaml_file] = cur_md5
                print 'Generating %40s -> %s' % (interface_name, interface_file)

            defs = yaml.load_all(yaml_text)
            for def_set in defs:
                # Handle object definitions first; make dummy constructors so tags will parse
                if 'obj' in def_set:
                    for obj_name in def_set['obj']:
                        tag = u'!%s' % (obj_name)
                        yaml.add_constructor(tag, lambda loader, node: {})
                    continue

                service_name = def_set.get('name', None)
                class_docstring = def_set.get('docstring', "class docstring")
                dependencies = def_set.get('dependencies', None)
                methods, class_methods = [], []

                # It seems that despite the get with default arg, there still can be None resulting (YAML?)
                meth_list = def_set.get('methods', {}) or {}
                for op_name,op_def in meth_list.iteritems():
                    if not op_def: continue
                    def_docstring, def_in, def_out = op_def.get('docstring', "method docstring"), op_def.get('in', None), op_def.get('out', None)

                    args_str, class_args_str = build_args_str(def_in, False), build_args_str(def_in, True)
                    docstring_str = templates['methdocstr'].format(methoddocstr=def_docstring)
                    outargs_str = '\n        # '.join(yaml.dump(def_out).split('\n'))

                    methods.append(templates['method'].format(name=op_name, args=args_str, methoddocstring=docstring_str, outargs=outargs_str))
                    class_methods.append(templates['method'].format(name=op_name, args=class_args_str, methoddocstring=docstring_str, outargs=outargs_str))

                if service_name is None:
                    raise IonServiceDefinitionError("Service definition file %s does not define name attribute" % yaml_file)
                service_name_str = templates['svcname'].format(name=service_name)
                class_docstring_str = templates['clssdocstr'].format(classdocstr=class_docstring)
                dependencies_str = templates['depends'].format(namelist=dependencies)
                methods_str = ''.join(methods) or '    pass\n'
                classmethods_str = ''.join(class_methods)
                class_name = service_name_from_file_name(interface_name)
                _class = templates['class'].format(name=class_name, classdocstring=class_docstring_str, servicename=service_name_str, dependencies=dependencies_str,
                                                       methods=methods_str, classmethods=classmethods_str)

                interface_contents = templates['file'].format(classes=_class, when_generated=currtime)
                open(interface_file, 'w').write(interface_contents)

                count+=1

    # write current svc_signatures
    if count > 0 and not opts.dryrun:
        print "Writing signature file to ", sigfile
        with open(sigfile, 'w') as f:
            f.write(yaml.dump(svc_signatures))

        # Load interface base classes
        load_mods("interface/services", True)
        base_subtypes = find_subtypes(BaseService)
        # Load impl classes
        load_mods("ion", False)

    # Generate validation report
    validation_results = "Report generated on " + currtime + "\n"
    load_mods("interface/services", True)
    base_subtypes = find_subtypes(BaseService)
    load_mods("ion", False)
    load_mods("examples", False)
    for base_subtype in base_subtypes:
        base_subtype_name = base_subtype.__module__ + "." + base_subtype.__name__
        compare_methods = {}
        for method_tuple in inspect.getmembers(base_subtype, inspect.ismethod):
            method_name = method_tuple[0]
            method = method_tuple[1]
            # Ignore private methods
            if method_name.startswith("_"):
                continue
            # Ignore methods not implemented in the class
            if method_name not in base_subtype.__dict__:
                continue
            compare_methods[method_name] = method

        # Find implementing subtypes of each base interface
        impl_subtypes = find_subtypes(base_subtype)
        if len(impl_subtypes) == 0:
            validation_results += "\nBase service: %s \n" % base_subtype_name
            validation_results += "  No impl subtypes found\n"
        for impl_subtype in find_subtypes(base_subtype):
            impl_subtype_name = impl_subtype.__module__ + "." + impl_subtype.__name__

            # Compare parameters
            added_class_names = False
            found_error = False
            for key in compare_methods:
                if key not in impl_subtype.__dict__:
                    if not added_class_names:
                        added_class_names = True
                        validation_results += "\nBase service: %s\n" % base_subtype_name
                        validation_results += "Impl subtype: %s\n" % impl_subtype_name
                    validation_results += "  Method '%s' not implemented" % key
                else:
                    base_params = inspect.getargspec(compare_methods[key])
                    impl_params = inspect.getargspec(impl_subtype.__dict__[key])

                    if base_params != impl_params:
                        if not added_class_names:
                            added_class_names = True
                            validation_results += "\nBase service: %s\n" % base_subtype_name
                            validation_results += "Impl subtype: %s\n" % impl_subtype_name
                        validation_results +=  "  Method '%s' implementation is out of sync\n" % key
                        validation_results +=  "    Base: %s\n" % str(base_params)
                        validation_results +=  "    Impl: %s\n" % str(impl_params)

            if found_error == False:
                validation_results += "\nBase service: %s\n" % base_subtype_name
                validation_results += "Impl subtype: %s\n" % impl_subtype_name
                validation_results += "  OK\n"

    reportfile = os.path.join('interface', 'validation_report.txt')
    try:
        os.unlink(reportfile)
    except:
        pass
    print "Writing validation report to '" + reportfile + "'"
    with open(reportfile, 'w') as f:
        f.write(validation_results)

    exitcode = 0
    if count > 0:
        exitcode = 1

    sys.exit(exitcode)

def load_mods(path, interfaces):
    import pkgutil
    import string
    mod_prefix = string.replace(path, "/", ".")

    for mod_imp, mod_name, is_pkg in pkgutil.iter_modules([path]):
        if is_pkg:
            load_mods(path+"/"+mod_name, interfaces)
        else:
            mod_qual = "%s.%s" % (mod_prefix, mod_name)
            try:
                named_any(mod_qual)
            except Exception, ex:
                print "Import module '%s' failed: %s" % (mod_qual, ex)
                if not interfaces:
                    print "Make sure that you have defined an __init__.py in your directory and that you have imported the correct base type"

def find_subtypes(clz):
    res = []
    for cls in clz.__subclasses__():
        assert hasattr(cls,'name'), 'Service class must define name value. Service class in error: %s' % cls
        res.append(cls)
    return res

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from booleanOperations import BooleanOperationManager
from robofab.world import OpenFont
from fontbuild.mix import Mix,Master,narrowFLGlyph
from fontbuild.instanceNames import setNamesRF
from fontbuild.italics import italicizeGlyph
from fontbuild.convertCurves import glyphCurvesToQuadratic
from fontbuild.mitreGlyph import mitreGlyph
from fontbuild.generateGlyph import generateGlyph
from fontTools.misc.transform import Transform
from fontbuild.kerning import makeKernFeature
from fontbuild.features import readFeatureFile, writeFeatureFile
from fontbuild.markFeature import GenerateFeature_mark
from fontbuild.mkmkFeature import GenerateFeature_mkmk
from fontbuild.decomposeGlyph import decomposeGlyph
import ConfigParser
import os

class FontProject:

    def __init__(self, basefont, basedir, configfile, thinfont = None):
        self.basefont = basefont
        self.thinfont = thinfont
        self.basedir = basedir
        self.config = ConfigParser.RawConfigParser()
        self.configfile = self.basedir+"/"+configfile
        self.config.read(self.configfile)

        diacriticList = open(self.basedir + "/" + self.config.get("res","diacriticfile")).readlines()
        self.diacriticList = [line.strip() for line in diacriticList if not line.startswith("#")]
        self.ot_classes = open(self.basedir + "/" + self.config.get("res","ot_classesfile")).read()
        self.ot_kerningclasses = open(self.basedir + "/" + self.config.get("res","ot_kerningclassesfile")).read()
        #self.ot_features = open(self.basedir + "/" + self.config.get("res","ot_featuresfile")).read()
        adobeGlyphList = open(self.basedir + "/" + self.config.get("res", "agl_glyphlistfile")).readlines()
        self.adobeGlyphList = dict([line.split(";") for line in adobeGlyphList if not line.startswith("#")])

        # map exceptional glyph names in Roboto to names in the AGL
        roboNames = (
            ('Obar', 'Ocenteredtilde'), ('obar', 'obarred'),
            ('eturn', 'eturned'), ('Iota1', 'Iotaafrican'))
        for roboName, aglName in roboNames:
            self.adobeGlyphList[roboName] = self.adobeGlyphList[aglName]

        self.builddir = "out"
        self.decompose = self.config.get("glyphs","decompose").split()
        self.predecompose = self.config.get("glyphs","predecompose").split()
        self.lessItalic = self.config.get("glyphs","lessitalic").split()
        self.deleteList = self.config.get("glyphs","delete").split()
        self.buildnumber = self.loadBuildNumber()

        self.buildOTF = False
        self.autohintOTF = False
        self.buildTTF = False

    def loadBuildNumber(self):
        versionFile = open(self.basedir + "/" + self.config.get("main","buildnumberfile"), "r+")
        buildnumber = int(versionFile.read().strip())
        buildnumber = "%05d" %(int(buildnumber) + 1)
        print "BuildNumber: %s" %(buildnumber)
        versionFile.close()
        return buildnumber

    def incrementBuildNumber(self):
        if len(self.buildnumber) > 0:
            versionFile = open(self.basedir + "/" + self.config.get("main","buildnumberfile"), "r+")
            versionFile.seek(0)
            versionFile.write(self.buildnumber)
            versionFile.truncate()
            versionFile.close()
        else:
            raise Exception("Empty build number")

    def generateOutputPath(self, font, ext):
        family = font.info.familyName.replace(" ", "")
        style = font.info.styleName.replace(" ", "")
        path = os.path.join(self.basedir, self.builddir, family + ext.upper())
        if not os.path.exists(path):
            os.makedirs(path)
        return os.path.join(path, "%s-%s.%s" % (family, style, ext))

    def generateFont(self, mix, names, italic=False, swapSuffixes=None, stemWidth=185, kern=True):

        n = names.split("/")
        log("---------------------\n%s %s\n----------------------" %(n[0],n[1]))
        log(">> Mixing masters")
        if isinstance( mix, Mix):
            f = mix.generateFont(self.basefont)
        else:
            f = mix.copy()
        if italic == True:
            log(">> Italicizing")
            tweakAmmount = .085
            narrowAmmount = .93
            if names.find("Thin") != -1:
                tweakAmmount = .05
            if names.find("Condensed") != -1:
                narrowAmmount = .96
            i = 0
            for g in f:
                i += 1
                if i % 10 == 0: print g.name

                if g.name == "uniFFFD":
                    continue

                # if i < 24:
                #     continue
                # if i > 86:
                #     for i,g in enumerate(fl.font.glyphs):
                #       fl.UpdateGlyph(i)
                #     # break
                #     assert False

                # print g.name
                # if self.thinfont != None:
                #                     narrowFLGlyph(g,self.thinfont.getGlyph(g.name),factor=narrowAmmount)

                if g.name != "eight" or g.name != "Q":
                    removeGlyphOverlap(g)

                    # not sure why FontLab sometimes refuses, seems to work if called twice

                if g.name in self.lessItalic:
                    italicizeGlyph(f, g, 9, stemWidth=stemWidth)
                elif g.name != ".notdef":
                    italicizeGlyph(f, g, 10, stemWidth=stemWidth)
                removeGlyphOverlap(g)
                if g.width != 0:
                    g.width += 10

        if swapSuffixes != None:
            for swap in swapSuffixes:
                swapList = [g.name for g in f if g.name.endswith(swap)]
                for gname in swapList:
                    print gname
                    swapContours(f, gname.replace(swap,""), gname)
        for gname in self.predecompose:
            if f.has_key(gname):
                decomposeGlyph(f[gname])

        log(">> Generating glyphs")
        generateGlyphs(f, self.diacriticList, self.adobeGlyphList)
        log(">> Copying features")
        readFeatureFile(f, self.ot_classes + self.basefont.features.text)
        log(">> Decomposing")
        for gname in self.decompose:
            if f.has_key(gname):
                decomposeGlyph(f[gname])

        setNamesRF(f, n, foundry=self.config.get('main', 'foundry'),
                         version=self.config.get('main', 'version'))
        cleanCurves(f)
        deleteGlyphs(f, self.deleteList)

        if kern:
            log(">> Generating kern classes")
            readFeatureFile(f, self.ot_kerningclasses)
            makeKernFeature(f, self.ot_kerningclasses)

        log(">> Generating font files")
        GenerateFeature_mark(f)
        GenerateFeature_mkmk(f)
        ufoName = self.generateOutputPath(f, "ufo")
        f.save(ufoName)

        if self.buildOTF:
            log(">> Generating OTF file")
            newFont = OpenFont(ufoName)
            otfName = self.generateOutputPath(f, "otf")
            builtSuccessfully = saveOTF(newFont, otfName, autohint=self.autohintOTF)
            if builtSuccessfully and self.buildTTF:
                log(">> Generating TTF file")
                import fontforge
                otFont = fontforge.open(otfName)
                otFont.generate(self.generateOutputPath(f, "ttf"))

def transformGlyphMembers(g, m):
    g.width = int(g.width * m.a)
    g.Transform(m)
    for a in g.anchors:
        p = Point(a.p)
        p.Transform(m)
        a.p = p
    for c in g.components:
        # Assumes that components have also been individually transformed
        p = Point(0,0)
        d = Point(c.deltas[0])
        d.Transform(m)
        p.Transform(m)
        d1 = d - p
        c.deltas[0].x = d1.x
        c.deltas[0].y = d1.y
        s = Point(c.scale)
        s.Transform(m)
        #c.scale = s

def swapContours(f,gName1,gName2):
    try:
        g1 = f[gName1]
        g2 = f[gName2]
    except KeyError:
        log("swapGlyphs failed for %s %s" % (gName1, gName2))
        return
    g3 = g1.copy()

    while g1.contours:
        g1.removeContour(0)
    for contour in g2.contours:
        g1.appendContour(contour)
    g1.width = g2.width

    while g2.contours:
        g2.removeContour(0)
    for contour in g3.contours:
        g2.appendContour(contour)
    g2.width = g3.width

def log(msg):
    print msg

def generateGlyphs(f, glyphNames, glyphList={}):
    log(">> Generating diacritics")
    glyphnames = [gname for gname in glyphNames if not gname.startswith("#") and gname != ""]

    for glyphName in glyphNames:
        generateGlyph(f, glyphName, glyphList)

def cleanCurves(f):
    log(">> Removing overlaps")
    for g in f:
        removeGlyphOverlap(g)

    # log(">> Mitring sharp corners")
    # for g in f:
    #     mitreGlyph(g, 3., .7)

    # log(">> Converting curves to quadratic")
    # for g in f:
    #     glyphCurvesToQuadratic(g)

def deleteGlyphs(f, deleteList):
    for name in deleteList:
        if f.has_key(name):
            f.removeGlyph(name)

def removeGlyphOverlap(glyph):
    """Remove overlaps in contours from a glyph."""
    #TODO(jamesgk) verify overlaps exist first, as per library's recommendation
    manager = BooleanOperationManager()
    contours = glyph.contours
    glyph.clearContours()
    manager.union(contours, glyph.getPointPen())

def saveOTF(font, destFile, autohint=False):
    """Save a RoboFab font as an OTF binary using ufo2fdk.

    Returns True on success, False otherwise.
    """

    from ufo2fdk import OTFCompiler

    # glyphs with multiple unicode values must be split up, due to FontTool's
    # use of a name -> UV dictionary during cmap compilation
    for glyph in font:
        if len(glyph.unicodes) > 1:
            newUV = glyph.unicodes.pop()
            newGlyph = font.newGlyph("uni%04X" % newUV)
            newGlyph.appendComponent(glyph.name)
            newGlyph.unicode = newUV
            newGlyph.width = glyph.width

    compiler = OTFCompiler()
    reports = compiler.compile(font, destFile, autohint=autohint)
    if autohint:
        print reports["autohint"]
    print reports["makeotf"]

    successMsg = ("makeotfexe [NOTE] Wrote new font file '%s'." %
                  os.path.basename(destFile))
    return successMsg in reports["makeotf"]
<commit_msg>Exit after failing to generate OTF.<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from booleanOperations import BooleanOperationManager
from robofab.world import OpenFont
from fontbuild.mix import Mix,Master,narrowFLGlyph
from fontbuild.instanceNames import setNamesRF
from fontbuild.italics import italicizeGlyph
from fontbuild.convertCurves import glyphCurvesToQuadratic
from fontbuild.mitreGlyph import mitreGlyph
from fontbuild.generateGlyph import generateGlyph
from fontTools.misc.transform import Transform
from fontbuild.kerning import makeKernFeature
from fontbuild.features import readFeatureFile, writeFeatureFile
from fontbuild.markFeature import GenerateFeature_mark
from fontbuild.mkmkFeature import GenerateFeature_mkmk
from fontbuild.decomposeGlyph import decomposeGlyph
import ConfigParser
import os
import sys

class FontProject:

    def __init__(self, basefont, basedir, configfile, thinfont = None):
        self.basefont = basefont
        self.thinfont = thinfont
        self.basedir = basedir
        self.config = ConfigParser.RawConfigParser()
        self.configfile = self.basedir+"/"+configfile
        self.config.read(self.configfile)

        diacriticList = open(self.basedir + "/" + self.config.get("res","diacriticfile")).readlines()
        self.diacriticList = [line.strip() for line in diacriticList if not line.startswith("#")]
        self.ot_classes = open(self.basedir + "/" + self.config.get("res","ot_classesfile")).read()
        self.ot_kerningclasses = open(self.basedir + "/" + self.config.get("res","ot_kerningclassesfile")).read()
        #self.ot_features = open(self.basedir + "/" + self.config.get("res","ot_featuresfile")).read()
        adobeGlyphList = open(self.basedir + "/" + self.config.get("res", "agl_glyphlistfile")).readlines()
        self.adobeGlyphList = dict([line.split(";") for line in adobeGlyphList if not line.startswith("#")])

        # map exceptional glyph names in Roboto to names in the AGL
        roboNames = (
            ('Obar', 'Ocenteredtilde'), ('obar', 'obarred'),
            ('eturn', 'eturned'), ('Iota1', 'Iotaafrican'))
        for roboName, aglName in roboNames:
            self.adobeGlyphList[roboName] = self.adobeGlyphList[aglName]

        self.builddir = "out"
        self.decompose = self.config.get("glyphs","decompose").split()
        self.predecompose = self.config.get("glyphs","predecompose").split()
        self.lessItalic = self.config.get("glyphs","lessitalic").split()
        self.deleteList = self.config.get("glyphs","delete").split()
        self.buildnumber = self.loadBuildNumber()

        self.buildOTF = False
        self.autohintOTF = False
        self.buildTTF = False

    def loadBuildNumber(self):
        versionFile = open(self.basedir + "/" + self.config.get("main","buildnumberfile"), "r+")
        buildnumber = int(versionFile.read().strip())
        buildnumber = "%05d" %(int(buildnumber) + 1)
        print "BuildNumber: %s" %(buildnumber)
        versionFile.close()
        return buildnumber

    def incrementBuildNumber(self):
        if len(self.buildnumber) > 0:
            versionFile = open(self.basedir + "/" + self.config.get("main","buildnumberfile"), "r+")
            versionFile.seek(0)
            versionFile.write(self.buildnumber)
            versionFile.truncate()
            versionFile.close()
        else:
            raise Exception("Empty build number")

    def generateOutputPath(self, font, ext):
        family = font.info.familyName.replace(" ", "")
        style = font.info.styleName.replace(" ", "")
        path = os.path.join(self.basedir, self.builddir, family + ext.upper())
        if not os.path.exists(path):
            os.makedirs(path)
        return os.path.join(path, "%s-%s.%s" % (family, style, ext))

    def generateFont(self, mix, names, italic=False, swapSuffixes=None, stemWidth=185, kern=True):

        n = names.split("/")
        log("---------------------\n%s %s\n----------------------" %(n[0],n[1]))
        log(">> Mixing masters")
        if isinstance( mix, Mix):
            f = mix.generateFont(self.basefont)
        else:
            f = mix.copy()
        if italic == True:
            log(">> Italicizing")
            tweakAmmount = .085
            narrowAmmount = .93
            if names.find("Thin") != -1:
                tweakAmmount = .05
            if names.find("Condensed") != -1:
                narrowAmmount = .96
            i = 0
            for g in f:
                i += 1
                if i % 10 == 0: print g.name

                if g.name == "uniFFFD":
                    continue

                # if i < 24:
                #     continue
                # if i > 86:
                #     for i,g in enumerate(fl.font.glyphs):
                #       fl.UpdateGlyph(i)
                #     # break
                #     assert False

                # print g.name
                # if self.thinfont != None:
                #                     narrowFLGlyph(g,self.thinfont.getGlyph(g.name),factor=narrowAmmount)

                if g.name != "eight" or g.name != "Q":
                    removeGlyphOverlap(g)

                    # not sure why FontLab sometimes refuses, seems to work if called twice

                if g.name in self.lessItalic:
                    italicizeGlyph(f, g, 9, stemWidth=stemWidth)
                elif g.name != ".notdef":
                    italicizeGlyph(f, g, 10, stemWidth=stemWidth)
                removeGlyphOverlap(g)
                if g.width != 0:
                    g.width += 10

        if swapSuffixes != None:
            for swap in swapSuffixes:
                swapList = [g.name for g in f if g.name.endswith(swap)]
                for gname in swapList:
                    print gname
                    swapContours(f, gname.replace(swap,""), gname)
        for gname in self.predecompose:
            if f.has_key(gname):
                decomposeGlyph(f[gname])

        log(">> Generating glyphs")
        generateGlyphs(f, self.diacriticList, self.adobeGlyphList)
        log(">> Copying features")
        readFeatureFile(f, self.ot_classes + self.basefont.features.text)
        log(">> Decomposing")
        for gname in self.decompose:
            if f.has_key(gname):
                decomposeGlyph(f[gname])

        setNamesRF(f, n, foundry=self.config.get('main', 'foundry'),
                         version=self.config.get('main', 'version'))
        cleanCurves(f)
        deleteGlyphs(f, self.deleteList)

        if kern:
            log(">> Generating kern classes")
            readFeatureFile(f, self.ot_kerningclasses)
            makeKernFeature(f, self.ot_kerningclasses)

        log(">> Generating font files")
        GenerateFeature_mark(f)
        GenerateFeature_mkmk(f)
        ufoName = self.generateOutputPath(f, "ufo")
        f.save(ufoName)

        if self.buildOTF:
            log(">> Generating OTF file")
            newFont = OpenFont(ufoName)
            otfName = self.generateOutputPath(f, "otf")
            builtSuccessfully = saveOTF(newFont, otfName, autohint=self.autohintOTF)
            if not builtSuccessfully:
                sys.exit(1)

            if self.buildTTF:
                log(">> Generating TTF file")
                import fontforge
                otFont = fontforge.open(otfName)
                otFont.generate(self.generateOutputPath(f, "ttf"))

def transformGlyphMembers(g, m):
    g.width = int(g.width * m.a)
    g.Transform(m)
    for a in g.anchors:
        p = Point(a.p)
        p.Transform(m)
        a.p = p
    for c in g.components:
        # Assumes that components have also been individually transformed
        p = Point(0,0)
        d = Point(c.deltas[0])
        d.Transform(m)
        p.Transform(m)
        d1 = d - p
        c.deltas[0].x = d1.x
        c.deltas[0].y = d1.y
        s = Point(c.scale)
        s.Transform(m)
        #c.scale = s

def swapContours(f,gName1,gName2):
    try:
        g1 = f[gName1]
        g2 = f[gName2]
    except KeyError:
        log("swapGlyphs failed for %s %s" % (gName1, gName2))
        return
    g3 = g1.copy()

    while g1.contours:
        g1.removeContour(0)
    for contour in g2.contours:
        g1.appendContour(contour)
    g1.width = g2.width

    while g2.contours:
        g2.removeContour(0)
    for contour in g3.contours:
        g2.appendContour(contour)
    g2.width = g3.width

def log(msg):
    print msg

def generateGlyphs(f, glyphNames, glyphList={}):
    log(">> Generating diacritics")
    glyphnames = [gname for gname in glyphNames if not gname.startswith("#") and gname != ""]

    for glyphName in glyphNames:
        generateGlyph(f, glyphName, glyphList)

def cleanCurves(f):
    log(">> Removing overlaps")
    for g in f:
        removeGlyphOverlap(g)

    # log(">> Mitring sharp corners")
    # for g in f:
    #     mitreGlyph(g, 3., .7)

    # log(">> Converting curves to quadratic")
    # for g in f:
    #     glyphCurvesToQuadratic(g)

def deleteGlyphs(f, deleteList):
    for name in deleteList:
        if f.has_key(name):
            f.removeGlyph(name)

def removeGlyphOverlap(glyph):
    """Remove overlaps in contours from a glyph."""
    #TODO(jamesgk) verify overlaps exist first, as per library's recommendation
    manager = BooleanOperationManager()
    contours = glyph.contours
    glyph.clearContours()
    manager.union(contours, glyph.getPointPen())

def saveOTF(font, destFile, autohint=False):
    """Save a RoboFab font as an OTF binary using ufo2fdk.

    Returns True on success, False otherwise.
    """

    from ufo2fdk import OTFCompiler

    # glyphs with multiple unicode values must be split up, due to FontTool's
    # use of a name -> UV dictionary during cmap compilation
    for glyph in font:
        if len(glyph.unicodes) > 1:
            newUV = glyph.unicodes.pop()
            newGlyph = font.newGlyph("uni%04X" % newUV)
            newGlyph.appendComponent(glyph.name)
            newGlyph.unicode = newUV
            newGlyph.width = glyph.width

    compiler = OTFCompiler()
    reports = compiler.compile(font, destFile, autohint=autohint)
    if autohint:
        print reports["autohint"]
    print reports["makeotf"]

    successMsg = ("makeotfexe [NOTE] Wrote new font file '%s'." %
                  os.path.basename(destFile))
    return successMsg in reports["makeotf"]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# List Available Language Packs
# Used for updating the Languages page of the documentation.
# https://sourcethemes.com/academic/
#
# Prerequisites: pip3 install PyYAML

import yaml

LANG_PATH = Path(__file__).resolve().parent.parent.joinpath('data').joinpath('i18n')
LANG_YAML = LANG_PATH.joinpath('languages.yaml')

# Iterate over languages.
with open(LANG_YAML) as f:
  master_map = yaml.safe_load(f)

  # Print languages as a plaintext list.
  # lang_list = [master_map[master_item] for master_item in master_map]
  # print(', '.join(lang_list))

  # Print languages as a Markdown list.
  for master_item in master_map:
    print(f'- **{master_map[master_item]}** ({master_item})')
<commit_msg>docs: update language pack lister<commit_after>#!/usr/bin/env python3

# List Available Language Packs
# Used for updating the Languages page of the documentation.
# https://sourcethemes.com/academic/
#
# Prerequisites: pip3 install PyYAML

import yaml
from pathlib import Path

LANG_PATH = Path(__file__).resolve().parent.parent.joinpath('data').joinpath('i18n')
LANG_YAML = LANG_PATH.joinpath('languages.yaml')

# Iterate over languages.
with open(LANG_YAML) as f:
  master_map = yaml.safe_load(f)

  # Print languages as a plaintext list.
  # lang_list = [master_map[master_item] for master_item in master_map]
  # print(', '.join(lang_list))

  # Print languages as a Markdown list.
  i = 0
  for master_item in master_map:
    print(f'- **{master_map[master_item]}** ({master_item})')
    i += 1

print("\n")
print(f"{i} language packs found!")
<|endoftext|>


<commit_before>import json
from lxml import etree

from sys import argv, stdout
fpath = argv[1]
with open(fpath) as f:
    root = etree.parse(f)

MARC = '{http://www.loc.gov/MARC21/slim}'

data = {}
fields = []

for elem in root.findall('*'):
    if elem.tag == MARC+'leader':
        data['leader'] = elem.text
    elif elem.tag == MARC+'controlfield':
        fields.append({elem.get('tag'): elem.text})
    else:
        fields.append({elem.get('tag'): {
                'ind1': elem.get('ind1'),
                'ind2': elem.get('ind2'),
                'subfields': [{sub.get('code'): sub.text}
                              for sub in elem.findall('*')]}})

data['fields'] = fields

json.dump(data, stdout)
<commit_msg>Add support for stdin to marcxml_to_marcjson.py<commit_after>import json
from lxml import etree
import sys

if len(sys.argv) > 1:
    fpath = sys.argv[1]
    with open(fpath) as f:
        root = etree.parse(f)
else:
    root = etree.parse(sys.stdin)

MARC = '{http://www.loc.gov/MARC21/slim}'

data = {}
fields = []

for elem in root.findall('*'):
    if elem.tag == MARC+'leader':
        data['leader'] = elem.text
    elif elem.tag == MARC+'controlfield':
        fields.append({elem.get('tag'): elem.text})
    else:
        fields.append({elem.get('tag'): {
                'ind1': elem.get('ind1'),
                'ind2': elem.get('ind2'),
                'subfields': [{sub.get('code'): sub.text}
                              for sub in elem.findall('*')]}})

data['fields'] = fields

json.dump(data, sys.stdout)
<|endoftext|>


<commit_before># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os

from buildbot import interfaces, util
from buildbot.buildslave import BuildSlave
from buildbot.status import mail
from buildbot.status.builder import BuildStatus
from twisted.python import log
from zope.interface import implements

from master.autoreboot_buildslave import AutoRebootBuildSlave
try:
  from master.chromium_status import WebStatus
except ImportError:
  from buildbot.status.web.baseweb import WebStatus

from master import slaves_list
import config

def HackMaxTime(maxTime=8*60*60):
  """Set maxTime default value to 8 hours. This function must be called before
  adding steps."""
  from buildbot.process.buildstep import RemoteShellCommand
  assert RemoteShellCommand.__init__.func_defaults == (None, 1, 1, 1200, None,
      {}, 'slave-config', True)
  RemoteShellCommand.__init__.im_func.func_defaults = (None, 1, 1, 1200,
      maxTime, {}, 'slave-config', True)
  assert RemoteShellCommand.__init__.func_defaults == (None, 1, 1, 1200,
      maxTime, {}, 'slave-config', True)

HackMaxTime()

def HackBuildStatus():
  """Adds a property to Build named 'blamelist' with the blamelist."""
  old_setBlamelist = BuildStatus.setBlamelist
  def setBlamelist(self, blamelist):
    self.setProperty('blamelist', blamelist, 'Build')
    old_setBlamelist(self, blamelist)
  BuildStatus.setBlamelist = setBlamelist

HackBuildStatus()

class InvalidConfig(Exception):
  """Used by VerifySetup."""
  pass

def AutoSetupSlaves(builders, bot_password, max_builds=1,
                    missing_recipients=None, missing_timeout=300):
  """Helper function for master.cfg to quickly setup c['slaves']."""
  slaves_dict = {}
  for builder in builders:
    auto_reboot = builder.get('auto_reboot', False)
    notify_on_missing = builder.get('notify_on_missing', False)
    slavenames = builder.get('slavenames', [])[:]
    if 'slavename' in builder:
      slavenames.append(builder['slavename'])
    for slavename in slavenames:
      slaves_dict[slavename] = (auto_reboot, notify_on_missing)
  slaves = []
  for (slavename, (auto_reboot, notify_on_missing)) in slaves_dict.iteritems():
    if auto_reboot:
      slave_class = AutoRebootBuildSlave
    else:
      slave_class = BuildSlave

    if notify_on_missing:
      slaves.append(slave_class(slavename, bot_password, max_builds=max_builds,
                                notify_on_missing=missing_recipients,
                                missing_timeout=missing_timeout))
    else:
      slaves.append(slave_class(slavename, bot_password, max_builds=max_builds))
  return slaves

def VerifySetup(c, slaves):
  """Verify all the available slaves in the slave configuration are used and
  that all the builders have a slave."""
  # Extract the list of slaves associated to a builder and make sure each
  # builder has its slaves connected.
  # Verify each builder has at least one slave.
  builders_slaves = set()
  slaves_name = [s.slavename for s in c['slaves']]
  for b in c['builders']:
    builder_slaves = set()
    slavename = b.get('slavename')
    if slavename:
      builder_slaves.add(slavename)
    slavenames = b.get('slavenames', [])
    for s in slavenames:
      builder_slaves.add(s)
    if not slavename and not slavenames:
      raise InvalidConfig('Builder %s has no slave' % b['name'])
    # Now test.
    for s in builder_slaves:
      if not s in slaves_name:
        raise InvalidConfig('Builder %s using undefined slave %s' % (b['name'],
                                                                     s))
    builders_slaves |= builder_slaves
  if len(builders_slaves) != len(slaves_name):
    raise InvalidConfig('Same slave defined multiple times')

  # Make sure each slave has their builder.
  builders_name = [b['name'] for b in c['builders']]
  for s in c['slaves']:
    name = s.slavename
    if not name in builders_slaves:
      raise InvalidConfig('Slave %s not associated with any builder' % name)

  # Make sure every defined slave is used.
  for s in slaves.GetSlaves():
    name = slaves_list.EntryToSlaveName(s)
    if not name in slaves_name:
      raise InvalidConfig('Slave %s defined in your slaves_list is not '
                          'referenced at all' % name)
    builders = s.get('builder', [])
    if not isinstance(builders, (list, tuple)):
      builders = [builders]
    testers = s.get('testers', [])
    if not isinstance(testers, (list, tuple)):
      testers = [testers]
    builders.extend(testers)
    for b in builders:
      if not b in builders_name:
        raise InvalidConfig('Slave %s uses non-existent builder %s' % (name,
                                                                       b))

class FilterDomain(util.ComparableMixin):
  """Similar to buildbot.mail.Domain but permits filtering out people we don't
  want to spam.

  Also loads default values from chromium_config."""
  implements(interfaces.IEmailLookup)

  compare_attrs = ['domain', 'permitted_domains']

  def __init__(self, domain=None, permitted_domains=None):
    """domain is the default domain to append when only the naked username is
    available.
    permitted_domains is a whitelist of domains that emails will be sent to."""
    # pylint: disable=E1101
    self.domain = domain or config.Master.master_domain
    self.permitted_domains = (permitted_domains or
                              config.Master.permitted_domains)

  def getAddress(self, name):
    """If name is already an email address, pass it through."""
    result = name
    if self.domain and not '@' in result:
      result = '%s@%s' % (name, self.domain)
    if not '@' in result:
      log.msg('Invalid blame email address "%s"' % result)
      return None
    if self.permitted_domains:
      for p in self.permitted_domains:
        if result.endswith(p):
          return result
      return None
    return result

def AutoSetupMaster(c, active_master, mail_notifier=False,
                    public_html=None, order_console_by_time=False):
  """Add common settings and status services to a master.

  If you wonder what all these mean, PLEASE go check the official doc!
  http://buildbot.net/buildbot/docs/0.7.12/ or
  http://buildbot.net/buildbot/docs/latest/full.html

  - Default number of logs to keep
  - WebStatus and MailNotifier
  - Debug ssh port. Just add a file named .manhole beside master.cfg and
    simply include one line containing 'port = 10101', then you can
    'ssh localhost -p' and you can access your buildbot from the inside."""
  c['slavePortnum'] = active_master.slave_port
  c['projectName'] = active_master.project_name
  c['projectURL'] = config.Master.project_url

  # 'status' is a list of Status Targets. The results of each build will be
  # pushed to these targets. buildbot/status/*.py has a variety to choose from,
  # including web pages, email senders, and IRC bots.
  c.setdefault('status', [])
  if mail_notifier:
    # pylint: disable=E1101
    c['status'].append(mail.MailNotifier(
        fromaddr=active_master.from_address,
        mode='problem',
        relayhost=config.Master.smtp,
        lookup=FilterDomain()))

  kwargs = {}
  if public_html:
    kwargs['public_html'] = public_html
  kwargs['order_console_by_time'] = order_console_by_time
  if active_master.master_port:
    c['status'].append(WebStatus(active_master.master_port, allowForce=True,
                                 **kwargs))
  if active_master.master_port_alt:
    c['status'].append(WebStatus(active_master.master_port_alt,
                                 allowForce=False,
                                 **kwargs))

  # Keep last build logs, the default is too low.
  c['buildHorizon'] = 1000
  c['logHorizon'] = 500
  # Must be at least 2x the number of slaves.
  c['eventHorizon'] = 200
  # Must be at least 1x the number of builds listed in console.
  c['buildCacheSize'] = 60

  # See http://buildbot.net/buildbot/docs/0.8.1/Debug-Options.html for more
  # details.
  if os.path.isfile('.manhole'):
    try:
      from buildbot import manhole
    except ImportError:
      log.msg('Using manhole has an implicit dependency on Crypto.Cipher. You '
              'need to install it manually: sudo apt-get install python-crypto '
              'on ubuntu.')
      raise

    # If 'port' is defined, it uses the same valid keys as the current user.
    values = {}
    execfile('.manhole', values)
    if 'debugPassword' in values:
      c['debugPassword'] = values['debugPassword']
    interface = 'tcp:%s:interface=127.0.0.1' % values.get('port', 0)
    if 'port' in values and 'user' in values and 'password' in values:
      c['manhole'] = manhole.PasswordManhole(interface, values['user'],
                                            values['password'])
    elif 'port' in values:
      c['manhole'] = manhole.AuthorizedKeysManhole(interface,
          os.path.expanduser("~/.ssh/authorized_keys"))
<commit_msg>Set num_events_max in Buildbot's WebStatus.<commit_after># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os

from buildbot import interfaces, util
from buildbot.buildslave import BuildSlave
from buildbot.status import mail
from buildbot.status.builder import BuildStatus
from twisted.python import log
from zope.interface import implements

from master.autoreboot_buildslave import AutoRebootBuildSlave
try:
  from master.chromium_status import WebStatus
except ImportError:
  from buildbot.status.web.baseweb import WebStatus

from master import slaves_list
import config

def HackMaxTime(maxTime=8*60*60):
  """Set maxTime default value to 8 hours. This function must be called before
  adding steps."""
  from buildbot.process.buildstep import RemoteShellCommand
  assert RemoteShellCommand.__init__.func_defaults == (None, 1, 1, 1200, None,
      {}, 'slave-config', True)
  RemoteShellCommand.__init__.im_func.func_defaults = (None, 1, 1, 1200,
      maxTime, {}, 'slave-config', True)
  assert RemoteShellCommand.__init__.func_defaults == (None, 1, 1, 1200,
      maxTime, {}, 'slave-config', True)

HackMaxTime()

def HackBuildStatus():
  """Adds a property to Build named 'blamelist' with the blamelist."""
  old_setBlamelist = BuildStatus.setBlamelist
  def setBlamelist(self, blamelist):
    self.setProperty('blamelist', blamelist, 'Build')
    old_setBlamelist(self, blamelist)
  BuildStatus.setBlamelist = setBlamelist

HackBuildStatus()

class InvalidConfig(Exception):
  """Used by VerifySetup."""
  pass

def AutoSetupSlaves(builders, bot_password, max_builds=1,
                    missing_recipients=None, missing_timeout=300):
  """Helper function for master.cfg to quickly setup c['slaves']."""
  slaves_dict = {}
  for builder in builders:
    auto_reboot = builder.get('auto_reboot', False)
    notify_on_missing = builder.get('notify_on_missing', False)
    slavenames = builder.get('slavenames', [])[:]
    if 'slavename' in builder:
      slavenames.append(builder['slavename'])
    for slavename in slavenames:
      slaves_dict[slavename] = (auto_reboot, notify_on_missing)
  slaves = []
  for (slavename, (auto_reboot, notify_on_missing)) in slaves_dict.iteritems():
    if auto_reboot:
      slave_class = AutoRebootBuildSlave
    else:
      slave_class = BuildSlave

    if notify_on_missing:
      slaves.append(slave_class(slavename, bot_password, max_builds=max_builds,
                                notify_on_missing=missing_recipients,
                                missing_timeout=missing_timeout))
    else:
      slaves.append(slave_class(slavename, bot_password, max_builds=max_builds))
  return slaves

def VerifySetup(c, slaves):
  """Verify all the available slaves in the slave configuration are used and
  that all the builders have a slave."""
  # Extract the list of slaves associated to a builder and make sure each
  # builder has its slaves connected.
  # Verify each builder has at least one slave.
  builders_slaves = set()
  slaves_name = [s.slavename for s in c['slaves']]
  for b in c['builders']:
    builder_slaves = set()
    slavename = b.get('slavename')
    if slavename:
      builder_slaves.add(slavename)
    slavenames = b.get('slavenames', [])
    for s in slavenames:
      builder_slaves.add(s)
    if not slavename and not slavenames:
      raise InvalidConfig('Builder %s has no slave' % b['name'])
    # Now test.
    for s in builder_slaves:
      if not s in slaves_name:
        raise InvalidConfig('Builder %s using undefined slave %s' % (b['name'],
                                                                     s))
    builders_slaves |= builder_slaves
  if len(builders_slaves) != len(slaves_name):
    raise InvalidConfig('Same slave defined multiple times')

  # Make sure each slave has their builder.
  builders_name = [b['name'] for b in c['builders']]
  for s in c['slaves']:
    name = s.slavename
    if not name in builders_slaves:
      raise InvalidConfig('Slave %s not associated with any builder' % name)

  # Make sure every defined slave is used.
  for s in slaves.GetSlaves():
    name = slaves_list.EntryToSlaveName(s)
    if not name in slaves_name:
      raise InvalidConfig('Slave %s defined in your slaves_list is not '
                          'referenced at all' % name)
    builders = s.get('builder', [])
    if not isinstance(builders, (list, tuple)):
      builders = [builders]
    testers = s.get('testers', [])
    if not isinstance(testers, (list, tuple)):
      testers = [testers]
    builders.extend(testers)
    for b in builders:
      if not b in builders_name:
        raise InvalidConfig('Slave %s uses non-existent builder %s' % (name,
                                                                       b))

class FilterDomain(util.ComparableMixin):
  """Similar to buildbot.mail.Domain but permits filtering out people we don't
  want to spam.

  Also loads default values from chromium_config."""
  implements(interfaces.IEmailLookup)

  compare_attrs = ['domain', 'permitted_domains']

  def __init__(self, domain=None, permitted_domains=None):
    """domain is the default domain to append when only the naked username is
    available.
    permitted_domains is a whitelist of domains that emails will be sent to."""
    # pylint: disable=E1101
    self.domain = domain or config.Master.master_domain
    self.permitted_domains = (permitted_domains or
                              config.Master.permitted_domains)

  def getAddress(self, name):
    """If name is already an email address, pass it through."""
    result = name
    if self.domain and not '@' in result:
      result = '%s@%s' % (name, self.domain)
    if not '@' in result:
      log.msg('Invalid blame email address "%s"' % result)
      return None
    if self.permitted_domains:
      for p in self.permitted_domains:
        if result.endswith(p):
          return result
      return None
    return result

def AutoSetupMaster(c, active_master, mail_notifier=False,
                    public_html=None, order_console_by_time=False):
  """Add common settings and status services to a master.

  If you wonder what all these mean, PLEASE go check the official doc!
  http://buildbot.net/buildbot/docs/0.7.12/ or
  http://buildbot.net/buildbot/docs/latest/full.html

  - Default number of logs to keep
  - WebStatus and MailNotifier
  - Debug ssh port. Just add a file named .manhole beside master.cfg and
    simply include one line containing 'port = 10101', then you can
    'ssh localhost -p' and you can access your buildbot from the inside."""
  c['slavePortnum'] = active_master.slave_port
  c['projectName'] = active_master.project_name
  c['projectURL'] = config.Master.project_url

  # 'status' is a list of Status Targets. The results of each build will be
  # pushed to these targets. buildbot/status/*.py has a variety to choose from,
  # including web pages, email senders, and IRC bots.
  c.setdefault('status', [])
  if mail_notifier:
    # pylint: disable=E1101
    c['status'].append(mail.MailNotifier(
        fromaddr=active_master.from_address,
        mode='problem',
        relayhost=config.Master.smtp,
        lookup=FilterDomain()))

  kwargs = {}
  if public_html:
    kwargs['public_html'] = public_html
  kwargs['order_console_by_time'] = order_console_by_time
  if active_master.master_port:
    c['status'].append(WebStatus(active_master.master_port, allowForce=True,
                                 num_events_max=3000, **kwargs))
  if active_master.master_port_alt:
    c['status'].append(WebStatus(active_master.master_port_alt,
                                 allowForce=False, num_events_max=3000,
                                 **kwargs))

  # Keep last build logs, the default is too low.
  c['buildHorizon'] = 1000
  c['logHorizon'] = 500
  # Must be at least 2x the number of slaves.
  c['eventHorizon'] = 200
  # Must be at least 1x the number of builds listed in console.
  c['buildCacheSize'] = 60

  # See http://buildbot.net/buildbot/docs/0.8.1/Debug-Options.html for more
  # details.
  if os.path.isfile('.manhole'):
    try:
      from buildbot import manhole
    except ImportError:
      log.msg('Using manhole has an implicit dependency on Crypto.Cipher. You '
              'need to install it manually: sudo apt-get install python-crypto '
              'on ubuntu.')
      raise

    # If 'port' is defined, it uses the same valid keys as the current user.
    values = {}
    execfile('.manhole', values)
    if 'debugPassword' in values:
      c['debugPassword'] = values['debugPassword']
    interface = 'tcp:%s:interface=127.0.0.1' % values.get('port', 0)
    if 'port' in values and 'user' in values and 'password' in values:
      c['manhole'] = manhole.PasswordManhole(interface, values['user'],
                                            values['password'])
    elif 'port' in values:
      c['manhole'] = manhole.AuthorizedKeysManhole(interface,
          os.path.expanduser("~/.ssh/authorized_keys"))
<|endoftext|>