<commit_before>#!/usr/bin/python
#-*- coding: utf-8 -*-

# ======================================================================
# Copyright 2016 Julien LE CLEACH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================================================

from collections import OrderedDict
from StringIO import StringIO
from sys import stderr

from supervisor.datatypes import boolean, list_of_strings

from supvisors.ttypes import StartingFailureStrategies
from supvisors.utils import supvisors_short_cuts

# XSD contents for XML validation
XSDContents = StringIO('''\
<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
    <xs:simpleType name="Loading">
        <xs:restriction base="xs:int">
            <xs:minInclusive value="0"/>
            <xs:maxInclusive value="100"/>
        </xs:restriction>
    </xs:simpleType>
    <xs:simpleType name="StartingFailureStrategy" final="restriction" >
        <xs:restriction base="xs:string">
            <xs:enumeration value="ABORT" />
            <xs:enumeration value="CONTINUE" />
            <xs:enumeration value="STOP" />
        </xs:restriction>
    </xs:simpleType>
    <xs:complexType name="ProgramModel">
        <xs:choice>
            <xs:element type="xs:string" name="reference"/>
            <xs:sequence>
                <xs:element type="xs:string" name="addresses" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:boolean" name="required" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:boolean" name="wait_exit" minOccurs="0" maxOccurs="1"/>
                <xs:element type="Loading" name="expected_loading" minOccurs="0" maxOccurs="1"/>
            </xs:sequence>
        </xs:choice>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:complexType name="ApplicationModel">
        <xs:sequence>
            <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="StartingFailureStrategy" name="starting_failure_strategy" minOccurs="0" maxOccurs="1"/>
            <xs:choice minOccurs="0" maxOccurs="unbounded">
                <xs:element type="ProgramModel" name="program"/>
                <xs:element type="ProgramModel" name="pattern"/>
            </xs:choice>
        </xs:sequence>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:element name="root">
        <xs:complexType>
            <xs:sequence>
                <xs:choice minOccurs="0" maxOccurs="unbounded">
                    <xs:element type="ProgramModel" name="model"/>
                    <xs:element type="ApplicationModel" name="application"/>
                </xs:choice>
            </xs:sequence>
        </xs:complexType>
    </xs:element>
</xs:schema>
''')

class Parser(object):

    def __init__(self, supvisors):
        self.supvisors = supvisors
        supvisors_short_cuts(self, ['logger'])
        self.tree = self.parse(supvisors.options.deployment_file)
        self.root = self.tree.getroot()
        # get models
        elements = self.root.findall("./model[@name]")
        self.models = {element.get('name'): element for element in elements}
        self.logger.debug(self.models)
        # get patterns
        elements = self.root.findall(".//pattern[@name]")
        self.patterns = {element.get('name'): element for element in elements}
        self.logger.debug(self.patterns)

    def load_application_rules(self, application):
        # find application element
        self.logger.trace('searching application element for {}'.format(application.application_name))
        application_elt = self.root.find("./application[@name='{}']".format(application.application_name))
        if application_elt is not None:
            # get start_sequence rule
            value = application_elt.findtext('start_sequence')
            application.rules.start_sequence = int(value) if value and int(value)>0 else 0
            # get stop_sequence rule
            value = application_elt.findtext('stop_sequence')
            application.rules.stop_sequence = int(value) if value and int(value)>0 else 0
            # get starting_failure_strategy rule
            value = application_elt.findtext('starting_failure_strategy')
            application.rules.starting_failure_strategy = (StartingFailureStrategies._from_string(value)
                if value else StartingFailureStrategies.ABORT)
            # final print
            self.logger.debug('application {} - rules {}'.format(application.application_name, application.rules))

    def load_process_rules(self, process):
        self.logger.trace('searching program element for {}'.format(process.namespec()))
        program_elt = self.get_program_element(process)
        if program_elt is not None:
            # get addresses rule
            self.get_program_addresses(program_elt, process.rules)
            # get start_sequence rule
            value = program_elt.findtext('start_sequence')
            process.rules.start_sequence = int(value) if value and int(value)>0 else 0
            # get stop_sequence rule
            value = program_elt.findtext('stop_sequence')
            process.rules.stop_sequence = int(value) if value and int(value)>0 else 0
            # get required rule
            value = program_elt.findtext('required')
            process.rules.required = boolean(value) if value else False
            # get wait_exit rule
            value = program_elt.findtext('wait_exit')
            process.rules.wait_exit = boolean(value) if value else False
            # get expected_loading rule
            value = program_elt.findtext('expected_loading')
            process.rules.expected_loading = int(value) if value and 0 <= int(value) <= 100 else 1
            # check that rules are compliant with dependencies
            process.rules.check_dependencies(process.namespec())
            self.logger.info('process {} - rules {}'.format(process.namespec(), process.rules))

    def get_program_addresses(self, program_elt, rules):
        value = program_elt.findtext('addresses')
        if value:
            # sort and trim
            addresses = list(OrderedDict.fromkeys(filter(None, list_of_strings(value))))
            if '*' in addresses:
                rules.addresses = [ '*' ]
            elif '#' in addresses:
                rules.addresses = [ '#' ]
            else:
                rules.addresses = self.supvisors.address_mapper.filter(addresses)

    def get_program_element(self, process):
        # try to find program name in file
        program_elt = self.root.find("./application[@name='{}']/program[@name='{}']".format(process.application_name, process.process_name))
        self.logger.trace('{} - direct search program element {}'.format(process.namespec(), program_elt))
        if program_elt is None:
            # try to find a corresponding pattern
            patterns = [name for name, element in self.patterns.items() if name in process.namespec()]
            self.supvisors.logger.trace('{} - found patterns {}'.format(process.namespec(), patterns))
            if patterns:
                pattern = max(patterns, key=len)
                program_elt = self.patterns[pattern]
            self.logger.trace('{} - pattern search program element {}'.format(process.namespec(), program_elt))
        if program_elt is not None:
            # find if model referenced in element
            model = program_elt.findtext('reference')
            if model in self.models.keys():
                program_elt = self.models[model]
            self.logger.trace('{} - model search ({}) program element {}'.format(process.namespec(), model, program_elt))
        return program_elt

    def parse(self, filename):
        self.parser = None
        # find parser
        try:
            from lxml.etree import parse, XMLSchema
            self.logger.info('using lxml.etree parser')
            # parse XML and validate it
            tree = parse(filename)
            # get XSD
            schemaDoc = parse(XSDContents)
            schema = XMLSchema(schemaDoc)
            xml_valid = schema.validate(tree)
            if xml_valid:
                self.logger.info('XML validated')
            else:
                self.logger.error('XML NOT validated: {}'.format(filename))
                print >> stderr,  schema.error_log
            return tree if xml_valid else None
        except ImportError:
            try:
                from xml.etree.ElementTree import parse
                self.logger.info('using xml.etree.ElementTree parser')
                return parse(filename)
            except ImportError:
                self.logger.critical("Failed to import ElementTree from any known place")
                raise
<commit_msg>log level changed for process rules<commit_after>#!/usr/bin/python
#-*- coding: utf-8 -*-

# ======================================================================
# Copyright 2016 Julien LE CLEACH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================================================

from collections import OrderedDict
from StringIO import StringIO
from sys import stderr

from supervisor.datatypes import boolean, list_of_strings

from supvisors.ttypes import StartingFailureStrategies
from supvisors.utils import supvisors_short_cuts

# XSD contents for XML validation
XSDContents = StringIO('''\
<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
    <xs:simpleType name="Loading">
        <xs:restriction base="xs:int">
            <xs:minInclusive value="0"/>
            <xs:maxInclusive value="100"/>
        </xs:restriction>
    </xs:simpleType>
    <xs:simpleType name="StartingFailureStrategy" final="restriction" >
        <xs:restriction base="xs:string">
            <xs:enumeration value="ABORT" />
            <xs:enumeration value="CONTINUE" />
            <xs:enumeration value="STOP" />
        </xs:restriction>
    </xs:simpleType>
    <xs:complexType name="ProgramModel">
        <xs:choice>
            <xs:element type="xs:string" name="reference"/>
            <xs:sequence>
                <xs:element type="xs:string" name="addresses" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:boolean" name="required" minOccurs="0" maxOccurs="1"/>
                <xs:element type="xs:boolean" name="wait_exit" minOccurs="0" maxOccurs="1"/>
                <xs:element type="Loading" name="expected_loading" minOccurs="0" maxOccurs="1"/>
            </xs:sequence>
        </xs:choice>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:complexType name="ApplicationModel">
        <xs:sequence>
            <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="StartingFailureStrategy" name="starting_failure_strategy" minOccurs="0" maxOccurs="1"/>
            <xs:choice minOccurs="0" maxOccurs="unbounded">
                <xs:element type="ProgramModel" name="program"/>
                <xs:element type="ProgramModel" name="pattern"/>
            </xs:choice>
        </xs:sequence>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:element name="root">
        <xs:complexType>
            <xs:sequence>
                <xs:choice minOccurs="0" maxOccurs="unbounded">
                    <xs:element type="ProgramModel" name="model"/>
                    <xs:element type="ApplicationModel" name="application"/>
                </xs:choice>
            </xs:sequence>
        </xs:complexType>
    </xs:element>
</xs:schema>
''')

class Parser(object):

    def __init__(self, supvisors):
        self.supvisors = supvisors
        supvisors_short_cuts(self, ['logger'])
        self.tree = self.parse(supvisors.options.deployment_file)
        self.root = self.tree.getroot()
        # get models
        elements = self.root.findall("./model[@name]")
        self.models = {element.get('name'): element for element in elements}
        self.logger.debug(self.models)
        # get patterns
        elements = self.root.findall(".//pattern[@name]")
        self.patterns = {element.get('name'): element for element in elements}
        self.logger.debug(self.patterns)

    def load_application_rules(self, application):
        # find application element
        self.logger.trace('searching application element for {}'.format(application.application_name))
        application_elt = self.root.find("./application[@name='{}']".format(application.application_name))
        if application_elt is not None:
            # get start_sequence rule
            value = application_elt.findtext('start_sequence')
            application.rules.start_sequence = int(value) if value and int(value)>0 else 0
            # get stop_sequence rule
            value = application_elt.findtext('stop_sequence')
            application.rules.stop_sequence = int(value) if value and int(value)>0 else 0
            # get starting_failure_strategy rule
            value = application_elt.findtext('starting_failure_strategy')
            application.rules.starting_failure_strategy = (StartingFailureStrategies._from_string(value)
                if value else StartingFailureStrategies.ABORT)
            # final print
            self.logger.debug('application {} - rules {}'.format(application.application_name, application.rules))

    def load_process_rules(self, process):
        self.logger.trace('searching program element for {}'.format(process.namespec()))
        program_elt = self.get_program_element(process)
        if program_elt is not None:
            # get addresses rule
            self.get_program_addresses(program_elt, process.rules)
            # get start_sequence rule
            value = program_elt.findtext('start_sequence')
            process.rules.start_sequence = int(value) if value and int(value)>0 else 0
            # get stop_sequence rule
            value = program_elt.findtext('stop_sequence')
            process.rules.stop_sequence = int(value) if value and int(value)>0 else 0
            # get required rule
            value = program_elt.findtext('required')
            process.rules.required = boolean(value) if value else False
            # get wait_exit rule
            value = program_elt.findtext('wait_exit')
            process.rules.wait_exit = boolean(value) if value else False
            # get expected_loading rule
            value = program_elt.findtext('expected_loading')
            process.rules.expected_loading = int(value) if value and 0 <= int(value) <= 100 else 1
            # check that rules are compliant with dependencies
            process.rules.check_dependencies(process.namespec())
            self.logger.debug('process {} - rules {}'.format(process.namespec(), process.rules))

    def get_program_addresses(self, program_elt, rules):
        value = program_elt.findtext('addresses')
        if value:
            # sort and trim
            addresses = list(OrderedDict.fromkeys(filter(None, list_of_strings(value))))
            if '*' in addresses:
                rules.addresses = [ '*' ]
            elif '#' in addresses:
                rules.addresses = [ '#' ]
            else:
                rules.addresses = self.supvisors.address_mapper.filter(addresses)

    def get_program_element(self, process):
        # try to find program name in file
        program_elt = self.root.find("./application[@name='{}']/program[@name='{}']".format(process.application_name, process.process_name))
        self.logger.trace('{} - direct search program element {}'.format(process.namespec(), program_elt))
        if program_elt is None:
            # try to find a corresponding pattern
            patterns = [name for name, element in self.patterns.items() if name in process.namespec()]
            self.supvisors.logger.trace('{} - found patterns {}'.format(process.namespec(), patterns))
            if patterns:
                pattern = max(patterns, key=len)
                program_elt = self.patterns[pattern]
            self.logger.trace('{} - pattern search program element {}'.format(process.namespec(), program_elt))
        if program_elt is not None:
            # find if model referenced in element
            model = program_elt.findtext('reference')
            if model in self.models.keys():
                program_elt = self.models[model]
            self.logger.trace('{} - model search ({}) program element {}'.format(process.namespec(), model, program_elt))
        return program_elt

    def parse(self, filename):
        self.parser = None
        # find parser
        try:
            from lxml.etree import parse, XMLSchema
            self.logger.info('using lxml.etree parser')
            # parse XML and validate it
            tree = parse(filename)
            # get XSD
            schemaDoc = parse(XSDContents)
            schema = XMLSchema(schemaDoc)
            xml_valid = schema.validate(tree)
            if xml_valid:
                self.logger.info('XML validated')
            else:
                self.logger.error('XML NOT validated: {}'.format(filename))
                print >> stderr,  schema.error_log
            return tree if xml_valid else None
        except ImportError:
            try:
                from xml.etree.ElementTree import parse
                self.logger.info('using xml.etree.ElementTree parser')
                return parse(filename)
            except ImportError:
                self.logger.critical("Failed to import ElementTree from any known place")
                raise
<|endoftext|>


<commit_before>from boto.connection import SQSConnection, S3Connection
from boto.s3.key import Key
from boto.sqs.message import MHMessage
from boto.exception import SQSError, S3ResponseError
import StringIO
import time
import os
import sys, traceback
import md5

class Service:

    RetryCount = 5
    ProcessingTime = 60

    def __init__(self, aws_access_key_id=None,
                 aws_secret_access_key=None, input_queue_name=None,
                 working_dir='work'):
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.input_queue_name = input_queue_name
        self.create_working_dir(working_dir)
        self.create_connections()

    def create_connections(self):
        self.queue_cache = {}
        self.bucket_cache = {}
        self.sqs_conn = SQSConnection(self.aws_access_key_id,
                                      self.aws_secret_access_key)
        if self.input_queue_name:
            self.input_queue = self.get_queue(self.input_queue_name)
        self.s3_conn = S3Connection(self.aws_access_key_id,
                                    self.aws_secret_access_key)

    def create_working_dir(self, working_dir):
        self.working_dir = os.path.expanduser('~/%s' % working_dir)
        if not os.path.exists(self.working_dir):
            os.mkdir(self.working_dir)
        os.chdir(self.working_dir)

    def notify(self, msg):
        pass

    def compute_key(self, filename):
        fp = open(filename, 'rb')
        m = md5.new()
        s = fp.read(4096)
        while s:
            m.update(s)
            s = fp.read(4096)
        fp.close()
        return m.hexdigest()

    def split_key(key):
        if key.find(';') < 0:
            t = (key, '')
        else:
            key, type = key.split(';')
            label, mtype = type.split('=')
            t = (key, mtype)
        return t

    def get_queue(self, queue_name):
        if queue_name in self.queue_cache.keys():
            return self.queue_cache[queue_name]
        else:
            queue = self.sqs_conn.create_queue(queue_name)
            queue.set_message_class(SDSMessage)
            self.queue_cache[queue_name] = queue
            return queue

    def get_bucket(self, bucket_name):
        if bucket_name in self.bucket_cache.keys():
            return self.bucket_cache[bucket_name]
        else:
            bucket = self.s3_conn.create_bucket(bucket_name)
            self.bucket_cache[bucket_name] = bucket
            return bucket

    def key_exists(self, bucket_name, key):
        bucket = self.get_bucket(bucket_name)
        return bucket.lookup(key)

    # read a new message from our queue
    def read_message(self):
        try:
            message = self.input_queue.read(self.ProcessingTime)
            if message:
                print message.get_body()
                key = 'Service-Read'
                message[key] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                             time.gmtime())
        except SQSError, e:
            print 'caught SQSError[%s]: %s' % (e.status, e.reason)
            message = None
        return message

    # retrieve the source file from S3
    def get_file(self, bucket_name, key, file_name):
        successful = False
        while not successful:
            try:
                print 'getting file %s.%s' % (bucket_name, key)
                bucket = self.get_bucket(bucket_name)
                k = Key(bucket)
                k.key = key
                k.get_contents_to_filename(file_name)
                successful = True
            except S3ResponseError, e:
                print 'caught S3Error[%s]: %s' % (e.status, e.reason)

    # process source file, return list of output files
    def process_file(self, in_file_name, msg):
        return []

    # store result file in S3
    def put_file(self, bucket_name, file_name, key):
        successful = False
        while not successful:
            try:
                bucket = self.get_bucket(bucket_name)
                k = Key(bucket)
                k.key = key
                k.set_contents_from_filename(file_name)
                print 'putting file %s as %s.%s' % (file_name, bucket_name, k.key)
                successful = True
            except S3ResponseError, e:
                print 'caught S3Error[%s]: %s' % (e.status, e.reason)
        return k.key

    # write message to each output queue
    def write_message(self, queue_names, message):
        message['Service-Write'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                                 time.gmtime())
        message['Server'] = self.__class__.__name__
        if os.environ.has_key('HOSTNAME'):
            message['Host'] = os.environ['HOSTNAME']
        else:
            message['Host'] = 'unknown'
        l = queue_names.split(',')
        for queue_name in l:
            queue = self.get_queue(queue_name)
            print 'writing message to %s' % queue.id
            successful = False
            num_tries = 0
            while not successful and num_tries < self.RetryCount:
                try:
                    num_tries += 1
                    print message.get_body()
                    queue.write(message)
                    successful = True
                except SQSError, e:
                    print 'caught SQSError[%s]: %s' % (e.status, e.reason)

    # delete message from input queue
    def delete_message(self, message):
        print 'deleting message from %s' % self.input_queue.id
        successful = False
        num_tries = 0
        while not successful and num_tries < self.RetryCount:
            try:
                num_tries += 1
                self.input_queue.delete_message(message)
                successful = True
            except SQSError, e:
                print 'caught SQSError[%s]: %s' % (e.status, e.reason)

    # to clean up any files, etc. after each iteration
    def cleanup(self):
        pass

    def run(self, notify=False):
        if notify:
            self.notify('Service Starting')
        num_tries = 0
        while num_tries < self.RetryCount:
            try:
                input_message = self.read_message()
                if input_message:
                    num_tries = 0
                    output_message = SDSMessage(None, input_message.get_body())
                    in_key = self.split_key(input_message['InputKey'])[0]
                    self.get_file(input_message['InputBucket'], in_key,
                                  os.path.join(self.working_dir,'in_file'))
                    results = self.process_file(os.path.join(self.working_dir,
                                                             'in_file'),
                                                output_message)
                    output_keys = []
                    for file, type in results:
                        key = self.compute_key(file)
                        output_keys.append('%s;type=%s' % (key, type))
                        self.put_file(input_message['OutputBucket'],
                                      os.path.join(self.working_dir,
                                                   'out_file'),
                                      key)
                    output_message['OutputKey'] = ','.join(output_keys)
                    self.write_message(input_message['OutputQueues'],
                                       output_message)
                    self.delete_message(input_message)
                    self.cleanup()
                else:
                    num_tries += 1
                    time.sleep(30)
            except Exception, e:
                num_tries += 1
                if notify:
                    fp = StringIO.StringIO()
                    traceback.print_exc(None, fp)
                    s = fp.getvalue()
                    self.notify('Service failed\n%s' % s)
                self.create_connections()

        if notify:
            self.notify('Service Stopping')
<commit_msg>Adding copyright notice<commit_after># Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

#
# Parts of this code were copied or derived from sample code supplied by AWS.
# The following notice applies to that code.
#
#  This software code is made available "AS IS" without warranties of any
#  kind.  You may copy, display, modify and redistribute the software
#  code either by itself or as incorporated into your code; provided that
#  you do not remove any proprietary notices.  Your use of this software
#  code is at your own risk and you waive any claim against Amazon
#  Digital Services, Inc. or its affiliates with respect to your use of
#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
#  affiliates.

from boto.connection import SQSConnection, S3Connection
from boto.s3.key import Key
from boto.sqs.message import MHMessage
from boto.exception import SQSError, S3ResponseError
import StringIO
import time
import os
import sys, traceback
import md5

class Service:

    RetryCount = 5
    ProcessingTime = 60

    def __init__(self, aws_access_key_id=None,
                 aws_secret_access_key=None, input_queue_name=None,
                 working_dir='work'):
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.input_queue_name = input_queue_name
        self.create_working_dir(working_dir)
        self.create_connections()

    def create_connections(self):
        self.queue_cache = {}
        self.bucket_cache = {}
        self.sqs_conn = SQSConnection(self.aws_access_key_id,
                                      self.aws_secret_access_key)
        if self.input_queue_name:
            self.input_queue = self.get_queue(self.input_queue_name)
        self.s3_conn = S3Connection(self.aws_access_key_id,
                                    self.aws_secret_access_key)

    def create_working_dir(self, working_dir):
        self.working_dir = os.path.expanduser('~/%s' % working_dir)
        if not os.path.exists(self.working_dir):
            os.mkdir(self.working_dir)
        os.chdir(self.working_dir)

    def notify(self, msg):
        pass

    def compute_key(self, filename):
        fp = open(filename, 'rb')
        m = md5.new()
        s = fp.read(4096)
        while s:
            m.update(s)
            s = fp.read(4096)
        fp.close()
        return m.hexdigest()

    def split_key(key):
        if key.find(';') < 0:
            t = (key, '')
        else:
            key, type = key.split(';')
            label, mtype = type.split('=')
            t = (key, mtype)
        return t

    def get_queue(self, queue_name):
        if queue_name in self.queue_cache.keys():
            return self.queue_cache[queue_name]
        else:
            queue = self.sqs_conn.create_queue(queue_name)
            queue.set_message_class(SDSMessage)
            self.queue_cache[queue_name] = queue
            return queue

    def get_bucket(self, bucket_name):
        if bucket_name in self.bucket_cache.keys():
            return self.bucket_cache[bucket_name]
        else:
            bucket = self.s3_conn.create_bucket(bucket_name)
            self.bucket_cache[bucket_name] = bucket
            return bucket

    def key_exists(self, bucket_name, key):
        bucket = self.get_bucket(bucket_name)
        return bucket.lookup(key)

    # read a new message from our queue
    def read_message(self):
        try:
            message = self.input_queue.read(self.ProcessingTime)
            if message:
                print message.get_body()
                key = 'Service-Read'
                message[key] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                             time.gmtime())
        except SQSError, e:
            print 'caught SQSError[%s]: %s' % (e.status, e.reason)
            message = None
        return message

    # retrieve the source file from S3
    def get_file(self, bucket_name, key, file_name):
        successful = False
        while not successful:
            try:
                print 'getting file %s.%s' % (bucket_name, key)
                bucket = self.get_bucket(bucket_name)
                k = Key(bucket)
                k.key = key
                k.get_contents_to_filename(file_name)
                successful = True
            except S3ResponseError, e:
                print 'caught S3Error[%s]: %s' % (e.status, e.reason)

    # process source file, return list of output files
    def process_file(self, in_file_name, msg):
        return []

    # store result file in S3
    def put_file(self, bucket_name, file_name, key):
        successful = False
        while not successful:
            try:
                bucket = self.get_bucket(bucket_name)
                k = Key(bucket)
                k.key = key
                k.set_contents_from_filename(file_name)
                print 'putting file %s as %s.%s' % (file_name, bucket_name, k.key)
                successful = True
            except S3ResponseError, e:
                print 'caught S3Error[%s]: %s' % (e.status, e.reason)
        return k.key

    # write message to each output queue
    def write_message(self, queue_names, message):
        message['Service-Write'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                                 time.gmtime())
        message['Server'] = self.__class__.__name__
        if os.environ.has_key('HOSTNAME'):
            message['Host'] = os.environ['HOSTNAME']
        else:
            message['Host'] = 'unknown'
        l = queue_names.split(',')
        for queue_name in l:
            queue = self.get_queue(queue_name)
            print 'writing message to %s' % queue.id
            successful = False
            num_tries = 0
            while not successful and num_tries < self.RetryCount:
                try:
                    num_tries += 1
                    print message.get_body()
                    queue.write(message)
                    successful = True
                except SQSError, e:
                    print 'caught SQSError[%s]: %s' % (e.status, e.reason)

    # delete message from input queue
    def delete_message(self, message):
        print 'deleting message from %s' % self.input_queue.id
        successful = False
        num_tries = 0
        while not successful and num_tries < self.RetryCount:
            try:
                num_tries += 1
                self.input_queue.delete_message(message)
                successful = True
            except SQSError, e:
                print 'caught SQSError[%s]: %s' % (e.status, e.reason)

    # to clean up any files, etc. after each iteration
    def cleanup(self):
        pass

    def run(self, notify=False):
        if notify:
            self.notify('Service Starting')
        num_tries = 0
        while num_tries < self.RetryCount:
            try:
                input_message = self.read_message()
                if input_message:
                    num_tries = 0
                    output_message = SDSMessage(None, input_message.get_body())
                    in_key = self.split_key(input_message['InputKey'])[0]
                    self.get_file(input_message['InputBucket'], in_key,
                                  os.path.join(self.working_dir,'in_file'))
                    results = self.process_file(os.path.join(self.working_dir,
                                                             'in_file'),
                                                output_message)
                    output_keys = []
                    for file, type in results:
                        key = self.compute_key(file)
                        output_keys.append('%s;type=%s' % (key, type))
                        self.put_file(input_message['OutputBucket'],
                                      os.path.join(self.working_dir,
                                                   'out_file'),
                                      key)
                    output_message['OutputKey'] = ','.join(output_keys)
                    self.write_message(input_message['OutputQueues'],
                                       output_message)
                    self.delete_message(input_message)
                    self.cleanup()
                else:
                    num_tries += 1
                    time.sleep(30)
            except Exception, e:
                num_tries += 1
                if notify:
                    fp = StringIO.StringIO()
                    traceback.print_exc(None, fp)
                    s = fp.getvalue()
                    self.notify('Service failed\n%s' % s)
                self.create_connections()

        if notify:
            self.notify('Service Stopping')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
from babelfish import (LANGUAGES, Language, Country, Script, get_language_converter, LANGUAGE_CONVERTERS,
    LanguageReverseConverter, load_language_converters, clear_language_converters, register_language_converter,
    unregister_language_converter, LanguageConvertError, LanguageReverseError)
from pkg_resources import resource_stream  # @UnresolvedImport

class TestScript(TestCase):
    def test_wrong_script(self):
        with self.assertRaises(ValueError):
            Script('Azer')

    def test_eq(self):
        self.assertEqual(Script('Latn'), Script('Latn'))

    def test_ne(self):
        self.assertNotEqual(Script('Cyrl'), Script('Latn'))

    def test_hash(self):
        self.assertEqual(hash(Script('Hira')), hash('Hira'))

class TestCountry(TestCase):
    def test_wrong_country(self):
        with self.assertRaises(ValueError):
            Country('ZZ')

    def test_eq(self):
        self.assertEqual(Country('US'), Country('US'))

    def test_ne(self):
        self.assertNotEqual(Country('GB'), Country('US'))
        self.assertNotEqual(Country('US'), None)

    def test_hash(self):
        self.assertEqual(hash(Country('US')), hash('US'))

    def test_name(self):
        self.assertEqual(Country('US').name, 'UNITED STATES')
        self.assertEqual(Country('FR').name, 'FRANCE')

class TestLanguage(TestCase):
    def test_languages(self):
        self.assertEqual(len(LANGUAGES), 7874)

    def test_wrong_language(self):
        with self.assertRaises(ValueError):
            Language('zzz')

    def test_unknown_language(self):
        self.assertEqual(Language('zzzz', unknown='und'), Language('und'))

    def test_converter_alpha2(self):
        self.assertEqual(Language('eng').alpha2, 'en')
        self.assertEqual(Language.fromalpha2('en'), Language('eng'))
        self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
        with self.assertRaises(LanguageReverseError):
            Language.fromalpha2('zz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').alpha2
        self.assertEqual(len(get_language_converter('alpha2').codes), 184)

    def test_converter_alpha3b(self):
        self.assertEqual(Language('fra').alpha3b, 'fre')
        self.assertEqual(Language.fromalpha3b('fre'), Language('fra'))
        self.assertEqual(Language.fromcode('fre', 'alpha3b'), Language('fra'))
        with self.assertRaises(LanguageReverseError):
            Language.fromalpha3b('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').alpha3b
        self.assertEqual(len(get_language_converter('alpha3b').codes), 418)

    def test_converter_alpha3t(self):
        self.assertEqual(Language('fra').alpha3t, 'fra')
        self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
        self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
        with self.assertRaises(LanguageReverseError):
            Language.fromalpha3t('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').alpha3t
        self.assertEqual(len(get_language_converter('alpha3t').codes), 418)

    def test_converter_name(self):
        self.assertEqual(Language('eng').name, 'English')
        self.assertEqual(Language.fromname('English'), Language('eng'))
        self.assertEqual(Language.fromcode('English', 'name'), Language('eng'))
        with self.assertRaises(LanguageReverseError):
            Language.fromname('Zzzzzzzzz')
        self.assertEqual(len(get_language_converter('name').codes), 7874)

    def test_converter_scope(self):
        self.assertEqual(get_language_converter('scope').codes, {'I', 'S', 'M'})
        self.assertEqual(Language('eng').scope, 'individual')
        self.assertEqual(Language('und').scope, 'special')

    def test_converter_language_type(self):
        self.assertEqual(get_language_converter('type').codes, {'A', 'C', 'E', 'H', 'L', 'S'})
        self.assertEqual(Language('eng').type, 'living')
        self.assertEqual(Language('und').type, 'special')

    def test_converter_opensubtitles(self):
        self.assertEqual(Language('fra').opensubtitles, Language('fra').alpha3b)
        self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
        self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
        self.assertEqual(Language.fromopensubtitles('pob'), Language('por', 'BR'))
        self.assertEqual(Language.fromopensubtitles('pb'), Language('por', 'BR'))
        # Montenegrin is not recognized as an ISO language (yet?) but for now it is
        # unofficially accepted as Serbian from Montenegro
        self.assertEqual(Language.fromopensubtitles('mne'), Language('srp', 'ME'))
        self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
        with self.assertRaises(LanguageReverseError):
            Language.fromopensubtitles('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').opensubtitles
        self.assertEqual(len(get_language_converter('opensubtitles').codes), 606)

        # test with all the languages from the opensubtitles api
        # downloaded from: http://www.opensubtitles.org/addons/export_languages.php
        f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
        f.readline()
        for l in f:
            idlang, alpha2, _, upload_enabled, web_enabled = l.decode('utf-8').strip().split('\t')
            if not int(upload_enabled) and not int(web_enabled):
                # do not test languages that are too esoteric / not widely available
                continue
            self.assertEqual(Language.fromopensubtitles(idlang).opensubtitles, idlang)
            if alpha2:
                self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
        f.close()

    def test_fromietf_country_script(self):
        language = Language.fromietf('fra-FR-Latn')
        self.assertEqual(language.alpha3, 'fra')
        self.assertEqual(language.country, Country('FR'))
        self.assertEqual(language.script, Script('Latn'))

    def test_fromietf_country_no_script(self):
        language = Language.fromietf('fra-FR')
        self.assertEqual(language.alpha3, 'fra')
        self.assertEqual(language.country, Country('FR'))
        self.assertIs(language.script, None)

    def test_fromietf_no_country_no_script(self):
        language = Language.fromietf('fra-FR')
        self.assertEqual(language.alpha3, 'fra')
        self.assertEqual(language.country, Country('FR'))
        self.assertIs(language.script, None)

    def test_fromietf_no_country_script(self):
        language = Language.fromietf('fra-Latn')
        self.assertEqual(language.alpha3, 'fra')
        self.assertIs(language.country, None)
        self.assertEqual(language.script, Script('Latn'))

    def test_fromietf_alpha2_language(self):
        language = Language.fromietf('fr-Latn')
        self.assertEqual(language.alpha3, 'fra')
        self.assertIs(language.country, None)
        self.assertEqual(language.script, Script('Latn'))

    def test_fromietf_wrong_language(self):
        with self.assertRaises(ValueError):
            Language.fromietf('xyz-FR')

    def test_fromietf_wrong_country(self):
        with self.assertRaises(ValueError):
            Language.fromietf('fra-YZ')

    def test_fromietf_wrong_script(self):
        with self.assertRaises(ValueError):
            Language.fromietf('fra-FR-Wxyz')

    def test_eq(self):
        self.assertEqual(Language('eng'), Language('eng'))

    def test_ne(self):
        self.assertNotEqual(Language('fra'), Language('eng'))
        self.assertIsNot(Language('fra'), None)

    def test_nonzero(self):
        self.assertIs(bool(Language('und')), False)
        self.assertIs(bool(Language('eng')), True)

    def test_country(self):
        self.assertEqual(Language('por', 'BR').country, Country('BR'))
        self.assertEqual(Language('eng', Country('US')).country, Country('US'))

    def test_eq_with_country(self):
        self.assertEqual(Language('eng', 'US'), Language('eng', Country('US')))

    def test_ne_with_country(self):
        self.assertNotEqual(Language('eng', 'US'), Language('eng', Country('GB')))

    def test_script(self):
        self.assertEqual(Language('srp', script='Latn').script, Script('Latn'))
        self.assertEqual(Language('srp', script=Script('Cyrl')).script, Script('Cyrl'))

    def test_eq_with_script(self):
        self.assertEqual(Language('srp', script='Latn'), Language('srp', script=Script('Latn')))

    def test_ne_with_script(self):
        self.assertNotEqual(Language('srp', script='Latn'), Language('srp', script=Script('Cyrl')))

    def test_eq_with_country_and_script(self):
        self.assertEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Latn')))

    def test_ne_with_country_and_script(self):
        self.assertNotEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Cyrl')))

    def test_hash(self):
        self.assertEqual(hash(Language('fra')), hash('fr'))
        self.assertEqual(hash(Language('ace')), hash('ace'))
        self.assertEqual(hash(Language('por', 'BR')), hash('pt-BR'))
        self.assertEqual(hash(Language('srp', script='Cyrl')), hash('sr-Cyrl'))
        self.assertEqual(hash(Language('eng', 'US', 'Latn')), hash('en-US-Latn'))

    def test_str(self):
        self.assertEqual(Language.fromietf(str(Language('eng', 'US', 'Latn'))), Language('eng', 'US', 'Latn'))
        self.assertEqual(Language.fromietf(str(Language('fra', 'FR'))), Language('fra', 'FR'))
        self.assertEqual(Language.fromietf(str(Language('bel'))), Language('bel'))

    def test_register_converter(self):
        class TestConverter(LanguageReverseConverter):
            def __init__(self):
                self.to_test = {'fra': 'test1', 'eng': 'test2'}
                self.from_test = {'test1': 'fra', 'test2': 'eng'}

            def convert(self, alpha3, country=None, script=None):
                if alpha3 not in self.to_test:
                    raise LanguageConvertError(alpha3, country, script)
                return self.to_test[alpha3]

            def reverse(self, test):
                if test not in self.from_test:
                    raise LanguageReverseError(test)
                return (self.from_test[test], None)
        language = Language('fra')
        self.assertFalse(hasattr(language, 'test'))
        register_language_converter('test', TestConverter)
        self.assertTrue(hasattr(language, 'test'))
        self.assertIn('test', LANGUAGE_CONVERTERS)
        self.assertEqual(Language('fra').test, 'test1')
        self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
        unregister_language_converter('test')
        self.assertNotIn('test', LANGUAGE_CONVERTERS)
        with self.assertRaises(KeyError):
            Language.fromtest('test1')
        with self.assertRaises(AttributeError):
            Language('fra').test
        clear_language_converters()
        load_language_converters()

def suite():
    suite = TestSuite()
    suite.addTest(TestLoader().loadTestsFromTestCase(TestScript))
    suite.addTest(TestLoader().loadTestsFromTestCase(TestCountry))
    suite.addTest(TestLoader().loadTestsFromTestCase(TestLanguage))
    return suite

if __name__ == '__main__':
    TextTestRunner().run(suite())
<commit_msg>Rename converter language_type to type in tests<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2013 the BabelFish authors. All rights reserved.
# Use of this source code is governed by the 3-clause BSD license
# that can be found in the LICENSE file.
#
from __future__ import unicode_literals
from unittest import TestCase, TestSuite, TestLoader, TextTestRunner
from babelfish import (LANGUAGES, Language, Country, Script, get_language_converter, LANGUAGE_CONVERTERS,
    LanguageReverseConverter, load_language_converters, clear_language_converters, register_language_converter,
    unregister_language_converter, LanguageConvertError, LanguageReverseError)
from pkg_resources import resource_stream  # @UnresolvedImport

class TestScript(TestCase):
    def test_wrong_script(self):
        with self.assertRaises(ValueError):
            Script('Azer')

    def test_eq(self):
        self.assertEqual(Script('Latn'), Script('Latn'))

    def test_ne(self):
        self.assertNotEqual(Script('Cyrl'), Script('Latn'))

    def test_hash(self):
        self.assertEqual(hash(Script('Hira')), hash('Hira'))

class TestCountry(TestCase):
    def test_wrong_country(self):
        with self.assertRaises(ValueError):
            Country('ZZ')

    def test_eq(self):
        self.assertEqual(Country('US'), Country('US'))

    def test_ne(self):
        self.assertNotEqual(Country('GB'), Country('US'))
        self.assertNotEqual(Country('US'), None)

    def test_hash(self):
        self.assertEqual(hash(Country('US')), hash('US'))

    def test_name(self):
        self.assertEqual(Country('US').name, 'UNITED STATES')
        self.assertEqual(Country('FR').name, 'FRANCE')

class TestLanguage(TestCase):
    def test_languages(self):
        self.assertEqual(len(LANGUAGES), 7874)

    def test_wrong_language(self):
        with self.assertRaises(ValueError):
            Language('zzz')

    def test_unknown_language(self):
        self.assertEqual(Language('zzzz', unknown='und'), Language('und'))

    def test_converter_alpha2(self):
        self.assertEqual(Language('eng').alpha2, 'en')
        self.assertEqual(Language.fromalpha2('en'), Language('eng'))
        self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
        with self.assertRaises(LanguageReverseError):
            Language.fromalpha2('zz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').alpha2
        self.assertEqual(len(get_language_converter('alpha2').codes), 184)

    def test_converter_alpha3b(self):
        self.assertEqual(Language('fra').alpha3b, 'fre')
        self.assertEqual(Language.fromalpha3b('fre'), Language('fra'))
        self.assertEqual(Language.fromcode('fre', 'alpha3b'), Language('fra'))
        with self.assertRaises(LanguageReverseError):
            Language.fromalpha3b('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').alpha3b
        self.assertEqual(len(get_language_converter('alpha3b').codes), 418)

    def test_converter_alpha3t(self):
        self.assertEqual(Language('fra').alpha3t, 'fra')
        self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
        self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
        with self.assertRaises(LanguageReverseError):
            Language.fromalpha3t('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').alpha3t
        self.assertEqual(len(get_language_converter('alpha3t').codes), 418)

    def test_converter_name(self):
        self.assertEqual(Language('eng').name, 'English')
        self.assertEqual(Language.fromname('English'), Language('eng'))
        self.assertEqual(Language.fromcode('English', 'name'), Language('eng'))
        with self.assertRaises(LanguageReverseError):
            Language.fromname('Zzzzzzzzz')
        self.assertEqual(len(get_language_converter('name').codes), 7874)

    def test_converter_scope(self):
        self.assertEqual(get_language_converter('scope').codes, {'I', 'S', 'M'})
        self.assertEqual(Language('eng').scope, 'individual')
        self.assertEqual(Language('und').scope, 'special')

    def test_converter_type(self):
        self.assertEqual(get_language_converter('type').codes, {'A', 'C', 'E', 'H', 'L', 'S'})
        self.assertEqual(Language('eng').type, 'living')
        self.assertEqual(Language('und').type, 'special')

    def test_converter_opensubtitles(self):
        self.assertEqual(Language('fra').opensubtitles, Language('fra').alpha3b)
        self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
        self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
        self.assertEqual(Language.fromopensubtitles('pob'), Language('por', 'BR'))
        self.assertEqual(Language.fromopensubtitles('pb'), Language('por', 'BR'))
        # Montenegrin is not recognized as an ISO language (yet?) but for now it is
        # unofficially accepted as Serbian from Montenegro
        self.assertEqual(Language.fromopensubtitles('mne'), Language('srp', 'ME'))
        self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
        with self.assertRaises(LanguageReverseError):
            Language.fromopensubtitles('zzz')
        with self.assertRaises(LanguageConvertError):
            Language('aaa').opensubtitles
        self.assertEqual(len(get_language_converter('opensubtitles').codes), 606)

        # test with all the languages from the opensubtitles api
        # downloaded from: http://www.opensubtitles.org/addons/export_languages.php
        f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
        f.readline()
        for l in f:
            idlang, alpha2, _, upload_enabled, web_enabled = l.decode('utf-8').strip().split('\t')
            if not int(upload_enabled) and not int(web_enabled):
                # do not test languages that are too esoteric / not widely available
                continue
            self.assertEqual(Language.fromopensubtitles(idlang).opensubtitles, idlang)
            if alpha2:
                self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
        f.close()

    def test_fromietf_country_script(self):
        language = Language.fromietf('fra-FR-Latn')
        self.assertEqual(language.alpha3, 'fra')
        self.assertEqual(language.country, Country('FR'))
        self.assertEqual(language.script, Script('Latn'))

    def test_fromietf_country_no_script(self):
        language = Language.fromietf('fra-FR')
        self.assertEqual(language.alpha3, 'fra')
        self.assertEqual(language.country, Country('FR'))
        self.assertIs(language.script, None)

    def test_fromietf_no_country_no_script(self):
        language = Language.fromietf('fra-FR')
        self.assertEqual(language.alpha3, 'fra')
        self.assertEqual(language.country, Country('FR'))
        self.assertIs(language.script, None)

    def test_fromietf_no_country_script(self):
        language = Language.fromietf('fra-Latn')
        self.assertEqual(language.alpha3, 'fra')
        self.assertIs(language.country, None)
        self.assertEqual(language.script, Script('Latn'))

    def test_fromietf_alpha2_language(self):
        language = Language.fromietf('fr-Latn')
        self.assertEqual(language.alpha3, 'fra')
        self.assertIs(language.country, None)
        self.assertEqual(language.script, Script('Latn'))

    def test_fromietf_wrong_language(self):
        with self.assertRaises(ValueError):
            Language.fromietf('xyz-FR')

    def test_fromietf_wrong_country(self):
        with self.assertRaises(ValueError):
            Language.fromietf('fra-YZ')

    def test_fromietf_wrong_script(self):
        with self.assertRaises(ValueError):
            Language.fromietf('fra-FR-Wxyz')

    def test_eq(self):
        self.assertEqual(Language('eng'), Language('eng'))

    def test_ne(self):
        self.assertNotEqual(Language('fra'), Language('eng'))
        self.assertIsNot(Language('fra'), None)

    def test_nonzero(self):
        self.assertIs(bool(Language('und')), False)
        self.assertIs(bool(Language('eng')), True)

    def test_country(self):
        self.assertEqual(Language('por', 'BR').country, Country('BR'))
        self.assertEqual(Language('eng', Country('US')).country, Country('US'))

    def test_eq_with_country(self):
        self.assertEqual(Language('eng', 'US'), Language('eng', Country('US')))

    def test_ne_with_country(self):
        self.assertNotEqual(Language('eng', 'US'), Language('eng', Country('GB')))

    def test_script(self):
        self.assertEqual(Language('srp', script='Latn').script, Script('Latn'))
        self.assertEqual(Language('srp', script=Script('Cyrl')).script, Script('Cyrl'))

    def test_eq_with_script(self):
        self.assertEqual(Language('srp', script='Latn'), Language('srp', script=Script('Latn')))

    def test_ne_with_script(self):
        self.assertNotEqual(Language('srp', script='Latn'), Language('srp', script=Script('Cyrl')))

    def test_eq_with_country_and_script(self):
        self.assertEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Latn')))

    def test_ne_with_country_and_script(self):
        self.assertNotEqual(Language('srp', 'SR', 'Latn'), Language('srp', Country('SR'), Script('Cyrl')))

    def test_hash(self):
        self.assertEqual(hash(Language('fra')), hash('fr'))
        self.assertEqual(hash(Language('ace')), hash('ace'))
        self.assertEqual(hash(Language('por', 'BR')), hash('pt-BR'))
        self.assertEqual(hash(Language('srp', script='Cyrl')), hash('sr-Cyrl'))
        self.assertEqual(hash(Language('eng', 'US', 'Latn')), hash('en-US-Latn'))

    def test_str(self):
        self.assertEqual(Language.fromietf(str(Language('eng', 'US', 'Latn'))), Language('eng', 'US', 'Latn'))
        self.assertEqual(Language.fromietf(str(Language('fra', 'FR'))), Language('fra', 'FR'))
        self.assertEqual(Language.fromietf(str(Language('bel'))), Language('bel'))

    def test_register_converter(self):
        class TestConverter(LanguageReverseConverter):
            def __init__(self):
                self.to_test = {'fra': 'test1', 'eng': 'test2'}
                self.from_test = {'test1': 'fra', 'test2': 'eng'}

            def convert(self, alpha3, country=None, script=None):
                if alpha3 not in self.to_test:
                    raise LanguageConvertError(alpha3, country, script)
                return self.to_test[alpha3]

            def reverse(self, test):
                if test not in self.from_test:
                    raise LanguageReverseError(test)
                return (self.from_test[test], None)
        language = Language('fra')
        self.assertFalse(hasattr(language, 'test'))
        register_language_converter('test', TestConverter)
        self.assertTrue(hasattr(language, 'test'))
        self.assertIn('test', LANGUAGE_CONVERTERS)
        self.assertEqual(Language('fra').test, 'test1')
        self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
        unregister_language_converter('test')
        self.assertNotIn('test', LANGUAGE_CONVERTERS)
        with self.assertRaises(KeyError):
            Language.fromtest('test1')
        with self.assertRaises(AttributeError):
            Language('fra').test
        clear_language_converters()
        load_language_converters()

def suite():
    suite = TestSuite()
    suite.addTest(TestLoader().loadTestsFromTestCase(TestScript))
    suite.addTest(TestLoader().loadTestsFromTestCase(TestCountry))
    suite.addTest(TestLoader().loadTestsFromTestCase(TestLanguage))
    return suite

if __name__ == '__main__':
    TextTestRunner().run(suite())
<|endoftext|>


<commit_before>"""Parse (absolute and relative) URLs.

See RFC 1808: "Relative Uniform Resource Locators", by R. Fielding,
UC Irvine, June 1995.
"""

__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
           "urlsplit", "urlunsplit"]

# A classification of schemes ('' means apply by default)
uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap',
                 'wais', 'file', 'https', 'shttp', 'mms',
                 'prospero', 'rtsp', 'rtspu', '', 'sftp']
uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet',
               'imap', 'wais', 'file', 'mms', 'https', 'shttp',
               'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '',
               'svn', 'svn+ssh', 'sftp', 'pbs', 'pbs+ssh', 'sge', 'sge+ssh',
               'fork', 'ssh', 'xt5torque', 'ec2', 'euca', 'pbs+gsissh', 'sge+gsissh',
               'xt5torque+gsissh', 'xt5torque+ssh']
non_hierarchical = ['gopher', 'hdl', 'mailto', 'news',
                    'telnet', 'wais', 'imap', 'snews', 'sip', 'sips']
uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap',
               'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips',
               'mms', '', 'sftp']
uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms',
              'gopher', 'rtsp', 'rtspu', 'sip', 'sips', '']
uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news',
                 'nntp', 'wais', 'https', 'shttp', 'snews',
                 'file', 'prospero', '']

# Characters valid in scheme names
scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
                'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                '0123456789'
                '+-.')

MAX_CACHE_SIZE = 20
_parse_cache = {}

def clear_cache():
    """Clear the parse cache."""
    global _parse_cache
    _parse_cache = {}

class BaseResult(tuple):
    """Base class for the parsed result objects.

    This provides the attributes shared by the two derived result
    objects as read-only properties.  The derived classes are
    responsible for checking the right number of arguments were
    supplied to the constructor.

    """

    __slots__ = ()

    # Attributes that access the basic components of the URL:

    @property
    def scheme(self):
        return self[0]

    @property
    def netloc(self):
        return self[1]

    @property
    def path(self):
        return self[2]

    @property
    def query(self):
        return self[-2]

    @property
    def fragment(self):
        return self[-1]

    # Additional attributes that provide access to parsed-out portions
    # of the netloc:

    @property
    def username(self):
        netloc = self.netloc
        if "@" in netloc:
            userinfo = netloc.split("@", 1)[0]
            if ":" in userinfo:
                userinfo = userinfo.split(":", 1)[0]
            return userinfo
        return None

    @property
    def password(self):
        netloc = self.netloc
        if "@" in netloc:
            userinfo = netloc.split("@", 1)[0]
            if ":" in userinfo:
                return userinfo.split(":", 1)[1]
        return None

    @property
    def hostname(self):
        netloc = self.netloc
        if "@" in netloc:
            netloc = netloc.split("@", 1)[1]
        if ":" in netloc:
            netloc = netloc.split(":", 1)[0]
        return netloc.lower() or None

    @property
    def port(self):
        netloc = self.netloc
        if "@" in netloc:
            netloc = netloc.split("@", 1)[1]
        if ":" in netloc:
            port = netloc.split(":", 1)[1]
            return int(port, 10)
        return None

class SplitResult(BaseResult):

    __slots__ = ()

    def __new__(cls, scheme, netloc, path, query, fragment):
        return BaseResult.__new__(
            cls, (scheme, netloc, path, query, fragment))

    def geturl(self):
        return urlunsplit(self)

class ParseResult(BaseResult):

    __slots__ = ()

    def __new__(cls, scheme, netloc, path, params, query, fragment):
        return BaseResult.__new__(
            cls, (scheme, netloc, path, params, query, fragment))

    @property
    def params(self):
        return self[3]

    def geturl(self):
        return urlunparse(self)

def urlparse(url, scheme='', allow_fragments=True):
    """Parse a URL into 6 components:
    <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
    Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    tuple = urlsplit(url, scheme, allow_fragments)
    scheme, netloc, url, query, fragment = tuple
    if scheme in uses_params and ';' in url:
        url, params = _splitparams(url)
    else:
        params = ''
    return ParseResult(scheme, netloc, url, params, query, fragment)

def _splitparams(url):
    if '/'  in url:
        i = url.find(';', url.rfind('/'))
        if i < 0:
            return url, ''
    else:
        i = url.find(';')
    return url[:i], url[i+1:]

def _splitnetloc(url, start=0):
    delim = len(url)   # position of end of domain part of url, default is end
    for c in '/?#':    # look for delimiters; the order is NOT important
        wdelim = url.find(c, start)        # find first of this delim
        if wdelim >= 0:                    # if found
            delim = min(delim, wdelim)     # use earliest delim position
    return url[start:delim], url[delim:]   # return (domain, rest)

def urlsplit(url, scheme='', allow_fragments=True):
    """Parse a URL into 5 components:
    <scheme>://<netloc>/<path>?<query>#<fragment>
    Return a 5-tuple: (scheme, netloc, path, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    allow_fragments = bool(allow_fragments)
    key = url, scheme, allow_fragments, type(url), type(scheme)
    cached = _parse_cache.get(key, None)
    if cached:
        return cached
    if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
        clear_cache()
    netloc = query = fragment = ''
    i = url.find(':')
    if i > 0:
        if url[:i] == 'http': # optimize the common case
            scheme = url[:i].lower()
            url = url[i+1:]
            if url[:2] == '//':
                netloc, url = _splitnetloc(url, 2)
            if allow_fragments and '#' in url:
                url, fragment = url.split('#', 1)
            if '?' in url:
                url, query = url.split('?', 1)
            v = SplitResult(scheme, netloc, url, query, fragment)
            _parse_cache[key] = v
            return v
        for c in url[:i]:
            if c not in scheme_chars:
                break
        else:
            scheme, url = url[:i].lower(), url[i+1:]
    if scheme in uses_netloc and url[:2] == '//':
        netloc, url = _splitnetloc(url, 2)
    if allow_fragments and scheme in uses_fragment and '#' in url:
        url, fragment = url.split('#', 1)
    if scheme in uses_query and '?' in url:
        url, query = url.split('?', 1)
    v = SplitResult(scheme, netloc, url, query, fragment)
    _parse_cache[key] = v
    return v

def urlunparse((scheme, netloc, url, params, query, fragment)):
    """Put a parsed URL back together again.  This may result in a
    slightly different, but equivalent URL, if the URL that was parsed
    originally had redundant delimiters, e.g. a ? with an empty query
    (the draft states that these are equivalent)."""
    if params:
        url = "%s;%s" % (url, params)
    return urlunsplit((scheme, netloc, url, query, fragment))

def urlunsplit((scheme, netloc, url, query, fragment)):
    if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'):
        if url and url[:1] != '/': url = '/' + url
        url = '//' + (netloc or '') + url
    if scheme:
        url = scheme + ':' + url
    if query:
        url = url + '?' + query
    if fragment:
        url = url + '#' + fragment
    return url

def urljoin(base, url, allow_fragments=True):
    """Join a base URL and a possibly relative URL to form an absolute
    interpretation of the latter."""
    if not base:
        return url
    if not url:
        return base
    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
            urlparse(base, '', allow_fragments)
    scheme, netloc, path, params, query, fragment = \
            urlparse(url, bscheme, allow_fragments)
    if scheme != bscheme or scheme not in uses_relative:
        return url
    if scheme in uses_netloc:
        if netloc:
            return urlunparse((scheme, netloc, path,
                               params, query, fragment))
        netloc = bnetloc
    if path[:1] == '/':
        return urlunparse((scheme, netloc, path,
                           params, query, fragment))
    if not (path or params or query):
        return urlunparse((scheme, netloc, bpath,
                           bparams, bquery, fragment))
    segments = bpath.split('/')[:-1] + path.split('/')
    # XXX The stuff below is bogus in various ways...
    if segments[-1] == '.':
        segments[-1] = ''
    while '.' in segments:
        segments.remove('.')
    while 1:
        i = 1
        n = len(segments) - 1
        while i < n:
            if (segments[i] == '..'
                and segments[i-1] not in ('', '..')):
                del segments[i-1:i+1]
                break
            i = i+1
        else:
            break
    if segments == ['', '..']:
        segments[-1] = ''
    elif len(segments) >= 2 and segments[-1] == '..':
        segments[-2:] = ['']
    return urlunparse((scheme, netloc, '/'.join(segments),
                       params, query, fragment))

def urldefrag(url):
    """Removes any existing fragment from URL.

    Returns a tuple of the defragmented URL and the fragment.  If
    the URL contained no fragments, the second element is the
    empty string.
    """
    if '#' in url:
        s, n, p, a, q, frag = urlparse(url)
        defrag = urlunparse((s, n, p, a, q, ''))
        return defrag, frag
    else:
        return url, ''

test_input = """
      http://a/b/c/d

      g:h        = <URL:g:h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      g          = <URL:http://a/b/c/g>
      ./g        = <URL:http://a/b/c/g>
      g/         = <URL:http://a/b/c/g/>
      /g         = <URL:http://a/g>
      //g        = <URL:http://g>
      ?y         = <URL:http://a/b/c/d?y>
      g?y        = <URL:http://a/b/c/g?y>
      g?y/./x    = <URL:http://a/b/c/g?y/./x>
      .          = <URL:http://a/b/c/>
      ./         = <URL:http://a/b/c/>
      ..         = <URL:http://a/b/>
      ../        = <URL:http://a/b/>
      ../g       = <URL:http://a/b/g>
      ../..      = <URL:http://a/>
      ../../g    = <URL:http://a/g>
      ../../../g = <URL:http://a/../g>
      ./../g     = <URL:http://a/b/g>
      ./g/.      = <URL:http://a/b/c/g/>
      /./g       = <URL:http://a/./g>
      g/./h      = <URL:http://a/b/c/g/h>
      g/../h     = <URL:http://a/b/c/h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      http:?y         = <URL:http://a/b/c/d?y>
      http:g?y        = <URL:http://a/b/c/g?y>
      http:g?y/./x    = <URL:http://a/b/c/g?y/./x>
"""

def test():
    import sys
    base = ''
    if sys.argv[1:]:
        fn = sys.argv[1]
        if fn == '-':
            fp = sys.stdin
        else:
            fp = open(fn)
    else:
        try:
            from cStringIO import StringIO
        except ImportError:
            from StringIO import StringIO
        fp = StringIO(test_input)
    while 1:
        line = fp.readline()
        if not line: break
        words = line.split()
        if not words:
            continue
        url = words[0]
        parts = urlparse(url)
        print '%-10s : %s' % (url, parts)
        abs = urljoin(base, url)
        if not base:
            base = abs
        wrapped = '<URL:%s>' % abs
        print '%-10s = %s' % (url, wrapped)
        if len(words) == 3 and words[1] == '=':
            if wrapped != words[2]:
                print 'EXPECTED', words[2], '!!!!!!!!!!'

if __name__ == '__main__':
    test()

<commit_msg>added redis:// as valid schema<commit_after>"""Parse (absolute and relative) URLs.

See RFC 1808: "Relative Uniform Resource Locators", by R. Fielding,
UC Irvine, June 1995.
"""

__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
           "urlsplit", "urlunsplit"]

# A classification of schemes ('' means apply by default)
uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap',
                 'wais', 'file', 'https', 'shttp', 'mms',
                 'prospero', 'rtsp', 'rtspu', '', 'sftp']
uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet',
               'imap', 'wais', 'file', 'mms', 'https', 'shttp',
               'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '',
               'svn', 'svn+ssh', 'sftp', 'pbs', 'pbs+ssh', 'sge', 'sge+ssh',
               'fork', 'ssh', 'xt5torque', 'ec2', 'euca', 'pbs+gsissh', 'sge+gsissh',
               'xt5torque+gsissh', 'xt5torque+ssh', 'redis']
non_hierarchical = ['gopher', 'hdl', 'mailto', 'news',
                    'telnet', 'wais', 'imap', 'snews', 'sip', 'sips']
uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap',
               'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips',
               'mms', '', 'sftp']
uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms',
              'gopher', 'rtsp', 'rtspu', 'sip', 'sips', '']
uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news',
                 'nntp', 'wais', 'https', 'shttp', 'snews',
                 'file', 'prospero', '']

# Characters valid in scheme names
scheme_chars = ('abcdefghijklmnopqrstuvwxyz'
                'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                '0123456789'
                '+-.')

MAX_CACHE_SIZE = 20
_parse_cache = {}

def clear_cache():
    """Clear the parse cache."""
    global _parse_cache
    _parse_cache = {}

class BaseResult(tuple):
    """Base class for the parsed result objects.

    This provides the attributes shared by the two derived result
    objects as read-only properties.  The derived classes are
    responsible for checking the right number of arguments were
    supplied to the constructor.

    """

    __slots__ = ()

    # Attributes that access the basic components of the URL:

    @property
    def scheme(self):
        return self[0]

    @property
    def netloc(self):
        return self[1]

    @property
    def path(self):
        return self[2]

    @property
    def query(self):
        return self[-2]

    @property
    def fragment(self):
        return self[-1]

    # Additional attributes that provide access to parsed-out portions
    # of the netloc:

    @property
    def username(self):
        netloc = self.netloc
        if "@" in netloc:
            userinfo = netloc.split("@", 1)[0]
            if ":" in userinfo:
                userinfo = userinfo.split(":", 1)[0]
            return userinfo
        return None

    @property
    def password(self):
        netloc = self.netloc
        if "@" in netloc:
            userinfo = netloc.split("@", 1)[0]
            if ":" in userinfo:
                return userinfo.split(":", 1)[1]
        return None

    @property
    def hostname(self):
        netloc = self.netloc
        if "@" in netloc:
            netloc = netloc.split("@", 1)[1]
        if ":" in netloc:
            netloc = netloc.split(":", 1)[0]
        return netloc.lower() or None

    @property
    def port(self):
        netloc = self.netloc
        if "@" in netloc:
            netloc = netloc.split("@", 1)[1]
        if ":" in netloc:
            port = netloc.split(":", 1)[1]
            return int(port, 10)
        return None

class SplitResult(BaseResult):

    __slots__ = ()

    def __new__(cls, scheme, netloc, path, query, fragment):
        return BaseResult.__new__(
            cls, (scheme, netloc, path, query, fragment))

    def geturl(self):
        return urlunsplit(self)

class ParseResult(BaseResult):

    __slots__ = ()

    def __new__(cls, scheme, netloc, path, params, query, fragment):
        return BaseResult.__new__(
            cls, (scheme, netloc, path, params, query, fragment))

    @property
    def params(self):
        return self[3]

    def geturl(self):
        return urlunparse(self)

def urlparse(url, scheme='', allow_fragments=True):
    """Parse a URL into 6 components:
    <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
    Return a 6-tuple: (scheme, netloc, path, params, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    tuple = urlsplit(url, scheme, allow_fragments)
    scheme, netloc, url, query, fragment = tuple
    if scheme in uses_params and ';' in url:
        url, params = _splitparams(url)
    else:
        params = ''
    return ParseResult(scheme, netloc, url, params, query, fragment)

def _splitparams(url):
    if '/'  in url:
        i = url.find(';', url.rfind('/'))
        if i < 0:
            return url, ''
    else:
        i = url.find(';')
    return url[:i], url[i+1:]

def _splitnetloc(url, start=0):
    delim = len(url)   # position of end of domain part of url, default is end
    for c in '/?#':    # look for delimiters; the order is NOT important
        wdelim = url.find(c, start)        # find first of this delim
        if wdelim >= 0:                    # if found
            delim = min(delim, wdelim)     # use earliest delim position
    return url[start:delim], url[delim:]   # return (domain, rest)

def urlsplit(url, scheme='', allow_fragments=True):
    """Parse a URL into 5 components:
    <scheme>://<netloc>/<path>?<query>#<fragment>
    Return a 5-tuple: (scheme, netloc, path, query, fragment).
    Note that we don't break the components up in smaller bits
    (e.g. netloc is a single string) and we don't expand % escapes."""
    allow_fragments = bool(allow_fragments)
    key = url, scheme, allow_fragments, type(url), type(scheme)
    cached = _parse_cache.get(key, None)
    if cached:
        return cached
    if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth
        clear_cache()
    netloc = query = fragment = ''
    i = url.find(':')
    if i > 0:
        if url[:i] == 'http': # optimize the common case
            scheme = url[:i].lower()
            url = url[i+1:]
            if url[:2] == '//':
                netloc, url = _splitnetloc(url, 2)
            if allow_fragments and '#' in url:
                url, fragment = url.split('#', 1)
            if '?' in url:
                url, query = url.split('?', 1)
            v = SplitResult(scheme, netloc, url, query, fragment)
            _parse_cache[key] = v
            return v
        for c in url[:i]:
            if c not in scheme_chars:
                break
        else:
            scheme, url = url[:i].lower(), url[i+1:]
    if scheme in uses_netloc and url[:2] == '//':
        netloc, url = _splitnetloc(url, 2)
    if allow_fragments and scheme in uses_fragment and '#' in url:
        url, fragment = url.split('#', 1)
    if scheme in uses_query and '?' in url:
        url, query = url.split('?', 1)
    v = SplitResult(scheme, netloc, url, query, fragment)
    _parse_cache[key] = v
    return v

def urlunparse((scheme, netloc, url, params, query, fragment)):
    """Put a parsed URL back together again.  This may result in a
    slightly different, but equivalent URL, if the URL that was parsed
    originally had redundant delimiters, e.g. a ? with an empty query
    (the draft states that these are equivalent)."""
    if params:
        url = "%s;%s" % (url, params)
    return urlunsplit((scheme, netloc, url, query, fragment))

def urlunsplit((scheme, netloc, url, query, fragment)):
    if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'):
        if url and url[:1] != '/': url = '/' + url
        url = '//' + (netloc or '') + url
    if scheme:
        url = scheme + ':' + url
    if query:
        url = url + '?' + query
    if fragment:
        url = url + '#' + fragment
    return url

def urljoin(base, url, allow_fragments=True):
    """Join a base URL and a possibly relative URL to form an absolute
    interpretation of the latter."""
    if not base:
        return url
    if not url:
        return base
    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
            urlparse(base, '', allow_fragments)
    scheme, netloc, path, params, query, fragment = \
            urlparse(url, bscheme, allow_fragments)
    if scheme != bscheme or scheme not in uses_relative:
        return url
    if scheme in uses_netloc:
        if netloc:
            return urlunparse((scheme, netloc, path,
                               params, query, fragment))
        netloc = bnetloc
    if path[:1] == '/':
        return urlunparse((scheme, netloc, path,
                           params, query, fragment))
    if not (path or params or query):
        return urlunparse((scheme, netloc, bpath,
                           bparams, bquery, fragment))
    segments = bpath.split('/')[:-1] + path.split('/')
    # XXX The stuff below is bogus in various ways...
    if segments[-1] == '.':
        segments[-1] = ''
    while '.' in segments:
        segments.remove('.')
    while 1:
        i = 1
        n = len(segments) - 1
        while i < n:
            if (segments[i] == '..'
                and segments[i-1] not in ('', '..')):
                del segments[i-1:i+1]
                break
            i = i+1
        else:
            break
    if segments == ['', '..']:
        segments[-1] = ''
    elif len(segments) >= 2 and segments[-1] == '..':
        segments[-2:] = ['']
    return urlunparse((scheme, netloc, '/'.join(segments),
                       params, query, fragment))

def urldefrag(url):
    """Removes any existing fragment from URL.

    Returns a tuple of the defragmented URL and the fragment.  If
    the URL contained no fragments, the second element is the
    empty string.
    """
    if '#' in url:
        s, n, p, a, q, frag = urlparse(url)
        defrag = urlunparse((s, n, p, a, q, ''))
        return defrag, frag
    else:
        return url, ''

test_input = """
      http://a/b/c/d

      g:h        = <URL:g:h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      g          = <URL:http://a/b/c/g>
      ./g        = <URL:http://a/b/c/g>
      g/         = <URL:http://a/b/c/g/>
      /g         = <URL:http://a/g>
      //g        = <URL:http://g>
      ?y         = <URL:http://a/b/c/d?y>
      g?y        = <URL:http://a/b/c/g?y>
      g?y/./x    = <URL:http://a/b/c/g?y/./x>
      .          = <URL:http://a/b/c/>
      ./         = <URL:http://a/b/c/>
      ..         = <URL:http://a/b/>
      ../        = <URL:http://a/b/>
      ../g       = <URL:http://a/b/g>
      ../..      = <URL:http://a/>
      ../../g    = <URL:http://a/g>
      ../../../g = <URL:http://a/../g>
      ./../g     = <URL:http://a/b/g>
      ./g/.      = <URL:http://a/b/c/g/>
      /./g       = <URL:http://a/./g>
      g/./h      = <URL:http://a/b/c/g/h>
      g/../h     = <URL:http://a/b/c/h>
      http:g     = <URL:http://a/b/c/g>
      http:      = <URL:http://a/b/c/d>
      http:?y         = <URL:http://a/b/c/d?y>
      http:g?y        = <URL:http://a/b/c/g?y>
      http:g?y/./x    = <URL:http://a/b/c/g?y/./x>
"""

def test():
    import sys
    base = ''
    if sys.argv[1:]:
        fn = sys.argv[1]
        if fn == '-':
            fp = sys.stdin
        else:
            fp = open(fn)
    else:
        try:
            from cStringIO import StringIO
        except ImportError:
            from StringIO import StringIO
        fp = StringIO(test_input)
    while 1:
        line = fp.readline()
        if not line: break
        words = line.split()
        if not words:
            continue
        url = words[0]
        parts = urlparse(url)
        print '%-10s : %s' % (url, parts)
        abs = urljoin(base, url)
        if not base:
            base = abs
        wrapped = '<URL:%s>' % abs
        print '%-10s = %s' % (url, wrapped)
        if len(words) == 3 and words[1] == '=':
            if wrapped != words[2]:
                print 'EXPECTED', words[2], '!!!!!!!!!!'

if __name__ == '__main__':
    test()

<|endoftext|>


<commit_before><commit_msg>Add configuration for bokeh server<commit_after>from blocks.config import config

config.add_config('bokeh_server', type_=str, default='http://localhost:5006/')
config.load_yaml()
<|endoftext|>