<commit_before># -*- coding: utf-8 -*-

from __future__ import with_statement

import hashlib
import os
import re
import zlib

from module.plugins.internal.Addon import Addon
from module.utils import save_join as fs_join, fs_encode

def compute_checksum(local_file, algorithm):
    if algorithm in getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")):
        h = getattr(hashlib, algorithm)()

        with open(local_file, 'rb') as f:
            for chunk in iter(lambda: f.read(128 * h.block_size), ''):
                h.update(chunk)

        return h.hexdigest()

    elif algorithm in ("adler32", "crc32"):
        hf = getattr(zlib, algorithm)
        last = 0

        with open(local_file, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), ''):
                last = hf(chunk, last)

        return "%x" % last

    else:
        return None

class Checksum(Addon):
    __name__    = "Checksum"
    __type__    = "hook"
    __version__ = "0.20"
    __status__  = "testing"

    __config__ = [("check_checksum", "bool"             , "Check checksum? (If False only size will be verified)", True   ),
                  ("check_action" , "fail;retry;nothing", "What to do if check fails?"                           , "retry"),
                  ("max_tries"    , "int"               , "Number of retries"                                    , 2      ),
                  ("retry_action" , "fail;nothing"      , "What to do if all retries fail?"                      , "fail" ),
                  ("wait_time"    , "int"               , "Time to wait before each retry (seconds)"             , 1      )]

    __description__ = """Verify downloaded file size and checksum"""
    __license__     = "GPLv3"
    __authors__     = [("zoidberg"      , "zoidberg@mujmail.cz"),
                       ("Walter Purcaro", "vuolter@gmail.com"  ),
                       ("stickell"      , "l.stickell@yahoo.it")]

    methods  = {'sfv' : 'crc32',
                'crc' : 'crc32',
                'hash': 'md5'}
    regexps  = {'sfv'    : r'^(?P<NAME>[^;].+)\s+(?P<HASH>[0-9A-Fa-f]{8})$',
                'md5'    : r'^(?P<NAME>[0-9A-Fa-f]{32})  (?P<FILE>.+)$',
                'crc'    : r'filename=(?P<NAME>.+)\nsize=(?P<SIZE>\d+)\ncrc32=(?P<HASH>[0-9A-Fa-f]{8})$',
                'default': r'^(?P<HASH>[0-9A-Fa-f]+)\s+\*?(?P<NAME>.+)$'}

    def activate(self):
        if not self.get_config('check_checksum'):
            self.log_info(_("Checksum validation is disabled in plugin configuration"))

    def init(self):
        self.algorithms = sorted(
            getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")), reverse=True)

        self.algorithms.extend(["crc32", "adler32"])

        self.formats = self.algorithms + ["sfv", "crc", "hash"]

    def download_finished(self, pyfile):
        """
        Compute checksum for the downloaded file and compare it with the hash provided by the hoster.
        pyfile.plugin.check_data should be a dictionary which can contain:
        a) if known, the exact filesize in bytes (e.g. 'size': 123456789)
        b) hexadecimal hash string with algorithm name as key (e.g. 'md5': "d76505d0869f9f928a17d42d66326307")
        """
        if hasattr(pyfile.plugin, "check_data") and isinstance(pyfile.plugin.check_data, dict):
            data = pyfile.plugin.check_data.copy()

        elif hasattr(pyfile.plugin, "api_data") and isinstance(pyfile.plugin.api_data, dict):
            data = pyfile.plugin.api_data.copy()

        elif hasattr(pyfile.plugin, "info") and isinstance(pyfile.plugin.info, dict):
            data = pyfile.plugin.info.copy()
            data.pop('size', None)  #@NOTE: Don't check file size until a similary matcher will be implemented

        else:
            return

        self.log_debug(data)

        if not pyfile.plugin.last_download:
            self.check_failed(pyfile, None, "No file downloaded")

        local_file = fs_encode(pyfile.plugin.last_download)
        # download_folder = self.pyload.config.get("general", "download_folder")
        # local_file = fs_encode(fs_join(download_folder, pyfile.package().folder, pyfile.name))

        if not os.path.isfile(local_file):
            self.check_failed(pyfile, None, "File does not exist")

        #: Validate file size
        if "size" in data:
            api_size  = int(data['size'])
            file_size = os.path.getsize(local_file)

            if api_size is not file_size:
                self.log_warning(_("File %s has incorrect size: %d B (%d expected)") % (pyfile.name, file_size, api_size))
                self.check_failed(pyfile, local_file, "Incorrect file size")

            data.pop('size', None)

        #: Validate checksum
        if data and self.get_config('check_checksum'):

            if not 'md5' in data:
                for type in ("checksum", "hashsum", "hash"):
                    if type in data:
                        data['md5'] = data[type]  #@NOTE: What happens if it's not an md5 hash?
                        break

            for key in self.algorithms:
                if key in data:
                    checksum = compute_checksum(local_file, key.replace("-", "").lower())
                    if checksum:
                        if checksum == data[key].lower():
                            self.log_info(_('File integrity of "%s" verified by %s checksum (%s)') %
                                        (pyfile.name, key.upper(), checksum))
                            break
                        else:
                            self.log_warning(_("%s checksum for file %s does not match (%s != %s)") %
                                           (key.upper(), pyfile.name, checksum, data[key]))
                            self.check_failed(pyfile, local_file, "Checksums do not match")
                    else:
                        self.log_warning(_("Unsupported hashing algorithm"), key.upper())
            else:
                self.log_warning(_("Unable to validate checksum for file: ") + pyfile.name)

    def check_failed(self, pyfile, local_file, msg):
        check_action = self.get_config('check_action')
        if check_action == "retry":
            max_tries = self.get_config('max_tries')
            retry_action = self.get_config('retry_action')
            if pyfile.plugin.retries < max_tries:
                if local_file:
                    os.remove(local_file)
                pyfile.plugin.retry(max_tries, self.get_config('wait_time'), msg)
            elif retry_action == "nothing":
                return
        elif check_action == "nothing":
            return
        pyfile.plugin.fail(reason=msg)

    def package_finished(self, pypack):
        download_folder = fs_join(self.pyload.config.get("general", "download_folder"), pypack.folder, "")

        for link in pypack.getChildren().values():
            file_type = os.path.splitext(link['name'])[1][1:].lower()

            if file_type not in self.formats:
                continue

            hash_file = fs_encode(fs_join(download_folder, link['name']))
            if not os.path.isfile(hash_file):
                self.log_warning(_("File not found"), link['name'])
                continue

            with open(hash_file) as f:
                text = f.read()

            for m in re.finditer(self.regexps.get(file_type, self.regexps['default']), text):
                data = m.groupdict()
                self.log_debug(link['name'], data)

                local_file = fs_encode(fs_join(download_folder, data['NAME']))
                algorithm = self.methods.get(file_type, file_type)
                checksum = compute_checksum(local_file, algorithm)
                if checksum is data['HASH']:
                    self.log_info(_('File integrity of "%s" verified by %s checksum (%s)') %
                                (data['NAME'], algorithm, checksum))
                else:
                    self.log_warning(_("%s checksum for file %s does not match (%s != %s)") %
                                   (algorithm, data['NAME'], checksum, data['HASH']))
<commit_msg>Minor change to log output<commit_after># -*- coding: utf-8 -*-

from __future__ import with_statement

import hashlib
import os
import re
import zlib

from module.plugins.internal.Addon import Addon
from module.utils import save_join as fs_join, fs_encode

def compute_checksum(local_file, algorithm):
    if algorithm in getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")):
        h = getattr(hashlib, algorithm)()

        with open(local_file, 'rb') as f:
            for chunk in iter(lambda: f.read(128 * h.block_size), ''):
                h.update(chunk)

        return h.hexdigest()

    elif algorithm in ("adler32", "crc32"):
        hf = getattr(zlib, algorithm)
        last = 0

        with open(local_file, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), ''):
                last = hf(chunk, last)

        return "%x" % last

    else:
        return None

class Checksum(Addon):
    __name__    = "Checksum"
    __type__    = "hook"
    __version__ = "0.20"
    __status__  = "testing"

    __config__ = [("check_checksum", "bool"             , "Check checksum? (If False only size will be verified)", True   ),
                  ("check_action" , "fail;retry;nothing", "What to do if check fails?"                           , "retry"),
                  ("max_tries"    , "int"               , "Number of retries"                                    , 2      ),
                  ("retry_action" , "fail;nothing"      , "What to do if all retries fail?"                      , "fail" ),
                  ("wait_time"    , "int"               , "Time to wait before each retry (seconds)"             , 1      )]

    __description__ = """Verify downloaded file size and checksum"""
    __license__     = "GPLv3"
    __authors__     = [("zoidberg"      , "zoidberg@mujmail.cz"),
                       ("Walter Purcaro", "vuolter@gmail.com"  ),
                       ("stickell"      , "l.stickell@yahoo.it")]

    methods  = {'sfv' : 'crc32',
                'crc' : 'crc32',
                'hash': 'md5'}
    regexps  = {'sfv'    : r'^(?P<NAME>[^;].+)\s+(?P<HASH>[0-9A-Fa-f]{8})$',
                'md5'    : r'^(?P<NAME>[0-9A-Fa-f]{32})  (?P<FILE>.+)$',
                'crc'    : r'filename=(?P<NAME>.+)\nsize=(?P<SIZE>\d+)\ncrc32=(?P<HASH>[0-9A-Fa-f]{8})$',
                'default': r'^(?P<HASH>[0-9A-Fa-f]+)\s+\*?(?P<NAME>.+)$'}

    def activate(self):
        if not self.get_config('check_checksum'):
            self.log_info(_("Checksum validation is disabled in plugin configuration"))

    def init(self):
        self.algorithms = sorted(
            getattr(hashlib, "algorithms", ("md5", "sha1", "sha224", "sha256", "sha384", "sha512")), reverse=True)

        self.algorithms.extend(["crc32", "adler32"])

        self.formats = self.algorithms + ["sfv", "crc", "hash"]

    def download_finished(self, pyfile):
        """
        Compute checksum for the downloaded file and compare it with the hash provided by the hoster.
        pyfile.plugin.check_data should be a dictionary which can contain:
        a) if known, the exact filesize in bytes (e.g. 'size': 123456789)
        b) hexadecimal hash string with algorithm name as key (e.g. 'md5': "d76505d0869f9f928a17d42d66326307")
        """
        if hasattr(pyfile.plugin, "check_data") and isinstance(pyfile.plugin.check_data, dict):
            data = pyfile.plugin.check_data.copy()

        elif hasattr(pyfile.plugin, "api_data") and isinstance(pyfile.plugin.api_data, dict):
            data = pyfile.plugin.api_data.copy()

        elif hasattr(pyfile.plugin, "info") and isinstance(pyfile.plugin.info, dict):
            data = pyfile.plugin.info.copy()
            data.pop('size', None)  #@NOTE: Don't check file size until a similary matcher will be implemented

        else:
            return

        self.log_debug(data)

        if not pyfile.plugin.last_download:
            self.check_failed(pyfile, None, "No file downloaded")

        local_file = fs_encode(pyfile.plugin.last_download)
        # download_folder = self.pyload.config.get("general", "download_folder")
        # local_file = fs_encode(fs_join(download_folder, pyfile.package().folder, pyfile.name))

        if not os.path.isfile(local_file):
            self.check_failed(pyfile, None, "File does not exist")

        #: Validate file size
        if "size" in data:
            api_size  = int(data['size'])
            file_size = os.path.getsize(local_file)

            if api_size is not file_size:
                self.log_warning(_("File %s has incorrect size: %d B (%d expected)") % (pyfile.name, file_size, api_size))
                self.check_failed(pyfile, local_file, "Incorrect file size")

            data.pop('size', None)

        #: Validate checksum
        if data and self.get_config('check_checksum'):

            if not 'md5' in data:
                for type in ("checksum", "hashsum", "hash"):
                    if type in data:
                        data['md5'] = data[type]  #@NOTE: What happens if it's not an md5 hash?
                        break

            for key in self.algorithms:
                if key in data:
                    checksum = compute_checksum(local_file, key.replace("-", "").lower())
                    if checksum:
                        if checksum == data[key].lower():
                            self.log_info(_('File integrity of "%s" verified by %s checksum (%s)') %
                                        (pyfile.name, key.upper(), checksum))
                            break
                        else:
                            self.log_warning(_("%s checksum for file %s does not match (%s != %s)") %
                                           (key.upper(), pyfile.name, checksum, data[key].lower()))
                            self.check_failed(pyfile, local_file, "Checksums do not match")
                    else:
                        self.log_warning(_("Unsupported hashing algorithm"), key.upper())
            else:
                self.log_warning(_("Unable to validate checksum for file: ") + pyfile.name)

    def check_failed(self, pyfile, local_file, msg):
        check_action = self.get_config('check_action')
        if check_action == "retry":
            max_tries = self.get_config('max_tries')
            retry_action = self.get_config('retry_action')
            if pyfile.plugin.retries < max_tries:
                if local_file:
                    os.remove(local_file)
                pyfile.plugin.retry(max_tries, self.get_config('wait_time'), msg)
            elif retry_action == "nothing":
                return
        elif check_action == "nothing":
            return
        pyfile.plugin.fail(reason=msg)

    def package_finished(self, pypack):
        download_folder = fs_join(self.pyload.config.get("general", "download_folder"), pypack.folder, "")

        for link in pypack.getChildren().values():
            file_type = os.path.splitext(link['name'])[1][1:].lower()

            if file_type not in self.formats:
                continue

            hash_file = fs_encode(fs_join(download_folder, link['name']))
            if not os.path.isfile(hash_file):
                self.log_warning(_("File not found"), link['name'])
                continue

            with open(hash_file) as f:
                text = f.read()

            for m in re.finditer(self.regexps.get(file_type, self.regexps['default']), text):
                data = m.groupdict()
                self.log_debug(link['name'], data)

                local_file = fs_encode(fs_join(download_folder, data['NAME']))
                algorithm = self.methods.get(file_type, file_type)
                checksum = compute_checksum(local_file, algorithm)
                if checksum is data['HASH']:
                    self.log_info(_('File integrity of "%s" verified by %s checksum (%s)') %
                                (data['NAME'], algorithm, checksum))
                else:
                    self.log_warning(_("%s checksum for file %s does not match (%s != %s)") %
                                   (algorithm, data['NAME'], checksum, data['HASH']))
<|endoftext|>


<commit_before>"""\
Example.

To load the initial data:

    %(prog)s production.ini

"""
from pyramid.paster import get_app
from pyramid.traversal import find_root
from pyelasticsearch import IndexAlreadyExistsError
from ..indexing import ELASTIC_SEARCH
import collections
import json
import logging

EPILOG = __doc__

log = logging.getLogger(__name__)

# An index to store non-content metadata
META_MAPPING = {
    'dynamic_templates': [
        {
            'store_generic': {
                'match': '*',
                'mapping': {
                    'index': 'no',
                    'store': 'yes',
                },
            },
        },
    ],
}

def sorted_pairs_hook(pairs):
    return collections.OrderedDict(sorted(pairs))

def sorted_dict(d):
    return json.loads(json.dumps(d), object_pairs_hook=sorted_pairs_hook)

def schema_mapping(name, schema):
    type_ = schema['type']

    # Elasticsearch handles multiple values for a field
    if type_ == 'array':
        return schema_mapping(name, schema['items'])

    if type_ == 'object':
        properties = {
            k: schema_mapping(k, v) for k, v in schema['properties'].items()
        }
        return {'properties': properties}

    if type_ == ["number", "string"]:
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
            'fields': {
                'value': {
                    'type': 'float',
                    'copy_to': '',
                    'ignore_malformed': True,
                    'include_in_all': False,
                    'copy_to': []
                }
            }
        }

    if type_ == 'string':
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
        }

    if type_ == 'number':
        return {
            'type': 'float',
            'copy_to': [],
            'include_in_all': False
        }

    if type_ in ('boolean', 'integer'):
        return {
            'type': type_,
            'copy_to': [],
            'include_in_all': False
        }

def index_settings():
    return {
        'index': {
            'analysis': {
                'filter': {
                    'substring': {
                        'type': 'nGram',
                        'min_gram': 1,
                        'max_gram': 33
                    }
                },
                'analyzer': {
                    'encoded_index_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding',
                            'substring'
                        ]
                    },
                    'encoded_search_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding'
                        ]
                    }
                }
            }
        }
    }

def es_mapping(mapping):
    return {
        '_all': {
            'analyzer': 'encoded_index_analyzer'
        },
        'properties': {
            'embedded': mapping,
            'encoded_all_ngram': {
                'type': 'string',
                'include_in_all': False,
                'boost': 1,
                'search_analyzer': 'encoded_search_analyzer',
                'index_analyzer': 'encoded_index_analyzer'
            },
            'encoded_all_standard': {
                'type': 'string',
                'include_in_all': False,
                'boost': 2
            },
            'encoded_all_untouched': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed',
                'boost': 3
            },
            'object': {
                'type': 'object',
                'include_in_all': False,
                'properties': {}
            },
            'principals_allowed_view': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'embedded_uuid_closure': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'link_uuid_closure': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'keys': {
                'type': 'object',
                'include_in_all': False,
                'properties': {}
            },
            'links': {
                'type': 'object',
                'include_in_all': False,
                'properties': {}
            },
            'url': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'audit': {
                'type': 'object',
                'include_in_all': False,
                'properties': {
                    'category': {
                        'type': 'string',
                        'index': 'not_analyzed',
                    },
                    'detail': {
                        'type': 'string',
                        'index': 'not_analyzed',
                    },
                    'level_name': {
                        'type': 'string',
                        'index': 'not_analyzed',
                    },
                    'level': {
                        'type': 'integer',
                    },
                },
            }
        }
    }

def collection_mapping(collection, embed=True):
    schema = collection.schema
    if schema is None:
        return None

    mapping = schema_mapping(collection.item_type, schema)

    merged_rev = collection.Item.merged_rev
    merged_template_type = collection.Item.merged_template_type

    calculated_props = list(schema.get('calculated_props', ()))
    calculated_props.extend(['@id', '@type'])
    calculated_props.extend(merged_rev.keys())

    for name in calculated_props:
        mapping['properties'][name] = schema_mapping(name, {'type': 'string'})

    if not embed:
        return mapping

    root = find_root(collection)

    for prop in collection.Item.embedded:
        new_mapping = mapping
        new_schema = schema

        for i, p in enumerate(prop.split('.')):
            if i == 0 and p in merged_rev:
                name = merged_rev[p][0]
            elif i == 0 and p in merged_template_type:
                name = merged_template_type[p]
            else:
                try:
                    name = new_schema['properties'][p]['linkTo']
                except KeyError:
                    name = new_schema['properties'][p]['items']['linkTo']

            # XXX Need to union with mouse_donor here
            if name == 'donor':
                name = 'human_donor'

            # XXX file has "linkTo": ["experiment", "dataset"]

            # Check if mapping for property is already an object
            # multiple subobjects may be embedded, so be carful here
            try:
                new_mapping['properties'][p]['properties']
            except KeyError:
                new_mapping['properties'][p] = collection_mapping(
                    root.by_item_type[name], embed=False)

            new_mapping = new_mapping['properties'][p]
            new_schema = root[name].schema

    boost_values = schema.get('boost_values', ())
    for value in boost_values:
        props = value.split('.')
        new_mapping = mapping['properties']
        for prop in props:
            if len(props) == props.index(prop) + 1:
                new_mapping[prop]['boost'] = boost_values[value]
                new_mapping[prop]['copy_to'] = ['encoded_all_ngram', 'encoded_all_standard', 'encoded_all_untouched']
                new_mapping = mapping['properties']
            else:
                new_mapping = new_mapping[prop]['properties']
    return mapping

def run(app, collections=None, dry_run=False):
    index = 'encoded'
    root = app.root_factory(app)

    if not dry_run:
        es = app.registry[ELASTIC_SEARCH]
        try:
            es.create_index(index, index_settings())
        except IndexAlreadyExistsError:
            if collections is None:
                es.delete_index(index)
                es.create_index(index, index_settings())

    if not collections:
        collections = ['meta'] + root.by_item_type.keys()

    for collection_name in collections:
        if collection_name == 'meta':
            doc_type = 'meta'
            mapping = META_MAPPING
        else:
            doc_type = collection_name
            collection = root.by_item_type[collection_name]
            mapping = collection_mapping(collection)

        if mapping is None:
            continue  # Testing collections
        if dry_run:
            print json.dumps(
                sorted_dict({index: {doc_type: mapping}}), indent=4)
            continue

        if collection_name is not 'meta':
            mapping = es_mapping(mapping)

        try:
            es.put_mapping(index, doc_type, {doc_type: mapping})
        except:
            log.info("Could not create mapping for the collection %s", doc_type)
        else:
            es.refresh(index)

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Create Elasticsearch mapping", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--item-type', action='append', help="Item type")
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument(
        '--dry-run', action='store_true', help="Don't post to ES, just print")
    parser.add_argument('config_uri', help="path to configfile")
    args = parser.parse_args()

    logging.basicConfig()
    app = get_app(args.config_uri, args.app_name)

    # Loading app will have configured from config file. Reconfigure here:
    logging.getLogger('encoded').setLevel(logging.DEBUG)

    return run(app, args.item_type, args.dry_run)

if __name__ == '__main__':
    main()
<commit_msg>Fix for inner object merged properties<commit_after>"""\
Example.

To load the initial data:

    %(prog)s production.ini

"""
from pyramid.paster import get_app
from pyramid.traversal import find_root
from pyelasticsearch import IndexAlreadyExistsError
from ..indexing import ELASTIC_SEARCH
import collections
import json
import logging

EPILOG = __doc__

log = logging.getLogger(__name__)

# An index to store non-content metadata
META_MAPPING = {
    'dynamic_templates': [
        {
            'store_generic': {
                'match': '*',
                'mapping': {
                    'index': 'no',
                    'store': 'yes',
                },
            },
        },
    ],
}

def sorted_pairs_hook(pairs):
    return collections.OrderedDict(sorted(pairs))

def sorted_dict(d):
    return json.loads(json.dumps(d), object_pairs_hook=sorted_pairs_hook)

def schema_mapping(name, schema):
    type_ = schema['type']

    # Elasticsearch handles multiple values for a field
    if type_ == 'array':
        return schema_mapping(name, schema['items'])

    if type_ == 'object':
        properties = {
            k: schema_mapping(k, v) for k, v in schema['properties'].items()
        }
        return {'properties': properties}

    if type_ == ["number", "string"]:
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
            'fields': {
                'value': {
                    'type': 'float',
                    'copy_to': '',
                    'ignore_malformed': True,
                    'include_in_all': False,
                    'copy_to': []
                }
            }
        }

    if type_ == 'string':
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
        }

    if type_ == 'number':
        return {
            'type': 'float',
            'copy_to': [],
            'include_in_all': False
        }

    if type_ in ('boolean', 'integer'):
        return {
            'type': type_,
            'copy_to': [],
            'include_in_all': False
        }

def index_settings():
    return {
        'index': {
            'analysis': {
                'filter': {
                    'substring': {
                        'type': 'nGram',
                        'min_gram': 1,
                        'max_gram': 33
                    }
                },
                'analyzer': {
                    'encoded_index_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding',
                            'substring'
                        ]
                    },
                    'encoded_search_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding'
                        ]
                    }
                }
            }
        }
    }

def es_mapping(mapping):
    return {
        '_all': {
            'analyzer': 'encoded_index_analyzer'
        },
        'properties': {
            'embedded': mapping,
            'encoded_all_ngram': {
                'type': 'string',
                'include_in_all': False,
                'boost': 1,
                'search_analyzer': 'encoded_search_analyzer',
                'index_analyzer': 'encoded_index_analyzer'
            },
            'encoded_all_standard': {
                'type': 'string',
                'include_in_all': False,
                'boost': 2
            },
            'encoded_all_untouched': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed',
                'boost': 3
            },
            'object': {
                'type': 'object',
                'include_in_all': False,
                'properties': {}
            },
            'principals_allowed_view': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'embedded_uuid_closure': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'link_uuid_closure': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'keys': {
                'type': 'object',
                'include_in_all': False,
                'properties': {}
            },
            'links': {
                'type': 'object',
                'include_in_all': False,
                'properties': {}
            },
            'url': {
                'type': 'string',
                'include_in_all': False,
                'index': 'not_analyzed'
            },
            'audit': {
                'type': 'object',
                'include_in_all': False,
                'properties': {
                    'category': {
                        'type': 'string',
                        'index': 'not_analyzed',
                    },
                    'detail': {
                        'type': 'string',
                        'index': 'not_analyzed',
                    },
                    'level_name': {
                        'type': 'string',
                        'index': 'not_analyzed',
                    },
                    'level': {
                        'type': 'integer',
                    },
                },
            }
        }
    }

def collection_mapping(collection, embed=True):
    schema = collection.schema
    if schema is None:
        return None

    mapping = schema_mapping(collection.item_type, schema)

    merged_rev = collection.Item.merged_rev
    merged_template_type = collection.Item.merged_template_type

    calculated_props = list(schema.get('calculated_props', ()))
    calculated_props.extend(['@id', '@type'])
    calculated_props.extend(merged_rev.keys())

    for name in calculated_props:
        mapping['properties'][name] = schema_mapping(name, {'type': 'string'})

    if not embed:
        return mapping

    root = find_root(collection)

    for prop in collection.Item.embedded:
        new_mapping = mapping
        new_schema = schema

        for i, p in enumerate(prop.split('.')):
            if i == 0 and p in merged_rev:
                name = merged_rev[p][0]
            elif i == 0 and p in merged_template_type:
                name = merged_template_type[p]
            else:
                if p not in new_schema['properties']:
                    if p in root[name].Item.merged_rev:
                        name = root[name].Item.merged_rev[p][0]
                    else:
                        name = root[name].Item.merged_template_type[p]
                else:
                    try:
                        name = new_schema['properties'][p]['linkTo']
                    except KeyError:
                        name = new_schema['properties'][p]['items']['linkTo']

            # XXX Need to union with mouse_donor here
            if name == 'donor':
                name = 'human_donor'

            # XXX file has "linkTo": ["experiment", "dataset"]

            # Check if mapping for property is already an object
            # multiple subobjects may be embedded, so be carful here
            try:
                new_mapping['properties'][p]['properties']
            except KeyError:
                new_mapping['properties'][p] = collection_mapping(
                    root.by_item_type[name], embed=False)

            new_mapping = new_mapping['properties'][p]
            new_schema = root[name].schema

    boost_values = schema.get('boost_values', ())
    for value in boost_values:
        props = value.split('.')
        new_mapping = mapping['properties']
        for prop in props:
            if len(props) == props.index(prop) + 1:
                new_mapping[prop]['boost'] = boost_values[value]
                new_mapping[prop]['copy_to'] = ['encoded_all_ngram', 'encoded_all_standard', 'encoded_all_untouched']
                new_mapping = mapping['properties']
            else:
                new_mapping = new_mapping[prop]['properties']
    return mapping

def run(app, collections=None, dry_run=False):
    index = 'encoded'
    root = app.root_factory(app)

    if not dry_run:
        es = app.registry[ELASTIC_SEARCH]
        try:
            es.create_index(index, index_settings())
        except IndexAlreadyExistsError:
            if collections is None:
                es.delete_index(index)
                es.create_index(index, index_settings())

    if not collections:
        collections = ['meta'] + root.by_item_type.keys()

    for collection_name in collections:
        if collection_name == 'meta':
            doc_type = 'meta'
            mapping = META_MAPPING
        else:
            doc_type = collection_name
            collection = root.by_item_type[collection_name]
            mapping = collection_mapping(collection)

        if mapping is None:
            continue  # Testing collections
        if dry_run:
            print json.dumps(
                sorted_dict({index: {doc_type: mapping}}), indent=4)
            continue

        if collection_name is not 'meta':
            mapping = es_mapping(mapping)

        try:
            es.put_mapping(index, doc_type, {doc_type: mapping})
        except:
            log.info("Could not create mapping for the collection %s", doc_type)
        else:
            es.refresh(index)

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Create Elasticsearch mapping", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--item-type', action='append', help="Item type")
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument(
        '--dry-run', action='store_true', help="Don't post to ES, just print")
    parser.add_argument('config_uri', help="path to configfile")
    args = parser.parse_args()

    logging.basicConfig()
    app = get_app(args.config_uri, args.app_name)

    # Loading app will have configured from config file. Reconfigure here:
    logging.getLogger('encoded').setLevel(logging.DEBUG)

    return run(app, args.item_type, args.dry_run)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from urlparse import urlparse

from django.conf import settings

from pymongo import Connection

mongodb_string = getattr(settings, 'MONGODB_STRING', '')
s = urlparse(mongodb_string)

connection = Connection(mongodb_string)

db = connection.hit_counter

db.authenticate(s.username, s.password)

hits = db.hits
single_hits = db.single_hits

def insert_hit(content_type, id, ip):
    h = hits.find({
        'contentType': content_type,
        'objId': id,
        'ip': ip
    })

    print h.count()
    print

    if h.count() == 0:
        single_hits.update({
            'contentType': content_type,
            'objId': id,
        }, {
            '$inc': {'hits': 1}
        }, True)

    hits.insert({
        'contentType': content_type,
        'objId': id,
        'ip': ip
    })
<commit_msg>removed debug print<commit_after>from urlparse import urlparse

from django.conf import settings

from pymongo import Connection

mongodb_string = getattr(settings, 'MONGODB_STRING', '')
s = urlparse(mongodb_string)

connection = Connection(mongodb_string)

db = connection.hit_counter

db.authenticate(s.username, s.password)

hits = db.hits
single_hits = db.single_hits

def insert_hit(content_type, id, ip):
    h = hits.find({
        'contentType': content_type,
        'objId': id,
        'ip': ip
    })

    if h.count() == 0:
        single_hits.update({
            'contentType': content_type,
            'objId': id,
        }, {
            '$inc': {'hits': 1}
        }, True)

    hits.insert({
        'contentType': content_type,
        'objId': id,
        'ip': ip
    })
<|endoftext|>


<commit_before>__author__ = 'abdul'

import mongoctl.repository as repository
import server

from mongoctl.utils import resolve_path
from mongoctl.mongoctl_logging import log_verbose

from bson.son import SON
from mongoctl.errors import MongoctlException
from cluster import get_member_repl_lag

###############################################################################
# CONSTANTS
###############################################################################

# This is mongodb's default dbpath
DEFAULT_DBPATH='/data/db'

LOCK_FILE_NAME = "mongod.lock"

###############################################################################
# MongodServer Class
###############################################################################

class MongodServer(server.Server):

    ###########################################################################
    # Constructor
    ###########################################################################
    def __init__(self, server_doc):
        super(MongodServer, self).__init__(server_doc)

    ###########################################################################
    # Properties
    ###########################################################################

    def get_db_path(self):
        dbpath = self.get_cmd_option("dbpath")
        if dbpath is None:
            dbpath = DEFAULT_DBPATH

        return resolve_path(dbpath)

    ###########################################################################
    def get_server_root_folder(self):
        """
            Override!
        :return:
        """
        return self.get_db_path()

    ###########################################################################
    def export_cmd_options(self):
        """
            Override!
        :return:
        """
        cmd_options =  super(MongodServer, self).export_cmd_options()
        # reset some props to exporting vals
        cmd_options['dbpath'] = self.get_db_path()

        return cmd_options

    ###########################################################################
    def get_lock_file_path(self):
        return self.get_default_file_path(LOCK_FILE_NAME)

    ###########################################################################
    def is_master(self):
        return self.get_cmd_option("master")

    ###########################################################################
    def is_slave(self):
        return self.get_cmd_option("slave")

    ###########################################################################
    def is_auth(self):
        if self.get_cmd_option("auth") or self.get_cmd_option("keyFile"):
            return True
        else:
            cluster = self.get_cluster()
            if cluster:
                return cluster.get_repl_key()

    ###########################################################################
    def set_auth(self,auth):
        self.set_cmd_option("auth", auth)

    ###########################################################################
    def get_cluster(self):
        return repository.lookup_cluster_by_server(self)

    ###########################################################################
    def is_cluster_member(self):
        return self.get_cluster() is not None

    ###########################################################################
    def is_administrable(self):
        return not self.is_arbiter_server() and self.can_function()

    ###########################################################################
    def get_status(self, admin=False):

        # get status for super
        status = super(MongodServer, self).get_status(admin=admin)

        if "error" not in status and admin and not self.is_arbiter_server():
            rs_summary = self.get_rs_status_summary()
            if rs_summary:
                status["selfReplicaSetStatusSummary"] = rs_summary

        return status

    ###########################################################################
    def get_server_status_summary(self):
        server_status = self.db_command(SON([('serverStatus', 1)]), "admin")
        server_summary = {
            "host": server_status['host'],
            "connections": server_status['connections'],
            "version": server_status['version']
        }
        return server_summary

    ###########################################################################
    def get_rs_status_summary(self):
        if self.is_cluster_member():
            member_rs_status = self.get_member_rs_status()
            if member_rs_status:
                return {
                    "name": member_rs_status['name'],
                    "stateStr": member_rs_status['stateStr']
                }

    ###########################################################################
    def is_arbiter_server(self):
        cluster = repository.lookup_cluster_by_server(self)
        if cluster is not None:
            return cluster.get_member_for(self).is_arbiter()
        else:
            return False

    ###########################################################################
    def command_needs_auth(self, dbname, cmd):
        # isMaster command does not need auth
        if "isMaster" in cmd or "ismaster" in cmd:
            return False
        if 'shutdown' in cmd and self.is_arbiter_server():
            return False

        # otherwise use default behavior
        return super(MongodServer, self).command_needs_auth(dbname, cmd)

    ###########################################################################
    def get_mongo_uri_template(self, db=None):
        if not db:
            if self.is_auth():
                db = "/[dbname]"
            else:
                db = ""
        else:
            db = "/" + db

        creds = "[dbuser]:[dbpass]@" if self.is_auth() else ""
        return "mongodb://%s%s%s" % (creds, self.get_address_display(), db)

    ###########################################################################
    def get_rs_config(self):
        try:
            return self.get_db('local')['system.replset'].find_one()
        except (Exception,RuntimeError), e:
            if type(e) == MongoctlException:
                raise e
            else:
                log_verbose("Cannot get rs config from server '%s'. "
                            "cause: %s" % (self.id, e))
                return None

    ###########################################################################
    def get_rs_status(self):
        try:
            rs_status_cmd = SON([('replSetGetStatus', 1)])
            rs_status =  self.db_command(rs_status_cmd, 'admin')
            return rs_status
        except (Exception,RuntimeError), e:
            log_verbose("Cannot get rs status from server '%s'. cause: %s" %
                        (self.id, e))
            return None

    ###########################################################################
    def get_member_rs_status(self):
        rs_status =  self.get_rs_status()
        if rs_status:
            try:
                for member in rs_status['members']:
                    if 'self' in member and member['self']:
                        return member
            except (Exception,RuntimeError), e:
                log_verbose("Cannot get member rs status from server '%s'. cause: %s" %
                            (self.id, e))
                return None

    ###########################################################################
    def is_primary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("ismaster")

    ###########################################################################
    def is_secondary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("secondary")

    ###########################################################################
    def is_master_command(self):
        try:
            if self.is_online():
                result = self.db_command({"isMaster" : 1}, "admin")
                return result

        except(Exception, RuntimeError),e:
            log_verbose("isMaster command failed on server '%s'. Cause %s" %
                        (self.id, e))

    ###########################################################################
    def read_replicaset_name(self):
        master_result = self.is_master_command()
        if master_result:
            return "setName" in master_result and master_result["setName"]

    ###########################################################################
    def get_repl_lag(self, master_status):
        """
            Given two 'members' elements from rs.status(),
            return lag between their optimes (in secs).
        """
        member_status = self.get_member_rs_status()

        if not member_status:
            raise MongoctlException("Unable to determine replicaset status for"
                                    " member '%s'" %
                                    self.id)

        return get_member_repl_lag(member_status, master_status)

    ###########################################################################
    def needs_repl_key(self):
        """
         We need a repl key if you are auth + a cluster member +
         version is None or >= 2.0.0
        """
        return (self.supports_repl_key() and
                self.is_auth() and self.is_cluster_member())

<commit_msg>exempting non admin/local/config database users for config servers<commit_after>__author__ = 'abdul'

import mongoctl.repository as repository
import server

from mongoctl.utils import resolve_path
from mongoctl.mongoctl_logging import log_verbose

from bson.son import SON
from mongoctl.errors import MongoctlException
from cluster import get_member_repl_lag

###############################################################################
# CONSTANTS
###############################################################################

# This is mongodb's default dbpath
DEFAULT_DBPATH='/data/db'

LOCK_FILE_NAME = "mongod.lock"

###############################################################################
# MongodServer Class
###############################################################################

class MongodServer(server.Server):

    ###########################################################################
    # Constructor
    ###########################################################################
    def __init__(self, server_doc):
        super(MongodServer, self).__init__(server_doc)

    ###########################################################################
    # Properties
    ###########################################################################

    def get_db_path(self):
        dbpath = self.get_cmd_option("dbpath")
        if dbpath is None:
            dbpath = DEFAULT_DBPATH

        return resolve_path(dbpath)

    ###########################################################################
    def get_server_root_folder(self):
        """
            Override!
        :return:
        """
        return self.get_db_path()

    ###########################################################################
    def export_cmd_options(self):
        """
            Override!
        :return:
        """
        cmd_options =  super(MongodServer, self).export_cmd_options()
        # reset some props to exporting vals
        cmd_options['dbpath'] = self.get_db_path()

        return cmd_options

    def get_seed_users(self):
        """
            Override!
        :return:
        """
        seed_users = super(MongodServer, self).get_seed_users()
        # exempt database users for config servers
        if seed_users and self.is_config_server():
            for dbname in seed_users.keys():
                if dbname not in ["admin", "local", "config"]:
                    del seed_users[dbname]

        return seed_users

    ###########################################################################
    def get_lock_file_path(self):
        return self.get_default_file_path(LOCK_FILE_NAME)

    ###########################################################################
    def is_master(self):
        return self.get_cmd_option("master")

    ###########################################################################
    def is_slave(self):
        return self.get_cmd_option("slave")

    ###########################################################################
    def is_auth(self):
        if self.get_cmd_option("auth") or self.get_cmd_option("keyFile"):
            return True
        else:
            cluster = self.get_cluster()
            if cluster:
                return cluster.get_repl_key()

    ###########################################################################
    def set_auth(self,auth):
        self.set_cmd_option("auth", auth)

    ###########################################################################
    def get_cluster(self):
        return repository.lookup_cluster_by_server(self)

    ###########################################################################
    def is_cluster_member(self):
        return self.get_cluster() is not None

    ###########################################################################
    def is_config_server(self):
        return self.get_cmd_option("configsvr")

    ###########################################################################
    def is_administrable(self):
        return not self.is_arbiter_server() and self.can_function()

    ###########################################################################
    def get_status(self, admin=False):

        # get status for super
        status = super(MongodServer, self).get_status(admin=admin)

        if "error" not in status and admin and not self.is_arbiter_server():
            rs_summary = self.get_rs_status_summary()
            if rs_summary:
                status["selfReplicaSetStatusSummary"] = rs_summary

        return status

    ###########################################################################
    def get_server_status_summary(self):
        server_status = self.db_command(SON([('serverStatus', 1)]), "admin")
        server_summary = {
            "host": server_status['host'],
            "connections": server_status['connections'],
            "version": server_status['version']
        }
        return server_summary

    ###########################################################################
    def get_rs_status_summary(self):
        if self.is_cluster_member():
            member_rs_status = self.get_member_rs_status()
            if member_rs_status:
                return {
                    "name": member_rs_status['name'],
                    "stateStr": member_rs_status['stateStr']
                }

    ###########################################################################
    def is_arbiter_server(self):
        cluster = repository.lookup_cluster_by_server(self)
        if cluster is not None:
            return cluster.get_member_for(self).is_arbiter()
        else:
            return False

    ###########################################################################
    def command_needs_auth(self, dbname, cmd):
        # isMaster command does not need auth
        if "isMaster" in cmd or "ismaster" in cmd:
            return False
        if 'shutdown' in cmd and self.is_arbiter_server():
            return False

        # otherwise use default behavior
        return super(MongodServer, self).command_needs_auth(dbname, cmd)

    ###########################################################################
    def get_mongo_uri_template(self, db=None):
        if not db:
            if self.is_auth():
                db = "/[dbname]"
            else:
                db = ""
        else:
            db = "/" + db

        creds = "[dbuser]:[dbpass]@" if self.is_auth() else ""
        return "mongodb://%s%s%s" % (creds, self.get_address_display(), db)

    ###########################################################################
    def get_rs_config(self):
        try:
            return self.get_db('local')['system.replset'].find_one()
        except (Exception,RuntimeError), e:
            if type(e) == MongoctlException:
                raise e
            else:
                log_verbose("Cannot get rs config from server '%s'. "
                            "cause: %s" % (self.id, e))
                return None

    ###########################################################################
    def get_rs_status(self):
        try:
            rs_status_cmd = SON([('replSetGetStatus', 1)])
            rs_status =  self.db_command(rs_status_cmd, 'admin')
            return rs_status
        except (Exception,RuntimeError), e:
            log_verbose("Cannot get rs status from server '%s'. cause: %s" %
                        (self.id, e))
            return None

    ###########################################################################
    def get_member_rs_status(self):
        rs_status =  self.get_rs_status()
        if rs_status:
            try:
                for member in rs_status['members']:
                    if 'self' in member and member['self']:
                        return member
            except (Exception,RuntimeError), e:
                log_verbose("Cannot get member rs status from server '%s'. cause: %s" %
                            (self.id, e))
                return None

    ###########################################################################
    def is_primary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("ismaster")

    ###########################################################################
    def is_secondary(self):
        master_result = self.is_master_command()

        if master_result:
            return master_result.get("secondary")

    ###########################################################################
    def is_master_command(self):
        try:
            if self.is_online():
                result = self.db_command({"isMaster" : 1}, "admin")
                return result

        except(Exception, RuntimeError),e:
            log_verbose("isMaster command failed on server '%s'. Cause %s" %
                        (self.id, e))

    ###########################################################################
    def read_replicaset_name(self):
        master_result = self.is_master_command()
        if master_result:
            return "setName" in master_result and master_result["setName"]

    ###########################################################################
    def get_repl_lag(self, master_status):
        """
            Given two 'members' elements from rs.status(),
            return lag between their optimes (in secs).
        """
        member_status = self.get_member_rs_status()

        if not member_status:
            raise MongoctlException("Unable to determine replicaset status for"
                                    " member '%s'" %
                                    self.id)

        return get_member_repl_lag(member_status, master_status)

    ###########################################################################
    def needs_repl_key(self):
        """
         We need a repl key if you are auth + a cluster member +
         version is None or >= 2.0.0
        """
        return (self.supports_repl_key() and
                self.is_auth() and self.is_cluster_member())

<|endoftext|>


<commit_before>import urllib2
from bs4 import BeautifulSoup

nasaissurl = 'http://spaceflight.nasa.gov/realdata/sightings/SSapplications/Post/JavaSSOP/orbit/ISS/SVPOST.html'
#TLE = BeautifulSoup(urllib2.urlopen(nasaissurl), 'html.parser')
soup = BeautifulSoup(open('./EVA.html'), 'html.parser')

numEVAs = soup.find("td", text="Acaba").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").text
EVAtime_hours = soup.find("td", text="Acaba").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").text
EVAtime_minutes = soup.find("td", text="Acaba").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").text

print numEVAs
print EVAtime_hours
print EVAtime_minutes
<commit_msg>Update urltest3.py<commit_after>import urllib2
from bs4 import BeautifulSoup

nasaissurl = 'http://spaceflight.nasa.gov/realdata/sightings/SSapplications/Post/JavaSSOP/orbit/ISS/SVPOST.html'
#TLE = BeautifulSoup(urllib2.urlopen(nasaissurl), 'html.parser')
soup = BeautifulSoup(open('./EVA.html'), 'html.parser')

name = "Acaba"

numEVAs = soup.find("td", text=name).find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").text
EVAtime_hours = soup.find("td", text=name).find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").text
EVAtime_minutes = soup.find("td", text=name).find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").find_next_sibling("td").text

print numEVAs
print EVAtime_hours
print EVAtime_minutes
<|endoftext|>


<commit_before>import difflib, re
from pymongo.objectid import ObjectId

class CollectionDoc(object):

    def __init__(self, db, docs):
        self._db = db
        self._collection_names = db.collection_names()
        self._docs = docs

    @property
    def text(self):
        text = ''
        for doc in self._docs:
            text += '{0}\n\n'.format(doc.text)
        text = text.split('\n')
        self._find_relationships(text)
        return '\n'.join(text)

    def _find_relationships(self, text):
        level = 0
        for doc in self._docs:
            fields = self._find_link_fields(doc.doc)
            level = self._make_links(doc.collection, fields, text, level)

    def _make_links(self, doc_collection_name, fields, text, level):
        for key, value in fields.iteritems():
            collection_name = self._find_collection(key, value)
            if collection_name is not None:
                self._make_link(doc_collection_name, collection_name, key, value, text, level)
                level += 1
        return level

    def _make_link(self, doc_collection_name, collection_name, key, value, text, level):

        for collection_start, row in enumerate(text):
            j = row.find('{0} '.format(doc_collection_name))
            if -1 < j < 20:
                break

        for i, row in enumerate(text[collection_start:]):
            j = row.find('{0}:'.format(key))
            if -1 < j < 20:
                break

        for k, row in enumerate(text):
            j = row.find('{0} '.format(collection_name))
            if -1 < j < 20:
                break

        start, end = sorted([k, i + collection_start])
        for i in xrange(len(text) - 1):
            self._append_row(start, end, i, text, level)

    def _append_row(self, start, end, i, text, level):
        j = 3 * level
        prefix = text[i][:j]
        rest = text[i][j:]
        print 'prefix:', len(prefix), level, prefix
        if (i == start or i == end) and '|' not in prefix:
            prefix = '+--{0}'.format(prefix.replace(' ', '-'))
        elif (i == start or i == end) and '|' in prefix:
            prefix = '+--{0}'.format(prefix.replace(' ', '-').replace('|', '+'))
        elif start > i or i > end:
            prefix = '   {0}'.format(prefix)
        elif start < i or i < end:
            prefix = '|  {0}'.format(prefix)
        text[i] = prefix + rest

    def _find_collection(self, key, value):
        scores = []
        for collection_name in self._collection_names:
            scores.append((collection_name, value, difflib.SequenceMatcher(None, collection_name, key).ratio()))
        scores.sort(key=lambda x: x[2], reverse=True)
        for item in scores:
            collection_name = item[0]
            value = item[1]
            doc = self._db[collection_name].find_one({'_id': value})
            if doc is None:
                continue
            else:
                return collection_name

    def _find_link_fields(self, doc):
        fields = {}
        for key, value in doc.iteritems():
            if key == '_id':
                continue
            if isinstance(value, ObjectId):
                fields[key] = value
        return fields

<commit_msg>finished link function<commit_after>import difflib, re
from pymongo.objectid import ObjectId

class CollectionDoc(object):

    def __init__(self, db, docs):
        self._db = db
        self._collection_names = db.collection_names()
        self._docs = docs

    @property
    def text(self):
        text = ''
        for doc in self._docs:
            text += '{0}\n\n'.format(doc.text)
        text = text.split('\n')
        self._find_relationships(text)
        return '\n'.join(text)

    def _find_relationships(self, text):
        level = 0
        for doc in self._docs:
            fields = self._find_link_fields(doc.doc)
            level = self._make_links(doc.collection, fields, text, level)

    def _make_links(self, doc_collection_name, fields, text, level):
        for key, value in fields.iteritems():
            collection_name = self._find_collection(key, value)
            if collection_name is not None:
                self._make_link(doc_collection_name, collection_name, key, value, text, level)
                level += 1
        return level

    def _make_link(self, doc_collection_name, collection_name, key, value, text, level):
        for collection_start, row in enumerate(text):
            j = row.find('{0} '.format(doc_collection_name))
            if -1 < j < 20:
                break
        for i, row in enumerate(text[collection_start:]):
            j = row.find('{0}:'.format(key))
            if -1 < j < 20:
                break
        for k, row in enumerate(text):
            j = row.find('{0} '.format(collection_name))
            if -1 < j < 20:
                break
        start, end = sorted([k, i + collection_start])
        for i in xrange(len(text) - 1):
            self._append_row(start, end, i, text, level)

    def _append_row(self, start, end, i, text, level):
        j = 3 * level
        prefix = text[i][:j]
        rest = text[i][j:]
        print 'prefix:', len(prefix), level, prefix
        if (i == start or i == end) and '|' not in prefix:
            prefix = '+--{0}'.format(prefix.replace(' ', '-'))
        elif (i == start or i == end) and '|' in prefix:
            prefix = '+--{0}'.format(prefix.replace(' ', '-').replace('|', '+'))
        elif start > i or i > end:
            prefix = '   {0}'.format(prefix)
        elif start < i or i < end:
            prefix = '|  {0}'.format(prefix)
        text[i] = prefix + rest

    def _find_collection(self, key, value):
        scores = []
        for collection_name in self._collection_names:
            scores.append((collection_name, value, difflib.SequenceMatcher(None, collection_name, key).ratio()))
        scores.sort(key=lambda x: x[2], reverse=True)
        for item in scores:
            collection_name = item[0]
            value = item[1]
            doc = self._db[collection_name].find_one({'_id': value})
            if doc is None:
                continue
            else:
                return collection_name

    def _find_link_fields(self, doc):
        fields = {}
        for key, value in doc.iteritems():
            if key == '_id':
                continue
            if isinstance(value, ObjectId):
                fields[key] = value
        return fields

<|endoftext|>


<commit_before>"""
Test harness for the venv module.

Copyright (C) 2011-2012 Vinay Sajip.
Licensed to the PSF under a contributor agreement.
"""

import os
import os.path
import shutil
import subprocess
import sys
import tempfile
from test.support import (captured_stdout, captured_stderr, run_unittest,
                          can_symlink)
import unittest
import venv

class BaseTest(unittest.TestCase):
    """Base class for venv tests."""

    def setUp(self):
        self.env_dir = os.path.realpath(tempfile.mkdtemp())
        if os.name == 'nt':
            self.bindir = 'Scripts'
            self.pydocname = 'pydoc.py'
            self.lib = ('Lib',)
            self.include = 'Include'
        else:
            self.bindir = 'bin'
            self.pydocname = 'pydoc'
            self.lib = ('lib', 'python%s' % sys.version[:3])
            self.include = 'include'
        if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ:
            executable = os.environ['__PYVENV_LAUNCHER__']
        else:
            executable = sys.executable
        self.exe = os.path.split(executable)[-1]

    def tearDown(self):
        shutil.rmtree(self.env_dir)

    def run_with_capture(self, func, *args, **kwargs):
        with captured_stdout() as output:
            with captured_stderr() as error:
                func(*args, **kwargs)
        return output.getvalue(), error.getvalue()

    def get_env_file(self, *args):
        return os.path.join(self.env_dir, *args)

    def get_text_file_contents(self, *args):
        with open(self.get_env_file(*args), 'r') as f:
            result = f.read()
        return result

class BasicTest(BaseTest):
    """Test venv module functionality."""

    def isdir(self, *args):
        fn = self.get_env_file(*args)
        self.assertTrue(os.path.isdir(fn))

    def test_defaults(self):
        """
        Test the create function with default arguments.
        """
        shutil.rmtree(self.env_dir)
        self.run_with_capture(venv.create, self.env_dir)
        self.isdir(self.bindir)
        self.isdir(self.include)
        self.isdir(*self.lib)
        data = self.get_text_file_contents('pyvenv.cfg')
        if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__'
                                         in os.environ):
            executable =  os.environ['__PYVENV_LAUNCHER__']
        else:
            executable = sys.executable
        path = os.path.dirname(executable)
        self.assertIn('home = %s' % path, data)
        data = self.get_text_file_contents(self.bindir, self.pydocname)
        self.assertTrue(data.startswith('#!%s%s' % (self.env_dir, os.sep)))
        fn = self.get_env_file(self.bindir, self.exe)
        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
            bd = self.get_env_file(self.bindir)
            print('Contents of %r:' % bd)
            print('    %r' % os.listdir(bd))
        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)

    def test_prefixes(self):
        """
        Test that the prefix values are as expected.
        """
        #check our prefixes
        self.assertEqual(sys.base_prefix, sys.prefix)
        self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)

        # check a venv's prefixes
        shutil.rmtree(self.env_dir)
        self.run_with_capture(venv.create, self.env_dir)
        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
        cmd = [envpy, '-c', None]
        for prefix, expected in (
            ('prefix', self.env_dir),
            ('prefix', self.env_dir),
            ('base_prefix', sys.prefix),
            ('base_exec_prefix', sys.exec_prefix)):
            cmd[2] = 'import sys; print(sys.%s)' % prefix
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            out, err = p.communicate()
            self.assertEqual(out[:-1], expected.encode())

    def test_overwrite_existing(self):
        """
        Test control of overwriting an existing environment directory.
        """
        self.assertRaises(ValueError, venv.create, self.env_dir)
        builder = venv.EnvBuilder(clear=True)
        builder.create(self.env_dir)

    def test_upgrade(self):
        """
        Test upgrading an existing environment directory.
        """
        builder = venv.EnvBuilder(upgrade=True)
        self.run_with_capture(builder.create, self.env_dir)
        self.isdir(self.bindir)
        self.isdir(self.include)
        self.isdir(*self.lib)
        fn = self.get_env_file(self.bindir, self.exe)
        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
            bd = self.get_env_file(self.bindir)
            print('Contents of %r:' % bd)
            print('    %r' % os.listdir(bd))
        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)

    def test_isolation(self):
        """
        Test isolation from system site-packages
        """
        for ssp, s in ((True, 'true'), (False, 'false')):
            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
            builder.create(self.env_dir)
            data = self.get_text_file_contents('pyvenv.cfg')
            self.assertIn('include-system-site-packages = %s\n' % s, data)

    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
    def test_symlinking(self):
        """
        Test symlinking works as expected
        """
        for usl in (False, True):
            builder = venv.EnvBuilder(clear=True, symlinks=usl)
            if (usl and sys.platform == 'darwin' and
                '__PYVENV_LAUNCHER__' in os.environ):
                self.assertRaises(ValueError, builder.create, self.env_dir)
            else:
                builder.create(self.env_dir)
                fn = self.get_env_file(self.bindir, self.exe)
                # Don't test when False, because e.g. 'python' is always
                # symlinked to 'python3.3' in the env, even when symlinking in
                # general isn't wanted.
                if usl:
                    self.assertTrue(os.path.islink(fn))

def test_main():
    run_unittest(BasicTest)

if __name__ == "__main__":
    test_main()
<commit_msg>Issue #15307: Skipped test_venv:test_prefixes when run from a venv.<commit_after>"""
Test harness for the venv module.

Copyright (C) 2011-2012 Vinay Sajip.
Licensed to the PSF under a contributor agreement.
"""

import os
import os.path
import shutil
import subprocess
import sys
import tempfile
from test.support import (captured_stdout, captured_stderr, run_unittest,
                          can_symlink)
import unittest
import venv

class BaseTest(unittest.TestCase):
    """Base class for venv tests."""

    def setUp(self):
        self.env_dir = os.path.realpath(tempfile.mkdtemp())
        if os.name == 'nt':
            self.bindir = 'Scripts'
            self.pydocname = 'pydoc.py'
            self.lib = ('Lib',)
            self.include = 'Include'
        else:
            self.bindir = 'bin'
            self.pydocname = 'pydoc'
            self.lib = ('lib', 'python%s' % sys.version[:3])
            self.include = 'include'
        if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ:
            executable = os.environ['__PYVENV_LAUNCHER__']
        else:
            executable = sys.executable
        self.exe = os.path.split(executable)[-1]

    def tearDown(self):
        shutil.rmtree(self.env_dir)

    def run_with_capture(self, func, *args, **kwargs):
        with captured_stdout() as output:
            with captured_stderr() as error:
                func(*args, **kwargs)
        return output.getvalue(), error.getvalue()

    def get_env_file(self, *args):
        return os.path.join(self.env_dir, *args)

    def get_text_file_contents(self, *args):
        with open(self.get_env_file(*args), 'r') as f:
            result = f.read()
        return result

class BasicTest(BaseTest):
    """Test venv module functionality."""

    def isdir(self, *args):
        fn = self.get_env_file(*args)
        self.assertTrue(os.path.isdir(fn))

    def test_defaults(self):
        """
        Test the create function with default arguments.
        """
        shutil.rmtree(self.env_dir)
        self.run_with_capture(venv.create, self.env_dir)
        self.isdir(self.bindir)
        self.isdir(self.include)
        self.isdir(*self.lib)
        data = self.get_text_file_contents('pyvenv.cfg')
        if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__'
                                         in os.environ):
            executable =  os.environ['__PYVENV_LAUNCHER__']
        else:
            executable = sys.executable
        path = os.path.dirname(executable)
        self.assertIn('home = %s' % path, data)
        data = self.get_text_file_contents(self.bindir, self.pydocname)
        self.assertTrue(data.startswith('#!%s%s' % (self.env_dir, os.sep)))
        fn = self.get_env_file(self.bindir, self.exe)
        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
            bd = self.get_env_file(self.bindir)
            print('Contents of %r:' % bd)
            print('    %r' % os.listdir(bd))
        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)

    @unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate '
                     'in a venv')
    def test_prefixes(self):
        """
        Test that the prefix values are as expected.
        """
        #check our prefixes
        self.assertEqual(sys.base_prefix, sys.prefix)
        self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)

        # check a venv's prefixes
        shutil.rmtree(self.env_dir)
        self.run_with_capture(venv.create, self.env_dir)
        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
        cmd = [envpy, '-c', None]
        for prefix, expected in (
            ('prefix', self.env_dir),
            ('prefix', self.env_dir),
            ('base_prefix', sys.prefix),
            ('base_exec_prefix', sys.exec_prefix)):
            cmd[2] = 'import sys; print(sys.%s)' % prefix
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            out, err = p.communicate()
            self.assertEqual(out[:-1], expected.encode())

    def test_overwrite_existing(self):
        """
        Test control of overwriting an existing environment directory.
        """
        self.assertRaises(ValueError, venv.create, self.env_dir)
        builder = venv.EnvBuilder(clear=True)
        builder.create(self.env_dir)

    def test_upgrade(self):
        """
        Test upgrading an existing environment directory.
        """
        builder = venv.EnvBuilder(upgrade=True)
        self.run_with_capture(builder.create, self.env_dir)
        self.isdir(self.bindir)
        self.isdir(self.include)
        self.isdir(*self.lib)
        fn = self.get_env_file(self.bindir, self.exe)
        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
            bd = self.get_env_file(self.bindir)
            print('Contents of %r:' % bd)
            print('    %r' % os.listdir(bd))
        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)

    def test_isolation(self):
        """
        Test isolation from system site-packages
        """
        for ssp, s in ((True, 'true'), (False, 'false')):
            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
            builder.create(self.env_dir)
            data = self.get_text_file_contents('pyvenv.cfg')
            self.assertIn('include-system-site-packages = %s\n' % s, data)

    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
    def test_symlinking(self):
        """
        Test symlinking works as expected
        """
        for usl in (False, True):
            builder = venv.EnvBuilder(clear=True, symlinks=usl)
            if (usl and sys.platform == 'darwin' and
                '__PYVENV_LAUNCHER__' in os.environ):
                self.assertRaises(ValueError, builder.create, self.env_dir)
            else:
                builder.create(self.env_dir)
                fn = self.get_env_file(self.bindir, self.exe)
                # Don't test when False, because e.g. 'python' is always
                # symlinked to 'python3.3' in the env, even when symlinking in
                # general isn't wanted.
                if usl:
                    self.assertTrue(os.path.islink(fn))

def test_main():
    run_unittest(BasicTest)

if __name__ == "__main__":
    test_main()
<|endoftext|>


<commit_before>
# GOAL
  # Imagine you have started up a small restaurant and are trying to make it
  # easier to take and calculate orders. Since your restaurant only sells 9
  # different items, you assign each one to a number, as shown below.
	# Chicken Strips - $3.50
	# French Fries - $2.50
	# Hamburger - $4.00
	# Hotdog - $3.50
	# Large Drink - $1.75
	# Medium Drink - $1.50
	# Milk Shake - $2.25
	# Salad - $3.75
	# Small Drink - $1.25
  # To quickly take orders, your program should allow the user to type in a string
  # of numbers and then it should calculate the cost of the order. For example, if
  # one large drink, two small drinks, two hamburgers, one hotdog, and a salad are ordered,
  # the user should type in 5993348, and the program should say that it costs $19.50. Also,
  # make sure that the program loops so the user can take multiple orders without having to
  # restart the program each time.
# SUBGOALS
  # If you decide to, print out the items and prices every time before the user types in
  # an order. Once the user has entered an order, print out how many of each item have been
  # ordered, as well as the total price. If an item was not ordered at all, then it should not show up.

  # Add in a way to track the total number of transactions completed during the program's runtime
  # as well as the total sales dollars that should have been collected by the user
    # - As though someone was going to balance and check their till after finishing for the night
	# - Similar to how we kept track of the game history in the higher/lower game

import time

def _main_():
	# a welcome prompt that will only display on the initilization of the program
	print ('''
			Welcome to the easy-menu genie. I was designed to make your life
			as a waiter as easy as can be. Simply enter the numbers of the items
			ordered by the customer and I will do the rest.

			For exmaple, if the customer orders a Salad and a Small Drink, simply
			enter 89.

			In case you're having trouble remembering the menu numbers, they are:
				1: Chicken Strips
				2: French Fries
				3: Hamburger
				4: Hotdog
				5: Large Drink
				6: Medium Drink
				7: Milk Shake
				8: Salad
				9: Small Drink
			''')
	_CustomerOrder_()

def _CustomerOrder_():

	# Let's make use of Python's built in dictionary data type
	# Is it possible and/or beneficial to combine the two dicts below into one?
	menu_order = {
		1: 'Chicken Strips',
		2: 'French Fries',
		3: 'Hamburger',
		4: 'Hotdog',
		5: 'Large Drink',
		6: 'Medium Drink',
		7: 'Milk Shake',
		8: 'Salad',
		9: 'Small Drink'
		}

	menu_prices = {
		'Chicken Strips' : 3.50,
		'French Fries' : 2.50,
		'Hamburger' : 4.00,
		'Hotdog' : 3.50,
		'Large Drink' : 1.75,
		'Medium Drink' : 1.50,
		'Milk Shake' : 2.75,
		'Salad' : 3.75,
		'Small Drink' : 1.25
		}

	customer_order = input("Enter the customer's order number(s)	> ")
	# sanity check; did the user input a valid response
	try:
		int(customer_order)
	# in the event they did not, start the order over
	except ValueError:
		print ("Are you sure you entered numbers? Let's try that again!")
		customer_order = _RedoOrder_()

	# creating empty list that I will append with the order info
	# one for the items ordered and one for the corresponding prices
	order_prices =  []
	order_items = []
	for x in customer_order:
		order_prices.append(menu_prices[menu_order[int(x)]])
		order_items.append(menu_order[int(x)])

	# print out the entered order to the user to ensure accuracy
	for x in menu_prices:
		item_count = order_items.count(x)
		if item_count > 0:
			print ('Item: %s | Quantity: %s | Item Price: %s | Extended Price: %s'
			% (x,str(item_count), str(menu_prices[x]), str(menu_prices[x] * item_count)))

	order_status = False
	while order_status != True:
		order_status = _SanityCheck_()

	# sum the list containing the prices to provide the user with the order total price
	order_total = sum(order_prices)	* 1.07
	print ("The customer's order total is $%s" % str(format(order_total,'.2f')))

	# call on the _Again_ function to ask the user if they would like to begin another order
	_Again_()

def _SanityCheck_():
	response = input("Does the order above match your customer's order?" +
					" (y/n) >	")
	response = response.strip()
	if response != "y" and response != "n":
		print ("Please enter a valid response - y for yes or n for no")
		_SanityCheck_()
	elif response == "y":
		return True
	elif response == "n":
		new_order = _RedoOrder_()
		return new_order

def _RedoOrder_():
	customer_order = input("Please re-enter the customer's order number(s)	> ")
	try:
		int(customer_order)
	except ValueError:
		print ("Are you sure you entered a number? Let's try that again!")
		_RedoOrder_()
	return customer_order

def _Again_():
	response = input("Would you like to begin another order? (y/n)	>	")
	if response != "y" and response != "n":
		print ("Please enter a valid response - y for yes or n for no")
		_Again_()
	if response == "y":
		_CustomerOrder_()
	elif response == "n":
		print ("I hope I was of value to you today")
		print ("Shutting Down...")
		time.sleep(0.5)
		exit

_main_()
<commit_msg>Update menu.py<commit_after>

import time

def _main_():
    print ('''
    Welcome to the easy-menu genie. I was designed to make your life
    as easy as possible. Simply enter the number that corresponds to each
    menu item ordered by the customer and I will handle the rest.

    For example, if a customer orders a salad and a small drink, simply
    enter 89.

    For your reference, the menu numbers are as follows"
        1: Chicken Strips
        2: French Fries
        3: Hamburger
        4: Hotdog
        5: Large Drink
        6: Medium Drink
        7: Milk Shake
        8: Salad
        9: Small Drink
    ''')

    _CustomerOrder_()

def _CustomerOrder_():
    menu_order = {
        1: 'Chicken Strips',
        2: 'French Fries',
        3: 'Hamburger',
        4: 'Hotdog',
        5: 'Large Drink',
        6: 'Medium Drink',
        7: 'Milk Shake',
        8: 'Salad',
        9: 'Small Drink'
        }

    menu_prices = {
        'Chicken Strips': 3.50,
        'French Fries': 2.50,
        'Hamburger': 4.00,
        'Hotdog': 3.50,
        'Large Drink': 1.75,
        'Medium Drink': 1.50,
        'Milk Shake': 2.75,
        'Salad': 3.75,
        'Small Drink': 1.25
        }

    customer_order = input("Enter the customer's order  >  ")

    try:
        int(customer_order)
    except ValueError:
        print ("Are you sure you entered a valid order number? Please " +
            "enter the order number(s) again")
        _RedoOrder_()

    order_prices = []
    order_items = []

    for x in customer_order:
        order_prices.append(menu_prices[menu_order[int(x)]])
        order_items.append(menu_order[int(x)])

    for x in menu_prices:
        item_count = order_items.count(x)
        if item_count > 0:
            print ('Item: %s | Quantity: %s | Price: %s | Extended Price: %s'
            % (x,str(item_count), str(menu_prices[x]), str(menu_prices[x] * item_count)))

    order_status = False
    while order_status != True:
        order_status = _SanityCheck_()

    pretax_total = sum(order_prices)
    tax_amount = sum(order_prices) * 0.07
    order_total = pretax_total + tax_amount

    print (" Order Subtotal: $%s \n Sales Tax: $%s \n Order Total: $%s"
    % (pretax_total, tax_amount, order_total))

    _Again_()

def _SanityCheck_():
    response = input("Does the order above match your customer's " +
    "request? (y/n) >  ")
    response = response.lower().strip()
    if response != 'y' and response != 'n':
        print ("Please enter a valid response - y for yes or n for no")
        _SanityCheck_()
    elif response == 'y':
        return True
    elif response == 'n':
        new_order = _RedoOrder_()
        return new_order

def _RedoOrder_():
    customer_order = input("Please re-enter the customer's order number(s)  >  ")
    try:
        int(customer_order)
    except ValueError:
        print ("Are you sure you entered a valid order number?")
        _RedoOrder_()
    return customer_order

def _Again_():
    response = input("Would you like to begin another order (y/n)   >  ")
    response = response.lower().strip()
    if response != 'y' and response != 'n':
        print ("Please enter a valid response - y for yes or n for no")
        _Again_()
    elif response == 'y':
        _CustomerOrder_()
    elif response == 'n':
        print ("Thanks for letting me help you today.")
        print ("Shutting Down...")
        time.sleep(0.15)
        exit

_main_()
<|endoftext|>


<commit_before>"""Authentication and authorization."""

from haas.errors import AuthorizationError
from abc import ABCMeta, abstractmethod

import sys

_auth_backend = None

class AuthBackend(object):
    """An authentication/authorization backend.

    Extensions which implement authentication/authorization backends should
    inherit from this class, and invoke ``set_auth_backend()`` on an instance
    of the subclass
    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def authenticate(self):
        """Authenticate the api call, and prepare for later authorization checks.

        This method will be invoked inside of a request context, meaning
        ``haas.rest.local.request`` will be available. It is responsible for
        authenticating the request, and storing any data it will need later to
        determine whether the requested operation is authorized.

        The attribute ``haas.rest.local.auth`` is reserved for use by auth
        backends; A backend may store any information it needs as that
        attribute.
        """

    @abstractmethod
    def have_admin(self):
        """Check if the request is authorized to act as an administrator.

        Return True if so, False if not. This will be caled sometime after
        ``authenticate()``.
        """

    @abstractmethod
    def have_project_access(self, project):
        """Check if the request is authorized to act as the given project.

        Return True if so, False if not. This will be caled sometime after
        ``authenticate()``.

        ``project`` will be a ``Project`` object, *not* the name of the
        project.
        """

    def require_admin(self):
        """Ensure the request is authorized to act as an administrator.

        Raises an ``AuthorizationError`` on failure, instead of returning False.
        This is a convienence wrapper around ``have_admin``, and should not be
        overwritten by subclasses.
        """
        if not self.have_admin():
            raise AuthorizationError("This operation is administrator-only.")

    def require_project_access(self, project):
        """Like ``require_admin()``, but wraps ``have_project_access()``."""
        if not self.have_project_access(project):
            raise AuthorizationError("You do not have access to the required project.")

def set_auth_backend(backend):
    """Set the authentication backend to ``backend``.

    This should be called exactly once, on startup, with an instance of
    ``AuthBackend`` as it's argument.
    """
    global _auth_backend
    if _auth_backend is not None:
        sys.exit("Fatal Error: set_auth_backed() called twice. Make sure "
                 "you don't have conflicting extensions loaded.")
    _auth_backend = backend

def get_auth_backend():
    """Return the current auth backend."""
    return _auth_backend
<commit_msg>Clarify documentation for have_project_access.<commit_after>"""Authentication and authorization."""

from haas.errors import AuthorizationError
from abc import ABCMeta, abstractmethod

import sys

_auth_backend = None

class AuthBackend(object):
    """An authentication/authorization backend.

    Extensions which implement authentication/authorization backends should
    inherit from this class, and invoke ``set_auth_backend()`` on an instance
    of the subclass
    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def authenticate(self):
        """Authenticate the api call, and prepare for later authorization checks.

        This method will be invoked inside of a request context, meaning
        ``haas.rest.local.request`` will be available. It is responsible for
        authenticating the request, and storing any data it will need later to
        determine whether the requested operation is authorized.

        The attribute ``haas.rest.local.auth`` is reserved for use by auth
        backends; A backend may store any information it needs as that
        attribute.
        """

    @abstractmethod
    def have_admin(self):
        """Check if the request is authorized to act as an administrator.

        Return True if so, False if not. This will be caled sometime after
        ``authenticate()``.
        """

    @abstractmethod
    def have_project_access(self, project):
        """Check if the request is authorized to act as the given project.

        Return True if so, False if not. This will be caled sometime after
        ``authenticate()``.

        ``project`` will be a ``Project`` object, *not* the name of the
        project.

        Note that in general, have_admin should imply have_project_acccess,
        but the backend must specifically implement this logic.
        """

    def require_admin(self):
        """Ensure the request is authorized to act as an administrator.

        Raises an ``AuthorizationError`` on failure, instead of returning False.
        This is a convienence wrapper around ``have_admin``, and should not be
        overwritten by subclasses.
        """
        if not self.have_admin():
            raise AuthorizationError("This operation is administrator-only.")

    def require_project_access(self, project):
        """Like ``require_admin()``, but wraps ``have_project_access()``."""
        if not self.have_project_access(project):
            raise AuthorizationError("You do not have access to the required project.")

def set_auth_backend(backend):
    """Set the authentication backend to ``backend``.

    This should be called exactly once, on startup, with an instance of
    ``AuthBackend`` as it's argument.
    """
    global _auth_backend
    if _auth_backend is not None:
        sys.exit("Fatal Error: set_auth_backed() called twice. Make sure "
                 "you don't have conflicting extensions loaded.")
    _auth_backend = backend

def get_auth_backend():
    """Return the current auth backend."""
    return _auth_backend
<|endoftext|>