<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import socket
import sys

from oslo_config import cfg

from st2common.constants.system import VERSION_STRING

def do_register_opts(opts, group=None, ignore_errors=False):
    try:
        cfg.CONF.register_opts(opts, group=group)
    except:
        if not ignore_errors:
            raise

def do_register_cli_opts(opt, ignore_errors=False):
    # TODO: This function has broken name, it should work with lists :/
    if not isinstance(opt, (list, tuple)):
        opts = [opt]
    else:
        opts = opt

    try:
        cfg.CONF.register_cli_opts(opts)
    except:
        if not ignore_errors:
            raise

def register_opts(ignore_errors=False):
    rbac_opts = [
        cfg.BoolOpt('enable', default=False, help='Enable RBAC.'),
    ]
    do_register_opts(rbac_opts, 'rbac', ignore_errors)

    system_user_opts = [
        cfg.StrOpt('user',
                   default='stanley',
                   help='Default system user.'),
        cfg.StrOpt('ssh_key_file',
                   default='/home/vagrant/.ssh/stanley_rsa',
                   help='SSH private key for the system user.')
    ]
    do_register_opts(system_user_opts, 'system_user', ignore_errors)

    schema_opts = [
        cfg.IntOpt('version', default=4, help='Version of JSON schema to use.'),
        cfg.StrOpt('draft', default='http://json-schema.org/draft-04/schema#',
                   help='URL to the JSON schema draft.')
    ]
    do_register_opts(schema_opts, 'schema', ignore_errors)

    system_opts = [
        cfg.BoolOpt('debug', help='Enable debug mode.', default=False),
        cfg.StrOpt('base_path', default='/opt/stackstorm',
                   help='Base path to all st2 artifacts.')
    ]
    do_register_opts(system_opts, 'system', ignore_errors)

    system_packs_base_path = os.path.join(cfg.CONF.system.base_path, 'packs')
    content_opts = [
        cfg.StrOpt('system_packs_base_path', default=system_packs_base_path,
                   help='Path to the directory which contains system packs.'),
        cfg.StrOpt('packs_base_paths', default=None,
                   help='Paths which will be searched for integration packs.')
    ]
    do_register_opts(content_opts, 'content', ignore_errors)

    webui_opts = [
        cfg.StrOpt('webui_base_url', default='https://%s' % socket.gethostname(),
                   help='Base https URL to access st2 Web UI. This is used to construct' +
                        'history URLs that are sent out when chatops is used to kick off ' +
                        'executions.')
    ]
    do_register_opts(webui_opts, 'webui', ignore_errors)

    db_opts = [
        cfg.StrOpt('host', default='0.0.0.0', help='host of db server'),
        cfg.IntOpt('port', default=27017, help='port of db server'),
        cfg.StrOpt('db_name', default='st2', help='name of database'),
        cfg.StrOpt('username', help='username for db login'),
        cfg.StrOpt('password', help='password for db login'),
        cfg.IntOpt('connection_retry_max_delay_m', help='Connection retry total time (minutes).',
                   default=3),
        cfg.IntOpt('connection_retry_backoff_max_s', help='Connection retry backoff max (seconds).',
                   default=10),
        cfg.IntOpt('connection_retry_backoff_mul', help='Backoff multiplier (seconds).',
                   default=1),
        cfg.BoolOpt('ssl', help='Create the connection to mongodb using SSL', default=False),
        cfg.StrOpt('ssl_keyfile',
                   help='Private keyfile used to identify the local connection against MongoDB.',
                   default=None),
        cfg.StrOpt('ssl_certfile', help='Certificate file used to identify the localconnection',
                   default=None),
        cfg.StrOpt('ssl_cert_reqs', choices='none, optional, required',
                   help='Specifies whether a certificate is required from the other side of the ' +
                        'connection, and whether it will be validated if provided',
                   default=None),
        cfg.StrOpt('ssl_ca_certs',
                   help='ca_certs file contains a set of concatenated CA certificates, which are' +
                        ' used to validate certificates passed from MongoDB.',
                   default=None),
        cfg.BoolOpt('ssl_match_hostname',
                    help='If True and `ssl_cert_reqs` is not None, enables hostname verification',
                    default=True)
    ]
    do_register_opts(db_opts, 'database', ignore_errors)

    messaging_opts = [
        # It would be nice to be able to deprecate url and completely switch to using
        # url. However, this will be a breaking change and will have impact so allowing both.
        cfg.StrOpt('url', default='amqp://guest:guest@127.0.0.1:5672//',
                   help='URL of the messaging server.'),
        cfg.ListOpt('cluster_urls', default=[],
                    help='URL of all the nodes in a messaging service cluster.')
    ]
    do_register_opts(messaging_opts, 'messaging', ignore_errors)

    syslog_opts = [
        cfg.StrOpt('host', default='127.0.0.1',
                   help='Host for the syslog server.'),
        cfg.IntOpt('port', default=514,
                   help='Port for the syslog server.'),
        cfg.StrOpt('facility', default='local7',
                   help='Syslog facility level.'),
        cfg.StrOpt('protocol', default='udp',
                   help='Transport protocol to use (udp / tcp).')
    ]
    do_register_opts(syslog_opts, 'syslog', ignore_errors)

    log_opts = [
        cfg.ListOpt('excludes', default='',
                    help='Exclusion list of loggers to omit.'),
        cfg.BoolOpt('redirect_stderr', default=False,
                    help='Controls if stderr should be redirected to the logs.'),
        cfg.BoolOpt('mask_secrets', default=True,
                    help='True to mask secrets in the log files.')
    ]
    do_register_opts(log_opts, 'log', ignore_errors)

    # Common API options
    api_opts = [
        cfg.StrOpt('host', default='0.0.0.0', help='StackStorm API server host'),
        cfg.IntOpt('port', default=9101, help='StackStorm API server port'),
        cfg.ListOpt('allow_origin', default=['http://127.0.0.1:3000'],
                    help='List of origins allowed for api, auth and stream'),
        cfg.BoolOpt('mask_secrets', default=True,
                    help='True to mask secrets in the API responses')
    ]
    do_register_opts(api_opts, 'api', ignore_errors)

    # Key Value store options
    keyvalue_opts = [
        cfg.BoolOpt('enable_encryption', default=True,
                    help='Allow encryption of values in key value stored qualified as "secret".'),
        cfg.StrOpt('encryption_key_path', default='',
                   help='Location of the symmetric encryption key for encrypting values in ' +
                        'kvstore. This key should be in JSON and should\'ve been ' +
                        'generated using keyczar.')
    ]
    do_register_opts(keyvalue_opts, group='keyvalue')

    # Common auth options
    auth_opts = [
        cfg.StrOpt('api_url', default=None,
                   help='Base URL to the API endpoint excluding the version'),
        cfg.BoolOpt('enable', default=True, help='Enable authentication middleware.'),
        cfg.IntOpt('token_ttl', default=86400, help='Access token ttl in seconds.')
    ]
    do_register_opts(auth_opts, 'auth', ignore_errors)

    # Common action runner options
    default_python_bin_path = sys.executable
    base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
    default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv')
    action_runner_opts = [
        cfg.StrOpt('logging', default='conf/logging.conf',
                   help='location of the logging.conf file'),
        cfg.StrOpt('python_binary', default=default_python_bin_path,
                   help='Python binary which will be used by Python actions.'),
        cfg.StrOpt('virtualenv_binary', default=default_virtualenv_bin_path,
                   help='Virtualenv binary which should be used to create pack virtualenvs.'),
        cfg.ListOpt('virtualenv_opts', default=['--system-site-packages'],
                    help='List of virtualenv options to be passsed to "virtualenv" command that ' +
                         'creates pack virtualenv.')
    ]
    do_register_opts(action_runner_opts, group='actionrunner')

    # Common options (used by action runner and sensor container)
    action_sensor_opts = [
        cfg.BoolOpt('enable', default=True,
                    help='Whether to enable or disable the ability to post a trigger on action.'),
    ]
    do_register_opts(action_sensor_opts, group='action_sensor')

    # Coordination options
    coord_opts = [
        cfg.StrOpt('url', default=None, help='Endpoint for the coordination server.'),
        cfg.IntOpt('lock_timeout', default=60, help='TTL for the lock if backend suports it.')
    ]
    do_register_opts(coord_opts, 'coordination', ignore_errors)

    # Mistral options
    mistral_opts = [
        cfg.StrOpt('v2_base_url', default='http://127.0.0.1:8989/v2', help='v2 API root endpoint.'),
        cfg.IntOpt('retry_exp_msec', default=1000, help='Multiplier for the exponential backoff.'),
        cfg.IntOpt('retry_exp_max_msec', default=300000, help='Max time for each set of backoff.'),
        cfg.IntOpt('retry_stop_max_msec', default=600000, help='Max time to stop retrying.'),
        cfg.StrOpt('keystone_username', default=None, help='Username for authentication.'),
        cfg.StrOpt('keystone_password', default=None, help='Password for authentication.'),
        cfg.StrOpt('keystone_project_name', default=None, help='OpenStack project scope.'),
        cfg.StrOpt('keystone_auth_url', default=None, help='Auth endpoint for Keystone.'),
        cfg.StrOpt('cacert', default=None, help='Optional certificate to validate endpoint.'),
        cfg.BoolOpt('insecure', default=False, help='Allow insecure communication with Mistral.'),

        cfg.StrOpt('api_url', default=None, help=('URL Mistral uses to talk back to the API.'
            'If not provided it defaults to public API URL. Note: This needs to be a base '
            'URL without API version (e.g. http://127.0.0.1:9101)'))
    ]
    do_register_opts(mistral_opts, group='mistral', ignore_errors=ignore_errors)

    # Common CLI options
    debug = cfg.BoolOpt('debug', default=False,
        help='Enable debug mode. By default this will set all log levels to DEBUG.')
    profile = cfg.BoolOpt('profile', default=False,
        help=('Enable profile mode. In the profile mode all the MongoDB queries and related '
              'profile data are logged.'))
    use_debugger = cfg.BoolOpt('use-debugger', default=True,
        help='Enables debugger. Note that using this option changes how the '
             'eventlet library is used to support async IO. This could result in '
             'failures that do not occur under normal operation.')

    cli_opts = [debug, profile, use_debugger]
    do_register_cli_opts(cli_opts, ignore_errors=ignore_errors)

def parse_args(args=None):
    register_opts()
    cfg.CONF(args=args, version=VERSION_STRING)
<commit_msg>Let's try FQDN<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import socket
import sys

from oslo_config import cfg

from st2common.constants.system import VERSION_STRING

def do_register_opts(opts, group=None, ignore_errors=False):
    try:
        cfg.CONF.register_opts(opts, group=group)
    except:
        if not ignore_errors:
            raise

def do_register_cli_opts(opt, ignore_errors=False):
    # TODO: This function has broken name, it should work with lists :/
    if not isinstance(opt, (list, tuple)):
        opts = [opt]
    else:
        opts = opt

    try:
        cfg.CONF.register_cli_opts(opts)
    except:
        if not ignore_errors:
            raise

def register_opts(ignore_errors=False):
    rbac_opts = [
        cfg.BoolOpt('enable', default=False, help='Enable RBAC.'),
    ]
    do_register_opts(rbac_opts, 'rbac', ignore_errors)

    system_user_opts = [
        cfg.StrOpt('user',
                   default='stanley',
                   help='Default system user.'),
        cfg.StrOpt('ssh_key_file',
                   default='/home/vagrant/.ssh/stanley_rsa',
                   help='SSH private key for the system user.')
    ]
    do_register_opts(system_user_opts, 'system_user', ignore_errors)

    schema_opts = [
        cfg.IntOpt('version', default=4, help='Version of JSON schema to use.'),
        cfg.StrOpt('draft', default='http://json-schema.org/draft-04/schema#',
                   help='URL to the JSON schema draft.')
    ]
    do_register_opts(schema_opts, 'schema', ignore_errors)

    system_opts = [
        cfg.BoolOpt('debug', help='Enable debug mode.', default=False),
        cfg.StrOpt('base_path', default='/opt/stackstorm',
                   help='Base path to all st2 artifacts.')
    ]
    do_register_opts(system_opts, 'system', ignore_errors)

    system_packs_base_path = os.path.join(cfg.CONF.system.base_path, 'packs')
    content_opts = [
        cfg.StrOpt('system_packs_base_path', default=system_packs_base_path,
                   help='Path to the directory which contains system packs.'),
        cfg.StrOpt('packs_base_paths', default=None,
                   help='Paths which will be searched for integration packs.')
    ]
    do_register_opts(content_opts, 'content', ignore_errors)

    webui_opts = [
        cfg.StrOpt('webui_base_url', default='https://%s' % socket.getfqdn(),
                   help='Base https URL to access st2 Web UI. This is used to construct' +
                        'history URLs that are sent out when chatops is used to kick off ' +
                        'executions.')
    ]
    do_register_opts(webui_opts, 'webui', ignore_errors)

    db_opts = [
        cfg.StrOpt('host', default='0.0.0.0', help='host of db server'),
        cfg.IntOpt('port', default=27017, help='port of db server'),
        cfg.StrOpt('db_name', default='st2', help='name of database'),
        cfg.StrOpt('username', help='username for db login'),
        cfg.StrOpt('password', help='password for db login'),
        cfg.IntOpt('connection_retry_max_delay_m', help='Connection retry total time (minutes).',
                   default=3),
        cfg.IntOpt('connection_retry_backoff_max_s', help='Connection retry backoff max (seconds).',
                   default=10),
        cfg.IntOpt('connection_retry_backoff_mul', help='Backoff multiplier (seconds).',
                   default=1),
        cfg.BoolOpt('ssl', help='Create the connection to mongodb using SSL', default=False),
        cfg.StrOpt('ssl_keyfile',
                   help='Private keyfile used to identify the local connection against MongoDB.',
                   default=None),
        cfg.StrOpt('ssl_certfile', help='Certificate file used to identify the localconnection',
                   default=None),
        cfg.StrOpt('ssl_cert_reqs', choices='none, optional, required',
                   help='Specifies whether a certificate is required from the other side of the ' +
                        'connection, and whether it will be validated if provided',
                   default=None),
        cfg.StrOpt('ssl_ca_certs',
                   help='ca_certs file contains a set of concatenated CA certificates, which are' +
                        ' used to validate certificates passed from MongoDB.',
                   default=None),
        cfg.BoolOpt('ssl_match_hostname',
                    help='If True and `ssl_cert_reqs` is not None, enables hostname verification',
                    default=True)
    ]
    do_register_opts(db_opts, 'database', ignore_errors)

    messaging_opts = [
        # It would be nice to be able to deprecate url and completely switch to using
        # url. However, this will be a breaking change and will have impact so allowing both.
        cfg.StrOpt('url', default='amqp://guest:guest@127.0.0.1:5672//',
                   help='URL of the messaging server.'),
        cfg.ListOpt('cluster_urls', default=[],
                    help='URL of all the nodes in a messaging service cluster.')
    ]
    do_register_opts(messaging_opts, 'messaging', ignore_errors)

    syslog_opts = [
        cfg.StrOpt('host', default='127.0.0.1',
                   help='Host for the syslog server.'),
        cfg.IntOpt('port', default=514,
                   help='Port for the syslog server.'),
        cfg.StrOpt('facility', default='local7',
                   help='Syslog facility level.'),
        cfg.StrOpt('protocol', default='udp',
                   help='Transport protocol to use (udp / tcp).')
    ]
    do_register_opts(syslog_opts, 'syslog', ignore_errors)

    log_opts = [
        cfg.ListOpt('excludes', default='',
                    help='Exclusion list of loggers to omit.'),
        cfg.BoolOpt('redirect_stderr', default=False,
                    help='Controls if stderr should be redirected to the logs.'),
        cfg.BoolOpt('mask_secrets', default=True,
                    help='True to mask secrets in the log files.')
    ]
    do_register_opts(log_opts, 'log', ignore_errors)

    # Common API options
    api_opts = [
        cfg.StrOpt('host', default='0.0.0.0', help='StackStorm API server host'),
        cfg.IntOpt('port', default=9101, help='StackStorm API server port'),
        cfg.ListOpt('allow_origin', default=['http://127.0.0.1:3000'],
                    help='List of origins allowed for api, auth and stream'),
        cfg.BoolOpt('mask_secrets', default=True,
                    help='True to mask secrets in the API responses')
    ]
    do_register_opts(api_opts, 'api', ignore_errors)

    # Key Value store options
    keyvalue_opts = [
        cfg.BoolOpt('enable_encryption', default=True,
                    help='Allow encryption of values in key value stored qualified as "secret".'),
        cfg.StrOpt('encryption_key_path', default='',
                   help='Location of the symmetric encryption key for encrypting values in ' +
                        'kvstore. This key should be in JSON and should\'ve been ' +
                        'generated using keyczar.')
    ]
    do_register_opts(keyvalue_opts, group='keyvalue')

    # Common auth options
    auth_opts = [
        cfg.StrOpt('api_url', default=None,
                   help='Base URL to the API endpoint excluding the version'),
        cfg.BoolOpt('enable', default=True, help='Enable authentication middleware.'),
        cfg.IntOpt('token_ttl', default=86400, help='Access token ttl in seconds.')
    ]
    do_register_opts(auth_opts, 'auth', ignore_errors)

    # Common action runner options
    default_python_bin_path = sys.executable
    base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
    default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv')
    action_runner_opts = [
        cfg.StrOpt('logging', default='conf/logging.conf',
                   help='location of the logging.conf file'),
        cfg.StrOpt('python_binary', default=default_python_bin_path,
                   help='Python binary which will be used by Python actions.'),
        cfg.StrOpt('virtualenv_binary', default=default_virtualenv_bin_path,
                   help='Virtualenv binary which should be used to create pack virtualenvs.'),
        cfg.ListOpt('virtualenv_opts', default=['--system-site-packages'],
                    help='List of virtualenv options to be passsed to "virtualenv" command that ' +
                         'creates pack virtualenv.')
    ]
    do_register_opts(action_runner_opts, group='actionrunner')

    # Common options (used by action runner and sensor container)
    action_sensor_opts = [
        cfg.BoolOpt('enable', default=True,
                    help='Whether to enable or disable the ability to post a trigger on action.'),
    ]
    do_register_opts(action_sensor_opts, group='action_sensor')

    # Coordination options
    coord_opts = [
        cfg.StrOpt('url', default=None, help='Endpoint for the coordination server.'),
        cfg.IntOpt('lock_timeout', default=60, help='TTL for the lock if backend suports it.')
    ]
    do_register_opts(coord_opts, 'coordination', ignore_errors)

    # Mistral options
    mistral_opts = [
        cfg.StrOpt('v2_base_url', default='http://127.0.0.1:8989/v2', help='v2 API root endpoint.'),
        cfg.IntOpt('retry_exp_msec', default=1000, help='Multiplier for the exponential backoff.'),
        cfg.IntOpt('retry_exp_max_msec', default=300000, help='Max time for each set of backoff.'),
        cfg.IntOpt('retry_stop_max_msec', default=600000, help='Max time to stop retrying.'),
        cfg.StrOpt('keystone_username', default=None, help='Username for authentication.'),
        cfg.StrOpt('keystone_password', default=None, help='Password for authentication.'),
        cfg.StrOpt('keystone_project_name', default=None, help='OpenStack project scope.'),
        cfg.StrOpt('keystone_auth_url', default=None, help='Auth endpoint for Keystone.'),
        cfg.StrOpt('cacert', default=None, help='Optional certificate to validate endpoint.'),
        cfg.BoolOpt('insecure', default=False, help='Allow insecure communication with Mistral.'),

        cfg.StrOpt('api_url', default=None, help=('URL Mistral uses to talk back to the API.'
            'If not provided it defaults to public API URL. Note: This needs to be a base '
            'URL without API version (e.g. http://127.0.0.1:9101)'))
    ]
    do_register_opts(mistral_opts, group='mistral', ignore_errors=ignore_errors)

    # Common CLI options
    debug = cfg.BoolOpt('debug', default=False,
        help='Enable debug mode. By default this will set all log levels to DEBUG.')
    profile = cfg.BoolOpt('profile', default=False,
        help=('Enable profile mode. In the profile mode all the MongoDB queries and related '
              'profile data are logged.'))
    use_debugger = cfg.BoolOpt('use-debugger', default=True,
        help='Enables debugger. Note that using this option changes how the '
             'eventlet library is used to support async IO. This could result in '
             'failures that do not occur under normal operation.')

    cli_opts = [debug, profile, use_debugger]
    do_register_cli_opts(cli_opts, ignore_errors=ignore_errors)

def parse_args(args=None):
    register_opts()
    cfg.CONF(args=args, version=VERSION_STRING)
<|endoftext|>


<commit_before>import re
import traceback

def parseUserAgent(useragent):
    uasplit = splitUserAgent(useragent)
    if not uasplit:
        return {}

    if not seemsLikeANormalUA(useragent):
        return None#{'text': useragent}

    parsed = {}
    url = getURLFromUA(useragent)
    if url is not None:
        parsed.update({'url': url})

    agent = splitVersion(uasplit[0].strip(';'))

    if len(uasplit) == 1:
        parsed.update({'browser': agent})
        return parsed

    elif agent[0] == 'Mozilla':
        platform = uasplit[1].strip('()').split('; ')
        #if platform[0] == 'compatible':
        #    return parseUserAgent('; '.join(platform[1:])) # very hacky!
        info = uasplit[2:]
        try:
            new = parseMozilla(platform, info)
            if new is not None:
                parsed.update(new)
                return parsed
            else:
                return None
        except:
            traceback.print_exc()
            return None

    elif agent[0] == 'curl':
        parsed.update({'browser': splitVersion(uasplit[0]), 'os': uasplit[1].strip('()')})
        return parsed
    else:
        parsed.update({'browser': agent})
        return parsed

    #return parsed
    return None

def seemsLikeANormalUA(useragent):
    try:
        useragent.encode('ascii')
    except UnicodeDecodeError:
        return False
    if len(useragent.split()) == 1:
        return True
    if re.search('\.[0-9]', useragent):
        return True
    points = 0
    if getURLFromUA(useragent) is not None:
        points += 1
    if '; ' in useragent:
        points += 1
    if re.search('\([^)]+\)', useragent):
        points += 1
    if re.search('[0-9]\.[0-9]', useragent):
        points += 1
    if re.search('^[A-Z][a-z]', useragent):
        points += 1
    return points >= 3

def getURLFromUA(useragent):
    if 'http://' in useragent:
        url_start = useragent[useragent.find('http://'):]
    elif 'https://' in useragent:
        url_start = useragent[useragent.find('https://'):]
    else:
        return None
    url = url_start.split()[0].strip(')').strip(',')
    return url

def parseMozilla(platform, info):
    if platform[0] in ['compatible', 'Mobile']:
        if platform[1].startswith('MSIE'):
            browser = ('Internet Explorer', version(platform[1].split()[1]))
            os_string = platform[2]
            osname = os_string.rsplit(' ', 1)[0]
            osver = version(os_string.split()[-1])
            if osname == 'Windows Phone OS':
                osname = 'Windows Phone'
            if osname == 'Windows Phone':
                device = '%s %s' % (platform[-2], platform[-1])
                return {'browser': browser, 'os': (osname, osver), 'device': device}
            else:
                return {'browser': browser, 'os': (osname, osver)}
        else:
            return {'browser': splitVersion(platform[1])}

    if len(info) < 2:
        return None

    # Firefox
    if info[1].startswith('Firefox/') and info[0].startswith('Gecko/'):
        browser = splitVersion(info[-1]) # Gecko/20100101 Firefox/34.0.1 Waterfox/34.0
        revision = platform.pop()
        client = {'browser': browser}
        if revision.startswith('rv:'):
            os = None
            if platform[0] == 'Android':
                os = ('Android', [])
            elif platform[0] == 'Macintosh':
                osname = 'Mac OS X'
                h = re.match(r'Intel Mac OS X ([0-9_]*)', platform[1])
                if h is not None:
                    osver = h.group(1)
                else:
                    osver = ''
                os = (osname, version(osver))
            elif platform[0] == 'X11':
                os = (platform[1].split()[0], [])
            elif platform == ['Maemo', 'Linux', 'U', 'Jolla', 'Sailfish', 'Mobile']:
                os = ('Jolla', [])
            elif platform[0].startswith('Windows'):
                if platform[0] == 'Windows':
                    platform.pop(0)
                if platform[0] == 'U':
                    platform.pop(0)
                os_string = platform.pop(0)
                osname = os_string.rsplit(' ', 1)[0]
                osver = version(os_string.split()[-1])
                os = (osname, osver)
            else:
                print(platform)
            if os is not None:
                client.update({'os': os})
            #return 'bot', browser
        return client
        #return platform, browser
    if platform[0] in ['iPad', 'iPhone', 'iPod', 'iPod touch']:
        osname = 'iOS'
        osver = re.match(r'CPU (iPhone )?OS ([0-9_]*) like Mac OS X', platform[-1]).group(2)
        os = (osname, version(osver))
        device = platform[0]
        browser = parseWebKitTail(info)
        if browser is not None:
            bname, bver = browser
            if bname == 'Mobile':
                browser = 'unknown'
                # TODO how should we differentiate it is not possible to know
                # and we could not parse it
            else:
                if bname == 'Version':
                    bname = 'Safari'
                elif bname == 'CriOS':
                    bname = 'Chrome'
                browser = (bname, bver)
        #return
        return {'device': device, 'os': os, 'browser': browser}
    if platform[0] == 'Macintosh':
        osname = 'Mac OS X'
        h = re.match(r'Intel Mac OS X ([0-9_]*)', platform[-1])
        if h is not None:
            osver = h.group(1)
        else:
            return None
        os = (osname, version(osver))
        device = platform[0]
        browser = parseWebKitTail(info)
        if browser is not None:
            bname, bver = browser
            if bname == 'Version':
                bname = 'Safari'
            browser = (bname, bver)
        #return
        return {'device': device, 'os': os, 'browser': browser}
    if platform[0] == 'Linux': # android
        # in rare cases, Build code is informative
        # 'Android 4.4.2', 'LG-D620 Build/KOT49I.A1401238405'
        # 'Android 4.4.2', 'LG-D620 Build/KOT49I.A1404447153'
        if platform[1] == 'U':
            platform.pop(1)
        osname, osver = platform[1].split()
        osver = version(osver)
        os = (osname, osver)
        if ' ' in platform[-1]:
            device, build = platform[-1].rsplit(' ', 1)
        else:
            device = None
        browser = parseWebKitTail(info)
        if browser is not None:
            bname, bver = browser
            if bname == 'Version':
                bname = 'Android browser'
            browser = (bname, bver)
        #return
        return {'device': device, 'os': os, 'browser': browser }

    if platform[0].startswith('Windows'):
        if platform[0] == 'Windows':
            platform.pop(0)
        if platform[0] == 'U':
            platform.pop(0)
        os_string = platform.pop(0)
        osname = os_string.rsplit(' ', 1)[0]
        osver = version(os_string.split()[-1])
        os = (osname, osver)
        if 'Trident/7.0' in platform:
            browser = ('Internet Explorer', [11])
        else:
            browser = parseWebKitTail(info)
        if osname == 'Windows Phone':
            device = '%s %s' % (platform[-2], platform[-1])
            return {'browser': browser, 'os': os, 'device': device}
        else:
            return {'browser': browser, 'os': os}

    if platform[0] == 'MeeGo':
        browser = parseWebKitTail(info)
        device = platform[1]
        os = splitVersion(platform[0])
        return {'browser': browser, 'os': os, 'device': device}

    if platform[0].startswith('Symbian'):
        browser = parseWebKitTail(info)
        device = platform[1].split()[-1].split('/')[0]
        os = splitVersion(platform[0])
        return {'browser': browser, 'os': os, 'device': device}

    if platform[0] == 'X11':
        if platform[1] == 'U':
            platform.pop(1)
        osname = platform[1].split()[0]
        if osname == 'Linux':
            # TODO architecture
            browser = parseWebKitTail(info)
            return {'browser': browser, 'os': (osname, [])}
        elif osname == 'CrOS':
            osver = version(platform[1].split()[-1])
            browser = parseWebKitTail(info)
            return {'browser': browser, 'os': (osname, osver)}

    return# info
    #return platform, info

def parseWebKitTail(info):
    browser_name = None
    browser_version = None
    if len(info) < 2:
        return # unknown
    applewebkit = info.pop(0) # different versions
    khtml = info.pop(0)
    if not applewebkit.startswith('AppleWebKit/') or khtml != '(KHTML, like Gecko)':
        return
    if len(info) == 1:
        return splitVersion(info[0])
    fb = parseFacebookUAString(info[-1])
    if fb is not None:
        browser_version = version(fb['AV'])
        # different devices
        app_name = fb.get('AN')
        if app_name == 'MessengerForiOS':
            browser_name = 'Facebook Messenger'
        elif app_name == 'FBIOS':
            browser_name = 'Facebook App' # iOS
        else:
            if fb.get('_IAB') == 'FB4A':
                browser_name = 'Facebook App' # Android
            else:
                browser_name = 'Facebook (?)'
        return browser_name, browser_version

    # TODO: this function should probably know something about the operating system...

    priority = ['Mobile', 'Safari', 'Chrome', 'Version']
    browser = None
    best_priority = -1
    for i in info:
        name, ver = splitVersion(i)
        if not name in priority:
            best_priority = len(priority)
            browser = (name, ver)
            if ver:
                break
        elif priority.index(name) > best_priority:
            browser = (name, ver)
            best_priority = priority.index(name)

    return browser

def parseFacebookUAString(fbstr):
    if fbstr[0] != '[' or fbstr[-1] != ']':
        return
    fbstr = fbstr[1:-1].strip(';')
    dictionary = {}
    for pair in fbstr.split(';'):
        pair = pair.strip(' ')
        if not pair.startswith('FB'):
            return
        if pair.count('/') != 1:
            return
        key, value = pair.split('/')
        dictionary.update({key[2:]: value})
    return dictionary

def splitUserAgent(string):
    # "foo (bar foo) bar" -> ["foo", "(bar foo)", "bar"]
    return re.findall(r'(\([^()]+\)|\[[^\[\]]+\]|[^ ()]+)', string)

def splitVersion(string):
    if not '/' in string:
        return string, []
    #name, ver = string.split('/') # not tested
    name = string.split('/')[0]
    ver = string.split('/')[1]
    return name, version(ver)

def version(version):
    version = re.match(r'[0-9_.]*', version).group(0)
    if not version:
        return None
    #version = version.split('-')[0] # Google-HTTP-Java-Client/1.17.0-rc
    numbers = re.split(r'_|\.', version)
    return [int(x) for x in numbers]

if __name__ == '__main__':

    uafile = '/dev/stdin'

    for line in open(uafile):
        ua = line.strip('\n')
        parsed = parseUserAgent(ua)
        #print(repr(ua))
        if parsed is not None:
            print(parsed)
            0
        else:
            #print(ua)
            0
        #if parsed is not None:
            #print(parsed)
        #if parsed:
            #print(ua)
       #     print(parsed)
            #print()
<commit_msg>Fix windows phone 8.1<commit_after>import re
import traceback

def parseUserAgent(useragent):
    uasplit = splitUserAgent(useragent)
    if not uasplit:
        return {}

    if not seemsLikeANormalUA(useragent):
        return None#{'text': useragent}

    parsed = {}
    url = getURLFromUA(useragent)
    if url is not None:
        parsed.update({'url': url})

    agent = splitVersion(uasplit[0].strip(';'))

    if len(uasplit) == 1:
        parsed.update({'browser': agent})
        return parsed

    elif agent[0] == 'Mozilla':
        platform = uasplit[1].strip('()').split('; ')
        #if platform[0] == 'compatible':
        #    return parseUserAgent('; '.join(platform[1:])) # very hacky!
        info = uasplit[2:]
        try:
            new = parseMozilla(platform, info)
            if new is not None:
                parsed.update(new)
                return parsed
            else:
                return None
        except:
            traceback.print_exc()
            return None

    elif agent[0] == 'curl':
        parsed.update({'browser': splitVersion(uasplit[0]), 'os': uasplit[1].strip('()')})
        return parsed
    else:
        parsed.update({'browser': agent})
        return parsed

    #return parsed
    return None

def seemsLikeANormalUA(useragent):
    try:
        useragent.encode('ascii')
    except UnicodeDecodeError:
        return False
    if len(useragent.split()) == 1:
        return True
    if re.search('\.[0-9]', useragent):
        return True
    points = 0
    if getURLFromUA(useragent) is not None:
        points += 1
    if '; ' in useragent:
        points += 1
    if re.search('\([^)]+\)', useragent):
        points += 1
    if re.search('[0-9]\.[0-9]', useragent):
        points += 1
    if re.search('^[A-Z][a-z]', useragent):
        points += 1
    return points >= 3

def getURLFromUA(useragent):
    if 'http://' in useragent:
        url_start = useragent[useragent.find('http://'):]
    elif 'https://' in useragent:
        url_start = useragent[useragent.find('https://'):]
    else:
        return None
    url = url_start.split()[0].strip(')').strip(',')
    return url

# TODO email

def parseMozilla(platform, info):
    if platform[0] == 'compatible':
        if platform[1].startswith('MSIE'):
            browser = ('Internet Explorer', version(platform[1].split()[1]))
            os_string = platform[2]
            osname = os_string.rsplit(' ', 1)[0]
            osver = version(os_string.split()[-1])
            if osname == 'Windows Phone OS':
                osname = 'Windows Phone'
            if osname == 'Windows Phone':
                device = '%s %s' % (platform[-2], platform[-1])
                return {'browser': browser, 'os': (osname, osver), 'device': device}
            else:
                return {'browser': browser, 'os': (osname, osver)}
        else:
            return {'browser': splitVersion(platform[1])}

    if len(info) < 2:
        return None

    # Firefox
    if info[1].startswith('Firefox/') and info[0].startswith('Gecko/'):
        browser = splitVersion(info[-1]) # Gecko/20100101 Firefox/34.0.1 Waterfox/34.0
        revision = platform.pop()
        client = {'browser': browser}
        if revision.startswith('rv:'):
            os = None
            if platform[0] == 'Android':
                os = ('Android', [])
            elif platform[0] == 'Macintosh':
                osname = 'Mac OS X'
                h = re.match(r'Intel Mac OS X ([0-9_]*)', platform[1])
                if h is not None:
                    osver = h.group(1)
                else:
                    osver = ''
                os = (osname, version(osver))
            elif platform[0] == 'X11':
                os = (platform[1].split()[0], [])
            elif platform == ['Maemo', 'Linux', 'U', 'Jolla', 'Sailfish', 'Mobile']:
                os = ('Jolla', [])
            elif platform[0].startswith('Windows'):
                if platform[0] == 'Windows':
                    platform.pop(0)
                if platform[0] == 'U':
                    platform.pop(0)
                os_string = platform.pop(0)
                osname = os_string.rsplit(' ', 1)[0]
                osver = version(os_string.split()[-1])
                os = (osname, osver)
            else:
                print(platform)
            if os is not None:
                client.update({'os': os})
            #return 'bot', browser
        return client
        #return platform, browser

    if platform[0] == 'Mobile':
        platform.pop(0)

    if platform[0] in ['iPad', 'iPhone', 'iPod', 'iPod touch']:
        osname = 'iOS'
        osver = re.match(r'CPU (iPhone )?OS ([0-9_]*) like Mac OS X', platform[-1]).group(2)
        os = (osname, version(osver))
        device = platform[0]
        browser = parseWebKitTail(info)
        if browser is not None:
            bname, bver = browser
            if bname == 'Mobile':
                browser = 'unknown'
                # TODO how should we differentiate it is not possible to know
                # and we could not parse it
            else:
                if bname == 'Version':
                    bname = 'Safari'
                elif bname == 'CriOS':
                    bname = 'Chrome'
                browser = (bname, bver)
        #return
        return {'device': device, 'os': os, 'browser': browser}
    if platform[0] == 'Macintosh':
        osname = 'Mac OS X'
        h = re.match(r'Intel Mac OS X ([0-9_]*)', platform[-1])
        if h is not None:
            osver = h.group(1)
        else:
            return None
        os = (osname, version(osver))
        device = platform[0]
        browser = parseWebKitTail(info)
        if browser is not None:
            bname, bver = browser
            if bname == 'Version':
                bname = 'Safari'
            browser = (bname, bver)
        #return
        return {'device': device, 'os': os, 'browser': browser}
    if platform[0] == 'Linux': # android
        # in rare cases, Build code is informative
        # 'Android 4.4.2', 'LG-D620 Build/KOT49I.A1401238405'
        # 'Android 4.4.2', 'LG-D620 Build/KOT49I.A1404447153'
        if platform[1] == 'U':
            platform.pop(1)
        osname, osver = platform[1].split()
        osver = version(osver)
        os = (osname, osver)
        if ' ' in platform[-1]:
            device, build = platform[-1].rsplit(' ', 1)
        else:
            device = None
        browser = parseWebKitTail(info)
        if browser is not None:
            bname, bver = browser
            if bname == 'Version':
                bname = 'Android browser'
            browser = (bname, bver)
        #return
        return {'device': device, 'os': os, 'browser': browser }

    if platform[0].startswith('Windows'):
        if platform[0] == 'Windows':
            platform.pop(0)
        if platform[0] == 'U':
            platform.pop(0)
        os_string = platform.pop(0)
        osname = os_string.rsplit(' ', 1)[0]
        osver = version(os_string.split()[-1])
        os = (osname, osver)
        if 'Trident/7.0' in platform:
            browser = ('Internet Explorer', [11])
        else:
            browser = parseWebKitTail(info)
        if osname == 'Windows Phone':
            device = '%s %s' % (platform[-2], platform[-1])
            return {'browser': browser, 'os': os, 'device': device}
        else:
            return {'browser': browser, 'os': os}

    if platform[0] == 'MeeGo':
        browser = parseWebKitTail(info)
        device = platform[1]
        os = splitVersion(platform[0])
        return {'browser': browser, 'os': os, 'device': device}

    if platform[0].startswith('Symbian'):
        browser = parseWebKitTail(info)
        device = platform[1].split()[-1].split('/')[0]
        os = splitVersion(platform[0])
        return {'browser': browser, 'os': os, 'device': device}

    if platform[0] == 'X11':
        if platform[1] == 'U':
            platform.pop(1)
        osname = platform[1].split()[0]
        if osname == 'Linux':
            # TODO architecture
            browser = parseWebKitTail(info)
            return {'browser': browser, 'os': (osname, [])}
        elif osname == 'CrOS':
            osver = version(platform[1].split()[-1])
            browser = parseWebKitTail(info)
            return {'browser': browser, 'os': (osname, osver)}

    return# info
    #return platform, info

def parseWebKitTail(info):
    browser_name = None
    browser_version = None
    if len(info) < 2:
        return # unknown
    applewebkit = info.pop(0) # different versions
    khtml = info.pop(0)
    if not applewebkit.startswith('AppleWebKit/') or khtml != '(KHTML, like Gecko)':
        return
    if len(info) == 1:
        return splitVersion(info[0])
    fb = parseFacebookUAString(info[-1])
    if fb is not None:
        browser_version = version(fb['AV'])
        # different devices
        app_name = fb.get('AN')
        if app_name == 'MessengerForiOS':
            browser_name = 'Facebook Messenger'
        elif app_name == 'FBIOS':
            browser_name = 'Facebook App' # iOS
        else:
            if fb.get('_IAB') == 'FB4A':
                browser_name = 'Facebook App' # Android
            else:
                browser_name = 'Facebook (?)'
        return browser_name, browser_version

    # TODO: this function should probably know something about the operating system...

    priority = ['Mobile', 'Safari', 'Chrome', 'Version']
    browser = None
    best_priority = -1
    for i in info:
        name, ver = splitVersion(i)
        if not name in priority:
            best_priority = len(priority)
            browser = (name, ver)
            if ver:
                break
        elif priority.index(name) > best_priority:
            browser = (name, ver)
            best_priority = priority.index(name)

    return browser

def parseFacebookUAString(fbstr):
    if fbstr[0] != '[' or fbstr[-1] != ']':
        return
    fbstr = fbstr[1:-1].strip(';')
    dictionary = {}
    for pair in fbstr.split(';'):
        pair = pair.strip(' ')
        if not pair.startswith('FB'):
            return
        if pair.count('/') != 1:
            return
        key, value = pair.split('/')
        dictionary.update({key[2:]: value})
    return dictionary

def splitUserAgent(string):
    # "foo (bar foo) bar" -> ["foo", "(bar foo)", "bar"]
    return re.findall(r'(\([^()]+\)|\[[^\[\]]+\]|[^ ()]+)', string)

def splitVersion(string):
    if not '/' in string:
        return string, []
    #name, ver = string.split('/') # not tested
    name = string.split('/')[0]
    ver = string.split('/')[1]
    return name, version(ver)

def version(version):
    version = re.match(r'[0-9_.]*', version).group(0)
    if not version:
        return None
    #version = version.split('-')[0] # Google-HTTP-Java-Client/1.17.0-rc
    numbers = re.split(r'_|\.', version)
    return [int(x) for x in numbers]

if __name__ == '__main__':

    uafile = '/dev/stdin'

    for line in open(uafile):
        ua = line.strip('\n')
        parsed = parseUserAgent(ua)
        #print(repr(ua))
        if parsed is not None:
            #print(ua + '\t' + repr(parsed))
            #print(parsed)
            0
        else:
            print(ua)
            0
        #if parsed is not None:
            #print(parsed)
        #if parsed:
            #print(ua)
       #     print(parsed)
            #print()
<|endoftext|>


<commit_before>
import serial, time

class Communications(object):

    BAUD = 115200
    PORT = '/dev/ttyS1'
    TERM = '\n'
    TEST_MSG = 'Huston we have a problem'

    def __init__(s):

        s.port = serial.Serial(s.PORT, s.BAUD)

        print 'Connected to', s.port.portstr

    def testMaster(s):
        print 'Testing coms'
        msg = s.read()
        print 'Master recieved', msg
        time.sleep(.5)
        s.write(msg)
        print 'Master sent', msg
        print 'Communincations spec out'

    def testSlave(s):
        print 'Testing coms'
        msg = s.TEST_MSG
        s.write(msg)
        print 'Slave sent', msg
        rsp = s.read()
        print 'Slave recieved', rsp
        print 'Communincations spec out'

    def writeDict(s, d):
        print 'Coms write', d
        s.port.write(repr(d))

    def readDict(s):
        d = ''
        while len(d) <= 1:
            d = s.port.readline().strip()
        print 'Coms read', d
        return eval(d)

    def writeByte(s, b):
        print 'Coms write', b
        s.port.write(str(b))

    def readByte(s):
        b = s.port.read()
        print 'Coms read', b
        return b

    def write(s, msg):
        s.port.write(msg + s.TERM)

    def read(s):
        return s.port.readline()
<commit_msg>Adding termination<commit_after>
import serial, time

class Communications(object):

    BAUD = 115200
    PORT = '/dev/ttyS1'
    TERM = '\n'
    TEST_MSG = 'Huston we have a problem'

    def __init__(s):

        s.port = serial.Serial(s.PORT, s.BAUD)

        print 'Connected to', s.port.portstr

    def testMaster(s):
        print 'Testing coms'
        msg = s.read()
        print 'Master recieved', msg
        time.sleep(.5)
        s.write(msg)
        print 'Master sent', msg
        print 'Communincations spec out'

    def testSlave(s):
        print 'Testing coms'
        msg = s.TEST_MSG
        s.write(msg)
        print 'Slave sent', msg
        rsp = s.read()
        print 'Slave recieved', rsp
        print 'Communincations spec out'

    def writeDict(s, d):
        print 'Coms write', d
        s.port.write(repr(d) + s.TERM)

    def readDict(s):
        d = ''
        i = 0
        while len(d) <= 1:
            i += 1
            d = s.port.readline().strip()
            print 'Coms read', i, d
        return eval(d)

    def writeByte(s, b):
        print 'Coms write', b
        s.port.write(str(b))

    def readByte(s):
        b = s.port.read()
        print 'Coms read', b
        return b

    def write(s, msg):
        s.port.write(msg + s.TERM)

    def read(s):
        return s.port.readline()
<|endoftext|>


<commit_before>
# Template XML requests required by the USGS Inventory Service
# Requesting data like it's 1999

from xml.etree.ElementTree import Element, SubElement, tostring
from usgs import CATALOG_NODES, USGSApiKeyRequiredError, USGSCatalogNodeDoesNotExist

def create_root_request():
    root = Element("soapenv:Envelope")
    root.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
    root.set("xmlns:xsd", "http://www.w3.org/2001/XMLSchema")
    root.set("xmlns:soapenv", "http://schemas.xmlsoap.org/soap/envelope/")
    root.set("xmlns:soap", "http://earthexplorer.usgs.gov/inventory/soap")

    header = SubElement(root, "soapenv:Header")
    body = SubElement(root, "soapenv:Body")

    return (root, body)

def create_request_type(parent, request_type):
    el = SubElement(parent, "soap:%s" % request_type)
    el.set("soapenv:encodingStyle", "http://schemas.xmlsoap.org/soap/encoding/")
    return el

def create_api_key_element(parent, api_key):
    el = SubElement(parent, "apiKey")
    el.set("xsi:type", "xsd:string")
    el.text = api_key

def create_node_element(parent, node):

    if node not in CATALOG_NODES:
        raise USGSCatalogNodeDoesNotExist("Catalog nodes include %s" % ", ".join(CATALOG_NODES))

    el = SubElement(parent, "node")
    el.set("xsi:type", "xsd:string")
    el.text = node

def create_dataset_element(parent, dataset):
    el = SubElement(parent, "datasetName")
    el.set("xsi:type", "xsd:string")
    el.text = dataset

def create_entity_ids_element(parent, entityids):

    if isinstance(entityids, str):

        el = SubElement(parent, "entityId")
        el.set("xsi:type", "xsd:string")
        el.text = entityids

    else:

        el = SubElement(parent, "entityIds")
        el.set("xsi:type", "soap:ArrayOfString")

        for entityid in entityids:
            child = SubElement(el, "item")
            child.set("xsi:type","xsd:string")
            child.text = entityid

def create_service_class_coordinate(parent, name, latitude=None, longitude=None):

    el = SubElement(parent, name)
    el.set("xsi:type", "soap:Service_Class_Coordinate")

    lat_el = SubElement(el, "latitude")
    lat_el.set("xsi:type", "xsd:double")
    lat_el.text = str(latitude)

    lng_el = SubElement(el, "longitude")
    lng_el.set("xsi:type", "xsd:double")
    lng_el.text = str(longitude)

def clear_bulk_download_order(dataset, node, api_key=None):
    """
    This method is used to clear bulk download order information from the item basket.

    :param dataset:

    :param node:

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "clearBulkDownloadOrder")

    create_node_element(el, node)
    create_dataset_element(el, dataset)
    create_api_key_element(el, api_key)

    return tostring(root)

def clear_order(dataset, node, api_key=None):
    """
    This method is used to clear order information from the item basket.

    :param dataset:

    :param node:

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "clearOrder")

    create_node_element(el, node)
    create_dataset_element(el, dataset)
    create_api_key_element(el, api_key)

    return tostring(root)

def datasets(dataset, node, ll=None, ur=None, start_date=None, end_date=None, api_key=None):
    """
    This method is used to find datasets available for searching.
    By passing no parameters except node, all available datasets
    are returned. Additional parameters such as temporal range
    and spatial bounding box can be used to find datasets that
    provide more specific data. The dataset name parameter can
    be used to limit the results based on matching the supplied
    value against the dataset name with assumed wildcards at the
    beginning and end. All parameters are optional except for
    the 'node' parameter.

    :param dataset:
        Dataset Identifier

    :param ll:
        Lower left corner of an AOI bounding box - in decimal form
        Longitude/Latitude dictionary

        e.g. { "longitude": 0.0, "latitude": 0.0 }

    :param ur:
        Upper right corner of an AOI bounding box - in decimal form
        Longitude/Latitude dictionary

        e.g. { "longitude": 0.0, "latitude": 0.0 }

    :param start_date:
        Used for searching scene acquisition - will accept anything
        that the PHP strtotime function can understand

    :param end_date:
        Used for searching scene acquisition - will accept anything
        that the PHP strtotime function can understand

    :param node:
        The requested Catalog

    :param api_key:
        API key is not required.

    """

    root, body = create_root_request()

    el = create_request_type(body, "datasets")

    create_node_element(el, node)
    if dataset:
        create_dataset_element(el, dataset)

    if api_key:
        create_api_key_element(el, api_key)

    if ll and ur:

        create_service_class_coordinate(el, "lowerLeft", latitude=ll["latitude"], longitude=ll["longitude"])
        create_service_class_coordinate(el, "upperRight", latitude=ur["latitude"], longitude=ur["longitude"])

    if start_date:

        start_date_el = SubElement(el, "startDate")
        start_date_el.set("xsi:type", "xsd:string")
        start_date_el.text = start_date

    if end_date:

        end_date_el = SubElement(el, "endDate")
        end_date_el.set("xsi:type", "xsd:string")
        end_date_el.text = end_date

    return tostring(root)

def dataset_fields(dataset, node, api_key=None):
    """
    This request is used to return the metadata filter
    fields for the specified dataset. These values can
    be used as additional criteria when submitting search
    and hit queries.

    :param dataset:

    :param node:

    :param api_key:
        API key is not required.
    """

    root, body = create_root_request()

    el = create_request_type(body, "datasetFields")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    if api_key:
        create_api_key_element(el, api_key)

    return tostring(root)

def download(dataset, node, entityids, products, api_key=None):
    """
    The use of this request will be to obtain valid data download URLs.

    :param dataset:

    :param entityIds:
        list

    :param products:
        list

    :param node:

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "download")

    create_dataset_element(el, dataset)
    create_node_element(el, node)
    create_api_key_element(el, api_key)
    create_entity_ids_element(el, entityids)

    products_el = SubElement(el, "products")
    products_el.set("xsi:type", "soap:ArrayOfString")

    for product in products:
        product_el = SubElement(products_el, "item")
        product_el.set("xsi:type","xsd:string")
        product_el.text = product

    return tostring(root)

def download_options(dataset, node, entityids, api_key=None):
    """
    The use of the download options request is to discover the different download
    options for each scene. Some download options may exist but still be unavailable
    due to disk usage and many other factors. If a download is unavailable
    it may need to be ordered.

    :param dataset:

    :param node:

    :param entityIds:

    :param api_key:
        API key is not required.
    """

    root, body = create_root_request()

    el = create_request_type(body, "downloadOptions")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    if api_key:
        create_api_key_element(el, api_key)

    create_entity_ids_element(el, entityids)

    return tostring(root)

def get_bulk_download_products(dataset, node, entityids, api_key):
    """
    Retrieve bulk download products on a scene-by-scene basis.

    :param dataset:

    :param node:

    :param entityid:
        String or list of strings.

    :param api_key:
        API key is required.

    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "getBulkDownloadProducts")

    create_dataset_element(el, dataset)
    create_node_element(el, node)
    create_api_key_element(el, api_key)
    create_entity_ids_element(el, entityids)

    return tostring(root)

def get_order_products(dataset, node, entityids, api_key=None):
    """
    Retrieve orderable products on a scene-by-scene basis.

    :param dataset:

    :param node:

    :param entityid:

    :param api_key:
        API key is required.

    .. todo:: Support multiple scene request.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "getOrderProducts")

    create_dataset_element(el, dataset)
    create_node_element(el, node)
    create_api_key_element(el, api_key)
    create_entity_ids_element(el, entityids)

    return tostring(root)

def item_basket(api_key=None):
    """
    Returns the current item basket for the current user.

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "itemBasket")
    create_api_key_element(el, api_key)

    return tostring(root)

def login(username, password):
    """
    This method requires SSL be used due to the sensitive nature of
    users passwords. Upon a successful login, an API key will be
    returned. This key will be active for one hour and should be
    destroyed upon final use of the service by calling the logout
    method. Users must have "Machine to Machine" access based on
    a user-based role in the users profile.

    :param username:

    :param password:
    """

    root, body = create_root_request()

    el = create_request_type(body, "login")

    username_el = SubElement(el, "username")
    username_el.set("xsi:type", "xsd:string")
    username_el.text = username

    password_el = SubElement(el, "password")
    password_el.set("xsi:type", "xsd:string")
    password_el.text = password

    return tostring(root)

def logout(api_key=None):
    """
    Remove the users API key from being used in the future.

    :param api_key:
        API key is required.
    """

    root, body = create_root_request()

    el = create_request_type(body, "logout")
    create_api_key_element(el, api_key)

    return tostring(root)

def metadata(dataset, node, sceneids, api_key=None):
    """
    The use of the metadata request is intended for those who have
    acquired scene IDs from a different source. It will return the
    same metadata that is available via the search request.

    :param dataset:

    :param node:

    :param sceneid:

    :param api_key:
    """
    root, body = create_root_request()

    el = create_request_type(body, "metadata")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    if api_key:
        create_api_key_element(el, api_key)

    create_entity_ids_element(el, sceneids)

    return tostring(root)

def remove_bulk_download_scene():
    raise NotImplementedError

def remove_order_scene():
    raise NotImplementedError

def search(dataset, node, lat=None, lng=None, distance=100, ll=None, ur=None, start_date=None, end_date=None, max_results=50000, starting_number=1, sort_order="DESC", api_key=None):
    """
    :param dataset:

    :param node:

    :param lat:

    :param lng:

    :param ll:

    :param distance:

    :param ur:

    :param start_date:

    :param end_date:

    :param max_results:

    :param starting_number:

    :param sort_order:

    :param api_key:
        API key is not required.
    """
    root, body = create_root_request()

    el = create_request_type(body, "search")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    # Latitude and longitude take precedence over ll and ur
    if lat and lng:

        try:
            import pyproj
            from shapely import geometry
        except ImportError:
            raise USGSDependencyRequired("Shapely and PyProj are required for search.")

        prj = pyproj.Proj(proj='aeqd', lat_0=lat, lon_0=lng)
        half_distance = 0.5 * distance
        box = geometry.box(-half_distance, -half_distance, half_distance, half_distance)

        lats, lngs = prj(*box.exterior.xy, inverse=True)

        ll = { "longitude": min(*lngs), "latitude": min(*lats) }
        ur = { "longitude": max(*lngs), "latitude": max(*lats) }

    if ll and ur:

        create_service_class_coordinate(el, "lowerLeft", latitude=ll["latitude"], longitude=ll["longitude"])
        create_service_class_coordinate(el, "upperRight", latitude=ur["latitude"], longitude=ur["longitude"])

    if start_date:

        start_date_el = SubElement(el, "startDate")
        start_date_el.set("xsi:type", "xsd:string")
        start_date_el.text = start_date

    if end_date:

        end_date_el = SubElement(el, "endDate")
        end_date_el.set("xsi:type", "xsd:string")
        end_date_el.text = end_date

    if max_results:

        max_results_el = SubElement(el, "maxResults")
        max_results_el.set("xsi:type", "xsd:int")
        max_results_el.text = str(max_results)

    if starting_number:

        starting_number_el = SubElement(el, "startingNumber")
        starting_number_el.set("xsi:type", "xsd:int")
        starting_number_el.text = str(starting_number)

    if sort_order:

        sort_order_el = SubElement(el, "sortOrder")
        sort_order_el.set("xsi:type", "xsd:string")
        sort_order_el.text = sort_order

    if api_key:
        create_api_key_element(el, api_key)

    return tostring(root)

def submit_bulk_order():
    raise NotImplementedError

def submit_order():
    raise NotImplementedError

def update_bulk_download_scene():
    raise NotImplementedError

def update_order_scene():
    raise NotImplementedError
<commit_msg>swap lng/lat when searching by area<commit_after>
# Template XML requests required by the USGS Inventory Service
# Requesting data like it's 1999

from xml.etree.ElementTree import Element, SubElement, tostring
from usgs import CATALOG_NODES, USGSApiKeyRequiredError, USGSCatalogNodeDoesNotExist

def create_root_request():
    root = Element("soapenv:Envelope")
    root.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
    root.set("xmlns:xsd", "http://www.w3.org/2001/XMLSchema")
    root.set("xmlns:soapenv", "http://schemas.xmlsoap.org/soap/envelope/")
    root.set("xmlns:soap", "http://earthexplorer.usgs.gov/inventory/soap")

    header = SubElement(root, "soapenv:Header")
    body = SubElement(root, "soapenv:Body")

    return (root, body)

def create_request_type(parent, request_type):
    el = SubElement(parent, "soap:%s" % request_type)
    el.set("soapenv:encodingStyle", "http://schemas.xmlsoap.org/soap/encoding/")
    return el

def create_api_key_element(parent, api_key):
    el = SubElement(parent, "apiKey")
    el.set("xsi:type", "xsd:string")
    el.text = api_key

def create_node_element(parent, node):

    if node not in CATALOG_NODES:
        raise USGSCatalogNodeDoesNotExist("Catalog nodes include %s" % ", ".join(CATALOG_NODES))

    el = SubElement(parent, "node")
    el.set("xsi:type", "xsd:string")
    el.text = node

def create_dataset_element(parent, dataset):
    el = SubElement(parent, "datasetName")
    el.set("xsi:type", "xsd:string")
    el.text = dataset

def create_entity_ids_element(parent, entityids):

    if isinstance(entityids, str):

        el = SubElement(parent, "entityId")
        el.set("xsi:type", "xsd:string")
        el.text = entityids

    else:

        el = SubElement(parent, "entityIds")
        el.set("xsi:type", "soap:ArrayOfString")

        for entityid in entityids:
            child = SubElement(el, "item")
            child.set("xsi:type","xsd:string")
            child.text = entityid

def create_service_class_coordinate(parent, name, latitude=None, longitude=None):

    el = SubElement(parent, name)
    el.set("xsi:type", "soap:Service_Class_Coordinate")

    lat_el = SubElement(el, "latitude")
    lat_el.set("xsi:type", "xsd:double")
    lat_el.text = str(latitude)

    lng_el = SubElement(el, "longitude")
    lng_el.set("xsi:type", "xsd:double")
    lng_el.text = str(longitude)

def clear_bulk_download_order(dataset, node, api_key=None):
    """
    This method is used to clear bulk download order information from the item basket.

    :param dataset:

    :param node:

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "clearBulkDownloadOrder")

    create_node_element(el, node)
    create_dataset_element(el, dataset)
    create_api_key_element(el, api_key)

    return tostring(root)

def clear_order(dataset, node, api_key=None):
    """
    This method is used to clear order information from the item basket.

    :param dataset:

    :param node:

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "clearOrder")

    create_node_element(el, node)
    create_dataset_element(el, dataset)
    create_api_key_element(el, api_key)

    return tostring(root)

def datasets(dataset, node, ll=None, ur=None, start_date=None, end_date=None, api_key=None):
    """
    This method is used to find datasets available for searching.
    By passing no parameters except node, all available datasets
    are returned. Additional parameters such as temporal range
    and spatial bounding box can be used to find datasets that
    provide more specific data. The dataset name parameter can
    be used to limit the results based on matching the supplied
    value against the dataset name with assumed wildcards at the
    beginning and end. All parameters are optional except for
    the 'node' parameter.

    :param dataset:
        Dataset Identifier

    :param ll:
        Lower left corner of an AOI bounding box - in decimal form
        Longitude/Latitude dictionary

        e.g. { "longitude": 0.0, "latitude": 0.0 }

    :param ur:
        Upper right corner of an AOI bounding box - in decimal form
        Longitude/Latitude dictionary

        e.g. { "longitude": 0.0, "latitude": 0.0 }

    :param start_date:
        Used for searching scene acquisition - will accept anything
        that the PHP strtotime function can understand

    :param end_date:
        Used for searching scene acquisition - will accept anything
        that the PHP strtotime function can understand

    :param node:
        The requested Catalog

    :param api_key:
        API key is not required.

    """

    root, body = create_root_request()

    el = create_request_type(body, "datasets")

    create_node_element(el, node)
    if dataset:
        create_dataset_element(el, dataset)

    if api_key:
        create_api_key_element(el, api_key)

    if ll and ur:

        create_service_class_coordinate(el, "lowerLeft", latitude=ll["latitude"], longitude=ll["longitude"])
        create_service_class_coordinate(el, "upperRight", latitude=ur["latitude"], longitude=ur["longitude"])

    if start_date:

        start_date_el = SubElement(el, "startDate")
        start_date_el.set("xsi:type", "xsd:string")
        start_date_el.text = start_date

    if end_date:

        end_date_el = SubElement(el, "endDate")
        end_date_el.set("xsi:type", "xsd:string")
        end_date_el.text = end_date

    return tostring(root)

def dataset_fields(dataset, node, api_key=None):
    """
    This request is used to return the metadata filter
    fields for the specified dataset. These values can
    be used as additional criteria when submitting search
    and hit queries.

    :param dataset:

    :param node:

    :param api_key:
        API key is not required.
    """

    root, body = create_root_request()

    el = create_request_type(body, "datasetFields")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    if api_key:
        create_api_key_element(el, api_key)

    return tostring(root)

def download(dataset, node, entityids, products, api_key=None):
    """
    The use of this request will be to obtain valid data download URLs.

    :param dataset:

    :param entityIds:
        list

    :param products:
        list

    :param node:

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "download")

    create_dataset_element(el, dataset)
    create_node_element(el, node)
    create_api_key_element(el, api_key)
    create_entity_ids_element(el, entityids)

    products_el = SubElement(el, "products")
    products_el.set("xsi:type", "soap:ArrayOfString")

    for product in products:
        product_el = SubElement(products_el, "item")
        product_el.set("xsi:type","xsd:string")
        product_el.text = product

    return tostring(root)

def download_options(dataset, node, entityids, api_key=None):
    """
    The use of the download options request is to discover the different download
    options for each scene. Some download options may exist but still be unavailable
    due to disk usage and many other factors. If a download is unavailable
    it may need to be ordered.

    :param dataset:

    :param node:

    :param entityIds:

    :param api_key:
        API key is not required.
    """

    root, body = create_root_request()

    el = create_request_type(body, "downloadOptions")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    if api_key:
        create_api_key_element(el, api_key)

    create_entity_ids_element(el, entityids)

    return tostring(root)

def get_bulk_download_products(dataset, node, entityids, api_key):
    """
    Retrieve bulk download products on a scene-by-scene basis.

    :param dataset:

    :param node:

    :param entityid:
        String or list of strings.

    :param api_key:
        API key is required.

    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "getBulkDownloadProducts")

    create_dataset_element(el, dataset)
    create_node_element(el, node)
    create_api_key_element(el, api_key)
    create_entity_ids_element(el, entityids)

    return tostring(root)

def get_order_products(dataset, node, entityids, api_key=None):
    """
    Retrieve orderable products on a scene-by-scene basis.

    :param dataset:

    :param node:

    :param entityid:

    :param api_key:
        API key is required.

    .. todo:: Support multiple scene request.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "getOrderProducts")

    create_dataset_element(el, dataset)
    create_node_element(el, node)
    create_api_key_element(el, api_key)
    create_entity_ids_element(el, entityids)

    return tostring(root)

def item_basket(api_key=None):
    """
    Returns the current item basket for the current user.

    :param api_key:
        API key is required.
    """

    if api_key is None:
        raise USGSApiKeyRequiredError

    root, body = create_root_request()

    el = create_request_type(body, "itemBasket")
    create_api_key_element(el, api_key)

    return tostring(root)

def login(username, password):
    """
    This method requires SSL be used due to the sensitive nature of
    users passwords. Upon a successful login, an API key will be
    returned. This key will be active for one hour and should be
    destroyed upon final use of the service by calling the logout
    method. Users must have "Machine to Machine" access based on
    a user-based role in the users profile.

    :param username:

    :param password:
    """

    root, body = create_root_request()

    el = create_request_type(body, "login")

    username_el = SubElement(el, "username")
    username_el.set("xsi:type", "xsd:string")
    username_el.text = username

    password_el = SubElement(el, "password")
    password_el.set("xsi:type", "xsd:string")
    password_el.text = password

    return tostring(root)

def logout(api_key=None):
    """
    Remove the users API key from being used in the future.

    :param api_key:
        API key is required.
    """

    root, body = create_root_request()

    el = create_request_type(body, "logout")
    create_api_key_element(el, api_key)

    return tostring(root)

def metadata(dataset, node, sceneids, api_key=None):
    """
    The use of the metadata request is intended for those who have
    acquired scene IDs from a different source. It will return the
    same metadata that is available via the search request.

    :param dataset:

    :param node:

    :param sceneid:

    :param api_key:
    """
    root, body = create_root_request()

    el = create_request_type(body, "metadata")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    if api_key:
        create_api_key_element(el, api_key)

    create_entity_ids_element(el, sceneids)

    return tostring(root)

def remove_bulk_download_scene():
    raise NotImplementedError

def remove_order_scene():
    raise NotImplementedError

def search(dataset, node, lat=None, lng=None, distance=100, ll=None, ur=None, start_date=None, end_date=None, max_results=50000, starting_number=1, sort_order="DESC", api_key=None):
    """
    :param dataset:

    :param node:

    :param lat:

    :param lng:

    :param ll:

    :param distance:

    :param ur:

    :param start_date:

    :param end_date:

    :param max_results:

    :param starting_number:

    :param sort_order:

    :param api_key:
        API key is not required.
    """
    root, body = create_root_request()

    el = create_request_type(body, "search")

    create_dataset_element(el, dataset)
    create_node_element(el, node)

    # Latitude and longitude take precedence over ll and ur
    if lat and lng:

        try:
            import pyproj
            from shapely import geometry
        except ImportError:
            raise USGSDependencyRequired("Shapely and PyProj are required for search.")

        prj = pyproj.Proj(proj='aeqd', lat_0=lat, lon_0=lng)
        half_distance = 0.5 * distance
        box = geometry.box(-half_distance, -half_distance, half_distance, half_distance)

        lngs, lats = prj(*box.exterior.xy, inverse=True)

        ll = { "longitude": min(*lngs), "latitude": min(*lats) }
        ur = { "longitude": max(*lngs), "latitude": max(*lats) }

    if ll and ur:

        create_service_class_coordinate(el, "lowerLeft", latitude=ll["latitude"], longitude=ll["longitude"])
        create_service_class_coordinate(el, "upperRight", latitude=ur["latitude"], longitude=ur["longitude"])

    if start_date:

        start_date_el = SubElement(el, "startDate")
        start_date_el.set("xsi:type", "xsd:string")
        start_date_el.text = start_date

    if end_date:

        end_date_el = SubElement(el, "endDate")
        end_date_el.set("xsi:type", "xsd:string")
        end_date_el.text = end_date

    if max_results:

        max_results_el = SubElement(el, "maxResults")
        max_results_el.set("xsi:type", "xsd:int")
        max_results_el.text = str(max_results)

    if starting_number:

        starting_number_el = SubElement(el, "startingNumber")
        starting_number_el.set("xsi:type", "xsd:int")
        starting_number_el.text = str(starting_number)

    if sort_order:

        sort_order_el = SubElement(el, "sortOrder")
        sort_order_el.set("xsi:type", "xsd:string")
        sort_order_el.text = sort_order

    if api_key:
        create_api_key_element(el, api_key)

    return tostring(root)

def submit_bulk_order():
    raise NotImplementedError

def submit_order():
    raise NotImplementedError

def update_bulk_download_scene():
    raise NotImplementedError

def update_order_scene():
    raise NotImplementedError
<|endoftext|>


<commit_before>from django.shortcuts import render, HttpResponse, redirect
from django.views import View
from rdflib import Graph, RDFS
from oeplatform.settings import ONTOLOGY_FOLDER
from collections import OrderedDict

import os
import re

def collect_modules(path):
    modules = dict()
    for file in os.listdir(path):
        if not os.path.isdir(os.path.join(path,file)):
            match = re.match("^(?P<filename>.*)\.(?P<extension>\w+)$", file)
            filename, extension = match.groups()
            if filename not in modules:
                modules[filename] = dict(extensions=[], comment="No description found")
            if extension == "owl":
                g = Graph()
                g.parse(os.path.join(path, file))
                root = dict(g.namespaces())['']
                comments = g.objects(root, RDFS.comment)
                try:
                    modules[filename]["comment"] = next(comments)
                except StopIteration:
                    modules[filename]["comment"] = "No description found"
            modules[filename]["extensions"].append(extension)
    return modules

class OntologyOverview(View):
    def get(self, request, ontology, module_or_id=None, version=None, imports=False):
        versions = os.listdir(f"{ONTOLOGY_FOLDER}/{ontology}")
        if not version:
            version = max((d for d in versions), key=lambda d:[int(x) for x in d.split(".")])
        if request.headers.get("Accept") == "application/rdf+xml":
            module_name = None
            if module_or_id:
                if imports:
                    submodules = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports")
                else:
                    submodules = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules")
                # If module_or_id is the name of a valid submodule, use this module
                if module_or_id in submodules:
                    module_name = module_or_id
                if imports:
                    return redirect(f"/ontology/{ontology}/releases/{version}/imports/{module_name}.owl")
                else:
                    return redirect(f"/ontology/{ontology}/releases/{version}/{module_name}.owl")
            # If no module was requested or the requested id was not a module, serve main ontology
            if module_name is None:
                main_module = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}")
                module_name = list(main_module.keys())[0]
            return redirect(f"/ontology/{ontology}/releases/{version}/{module_name}.owl")
        else:
            main_module = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}")
            main_module_name = list(main_module.keys())[0]
            main_module = main_module[main_module_name]
            main_module["name"] = main_module_name
            submodules = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules")
            # Collect all file names

            imports = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports")

            return render(request, "ontology/oeo.html", dict(
                ontology=ontology,
                version=version,
                main_module=main_module,
                submodules=submodules.items(),
                imports=imports.items()
            ))

class OntologyStatics(View):
    def get(self, request, ontology, file, version=None, extension=None, imports=False):
        """
        Returns the requested file `{file}.{extension}` of version `version`
        of ontology `ontology`

        :param version: default: highest version in folder
        :param extension: default: `.owl`
        :return:
        """

        if not extension:
            extension = "owl"
        if not version:
            version = max((d for d in os.listdir(f"{ONTOLOGY_FOLDER}/{ontology}")), key=lambda d:[int(x) for x in d.split(".")])
        if imports:
            file_path = f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports/{file}.{extension}"
        else:
            file_path = f"{ONTOLOGY_FOLDER}/{ontology}/{version}/{file}.{extension}"
        if os.path.exists(file_path):
            with open(file_path) as f:
                response = HttpResponse(f, content_type="application/rdf+xml")
                response["Content-Disposition"] = f'attachment; filename="{file}.{extension}"'
                return response
        else:
            file_path = f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules/{file}.{extension}"
            with open(file_path) as f:
                response = HttpResponse(f, content_type="application/rdf+xml")
                response["Content-Disposition"] = f'attachment; filename="{file}.{extension}"'
                return response<commit_msg>Read file before responding<commit_after>from django.shortcuts import render, HttpResponse, redirect
from django.views import View
from rdflib import Graph, RDFS
from oeplatform.settings import ONTOLOGY_FOLDER
from collections import OrderedDict

import os
import re

def collect_modules(path):
    modules = dict()
    for file in os.listdir(path):
        if not os.path.isdir(os.path.join(path,file)):
            match = re.match("^(?P<filename>.*)\.(?P<extension>\w+)$", file)
            filename, extension = match.groups()
            if filename not in modules:
                modules[filename] = dict(extensions=[], comment="No description found")
            if extension == "owl":
                g = Graph()
                g.parse(os.path.join(path, file))
                root = dict(g.namespaces())['']
                comments = g.objects(root, RDFS.comment)
                try:
                    modules[filename]["comment"] = next(comments)
                except StopIteration:
                    modules[filename]["comment"] = "No description found"
            modules[filename]["extensions"].append(extension)
    return modules

class OntologyOverview(View):
    def get(self, request, ontology, module_or_id=None, version=None, imports=False):
        versions = os.listdir(f"{ONTOLOGY_FOLDER}/{ontology}")
        if not version:
            version = max((d for d in versions), key=lambda d:[int(x) for x in d.split(".")])
        if request.headers.get("Accept") == "application/rdf+xml":
            module_name = None
            if module_or_id:
                if imports:
                    submodules = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports")
                else:
                    submodules = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules")
                # If module_or_id is the name of a valid submodule, use this module
                if module_or_id in submodules:
                    module_name = module_or_id
                if imports:
                    return redirect(f"/ontology/{ontology}/releases/{version}/imports/{module_name}.owl")
                else:
                    return redirect(f"/ontology/{ontology}/releases/{version}/{module_name}.owl")
            # If no module was requested or the requested id was not a module, serve main ontology
            if module_name is None:
                main_module = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}")
                module_name = list(main_module.keys())[0]
            return redirect(f"/ontology/{ontology}/releases/{version}/{module_name}.owl")
        else:
            main_module = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}")
            main_module_name = list(main_module.keys())[0]
            main_module = main_module[main_module_name]
            main_module["name"] = main_module_name
            submodules = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules")
            # Collect all file names

            imports = collect_modules(f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports")

            return render(request, "ontology/oeo.html", dict(
                ontology=ontology,
                version=version,
                main_module=main_module,
                submodules=submodules.items(),
                imports=imports.items()
            ))

class OntologyStatics(View):
    def get(self, request, ontology, file, version=None, extension=None, imports=False):
        """
        Returns the requested file `{file}.{extension}` of version `version`
        of ontology `ontology`

        :param version: default: highest version in folder
        :param extension: default: `.owl`
        :return:
        """

        if not extension:
            extension = "owl"
        if not version:
            version = max((d for d in os.listdir(f"{ONTOLOGY_FOLDER}/{ontology}")), key=lambda d:[int(x) for x in d.split(".")])
        if imports:
            file_path = f"{ONTOLOGY_FOLDER}/{ontology}/{version}/imports/{file}.{extension}"
        else:
            file_path = f"{ONTOLOGY_FOLDER}/{ontology}/{version}/{file}.{extension}"
        if os.path.exists(file_path):
            with open(file_path) as f:
                response = HttpResponse(f.read(), content_type="application/rdf+xml")
                response["Content-Disposition"] = f'attachment; filename="{file}.{extension}"'
                return response
        else:
            file_path = f"{ONTOLOGY_FOLDER}/{ontology}/{version}/modules/{file}.{extension}"
            with open(file_path) as f:
                response = HttpResponse(f.read(), content_type="application/rdf+xml")
                response["Content-Disposition"] = f'attachment; filename="{file}.{extension}"'
                return response<|endoftext|>