<commit_before># -*- encoding: utf-8 -*-
'''
HubbleStack Nova plugin for auditing installed packages.

This module checks all of a system's local packages and reports if the package
is vulnerable to a known cve. The cve vunlerablities are gathered via the url in
the yaml profile, and that data cached at the path
/var/cache/salt/minion/cve_scan_cache/<os_name>_<version>.json

:maintainer: HubbleStack / jaredhanson11
:maturity: 2016.7.0
:platform: Linux
:requires: SaltStack

This audit module requires yaml data to execute. It will search the local
directory for any .yaml files, and if it finds a top-level 'cve_scan_v2' key, it
will use that data.

Sample YAML data with inline comments:

cve_scan_v2:
    # Seconds until the local cache expires
    ttl: 86400
    # Source of cve data
    url: http://vulners.com/
    # Optional control tag
    control:
        # minimum score, vulnerabilities with a smaller
        #   score added to 'Controlled' output
        score: 3

The source of the cve data can be http://vulners.com/, salt://path/to/json, and
any other url that returns cve data in json format. If the url contains
vulners.com, then this module will use the local system's os and os version to
dynamically query vulner.com/api/v3 for cve data specifically related to your
system. If the url doesn't contain vulners.com, it will query the exact url, so
that endpoint must return cve data specific to the system you are scanning.

The cve data json must be formatted as follows:

[

{'_source': {'affectedPackage': [{'OS': 'CentOS',
                                    'OSVersion': '7',
                                    'operator': 'lt',
                                    'packageFilename': 'krb5-server-1.13.2-12.el7_2.x86_64.rpm',
                                    'packageName': 'krb5-server',
                                    'packageVersion': '1.13.2-12.el7_2'},
                                   {'OS': 'CentOS',
                                    'OSVersion': '7',
                                    'operator': 'lt',
                                    'packageFilename': 'krb5-libs-1.13.2-12.el7_2.i686.rpm',
                                    'packageName': 'krb5-libs',
                                    'packageVersion': '1.13.2-12.el7_2'}
                                    ]
              'cvelist': ['CVE-2015-8631',
                           'CVE-2015-8630',
                           'CVE-2015-8629'],
              'cvss': {'score': 6.8}
              'href': 'http://lists.centos.org/pipermail/centos-announce/2016-March/021788.html',
              'reporter': 'CentOS Project',
              'title': 'Moderate krb5 Security Update'
            }
    },
...

 ]

'''
from __future__ import absolute_import
import logging

import fnmatch
import hashlib
import json
import os
import re
import requests

from distutils.version import LooseVersion
from time import time as current_time
from zipfile import ZipFile

import salt
import salt.utils

log = logging.getLogger(__name__)

def __virtual__():
    return not salt.utils.is_windows()

def audit(data_list, tags, debug=False, **kwargs):
    '''
    Main audit function. See module docstring for more information on usage.
    '''
    os_version = __grains__.get('osmajorrelease', None)
    if os_version is None:
        os_version = __grains__.get('osrelease', None)
    os_name = __grains__['os'].lower()

    log.debug("os_version: %s, os_name: %s", os_version, os_name)

    endpoints = []

    # Go through yaml to check for cve_scan_v2,
    #    if its present, check for a cached version
    #    of the scan.
    for profile, data in data_list:

        if 'cve_scan_v2' in data:

            ttl = data['cve_scan_v2']['ttl']
            url = data['cve_scan_v2']['url']
            control = data['cve_scan_v2'].get('control', {})
            # Ability to add more controls easily, in control dict
            min_score = float(control.get('score', 0))
            urlhash = hashlib.md5(url).hexdigest()
            cached_json = os.path.join(__opts__['cachedir'],
                                       'cve_scan_cache',
                                       '%s.json' % urlhash)
            cached_zip = os.path.join(__opts__['cachedir'],
                                      'cve_scan_cache',
                                      '%s.zip' % urlhash)
            # Make cache directory and all parent directories if it doesn't exist.
            if not os.path.exists(os.path.dirname(cached_json)):
                os.makedirs(os.path.dirname(cached_json))
            cache = _get_cache(ttl, cached_json)
            log.debug("valid cache: %s, for url: %s", cache != [], url)
            endpoints.append((url, cache, cached_json, cached_zip, min_score, profile))

    # If we don't find our module in the yaml
    if not endpoints:
        return {}

    ret = {'Success': [], 'Failure': [], 'Controlled': []}
    # Dictionary of {pkg_name: list(pkg_versions)}
    local_pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True)

    for url, cache, cached_json, cached_zip, min_score, profile in endpoints:
        log.debug("url: %s, min_score: %s", url, min_score)
        if cache: # Valid cached file
            master_json = cache
        else: # Query the url for cve's
            if url.startswith('http://') or url.startswith('https://'):
                if 'vulners.com' in url:
                    # Vulners api can only handles http:// requests from request.get
                    if url.startswith('https'):
                        url.replace('https', 'http', 1)
                    # Format the url for the request based on operating system.
                    if url.endswith('/'):
                        url = url[:-1]
                    url_final = '%s/api/v3/archive/distributive/?os=%s&version=%s' \
                                                                % (url, os_name, os_version)
                    log.debug('requesting: %s', url_final)
                    cve_query = requests.get(url_final)
                    # Confirm that the request was valid.
                    if cve_query.status_code != 200:
                        raise Exception('Vulners requests was not successful. Check the url.')
                    # Save vulners zip attachment in cache location and extract json
                    try:
                        with open(cached_zip, 'w') as zip_attachment:
                            zip_attachment.write(cve_query.content)
                        zip_file = ZipFile(cached_zip)
                        zip_file.extractall(os.path.dirname(cached_zip))
                        os.remove(cached_zip)
                        extracted_json = os.path.join(__opts__['cachedir'],
                                                      'cve_scan_cache',
                                                      '%s_%s.json' % (os_name, str(os_version).replace('.', '')))
                        log.debug('attempting to open %s', extracted_json)
                        with open(extracted_json, 'r') as json_file:
                            master_json = json.load(json_file)
                        os.remove(extracted_json)
                    except IOError as ioe:
                        log.error('The json zip attachment was not able to be extracted from vulners.')
                        raise ioe
                else: # Not a vulners request, external source for cve's
                    log.debug('requesting: %s', url)
                    cve_query = requests.get(url)
                    if cve_query.status_code != 200:
                        log.error('URL request was not successful.')
                        raise Exception('The url given is invalid.')
                    master_json = json.loads(cve_query.text)
                #Cache results.
                try:
                    with open(cached_json, 'w') as cache_file:
                        json.dump(master_json, cache_file)
                except IOError:
                    log.error('The cve results weren\'t able to be cached')
            elif url.startswith('salt://'):
                # Cache the file
                log.debug('getting file from %s', url)
                cache_file = __salt__['cp.get_file'](url, cached_json)
                if cache_file:
                    master_json = json.load(open(cache_file))
                else:
                    raise IOError('The file was not able to be retrieved from the salt file server.')
            else:
                raise Exception('The url is invalid. It does not begin with http(s):// or salt://')

        affected_pkgs = _get_cve_vulnerabilities(master_json, os_version)

        # Check all local packages against cve vulnerablities in affected_pkgs
        for local_pkg in local_pkgs:
            vulnerable = None
            if local_pkg in affected_pkgs:
                # There can be multiple versions for a single local package, check all
                for local_version in local_pkgs[local_pkg]:
                    # There can be multiple cve announcements for a single package, check against all
                    for affected_obj in affected_pkgs[local_pkg]:
                        affected_version = affected_obj.pkg_version
                        if _is_vulnerable(local_version, affected_version, affected_obj.operator):
                            # If the local pkg hasn't been found as vulnerable yet, vulnerable is None
                            if not vulnerable:
                                affected_obj.oudated_version = local_version
                                vulnerable = affected_obj
                            # If local_pkg has already been marked affected, vulnerable is set. We
                            #   want to report the cve with highest severity
                            else:
                                if affected_obj.score > vulnerable.score:
                                    affected_obj.oudated_version = local_version
                                    vulnerable = affected_obj
                if vulnerable:
                    if vulnerable.score < min_score:
                        ret['Controlled'].append(vulnerable.get_report(profile))
                    else:
                        ret['Failure'].append(vulnerable.get_report(profile))

    if tags != '*':
        log.debug("tags: %s", tags)
        remove = []
        for i, failure in enumerate(ret['Failure']):
            if not fnmatch.fnmatch(failure.keys()[0], tags):
                remove.append(i)
        remove.reverse()
        for i in remove:
            ret['Failure'].pop(i)

        remove = []
        for i, failure in enumerate(ret['Controlled']):
            if not fnmatch.fnmatch(failure.keys()[0], tags):
                remove.append(i)
        remove.reverse()
        for i in remove:
            ret['Controlled'].pop(i)

    if not ret['Controlled']:
        ret.pop('Controlled')

    return ret

def _get_cve_vulnerabilities(query_results, os_version):
    '''
    Returns dictionary of vulnerablities, mapped as pkg_name:pkgObj.
    '''

    vulnerable_pkgs = {}

    for report in query_results:
        try:
            reporter = report['_source'].get('reporter', '')
            cve_list = report['_source'].get('cvelist', [])
            href = report['_source'].get('href', '')
            score = report['_source']['cvss'].get('score', 0)
            title = report['_source'].get('title', 'No Title Given')

            for pkg in report['_source']['affectedPackage']:
                #_source:affectedPackages
                if pkg['OSVersion'] in ['any', os_version]: #Only use matching os
                    pkg_obj = VulnerablePkg(title, pkg['packageName'], pkg['packageVersion'], \
                                 score, pkg['operator'], reporter, href, cve_list)
                    if pkg_obj.pkg not in vulnerable_pkgs:
                        vulnerable_pkgs[pkg_obj.pkg] = [pkg_obj]
                    else:
                        vulnerable_pkgs[pkg_obj.pkg].append(pkg_obj)
        except KeyError, key_err:
            if key_err != '_source':
                log.error('Format error at: %s', report)
                raise KeyError('The cve data was not formatted correctly at: %s' % pkg)
            else:
                log.error('Format error at: %s', report)
                raise KeyError('The cve data was not formatted correctly')
    return vulnerable_pkgs

def _is_vulnerable(local_version, affected_version, operator):
    '''
    Given two version strings, and operator
        returns whether the package is vulnerable or not.
    '''
    # Get rid of prefix if version number has one, ex '1:3.4.52'
    if ':' in local_version:
        _, _, local_version = local_version.partition(':')
    if ':' in affected_version:
        _, _, affected_version = affected_version.partition(':')

    compare = None
    # Try to use salt's built in comparison module, if it exists for distro
    if 'pkg.version_cmp' in __salt__:
        compare = __salt__['pkg.version_cmp'](local_version, affected_version)

    # When salt can't compare, use LooseVersion
    if compare is None:
        #Compare from higher order to lower order based on '-' split.
        local_version_split = local_version.split('-')
        affected_version_split = affected_version.split('-')

        for (order_index, local_version_str) in enumerate(local_version_split):

            local_version_obj = LooseVersion(local_version_str)
            affected_version_obj = LooseVersion(affected_version_split[order_index])

            #Check lower order bits if higher order are equal.
            if local_version == affected_version:
                continue

            #Return when highest order version is not equal.
            elif local_version_obj > affected_version_obj:
                compare = 1
                break
            elif local_version_obj < affected_version_obj:
                compare = -1
                break
        # If for loop exits without break, the versions are equal.
        else:
            compare = 0

    # Return whether local_version is vulnerable to affected_verison
    if operator == 'le':
        return compare <= 0
    elif operator == 'lt':
        return compare < 0

def _get_cache(ttl, cache_path):
    '''
    If url contains valid cache, returns it, else returns empty list.
    '''
    # Check if we have a valid cached version.
    try:
        cached_time = os.path.getmtime(cache_path)
    except OSError:
        return []
    if current_time() - cached_time < ttl:
        log.debug('%s is less than ttl', cache_path)
        try:
            with open(cache_path) as json_file:
                loaded_json = json.load(json_file)
                return loaded_json
        except IOError:
            return []
        except ValueError:
            log.error('%s was not json formatted', cache_path)
            return []
    else:
        log.debug('%s was older than ttl', cache_path)
        return []

class VulnerablePkg:
    '''
    Object representing a vulnverable pkg for the current operating system.
    '''
    def __init__(self, title, pkg, pkg_version, score, operator, reporter, href, cve_list):
        self.title = title
        self.pkg = pkg
        self.pkg_version = pkg_version
        self.score = float(score)
        if operator not in ['lt', 'le']:
            log.error('pkg:%s contains an operator that\'s not supported and was changed to <')
            operator = 'lt'
        self.operator = operator
        self.href = href
        self.cve_list = cve_list
        self.reporter = reporter
        self.oudated_version = None

    def get_report(self, profile):
        '''
        Return the dictionary of what should be reported in failures, based on verbose.
        '''
        return {
            'tag': self.pkg + '-' + self.pkg_version,
            'href': self.href,
            'affected_version': self.pkg_version,
            'reporter': self.reporter,
            'score': self.score,
            'cve_list': self.cve_list,
            'affected_pkg': self.pkg,
            'local_version': self.oudated_version,
            'description': self.title,
            'nova_profile': profile
        }

<commit_msg>Cherry-pick hubble-salt PR109<commit_after># -*- encoding: utf-8 -*-
'''
HubbleStack Nova plugin for auditing installed packages.

This module checks all of a system's local packages and reports if the package
is vulnerable to a known cve. The cve vunlerablities are gathered via the url in
the yaml profile, and that data cached at the path
/var/cache/salt/minion/cve_scan_cache/<os_name>_<version>.json

:maintainer: HubbleStack / jaredhanson11
:maturity: 2016.7.0
:platform: Linux
:requires: SaltStack

This audit module requires yaml data to execute. It will search the local
directory for any .yaml files, and if it finds a top-level 'cve_scan_v2' key, it
will use that data.

Sample YAML data with inline comments:

cve_scan_v2:
    # Seconds until the local cache expires
    ttl: 86400
    # Source of cve data
    url: http://vulners.com/
    # Optional control tag
    control:
        # minimum score, vulnerabilities with a smaller
        #   score added to 'Controlled' output
        score: 3

The source of the cve data can be http://vulners.com/, salt://path/to/json, and
any other url that returns cve data in json format. If the url contains
vulners.com, then this module will use the local system's os and os version to
dynamically query vulner.com/api/v3 for cve data specifically related to your
system. If the url doesn't contain vulners.com, it will query the exact url, so
that endpoint must return cve data specific to the system you are scanning.

The cve data json must be formatted as follows:

[

{'_source': {'affectedPackage': [{'OS': 'CentOS',
                                    'OSVersion': '7',
                                    'operator': 'lt',
                                    'packageFilename': 'krb5-server-1.13.2-12.el7_2.x86_64.rpm',
                                    'packageName': 'krb5-server',
                                    'packageVersion': '1.13.2-12.el7_2'},
                                   {'OS': 'CentOS',
                                    'OSVersion': '7',
                                    'operator': 'lt',
                                    'packageFilename': 'krb5-libs-1.13.2-12.el7_2.i686.rpm',
                                    'packageName': 'krb5-libs',
                                    'packageVersion': '1.13.2-12.el7_2'}
                                    ]
              'cvelist': ['CVE-2015-8631',
                           'CVE-2015-8630',
                           'CVE-2015-8629'],
              'cvss': {'score': 6.8}
              'href': 'http://lists.centos.org/pipermail/centos-announce/2016-March/021788.html',
              'reporter': 'CentOS Project',
              'title': 'Moderate krb5 Security Update'
            }
    },
...

 ]

'''
from __future__ import absolute_import
import logging

import fnmatch
import hashlib
import json
import os
import re
import requests

from distutils.version import LooseVersion
from time import time as current_time
from zipfile import ZipFile

import salt
import salt.utils

log = logging.getLogger(__name__)

def __virtual__():
    return not salt.utils.is_windows()

def audit(data_list, tags, debug=False, **kwargs):
    '''
    Main audit function. See module docstring for more information on usage.
    '''
    os_version = __grains__.get('osrelease', None)
    if os_version is None:
        os_version = __grains__.get('osmajorrelease', None)
    os_name = __grains__['os'].lower()

    log.debug("os_version: %s, os_name: %s", os_version, os_name)

    endpoints = []

    # Go through yaml to check for cve_scan_v2,
    #    if its present, check for a cached version
    #    of the scan.
    for profile, data in data_list:

        if 'cve_scan_v2' in data:

            ttl = data['cve_scan_v2']['ttl']
            url = data['cve_scan_v2']['url']
            control = data['cve_scan_v2'].get('control', {})
            # Ability to add more controls easily, in control dict
            min_score = float(control.get('score', 0))
            urlhash = hashlib.md5(url).hexdigest()
            cached_json = os.path.join(__opts__['cachedir'],
                                       'cve_scan_cache',
                                       '%s.json' % urlhash)
            cached_zip = os.path.join(__opts__['cachedir'],
                                      'cve_scan_cache',
                                      '%s.zip' % urlhash)
            # Make cache directory and all parent directories if it doesn't exist.
            if not os.path.exists(os.path.dirname(cached_json)):
                os.makedirs(os.path.dirname(cached_json))
            cache = _get_cache(ttl, cached_json)
            log.debug("valid cache: %s, for url: %s", cache != [], url)
            endpoints.append((url, cache, cached_json, cached_zip, min_score, profile))

    # If we don't find our module in the yaml
    if not endpoints:
        return {}

    ret = {'Success': [], 'Failure': [], 'Controlled': []}
    # Dictionary of {pkg_name: list(pkg_versions)}
    local_pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True)

    for url, cache, cached_json, cached_zip, min_score, profile in endpoints:
        log.debug("url: %s, min_score: %s", url, min_score)
        if cache: # Valid cached file
            master_json = cache
        else: # Query the url for cve's
            if url.startswith('http://') or url.startswith('https://'):
                if 'vulners.com' in url:
                    # Vulners api can only handles http:// requests from request.get
                    if url.startswith('https'):
                        url.replace('https', 'http', 1)
                    # Format the url for the request based on operating system.
                    if url.endswith('/'):
                        url = url[:-1]
                    url_final = '%s/api/v3/archive/distributive/?os=%s&version=%s' \
                                                                % (url, os_name, os_version)
                    log.debug('requesting: %s', url_final)
                    cve_query = requests.get(url_final)
                    # Confirm that the request was valid.
                    if cve_query.status_code != 200:
                        raise Exception('Vulners requests was not successful. Check the url.')
                    # Save vulners zip attachment in cache location and extract json
                    try:
                        with open(cached_zip, 'w') as zip_attachment:
                            zip_attachment.write(cve_query.content)
                        zip_file = ZipFile(cached_zip)
                        zip_file.extractall(os.path.dirname(cached_zip))
                        os.remove(cached_zip)
                        extracted_json = os.path.join(__opts__['cachedir'],
                                                      'cve_scan_cache',
                                                      '%s_%s.json' % (os_name, str(os_version).replace('.', '')))
                        log.debug('attempting to open %s', extracted_json)
                        with open(extracted_json, 'r') as json_file:
                            master_json = json.load(json_file)
                        os.remove(extracted_json)
                    except IOError as ioe:
                        log.error('The json zip attachment was not able to be extracted from vulners.')
                        raise ioe
                else: # Not a vulners request, external source for cve's
                    log.debug('requesting: %s', url)
                    cve_query = requests.get(url)
                    if cve_query.status_code != 200:
                        log.error('URL request was not successful.')
                        raise Exception('The url given is invalid.')
                    master_json = json.loads(cve_query.text)
                #Cache results.
                try:
                    with open(cached_json, 'w') as cache_file:
                        json.dump(master_json, cache_file)
                except IOError:
                    log.error('The cve results weren\'t able to be cached')
            elif url.startswith('salt://'):
                # Cache the file
                log.debug('getting file from %s', url)
                cache_file = __salt__['cp.get_file'](url, cached_json)
                if cache_file:
                    master_json = json.load(open(cache_file))
                else:
                    raise IOError('The file was not able to be retrieved from the salt file server.')
            else:
                raise Exception('The url is invalid. It does not begin with http(s):// or salt://')

        affected_pkgs = _get_cve_vulnerabilities(master_json, os_version)

        # Check all local packages against cve vulnerablities in affected_pkgs
        for local_pkg in local_pkgs:
            vulnerable = None
            if local_pkg in affected_pkgs:
                # There can be multiple versions for a single local package, check all
                for local_version in local_pkgs[local_pkg]:
                    # There can be multiple cve announcements for a single package, check against all
                    for affected_obj in affected_pkgs[local_pkg]:
                        affected_version = affected_obj.pkg_version
                        if _is_vulnerable(local_version, affected_version, affected_obj.operator):
                            # If the local pkg hasn't been found as vulnerable yet, vulnerable is None
                            if not vulnerable:
                                affected_obj.oudated_version = local_version
                                vulnerable = affected_obj
                            # If local_pkg has already been marked affected, vulnerable is set. We
                            #   want to report the cve with highest severity
                            else:
                                if affected_obj.score > vulnerable.score:
                                    affected_obj.oudated_version = local_version
                                    vulnerable = affected_obj
                if vulnerable:
                    if vulnerable.score < min_score:
                        ret['Controlled'].append(vulnerable.get_report(profile))
                    else:
                        ret['Failure'].append(vulnerable.get_report(profile))

    if tags != '*':
        log.debug("tags: %s", tags)
        remove = []
        for i, failure in enumerate(ret['Failure']):
            if not fnmatch.fnmatch(failure.keys()[0], tags):
                remove.append(i)
        remove.reverse()
        for i in remove:
            ret['Failure'].pop(i)

        remove = []
        for i, failure in enumerate(ret['Controlled']):
            if not fnmatch.fnmatch(failure.keys()[0], tags):
                remove.append(i)
        remove.reverse()
        for i in remove:
            ret['Controlled'].pop(i)

    if not ret['Controlled']:
        ret.pop('Controlled')

    return ret

def _get_cve_vulnerabilities(query_results, os_version):
    '''
    Returns dictionary of vulnerablities, mapped as pkg_name:pkgObj.
    '''

    vulnerable_pkgs = {}

    for report in query_results:
        try:
            reporter = report['_source'].get('reporter', '')
            cve_list = report['_source'].get('cvelist', [])
            href = report['_source'].get('href', '')
            score = report['_source']['cvss'].get('score', 0)
            title = report['_source'].get('title', 'No Title Given')

            for pkg in report['_source']['affectedPackage']:
                #_source:affectedPackages
                if pkg['OSVersion'] in ['any', os_version]: #Only use matching os
                    pkg_obj = VulnerablePkg(title, pkg['packageName'], pkg['packageVersion'], \
                                 score, pkg['operator'], reporter, href, cve_list)
                    if pkg_obj.pkg not in vulnerable_pkgs:
                        vulnerable_pkgs[pkg_obj.pkg] = [pkg_obj]
                    else:
                        vulnerable_pkgs[pkg_obj.pkg].append(pkg_obj)
        except KeyError, key_err:
            if key_err != '_source':
                log.error('Format error at: %s', report)
                raise KeyError('The cve data was not formatted correctly at: %s' % pkg)
            else:
                log.error('Format error at: %s', report)
                raise KeyError('The cve data was not formatted correctly')
    return vulnerable_pkgs

def _is_vulnerable(local_version, affected_version, operator):
    '''
    Given two version strings, and operator
        returns whether the package is vulnerable or not.
    '''
    # Get rid of prefix if version number has one, ex '1:3.4.52'
    if ':' in local_version:
        _, _, local_version = local_version.partition(':')
    if ':' in affected_version:
        _, _, affected_version = affected_version.partition(':')

    compare = None
    # Try to use salt's built in comparison module, if it exists for distro
    if 'pkg.version_cmp' in __salt__:
        compare = __salt__['pkg.version_cmp'](local_version, affected_version)

    # When salt can't compare, use LooseVersion
    if compare is None:
        #Compare from higher order to lower order based on '-' split.
        local_version_split = local_version.split('-')
        affected_version_split = affected_version.split('-')

        for (order_index, local_version_str) in enumerate(local_version_split):

            local_version_obj = LooseVersion(local_version_str)
            affected_version_obj = LooseVersion(affected_version_split[order_index])

            #Check lower order bits if higher order are equal.
            if local_version == affected_version:
                continue

            #Return when highest order version is not equal.
            elif local_version_obj > affected_version_obj:
                compare = 1
                break
            elif local_version_obj < affected_version_obj:
                compare = -1
                break
        # If for loop exits without break, the versions are equal.
        else:
            compare = 0

    # Return whether local_version is vulnerable to affected_verison
    if operator == 'le':
        return compare <= 0
    elif operator == 'lt':
        return compare < 0

def _get_cache(ttl, cache_path):
    '''
    If url contains valid cache, returns it, else returns empty list.
    '''
    # Check if we have a valid cached version.
    try:
        cached_time = os.path.getmtime(cache_path)
    except OSError:
        return []
    if current_time() - cached_time < ttl:
        log.debug('%s is less than ttl', cache_path)
        try:
            with open(cache_path) as json_file:
                loaded_json = json.load(json_file)
                return loaded_json
        except IOError:
            return []
        except ValueError:
            log.error('%s was not json formatted', cache_path)
            return []
    else:
        log.debug('%s was older than ttl', cache_path)
        return []

class VulnerablePkg:
    '''
    Object representing a vulnverable pkg for the current operating system.
    '''
    def __init__(self, title, pkg, pkg_version, score, operator, reporter, href, cve_list):
        self.title = title
        self.pkg = pkg
        self.pkg_version = pkg_version
        self.score = float(score)
        if operator not in ['lt', 'le']:
            log.error('pkg:%s contains an operator that\'s not supported and was changed to <')
            operator = 'lt'
        self.operator = operator
        self.href = href
        self.cve_list = cve_list
        self.reporter = reporter
        self.oudated_version = None

    def get_report(self, profile):
        '''
        Return the dictionary of what should be reported in failures, based on verbose.
        '''
        return {
            'tag': self.pkg + '-' + self.pkg_version,
            'href': self.href,
            'affected_version': self.pkg_version,
            'reporter': self.reporter,
            'score': self.score,
            'cve_list': self.cve_list,
            'affected_pkg': self.pkg,
            'local_version': self.oudated_version,
            'description': self.title,
            'nova_profile': profile
        }

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import time

from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponseRedirect
from django.utils import timezone
from django.views.generic import ListView
from django.views.generic.base import TemplateView
from django.views.generic.edit import UpdateView, CreateView
from geopy import Nominatim
from os.path import isfile, join

from base.forms import MissionForm, BPTrainingForm, MessageForm, BPCarrierForm
from base.geo_locator import GeoLocator
from base.importer import UserImporter, TrainingImporter
from base.models import Mission, Training, BreathingProtectionTraining, Message, UserProfile
from feumgmt import settings

class Dashboard(TemplateView):
    template_name = 'base/dashboard.html'

    def get_context_data(self, **kwargs):
        ctx = super(Dashboard, self).get_context_data(**kwargs)

        try:
            next_training = Training.objects.filter(date__gt=timezone.now()).order_by(
                'date').first()
            responsibles = ', '.join([unicode(f) for f in next_training.responsibles.all()])
            ctx['next_training'] = {
                'date': next_training.date,
                'subject': next_training.subject,
                'responsibles': responsibles,
            }
        except ObjectDoesNotExist:
            ctx['next_training'] = None

        try:
            next_bpt = BreathingProtectionTraining.objects.filter(
                date__gt=timezone.now()).order_by('date').first()
            participants = ', '.join([unicode(f) for f in next_bpt.participants.all()])
            ctx['next_bpt'] = {
                'date': next_bpt.date,
                'location': next_bpt.location,
                'participants': participants,
            }
        except ObjectDoesNotExist:
            ctx['next_bpt'] = None

        next_messages = Message.objects.all().order_by('-creation_date')[:3]
        if next_messages:
            ctx['next_messages'] = next_messages

        return ctx

class MissionList(ListView):
    model = Mission

class MissionUpdate(UpdateView):
    model = Mission
    success_url = reverse_lazy('mission_list')
    form_class = MissionForm
    template_name = 'base/mission_form.html'

    def get_context_data(self, **kwargs):
        ctx = super(MissionUpdate, self).get_context_data(**kwargs)

        geolocator = Nominatim(format_string="%s, Landkreis Sächsische Schweiz-Osterzgebirge, Sachsen, 01833, Deutschland")
        address = '{0}, {1}'.format(self.object.street, self.object.location)

        # 11, Alte Hauptstraße, Wilschdorf, Dürrröhrsdorf-Dittersbach, Landkreis Sächsische Schweiz-Osterzgebirge, Sachsen, 01833, Deutschland
        locations = geolocator.geocode(address, False)
        if locations:
            location = locations[0]
            ctx['address'] = location.address
            ctx['lat'] = float(location.latitude)
            ctx['long'] = float(location.longitude)
            ctx['raw'] = location.raw
        return ctx

class MissionAlarm(TemplateView):
    model = Mission
    template_name = 'base/mission_alarm.html'

    def get_context_data(self, **kwargs):
        ctx = super(MissionAlarm, self).get_context_data(**kwargs)

        mission = Mission.objects.all().order_by('-alarm_time').first()
        ctx['object'] = mission

        geo_locator = GeoLocator()

        key = settings.MAP_QUEST_KEY
        address = '{0}, {1}'.format(mission.street, mission.location)

        ctx.update(geo_locator.get_location_coordinates(key, address))
        ctx.update(geo_locator.get_route(key, address))
        return ctx

class BPTrainingList(ListView):
    model = BreathingProtectionTraining
    template_name = 'base/bptraining_list.html'

class BPTrainingCreate(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    model = BreathingProtectionTraining
    success_url = reverse_lazy('bptraining_list')
    form_class = BPTrainingForm
    template_name = 'base/bptraining_form.html'
    permission_required = 'add_breathingprotectiontraining'

    def form_valid(self, form):
        form.instance.organizer_id = self.request.user.id
        form.instance.editor_id = self.request.user.id
        form.instance.creation_date = timezone.now()
        return super(BPTrainingCreate, self).form_valid(form)

class BPTrainingUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    model = BreathingProtectionTraining
    success_url = reverse_lazy('bptraining_list')
    form_class = BPTrainingForm
    template_name = 'base/bptraining_form.html'
    permission_required = 'change_breathingprotectiontraining'

    def form_valid(self, form):
        form.instance.editor_id = self.request.user.id
        return super(BPTrainingUpdate, self).form_valid(form)

class MessageList(ListView):
    model = Message
    template_name = 'base/message_list.html'

class MessageCreate(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    model = Message
    success_url = reverse_lazy('message_list')
    form_class = MessageForm
    template_name = 'base/message_form.html'
    permission_required = 'add_message'

    def form_valid(self, form):
        form.instance.author_id = self.request.user.id
        form.instance.editor_id = self.request.user.id
        form.instance.creation_date = timezone.now()
        return super(MessageCreate, self).form_valid(form)

class MessageUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    model = Message
    success_url = reverse_lazy('message_list')
    form_class = MessageForm
    template_name = 'base/message_form.html'
    permission_required = 'change_message'

    def form_valid(self, form):
        form.instance.editor_id = self.request.user.id
        return super(MessageUpdate, self).form_valid(form)

class BPCarrierList(ListView):
    model = UserProfile
    template_name = 'base/bpcarrier_list.html'

    def get_queryset(self):
        queryset = super(BPCarrierList, self).get_queryset()
        return queryset.filter(breathing_protection_carrier=True)

class BPCarrierUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    model = UserProfile
    success_url = reverse_lazy('bpcarrier_list')
    form_class = BPCarrierForm
    template_name = 'base/bpcarrier_form.html'
    permission_required = 'change_userprofile'

    def form_valid(self, form):
        form.instance.editor_id = self.request.user.id
        return super(BPCarrierUpdate, self).form_valid(form)

class UserImport(TemplateView):
    template_name = 'base/user_import.html'
    success_url = reverse_lazy('dashboard')

    def get_context_data(self, **kwargs):
        ctx = super(UserImport, self).get_context_data(**kwargs)
        importer = UserImporter()
        ctx['users'] = importer.get_users_from_csv_file()
        return ctx

    def post(self, request, *args, **kwargs):
        importer = UserImporter()
        users = importer.get_users_from_csv_file()
        importer.import_users(users, self.request.user)
        return HttpResponseRedirect(self.success_url)

class TrainingImport(TemplateView):
    template_name = 'base/training_import.html'
    success_url = reverse_lazy('dashboard')

    def get_context_data(self, **kwargs):
        ctx = super(TrainingImport, self).get_context_data(**kwargs)
        importer = TrainingImporter()
        ctx['trainings'] = importer.get_trainings_from_csv_file()
        return ctx

    def post(self, request, *args, **kwargs):
        importer = TrainingImporter()
        trainings = importer.get_trainings_from_csv_file()
        importer.import_trainings(trainings, self.request.user)
        return HttpResponseRedirect(self.success_url)

class GalleryList(TemplateView):
    template_name = 'base/gallery_list.html'

    def get_context_data(self, **kwargs):
        ctx = super(GalleryList, self).get_context_data(**kwargs)
        path = settings.MEDIA_ROOT + 'images/'
        files = []

        mtime = lambda f: os.stat(os.path.join(path, f)).st_ctime
        fls = list(sorted(os.listdir(path), key=mtime))
        index = 0
        for f in reversed(fls):
            index += 1
            file_path = join(path, f)
            if not isfile(file_path):
                continue
            files.append({
                'path': f,
                'time': time.ctime(os.path.getctime(file_path))
            })
            if index >= 15:
                break
        ctx['files'] = files
        return ctx
<commit_msg>fixed required permissions<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import time

from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponseRedirect
from django.utils import timezone
from django.views.generic import ListView
from django.views.generic.base import TemplateView
from django.views.generic.edit import UpdateView, CreateView
from geopy import Nominatim
from os.path import isfile, join

from base.forms import MissionForm, BPTrainingForm, MessageForm, BPCarrierForm
from base.geo_locator import GeoLocator
from base.importer import UserImporter, TrainingImporter
from base.models import Mission, Training, BreathingProtectionTraining, Message, UserProfile
from feumgmt import settings

class Dashboard(TemplateView):
    template_name = 'base/dashboard.html'

    def get_context_data(self, **kwargs):
        ctx = super(Dashboard, self).get_context_data(**kwargs)

        try:
            next_training = Training.objects.filter(date__gt=timezone.now()).order_by(
                'date').first()
            responsibles = ', '.join([unicode(f) for f in next_training.responsibles.all()])
            ctx['next_training'] = {
                'date': next_training.date,
                'subject': next_training.subject,
                'responsibles': responsibles,
            }
        except ObjectDoesNotExist:
            ctx['next_training'] = None

        try:
            next_bpt = BreathingProtectionTraining.objects.filter(
                date__gt=timezone.now()).order_by('date').first()
            participants = ', '.join([unicode(f) for f in next_bpt.participants.all()])
            ctx['next_bpt'] = {
                'date': next_bpt.date,
                'location': next_bpt.location,
                'participants': participants,
            }
        except ObjectDoesNotExist:
            ctx['next_bpt'] = None

        next_messages = Message.objects.all().order_by('-creation_date')[:3]
        if next_messages:
            ctx['next_messages'] = next_messages

        return ctx

class MissionList(ListView):
    model = Mission

class MissionUpdate(UpdateView):
    model = Mission
    success_url = reverse_lazy('mission_list')
    form_class = MissionForm
    template_name = 'base/mission_form.html'

    def get_context_data(self, **kwargs):
        ctx = super(MissionUpdate, self).get_context_data(**kwargs)

        geolocator = Nominatim(format_string="%s, Landkreis Sächsische Schweiz-Osterzgebirge, Sachsen, 01833, Deutschland")
        address = '{0}, {1}'.format(self.object.street, self.object.location)

        # 11, Alte Hauptstraße, Wilschdorf, Dürrröhrsdorf-Dittersbach, Landkreis Sächsische Schweiz-Osterzgebirge, Sachsen, 01833, Deutschland
        locations = geolocator.geocode(address, False)
        if locations:
            location = locations[0]
            ctx['address'] = location.address
            ctx['lat'] = float(location.latitude)
            ctx['long'] = float(location.longitude)
            ctx['raw'] = location.raw
        return ctx

class MissionAlarm(TemplateView):
    model = Mission
    template_name = 'base/mission_alarm.html'

    def get_context_data(self, **kwargs):
        ctx = super(MissionAlarm, self).get_context_data(**kwargs)

        mission = Mission.objects.all().order_by('-alarm_time').first()
        ctx['object'] = mission

        geo_locator = GeoLocator()

        key = settings.MAP_QUEST_KEY
        address = '{0}, {1}'.format(mission.street, mission.location)

        ctx.update(geo_locator.get_location_coordinates(key, address))
        ctx.update(geo_locator.get_route(key, address))
        return ctx

class BPTrainingList(ListView):
    model = BreathingProtectionTraining
    template_name = 'base/bptraining_list.html'

class BPTrainingCreate(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    model = BreathingProtectionTraining
    success_url = reverse_lazy('bptraining_list')
    form_class = BPTrainingForm
    template_name = 'base/bptraining_form.html'
    permission_required = 'base.add_breathingprotectiontraining'

    def form_valid(self, form):
        form.instance.organizer_id = self.request.user.id
        form.instance.editor_id = self.request.user.id
        form.instance.creation_date = timezone.now()
        return super(BPTrainingCreate, self).form_valid(form)

class BPTrainingUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    model = BreathingProtectionTraining
    success_url = reverse_lazy('bptraining_list')
    form_class = BPTrainingForm
    template_name = 'base/bptraining_form.html'
    permission_required = 'base.change_breathingprotectiontraining'

    def form_valid(self, form):
        form.instance.editor_id = self.request.user.id
        return super(BPTrainingUpdate, self).form_valid(form)

class MessageList(ListView):
    model = Message
    template_name = 'base/message_list.html'

class MessageCreate(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    model = Message
    success_url = reverse_lazy('message_list')
    form_class = MessageForm
    template_name = 'base/message_form.html'
    permission_required = 'base.add_message'

    def form_valid(self, form):
        form.instance.author_id = self.request.user.id
        form.instance.editor_id = self.request.user.id
        form.instance.creation_date = timezone.now()
        return super(MessageCreate, self).form_valid(form)

class MessageUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    model = Message
    success_url = reverse_lazy('message_list')
    form_class = MessageForm
    template_name = 'base/message_form.html'
    permission_required = 'base.change_message'

    def form_valid(self, form):
        form.instance.editor_id = self.request.user.id
        return super(MessageUpdate, self).form_valid(form)

class BPCarrierList(ListView):
    model = UserProfile
    template_name = 'base/bpcarrier_list.html'

    def get_queryset(self):
        queryset = super(BPCarrierList, self).get_queryset()
        return queryset.filter(breathing_protection_carrier=True)

class BPCarrierUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView):
    model = UserProfile
    success_url = reverse_lazy('bpcarrier_list')
    form_class = BPCarrierForm
    template_name = 'base/bpcarrier_form.html'
    permission_required = 'base.change_breathingprotectioncarrier'

    def form_valid(self, form):
        form.instance.editor_id = self.request.user.id
        return super(BPCarrierUpdate, self).form_valid(form)

class UserImport(TemplateView):
    template_name = 'base/user_import.html'
    success_url = reverse_lazy('dashboard')

    def get_context_data(self, **kwargs):
        ctx = super(UserImport, self).get_context_data(**kwargs)
        importer = UserImporter()
        ctx['users'] = importer.get_users_from_csv_file()
        return ctx

    def post(self, request, *args, **kwargs):
        importer = UserImporter()
        users = importer.get_users_from_csv_file()
        importer.import_users(users, self.request.user)
        return HttpResponseRedirect(self.success_url)

class TrainingImport(TemplateView):
    template_name = 'base/training_import.html'
    success_url = reverse_lazy('dashboard')

    def get_context_data(self, **kwargs):
        ctx = super(TrainingImport, self).get_context_data(**kwargs)
        importer = TrainingImporter()
        ctx['trainings'] = importer.get_trainings_from_csv_file()
        return ctx

    def post(self, request, *args, **kwargs):
        importer = TrainingImporter()
        trainings = importer.get_trainings_from_csv_file()
        importer.import_trainings(trainings, self.request.user)
        return HttpResponseRedirect(self.success_url)

class GalleryList(TemplateView):
    template_name = 'base/gallery_list.html'

    def get_context_data(self, **kwargs):
        ctx = super(GalleryList, self).get_context_data(**kwargs)
        path = settings.MEDIA_ROOT + 'images/'
        files = []

        mtime = lambda f: os.stat(os.path.join(path, f)).st_ctime
        fls = list(sorted(os.listdir(path), key=mtime))
        index = 0
        for f in reversed(fls):
            index += 1
            file_path = join(path, f)
            if not isfile(file_path):
                continue
            files.append({
                'path': f,
                'time': time.ctime(os.path.getctime(file_path))
            })
            if index >= 15:
                break
        ctx['files'] = files
        return ctx
<|endoftext|>


<commit_before>import json
import requests
from django.core import exceptions
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from fluent_contents.models import ContentItem

@python_2_unicode_compatible
class AbstractInstagramEmbedItem(ContentItem):
    """
    An embeded instagram image.
    """
    url = models.URLField(verbose_name=_('URL'))

    # Data return from OEmbed API call.
    provider_url = models.CharField(max_length=512, blank=True)
    media_id = models.CharField(max_length=255, blank=True)
    author_name = models.CharField(max_length=255, blank=True)
    height = models.PositiveIntegerField(blank=True, null=True)
    width = models.PositiveIntegerField(blank=True, null=True)
    thumbnail_url = models.CharField(max_length=255, blank=True)
    thumbnail_width = models.PositiveIntegerField(blank=True, null=True)
    thumbnail_height = models.PositiveIntegerField(blank=True, null=True)
    provider_name = models.CharField(max_length=255, blank=True)
    title = models.CharField(max_length=512, blank=True)
    html = models.TextField(blank=True)
    version = models.CharField(max_length=20, blank=True)
    author_url = models.CharField(max_length=255, blank=True)
    author_id = models.PositiveIntegerField(blank=True, null=True)
    type = models.CharField(max_length=50, blank=True)

    class Meta:
        abstract = True
        verbose_name = _('Instagram Embed')

    def __str__(self):
        return str('Instagram Embed: %s' % self.url)

    def clean(self, *args, **kwargs):
        """
        Prefetch instagram data and clean it.
        """
        # Get the instagram data.
        instagram_data = self.fetch_instagram_data()

        # If a dict is returned assume it is the JSON data returned from instagram.
        if isinstance(instagram_data, dict):
            # Set each of the data attributes.
            for key in instagram_data.keys():
                setattr(self, key, instagram_data[key])
        else:
            raise exceptions.ValidationError(instagram_data)

        super(InstagramEmbedItem, self).clean(*args, **kwargs)

    def fetch_instagram_data(self):
        """
        Get the instagram data for the url.

        :return: Dict of data if successful or String if error.
        """
        r = requests.get('http://api.instagram.com/publicapi/oembed/?url=%s' % self.url)
        if r.status_code is 200:
            return json.loads(r.content)
        return r.content

    def get_thumbnail(self):
        """
        Get the image thumbnail url if it exists.

        :return: String.
        """
        if self.thumbnail_url:
            return self.thumbnail_url
        return ''

    def get_default_embed(self):
        """
        Get the default embed if it exists.

        :return: HTML String marked safe.
        """
        if self.html:
            return mark_safe(self.html)
        return ''
<commit_msg>Fix super call.<commit_after>import json
import requests
from django.core import exceptions
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from fluent_contents.models import ContentItem

@python_2_unicode_compatible
class AbstractInstagramEmbedItem(ContentItem):
    """
    An embeded instagram image.
    """
    url = models.URLField(verbose_name=_('URL'))

    # Data return from OEmbed API call.
    provider_url = models.CharField(max_length=512, blank=True)
    media_id = models.CharField(max_length=255, blank=True)
    author_name = models.CharField(max_length=255, blank=True)
    height = models.PositiveIntegerField(blank=True, null=True)
    width = models.PositiveIntegerField(blank=True, null=True)
    thumbnail_url = models.CharField(max_length=255, blank=True)
    thumbnail_width = models.PositiveIntegerField(blank=True, null=True)
    thumbnail_height = models.PositiveIntegerField(blank=True, null=True)
    provider_name = models.CharField(max_length=255, blank=True)
    title = models.CharField(max_length=512, blank=True)
    html = models.TextField(blank=True)
    version = models.CharField(max_length=20, blank=True)
    author_url = models.CharField(max_length=255, blank=True)
    author_id = models.PositiveIntegerField(blank=True, null=True)
    type = models.CharField(max_length=50, blank=True)

    class Meta:
        abstract = True
        verbose_name = _('Instagram Embed')

    def __str__(self):
        return str('Instagram Embed: %s' % self.url)

    def clean(self, *args, **kwargs):
        """
        Prefetch instagram data and clean it.
        """
        # Get the instagram data.
        instagram_data = self.fetch_instagram_data()

        # If a dict is returned assume it is the JSON data returned from instagram.
        if isinstance(instagram_data, dict):
            # Set each of the data attributes.
            for key in instagram_data.keys():
                setattr(self, key, instagram_data[key])
        else:
            raise exceptions.ValidationError(instagram_data)

        super(AbstractInstagramEmbedItem, self).clean(*args, **kwargs)

    def fetch_instagram_data(self):
        """
        Get the instagram data for the url.

        :return: Dict of data if successful or String if error.
        """
        r = requests.get('http://api.instagram.com/publicapi/oembed/?url=%s' % self.url)
        if r.status_code is 200:
            return json.loads(r.content)
        return r.content

    def get_thumbnail(self):
        """
        Get the image thumbnail url if it exists.

        :return: String.
        """
        if self.thumbnail_url:
            return self.thumbnail_url
        return ''

    def get_default_embed(self):
        """
        Get the default embed if it exists.

        :return: HTML String marked safe.
        """
        if self.html:
            return mark_safe(self.html)
        return ''
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import logging
import unittest
import importlib
import time
from tests.test_general import GeneralGraphTestCase
# from dipper.utils.TestUtils import TestUtils
from dipper.utils.GraphUtils import GraphUtils

logging.basicConfig()
LOG = logging.getLogger(__name__)

TEST_SUITE = unittest.TestLoader().loadTestsFromTestCase(GeneralGraphTestCase)

def main():
    # TODO this should be generated by looking in the dipper/sources directory
    source_to_class_map = {
        # 'facebase_alpha': 'FaceBase_alpha',
        'hpoa': 'HPOAnnotations',
        'zfin': 'ZFIN',
        'omim': 'OMIM',
        'biogrid': 'BioGrid',
        'mgi': 'MGI',
        'impc': 'IMPC',
        'panther': 'Panther',
        'ncbigene': 'NCBIGene',
        'ucscbands': 'UCSCBands',
        'ctd': 'CTD',
        'genereviews': 'GeneReviews',
        'eom': 'EOM',
        'coriell': 'Coriell',
        # 'clinvar': 'ClinVar',      # needs integrating here
        'monochrom': 'Monochrom',
        'kegg': 'KEGG',
        'animalqtldb': 'AnimalQTLdb',
        'ensembl': 'Ensembl',
        'hgnc': 'HGNC',
        'orphanet': 'Orphanet',
        'omia': 'OMIA',
        'flybase': 'FlyBase',
        'mmrrc': 'MMRRC',
        'wormbase': 'WormBase',
        'mpd': 'MPD',
        'gwascatalog': 'GWASCatalog',
        'monarch': 'Monarch',
        'go': 'GeneOntology',
        'reactome': 'Reactome',
        'udp': 'UDP',
        'mgi-slim': 'MGISlim',
        'zfinslim': 'ZFINSlim',
        'bgee': 'Bgee',
        'mydrug': 'MyDrug',
        'stringdb': 'StringDB',
        'rgd': 'RGD',
        'sgd': 'SGD',
        'mychem': 'MyChem',
        'ebi': 'EBIGene2Phen',
    }

    parser = argparse.ArgumentParser(
        description='Dipper: Data Ingestion Pipeline for Monarch',
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        '-g', '--graph', type=str, default="rdf_graph",
        help='graph type: rdf_graph, streamed_graph')
    parser.add_argument(
        '-s', '--sources', type=str, default='?',
        help='comma separated list of sources')
    parser.add_argument('-l', '--limit', type=int, help='limit number of rows used')
    parser.add_argument(
        '--parse_only', action='store_true', help='parse files without writing RDF')
    parser.add_argument(
        '--fetch_only', action='store_true', help='fetch sources without parsing')
    parser.add_argument(
        '-f', '--force', action='store_true', help='force re-download of files')
    parser.add_argument(
        '--no_verify', help='ignore the verification step', action='store_true')
    # parser.add_argument( '--query', help='enter in a sparql query', type=str)
    parser.add_argument(
        '-q', '--quiet', help='turn off info logging', action="store_true")
    parser.add_argument(
        '--debug', help='turn on debug logging', action="store_true")
    parser.add_argument(
        '--skip_tests', help='skip any testing', action="store_true")
    # Blank Nodes can't be visualized in Protege, default to Skolemizing them
    parser.add_argument(
        '-b', '--use_bnodes',
        help="use blank nodes instead of skolemizing", action="store_true",
        default=False)
    #
    parser.add_argument(  # TODO help needs revisiting, push constraints off the the src
        '-t', '--taxon', type=str, help='''
            Constrain Source to supplied taxon identifier(s).
            Please enter comma delimited NCBITaxon numbers:
            Implemented taxa per source
            NCBIGene: 9606,10090,7955
            Panther: 9606,10090,10116,7227,7955,6239,8355
            BioGrid: 9606,10090,10116,7227,7955,6239,8355
            UCSCBands: 9606
            GO: 9606,10090,10116,7227,7955,6239,9615,9823,9031,9913,4896,5782,5052
    ''')
    parser.add_argument(
        '-o', '--test_only',
        help='only process and output the pre-configured test subset',
        action="store_true")

    parser.add_argument(
        '--dest_fmt',
        help='serialization format: [turtle], nt, nquads, rdfxml, n3, raw', type=str)

    parser.add_argument(
        '-v', '--version', help='version of source (deprecated)', type=str)

    parser.add_argument(
        '-d', '--data_release_version',
        help='''
            string indicating the version of data release, e.g. '\'201908\' (YYYYMM),
            used to construct metadata, including version and distribution IRIs
            and downloadURLs
            [defaults to date at start of runtime in ISO 8601 format]
        ''',
        type=str)

    args = parser.parse_args()
    tax_ids = None
    if args.taxon is not None:
        tax_ids = [str(t) for t in args.taxon.split(',') if t.isdigit()]

    species_specific = [
        'Panther', 'NCBIGene', 'BioGrid', 'UCSCBands',
        'GeneOntology', 'Bgee', 'StringDB', 'Ensembl']

    formats_supported = [
        'turtle', 'ttl',
        'ntriples', 'nt',
        'nquads', 'nq',
        'rdfxml', 'xml',
        'notation3', 'n3',
        'raw']

    if args.quiet:
        logging.getLogger().setLevel(logging.WARNING)
    else:
        if args.debug:
            logging.getLogger().setLevel(logging.DEBUG)
        else:
            logging.getLogger().setLevel(logging.INFO)

    if not args.use_bnodes:
        LOG.info("Will Skolemize Blank Nodes")

    # None of these query test utils exist in ./dipper/utils/TestUtils.py
    # if args.query is not None:
    #    test_query = TestUtils()
    #    for source in args.sources.split(','):
    #        source = source.lower()
    #        mysource = source_to_class_map[source]()
    #        # import source lib
    #        module = "dipper.sources.{0}".format(mysource)
    #        imported_module = importlib.import_module(module)
    #        source_class = getattr(imported_module, mysource)
    #        test_query.check_query_syntax(args.query, source_class)
    #        test_query.load_graph_from_turtle(source_class)
    #
    #    print(test_query.query_graph(args.query, True))
    #    exit(0)

    # run initial tests
    if (args.no_verify or args.skip_tests) is not True:
        unittest.TextTestRunner(verbosity=2).run(TEST_SUITE)

    # set serializer
    if args.dest_fmt is not None:
        if args.dest_fmt in formats_supported:
            if args.dest_fmt == 'ttl':
                args.dest_fmt = 'turtle'
            elif args.dest_fmt == 'ntriples':
                args.dest_fmt = 'nt'
            elif args.dest_fmt == 'nq':
                args.dest_fmt = 'nquads'
            elif args.dest_fmt == 'xml':
                args.dest_fmt = 'rdfxml'
            elif args.dest_fmt == 'notation3':
                args.dest_fmt = 'n3'
        else:
            LOG.error("You have specified an invalid serializer: %s", args.dest_fmt)
            exit(0)
    else:
        args.dest_fmt = 'turtle'

    # Provide feedback if we can't proceed
    if args.sources is None or args.sources.split(',')[0] not in source_to_class_map:
        LOG.info('Unknown Source %s', args.sources.split(',')[0])
        LOG.info('Sources Known are limited to:')
        for key in sorted(source_to_class_map):
            LOG.info('\t%s\t%s', key, source_to_class_map[key])
        exit(0)

    # iterate through all the sources
    for source in args.sources.split(','):
        LOG.info("\n******* %s *******", source)
        source = source.lower()
        src = source_to_class_map[source]

        # import source lib
        module = "dipper.sources.{0}".format(src)
        imported_module = importlib.import_module(module)
        source_class = getattr(imported_module, src)
        mysource = None

        LOG.info(
            'Command line arguments available to dipper-etl:\n%s',
            "\n".join(['\t{}: {}'.format(k, v) for k, v in vars(args).items()]))

        source_args = dict(graph_type=args.graph)
        source_args['are_bnodes_skolemized'] = not args.use_bnodes
        if src in species_specific:
            source_args['tax_ids'] = tax_ids
        if args.version:
            source_args['version'] = args.version
        if args.data_release_version:
            source_args['data_release_version'] = args.data_release_version

        mysource = source_class(**source_args)

        # WIP cli args should be available to source
        if hasattr(mysource, 'ARGV'):
            mysource.ARGV = vars(args)

        if args.parse_only is False:
            start_fetch = time.perf_counter()
            mysource.fetch(args.force)

            end_fetch = time.perf_counter()
            LOG.info("Fetching time: %d sec", end_fetch - start_fetch)

        mysource.settestonly(args.test_only)

        # run tests first
        if (args.no_verify or args.skip_tests) is not True:
            suite = mysource.getTestSuite()
            if suite is None:
                LOG.warning("No tests configured for this source: %s", source)
            else:
                unittest.TextTestRunner(verbosity=2).run(suite)
        else:
            LOG.info("Skipping Tests for source: %s", source)

        if args.test_only is False and args.fetch_only is False:
            start_parse = time.perf_counter()
            mysource.parse(args.limit)

            end_parse = time.perf_counter()
            LOG.info("Parsing time: %d sec", end_parse - start_parse)

            if args.graph == 'rdf_graph':
                LOG.info("Found %d nodes", len(mysource.graph))

                # Add property axioms
                start_axiom_exp = time.perf_counter()
                LOG.info("Adding property axioms")

                properties = GraphUtils.get_properties_from_graph(mysource.graph)
                GraphUtils.add_property_axioms(mysource.graph, properties)
                LOG.info(
                    "Property axioms added: %d sec",
                    time.perf_counter() - start_axiom_exp)

                start_write = time.perf_counter()
                mysource.write(fmt=args.dest_fmt)
                LOG.info("Writing time: %d sec", time.perf_counter() - start_write)
            # elif args.graph == 'streamed_graph': ...

        LOG.info('***** Finished with %s *****', source)

    LOG.info("All done.")

if __name__ == "__main__":
    main()

###########################
<commit_msg>reprioritize source generation; more logging<commit_after>#!/usr/bin/env python3

import argparse
import logging
import unittest
import importlib
import time
from tests.test_general import GeneralGraphTestCase
# from dipper.utils.TestUtils import TestUtils
from dipper.utils.GraphUtils import GraphUtils

logging.basicConfig()
LOG = logging.getLogger(__name__)

TEST_SUITE = unittest.TestLoader().loadTestsFromTestCase(GeneralGraphTestCase)

def main():
    # TODO this should be generated by looking in the dipper/sources directory
    source_to_class_map = {
        # 'facebase_alpha': 'FaceBase_alpha',
        'hpoa': 'HPOAnnotations',
        'zfin': 'ZFIN',
        'omim': 'OMIM',
        'biogrid': 'BioGrid',
        'mgi': 'MGI',
        'impc': 'IMPC',
        'panther': 'Panther',
        'ncbigene': 'NCBIGene',
        'ucscbands': 'UCSCBands',
        'ctd': 'CTD',
        'genereviews': 'GeneReviews',
        'eom': 'EOM',
        'coriell': 'Coriell',
        # 'clinvar': 'ClinVar',      # needs integrating here
        'monochrom': 'Monochrom',
        'kegg': 'KEGG',
        'animalqtldb': 'AnimalQTLdb',
        'ensembl': 'Ensembl',
        'hgnc': 'HGNC',
        'orphanet': 'Orphanet',
        'omia': 'OMIA',
        'flybase': 'FlyBase',
        'mmrrc': 'MMRRC',
        'wormbase': 'WormBase',
        'mpd': 'MPD',
        'gwascatalog': 'GWASCatalog',
        'monarch': 'Monarch',
        'go': 'GeneOntology',
        'reactome': 'Reactome',
        'udp': 'UDP',
        'mgi-slim': 'MGISlim',
        'zfinslim': 'ZFINSlim',
        'bgee': 'Bgee',
        'mydrug': 'MyDrug',
        'stringdb': 'StringDB',
        'rgd': 'RGD',
        'sgd': 'SGD',
        'mychem': 'MyChem',
        'ebi': 'EBIGene2Phen',
    }

    parser = argparse.ArgumentParser(
        description='Dipper: Data Ingestion Pipeline for Monarch',
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument(
        '-g', '--graph', type=str, default="rdf_graph",
        help='graph type: rdf_graph, streamed_graph')
    parser.add_argument(
        '-s', '--sources', type=str, default='?',
        help='comma separated list of sources')
    parser.add_argument('-l', '--limit', type=int, help='limit number of rows used')
    parser.add_argument(
        '--parse_only', action='store_true', help='parse files without writing RDF')
    parser.add_argument(
        '--fetch_only', action='store_true', help='fetch sources without parsing')
    parser.add_argument(
        '-f', '--force', action='store_true', help='force re-download of files')
    parser.add_argument(
        '--no_verify', help='ignore the verification step', action='store_true')
    # parser.add_argument( '--query', help='enter in a sparql query', type=str)
    parser.add_argument(
        '-q', '--quiet', help='turn off info logging', action="store_true")
    parser.add_argument(
        '--debug', help='turn on debug logging', action="store_true")
    parser.add_argument(
        '--skip_tests', help='skip any testing', action="store_true")
    # Blank Nodes can't be visualized in Protege, default to Skolemizing them
    parser.add_argument(
        '-b', '--use_bnodes',
        help="use blank nodes instead of skolemizing", action="store_true",
        default=False)
    #
    parser.add_argument(  # TODO help needs revisiting, push constraints off the the src
        '-t', '--taxon', type=str, help='''
            Constrain Source to supplied taxon identifier(s).
            Please enter comma delimited NCBITaxon numbers:
            Implemented taxa per source
            NCBIGene: 9606,10090,7955
            Panther: 9606,10090,10116,7227,7955,6239,8355
            BioGrid: 9606,10090,10116,7227,7955,6239,8355
            UCSCBands: 9606
            GO: 9606,10090,10116,7227,7955,6239,9615,9823,9031,9913,4896,5782,5052
    ''')
    parser.add_argument(
        '-o', '--test_only',
        help='only process and output the pre-configured test subset',
        action="store_true")

    parser.add_argument(
        '--dest_fmt',
        help='serialization format: [turtle], nt, nquads, rdfxml, n3, raw', type=str)

    parser.add_argument(
        '-v', '--version', help='version of source (deprecated)', type=str)

    parser.add_argument(
        '-d', '--data_release_version',
        help='''
            string indicating the version of data release, e.g. '\'201908\' (YYYYMM),
            used to construct metadata, including version and distribution IRIs
            and downloadURLs
            [defaults to date at start of runtime in ISO 8601 format]
        ''',
        type=str)

    args = parser.parse_args()
    tax_ids = None
    if args.taxon is not None:
        tax_ids = [str(t) for t in args.taxon.split(',') if t.isdigit()]

    species_specific = [
        'Panther', 'NCBIGene', 'BioGrid', 'UCSCBands',
        'GeneOntology', 'Bgee', 'StringDB', 'Ensembl']

    formats_supported = [
        'turtle', 'ttl',
        'ntriples', 'nt',
        'nquads', 'nq',
        'rdfxml', 'xml',
        'notation3', 'n3',
        'raw']

    if args.quiet:
        logging.getLogger().setLevel(logging.WARNING)
    else:
        if args.debug:
            logging.getLogger().setLevel(logging.DEBUG)
        else:
            logging.getLogger().setLevel(logging.INFO)

    if not args.use_bnodes:
        LOG.info("Will Skolemize Blank Nodes")

    # None of these query test utils exist in ./dipper/utils/TestUtils.py
    # if args.query is not None:
    #    test_query = TestUtils()
    #    for source in args.sources.split(','):
    #        source = source.lower()
    #        mysource = source_to_class_map[source]()
    #        # import source lib
    #        module = "dipper.sources.{0}".format(mysource)
    #        imported_module = importlib.import_module(module)
    #        source_class = getattr(imported_module, mysource)
    #        test_query.check_query_syntax(args.query, source_class)
    #        test_query.load_graph_from_turtle(source_class)
    #
    #    print(test_query.query_graph(args.query, True))
    #    exit(0)

    # run initial tests
    if (args.no_verify or args.skip_tests) is not True:
        unittest.TextTestRunner(verbosity=2).run(TEST_SUITE)

    # set serializer
    if args.dest_fmt is not None:
        if args.dest_fmt in formats_supported:
            if args.dest_fmt == 'ttl':
                args.dest_fmt = 'turtle'
            elif args.dest_fmt == 'ntriples':
                args.dest_fmt = 'nt'
            elif args.dest_fmt == 'nq':
                args.dest_fmt = 'nquads'
            elif args.dest_fmt == 'xml':
                args.dest_fmt = 'rdfxml'
            elif args.dest_fmt == 'notation3':
                args.dest_fmt = 'n3'
        else:
            LOG.error("You have specified an invalid serializer: %s", args.dest_fmt)
            exit(0)
    else:
        args.dest_fmt = 'turtle'

    # Provide feedback if we can't proceed
    if args.sources is None or args.sources.split(',')[0] not in source_to_class_map:
        LOG.info('Unknown Source %s', args.sources.split(',')[0])
        LOG.info('Sources Known are limited to:')
        for key in sorted(source_to_class_map):
            LOG.info('\t%s\t%s', key, source_to_class_map[key])
        exit(0)

    # iterate through all the sources
    for source in args.sources.split(','):
        LOG.info("\n******* %s *******", source)
        source = source.lower()
        src = source_to_class_map[source]

        # import source lib
        module = "dipper.sources.{0}".format(src)
        imported_module = importlib.import_module(module)
        source_class = getattr(imported_module, src)
        mysource = None

        LOG.info(
            'Command line arguments available to dipper-etl:\n%s',
            "\n".join(['\t{}: {}'.format(k, v) for k, v in vars(args).items()]))

        source_args = dict(graph_type=args.graph)
        source_args['are_bnodes_skolemized'] = not args.use_bnodes
        if src in species_specific:
            source_args['tax_ids'] = tax_ids
        if args.version:
            source_args['version'] = args.version
        if args.data_release_version:
            source_args['data_release_version'] = args.data_release_version

        mysource = source_class(**source_args)

        # WIP cli args should be available to source
        if hasattr(mysource, 'ARGV'):
            mysource.ARGV = vars(args)
        else:
            LOG.error('no where to to put args in %s', mysource.__class__)

        if args.parse_only is False:
            start_fetch = time.perf_counter()
            mysource.fetch(args.force)

            end_fetch = time.perf_counter()
            LOG.info("Fetching time: %d sec", end_fetch - start_fetch)

        mysource.settestonly(args.test_only)

        # create source ingest graph first (with pristine arguments)
        if args.test_only is False and args.fetch_only is False:
            start_parse = time.perf_counter()
            mysource.parse(args.limit)

            end_parse = time.perf_counter()
            LOG.info("Parsing time: %d sec", end_parse - start_parse)

            if args.graph == 'rdf_graph':
                LOG.info("Found %d nodes", len(mysource.graph))

                # Add property axioms
                start_axiom_exp = time.perf_counter()
                LOG.info("Adding property axioms")

                properties = GraphUtils.get_properties_from_graph(mysource.graph)
                GraphUtils.add_property_axioms(mysource.graph, properties)
                LOG.info(
                    "Property axioms added: %d sec",
                    time.perf_counter() - start_axiom_exp)

                start_write = time.perf_counter()
                mysource.write(fmt=args.dest_fmt)
                LOG.info("Writing time: %d sec", time.perf_counter() - start_write)
            # elif args.graph == 'streamed_graph': ...

        # '*_test.ttl' graphs if requested
        if (args.no_verify or args.skip_tests) is False:
            suite = mysource.getTestSuite()
            if suite is None:
                LOG.warning("No tests configured for this source: %s", source)
            else:
                unittest.TextTestRunner(verbosity=2).run(suite)
        else:
            LOG.info("Skipping Tests for source: %s", source)

        LOG.info('***** Finished with %s *****', source)

    LOG.info("All done.")

if __name__ == "__main__":
    main()

###########################
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Author: Milan Oberkirch <zvyn@oberkirch.org>

"""
Provides a `requires_auth` decorator which enables basic auth for the decorated
function by checking for config section called username and an entry
password_hash. If the lookup fails or the hmac hash of the given password is
not equivalent to the one in the `config_file`, the authentication fails and a
401 response is sent to the client.
"""

from crypt import crypt
from hmac import compare_digest as compare_hash
from configparser import ConfigParser
from functools import wraps
from flask import request, Response

config = ConfigParser()
config_file = 'basic_auth.ini'
config.read(config_file)

def check_auth(username, password):
    """
    Check user and password using hmac.
    """
    try:
        password_hash = config[username]["password_hash"]
        return compare_hash(password_hash, crypt(password, password_hash))
    except KeyError:
        return False

def authenticate():
    return Response(
        'Please log in to use this service.', 401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

def change_password(username, new_password):
    config[username] = crypt(new_password)
    config.write(config_file)

if __name__ == "__main__":
    from flask import Flask
    app = Flask(__name__)

    @app.route("/")
    @requires_auth
    def secret():
        return "Hello World!"

    app.debug = True
    app.run()
<commit_msg>Make location of config dependent on environment.<commit_after>#!/usr/bin/env python3
# Author: Milan Oberkirch <zvyn@oberkirch.org>

"""
Provides a `requires_auth` decorator which enables basic auth for the decorated
function by checking for config section called username and an entry
password_hash. If the lookup fails or the hmac hash of the given password is
not equivalent to the one in the `config_file`, the authentication fails and a
401 response is sent to the client.
"""

from os import environ
from crypt import crypt
from hmac import compare_digest as compare_hash
from configparser import ConfigParser
from functools import wraps
from flask import request, Response

config = ConfigParser()
config_file = (
    environ['BASIC_AUTH_INI'] if 'BASIC_AUTH_INI' in environ
    else 'basic_auth.ini')
config.read(config_file)

def check_auth(username, password):
    """
    Check user and password using hmac.
    """
    try:
        password_hash = config[username]["password_hash"]
        return compare_hash(password_hash, crypt(password, password_hash))
    except KeyError:
        return False

def authenticate():
    return Response(
        'Please log in to use this service.', 401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

def change_password(username, new_password):
    config[username] = crypt(new_password)
    config.write(config_file)

if __name__ == "__main__":
    from flask import Flask
    app = Flask(__name__)

    @app.route("/")
    @requires_auth
    def secret():
        return "Hello World!"

    app.debug = True
    app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# TODO
# Gatewayliste
# aliases.json

import json
import fileinput
import argparse

from nodedb import NodeDB
from geomapbuilder import GeoMapBuilder

parser = argparse.ArgumentParser()

parser.add_argument('-a', '--aliases',
                  help='read aliases from FILE',
                  metavar='FILE')

parser.add_argument('batmanjson', help='output of batman vd json')

args = parser.parse_args()

options = vars(args)

db = NodeDB()

db.import_batman(list(fileinput.input(options['batmanjson'])))

if options['aliases']:
  db.import_aliases(json.load(open(options['aliases'])))

db.import_wikigps("http://freifunk.metameute.de/Knoten")

#db.wilder_scheiss()

m = GeoMapBuilder(db)

print(m.build())
<commit_msg>encode kml using utf8<commit_after>#!/usr/bin/env python

# TODO
# Gatewayliste
# aliases.json

import json
import fileinput
import argparse

from nodedb import NodeDB
from geomapbuilder import GeoMapBuilder

parser = argparse.ArgumentParser()

parser.add_argument('-a', '--aliases',
                  help='read aliases from FILE',
                  metavar='FILE')

parser.add_argument('batmanjson', help='output of batman vd json')

args = parser.parse_args()

options = vars(args)

db = NodeDB()

db.import_batman(list(fileinput.input(options['batmanjson'])))

if options['aliases']:
  db.import_aliases(json.load(open(options['aliases'])))

db.import_wikigps("http://freifunk.metameute.de/Knoten")

#db.wilder_scheiss()

m = GeoMapBuilder(db)

print(m.build().encode("utf8"))
<|endoftext|>