<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# Copyright (C) 2003-2004 Jonas Borgström <jonas@edgewall.com>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Jonas Borgström <jonas@edgewall.com>

import pkg_resources
import re

from trac.config import IntOption, ListOption
from trac.core import *
from trac.perm import IPermissionRequestor
from trac.search.api import ISearchSource
from trac.util.datefmt import format_datetime, user_time
from trac.util.html import Markup, escape, find_element, tag
from trac.util.presentation import Paginator
from trac.util.text import quote_query_string
from trac.util.translation import _
from trac.web.api import IRequestHandler
from trac.web.chrome import (INavigationContributor, ITemplateProvider,
                             accesskey, add_link, add_notice, add_stylesheet,
                             add_warning, web_context)
from trac.wiki.api import IWikiSyntaxProvider
from trac.wiki.formatter import extract_link

class SearchModule(Component):
    """Controller for the search sub-system"""

    implements(INavigationContributor, IPermissionRequestor, IRequestHandler,
               ITemplateProvider, IWikiSyntaxProvider)

    search_sources = ExtensionPoint(ISearchSource)

    RESULTS_PER_PAGE = 10

    min_query_length = IntOption('search', 'min_query_length', 3,
        """Minimum length of query string allowed when performing a search.
        """)

    default_disabled_filters = ListOption('search',
                                          'default_disabled_filters',
        doc="""Specifies which search filters should be disabled by
               default on the search page. This will also restrict the
               filters for the quick search function. The filter names
               defined by default components are: `wiki`, `ticket`,
               `milestone` and `changeset`.  For plugins, look for
               their implementation of the ISearchSource interface, in
               the `get_search_filters()` method, the first member of
               returned tuple. Once disabled, search filters can still
               be manually enabled by the user on the search page.
               (''since 0.12'')""")

    # INavigationContributor methods

    def get_active_navigation_item(self, req):
        return 'search'

    def get_navigation_items(self, req):
        if 'SEARCH_VIEW' in req.perm:
            yield ('mainnav', 'search',
                   tag.a(_('Search'), href=req.href.search(),
                         accesskey=accesskey(req, 4)))

    # IPermissionRequestor methods

    def get_permission_actions(self):
        return ['SEARCH_VIEW']

    # IRequestHandler methods

    def match_request(self, req):
        return re.match(r'/search(?:/opensearch)?$', req.path_info) \
               is not None

    def process_request(self, req):
        req.perm.assert_permission('SEARCH_VIEW')

        if req.path_info == '/search/opensearch':
            return ('opensearch.xml', {},
                    {'content_type': 'application/opensearchdescription+xml'})

        query = req.args.get('q')
        available_filters = []
        for source in self.search_sources:
            available_filters.extend(source.get_search_filters(req) or [])
        available_filters.sort(key=lambda f: f[1].lower())

        filters = self._get_selected_filters(req, available_filters)
        data = self._prepare_data(req, query, available_filters, filters)
        if query:
            noquickjump = req.args.getbool('noquickjump', False)
            data['quickjump'] = self._check_quickjump(req, noquickjump, query)
            if query.startswith('!'):
                query = query[1:]

            terms = self._parse_query(req, query)
            if terms:
                results = self._do_search(req, terms, filters)
                if results:
                    data.update(self._prepare_results(req, filters, results))
            if noquickjump and filters:
                req.session['search.filters'] = ','.join(filters)

        add_stylesheet(req, 'common/css/search.css')
        return 'search.html', data

    # ITemplateProvider methods

    def get_htdocs_dirs(self):
        return []

    def get_templates_dirs(self):
        return [pkg_resources.resource_filename('trac.search', 'templates')]

    # IWikiSyntaxProvider methods

    def get_wiki_syntax(self):
        return []

    def get_link_resolvers(self):
        yield ('search', self._format_link)

    def _format_link(self, formatter, ns, target, label):
        path, query, fragment = formatter.split_link(target)
        if path:
            href = formatter.href.search(q=path)
            if query:
                href += '&' + quote_query_string(query[1:])
        else:
            href = formatter.href.search() + quote_query_string(query)
        href += fragment
        return tag.a(label, class_='search', href=href)

    # IRequestHandler helper methods

    def _get_selected_filters(self, req, available_filters):
        """Return selected filters or the default filters if none was
        selected.
        """
        filters = [f[0] for f in available_filters if f[0] in req.args]
        if not filters:
            session_filters = req.session.get('search.filters')
            if session_filters:
                available_filters_dict = dict([f[0:2]
                                               for f in available_filters])
                filters = [f for f in session_filters.split(',')
                           if f in available_filters_dict]
            else:
                filters = [f[0] for f in available_filters
                           if f[0] not in self.default_disabled_filters and
                           (len(f) < 3 or len(f) > 2 and f[2])]
        return filters

    def _prepare_data(self, req, query, available_filters, filters):
        return {'filters': [{'name': f[0], 'label': f[1],
                             'active': f[0] in filters}
                            for f in available_filters],
                'query': query, 'quickjump': None, 'results': []}

    def _check_quickjump(self, req, noquickjump, kwd):
        """Look for search shortcuts"""
        # Source quickjump  FIXME: delegate to ISearchSource.search_quickjump
        quickjump_href = None
        if kwd[0] == '/':
            quickjump_href = req.href.browser(kwd)
            name = kwd
            description = _('Browse repository path %(path)s', path=kwd)
        else:
            context = web_context(req, 'search')
            link = find_element(extract_link(self.env, context, kwd), 'href')
            if link is not None:
                quickjump_href = link.attrib.get('href')
                name = link.children
                description = link.attrib.get('title', '')
        if quickjump_href:
            # Only automatically redirect to local quickjump links
            if not quickjump_href.startswith(req.base_path or '/'):
                noquickjump = True
            if noquickjump:
                return {'href': quickjump_href, 'name': tag.em(name),
                        'description': description}
            else:
                help_url = req.href.wiki('TracSearch') + '#Quicksearches'
                search_url = req.href.search(q=kwd, noquickjump=1)
                # FIXME: use tag_
                add_notice(req, Markup(_(
                    'You arrived here through the '
                    'quick-jump search feature. To instead search for the '
                    'term %(term)s, click here.',
                    help_url=escape(help_url), term=escape(kwd),
                    search_url=escape(search_url))))
                req.redirect(quickjump_href)

    def _get_search_terms(self, query):
        """Break apart a search query into its various search terms.

        Terms are grouped implicitly by word boundary, or explicitly by (single
        or double) quotes.
        """
        terms = []
        for term in re.split('(".*?")|(\'.*?\')|(\s+)', query):
            if term is not None and term.strip():
                if term[0] == term[-1] and term[0] in "'\"":
                    term = term[1:-1]
                terms.append(term)
        return terms

    def _parse_query(self, req, query):
        """Parse query and refuse those which would result in a huge result set
        """
        terms = self._get_search_terms(query)
        if terms and (len(terms) > 1 or
                      len(terms[0]) >= self.min_query_length):
            return terms

        add_warning(req, _('Search query too short. '
                           'Query must be at least %(num)s characters long.',
                           num=self.min_query_length))

    def _do_search(self, req, terms, filters):
        results = []
        for source in self.search_sources:
            results.extend(source.get_search_results(req, terms, filters)
                           or [])
        return sorted(results, key=lambda x: x[2], reverse=True)

    def _prepare_results(self, req, filters, results):
        page = req.args.getint('page', 1, min=1)
        try:
            results = Paginator(results, page - 1, self.RESULTS_PER_PAGE)
        except TracError:
            add_warning(req, _("Page %(page)s is out of range.", page=page))
            page = 1
            results = Paginator(results, page - 1, self.RESULTS_PER_PAGE)

        for idx, result in enumerate(results):
            results[idx] = {'href': result[0], 'title': result[1],
                            'date': user_time(req, format_datetime, result[2]),
                            'author': result[3], 'excerpt': result[4]}

        pagedata = []
        shown_pages = results.get_shown_pages(21)
        for shown_page in shown_pages:
            page_href = req.href.search([(f, 'on') for f in filters],
                                        q=req.args.get('q'),
                                        page=shown_page, noquickjump=1)
            pagedata.append([page_href, None, str(shown_page),
                             _("Page %(num)d", num=shown_page)])

        fields = ['href', 'class', 'string', 'title']
        results.shown_pages = [dict(zip(fields, p)) for p in pagedata]

        results.current_page = {'href': None, 'class': 'current',
                                'string': str(results.page + 1),
                                'title': None}

        if results.has_next_page:
            next_href = req.href.search(zip(filters, ['on'] * len(filters)),
                                        q=req.args.get('q'), page=page + 1,
                                        noquickjump=1)
            add_link(req, 'next', next_href, _('Next Page'))

        if results.has_previous_page:
            prev_href = req.href.search(zip(filters, ['on'] * len(filters)),
                                        q=req.args.get('q'), page=page - 1,
                                        noquickjump=1)
            add_link(req, 'prev', prev_href, _('Previous Page'))

        page_href = req.href.search(
            zip(filters, ['on'] * len(filters)), q=req.args.get('q'),
            noquickjump=1)
        return {'results': results, 'page_href': page_href}
<commit_msg>1.3.2dev: Merge r15514 from 1.2-stable<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# Copyright (C) 2003-2004 Jonas Borgström <jonas@edgewall.com>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Jonas Borgström <jonas@edgewall.com>

import pkg_resources
import re

from trac.config import IntOption, ListOption
from trac.core import *
from trac.perm import IPermissionRequestor
from trac.search.api import ISearchSource
from trac.util.datefmt import format_datetime, user_time
from trac.util.html import Markup, escape, find_element, tag
from trac.util.presentation import Paginator
from trac.util.text import quote_query_string
from trac.util.translation import _
from trac.web.api import IRequestHandler
from trac.web.chrome import (INavigationContributor, ITemplateProvider,
                             accesskey, add_link, add_notice, add_stylesheet,
                             add_warning, web_context)
from trac.wiki.api import IWikiSyntaxProvider
from trac.wiki.formatter import extract_link

class SearchModule(Component):
    """Controller for the search sub-system"""

    implements(INavigationContributor, IPermissionRequestor, IRequestHandler,
               ITemplateProvider, IWikiSyntaxProvider)

    search_sources = ExtensionPoint(ISearchSource)

    RESULTS_PER_PAGE = 10

    min_query_length = IntOption('search', 'min_query_length', 3,
        """Minimum length of query string allowed when performing a search.
        """)

    default_disabled_filters = ListOption('search',
                                          'default_disabled_filters',
        doc="""Specifies which search filters should be disabled by
               default on the search page. This will also restrict the
               filters for the quick search function. The filter names
               defined by default components are: `wiki`, `ticket`,
               `milestone` and `changeset`.  For plugins, look for
               their implementation of the ISearchSource interface, in
               the `get_search_filters()` method, the first member of
               returned tuple. Once disabled, search filters can still
               be manually enabled by the user on the search page.
               (''since 0.12'')""")

    # INavigationContributor methods

    def get_active_navigation_item(self, req):
        return 'search'

    def get_navigation_items(self, req):
        if 'SEARCH_VIEW' in req.perm:
            yield ('mainnav', 'search',
                   tag.a(_('Search'), href=req.href.search(),
                         accesskey=accesskey(req, 4)))

    # IPermissionRequestor methods

    def get_permission_actions(self):
        return ['SEARCH_VIEW']

    # IRequestHandler methods

    def match_request(self, req):
        return re.match(r'/search(?:/opensearch)?$', req.path_info) \
               is not None

    def process_request(self, req):
        req.perm.assert_permission('SEARCH_VIEW')

        if req.path_info == '/search/opensearch':
            return ('opensearch.xml', {},
                    {'content_type': 'application/opensearchdescription+xml'})

        query = req.args.getfirst('q')
        available_filters = []
        for source in self.search_sources:
            available_filters.extend(source.get_search_filters(req) or [])
        available_filters.sort(key=lambda f: f[1].lower())

        filters = self._get_selected_filters(req, available_filters)
        data = self._prepare_data(req, query, available_filters, filters)
        if query:
            noquickjump = req.args.getbool('noquickjump', False)
            data['quickjump'] = self._check_quickjump(req, noquickjump, query)
            if query.startswith('!'):
                query = query[1:]

            terms = self._parse_query(req, query)
            if terms:
                results = self._do_search(req, terms, filters)
                if results:
                    data.update(self._prepare_results(req, filters, results))
            if noquickjump and filters:
                req.session['search.filters'] = ','.join(filters)

        add_stylesheet(req, 'common/css/search.css')
        return 'search.html', data

    # ITemplateProvider methods

    def get_htdocs_dirs(self):
        return []

    def get_templates_dirs(self):
        return [pkg_resources.resource_filename('trac.search', 'templates')]

    # IWikiSyntaxProvider methods

    def get_wiki_syntax(self):
        return []

    def get_link_resolvers(self):
        yield ('search', self._format_link)

    def _format_link(self, formatter, ns, target, label):
        path, query, fragment = formatter.split_link(target)
        if path:
            href = formatter.href.search(q=path)
            if query:
                href += '&' + quote_query_string(query[1:])
        else:
            href = formatter.href.search() + quote_query_string(query)
        href += fragment
        return tag.a(label, class_='search', href=href)

    # IRequestHandler helper methods

    def _get_selected_filters(self, req, available_filters):
        """Return selected filters or the default filters if none was
        selected.
        """
        filters = [f[0] for f in available_filters if f[0] in req.args]
        if not filters:
            session_filters = req.session.get('search.filters')
            if session_filters:
                available_filters_dict = dict([f[0:2]
                                               for f in available_filters])
                filters = [f for f in session_filters.split(',')
                           if f in available_filters_dict]
            else:
                filters = [f[0] for f in available_filters
                           if f[0] not in self.default_disabled_filters and
                           (len(f) < 3 or len(f) > 2 and f[2])]
        return filters

    def _prepare_data(self, req, query, available_filters, filters):
        return {'filters': [{'name': f[0], 'label': f[1],
                             'active': f[0] in filters}
                            for f in available_filters],
                'query': query, 'quickjump': None, 'results': []}

    def _check_quickjump(self, req, noquickjump, kwd):
        """Look for search shortcuts"""
        # Source quickjump  FIXME: delegate to ISearchSource.search_quickjump
        quickjump_href = None
        if kwd[0] == '/':
            quickjump_href = req.href.browser(kwd)
            name = kwd
            description = _('Browse repository path %(path)s', path=kwd)
        else:
            context = web_context(req, 'search')
            link = find_element(extract_link(self.env, context, kwd), 'href')
            if link is not None:
                quickjump_href = link.attrib.get('href')
                name = link.children
                description = link.attrib.get('title', '')
        if quickjump_href:
            # Only automatically redirect to local quickjump links
            if not quickjump_href.startswith(req.base_path or '/'):
                noquickjump = True
            if noquickjump:
                return {'href': quickjump_href, 'name': tag.em(name),
                        'description': description}
            else:
                help_url = req.href.wiki('TracSearch') + '#Quicksearches'
                search_url = req.href.search(q=kwd, noquickjump=1)
                # FIXME: use tag_
                add_notice(req, Markup(_(
                    'You arrived here through the '
                    'quick-jump search feature. To instead search for the '
                    'term %(term)s, click here.',
                    help_url=escape(help_url), term=escape(kwd),
                    search_url=escape(search_url))))
                req.redirect(quickjump_href)

    def _get_search_terms(self, query):
        """Break apart a search query into its various search terms.

        Terms are grouped implicitly by word boundary, or explicitly by (single
        or double) quotes.
        """
        terms = []
        for term in re.split('(".*?")|(\'.*?\')|(\s+)', query):
            if term is not None and term.strip():
                if term[0] == term[-1] and term[0] in "'\"":
                    term = term[1:-1]
                terms.append(term)
        return terms

    def _parse_query(self, req, query):
        """Parse query and refuse those which would result in a huge result set
        """
        terms = self._get_search_terms(query)
        if terms and (len(terms) > 1 or
                      len(terms[0]) >= self.min_query_length):
            return terms

        add_warning(req, _('Search query too short. '
                           'Query must be at least %(num)s characters long.',
                           num=self.min_query_length))

    def _do_search(self, req, terms, filters):
        results = []
        for source in self.search_sources:
            results.extend(source.get_search_results(req, terms, filters)
                           or [])
        return sorted(results, key=lambda x: x[2], reverse=True)

    def _prepare_results(self, req, filters, results):
        page = req.args.getint('page', 1, min=1)
        try:
            results = Paginator(results, page - 1, self.RESULTS_PER_PAGE)
        except TracError:
            add_warning(req, _("Page %(page)s is out of range.", page=page))
            page = 1
            results = Paginator(results, page - 1, self.RESULTS_PER_PAGE)

        for idx, result in enumerate(results):
            results[idx] = {'href': result[0], 'title': result[1],
                            'date': user_time(req, format_datetime, result[2]),
                            'author': result[3], 'excerpt': result[4]}

        pagedata = []
        shown_pages = results.get_shown_pages(21)
        for shown_page in shown_pages:
            page_href = req.href.search([(f, 'on') for f in filters],
                                        q=req.args.get('q'),
                                        page=shown_page, noquickjump=1)
            pagedata.append([page_href, None, str(shown_page),
                             _("Page %(num)d", num=shown_page)])

        fields = ['href', 'class', 'string', 'title']
        results.shown_pages = [dict(zip(fields, p)) for p in pagedata]

        results.current_page = {'href': None, 'class': 'current',
                                'string': str(results.page + 1),
                                'title': None}

        if results.has_next_page:
            next_href = req.href.search(zip(filters, ['on'] * len(filters)),
                                        q=req.args.get('q'), page=page + 1,
                                        noquickjump=1)
            add_link(req, 'next', next_href, _('Next Page'))

        if results.has_previous_page:
            prev_href = req.href.search(zip(filters, ['on'] * len(filters)),
                                        q=req.args.get('q'), page=page - 1,
                                        noquickjump=1)
            add_link(req, 'prev', prev_href, _('Previous Page'))

        page_href = req.href.search(
            zip(filters, ['on'] * len(filters)), q=req.args.get('q'),
            noquickjump=1)
        return {'results': results, 'page_href': page_href}
<|endoftext|>


<commit_before>#!/usr/bin/env python

debug = True

from Queue import Queue
import socket
import sys

irr_host = 'rr.ntt.net'
irr_port = 43

as_set = sys.argv[1]

try:
    import asciitree
except ImportError:
    print "ERROR: install asciitree: pip install asciitree"
    sys.exit(1)

def connect():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((irr_host, irr_port))
    sock_in = sock.makefile('r')
    sock_out = sock.makefile('w')
    return sock, sock_in, sock_out

def send(connection, command):
    sock, sock_in, sock_out = connection
    sock_out.write(command + '\r\n')
    sock_out.flush()

def receive(connection):
    sock, sock_in, sock_out = connection
    return sock_in.readline()[:-1]

def query(connection, cmd, as_set, recurse=False):
    cmd = "!%s%s%s" % (cmd, as_set, ",1" if recurse else "")
    send(connection, cmd)
    answer = receive(connection)
    if answer == "D":
        return list()
    elif answer[0] == "F":
        if debug:
            print "Error: %s" % answer[1:]
            print "Query was: %s" % cmd
    elif answer[0] == "A":
        if debug:
            print "Info: receiving %s bytes" % answer[1:]
        results = receive(connection).split()
        if not receive(connection) == "C":
            print "Error: something went wrong with: %s" % cmd
        return results

queue = Queue()
queue.put(as_set)

connection = connect()
send(connection, "!!")

db = {}

while not queue.empty():
    item = queue.get()
    print "Info: expanding %s" % item
    if not "-" in item:  # expand aut-nums
        prefixes = query(connection, "g", item)
        db[item] = prefixes
        queue.task_done()
        continue
    db.setdefault(item, {})['members'] = query(connection, "i", item)
    db[item]['origin_asns'] = query(connection, "i", item, True)
    for candidate in set(db[item]['members'] + db[item]['origin_asns']):
        if not candidate in db and candidate not in queue.queue:
            queue.put(candidate)
    queue.task_done()

print db
<commit_msg>Add progress indicator<commit_after>#!/usr/bin/env python

debug = False

import progressbar
from Queue import Queue
import socket
import sys

irr_host = 'rr.ntt.net'
irr_port = 43

as_set = sys.argv[1]

try:
    import asciitree
except ImportError:
    print "ERROR: install asciitree: pip install asciitree"
    sys.exit(1)

def connect():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((irr_host, irr_port))
    sock_in = sock.makefile('r')
    sock_out = sock.makefile('w')
    return sock, sock_in, sock_out

def send(connection, command):
    sock, sock_in, sock_out = connection
    sock_out.write(command + '\r\n')
    sock_out.flush()

def receive(connection):
    sock, sock_in, sock_out = connection
    return sock_in.readline()[:-1]

def query(connection, cmd, as_set, recurse=False):
    cmd = "!%s%s%s" % (cmd, as_set, ",1" if recurse else "")
    send(connection, cmd)
    answer = receive(connection)
    if answer == "D":
        return list()
    elif answer[0] == "F":
        if debug:
            print "Error: %s" % answer[1:]
            print "Query was: %s" % cmd
    elif answer[0] == "A":
        if debug:
            print "Info: receiving %s bytes" % answer[1:]
        results = receive(connection).split()
        if not receive(connection) == "C":
            print "Error: something went wrong with: %s" % cmd
        return results

queue = Queue()
queue.put(as_set)

connection = connect()
send(connection, "!!")

db = {}

widgets = ['Processed: ', progressbar.Counter(), ' objects (',
           progressbar.Timer(), ')']
pbar = progressbar.ProgressBar(widgets=widgets)
pbar.start()
counter = 0

while not queue.empty():
    item = queue.get()
    if debug:
        print "Info: expanding %s" % item
    if not "-" in item:  # expand aut-nums
        prefixes = query(connection, "g", item)
        db[item] = prefixes
        counter += 1
        pbar.update(counter)
        queue.task_done()
        continue
    db.setdefault(item, {})['members'] = query(connection, "i", item)
    db[item]['origin_asns'] = query(connection, "i", item, True)
    for candidate in set(db[item]['members'] + db[item]['origin_asns']):
        if not candidate in db and candidate not in queue.queue:
            queue.put(candidate)
    counter += 1
    pbar.update(counter)
    queue.task_done()

print db
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from trac.test import Mock
from trac.web.api import Request, RequestDone
from trac.web.clearsilver import HDFWrapper

from Cookie import SimpleCookie as Cookie
from StringIO import StringIO
import unittest

class RequestTestCase(unittest.TestCase):

    def _make_environ(self, scheme='http', server_name='example.org',
                      server_port=80, method='GET', script_name='/trac',
                      **kwargs):
        environ = {'wsgi.url_scheme': scheme, 'wsgi.input': StringIO(''),
                   'REQUEST_METHOD': method, 'SERVER_NAME': server_name,
                   'SERVER_PORT': server_port, 'SCRIPT_NAME': script_name}
        environ.update(kwargs)
        return environ

    def test_base_url(self):
        environ = self._make_environ()
        req = Request(environ, None)
        self.assertEqual('http://example.org/trac', req.base_url)

    def test_base_url_host(self):
        environ = self._make_environ(server_port=8080, HTTP_HOST='example.com')
        req = Request(environ, None)
        self.assertEqual('http://example.com/trac', req.base_url)

    def test_base_url_nondefaultport(self):
        environ = self._make_environ(server_port=8080)
        req = Request(environ, None)
        self.assertEqual('http://example.org:8080/trac', req.base_url)

    def test_base_url_https(self):
        environ = self._make_environ(scheme='https', server_port=443)
        req = Request(environ, None)
        self.assertEqual('https://example.org/trac', req.base_url)

    def test_base_url_https_host(self):
        environ = self._make_environ(scheme='https', server_port=443,
                                     HTTP_HOST='example.com')
        req = Request(environ, None)
        self.assertEqual('https://example.com/trac', req.base_url)

    def test_base_url_https_nondefaultport(self):
        environ = self._make_environ(scheme='https', server_port=8443)
        req = Request(environ, None)
        self.assertEqual('https://example.org:8443/trac', req.base_url)

    def test_base_url_proxy(self):
        environ = self._make_environ(HTTP_HOST='localhost',
                                     HTTP_X_FORWARDED_HOST='example.com')
        req = Request(environ, None)
        self.assertEqual('http://example.com/trac', req.base_url)

    def test_redirect(self):
        status_sent = []
        headers_sent = {}
        def start_response(status, headers):
            status_sent.append(status)
            headers_sent.update(dict(headers))
        environ = self._make_environ(method='HEAD')
        req = Request(environ, start_response)
        self.assertRaises(RequestDone, req.redirect, '/trac/test')
        self.assertEqual('302 Found', status_sent[0])
        self.assertEqual('http://example.org/trac/test',
                         headers_sent['Location'])

    def test_redirect_absolute(self):
        status_sent = []
        headers_sent = {}
        def start_response(status, headers):
            status_sent.append(status)
            headers_sent.update(dict(headers))
        environ = self._make_environ(method='HEAD')
        req = Request(environ, start_response)
        self.assertRaises(RequestDone, req.redirect,
                          'http://example.com/trac/test')
        self.assertEqual('302 Found', status_sent[0])
        self.assertEqual('http://example.com/trac/test',
                         headers_sent['Location'])

    def test_write_unicode(self):
        buf = StringIO()
        def write(data):
            buf.write(data)
        def start_response(status, headers):
            return write
        environ = self._make_environ(method='HEAD')
        req = Request(environ, start_response)
        req.send_header('Content-Type', 'text/plain;charset=utf-8')
        req.write(u'Föö')
        self.assertEqual('Föö', buf.getvalue())

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(RequestTestCase, 'test'))
    return suite

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix test failure introduced in [3119].<commit_after># -*- coding: utf-8 -*-

from trac.test import Mock
from trac.web.api import Request, RequestDone
from trac.web.clearsilver import HDFWrapper

from Cookie import SimpleCookie as Cookie
from StringIO import StringIO
import unittest

class RequestTestCase(unittest.TestCase):

    def _make_environ(self, scheme='http', server_name='example.org',
                      server_port=80, method='GET', script_name='/trac',
                      **kwargs):
        environ = {'wsgi.url_scheme': scheme, 'wsgi.input': StringIO(''),
                   'REQUEST_METHOD': method, 'SERVER_NAME': server_name,
                   'SERVER_PORT': server_port, 'SCRIPT_NAME': script_name}
        environ.update(kwargs)
        return environ

    def test_base_url(self):
        environ = self._make_environ()
        req = Request(environ, None)
        self.assertEqual('http://example.org/trac', req.base_url)

    def test_base_url_host(self):
        environ = self._make_environ(server_port=8080, HTTP_HOST='example.com')
        req = Request(environ, None)
        self.assertEqual('http://example.com/trac', req.base_url)

    def test_base_url_nondefaultport(self):
        environ = self._make_environ(server_port=8080)
        req = Request(environ, None)
        self.assertEqual('http://example.org:8080/trac', req.base_url)

    def test_base_url_https(self):
        environ = self._make_environ(scheme='https', server_port=443)
        req = Request(environ, None)
        self.assertEqual('https://example.org/trac', req.base_url)

    def test_base_url_https_host(self):
        environ = self._make_environ(scheme='https', server_port=443,
                                     HTTP_HOST='example.com')
        req = Request(environ, None)
        self.assertEqual('https://example.com/trac', req.base_url)

    def test_base_url_https_nondefaultport(self):
        environ = self._make_environ(scheme='https', server_port=8443)
        req = Request(environ, None)
        self.assertEqual('https://example.org:8443/trac', req.base_url)

    def test_base_url_proxy(self):
        environ = self._make_environ(HTTP_HOST='localhost',
                                     HTTP_X_FORWARDED_HOST='example.com')
        req = Request(environ, None)
        self.assertEqual('http://localhost/trac', req.base_url)

    def test_redirect(self):
        status_sent = []
        headers_sent = {}
        def start_response(status, headers):
            status_sent.append(status)
            headers_sent.update(dict(headers))
        environ = self._make_environ(method='HEAD')
        req = Request(environ, start_response)
        self.assertRaises(RequestDone, req.redirect, '/trac/test')
        self.assertEqual('302 Found', status_sent[0])
        self.assertEqual('http://example.org/trac/test',
                         headers_sent['Location'])

    def test_redirect_absolute(self):
        status_sent = []
        headers_sent = {}
        def start_response(status, headers):
            status_sent.append(status)
            headers_sent.update(dict(headers))
        environ = self._make_environ(method='HEAD')
        req = Request(environ, start_response)
        self.assertRaises(RequestDone, req.redirect,
                          'http://example.com/trac/test')
        self.assertEqual('302 Found', status_sent[0])
        self.assertEqual('http://example.com/trac/test',
                         headers_sent['Location'])

    def test_write_unicode(self):
        buf = StringIO()
        def write(data):
            buf.write(data)
        def start_response(status, headers):
            return write
        environ = self._make_environ(method='HEAD')
        req = Request(environ, start_response)
        req.send_header('Content-Type', 'text/plain;charset=utf-8')
        req.write(u'Föö')
        self.assertEqual('Föö', buf.getvalue())

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(RequestTestCase, 'test'))
    return suite

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>VERSION = (2, 15, 1)

def get_version():
    return '.'.join(map(str, VERSION))
<commit_msg>2.15.2<commit_after>VERSION = (2, 15, 2)

def get_version():
    return '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Main application file.
"""
<commit_msg>* ISSUE-5: Search for places<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Main application file.

This can eventually become a CherryPy server, but for now use this file to prototype a minimum working version a Twitterverse trends service.

Filter
    http://sqlobject.org/SelectResults.html#filter-expression
Builder expressions
    http://sqlobject.org/SQLBuilder.html
Like
    https://stackoverflow.com/questions/1003506/executing-sql-like-in-sqlobject
"""
#import sqlobject
import sqlobject.sqlbuilder as builder

from lib import database as db

def searchCountry(searchStr='', startswith=False):
    """
    LIKE, startswith and endswith seem to be case insensitive at least when using SQLite as db.
    """
    if startswith:
        condition = db.Country.q.name.startswith(searchStr)
    else:
        condition = builder.LIKE(db.Country.q.name, '%{}%'.format(searchStr))

    res = db.Country.select().filter(condition)

    return res

def searchTowns(searchStr='', startswith=False):
    """
    LIKE, startswith and endswith seem to be case insensitive at least when using SQLite as db.
    """
    if startswith:
        condition = db.Town.q.name.startswith(searchStr)
    else:
        condition = builder.LIKE(db.Town.q.name, '%{}%'.format(searchStr))

    res = db.Town.select().filter(condition)

    return res

def _test():
    print 'TOWNS SEARCH'
    searchStr = 'ca'
    res = searchTowns(searchStr)
    print
    print str(res)
    print
    for town in res:
        print u'{:10d} - {}'.format(town.woeid, town.name)
    print
    print '-------'

    print 'COUNTRY SEARCH'
    searchStr = 'ica'
    res = searchCountry(searchStr)
    print
    print str(res)
    print
    for country in res:
        print u'{} - {}'.format(country.woeid, country.name)
        for town in country.hasTowns:
            print u' * {:10d} - {}'.format(town.woeid, town.name)
        print

if __name__ == '__main__':
    _test()
<|endoftext|>


<commit_before>import math
import array

class Biquad:
    """
    A biquad filter is a 2-poles/2-zeros filter allowing to perform
    various kind of filtering.
    """

    def __init__(self):
        """
        Create an inactive biquad filter with a flat frequency response.
        To make the filter active, use one of the provided methods.
        """
        self.reset()

    def reset(self):
        """
        Make the filter inactive with a flat frequency response.
        """
        self._a_coeffs = array.array('f', [0] * 3)
        self._b_coeffs = array.array('f', [0] * 3)
        self._x1 = 0.0
        self._x2 = 0.0
        self._y1 = 0.0
        self._y2 = 0.0

    def low_pass(self, samplerate, cutoff, resonance):
        """
        Make a low-pass filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, cutoff, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = ((1.0 - self._cos_w0) / 2.0) / self._a_coeffs[0]
        self._b_coeffs[1] = (1.0 - self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = ((1.0 - self._cos_w0) / 2.0) / self._a_coeffs[0]

    def high_pass(self, samplerate, cutoff, resonance):
        """
        Make a high-pass filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, cutoff, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = ((1.0 + self._cos_w0) / 2.0) / self._a_coeffs[0]
        self._b_coeffs[1] = -(1.0 + self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = ((1.0 + self._cos_w0) / 2.0) / self._a_coeffs[0]

    def band_pass(self, samplerate, center, resonance):
        """
        Make a band-pass filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, center, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = (self._alpha) / self._a_coeffs[0]
        self._b_coeffs[1] = 0
        self._b_coeffs[2] = (- self._alpha) / self._a_coeffs[0]

    def all_pass(self, samplerate, center, resonance):
        """
        Make an all-pass filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, center, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = (1.0 + self._alpha) / self._a_coeffs[0]

    def notch(self, samplerate, center, resonance):
        """
        Make a notch filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, center, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = (1.0) / self._a_coeffs[0]
        self._b_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = (1.0) / self._a_coeffs[0]

    def peak(self, samplerate, center, resonance, dbgain):
        """
        Make a peak filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        :param dbgain: gain in dB
        """
        self._compute_constants(samplerate, center, resonance, dbgain)
        self._a_coeffs[0] = (1.0 + self._alpha / self._a)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha / self._a) / self._a_coeffs[0]
        self._b_coeffs[0] = (1.0 + self._alpha * self._a) / self._a_coeffs[0]
        self._b_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = (1.0 - self._alpha * self._a) / self._a_coeffs[0]

    def low_shelf(self, samplerate, cutoff, resonance, dbgain):
        """
        Make a low-shelf filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        :param dbgain: gain in dB
        """
        self._compute_constants(samplerate, cutoff, resonance, dbgain)
        self._a_coeffs[0] = ((self._a + 1) +
                             (self._a - 1) * self._cos_w0 + self._sqrtAlpha)
        self._a_coeffs[1] = ((-2.0 * ((self._a - 1) +
                             (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._a_coeffs[2] = (((self._a + 1) +
                              (self._a - 1) * self._cos_w0 - self._sqrtAlpha) /
                             self._a_coeffs[0])
        self._b_coeffs[0] = ((self._a *
                              ((self._a + 1) -
                               (self._a - 1) * self._cos_w0 +
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])
        self._b_coeffs[1] = ((2.0 * self._a *
                              ((self._a - 1) - (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._b_coeffs[2] = ((self._a *
                              ((self._a + 1) -
                               (self._a - 1) * self._cos_w0 -
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])

    def high_shelf(self, samplerate, cutoff, resonance, dbgain):
        """
        Make a high-shelf filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        :param dbgain: gain in dB
        """
        self._compute_constants(samplerate, cutoff, resonance, dbgain)
        self._a_coeffs[0] = ((self._a + 1) -
                             (self._a - 1) * self._cos_w0 + self._sqrtAlpha)
        self._a_coeffs[1] = ((2.0 *
                              ((self._a - 1) - (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._a_coeffs[2] = (((self._a + 1) -
                              (self._a - 1) * self._cos_w0 -
                              self._sqrtAlpha) /
                             self._a_coeffs[0])
        self._b_coeffs[0] = ((self._a *
                              ((self._a + 1) +
                               (self._a - 1) * self._cos_w0 +
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])
        self._b_coeffs[1] = ((-2.0 * self._a *
                              ((self._a - 1) + (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._b_coeffs[2] = ((self._a *
                              ((self._a + 1) +
                               (self._a - 1) * self._cos_w0 -
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])

    def custom(self, a0, a1, a2, b0, b1, b2):
        """
        Make a custom filter.

        :param a0: a[0] coefficient
        :param a1: a[1] coefficient
        :param a2: a[2] coefficient
        :param b0: b[0] coefficient
        :param b1: b[1] coefficient
        :param b2: b[2] coefficient
        """
        self._a_coeffs[0] = a0
        self._a_coeffs[1] = a1 / a0
        self._a_coeffs[2] = a2 / a0
        self._b_coeffs[0] = b0 / a0
        self._b_coeffs[1] = b1 / a0
        self._b_coeffs[2] = b2 / a0

    def process(self, x, y):
        """
        Filter an input signal. Can be used for in-place filtering.

        :param x: input buffer
        :param y: output buffer
        """
        num_samples = len(x)
        for n in range(0, num_samples):
            curr = x[n]
            y[n] = (self._b_coeffs[0] * x[n] +
                    self._b_coeffs[1] * self._x1 +
                    self._b_coeffs[2] * self._x2 -
                    self._a_coeffs[1] * self._y1 -
                    self._a_coeffs[2] * self._y2)
            self._x2 = self._x1
            self._x1 = curr
            self._y2 = self._y1
            self._y1 = y[n]

    def _compute_constants(self, fs, fc, q, dbgain=0):
        """
        Pre-compute internal mathematical constants
        """
        self._fc = fc
        self._q = q
        self._dbgain = dbgain
        self._fs = fs
        self._a = math.pow(10, (self._dbgain / 40.0))
        self._w0 = 2.0 * math.pi * self._fc / self._fs
        self._cos_w0 = math.cos(self._w0)
        self._sin_w0 = math.sin(self._w0)
        self._alpha = self._sin_w0 / (2.0 * self._q)
        self._sqrtAlpha = 2.0 * math.sqrt(self._a) * self._alpha
<commit_msg>[DEV] Allow sample-by-sample processing for the biquad filter<commit_after>import math
import array

class Biquad:
    """
    A biquad filter is a 2-poles/2-zeros filter allowing to perform
    various kind of filtering. Signal attenuation is at a rate of 12 dB
    per octave.
    """

    def __init__(self):
        """
        Create an inactive biquad filter with a flat frequency response.
        To make the filter active, use one of the provided methods.
        """
        self.reset()

    def reset(self):
        """
        Make the filter inactive with a flat frequency response.
        """
        self._a_coeffs = array.array('f', [0] * 3)
        self._b_coeffs = array.array('f', [0] * 3)
        self._x1 = 0.0
        self._x2 = 0.0
        self._y1 = 0.0
        self._y2 = 0.0

    def low_pass(self, samplerate, cutoff, resonance):
        """
        Make a low-pass filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, cutoff, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = ((1.0 - self._cos_w0) / 2.0) / self._a_coeffs[0]
        self._b_coeffs[1] = (1.0 - self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = ((1.0 - self._cos_w0) / 2.0) / self._a_coeffs[0]

    def high_pass(self, samplerate, cutoff, resonance):
        """
        Make a high-pass filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, cutoff, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = ((1.0 + self._cos_w0) / 2.0) / self._a_coeffs[0]
        self._b_coeffs[1] = -(1.0 + self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = ((1.0 + self._cos_w0) / 2.0) / self._a_coeffs[0]

    def band_pass(self, samplerate, center, resonance):
        """
        Make a band-pass filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, center, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = (self._alpha) / self._a_coeffs[0]
        self._b_coeffs[1] = 0
        self._b_coeffs[2] = (- self._alpha) / self._a_coeffs[0]

    def all_pass(self, samplerate, center, resonance):
        """
        Make an all-pass filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, center, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = (1.0 + self._alpha) / self._a_coeffs[0]

    def notch(self, samplerate, center, resonance):
        """
        Make a notch filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        """
        self._compute_constants(samplerate, center, resonance)
        self._a_coeffs[0] = (1.0 + self._alpha)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha) / self._a_coeffs[0]
        self._b_coeffs[0] = (1.0) / self._a_coeffs[0]
        self._b_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = (1.0) / self._a_coeffs[0]

    def peak(self, samplerate, center, resonance, dbgain):
        """
        Make a peak filter.

        :param samplerate: sample-rate in Hz
        :param center: center frequency in Hz
        :param resonance: resonance or Q-factor
        :param dbgain: gain in dB
        """
        self._compute_constants(samplerate, center, resonance, dbgain)
        self._a_coeffs[0] = (1.0 + self._alpha / self._a)
        self._a_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._a_coeffs[2] = (1.0 - self._alpha / self._a) / self._a_coeffs[0]
        self._b_coeffs[0] = (1.0 + self._alpha * self._a) / self._a_coeffs[0]
        self._b_coeffs[1] = (-2.0 * self._cos_w0) / self._a_coeffs[0]
        self._b_coeffs[2] = (1.0 - self._alpha * self._a) / self._a_coeffs[0]

    def low_shelf(self, samplerate, cutoff, resonance, dbgain):
        """
        Make a low-shelf filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        :param dbgain: gain in dB
        """
        self._compute_constants(samplerate, cutoff, resonance, dbgain)
        self._a_coeffs[0] = ((self._a + 1) +
                             (self._a - 1) * self._cos_w0 + self._sqrtAlpha)
        self._a_coeffs[1] = ((-2.0 * ((self._a - 1) +
                             (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._a_coeffs[2] = (((self._a + 1) +
                              (self._a - 1) * self._cos_w0 - self._sqrtAlpha) /
                             self._a_coeffs[0])
        self._b_coeffs[0] = ((self._a *
                              ((self._a + 1) -
                               (self._a - 1) * self._cos_w0 +
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])
        self._b_coeffs[1] = ((2.0 * self._a *
                              ((self._a - 1) - (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._b_coeffs[2] = ((self._a *
                              ((self._a + 1) -
                               (self._a - 1) * self._cos_w0 -
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])

    def high_shelf(self, samplerate, cutoff, resonance, dbgain):
        """
        Make a high-shelf filter.

        :param samplerate: sample-rate in Hz
        :param cutoff: cut-off frequency in Hz
        :param resonance: resonance or Q-factor
        :param dbgain: gain in dB
        """
        self._compute_constants(samplerate, cutoff, resonance, dbgain)
        self._a_coeffs[0] = ((self._a + 1) -
                             (self._a - 1) * self._cos_w0 + self._sqrtAlpha)
        self._a_coeffs[1] = ((2.0 *
                              ((self._a - 1) - (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._a_coeffs[2] = (((self._a + 1) -
                              (self._a - 1) * self._cos_w0 -
                              self._sqrtAlpha) /
                             self._a_coeffs[0])
        self._b_coeffs[0] = ((self._a *
                              ((self._a + 1) +
                               (self._a - 1) * self._cos_w0 +
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])
        self._b_coeffs[1] = ((-2.0 * self._a *
                              ((self._a - 1) + (self._a + 1) * self._cos_w0)) /
                             self._a_coeffs[0])
        self._b_coeffs[2] = ((self._a *
                              ((self._a + 1) +
                               (self._a - 1) * self._cos_w0 -
                               self._sqrtAlpha)) /
                             self._a_coeffs[0])

    def custom(self, a0, a1, a2, b0, b1, b2):
        """
        Make a custom filter.

        :param a0: a[0] coefficient
        :param a1: a[1] coefficient
        :param a2: a[2] coefficient
        :param b0: b[0] coefficient
        :param b1: b[1] coefficient
        :param b2: b[2] coefficient
        """
        self._a_coeffs[0] = a0
        self._a_coeffs[1] = a1 / a0
        self._a_coeffs[2] = a2 / a0
        self._b_coeffs[0] = b0 / a0
        self._b_coeffs[1] = b1 / a0
        self._b_coeffs[2] = b2 / a0

    def process_sample(self, x):
        curr = x
        y = (self._b_coeffs[0] * x +
             self._b_coeffs[1] * self._x1 +
             self._b_coeffs[2] * self._x2 -
             self._a_coeffs[1] * self._y1 -
             self._a_coeffs[2] * self._y2)
        self._x2 = self._x1
        self._x1 = curr
        self._y2 = self._y1
        self._y1 = y
        return y

    def process(self, x, y):
        """
        Filter an input signal. Can be used for in-place filtering.

        :param x: input buffer
        :param y: output buffer
        """
        num_samples = len(x)
        for n in range(0, num_samples):
            y[n] = self.process_sample(x[n])

    def _compute_constants(self, fs, fc, q, dbgain=0):
        """
        Pre-compute internal mathematical constants
        """
        self._fc = fc
        self._q = q
        self._dbgain = dbgain
        self._fs = fs
        self._a = math.pow(10, (self._dbgain / 40.0))
        self._w0 = 2.0 * math.pi * self._fc / self._fs
        self._cos_w0 = math.cos(self._w0)
        self._sin_w0 = math.sin(self._w0)
        self._alpha = self._sin_w0 / (2.0 * self._q)
        self._sqrtAlpha = 2.0 * math.sqrt(self._a) * self._alpha
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

from dash.orgs.views import OrgPermsMixin

from django.utils.translation import ugettext_lazy as _

from smartmin.users.views import SmartTemplateView

from tracpro.polls.models import Poll
from tracpro.baseline.models import BaselineTerm
from tracpro.baseline.utils import chart_baseline

class HomeView(OrgPermsMixin, SmartTemplateView):
    """TracPro homepage"""

    title = _("Home")
    template_name = 'home/home.html'

    def has_permission(self, request, *args, **kwargs):
        return request.user.is_authenticated()

    def get_context_data(self, **kwargs):
        context = super(HomeView, self).get_context_data(**kwargs)
        context['polls'] = Poll.get_all(self.request.org).order_by('name')
        # Loop through all baseline terms, until we find one with data
        baselineterms = BaselineTerm.get_all(self.request.org).order_by('-end_date')
        for baselineterm in baselineterms:
            data_found = baselineterm.check_for_data(self.request.data_regions)
            if data_found:
                (answers_dict, baseline_dict, all_regions, date_list,
                 baseline_mean, baseline_std, follow_up_mean, follow_up_std) = chart_baseline(
                    baselineterm=baselineterm, regions=self.request.data_regions, region_selected=0)
                context['all_regions'] = all_regions
                context['date_list'] = date_list
                context['baseline_list'] = baseline_dict
                context['follow_up_list'] = answers_dict
                context['baselineterm'] = baselineterm
                context['include_legend_data'] = 0
                break  # Found our baseline chart with data, send it back to the view

        # Return top 5 baseline terms only
        context['baselineterms'] = baselineterms[0:5]

        return context
<commit_msg>Renaming variables (lists not dicts).<commit_after>from __future__ import absolute_import, unicode_literals

from dash.orgs.views import OrgPermsMixin

from django.utils.translation import ugettext_lazy as _

from smartmin.users.views import SmartTemplateView

from tracpro.polls.models import Poll
from tracpro.baseline.models import BaselineTerm
from tracpro.baseline.utils import chart_baseline

class HomeView(OrgPermsMixin, SmartTemplateView):
    """TracPro homepage"""

    title = _("Home")
    template_name = 'home/home.html'

    def has_permission(self, request, *args, **kwargs):
        return request.user.is_authenticated()

    def get_context_data(self, **kwargs):
        context = super(HomeView, self).get_context_data(**kwargs)
        context['polls'] = Poll.get_all(self.request.org).order_by('name')
        # Loop through all baseline terms, until we find one with data
        baselineterms = BaselineTerm.get_all(self.request.org).order_by('-end_date')
        for baselineterm in baselineterms:
            data_found = baselineterm.check_for_data(self.request.data_regions)
            if data_found:
                (follow_up_list, baseline_list, all_regions, date_list,
                 baseline_mean, baseline_std, follow_up_mean, follow_up_std) = chart_baseline(
                    baselineterm=baselineterm, regions=self.request.data_regions, region_selected=0)
                context['all_regions'] = all_regions
                context['date_list'] = date_list
                context['baseline_list'] = baseline_list
                context['follow_up_list'] = follow_up_list
                context['baselineterm'] = baselineterm
                context['include_legend_data'] = 0
                break  # Found our baseline chart with data, send it back to the view

        # Return top 5 baseline terms only
        context['baselineterms'] = baselineterms[0:5]

        return context
<|endoftext|>


<commit_before>VERSION = (2, 18, 15)

def get_version():
    return '.'.join(map(str, VERSION))
<commit_msg>2.18.16<commit_after>VERSION = (2, 18, 16)

def get_version():
    return '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before>import datetime
import json
import os

from django.core.urlresolvers import reverse
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.http import HttpResponse, HttpResponseForbidden, Http404, \
    HttpResponseRedirect
from django.utils.encoding import smart_str

from celery.result import AsyncResult

from core.forms import DownloadForm

from api import tasks, utils
from api.models import ActivityLog, YouTube
from api.utils import get_client_ip

def extract_audio(request):
    if request.method == 'POST':
        form = DownloadForm(request.POST)

        if form.is_valid():
            client_ip = utils.get_client_ip(request)

            url = form.cleaned_data['url']

            # Truncate the URL to remove the ampersand and anything after it.
            # This will prevent passing a playlist URL.
            if url:
                url = url.split('&')[0]

            task = tasks.extract_audio.delay(url, client_ip)
            result = AsyncResult(task.id)
            result.wait()

            video_id = result.result['video_id']
            filename = result.result['filename']
            download_link = reverse(
                'download_file',
                kwargs={'video_id': video_id, 'filename': filename})
            data = {
                'success': True,
                'id': task.id,
                'filename': filename,
                'download_link': download_link
            }

            return HttpResponse(json.dumps(data))
        else:
            message = 'Please enter a URL.'
            return HttpResponse(json.dumps({'form_valid': False,
                                            'detail': message}))

    return HttpResponseForbidden

def download_file(request, video_id, filename):
    filepath = '%s%s' % (settings.MEDIA_ROOT, filename)
    file_exists = os.path.exists(filepath)

    youtube = None
    try:
        youtube = YouTube.objects.get(video_id=video_id, audio_filename=filename)
    except ObjectDoesNotExist:
        pass

    if youtube and file_exists:
        ActivityLog.objects.create(
            client_ip=get_client_ip(request),
            video_id=video_id
        )

        youtube.download_count += 1
        youtube.last_download_date = datetime.datetime.now()
        youtube.save()

        if settings.DEBUG:
            with open(filepath, 'rb') as file_data:
                response = HttpResponse(file_data.read(),
                                        content_type='audio/mpeg')

            response['Content-Disposition'] = 'attachment; filename=%s' % \
                smart_str(filename)
            response['Content-Length'] = os.path.getsize(filepath)

            return response
        else:
            # Have Nginx serve the file in production.
            response = HttpResponse(mimetype='application/force-download')
            response['Content-Length'] = os.path.getsize(filepath)
            response['X-Accel-Redirect'] = '%s%s' % (settings.MEDIA_URL,
                                                     smart_str(filename))

            return response

    return HttpResponseRedirect(reverse('home'))<commit_msg>Added action logging in download view.<commit_after>import datetime
import json
import os

from django.core.urlresolvers import reverse
from django.core.exceptions import ObjectDoesNotExist
from django.conf import settings
from django.http import HttpResponse, HttpResponseForbidden, Http404, \
    HttpResponseRedirect
from django.utils.encoding import smart_str

from celery.result import AsyncResult

from core.forms import DownloadForm

from api import tasks, utils
from api.models import ActivityLog, YouTube
from api.utils import get_client_ip

def extract_audio(request):
    if request.method == 'POST':
        form = DownloadForm(request.POST)

        if form.is_valid():
            client_ip = utils.get_client_ip(request)

            url = form.cleaned_data['url']

            # Truncate the URL to remove the ampersand and anything after it.
            # This will prevent passing a playlist URL.
            if url:
                url = url.split('&')[0]

            task = tasks.extract_audio.delay(url, client_ip)
            result = AsyncResult(task.id)
            result.wait()

            video_id = result.result['video_id']
            filename = result.result['filename']
            download_link = reverse(
                'download_file',
                kwargs={'video_id': video_id, 'filename': filename})
            data = {
                'success': True,
                'id': task.id,
                'filename': filename,
                'download_link': download_link
            }

            return HttpResponse(json.dumps(data))
        else:
            message = 'Please enter a URL.'
            return HttpResponse(json.dumps({'form_valid': False,
                                            'detail': message}))

    return HttpResponseForbidden

def download_file(request, video_id, filename):
    filepath = '%s%s' % (settings.MEDIA_ROOT, filename)
    file_exists = os.path.exists(filepath)

    youtube = None
    try:
        youtube = YouTube.objects.get(video_id=video_id, audio_filename=filename)
    except ObjectDoesNotExist:
        pass

    if youtube and file_exists:
        ActivityLog.objects.create(
            action='DOWNLOAD',
            client_ip=get_client_ip(request),
            video_id=video_id
        )

        youtube.download_count += 1
        youtube.last_download_date = datetime.datetime.now()
        youtube.save()

        if settings.DEBUG:
            with open(filepath, 'rb') as file_data:
                response = HttpResponse(file_data.read(),
                                        content_type='audio/mpeg')

            response['Content-Disposition'] = 'attachment; filename=%s' % \
                smart_str(filename)
            response['Content-Length'] = os.path.getsize(filepath)

            return response
        else:
            # Have Nginx serve the file in production.
            response = HttpResponse(mimetype='application/force-download')
            response['Content-Length'] = os.path.getsize(filepath)
            response['X-Accel-Redirect'] = '%s%s' % (settings.MEDIA_URL,
                                                     smart_str(filename))

            return response

    return HttpResponseRedirect(reverse('home'))<|endoftext|>


<commit_before>VERSION = (2, 8, 1)

def get_version():
    return '.'.join(map(str, VERSION))
<commit_msg>2.8.2<commit_after>VERSION = (2, 8, 2)

def get_version():
    return '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before># Copyright 2019 The Forseti Real Time Enforcer Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import time
import traceback

from google.cloud import pubsub

from rpe import RPE

from parsers.stackdriver import StackdriverParser
from parsers.cai import CaiParser
from lib.logger import Logger
from lib.credentials import CredentialsBroker
from lib.enforcement import EnforcementDecision
import hooks

# Load configuration
app_name = os.environ.get('APP_NAME', 'forseti-realtime-enforcer')
project_id = os.environ.get('PROJECT_ID')
subscription_name = os.environ.get('SUBSCRIPTION_NAME')
opa_url = os.environ.get('OPA_URL')
python_policy_path = os.environ.get('PYTHON_POLICY_PATH')
enforce_policy = os.environ.get('ENFORCE', '').lower() == 'true'
enforcement_delay = int(os.environ.get('ENFORCEMENT_DELAY', 0))
stackdriver_logging = os.environ.get('STACKDRIVER_LOGGING', '').lower() == 'true'
per_project_logging = os.environ.get('PER_PROJECT_LOGGING', '').lower() == 'true'
debug_logging = os.environ.get('DEBUG_LOGGING', '').lower() == 'true'

# Build a dict of pubsub flow control settings from env vars if they're set
flow_control_config = {k: int(v) for k, v in dict(
    max_messages=os.environ.get('PUBSUB_MAX_MESSAGES'),
    max_bytes=os.environ.get('PUBSUB_MAX_BYTES'),
    max_lease_duration=os.environ.get('PUBSUB_MAX_LEASE_DURATION'),
).items() if v is not None}

# We're using the application default credentials, but defining them
# explicitly so its easy to plug-in credentials using your own preferred
# method
cb = CredentialsBroker()
app_creds = cb.get_credentials()

# Setup logging helper
logger = Logger(
    app_name,
    stackdriver_logging,
    project_id,
    app_creds,
    debug_logging
)

# Setup message handlers
message_parsers = [
    CaiParser,
    StackdriverParser,
]

# Instantiate our rpe
rpeconfig = {
    'policy_engines': [
        {
            'type': 'opa',
            'url': opa_url
        }
    ]
}

if python_policy_path is not None:
    python_engine = {
        'type': 'python',
        'path': python_policy_path,
    }
    rpeconfig['policy_engines'].append(python_engine)

rpe = RPE(rpeconfig)

running_config = {
    'policies': rpe.policies(),
    'policy_enforcement': "enabled" if enforce_policy else "disabled",
    'stackdriver_logging': "enabled" if stackdriver_logging else "disabled",
    'enforcement_delay': enforcement_delay,
    'debug_logging': "enabled" if debug_logging else "disabled",
    'python_policy_path': python_policy_path,
    'flow_control_config': flow_control_config,
}
logger(running_config)

def callback(pubsub_message):

    # We use the message ID in all logs we emit
    message_id = pubsub_message.message_id

    # Only one message parser should be able to parse a given message, lets capture which one it is
    parser_match = None

    for parser in message_parsers:

        try:
            if not parser.match(pubsub_message):
                logger.debug({'message_id': message_id, 'message': f'Message not matched by parser {parser.__name__}'})
                continue

            logger.debug({'message_id': message_id, 'message': f'Message matched by parser {parser.__name__}'})

            parsed_message = parser.parse_message(pubsub_message)
            parser_match = parser

            if len(parsed_message.resources) == 0:
                # We did not recognize any assets in this message
                logger.debug({
                    'message_id':
                    message_id, 'message': f'No resources identified in message parsed by {parser.__name__}',
                    'metadata': parsed_message.metadata.dict()
                })

                pubsub_message.ack()
                return
            break

        except Exception as e:
            logger({
                'message_id': message_id,
                'message': f'Exception while parsing message with {parser.__name__}',
                **exc_info(e)
            })

    # If no message parsers were able to parse the message, log and return
    if parser_match is None:
        logger({'message_id': message_id, 'message': 'No parsers recognized the message format, discarding message'})
        pubsub_message.ack()
        return

    for resource in parsed_message.resources:

        try:

            logger.debug({
                'message_id': message_id,
                'message': 'Processing resource',
                'resource_data': resource.to_dict(),
            })

            # Set the resource's credentials before any API calls
            resource_creds = cb.get_credentials(
                **resource.to_dict(),
            )
            resource.client_kwargs = {
                'credentials': resource_creds
            }

            project_logger = None
            if per_project_logging and resource.project_id is not None:
                project_logger = Logger(
                    app_name,
                    True,  # per-project logging is always stackdriver
                    resource.project_id,
                    resource_creds
                )

        except Exception as e:
            logger({
                'message_id': message_id,
                'message': 'Exception while getting credentials for resource',
                'metadata': parsed_message.metadata.dict(),
                'resource_data': resource.to_dict(),
                **exc_info(e),
            })
            pubsub_message.ack()
            continue

        logger.debug({'message_id': message_id, 'message': 'Evaluating resource against policies'})

        try:
            evaluations = rpe.evaluate(resource)
            eval_time = int(time.time())
        except Exception as e:
            logger({
                'message_id': message_id,
                'resource_data': resource.to_dict(),
                'message': 'Exception while evaluating resource',
                'metadata': parsed_message.metadata.dict(),
                **exc_info(e),
            })
            pubsub_message.ack()
            continue

        if len(evaluations) < 1:
            logger.debug({'message_id': message_id, 'message': 'No policies matched resource'})

        enforcements = []

        # Log the results of evaluations on each policy for this resource
        for evaluation in evaluations:

            # Call hook to allow for customization
            hooks.process_evaluation(evaluation, parsed_message)

            # decide if we should remediate
            decision = EnforcementDecision(evaluation, parsed_message)
            hooks.process_enforcement_decision(decision, parsed_message)

            if decision.enforce:
                enforcements.append(decision)

            evaluation_log = {
                'compliant': evaluation.compliant,
                'event': 'evaluation',
                'excluded': evaluation.excluded,
                'message_id': message_id,
                'metadata': parsed_message.metadata.dict(),
                'policy_id': evaluation.policy_id,
                'remediable': evaluation.remediable,
                'resource_data': resource.to_dict(),
                'enforce': decision.enforce,
                'non_enforcement_conditions': decision.reasons,
                'timestamp': eval_time,
            }

            logger(evaluation_log)

            if project_logger is not None:
                project_logger(evaluation_log)

        # Skip all this if application-level enforcement is disabled
        if enforce_policy:

            for enforcement in enforcements:

                delay(parsed_message)
                logger.debug({'message_id': message_id, 'message': 'Executing remediation'})

                try:
                    enforcement.evaluation.remediate()
                    remediation_timestamp = int(time.time())

                    remediation_log = {
                        'event': 'remediation',
                        'message_id': message_id,
                        'metadata': parsed_message.metadata.dict(),
                        'policy_id': enforcement.evaluation.policy_id,
                        'resource_data': resource.to_dict(),
                        'timestamp': remediation_timestamp,
                    }

                    logger(remediation_log)

                    if project_logger is not None:
                        project_logger(remediation_log)

                except Exception as e:
                    # Catch any other exceptions so we can acknowledge the message.
                    # Otherwise they start to fill up the buffer of unacknowledged messages
                    logger(dict(
                        message='Execption while attempting to remediate',
                        message_id=message_id,
                        metadata=parsed_message.metadata.dict(),
                        policy_id=evaluation.policy_id,
                        resource_data=resource.to_dict(),
                        **exc_info(e),
                    ))

        else:
            logger.debug({'message_id': message_id, 'message': 'Enforcement is disabled, processing complete'})

    # Finally ack the message after we're done with all of the assets
    pubsub_message.ack()

def delay(trigger):
    global enforcement_delay

    if enforcement_delay and trigger.control_data.delay_enforcement:

        # If the log is old, subtract that from the enforcement delay
        delay = max(0, enforcement_delay - trigger.age)
        time.sleep(delay)

def exc_info(exception):
    return {
        'event': 'exception',
        'details': str(exception),
        'trace': traceback.format_exc(),
    }

if __name__ == "__main__":

    subscriber = pubsub.SubscriberClient(credentials=app_creds)

    subscription_path = 'projects/{project_id}/subscriptions/{sub}'.format(
        project_id=project_id,
        sub=subscription_name
    )

    flow_control = pubsub.types.FlowControl(
        **flow_control_config
    )

    future = subscriber.subscribe(
        subscription_path,
        callback=callback,
        flow_control=flow_control,
    )

    logger("Listening for pubsub messages on {}...".format(subscription_path))

    try:
        future.result()
    except Exception:
        future.cancel()
        raise
<commit_msg>Log the correct policy_id<commit_after># Copyright 2019 The Forseti Real Time Enforcer Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import time
import traceback

from google.cloud import pubsub

from rpe import RPE

from parsers.stackdriver import StackdriverParser
from parsers.cai import CaiParser
from lib.logger import Logger
from lib.credentials import CredentialsBroker
from lib.enforcement import EnforcementDecision
import hooks

# Load configuration
app_name = os.environ.get('APP_NAME', 'forseti-realtime-enforcer')
project_id = os.environ.get('PROJECT_ID')
subscription_name = os.environ.get('SUBSCRIPTION_NAME')
opa_url = os.environ.get('OPA_URL')
python_policy_path = os.environ.get('PYTHON_POLICY_PATH')
enforce_policy = os.environ.get('ENFORCE', '').lower() == 'true'
enforcement_delay = int(os.environ.get('ENFORCEMENT_DELAY', 0))
stackdriver_logging = os.environ.get('STACKDRIVER_LOGGING', '').lower() == 'true'
per_project_logging = os.environ.get('PER_PROJECT_LOGGING', '').lower() == 'true'
debug_logging = os.environ.get('DEBUG_LOGGING', '').lower() == 'true'

# Build a dict of pubsub flow control settings from env vars if they're set
flow_control_config = {k: int(v) for k, v in dict(
    max_messages=os.environ.get('PUBSUB_MAX_MESSAGES'),
    max_bytes=os.environ.get('PUBSUB_MAX_BYTES'),
    max_lease_duration=os.environ.get('PUBSUB_MAX_LEASE_DURATION'),
).items() if v is not None}

# We're using the application default credentials, but defining them
# explicitly so its easy to plug-in credentials using your own preferred
# method
cb = CredentialsBroker()
app_creds = cb.get_credentials()

# Setup logging helper
logger = Logger(
    app_name,
    stackdriver_logging,
    project_id,
    app_creds,
    debug_logging
)

# Setup message handlers
message_parsers = [
    CaiParser,
    StackdriverParser,
]

# Instantiate our rpe
rpeconfig = {
    'policy_engines': [
        {
            'type': 'opa',
            'url': opa_url
        }
    ]
}

if python_policy_path is not None:
    python_engine = {
        'type': 'python',
        'path': python_policy_path,
    }
    rpeconfig['policy_engines'].append(python_engine)

rpe = RPE(rpeconfig)

running_config = {
    'policies': rpe.policies(),
    'policy_enforcement': "enabled" if enforce_policy else "disabled",
    'stackdriver_logging': "enabled" if stackdriver_logging else "disabled",
    'enforcement_delay': enforcement_delay,
    'debug_logging': "enabled" if debug_logging else "disabled",
    'python_policy_path': python_policy_path,
    'flow_control_config': flow_control_config,
}
logger(running_config)

def callback(pubsub_message):

    # We use the message ID in all logs we emit
    message_id = pubsub_message.message_id

    # Only one message parser should be able to parse a given message, lets capture which one it is
    parser_match = None

    for parser in message_parsers:

        try:
            if not parser.match(pubsub_message):
                logger.debug({'message_id': message_id, 'message': f'Message not matched by parser {parser.__name__}'})
                continue

            logger.debug({'message_id': message_id, 'message': f'Message matched by parser {parser.__name__}'})

            parsed_message = parser.parse_message(pubsub_message)
            parser_match = parser

            if len(parsed_message.resources) == 0:
                # We did not recognize any assets in this message
                logger.debug({
                    'message_id':
                    message_id, 'message': f'No resources identified in message parsed by {parser.__name__}',
                    'metadata': parsed_message.metadata.dict()
                })

                pubsub_message.ack()
                return
            break

        except Exception as e:
            logger({
                'message_id': message_id,
                'message': f'Exception while parsing message with {parser.__name__}',
                **exc_info(e)
            })

    # If no message parsers were able to parse the message, log and return
    if parser_match is None:
        logger({'message_id': message_id, 'message': 'No parsers recognized the message format, discarding message'})
        pubsub_message.ack()
        return

    for resource in parsed_message.resources:

        try:

            logger.debug({
                'message_id': message_id,
                'message': 'Processing resource',
                'resource_data': resource.to_dict(),
            })

            # Set the resource's credentials before any API calls
            resource_creds = cb.get_credentials(
                **resource.to_dict(),
            )
            resource.client_kwargs = {
                'credentials': resource_creds
            }

            project_logger = None
            if per_project_logging and resource.project_id is not None:
                project_logger = Logger(
                    app_name,
                    True,  # per-project logging is always stackdriver
                    resource.project_id,
                    resource_creds
                )

        except Exception as e:
            logger({
                'message_id': message_id,
                'message': 'Exception while getting credentials for resource',
                'metadata': parsed_message.metadata.dict(),
                'resource_data': resource.to_dict(),
                **exc_info(e),
            })
            pubsub_message.ack()
            continue

        logger.debug({'message_id': message_id, 'message': 'Evaluating resource against policies'})

        try:
            evaluations = rpe.evaluate(resource)
            eval_time = int(time.time())
        except Exception as e:
            logger({
                'message_id': message_id,
                'resource_data': resource.to_dict(),
                'message': 'Exception while evaluating resource',
                'metadata': parsed_message.metadata.dict(),
                **exc_info(e),
            })
            pubsub_message.ack()
            continue

        if len(evaluations) < 1:
            logger.debug({'message_id': message_id, 'message': 'No policies matched resource'})

        enforcements = []

        # Log the results of evaluations on each policy for this resource
        for evaluation in evaluations:

            # Call hook to allow for customization
            hooks.process_evaluation(evaluation, parsed_message)

            # decide if we should remediate
            decision = EnforcementDecision(evaluation, parsed_message)
            hooks.process_enforcement_decision(decision, parsed_message)

            if decision.enforce:
                enforcements.append(decision)

            evaluation_log = {
                'compliant': evaluation.compliant,
                'event': 'evaluation',
                'excluded': evaluation.excluded,
                'message_id': message_id,
                'metadata': parsed_message.metadata.dict(),
                'policy_id': evaluation.policy_id,
                'remediable': evaluation.remediable,
                'resource_data': resource.to_dict(),
                'enforce': decision.enforce,
                'non_enforcement_conditions': decision.reasons,
                'timestamp': eval_time,
            }

            logger(evaluation_log)

            if project_logger is not None:
                project_logger(evaluation_log)

        # Skip all this if application-level enforcement is disabled
        if enforce_policy:

            for enforcement in enforcements:

                delay(parsed_message)
                logger.debug({'message_id': message_id, 'message': 'Executing remediation'})

                try:
                    enforcement.evaluation.remediate()
                    remediation_timestamp = int(time.time())

                    remediation_log = {
                        'event': 'remediation',
                        'message_id': message_id,
                        'metadata': parsed_message.metadata.dict(),
                        'policy_id': enforcement.evaluation.policy_id,
                        'resource_data': resource.to_dict(),
                        'timestamp': remediation_timestamp,
                    }

                    logger(remediation_log)

                    if project_logger is not None:
                        project_logger(remediation_log)

                except Exception as e:
                    # Catch any other exceptions so we can acknowledge the message.
                    # Otherwise they start to fill up the buffer of unacknowledged messages
                    logger(dict(
                        message='Execption while attempting to remediate',
                        message_id=message_id,
                        metadata=parsed_message.metadata.dict(),
                        policy_id=enforcement.evaluation.policy_id,
                        resource_data=resource.to_dict(),
                        **exc_info(e),
                    ))

        else:
            logger.debug({'message_id': message_id, 'message': 'Enforcement is disabled, processing complete'})

    # Finally ack the message after we're done with all of the assets
    pubsub_message.ack()

def delay(trigger):
    global enforcement_delay

    if enforcement_delay and trigger.control_data.delay_enforcement:

        # If the log is old, subtract that from the enforcement delay
        delay = max(0, enforcement_delay - trigger.age)
        time.sleep(delay)

def exc_info(exception):
    return {
        'event': 'exception',
        'details': str(exception),
        'trace': traceback.format_exc(),
    }

if __name__ == "__main__":

    subscriber = pubsub.SubscriberClient(credentials=app_creds)

    subscription_path = 'projects/{project_id}/subscriptions/{sub}'.format(
        project_id=project_id,
        sub=subscription_name
    )

    flow_control = pubsub.types.FlowControl(
        **flow_control_config
    )

    future = subscriber.subscribe(
        subscription_path,
        callback=callback,
        flow_control=flow_control,
    )

    logger("Listening for pubsub messages on {}...".format(subscription_path))

    try:
        future.result()
    except Exception:
        future.cancel()
        raise
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
# Name:        Lasso Regression
# Purpose:
#
# Author:      Nonlining
#
# Created:     26/01/2017
# Copyright:   (c) Nonlining 2017
# Licence:     <your licence>
#-------------------------------------------------------------------------------

import numpy as np

def lasso_coordinate_descent_step(i, feature_matrix, output, weights, l1_penalty):
    predictions = feature_matrix.dot(weights)
    rho = (feature_matrix[:, i].T).dot(output - predictions + (weights[i] * feature_matrix[:, i]))
    if i==0:
        new_weight = rho
    elif rho < (-l1_penalty/2.0):
        new_weight = rho + (l1_penalty/2.0)
    elif rho > (l1_penalty/2.0):
        new_weight = rho - (l1_penalty/2.0)
    else:
        new_weight = 0.0
    return new_weight

def lasso_cyclical_coordinate_descent(feature_matrix, output, initial_weights, l1_penalty, tolerance):

    D = feature_matrix.shape[1]
    weights = copy.copy(initial_weights)
    change = np.zeros(initial_weights.shape)
    converged = False

    while not converged:
        # Evaluate over all features
        for idx in range(D):
            # New weight for feature
            new_weight = lasso_coordinate_descent_step(idx, feature_matrix,
                                                       output, weights,
                                                       l1_penalty)
            # Compute change in weight for feature
            change[idx] = np.abs(new_weight - weights[idx])
            # assign new weight
            weights[idx] = new_weight
        # Maximum change in weight, after all changes have been computed
        max_change = max(change)
        if max_change < tolerance:
            converged = True
    return weights<commit_msg>some fix<commit_after>#-------------------------------------------------------------------------------
# Name:        Lasso Regression
# Purpose:
#
# Author:      Nonlining
#
# Created:     26/01/2017
# Copyright:   (c) Nonlining 2017
# Licence:     <your licence>
#-------------------------------------------------------------------------------

import numpy as np

def lasso_coordinate_descent_step(i, feature_matrix, output, weights, l1_penalty):
    predictions = feature_matrix.dot(weights)
    rho = (feature_matrix[:, i].T).dot(output - predictions + (weights[i] * feature_matrix[:, i]))
    if i==0:
        new_weight = rho
    elif rho < (-l1_penalty/2.0):
        new_weight = rho + (l1_penalty/2.0)
    elif rho > (l1_penalty/2.0):
        new_weight = rho - (l1_penalty/2.0)
    else:
        new_weight = 0.0
    return new_weight

def lasso_cyclical_coordinate_descent(feature_matrix, output, initial_weights, l1_penalty, tolerance):

    D = feature_matrix.shape[1]
    weights = copy.copy(initial_weights)
    change = np.zeros(initial_weights.shape)
    converged = False

    while not converged:
        for idx in range(D):
            new_weight = lasso_coordinate_descent_step(idx, feature_matrix,
                                                       output, weights,
                                                       l1_penalty)
            change[idx] = np.abs(new_weight - weights[idx])
            weights[idx] = new_weight
        max_change = max(change)
        if max_change < tolerance:
            converged = True
    return weights<|endoftext|>