<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

import optparse
import sys

from invenio.legacy.docextract.record import create_records, print_records
from invenio.refextract_kbs import get_kbs

from invenio.docextract_text import re_group_captured_multiple_space
from invenio.refextract_re import re_punctuation

DESCRIPTION = """Utility to convert journal names from abbreviations
or full names to their short form"""

HELP_MESSAGE = """
  -o, --out            Write the extracted references, in xml form, to a file
                       rather than standard output.
"""

USAGE_MESSAGE = """Usage: convert_journals [options] file1 [file2 ...]
Command options: %s
Examples:
    convert_journals -o /home/chayward/thesis-out.xml /home/chayward/thesis.xml
""" % HELP_MESSAGE

def mangle_value(kb, value):
    value = re_punctuation.sub(u' ', value.upper())
    value = re_group_captured_multiple_space.sub(u' ', value)
    value = value.strip()

    standardized_titles = kb[1]
    if value in standardized_titles:
        value = standardized_titles[value]

    return value

def mangle(kb, value):
    try:
        title, volume, page = value.split(',')
    except ValueError:
        pass
    else:
        value = '%s,%s,%s' % (mangle_value(kb, title), volume, page)
    return value

def convert_journals(kb, record):
    for subfield in record.find_subfields('999C5s'):
        subfield.value = mangle(kb, subfield.value)
    for subfield in record.find_subfields('773__p'):
        subfield.value = mangle_value(kb, subfield.value)
    return record

def convert_journals_list(kb, records):
    return [convert_journals(kb, record) for record in records]

def write_records(config, records):
    """Write marcxml to file

    * Output xml header
    * Output collection opening tag
    * Output xml for each record
    * Output collection closing tag
    """
    if config.xmlfile:
        out = open(config.xmlfile, 'w')
    else:
        out = sys.stdout

    xml = print_records(records)

    try:
        print >>out, xml
        out.flush()
    finally:
        if config.xmlfile:
            out.close()

def usage(wmsg=None, err_code=0):
    """Display a usage message for refextract on the standard error stream and
       then exit.
       @param wmsg: (string) some kind of brief warning message for the user.
       @param err_code: (integer) an error code to be passed to halt,
        which is called after the usage message has been printed.
       @return: None.
    """
    # Display the help information and the warning in the stderr stream
    # 'help_message' is global
    if wmsg:
        print >> sys.stderr, wmsg
    print >> sys.stderr, USAGE_MESSAGE
    sys.exit(err_code)

def cli_main(options, args):
    if options.help or not args:
        usage()
        return

    if options.kb_journals:
        kbs_files = {'journals': options.kb_journals}
    else:
        kbs_files = {}

    kb = get_kbs(custom_kbs_files=kbs_files)['journals']

    out_records = []
    for path in args:
        f = open(path)
        try:
            xml = f.read()
        finally:
            f.close()

        out_records += convert_journals_list(kb, create_records(xml))

    write_records(options, out_records)

def get_cli_options():
    """Get the various arguments and options from the command line and populate
       a dictionary of cli_options.
       @return: (tuple) of 2 elements. First element is a dictionary of cli
        options and flags, set as appropriate; Second element is a list of cli
        arguments.
    """
    parser = optparse.OptionParser(description=DESCRIPTION,
                                   usage=USAGE_MESSAGE,
                                   add_help_option=False)
    # Display help and exit
    parser.add_option('-h', '--help', action='store_true')
    # Write out MARC XML references to the specified file
    parser.add_option('-o', '--out', dest='xmlfile')
    # Handle verbosity
    parser.add_option('-v', '--verbose', type=int, dest='verbosity', default=0)
    # Specify a different journals database
    parser.add_option('--kb-journals', dest='kb_journals')

    return parser.parse_args()
<commit_msg>legacy: docextract imports fix<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

import optparse
import sys

from invenio.legacy.docextract.record import create_records, print_records
from invenio.refextract_kbs import get_kbs

from invenio.legacy.docextract.text import re_group_captured_multiple_space
from invenio.refextract_re import re_punctuation

DESCRIPTION = """Utility to convert journal names from abbreviations
or full names to their short form"""

HELP_MESSAGE = """
  -o, --out            Write the extracted references, in xml form, to a file
                       rather than standard output.
"""

USAGE_MESSAGE = """Usage: convert_journals [options] file1 [file2 ...]
Command options: %s
Examples:
    convert_journals -o /home/chayward/thesis-out.xml /home/chayward/thesis.xml
""" % HELP_MESSAGE

def mangle_value(kb, value):
    value = re_punctuation.sub(u' ', value.upper())
    value = re_group_captured_multiple_space.sub(u' ', value)
    value = value.strip()

    standardized_titles = kb[1]
    if value in standardized_titles:
        value = standardized_titles[value]

    return value

def mangle(kb, value):
    try:
        title, volume, page = value.split(',')
    except ValueError:
        pass
    else:
        value = '%s,%s,%s' % (mangle_value(kb, title), volume, page)
    return value

def convert_journals(kb, record):
    for subfield in record.find_subfields('999C5s'):
        subfield.value = mangle(kb, subfield.value)
    for subfield in record.find_subfields('773__p'):
        subfield.value = mangle_value(kb, subfield.value)
    return record

def convert_journals_list(kb, records):
    return [convert_journals(kb, record) for record in records]

def write_records(config, records):
    """Write marcxml to file

    * Output xml header
    * Output collection opening tag
    * Output xml for each record
    * Output collection closing tag
    """
    if config.xmlfile:
        out = open(config.xmlfile, 'w')
    else:
        out = sys.stdout

    xml = print_records(records)

    try:
        print >>out, xml
        out.flush()
    finally:
        if config.xmlfile:
            out.close()

def usage(wmsg=None, err_code=0):
    """Display a usage message for refextract on the standard error stream and
       then exit.
       @param wmsg: (string) some kind of brief warning message for the user.
       @param err_code: (integer) an error code to be passed to halt,
        which is called after the usage message has been printed.
       @return: None.
    """
    # Display the help information and the warning in the stderr stream
    # 'help_message' is global
    if wmsg:
        print >> sys.stderr, wmsg
    print >> sys.stderr, USAGE_MESSAGE
    sys.exit(err_code)

def cli_main(options, args):
    if options.help or not args:
        usage()
        return

    if options.kb_journals:
        kbs_files = {'journals': options.kb_journals}
    else:
        kbs_files = {}

    kb = get_kbs(custom_kbs_files=kbs_files)['journals']

    out_records = []
    for path in args:
        f = open(path)
        try:
            xml = f.read()
        finally:
            f.close()

        out_records += convert_journals_list(kb, create_records(xml))

    write_records(options, out_records)

def get_cli_options():
    """Get the various arguments and options from the command line and populate
       a dictionary of cli_options.
       @return: (tuple) of 2 elements. First element is a dictionary of cli
        options and flags, set as appropriate; Second element is a list of cli
        arguments.
    """
    parser = optparse.OptionParser(description=DESCRIPTION,
                                   usage=USAGE_MESSAGE,
                                   add_help_option=False)
    # Display help and exit
    parser.add_option('-h', '--help', action='store_true')
    # Write out MARC XML references to the specified file
    parser.add_option('-o', '--out', dest='xmlfile')
    # Handle verbosity
    parser.add_option('-v', '--verbose', type=int, dest='verbosity', default=0)
    # Specify a different journals database
    parser.add_option('--kb-journals', dest='kb_journals')

    return parser.parse_args()
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
@file ion/services/dm/ingestion/test/test_ingestion.py
@author Swarbhanu Chatterjee
@test ion.services.dm.ingestion.ingestion_management_service test suite to cover all ingestion mgmt service code
'''

import gevent
from mock import Mock, sentinel, patch
from pyon.util.unit_test import PyonTestCase
from pyon.util.int_test import IonIntegrationTestCase
from nose.plugins.attrib import attr
from pyon.core.exception import NotFound, BadRequest
from pyon.public import CFG, IonObject, log, RT, PRED, LCS, StreamPublisher, StreamSubscriber, StreamPublisherRegistrar, StreamSubscriberRegistrar
from interface.services.dm.iingestion_management_service import IngestionManagementServiceClient
from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient
from interface.services.dm.itransform_management_service import TransformManagementServiceClient
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
from interface.services.dm.idata_retriever_service import DataRetrieverServiceClient
from interface.services.icontainer_agent import ContainerAgentClient

from interface.objects import StreamQuery, ExchangeQuery, ProcessDefinition
from ion.services.dm.inventory.data_retriever_service import DataRetrieverService
from interface.objects import BlogPost, BlogComment
import time
from sets import Set

class BlogListener(object):

    def __init__(self):

        #-----------------------------------------------------------------------------------------------------------
        # Variables that are used for debugging tests... use find to see how these variables in the course of tests
        #-----------------------------------------------------------------------------------------------------------
        self.num_of_messages = 0

        self.blogs ={}

        # Contains the subscriber that calls back on the blog_store method
        self.subscriber = None

    def blog_store(self, message, headers):
        """
        Use a method in BlogListener object to hold state as we receive blog messages
        """

        log.debug('blog_store message received' )

        # store all posts... since there are very few posts
        if isinstance(message, BlogPost):

            # make a dictionary to contain the post and comments if it does not already exist
            self.blogs[message.post_id] = self.blogs.get(message.post_id, {})

            self.blogs[message.post_id]['post'] = message

        # store only 3 comments
        elif isinstance(message, BlogComment):

            # make a dictionary to contain the post and comments if it does not already exist
            self.blogs[message.ref_id] = self.blogs.get(message.ref_id, {})

            # make a dictionary to contain the comments if it doesn't already exist...
            self.blogs[message.ref_id]['comment'] = self.blogs[message.ref_id].get('comment', {})

            self.blogs[message.ref_id]['comment'][message.updated] = (message)

@attr('INT', group='dm')
class BlogIntegrationTest(IonIntegrationTestCase):

    def setUp(self):
        # set up the container for testing

        #-------------------------------------------------------------------------------------------------------
        # Container
        #-------------------------------------------------------------------------------------------------------
        self._start_container()

        self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)

        self.cc.start_rel_from_url('res/deploy/r2dm.yml')

        # make a registrar object - this is work usually done for you by the container in a transform or data stream process
        self.subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)

        #-----------------------------------------------------------------------------------------------------
        # Service clients
        #-----------------------------------------------------------------------------------------------------
        self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
        self.tms_cli = TransformManagementServiceClient(node=self.cc.node)
        self.ingestion_cli = IngestionManagementServiceClient(node=self.cc.node)
        self.rr_cli = ResourceRegistryServiceClient(node=self.cc.node)
        self.dr_cli = DataRetrieverServiceClient(node=self.cc.node)

    def test_blog_ingestion_replay(self):

        #-------------------------------------------------------------------------------------------------------
        # Create and activate ingestion configuration
        #-------------------------------------------------------------------------------------------------------

        ingestion_configuration_id = self.ingestion_cli.create_ingestion_configuration(exchange_point_id='science_data', couch_storage={},\
            hdf_storage={},  number_of_workers=6, default_policy={})
        # activates the transforms... so bindings will be created in this step
        self.ingestion_cli.activate_ingestion_configuration(ingestion_configuration_id)

        #------------------------------------------------------------------------------------------------------
        # Create subscriber to listen to the messages published to the ingestion
        #------------------------------------------------------------------------------------------------------

        # Define the query we want
        query = ExchangeQuery()

        # Create the stateful listener to hold the captured data for comparison with replay
        captured_input = BlogListener()

        # Start the subscription
        self._create_subscriber(subscription_name='input_capture_queue', subscription_query=query , blog_listener = captured_input)

        #-------------------------------------------------------------------------------------------------------
        # Launching blog scraper
        #-------------------------------------------------------------------------------------------------------
        self._launch_blog_scraper()

        # wait five seconds for some data to come in...
        log.warn('Sleeping for 5 seconds to wait for some input')
        time.sleep(5)

        #------------------------------------------------------------------------------------------------------
        # For 3 posts captured, make 3 replays and verify we get back what came in
        #------------------------------------------------------------------------------------------------------

        self.assertTrue(len(captured_input.blogs)>3)
        post_ids = [id for idx, id in enumerate(captured_input.blogs.iterkeys()) if idx < 3]

        #------------------------------------------------------------------------------------------------------
        # Create subscriber to listen to the replays
        #------------------------------------------------------------------------------------------------------

        captured_replays = {}

        for idx, post_id in enumerate(post_ids):
            # Create the stateful listener to hold the captured data for comparison with replay

            replay_id, stream_id = self._create_replay(post_id)

            query = StreamQuery(stream_ids=[stream_id])

            captured_replay = BlogListener()

            self._create_subscriber(subscription_name='replay_capture_queue_%d' % idx, subscription_query=query , blog_listener = captured_replay)

            self.dr_cli.start_replay(replay_id)

            captured_replays[post_id] = captured_replay

        # wait five seconds for some data to come in...
        log.warn('Sleeping for 5 seconds to wait for some output')
        time.sleep(5)

        matched_comments={}
        for post_id, captured_replay in captured_replays.iteritems():

            # There should be only one blog in here!
            self.assertEqual(len(captured_replay.blogs),1)

            replayed_blog = captured_replay.blogs[post_id]

            input_blog = captured_input.blogs[post_id]

            self.assertEqual(replayed_blog['post'].content, input_blog['post'].content)

            # can't deterministically assert that the number of comments is the same...
            matched_comments[post_id] = 0

            for updated, comment in replayed_blog.get('comment',{}).iteritems():
                self.assertIn(updated, input_blog['comment'])
                matched_comments[post_id] += 1

        log.info('Matched comments on the following blogs: %s' % matched_comments)

    def _create_subscriber(self, subscription_name, subscription_query, blog_listener):
        #------------------------------------------------------------------------------------------------------
        # Create subscriber to listen to the messages published to the ingestion
        #------------------------------------------------------------------------------------------------------

        # Make a subscription to the input stream to ingestion
        subscription_id = self.pubsub_cli.create_subscription(query = subscription_query, exchange_name=subscription_name ,name = subscription_name)

        # It is not required or even generally a good idea to use the subscription resource name as the queue name, but it makes things simple here
        # Normally the container creates and starts subscribers for you when a transform process is spawned
        subscriber = self.subscriber_registrar.create_subscriber(exchange_name=subscription_name, callback=blog_listener.blog_store)
        subscriber.start()

        blog_listener.subscriber = subscriber

        self.pubsub_cli.activate_subscription(subscription_id)

        return subscription_id

    def _launch_blog_scraper(self):
        #-------------------------------------------------------------------------------------------------------
        # Launching blog scraper
        #-------------------------------------------------------------------------------------------------------

        blogs = [
            'voodoofunk',
            'google-code-featured',
            'googleblog',
            'deerhuntertheband',
            'googlewebmastercentral',
            'strobist',
            'chrome',
            'hellohighprices',
            'saintsandspinners']

        blogs = [
            'google-code-featured',
            'googleblog',
            ]

        log.debug('before spawning blog scraper')

        for blog in blogs:
            config = {'process':{'type':'stream_process','blog':blog}}
            self.cc.spawn_process(name=blog,
                module='ion.services.dm.ingestion.example.blog_scraper',
                cls='FeedStreamer',
                config=config)

    def _create_replay(self, post_id):
        """
        Define the replay
        """

        query = {

            'datastore_name':'dm_datastore',
            'post_id':post_id
            #'options':{
            #    'limit':3,
            #    'include_docs':True
            #}
        }

        return self.dr_cli.define_replay(query=query)

<commit_msg>Fixed key error in test logic<commit_after>#!/usr/bin/env python

'''
@file ion/services/dm/test/test_blog_integration.py
@author Swarbhanu Chatterjee
@author David Stuebe
@test ion.services.dm.test.test_blog_integration Covers a demonstration of the basic capability to ingest and replay
simple data from a blog consisting of posts and comments.

This test starts the DM services and sets up ingestion for the science_data exachange point. Data is published by a
scapper providing input to the system and then replayed as part of the test. The test also subscribes to the incoming
data and verifies that the initial input matched the replayed data.

'''

import gevent
from pyon.util.int_test import IonIntegrationTestCase
from nose.plugins.attrib import attr
from pyon.public import CFG, IonObject, log, RT, PRED, LCS, StreamPublisher, StreamSubscriber, StreamPublisherRegistrar, StreamSubscriberRegistrar
from interface.services.dm.iingestion_management_service import IngestionManagementServiceClient
from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient
from interface.services.dm.itransform_management_service import TransformManagementServiceClient
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
from interface.services.dm.idata_retriever_service import DataRetrieverServiceClient
from interface.services.icontainer_agent import ContainerAgentClient

from interface.objects import StreamQuery, ExchangeQuery, ProcessDefinition
from ion.services.dm.inventory.data_retriever_service import DataRetrieverService
from interface.objects import BlogPost, BlogComment
import time

class BlogListener(object):

    def __init__(self):

        #-----------------------------------------------------------------------------------------------------------
        # Variables that are used for debugging tests... use find to see how these variables in the course of tests
        #-----------------------------------------------------------------------------------------------------------
        self.num_of_messages = 0

        self.blogs ={}

        # Contains the subscriber that calls back on the blog_store method
        self.subscriber = None

    def blog_store(self, message, headers):
        """
        Use a method in BlogListener object to hold state as we receive blog messages
        """

        log.debug('blog_store message received' )

        # store all posts... since there are very few posts
        if isinstance(message, BlogPost):

            # make a dictionary to contain the post and comments if it does not already exist
            self.blogs[message.post_id] = self.blogs.get(message.post_id, {})

            self.blogs[message.post_id]['post'] = message

        # store only 3 comments
        elif isinstance(message, BlogComment):

            # make a dictionary to contain the post and comments if it does not already exist
            self.blogs[message.ref_id] = self.blogs.get(message.ref_id, {})

            # make a dictionary to contain the comments if it doesn't already exist...
            self.blogs[message.ref_id]['comments'] = self.blogs[message.ref_id].get('comments', {})

            self.blogs[message.ref_id]['comments'][message.updated] = (message)

@attr('INT', group='dm')
class BlogIntegrationTest(IonIntegrationTestCase):

    def setUp(self):
        # set up the container for testing

        #-------------------------------------------------------------------------------------------------------
        # Container
        #-------------------------------------------------------------------------------------------------------
        self._start_container()

        self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)

        self.cc.start_rel_from_url('res/deploy/r2dm.yml')

        # make a registrar object - this is work usually done for you by the container in a transform or data stream process
        self.subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node)

        #-----------------------------------------------------------------------------------------------------
        # Service clients
        #-----------------------------------------------------------------------------------------------------
        self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
        self.tms_cli = TransformManagementServiceClient(node=self.cc.node)
        self.ingestion_cli = IngestionManagementServiceClient(node=self.cc.node)
        self.rr_cli = ResourceRegistryServiceClient(node=self.cc.node)
        self.dr_cli = DataRetrieverServiceClient(node=self.cc.node)

    def test_blog_ingestion_replay(self):

        #-------------------------------------------------------------------------------------------------------
        # Create and activate ingestion configuration
        #-------------------------------------------------------------------------------------------------------

        ingestion_configuration_id = self.ingestion_cli.create_ingestion_configuration(exchange_point_id='science_data', couch_storage={},\
            hdf_storage={},  number_of_workers=6, default_policy={})
        # activates the transforms... so bindings will be created in this step
        self.ingestion_cli.activate_ingestion_configuration(ingestion_configuration_id)

        #------------------------------------------------------------------------------------------------------
        # Create subscriber to listen to the messages published to the ingestion
        #------------------------------------------------------------------------------------------------------

        # Define the query we want
        query = ExchangeQuery()

        # Create the stateful listener to hold the captured data for comparison with replay
        captured_input = BlogListener()

        # Start the subscription
        self._create_subscriber(subscription_name='input_capture_queue', subscription_query=query , blog_listener = captured_input)

        #-------------------------------------------------------------------------------------------------------
        # Launching blog scraper
        #-------------------------------------------------------------------------------------------------------
        self._launch_blog_scraper()

        # wait ten seconds for some data to come in...
        log.warn('Sleeping for 10 seconds to wait for some input')
        time.sleep(10)

        #------------------------------------------------------------------------------------------------------
        # For 3 posts captured, make 3 replays and verify we get back what came in
        #------------------------------------------------------------------------------------------------------

        # Cute list comprehension method does not give enough control
        #self.assertTrue(len(captured_input.blogs)>3)
        #post_ids = [id for idx, id in enumerate(captured_input.blogs.iterkeys()) if idx < 3]

        post_ids = []

        if len(captured_input.blogs) < 1:
            self.fail('No data returned in ten seconds by the blog scrappers!')

        for post_id, blog in captured_input.blogs.iteritems(): # Use items not iter items - I copy of fixed length

            log.info('Captured Input: %s' % post_id)
            if len(blog.get('comments',[])) > 2:
                post_ids.append(post_id)

            if len(post_ids) >3:
                break

        else:
            self.fail('Not enough comments returned by the blog scrappers in 30 seconds')

        #------------------------------------------------------------------------------------------------------
        # Create subscriber to listen to the replays
        #------------------------------------------------------------------------------------------------------

        captured_replays = {}

        for idx, post_id in enumerate(post_ids):
            # Create the stateful listener to hold the captured data for comparison with replay

            replay_id, stream_id = self._create_replay(post_id)

            query = StreamQuery(stream_ids=[stream_id])

            captured_replay = BlogListener()

            self._create_subscriber(subscription_name='replay_capture_queue_%d' % idx, subscription_query=query , blog_listener = captured_replay)

            self.dr_cli.start_replay(replay_id)

            captured_replays[post_id] = captured_replay

        # wait five seconds for some data to come in...
        log.warn('Sleeping for 5 seconds to wait for some output')
        time.sleep(5)

        matched_comments={}
        for post_id, captured_replay in captured_replays.iteritems():

            # There should be only one blog in here!
            self.assertEqual(len(captured_replay.blogs),1)

            replayed_blog = captured_replay.blogs[post_id]

            input_blog = captured_input.blogs[post_id]

            self.assertEqual(replayed_blog['post'].content, input_blog['post'].content)

            # can't deterministically assert that the number of comments is the same...
            matched_comments[post_id] = 0

            for updated, comment in replayed_blog.get('comments',{}).iteritems():
                self.assertIn(updated, input_blog['comments'])
                matched_comments[post_id] += 1

        # Assert that we got some comments back!
        self.assertTrue(sum(matched_comments.values()) > 0)

        log.info('Matched comments on the following blogs: %s' % matched_comments)

    def _create_subscriber(self, subscription_name, subscription_query, blog_listener):
        #------------------------------------------------------------------------------------------------------
        # Create subscriber to listen to the messages published to the ingestion
        #------------------------------------------------------------------------------------------------------

        # Make a subscription to the input stream to ingestion
        subscription_id = self.pubsub_cli.create_subscription(query = subscription_query, exchange_name=subscription_name ,name = subscription_name)

        # It is not required or even generally a good idea to use the subscription resource name as the queue name, but it makes things simple here
        # Normally the container creates and starts subscribers for you when a transform process is spawned
        subscriber = self.subscriber_registrar.create_subscriber(exchange_name=subscription_name, callback=blog_listener.blog_store)
        subscriber.start()

        blog_listener.subscriber = subscriber

        self.pubsub_cli.activate_subscription(subscription_id)

        return subscription_id

    def _launch_blog_scraper(self):
        #-------------------------------------------------------------------------------------------------------
        # Launching blog scraper
        #-------------------------------------------------------------------------------------------------------

        blogs = [
            'voodoofunk',
            'google-code-featured',
            'googleblog',
            'deerhuntertheband',
            'googlewebmastercentral',
            'strobist',
            'chrome',
            'hellohighprices',
            'saintsandspinners',
            'warnakubrunei']

        blogs = [
            'saintsandspinners',
            'strobist',
            'voodoofunk'
            ]

        log.debug('before spawning blog scraper')

        for blog in blogs:
            config = {'process':{'type':'stream_process','blog':blog}}
            self.cc.spawn_process(name=blog,
                module='ion.services.dm.ingestion.example.blog_scraper',
                cls='FeedStreamer',
                config=config)

    def _create_replay(self, post_id):
        """
        Define the replay
        """

        query = {

            'datastore_name':'dm_datastore',
            'post_id':post_id
            #'options':{
            #    'limit':3,
            #    'include_docs':True
            #}
        }

        return self.dr_cli.define_replay(query=query)

<|endoftext|>


<commit_before>from templating import RendererFactory
from routing import lookup_controller, NonCanonicalPath
from util import _cfg, encode_if_needed
from middleware.recursive import ForwardRequestException

from webob import Request, Response, exc
from threading import local
from itertools import chain
from mimetypes import guess_type, add_type
from urlparse import urlsplit, urlunsplit
from os.path import splitext

try:
    from simplejson import loads
except ImportError:             # pragma: no cover
    from json import loads      # noqa

import urllib

# make sure that json is defined in mimetypes
add_type('application/json', '.json', True)

state = local()

def proxy(key):
    class ObjectProxy(object):
        def __getattr__(self, attr):
            obj = getattr(state, key)
            return getattr(obj, attr)

        def __setattr__(self, attr, value):
            obj = getattr(state, key)
            return setattr(obj, attr, value)

        def __delattr__(self, attr):
            obj = getattr(state, key)
            return delattr(obj, attr)
    return ObjectProxy()

request = proxy('request')
response = proxy('response')

def override_template(template, content_type=None):
    '''
    Call within a controller to override the template that is used in
    your response.

    :param template: a valid path to a template file, just as you would specify
                     in an ``@expose``.
    :param content_type: a valid MIME type to use for the response.func_closure
    '''

    request.pecan['override_template'] = template
    if content_type:
        request.pecan['override_content_type'] = content_type

def abort(status_code=None, detail='', headers=None, comment=None, **kw):
    '''
    Raise an HTTP status code, as specified. Useful for returning status
    codes like 401 Unauthorized or 403 Forbidden.

    :param status_code: The HTTP status code as an integer.
    :param detail: The message to send along, as a string.
    :param headers: A dictionary of headers to send along with the response.
    :param comment: A comment to include in the response.
    '''

    raise exc.status_map[status_code](
        detail=detail,
        headers=headers,
        comment=comment,
        **kw
    )

def redirect(location=None, internal=False, code=None, headers={},
             add_slash=False):
    '''
    Perform a redirect, either internal or external. An internal redirect
    performs the redirect server-side, while the external redirect utilizes
    an HTTP 302 status code.

    :param location: The HTTP location to redirect to.
    :param internal: A boolean indicating whether the redirect should be
                     internal.
    :param code: The HTTP status code to use for the redirect. Defaults to 302.
    :param headers: Any HTTP headers to send with the response, as a
                    dictionary.
    '''

    if add_slash:
        if location is None:
            split_url = list(urlsplit(state.request.url))
            new_proto = state.request.environ.get(
                'HTTP_X_FORWARDED_PROTO', split_url[0]
            )
            split_url[0] = new_proto
        else:
            split_url = urlsplit(location)

        split_url[2] = split_url[2].rstrip('/') + '/'
        location = urlunsplit(split_url)

    if not headers:
        headers = {}
    if internal:
        if code is not None:
            raise ValueError('Cannot specify a code for internal redirects')
        raise ForwardRequestException(location)
    if code is None:
        code = 302
    raise exc.status_map[code](location=location, headers=headers)

def render(template, namespace):
    '''
    Render the specified template using the Pecan rendering framework
    with the specified template namespace as a dictionary. Useful in a
    controller where you have no template specified in the ``@expose``.

    :param template: The path to your template, as you would specify in
                     ``@expose``.
    :param namespace: The namespace to use for rendering the template, as a
                      dictionary.
    '''

    return state.app.render(template, namespace)

def load_app(config):
    '''
    Used to load a ``Pecan`` application and its environment based on passed
    configuration.

    :param config: Can be a dictionary containing configuration, or a string
                   which represents a (relative) configuration filename.

    returns a pecan.Pecan object
    '''
    from configuration import _runtime_conf, set_config
    set_config(config, overwrite=True)

    for package_name in getattr(_runtime_conf.app, 'modules', []):
        module = __import__(package_name, fromlist=['app'])
        if hasattr(module, 'app') and hasattr(module.app, 'setup_app'):
            app = module.app.setup_app(_runtime_conf)
            app.config = _runtime_conf
            return app
    raise RuntimeError(
        'No app.setup_app found in any of the configured app.modules'
    )

class Pecan(object):
    '''
    Base Pecan application object. Generally created using ``pecan.make_app``,
    rather than being created manually.

    Creates a Pecan application instance, which is a WSGI application.

    :param root: A string representing a root controller object (e.g.,
                "myapp.controller.root.RootController")
    :param default_renderer: The default rendering engine to use. Defaults
                             to mako.
    :param template_path: The default relative path to use for templates.
                          Defaults to 'templates'.
    :param hooks: A list of Pecan hook objects to use for this application.
    :param custom_renderers: Custom renderer objects, as a dictionary keyed
                             by engine name.
    :param extra_template_vars: Any variables to inject into the template
                                namespace automatically.
    :param force_canonical: A boolean indicating if this project should
                            require canonical URLs.
    '''

    def __init__(self, root,
                 default_renderer='mako',
                 template_path='templates',
                 hooks=[],
                 custom_renderers={},
                 extra_template_vars={},
                 force_canonical=True
        ):
        '''
        '''

        if isinstance(root, basestring):
            root = self.__translate_root__(root)

        self.root = root
        self.renderers = RendererFactory(custom_renderers, extra_template_vars)
        self.default_renderer = default_renderer
        self.hooks = hooks
        self.template_path = template_path
        self.force_canonical = force_canonical

    def __translate_root__(self, item):
        '''
        Creates a root controller instance from a string root, e.g.,

        > __translate_root__("myproject.controllers.RootController")
        myproject.controllers.RootController()

        :param item: The string to the item
        '''

        if '.' in item:
            parts = item.split('.')
            name = '.'.join(parts[:-1])
            fromlist = parts[-1:]

            try:
                module = __import__(name, fromlist=fromlist)
                kallable = getattr(module, parts[-1])
                msg = "%s does not represent a callable class or function."
                assert hasattr(kallable, '__call__'), msg % item
                return kallable()
            except AttributeError:
                raise ImportError('No item named %s' % item)

        raise ImportError('No item named %s' % item)

    def route(self, node, path):
        '''
        Looks up a controller from a node based upon the specified path.

        :param node: The node, such as a root controller object.
        :param path: The path to look up on this node.
        '''

        path = path.split('/')[1:]
        try:
            node, remainder = lookup_controller(node, path)
            return node, remainder
        except NonCanonicalPath, e:
            if self.force_canonical and \
                not _cfg(e.controller).get('accept_noncanonical', False):
                if request.method == 'POST':
                    raise RuntimeError(
                        "You have POSTed to a URL '%s' which '\
                        'requires a slash. Most browsers will not maintain '\
                        'POST data when redirected. Please update your code '\
                        'to POST to '%s/' or set force_canonical to False" % \
                        (request.pecan['routing_path'],
                            request.pecan['routing_path'])
                    )
                redirect(code=302, add_slash=True)
            return e.controller, e.remainder

    def determine_hooks(self, controller=None):
        '''
        Determines the hooks to be run, in which order.

        :param controller: If specified, includes hooks for a specific
                           controller.
        '''

        controller_hooks = []
        if controller:
            controller_hooks = _cfg(controller).get('hooks', [])
        return list(
            sorted(
                chain(controller_hooks, self.hooks),
                lambda x, y: cmp(x.priority, y.priority)
            )
        )

    def handle_hooks(self, hook_type, *args):
        '''
        Processes hooks of the specified type.

        :param hook_type: The type of hook, including ``before``, ``after``,
                          ``on_error``, and ``on_route``.
        :param \*args: Arguments to pass to the hooks.
        '''

        if hook_type in ['before', 'on_route']:
            hooks = state.hooks
        else:
            hooks = reversed(state.hooks)

        for hook in hooks:
            getattr(hook, hook_type)(*args)

    def get_args(self, all_params, remainder, argspec, im_self):
        '''
        Determines the arguments for a controller based upon parameters
        passed the argument specification for the controller.
        '''
        args = []
        kwargs = dict()
        valid_args = argspec[0][1:]

        def _decode(x):
            return urllib.unquote_plus(x) if isinstance(x, basestring) else x

        remainder = [_decode(x) for x in remainder]

        if im_self is not None:
            args.append(im_self)

        # grab the routing args from nested REST controllers
        if 'routing_args' in request.pecan:
            remainder = request.pecan['routing_args'] + list(remainder)
            del request.pecan['routing_args']

        # handle positional arguments
        if valid_args and remainder:
            args.extend(remainder[:len(valid_args)])
            remainder = remainder[len(valid_args):]
            valid_args = valid_args[len(args):]

        # handle wildcard arguments
        if remainder:
            if not argspec[1]:
                abort(404)
            args.extend(remainder)

        # get the default positional arguments
        if argspec[3]:
            defaults = dict(zip(argspec[0][-len(argspec[3]):], argspec[3]))
        else:
            defaults = dict()

        # handle positional GET/POST params
        for name in valid_args:
            if name in all_params:
                args.append(all_params.pop(name))
            elif name in defaults:
                args.append(defaults[name])
            else:
                break

        # handle wildcard GET/POST params
        if argspec[2]:
            for name, value in all_params.iteritems():
                if name not in argspec[0]:
                    kwargs[encode_if_needed(name)] = value

        return args, kwargs

    def render(self, template, namespace):
        renderer = self.renderers.get(
            self.default_renderer,
            self.template_path
        )
        if template == 'json':
            renderer = self.renderers.get('json', self.template_path)
        if ':' in template:
            renderer = self.renderers.get(
                template.split(':')[0],
                self.template_path
            )
            template = template.split(':')[1]
        return renderer.render(template, namespace)

    def handle_request(self):
        '''
        The main request handler for Pecan applications.
        '''

        # get a sorted list of hooks, by priority (no controller hooks yet)
        state.hooks = self.determine_hooks()

        # store the routing path to allow hooks to modify it
        request.pecan['routing_path'] = request.path

        # handle "on_route" hooks
        self.handle_hooks('on_route', state)

        # lookup the controller, respecting content-type as requested
        # by the file extension on the URI
        path = request.pecan['routing_path']

        if not request.pecan['content_type'] and '.' in path.split('/')[-1]:
            path, extension = splitext(path)
            request.pecan['extension'] = extension
            # preface with a letter to ensure compat for 2.5
            request.pecan['content_type'] = guess_type('x' + extension)[0]

        controller, remainder = self.route(self.root, path)
        cfg = _cfg(controller)

        if cfg.get('generic_handler'):
            raise exc.HTTPNotFound

        # handle generic controllers
        im_self = None
        if cfg.get('generic'):
            im_self = controller.im_self
            handlers = cfg['generic_handlers']
            controller = handlers.get(request.method, handlers['DEFAULT'])
            cfg = _cfg(controller)

        # add the controller to the state so that hooks can use it
        state.controller = controller

        # if unsure ask the controller for the default content type
        if not request.pecan['content_type']:
            request.pecan['content_type'] = cfg.get(
                'content_type',
                'text/html'
            )
        elif cfg.get('content_type') is not None and \
            request.pecan['content_type'] not in cfg.get('content_types', {}):

            import warnings
            msg = "Controller '%s' defined does not support content_type " + \
                  "'%s'. Supported type(s): %s"
            warnings.warn(
                msg % (
                    controller.__name__,
                    request.pecan['content_type'],
                    cfg.get('content_types', {}).keys()
                ),
                RuntimeWarning
            )
            raise exc.HTTPNotFound

        # get a sorted list of hooks, by priority
        state.hooks = self.determine_hooks(controller)

        # handle "before" hooks
        self.handle_hooks('before', state)

        # fetch any parameters
        params = dict(request.params)

        # fetch the arguments for the controller
        args, kwargs = self.get_args(
            params,
            remainder,
            cfg['argspec'],
            im_self
        )

        # get the result from the controller
        result = controller(*args, **kwargs)

        # a controller can return the response object which means they've taken
        # care of filling it out
        if result == response:
            return

        raw_namespace = result

        # pull the template out based upon content type and handle overrides
        template = cfg.get('content_types', {}).get(
            request.pecan['content_type']
        )

        # check if for controller override of template
        template = request.pecan.get('override_template', template)
        request.pecan['content_type'] = request.pecan.get(
            'override_content_type',
            request.pecan['content_type']
        )

        # if there is a template, render it
        if template:
            if template == 'json':
                request.pecan['content_type'] = 'application/json'
            result = self.render(template, result)

        if 'pecan.params' in request.environ:
            params = request.environ.pop('pecan.params')

        # If we are in a test request put the namespace where it can be
        # accessed directly
        if request.environ.get('paste.testing'):
            testing_variables = request.environ['paste.testing_variables']
            testing_variables['namespace'] = raw_namespace
            testing_variables['template_name'] = template
            testing_variables['controller_output'] = result

        # set the body content
        if isinstance(result, unicode):
            response.unicode_body = result
        else:
            response.body = result

        # set the content type
        if request.pecan['content_type']:
            response.content_type = request.pecan['content_type']

    def __call__(self, environ, start_response):
        '''
        Implements the WSGI specification for Pecan applications, utilizing
        ``WebOb``.
        '''

        # create the request and response object
        state.request = Request(environ)
        state.response = Response()
        state.hooks = []
        state.app = self
        state.controller = None

        # handle the request
        try:
            # add context and environment to the request
            state.request.context = {}
            state.request.pecan = dict(content_type=None, validation_errors={})

            self.handle_request()
        except Exception, e:
            # if this is an HTTP Exception, set it as the response
            if isinstance(e, exc.HTTPException):
                state.response = e

            # if this is not an internal redirect, run error hooks
            if not isinstance(e, ForwardRequestException):
                self.handle_hooks('on_error', state, e)

            if not isinstance(e, exc.HTTPException):
                raise
        finally:
            # handle "after" hooks
            self.handle_hooks('after', state)

        # get the response
        try:
            return state.response(environ, start_response)
        finally:
            # clean up state
            del state.hooks
            del state.request
            del state.response
            del state.controller
<commit_msg>Removing some old validation cruft.<commit_after>from templating import RendererFactory
from routing import lookup_controller, NonCanonicalPath
from util import _cfg, encode_if_needed
from middleware.recursive import ForwardRequestException

from webob import Request, Response, exc
from threading import local
from itertools import chain
from mimetypes import guess_type, add_type
from urlparse import urlsplit, urlunsplit
from os.path import splitext

try:
    from simplejson import loads
except ImportError:             # pragma: no cover
    from json import loads      # noqa

import urllib

# make sure that json is defined in mimetypes
add_type('application/json', '.json', True)

state = local()

def proxy(key):
    class ObjectProxy(object):
        def __getattr__(self, attr):
            obj = getattr(state, key)
            return getattr(obj, attr)

        def __setattr__(self, attr, value):
            obj = getattr(state, key)
            return setattr(obj, attr, value)

        def __delattr__(self, attr):
            obj = getattr(state, key)
            return delattr(obj, attr)
    return ObjectProxy()

request = proxy('request')
response = proxy('response')

def override_template(template, content_type=None):
    '''
    Call within a controller to override the template that is used in
    your response.

    :param template: a valid path to a template file, just as you would specify
                     in an ``@expose``.
    :param content_type: a valid MIME type to use for the response.func_closure
    '''

    request.pecan['override_template'] = template
    if content_type:
        request.pecan['override_content_type'] = content_type

def abort(status_code=None, detail='', headers=None, comment=None, **kw):
    '''
    Raise an HTTP status code, as specified. Useful for returning status
    codes like 401 Unauthorized or 403 Forbidden.

    :param status_code: The HTTP status code as an integer.
    :param detail: The message to send along, as a string.
    :param headers: A dictionary of headers to send along with the response.
    :param comment: A comment to include in the response.
    '''

    raise exc.status_map[status_code](
        detail=detail,
        headers=headers,
        comment=comment,
        **kw
    )

def redirect(location=None, internal=False, code=None, headers={},
             add_slash=False):
    '''
    Perform a redirect, either internal or external. An internal redirect
    performs the redirect server-side, while the external redirect utilizes
    an HTTP 302 status code.

    :param location: The HTTP location to redirect to.
    :param internal: A boolean indicating whether the redirect should be
                     internal.
    :param code: The HTTP status code to use for the redirect. Defaults to 302.
    :param headers: Any HTTP headers to send with the response, as a
                    dictionary.
    '''

    if add_slash:
        if location is None:
            split_url = list(urlsplit(state.request.url))
            new_proto = state.request.environ.get(
                'HTTP_X_FORWARDED_PROTO', split_url[0]
            )
            split_url[0] = new_proto
        else:
            split_url = urlsplit(location)

        split_url[2] = split_url[2].rstrip('/') + '/'
        location = urlunsplit(split_url)

    if not headers:
        headers = {}
    if internal:
        if code is not None:
            raise ValueError('Cannot specify a code for internal redirects')
        raise ForwardRequestException(location)
    if code is None:
        code = 302
    raise exc.status_map[code](location=location, headers=headers)

def render(template, namespace):
    '''
    Render the specified template using the Pecan rendering framework
    with the specified template namespace as a dictionary. Useful in a
    controller where you have no template specified in the ``@expose``.

    :param template: The path to your template, as you would specify in
                     ``@expose``.
    :param namespace: The namespace to use for rendering the template, as a
                      dictionary.
    '''

    return state.app.render(template, namespace)

def load_app(config):
    '''
    Used to load a ``Pecan`` application and its environment based on passed
    configuration.

    :param config: Can be a dictionary containing configuration, or a string
                   which represents a (relative) configuration filename.

    returns a pecan.Pecan object
    '''
    from configuration import _runtime_conf, set_config
    set_config(config, overwrite=True)

    for package_name in getattr(_runtime_conf.app, 'modules', []):
        module = __import__(package_name, fromlist=['app'])
        if hasattr(module, 'app') and hasattr(module.app, 'setup_app'):
            app = module.app.setup_app(_runtime_conf)
            app.config = _runtime_conf
            return app
    raise RuntimeError(
        'No app.setup_app found in any of the configured app.modules'
    )

class Pecan(object):
    '''
    Base Pecan application object. Generally created using ``pecan.make_app``,
    rather than being created manually.

    Creates a Pecan application instance, which is a WSGI application.

    :param root: A string representing a root controller object (e.g.,
                "myapp.controller.root.RootController")
    :param default_renderer: The default rendering engine to use. Defaults
                             to mako.
    :param template_path: The default relative path to use for templates.
                          Defaults to 'templates'.
    :param hooks: A list of Pecan hook objects to use for this application.
    :param custom_renderers: Custom renderer objects, as a dictionary keyed
                             by engine name.
    :param extra_template_vars: Any variables to inject into the template
                                namespace automatically.
    :param force_canonical: A boolean indicating if this project should
                            require canonical URLs.
    '''

    def __init__(self, root,
                 default_renderer='mako',
                 template_path='templates',
                 hooks=[],
                 custom_renderers={},
                 extra_template_vars={},
                 force_canonical=True
        ):
        '''
        '''

        if isinstance(root, basestring):
            root = self.__translate_root__(root)

        self.root = root
        self.renderers = RendererFactory(custom_renderers, extra_template_vars)
        self.default_renderer = default_renderer
        self.hooks = hooks
        self.template_path = template_path
        self.force_canonical = force_canonical

    def __translate_root__(self, item):
        '''
        Creates a root controller instance from a string root, e.g.,

        > __translate_root__("myproject.controllers.RootController")
        myproject.controllers.RootController()

        :param item: The string to the item
        '''

        if '.' in item:
            parts = item.split('.')
            name = '.'.join(parts[:-1])
            fromlist = parts[-1:]

            try:
                module = __import__(name, fromlist=fromlist)
                kallable = getattr(module, parts[-1])
                msg = "%s does not represent a callable class or function."
                assert hasattr(kallable, '__call__'), msg % item
                return kallable()
            except AttributeError:
                raise ImportError('No item named %s' % item)

        raise ImportError('No item named %s' % item)

    def route(self, node, path):
        '''
        Looks up a controller from a node based upon the specified path.

        :param node: The node, such as a root controller object.
        :param path: The path to look up on this node.
        '''

        path = path.split('/')[1:]
        try:
            node, remainder = lookup_controller(node, path)
            return node, remainder
        except NonCanonicalPath, e:
            if self.force_canonical and \
                not _cfg(e.controller).get('accept_noncanonical', False):
                if request.method == 'POST':
                    raise RuntimeError(
                        "You have POSTed to a URL '%s' which '\
                        'requires a slash. Most browsers will not maintain '\
                        'POST data when redirected. Please update your code '\
                        'to POST to '%s/' or set force_canonical to False" % \
                        (request.pecan['routing_path'],
                            request.pecan['routing_path'])
                    )
                redirect(code=302, add_slash=True)
            return e.controller, e.remainder

    def determine_hooks(self, controller=None):
        '''
        Determines the hooks to be run, in which order.

        :param controller: If specified, includes hooks for a specific
                           controller.
        '''

        controller_hooks = []
        if controller:
            controller_hooks = _cfg(controller).get('hooks', [])
        return list(
            sorted(
                chain(controller_hooks, self.hooks),
                lambda x, y: cmp(x.priority, y.priority)
            )
        )

    def handle_hooks(self, hook_type, *args):
        '''
        Processes hooks of the specified type.

        :param hook_type: The type of hook, including ``before``, ``after``,
                          ``on_error``, and ``on_route``.
        :param \*args: Arguments to pass to the hooks.
        '''

        if hook_type in ['before', 'on_route']:
            hooks = state.hooks
        else:
            hooks = reversed(state.hooks)

        for hook in hooks:
            getattr(hook, hook_type)(*args)

    def get_args(self, all_params, remainder, argspec, im_self):
        '''
        Determines the arguments for a controller based upon parameters
        passed the argument specification for the controller.
        '''
        args = []
        kwargs = dict()
        valid_args = argspec[0][1:]

        def _decode(x):
            return urllib.unquote_plus(x) if isinstance(x, basestring) else x

        remainder = [_decode(x) for x in remainder]

        if im_self is not None:
            args.append(im_self)

        # grab the routing args from nested REST controllers
        if 'routing_args' in request.pecan:
            remainder = request.pecan['routing_args'] + list(remainder)
            del request.pecan['routing_args']

        # handle positional arguments
        if valid_args and remainder:
            args.extend(remainder[:len(valid_args)])
            remainder = remainder[len(valid_args):]
            valid_args = valid_args[len(args):]

        # handle wildcard arguments
        if remainder:
            if not argspec[1]:
                abort(404)
            args.extend(remainder)

        # get the default positional arguments
        if argspec[3]:
            defaults = dict(zip(argspec[0][-len(argspec[3]):], argspec[3]))
        else:
            defaults = dict()

        # handle positional GET/POST params
        for name in valid_args:
            if name in all_params:
                args.append(all_params.pop(name))
            elif name in defaults:
                args.append(defaults[name])
            else:
                break

        # handle wildcard GET/POST params
        if argspec[2]:
            for name, value in all_params.iteritems():
                if name not in argspec[0]:
                    kwargs[encode_if_needed(name)] = value

        return args, kwargs

    def render(self, template, namespace):
        renderer = self.renderers.get(
            self.default_renderer,
            self.template_path
        )
        if template == 'json':
            renderer = self.renderers.get('json', self.template_path)
        if ':' in template:
            renderer = self.renderers.get(
                template.split(':')[0],
                self.template_path
            )
            template = template.split(':')[1]
        return renderer.render(template, namespace)

    def handle_request(self):
        '''
        The main request handler for Pecan applications.
        '''

        # get a sorted list of hooks, by priority (no controller hooks yet)
        state.hooks = self.determine_hooks()

        # store the routing path to allow hooks to modify it
        request.pecan['routing_path'] = request.path

        # handle "on_route" hooks
        self.handle_hooks('on_route', state)

        # lookup the controller, respecting content-type as requested
        # by the file extension on the URI
        path = request.pecan['routing_path']

        if not request.pecan['content_type'] and '.' in path.split('/')[-1]:
            path, extension = splitext(path)
            request.pecan['extension'] = extension
            # preface with a letter to ensure compat for 2.5
            request.pecan['content_type'] = guess_type('x' + extension)[0]

        controller, remainder = self.route(self.root, path)
        cfg = _cfg(controller)

        if cfg.get('generic_handler'):
            raise exc.HTTPNotFound

        # handle generic controllers
        im_self = None
        if cfg.get('generic'):
            im_self = controller.im_self
            handlers = cfg['generic_handlers']
            controller = handlers.get(request.method, handlers['DEFAULT'])
            cfg = _cfg(controller)

        # add the controller to the state so that hooks can use it
        state.controller = controller

        # if unsure ask the controller for the default content type
        if not request.pecan['content_type']:
            request.pecan['content_type'] = cfg.get(
                'content_type',
                'text/html'
            )
        elif cfg.get('content_type') is not None and \
            request.pecan['content_type'] not in cfg.get('content_types', {}):

            import warnings
            msg = "Controller '%s' defined does not support content_type " + \
                  "'%s'. Supported type(s): %s"
            warnings.warn(
                msg % (
                    controller.__name__,
                    request.pecan['content_type'],
                    cfg.get('content_types', {}).keys()
                ),
                RuntimeWarning
            )
            raise exc.HTTPNotFound

        # get a sorted list of hooks, by priority
        state.hooks = self.determine_hooks(controller)

        # handle "before" hooks
        self.handle_hooks('before', state)

        # fetch any parameters
        params = dict(request.params)

        # fetch the arguments for the controller
        args, kwargs = self.get_args(
            params,
            remainder,
            cfg['argspec'],
            im_self
        )

        # get the result from the controller
        result = controller(*args, **kwargs)

        # a controller can return the response object which means they've taken
        # care of filling it out
        if result == response:
            return

        raw_namespace = result

        # pull the template out based upon content type and handle overrides
        template = cfg.get('content_types', {}).get(
            request.pecan['content_type']
        )

        # check if for controller override of template
        template = request.pecan.get('override_template', template)
        request.pecan['content_type'] = request.pecan.get(
            'override_content_type',
            request.pecan['content_type']
        )

        # if there is a template, render it
        if template:
            if template == 'json':
                request.pecan['content_type'] = 'application/json'
            result = self.render(template, result)

        if 'pecan.params' in request.environ:
            params = request.environ.pop('pecan.params')

        # If we are in a test request put the namespace where it can be
        # accessed directly
        if request.environ.get('paste.testing'):
            testing_variables = request.environ['paste.testing_variables']
            testing_variables['namespace'] = raw_namespace
            testing_variables['template_name'] = template
            testing_variables['controller_output'] = result

        # set the body content
        if isinstance(result, unicode):
            response.unicode_body = result
        else:
            response.body = result

        # set the content type
        if request.pecan['content_type']:
            response.content_type = request.pecan['content_type']

    def __call__(self, environ, start_response):
        '''
        Implements the WSGI specification for Pecan applications, utilizing
        ``WebOb``.
        '''

        # create the request and response object
        state.request = Request(environ)
        state.response = Response()
        state.hooks = []
        state.app = self
        state.controller = None

        # handle the request
        try:
            # add context and environment to the request
            state.request.context = {}
            state.request.pecan = dict(content_type=None)

            self.handle_request()
        except Exception, e:
            # if this is an HTTP Exception, set it as the response
            if isinstance(e, exc.HTTPException):
                state.response = e

            # if this is not an internal redirect, run error hooks
            if not isinstance(e, ForwardRequestException):
                self.handle_hooks('on_error', state, e)

            if not isinstance(e, exc.HTTPException):
                raise
        finally:
            # handle "after" hooks
            self.handle_hooks('after', state)

        # get the response
        try:
            return state.response(environ, start_response)
        finally:
            # clean up state
            del state.hooks
            del state.request
            del state.response
            del state.controller
<|endoftext|>


<commit_before># Copyright 2016 Uri Laserson
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from logging import captureWarnings

# biopython has a bunch of annoying warnings bc Seq comparisons changed
captureWarnings(True)

from click import group, command, option, argument, File, Choice
from Bio import SeqIO
from Bio.SeqRecord import SeqRecord

from pepsyn.operations import (
    reverse_translate, remove_site_from_cds, x_to_ggsg, disambiguate_iupac_aa,
    tile as tile_op)
from pepsyn.codons import (
    FreqWeightedCodonSampler, UniformCodonSampler, ecoli_codon_usage)
from pepsyn.util import site2dna

@group(context_settings={'help_option_names': ['-h', '--help']})
def cli():
    """pepsyn -- peptide synthesis design"""
    pass

# reusable args
argument_input = argument('input', type=File('r'))
argument_output = argument('output', type=File('w'))

@cli.command()
@argument_input
@argument_output
@option('--length', '-l', type=int, help='Length of output oligos')
@option('--overlap', '-p', type=int, help='Overlap of oligos')
def tile(input, output, length, overlap):
    """tile a set of sequences"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        for (start, end, t) in tile_op(seqrecord.seq, length, overlap):
                output_title = '{}|{}-{}'.format(seqrecord.id, start, end)
                output_record = SeqRecord(t, output_title, description='')
                SeqIO.write(output_record, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--prefix', '-p', help='DNA sequence to prefix each oligo')
def prefix(input, output, prefix):
    """add a prefix to each sequence"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        newseq = prefix + seqrecord.seq
        seqrecord.seq = newseq
        SeqIO.write(seqrecord, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--suffix', '-s', help='DNA sequence to suffix each oligo')
def suffix(input, output, suffix):
    """add a suffix to each sequence"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        newseq = seqrecord.seq + suffix
        seqrecord.seq = newseq
        SeqIO.write(seqrecord, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--codon-table', '-t', default='standard',
        help='ONLY STANDARD TABLE IMPLEMENTED')
@option('--codon-usage', '-u', default='ecoli', help='ONLY ECOLI IMPLEMENTED')
@option('--sampler', default='weighted', show_default=True,
        type=Choice(['weighted', 'uniform']), help='Codon sampling method')
def revtrans(input, output, codon_table, codon_usage, sampler):
    """reverse translate amino acid sequences into DNA"""
    if sampler == 'weighted':
        codon_sampler = FreqWeightedCodonSampler(usage=ecoli_codon_usage)
    elif sampler == 'uniform':
        codon_sampler = UniformCodonSampler()
    for seqrecord in SeqIO.parse(input, 'fasta'):
        dna_id = seqrecord.id
        dna_seq = reverse_translate(seqrecord.seq, codon_sampler)
        SeqIO.write(SeqRecord(dna_seq, dna_id, description=''), output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--site', help='Site to remove (e.g., EcoRI, AGCCT); case sensitive')
@option('--clip-left', type=int,
        help='Number of bases to clip from start of sequence to get to CDS')
@option('--clip-right', type=int,
        help='Number of bases to clip from end of sequence to get to CDS')
@option('--codon-table', '-t', default='standard',
        help='ONLY STANDARD TABLE IMPLEMENTED')
@option('--codon-usage', '-u', default='ecoli', help='ONLY ECOLI IMPLEMENTED')
@option('--sampler', default='weighted', show_default=True,
        type=Choice(['weighted', 'uniform']), help='Codon sampling method')
def removesite(input, output, site, clip_left, clip_right, codon_table,
               codon_usage, sampler):
    """remove site from each sequence's CDS by recoding"""
    if sampler == 'weighted':
        codon_sampler = FreqWeightedCodonSampler(usage=ecoli_codon_usage)
    elif sampler == 'uniform':
        codon_sampler = UniformCodonSampler()

    # import pdb; pdb.set_trace()
    site = site2dna(site)
    # site is now a Bio.Seq.Seq

    for seqrecord in SeqIO.parse(input, 'fasta'):
        id_ = seqrecord.id
        cds_start = clip_left
        cds_end = len(seqrecord) - clip_right
        seq = remove_site_from_cds(seqrecord.seq, site, codon_sampler,
                                   cds_start, cds_end)
        SeqIO.write(SeqRecord(seq, id_, description=''), output, 'fasta')

@cli.command()
@argument_input
@argument_output
def x2ggsg(input, output):
    """replace stretches of Xs with Serine-Glycine linker (GGSG pattern)"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        seq = seqrecord.seq
        replacement = x_to_ggsg(seq)
        seqrecord.seq = replacement
        if replacement != seq:
            seqrecord.id = '{}|{}'.format(seqrecord.id, 'withGSlinker')
        SeqIO.write(seqrecord, output, 'fasta')

@cli.command()
@argument_input
@argument_output
def disambiguateaa(input, output):
    """replace ambiguous (IUPAC) AAs with unambiguous ones (e.g. Z => E/Q)

    B => DN, X => ACDEFGHIKLMNPQRSTVWY, Z => EQ, J => LI,
    U => C (selenocysteine), O => K (pyrrolysine)
    """
    for seqrecord in SeqIO.parse(input, 'fasta'):
        id_ = seqrecord.id
        ambig = seqrecord.seq
        for (i, unambig) in enumerate(disambiguate_iupac_aa(ambig)):
            if unambig != ambig:
                seqrecord.id = '{}|disambig_{}'.format(id_, i + 1)
                seqrecord.seq = unambig
            SeqIO.write(seqrecord, output, 'fasta')

@cli.command()
@argument_input
@option('--site', help='Site to find (e.g., EcoRI, AGCCT); case sensitive')
def findsite(input, site):
    """find locations of a site"""
    query = site2dna(site)
    for seqrecord in SeqIO.parse(input, 'fasta'):
        id_ = seqrecord.id
        idx = seqrecord.seq.find(query)
        if idx >= 0:
            print('{}|{}|{}'.format(id_, site, idx), flush=True)

@cli.command()
@argument_input
def stats(input):
    """NOT IMPL'd: compute some sequence statistics"""
    pass
<commit_msg>Mainly added ctermpep and pad<commit_after># Copyright 2016 Uri Laserson
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from logging import captureWarnings

# biopython has a bunch of annoying warnings bc Seq comparisons changed
captureWarnings(True)

from click import group, command, option, argument, File, Choice
from Bio import SeqIO
from Bio.SeqRecord import SeqRecord
from Bio.Data.CodonTable import standard_dna_table

from pepsyn.operations import (
    reverse_translate, remove_site_from_cds, x_to_ggsg, disambiguate_iupac_aa,
    tile as tile_op, ctermpep as cterm_oligo, pad_ggsg)
from pepsyn.codons import (
    FreqWeightedCodonSampler, UniformCodonSampler, ecoli_codon_usage,
    zero_non_amber_stops, zero_low_freq_codons)
from pepsyn.util import site2dna

@group(context_settings={'help_option_names': ['-h', '--help']})
def cli():
    """pepsyn -- peptide synthesis design"""
    pass

# reusable args
argument_input = argument('input', type=File('r'))
argument_output = argument('output', type=File('w'))

@cli.command()
@argument_input
@argument_output
@option('--length', '-l', type=int, help='Length of output oligos')
@option('--overlap', '-p', type=int, help='Overlap of oligos')
def tile(input, output, length, overlap):
    """tile a set of sequences"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        for (start, end, t) in tile_op(seqrecord.seq, length, overlap):
                output_title = '{}|{}-{}'.format(seqrecord.id, start, end)
                output_record = SeqRecord(t, output_title, description='')
                SeqIO.write(output_record, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--length', '-l', type=int, help='Length of output C-terminal oligo')
@option('--add-stop', '-s', is_flag=True, help='Add a stop codon to peptide')
def ctermpep(input, output, length, add_stop):
    """extract C-terminal peptide (AA alphabets)"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        oligo = cterm_oligo(seqrecord.seq, length, add_stop=add_stop)
        output_title = '{}|CTERM'.format(seqrecord.id)
        if add_stop:
            output_title = '{}|STOP'.format(output_title)
        output_record = SeqRecord(oligo, output_title, description='')
        SeqIO.write(output_record, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--length', '-l', type=int, help='Target length for peptide')
@option('--n-term', '-n', 'terminus', flag_value='N',
        help='Pad the N-terminus')
@option('--c-term', '-c', 'terminus', flag_value='C', default=True,
        help='Pad the C-terminus')
def pad(input, output, length, terminus):
    """pad peptide to target length with GSGG"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        padded = pad_ggsg(seqrecord.seq, length, terminus)
        pad_len = len(padded) - len(seqrecord)
        if pad_len > 0:
            output_title = '{}|{}-PADDED-{}'.format(seqrecord.id, terminus,
                                                    pad_len)
        else:
            output_title = seqrecord.id
        output_record = SeqRecord(padded, output_title, description='')
        SeqIO.write(output_record, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--prefix', '-p', help='DNA sequence to prefix each oligo')
def prefix(input, output, prefix):
    """add a prefix to each sequence"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        newseq = prefix + seqrecord.seq
        seqrecord.seq = newseq
        SeqIO.write(seqrecord, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--suffix', '-s', help='DNA sequence to suffix each oligo')
def suffix(input, output, suffix):
    """add a suffix to each sequence"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        newseq = seqrecord.seq + suffix
        seqrecord.seq = newseq
        SeqIO.write(seqrecord, output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--codon-table', '-t', default='standard',
        help='ONLY STANDARD TABLE IMPLEMENTED')
@option('--codon-usage', '-u', default='ecoli', help='ONLY ECOLI IMPLEMENTED')
@option('--sampler', default='weighted', show_default=True,
        type=Choice(['weighted', 'uniform']), help='Codon sampling method')
@option('--codon-freq-threshold', type=float, default=None,
        help='Minimum codon frequency')
@option('--amber-only', is_flag=True, help='Use only amber stop codon')
def revtrans(input, output, codon_table, codon_usage, sampler,
             codon_freq_threshold, amber_only):
    """reverse translate amino acid sequences into DNA"""
    if sampler == 'weighted':
        usage = ecoli_codon_usage
        if codon_freq_threshold is not None:
            # TODO: this is hardcoded in and there's a leaky abstraction here
            table = standard_dna_table
            usage = zero_low_freq_codons(usage, table, codon_freq_threshold)
        if amber_only:
            usage = zero_non_amber_stops(usage)
        codon_sampler = FreqWeightedCodonSampler(usage=usage)
    elif sampler == 'uniform':
        codon_sampler = UniformCodonSampler()
    for seqrecord in SeqIO.parse(input, 'fasta'):
        dna_id = seqrecord.id
        dna_seq = reverse_translate(seqrecord.seq, codon_sampler)
        SeqIO.write(SeqRecord(dna_seq, dna_id, description=''), output, 'fasta')

@cli.command()
@argument_input
@argument_output
@option('--site', help='Site to remove (e.g., EcoRI, AGCCT); case sensitive')
@option('--clip-left', type=int,
        help='Number of bases to clip from start of sequence to get to CDS')
@option('--clip-right', type=int,
        help='Number of bases to clip from end of sequence to get to CDS')
@option('--codon-table', '-t', default='standard',
        help='ONLY STANDARD TABLE IMPLEMENTED')
@option('--codon-usage', '-u', default='ecoli', help='ONLY ECOLI IMPLEMENTED')
@option('--sampler', default='weighted', show_default=True,
        type=Choice(['weighted', 'uniform']), help='Codon sampling method')
@option('--codon-freq-threshold', type=float, default=None,
        help='Minimum codon frequency')
@option('--amber-only', is_flag=True, help='Use only amber stop codon')
def removesite(input, output, site, clip_left, clip_right, codon_table,
               codon_usage, sampler, codon_freq_threshold, amber_only):
    """remove site from each sequence's CDS by recoding"""
    if sampler == 'weighted':
        usage = ecoli_codon_usage
        if codon_freq_threshold is not None:
            # TODO: this is hardcoded in and there's a leaky abstraction here
            table = standard_dna_table
            usage = zero_low_freq_codons(usage, table, codon_freq_threshold)
        if amber_only:
            usage = zero_non_amber_stops(usage)
        codon_sampler = FreqWeightedCodonSampler(usage=usage)
    elif sampler == 'uniform':
        codon_sampler = UniformCodonSampler()

    # import pdb; pdb.set_trace()
    site = site2dna(site)
    # site is now a Bio.Seq.Seq

    for seqrecord in SeqIO.parse(input, 'fasta'):
        id_ = seqrecord.id
        cds_start = clip_left
        cds_end = len(seqrecord) - clip_right
        seq = remove_site_from_cds(seqrecord.seq, site, codon_sampler,
                                   cds_start, cds_end)
        SeqIO.write(SeqRecord(seq, id_, description=''), output, 'fasta')

@cli.command()
@argument_input
@argument_output
def x2ggsg(input, output):
    """replace stretches of Xs with Serine-Glycine linker (GGSG pattern)"""
    for seqrecord in SeqIO.parse(input, 'fasta'):
        replacement = x_to_ggsg(seqrecord.seq)
        if replacement != seqrecord.seq:
            output_title = '{}|{}'.format(seqrecord.id, 'withGSlinker')
        else:
            output_title = seqrecord.id
        output_record = SeqRecord(replacement, id=output_title, description='')
        SeqIO.write(output_record, output, 'fasta')

@cli.command()
@argument_input
@argument_output
def disambiguateaa(input, output):
    """replace ambiguous (IUPAC) AAs with unambiguous ones (e.g. Z => E/Q)

    B => DN, X => ACDEFGHIKLMNPQRSTVWY, Z => EQ, J => LI,
    U => C (selenocysteine), O => K (pyrrolysine)
    """
    for seqrecord in SeqIO.parse(input, 'fasta'):
        id_ = seqrecord.id
        ambig = seqrecord.seq
        for (i, unambig) in enumerate(disambiguate_iupac_aa(ambig)):
            if unambig != ambig:
                output_title = '{}|disambig_{}'.format(id_, i + 1)
            else:
                output_title = id_
            output_record = SeqRecord(unambig, output_title, description='')
            SeqIO.write(output_record, output, 'fasta')

@cli.command()
@argument_input
@option('--site', help='Site to find (e.g., EcoRI, AGCCT); case sensitive')
def findsite(input, site):
    """find locations of a site"""
    query = site2dna(site)
    for seqrecord in SeqIO.parse(input, 'fasta'):
        id_ = seqrecord.id
        idx = seqrecord.seq.find(query)
        if idx >= 0:
            print('{}|{}|{}'.format(id_, site, idx), flush=True)

@cli.command()
@argument_input
def stats(input):
    """NOT IMPL'd: compute some sequence statistics"""
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
doing stuff i really wish we could have done natively in influxdb -- what a disappointment
"""

import logging
import sys

from influxdb import InfluxDBClient

def get_thresholds(client: InfluxDBClient, percentile_series: str, time_offset: int) -> [(str, str)]:
    """gets timestamp and threshold tuples from a given series within a given offset

    :param client: an influxdb client
    :param percentile_series: the name of series containing the percentile values
    :param time_offset: the influxdb formatted time offset
    :return: a list of timestamp, threshold tuples
    """
    query = 'select * from {} where time > now() - {}'.format(percentile_series, time_offset)
    results = client.query(query)

    if len(results):
        results = results[0]
    else:
        logging.error('`get_thresholds` query results are empty')
        raise Exception('results are empty')

    points = results.get('points', [])
    return [(timestamp, threshold) for timestamp, _, threshold in points]

def get_content(client: InfluxDBClient, content_series: str, time_offset: int, threshold: int) -> [[int, int, int, str]]:
    """gets content points that match a given time offset and have click values within the threshold

    :param client: an influxdb client
    :param content_series: the name of the series containing the content
    :param time_offset: the influxdb formatted time offset
    :param threshold: the minimum number of clicks allowable
    :return: a list of time, sequence_number, clicks, content_id lists
    """
    query = 'select * from {} where clicks >= {} and time > now() - {}'.format(content_series, threshold, time_offset)
    results = client.query(query)

    if len(results):
        results = results[0]
    else:
        logging.error('`get_content` query results are empty')
        raise Exception('results are empty')

    points = results.get('points', [])
    return points

def write_trend_point(client: InfluxDBClient, trending_series: str, content: [[int, int, int, str]], threshold: int):
    """writes trend values to a given series so that it can be later recalled or rendered

    :param client: and influxdb client
    :param trending_series: the name of the series to write to
    :param content: the content/point lists from `get_content`
    :param threshold: the minimum number of clicks allowable
    """
    body = [{
        'name': trending_series,
        'columns': ['time', 'clicks', 'content_id', 'threshold'],
        'points': [[t[0], t[2], t[3], threshold] for t in content],
    }]
    try:
        client.write_points(body)
    except Exception as e:
        logging.error('`write_trend_point` write operation failed: {}'.format(str(e)))

if __name__ == '__main__':
    # get values
    _, host, port, username, password, db, percentile_series, content_series, trending_series, offset = sys.argv

    # create client
    client = InfluxDBClient(host, port, username, password, db)

    # get thresholds; iterate
    thresholds = get_thresholds(client, percentile_series, offset)
    for timestamp, threshold in thresholds:
        # log it
        logging.info('getting content: >= {} on {}'.format(threshold, timestamp))

        # get content
        content = get_content(client, content_series, offset, threshold)
        # log it
        logging.info('found {} points that match'.format(len(content)))

        # write to series
        write_trend_point(client, trending_series, content, threshold)
<commit_msg>added sequence number to stop doubling up every run<commit_after>#!/usr/bin/env python3

"""
doing stuff i really wish we could have done natively in influxdb -- what a disappointment
"""

import logging
import sys

from influxdb import InfluxDBClient

def get_thresholds(client: InfluxDBClient, percentile_series: str, time_offset: int) -> [(str, str)]:
    """gets timestamp and threshold tuples from a given series within a given offset

    :param client: an influxdb client
    :param percentile_series: the name of series containing the percentile values
    :param time_offset: the influxdb formatted time offset
    :return: a list of timestamp, threshold tuples
    """
    query = 'select * from {} where time > now() - {}'.format(percentile_series, time_offset)
    results = client.query(query)

    if len(results):
        results = results[0]
    else:
        logging.error('`get_thresholds` query results are empty')
        raise Exception('results are empty')

    points = results.get('points', [])
    return [(timestamp, threshold) for timestamp, _, threshold in points]

def get_content(client: InfluxDBClient, content_series: str, time_offset: int, threshold: int) -> [[int, int, int, str]]:
    """gets content points that match a given time offset and have click values within the threshold

    :param client: an influxdb client
    :param content_series: the name of the series containing the content
    :param time_offset: the influxdb formatted time offset
    :param threshold: the minimum number of clicks allowable
    :return: a list of time, sequence_number, clicks, content_id lists
    """
    query = 'select * from {} where clicks >= {} and time > now() - {}'.format(content_series, threshold, time_offset)
    results = client.query(query)

    if len(results):
        results = results[0]
    else:
        logging.error('`get_content` query results are empty')
        raise Exception('results are empty')

    points = results.get('points', [])
    return points

def write_trend_point(client: InfluxDBClient, trending_series: str, content: [[int, int, int, str]], threshold: int):
    """writes trend values to a given series so that it can be later recalled or rendered

    :param client: and influxdb client
    :param trending_series: the name of the series to write to
    :param content: the content/point lists from `get_content`
    :param threshold: the minimum number of clicks allowable
    """
    body = [{
        'name': trending_series,
        'columns': ['time', 'sequence_number', 'clicks', 'content_id', 'threshold'],
        'points': [[t[0], 1, t[2], t[3], threshold] for t in content],
    }]
    try:
        client.write_points(body)
    except Exception as e:
        logging.error('`write_trend_point` write operation failed: {}'.format(str(e)))

if __name__ == '__main__':
    # get values
    _, host, port, username, password, db, percentile_series, content_series, trending_series, offset = sys.argv

    # create client
    client = InfluxDBClient(host, port, username, password, db)

    # get thresholds; iterate
    thresholds = get_thresholds(client, percentile_series, offset)
    for timestamp, threshold in thresholds:
        # log it
        logging.info('getting content: >= {} on {}'.format(threshold, timestamp))

        # get content
        content = get_content(client, content_series, offset, threshold)
        # log it
        logging.info('found {} points that match'.format(len(content)))

        # write to series
        write_trend_point(client, trending_series, content, threshold)
<|endoftext|>


<commit_before>"""
Implementation of a Perceptron Learning Algorithm, that can be abstracted to
various input sizes (NUM) or dimensions (DIM). Displays using pyplot.
"""
from matplotlib import pyplot as plt
from math import ceil
import numpy as np

NUM = 100  # Sample size
DIM = 11  # Number of Dimensions
w = []
y = []
matrix = []
if DIM == 3:
    PLOT = True  # whether or not to show the plot (only works in 2D)
else:
    PLOT = False

def f(x):  # Target Function
    return 0.5 * x + 1.25

def setup():
    global w, y, matrix
    w = np.random.rand(DIM)  # randomly selected weights
    matrix = np.random.rand(NUM, DIM) * 5  # randomly selected points
    y = [1] * NUM  # default values for y
    for i in range(NUM):
        matrix[i][0] = 1
        if (f(matrix[i][1])) > matrix[i][2]:  # determine if point is above line/hyperplane formed by f()
            y[i] = -1  # correct y[i] if below line
    if PLOT:
        # for x in range(NUM):
            # if y[x] == 1:
            #     plt.plot(matrix[x][1], matrix[x][2], 'Dg')
            # else:
            #     plt.plot(matrix[x][1], matrix[x][2], 'ro')
        # plt.plot([0,5], [f(0), f(5)], label='F')  # plot Target function f(x)
        plt.ylabel('X_2 Axis')
        plt.xlabel('X_1 Axis')

def next_w(w, y, x):  # Perceptron update function
    r = []
    for i in range(DIM):
        r.append(w[i] + y*x[i])
    return r

def sign(k):  # returns 1 if k > 0, -1 otherwise
    return 1 if float(k) > 0 else -1

def perceptron(w, x):  # Inner Product/Dot Product of vectors w and x
    return sign(sum(w[i] * x[i] for i in range(DIM)))

def check():  # verify if all points are classified correctly
    global y, matrix
    for n in range(NUM):
        if y[n] != perceptron(w, matrix[n]):
            return n
    return -1

def random_check():
    global y, matrix
    misclass = []
    for n in range(NUM):
        if y[n] != perceptron(w, matrix[n]):
            misclass.append(n)
    if len(misclass) > 0:
        return misclass[ceil(np.random.rand(1) * (len(misclass) - 1))]
    else:
        return -1

def g(vector_x):  # used to graph selected hypothesis g, which should emulate f with some error
    s = 0
    for i in range(len(vector_x) - 1):
        s += (-w[i]/w[len(w) - 1]) * vector_x[i]
    return s

def run():
    global w
    setup()
    t = 0
    c = True
    while c:
        n = random_check()
        if n == -1:
            c = False
        else:
            w = next_w(w, y[n], matrix[n])
        t += 1
        print("t: {0}, w: {1}".format(t, w))
    # if PLOT:
    #     plt.plot([0, 5], [g([1, 0, 0]), g([1, 5, 0])])  # In calling g(), the 0th value is 1, corresponding to w_0
        #plt.show()                                  # and the last value is not used in calculation, so is set as 0

    return t                                                # This is just to properly display the line formed by g().
plt.clf()
j=[]
for x in range(100):
    j.append(run())
plt.ylabel("Trials")
plt.xlabel("Updates to converge")

plt.hist(j, bins=30,data=j)
plt.show()
print(j)<commit_msg>Added "run-trials" function<commit_after>"""
Implementation of a Perceptron Learning Algorithm, that can be abstracted to
various input sizes (NUM) or dimensions (DIM). Displays using pyplot.
"""
from matplotlib import pyplot as plt
from math import ceil
import numpy as np

NUM = 100  # Sample size
DIM = 11  # Number of Dimensions
w = []
y = []
matrix = []
if DIM == 3:
    PLOT = True  # whether or not to show the plot (only works in 2D)
else:
    PLOT = False

def f(x):  # Target Function
    return 0.5 * x + 1.25

def setup():
    global w, y, matrix
    w = np.random.rand(DIM)  # randomly selected weights
    matrix = np.random.rand(NUM, DIM) * 5  # randomly selected points
    y = [1] * NUM  # default values for y
    for i in range(NUM):
        matrix[i][0] = 1
        if (f(matrix[i][1])) > matrix[i][2]:  # determine if point is above line/hyperplane formed by f()
            y[i] = -1  # correct y[i] if below line
    # if PLOT:
        # for x in range(NUM):
            # if y[x] == 1:
            #     plt.plot(matrix[x][1], matrix[x][2], 'Dg')
            # else:
            #     plt.plot(matrix[x][1], matrix[x][2], 'ro')
        # plt.plot([0,5], [f(0), f(5)], label='F')  # plot Target function f(x)
        # plt.ylabel('X_2 Axis')
        # plt.xlabel('X_1 Axis')

def next_w(w, y, x):  # Perceptron update function
    r = []
    for i in range(DIM):
        r.append(w[i] + y*x[i])
    return r

def sign(k):  # returns 1 if k > 0, -1 otherwise
    return 1 if float(k) > 0 else -1

def perceptron(w, x):  # Inner Product/Dot Product of vectors w and x
    return sign(sum(w[i] * x[i] for i in range(DIM)))

def check():  # verify if all points are classified correctly
    global y, matrix
    for n in range(NUM):
        if y[n] != perceptron(w, matrix[n]):
            return n
    return -1

def random_check():
    global y, matrix
    misclass = []
    for n in range(NUM):
        if y[n] != perceptron(w, matrix[n]):
            misclass.append(n)
    if len(misclass) > 0:
        return misclass[ceil(np.random.rand(1) * (len(misclass) - 1))]
    else:
        return -1

def g(vector_x):  # used to graph selected hypothesis g, which should emulate f with some error
    s = 0
    for i in range(len(vector_x) - 1):
        s += (-w[i]/w[len(w) - 1]) * vector_x[i]
    return s

def run():
    global w
    setup()
    t = 0
    c = True
    while c:
        n = random_check()
        if n == -1:
            c = False
        else:
            w = next_w(w, y[n], matrix[n])
        t += 1
        print("t: {0}, w: {1}".format(t, w))
    # if PLOT:
    #     plt.plot([0, 5], [g([1, 0, 0]), g([1, 5, 0])])  # In calling g(), the 0th value is 1, corresponding to w_0
        #plt.show()                                  # and the last value is not used in calculation, so is set as 0

    return t                                                # This is just to properly display the line formed by g().

def run_trials(i=100):
    j=[]
    for x in range(i):
        j.append(run())
    plt.ylabel("Trials")
    plt.xlabel("Updates to converge")
    plt.hist(j, bins=40,data=j)
    plt.show()
    print(j)

run_trials()<|endoftext|>