<commit_before>from typing import Dict, Optional, Any

from django.conf import settings
from django.template import Library, loader
from django.utils.safestring import mark_safe
from django.utils.lru_cache import lru_cache

from zerver.lib.utils import force_text
from typing import List
import zerver.lib.bugdown.fenced_code

import markdown
import markdown.extensions.admonition
import markdown.extensions.codehilite
import markdown.extensions.toc
import markdown_include.include

register = Library()

def and_n_others(values, limit):
    # type: (List[str], int) -> str
    # A helper for the commonly appended "and N other(s)" string, with
    # the appropriate pluralization.
    return " and %d other%s" % (len(values) - limit,
                                "" if len(values) == limit + 1 else "s")

@register.filter(name='display_list', is_safe=True)
def display_list(values, display_limit):
    # type: (List[str], int) -> str
    """
    Given a list of values, return a string nicely formatting those values,
    summarizing when you have more than `display_limit`. Eg, for a
    `display_limit` of 3 we get the following possible cases:

    Jessica
    Jessica and Waseem
    Jessica, Waseem, and Tim
    Jessica, Waseem, Tim, and 1 other
    Jessica, Waseem, Tim, and 2 others
    """
    if len(values) == 1:
        # One value, show it.
        display_string = "%s" % (values[0],)
    elif len(values) <= display_limit:
        # Fewer than `display_limit` values, show all of them.
        display_string = ", ".join(
            "%s" % (value,) for value in values[:-1])
        display_string += " and %s" % (values[-1],)
    else:
        # More than `display_limit` values, only mention a few.
        display_string = ", ".join(
            "%s" % (value,) for value in values[:display_limit])
        display_string += and_n_others(values, display_limit)

    return display_string

md_extensions = None

@lru_cache(512 if settings.PRODUCTION else 0)
@register.filter(name='render_markdown_path', is_safe=True)
def render_markdown_path(markdown_file_path):
    # type: (str) -> str
    """Given a path to a markdown file, return the rendered html.

    Note that this assumes that any HTML in the markdown file is
    trusted; it is intended to be used for documentation, not user
    data."""
    global md_extensions
    if md_extensions is None:
        md_extensions = [
            markdown.extensions.toc.makeExtension(),
            markdown.extensions.admonition.makeExtension(),
            markdown.extensions.codehilite.makeExtension(
                linenums=False,
                guess_lang=False
            ),
            zerver.lib.bugdown.fenced_code.makeExtension(),
            markdown_include.include.makeExtension(base_path='templates/zerver/help/include/'),
        ]
    md_engine = markdown.Markdown(extensions=md_extensions)
    md_engine.reset()

    template = loader.get_template(markdown_file_path)
    markdown_string = template.render()
    html = md_engine.convert(markdown_string)
    return mark_safe(html)
<commit_msg>app_filters.py: Accept context in render_markdown_path.<commit_after>from typing import Dict, Optional, Any

from django.conf import settings
from django.template import Library, loader
from django.utils.safestring import mark_safe
from django.utils.lru_cache import lru_cache

from zerver.lib.utils import force_text
from typing import List
import zerver.lib.bugdown.fenced_code

import markdown
import markdown.extensions.admonition
import markdown.extensions.codehilite
import markdown.extensions.toc
import markdown_include.include

register = Library()

def and_n_others(values, limit):
    # type: (List[str], int) -> str
    # A helper for the commonly appended "and N other(s)" string, with
    # the appropriate pluralization.
    return " and %d other%s" % (len(values) - limit,
                                "" if len(values) == limit + 1 else "s")

@register.filter(name='display_list', is_safe=True)
def display_list(values, display_limit):
    # type: (List[str], int) -> str
    """
    Given a list of values, return a string nicely formatting those values,
    summarizing when you have more than `display_limit`. Eg, for a
    `display_limit` of 3 we get the following possible cases:

    Jessica
    Jessica and Waseem
    Jessica, Waseem, and Tim
    Jessica, Waseem, Tim, and 1 other
    Jessica, Waseem, Tim, and 2 others
    """
    if len(values) == 1:
        # One value, show it.
        display_string = "%s" % (values[0],)
    elif len(values) <= display_limit:
        # Fewer than `display_limit` values, show all of them.
        display_string = ", ".join(
            "%s" % (value,) for value in values[:-1])
        display_string += " and %s" % (values[-1],)
    else:
        # More than `display_limit` values, only mention a few.
        display_string = ", ".join(
            "%s" % (value,) for value in values[:display_limit])
        display_string += and_n_others(values, display_limit)

    return display_string

md_extensions = None

@lru_cache(512 if settings.PRODUCTION else 0)
@register.filter(name='render_markdown_path', is_safe=True)
def render_markdown_path(markdown_file_path, context=None):
    # type: (str, Optional[Dict[Any, Any]]) -> str
    """Given a path to a markdown file, return the rendered html.

    Note that this assumes that any HTML in the markdown file is
    trusted; it is intended to be used for documentation, not user
    data."""
    global md_extensions
    if md_extensions is None:
        md_extensions = [
            markdown.extensions.toc.makeExtension(),
            markdown.extensions.admonition.makeExtension(),
            markdown.extensions.codehilite.makeExtension(
                linenums=False,
                guess_lang=False
            ),
            zerver.lib.bugdown.fenced_code.makeExtension(),
            markdown_include.include.makeExtension(base_path='templates/zerver/help/include/'),
        ]
    md_engine = markdown.Markdown(extensions=md_extensions)
    md_engine.reset()

    if context is None:
        context = {}

    template = loader.get_template(markdown_file_path)
    markdown_string = template.render(context)
    html = md_engine.convert(markdown_string)
    return mark_safe(html)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

from pages.feedback import FeedbackPage
from pages.generic_feedback_picker import PickerPage
from pages.generic_feedback_thanks import ThanksPage

class GenericFeedbackFormPage(FeedbackPage):
    _page_title = 'Submit Your Feedback :: Firefox Input'

    _intro_card_locator = (By.ID, 'intro')
    _picker_link_locator = (By.CSS_SELECTOR, '#back-to-picker a')

    _back_locator = (By.ID, 'back-button')

    _happy_button_locator = (By.ID, 'happy-button')
    _sad_button_locator = (By.ID, 'sad-button')

    _moreinfo_card_locator = (By.ID, 'moreinfo')
    _description_locator = (By.ID, 'description')
    _description_character_count_locator = (By.ID, 'description-counter')
    _url_locator = (By.ID, 'id_url')
    _email_checkbox_locator = (By.ID, 'email-ok')
    _email_locator = (By.ID, 'id_email')

    _submit_locator = (By.ID, 'form-submit-btn')

    _support_page_locator = (By.LINK_TEXT, 'Firefox Support')

    def go_to_feedback_page(self, product_name, querystring=None):
        url = self.base_url + '/feedback/' + product_name + '/'
        if querystring:
            url = url + '?' + querystring
        self.selenium.get(url)
        self.is_the_current_page

    def is_product(self, product_name):
        # Make sure we're on the intro card
        self.selenium.find_element(*self._intro_card_locator)

        # Get the first ask and make sure it has the product name in
        # it
        first_ask = self.selenium.find_element(By.CSS_SELECTOR, 'div.ask:first-child').text
        return product_name in first_ask

    def go_to_picker_page(self):
        picker_pg = PickerPage(self.testsetup)
        self.selenium.find_element(*self._picker_link_locator).click()
        WebDriverWait(self.selenium, 10).until(lambda s: picker_pg.is_the_current_page)
        return picker_pg

    def type_feedback(self, feedback):
        self.selenium.find_element(*self._feedback_locator).send_keys(feedback)

    def click_support_page(self):
        self.selenium.find_element(*self._support_page_locator).click()

    def click_happy_feedback(self):
        self.selenium.find_element(*self._happy_button_locator).click()
        self.wait_for(self._description_locator)

    def click_sad_feedback(self):
        self.selenium.find_element(*self._sad_button_locator).click()
        self.wait_for(self._description_locator)

    def click_back(self):
        self.selenium.find_element(*self._back_locator).click()
        self.wait_for(self._happy_button_locator)

    def set_description_execute_script(self, text):
        """Sets the value of the description textarea using execute_script

        :arg text: The text to set

        We use ``execute_script`` here because sending keys one at a time
        takes a crazy amount of time for texts > 200 characters.

        """
        text = text.replace("'", "\\'").replace('"', '\\"')
        self.selenium.execute_script("$('#description').val('" + text + "')")

    def set_description(self, text):
        desc = self.selenium.find_element(*self._description_locator)
        desc.clear()
        desc.send_keys(text)

    def update_description(self, text):
        desc = self.selenium.find_element(*self._description_locator)
        desc.send_keys(text)

    def set_url(self, text):
        url = self.selenium.find_element(*self._url_locator)
        url.clear()
        url.send_keys(text)

    def check_email_checkbox(self, checked=True):
        self.wait_for(self._email_checkbox_locator)
        checkbox = self.selenium.find_element(*self._email_checkbox_locator)
        if checked != checkbox.is_selected():
            checkbox.click()

    def set_email(self, text):
        if not self.is_element_visible(self._email_locator):
            self.wait_for(self._email_locator)

        email = self.selenium.find_element(*self._email_locator)
        email.clear()
        email.send_keys(text)

    @property
    def is_submit_enabled(self):
        # return not 'disabled' in self.selenium.find_element(*self._submit_feedback_locator).get_attribute('class'
        return self.selenium.find_element(*self._submit_locator).is_enabled()

    def submit(self, expect_success=True):
        self.selenium.find_element(*self._submit_locator).click()
        if expect_success:
            return ThanksPage(self.testsetup)

    @property
    def support_page_link_address(self):
        return self.selenium.find_element(*self._support_page_locator).get_attribute('href')

    @property
    def is_url_valid(self):
        return not 'invalid' in self.selenium.find_element(*self._url_locator).get_attribute('class')

    @property
    def is_email_valid(self):
        return not 'invalid' in self.selenium.find_element(*self._email_locator).get_attribute('class')

    @property
    def remaining_character_count(self):
        return int(self.selenium.find_element(*self._description_character_count_locator).text)
<commit_msg>[bug 1183130] Fix test_picker.py smoketest<commit_after>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

from pages.feedback import FeedbackPage
from pages.generic_feedback_picker import PickerPage
from pages.generic_feedback_thanks import ThanksPage

class GenericFeedbackFormPage(FeedbackPage):
    _page_title = 'Submit Your Feedback :: Firefox Input'

    _intro_card_locator = (By.ID, 'intro')
    _picker_link_locator = (By.CSS_SELECTOR, '#back-to-picker a')

    _back_locator = (By.ID, 'back-button')

    _happy_button_locator = (By.ID, 'happy-button')
    _sad_button_locator = (By.ID, 'sad-button')

    _moreinfo_card_locator = (By.ID, 'moreinfo')
    _description_locator = (By.ID, 'description')
    _description_character_count_locator = (By.ID, 'description-counter')
    _url_locator = (By.ID, 'id_url')
    _email_checkbox_locator = (By.ID, 'email-ok')
    _email_locator = (By.ID, 'id_email')

    _submit_locator = (By.ID, 'form-submit-btn')

    _support_page_locator = (By.LINK_TEXT, 'Firefox Support')

    def go_to_feedback_page(self, product_name, querystring=None):
        url = self.base_url + '/feedback/' + product_name + '/'
        if querystring:
            url = url + '?' + querystring
        self.selenium.get(url)
        self.is_the_current_page

    def is_product(self, product_name):
        # Make sure we're on the intro card
        self.selenium.find_element(*self._intro_card_locator)

        # Get the first ask and make sure it has the product name in
        # it
        first_ask = self.selenium.find_element(By.CSS_SELECTOR, 'div.ask-large').text
        return product_name in first_ask

    def go_to_picker_page(self):
        picker_pg = PickerPage(self.testsetup)
        self.selenium.find_element(*self._picker_link_locator).click()
        WebDriverWait(self.selenium, 10).until(lambda s: picker_pg.is_the_current_page)
        return picker_pg

    def type_feedback(self, feedback):
        self.selenium.find_element(*self._feedback_locator).send_keys(feedback)

    def click_support_page(self):
        self.selenium.find_element(*self._support_page_locator).click()

    def click_happy_feedback(self):
        self.selenium.find_element(*self._happy_button_locator).click()
        self.wait_for(self._description_locator)

    def click_sad_feedback(self):
        self.selenium.find_element(*self._sad_button_locator).click()
        self.wait_for(self._description_locator)

    def click_back(self):
        self.selenium.find_element(*self._back_locator).click()
        self.wait_for(self._happy_button_locator)

    def set_description_execute_script(self, text):
        """Sets the value of the description textarea using execute_script

        :arg text: The text to set

        We use ``execute_script`` here because sending keys one at a time
        takes a crazy amount of time for texts > 200 characters.

        """
        text = text.replace("'", "\\'").replace('"', '\\"')
        self.selenium.execute_script("$('#description').val('" + text + "')")

    def set_description(self, text):
        desc = self.selenium.find_element(*self._description_locator)
        desc.clear()
        desc.send_keys(text)

    def update_description(self, text):
        desc = self.selenium.find_element(*self._description_locator)
        desc.send_keys(text)

    def set_url(self, text):
        url = self.selenium.find_element(*self._url_locator)
        url.clear()
        url.send_keys(text)

    def check_email_checkbox(self, checked=True):
        self.wait_for(self._email_checkbox_locator)
        checkbox = self.selenium.find_element(*self._email_checkbox_locator)
        if checked != checkbox.is_selected():
            checkbox.click()

    def set_email(self, text):
        if not self.is_element_visible(self._email_locator):
            self.wait_for(self._email_locator)

        email = self.selenium.find_element(*self._email_locator)
        email.clear()
        email.send_keys(text)

    @property
    def is_submit_enabled(self):
        # return not 'disabled' in self.selenium.find_element(*self._submit_feedback_locator).get_attribute('class'
        return self.selenium.find_element(*self._submit_locator).is_enabled()

    def submit(self, expect_success=True):
        self.selenium.find_element(*self._submit_locator).click()
        if expect_success:
            return ThanksPage(self.testsetup)

    @property
    def support_page_link_address(self):
        return self.selenium.find_element(*self._support_page_locator).get_attribute('href')

    @property
    def is_url_valid(self):
        return not 'invalid' in self.selenium.find_element(*self._url_locator).get_attribute('class')

    @property
    def is_email_valid(self):
        return not 'invalid' in self.selenium.find_element(*self._email_locator).get_attribute('class')

    @property
    def remaining_character_count(self):
        return int(self.selenium.find_element(*self._description_character_count_locator).text)
<|endoftext|>


<commit_before>from soccermetrics.rest.resources import Resource

class Personnel(Resource):
    """
    Represents a Personnel REST resource (/personnel/<resource> endpoint).

    The Personnel resources let you access biographic and demographic
    data on all personnel involved in a football match – players,
    managers, and match referees.

    Derived from :class:`Resource`.
    """
    def __init__(self, resource, base_uri, auth):
        """
        Constructor of Personnel class.

        :param resource: Name of resource.
        :type resource: string
        :param base_uri: Base URI of API.
        :type base_uri: string
        :param auth: Authentication credential.
        :type auth: tuple
        """
        super(Personnel, self).__init__(base_uri,auth)

        self.endpoint += "/personnel/%s" % resource<commit_msg>Remove stray non-ASCII character in docstring<commit_after>from soccermetrics.rest.resources import Resource

class Personnel(Resource):
    """
    Represents a Personnel REST resource (/personnel/<resource> endpoint).

    The Personnel resources let you access biographic and demographic
    data on the following personnel involved in a football match:

        * Players,
        * Managers,
        * Match referees.

    Derived from :class:`Resource`.
    """
    def __init__(self, resource, base_uri, auth):
        """
        Constructor of Personnel class.

        :param resource: Name of resource.
        :type resource: string
        :param base_uri: Base URI of API.
        :type base_uri: string
        :param auth: Authentication credential.
        :type auth: tuple
        """
        super(Personnel, self).__init__(base_uri,auth)

        self.endpoint += "/personnel/%s" % resource<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from configman import Namespace
from configman.dotdict import DotDict
import pika
from six.moves.queue import Queue, Empty

from socorro.external.crashstorage_base import CrashStorageBase
from socorro.external.rabbitmq.connection_context import (
    ConnectionContext,
    ConnectionContextPooled
)
from socorro.lib.converters import change_default
from socorro.lib.transaction import retry

class RabbitMQCrashStorage(CrashStorageBase):
    """This class is an implementation of a Socorro Crash Storage system.
    It is used as a crash queing methanism for raw crashes.  It implements
    the save_raw_crash method as a queue submission function, and the
    new_crashes generator as a queue consumption function.  Please note: as
    it only queues the crash_id and not the whole raw crash, it is not suitable
    to actually save a crash.  It is a very lossy container.  This class
    should be used in conjuction with a more persistant storage mechanism.

    The implementations CrashStorage classes can use arbitrarly high or low
    level semantics to talk to their underlying resource.

    The 'new_crashes' generator has a lower level relationship with the
    underlying connection object

    """

    required_config = Namespace()
    required_config.add_option(
        'rabbitmq_class',
        # we choose a pooled connection because we need thread safe connection
        # behaviors
        default=ConnectionContextPooled,
        doc='the class responsible for connecting to RabbitMQ',
        reference_value_from='resource.rabbitmq',
    )
    required_config.add_option(
        'routing_key',
        default='socorro.normal',
        doc='the name of the queue to recieve crashes',
        reference_value_from='resource.rabbitmq',
    )
    required_config.add_option(
        'filter_on_legacy_processing',
        default=True,
        doc='toggle for using or ignoring the throttling flag',
        reference_value_from='resource.rabbitmq',
    )

    def __init__(self, config, namespace='', quit_check_callback=None):
        super(RabbitMQCrashStorage, self).__init__(
            config,
            namespace=namespace,
            quit_check_callback=quit_check_callback
        )

        # Note: this may continue to grow if we aren't acking certain UUIDs.
        # We should find a way to time out UUIDs after a certain time.
        self.acknowledgement_token_cache = {}
        self.acknowledgment_queue = Queue()

        self.rabbitmq = config.rabbitmq_class(config)
        self._basic_properties = pika.BasicProperties(
            # Make message persistent
            delivery_mode=2,
        )

    def save_raw_crash(self, raw_crash, dumps, crash_id):
        try:
            this_crash_should_be_queued = (
                not self.config.filter_on_legacy_processing or
                raw_crash.legacy_processing == 0
            )
        except KeyError:
            self.config.logger.debug(
                'RabbitMQCrashStorage legacy_processing key absent in crash '
                '%s', crash_id
            )
            return

        if this_crash_should_be_queued:
            self.config.logger.debug('RabbitMQCrashStorage saving crash %s', crash_id)
            retry(
                self.rabbitmq,
                self.quit_check,
                self._save_raw_crash,
                crash_id=crash_id
            )
            return True
        else:
            self.config.logger.debug(
                'RabbitMQCrashStorage not saving crash %s, legacy processing '
                'flag is %s', crash_id, raw_crash.legacy_processing
            )

    def _save_raw_crash(self, connection, crash_id):
        connection.channel.basic_publish(
            exchange='',
            routing_key=self.config.routing_key,
            body=crash_id,
            properties=self._basic_properties
        )

    def _basic_get(self, conn, queue):
        return conn.channel.basic_get(queue=queue)

    def new_crashes(self):
        """This generator fetches crash_ids from RabbitMQ."""

        # We've set up RabbitMQ to require acknowledgement of processing of a
        # crash_id from this generator.  It is the responsibility of the
        # consumer of the crash_id to tell this instance of the class when has
        # completed its work on the crash_id.  That is done with the call to
        # 'ack_crash' below.  Because RabbitMQ connections are not thread safe,
        # only the thread that read the crash may acknowledge it.  'ack_crash'
        # queues the crash_id. The '_consume_acknowledgement_queue' function
        # is run to send acknowledgments back to RabbitMQ
        self._consume_acknowledgement_queue()
        queues = [
            self.rabbitmq.config.priority_queue_name,
            self.rabbitmq.config.standard_queue_name,
            self.rabbitmq.config.reprocessing_queue_name,
            self.rabbitmq.config.priority_queue_name,
        ]
        while True:
            for queue in queues:
                method_frame, header_frame, body = retry(
                    self.rabbitmq,
                    self.quit_check,
                    self._basic_get,
                    queue=queue
                )
                if method_frame and self._suppress_duplicate_jobs(body, method_frame):
                    continue
                if method_frame:
                    break
            # must consume ack queue before testing for end of iterator
            # or the last job won't get ack'd
            self._consume_acknowledgement_queue()
            if not method_frame:
                # there was nothing in the queue - leave the iterator
                return
            self.acknowledgement_token_cache[body] = method_frame
            yield body
            queues.reverse()

    def ack_crash(self, crash_id):
        self.acknowledgment_queue.put(crash_id)

    def _suppress_duplicate_jobs(self, crash_id, acknowledgement_token):
        """if this crash is in the cache, then it is already in progress
        and this is a duplicate.  Acknowledge it, then return to True
        to let the caller know to skip on to the next crash."""
        if crash_id in self.acknowledgement_token_cache:
            # reject this crash - it's already being processsed
            self.config.logger.info('duplicate job: %s is already in progress', crash_id)
            # ack this
            retry(
                self.rabbitmq,
                self.quit_check,
                self._ack_crash,
                crash_id=crash_id,
                acknowledgement_token=acknowledgement_token
            )
            return True
        return False

    def _consume_acknowledgement_queue(self):
        """The acknowledgement of the processing of each crash_id yielded
        from the 'new_crashes' method must take place on the same connection
        that the crash_id came from.  The crash_ids are queued in the
        'acknowledgment_queue'.  That queue is consumed by the QueuingThread"""
        try:
            while True:
                crash_id_to_be_acknowledged = self.acknowledgment_queue.get_nowait()

                try:
                    acknowledgement_token = self.acknowledgement_token_cache[
                        crash_id_to_be_acknowledged
                    ]
                    retry(
                        self.rabbitmq,
                        self.quit_check,
                        self._ack_crash,
                        crash_id=crash_id_to_be_acknowledged,
                        acknowledgement_token=acknowledgement_token
                    )
                    del self.acknowledgement_token_cache[crash_id_to_be_acknowledged]
                except KeyError:
                    self.config.logger.warning(
                        'RabbitMQCrashStorage tried to acknowledge crash %s'
                        ', which was not in the cache',
                        crash_id_to_be_acknowledged,
                        exc_info=True
                    )
                except Exception:
                    self.config.logger.error(
                        'RabbitMQCrashStorage unexpected failure on %s',
                        crash_id_to_be_acknowledged,
                        exc_info=True
                    )

        except Empty:
            pass  # nothing to do with an empty queue

    def _ack_crash(self, connection, crash_id, acknowledgement_token):
        connection.channel.basic_ack(delivery_tag=acknowledgement_token.delivery_tag)
        self.config.logger.debug(
            'RabbitMQCrashStorage acking %s with delivery_tag %s',
            crash_id,
            acknowledgement_token.delivery_tag
        )

class ReprocessingRabbitMQCrashStore(RabbitMQCrashStorage):
    required_config = Namespace()
    required_config.routing_key = change_default(
        RabbitMQCrashStorage,
        'routing_key',
        'socorro.reprocessing'
    )
    required_config.filter_on_legacy_processing = change_default(
        RabbitMQCrashStorage,
        'filter_on_legacy_processing',
        False
    )

class ReprocessingOneRabbitMQCrashStore(ReprocessingRabbitMQCrashStore):
    required_config = Namespace()
    required_config.rabbitmq_class = change_default(
        RabbitMQCrashStorage,
        'rabbitmq_class',
        ConnectionContext,
    )
    required_config.routing_key = change_default(
        RabbitMQCrashStorage,
        'routing_key',
        'socorro.reprocessing'
    )

    def reprocess(self, crash_ids):
        if not isinstance(crash_ids, (list, tuple)):
            crash_ids = [crash_ids]
        success = bool(crash_ids)
        for crash_id in crash_ids:
            if not self.save_raw_crash(
                DotDict({'legacy_processing': 0}),
                [],
                crash_id
            ):
                success = False
        return success

class PriorityjobRabbitMQCrashStore(RabbitMQCrashStorage):
    required_config = Namespace()
    required_config.rabbitmq_class = change_default(
        RabbitMQCrashStorage,
        'rabbitmq_class',
        ConnectionContext,
    )
    required_config.add_option(
        'routing_key',
        default='socorro.priority',
        doc='the name of the queue to receive crashes',
    )

    def process(self, crash_ids):
        if not isinstance(crash_ids, (list, tuple)):
            crash_ids = [crash_ids]
        success = bool(crash_ids)
        for crash_id in crash_ids:
            if not self.save_raw_crash(
                DotDict({'legacy_processing': 0}),
                [],
                crash_id
            ):
                success = False
        return success
<commit_msg>bug 1406703: fix rabbitmq crashstorage<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from configman import Namespace
from configman.dotdict import DotDict
import pika
from six.moves.queue import Queue, Empty

from socorro.external.crashstorage_base import CrashStorageBase
from socorro.external.rabbitmq.connection_context import (
    ConnectionContext,
    ConnectionContextPooled
)
from socorro.lib.converters import change_default
from socorro.lib.transaction import retry

class RabbitMQCrashStorage(CrashStorageBase):
    """This class is an implementation of a Socorro Crash Storage system.
    It is used as a crash queing methanism for raw crashes.  It implements
    the save_raw_crash method as a queue submission function, and the
    new_crashes generator as a queue consumption function.  Please note: as
    it only queues the crash_id and not the whole raw crash, it is not suitable
    to actually save a crash.  It is a very lossy container.  This class
    should be used in conjuction with a more persistant storage mechanism.

    The implementations CrashStorage classes can use arbitrarly high or low
    level semantics to talk to their underlying resource.

    The 'new_crashes' generator has a lower level relationship with the
    underlying connection object

    """

    required_config = Namespace()
    required_config.add_option(
        'rabbitmq_class',
        # we choose a pooled connection because we need thread safe connection
        # behaviors
        default=ConnectionContextPooled,
        doc='the class responsible for connecting to RabbitMQ',
        reference_value_from='resource.rabbitmq',
    )
    required_config.add_option(
        'routing_key',
        default='socorro.normal',
        doc='the name of the queue to recieve crashes',
        reference_value_from='resource.rabbitmq',
    )
    required_config.add_option(
        'filter_on_legacy_processing',
        default=True,
        doc='toggle for using or ignoring the throttling flag',
        reference_value_from='resource.rabbitmq',
    )

    def __init__(self, config, namespace='', quit_check_callback=None):
        super(RabbitMQCrashStorage, self).__init__(
            config,
            namespace=namespace,
            quit_check_callback=quit_check_callback
        )

        # Note: this may continue to grow if we aren't acking certain UUIDs.
        # We should find a way to time out UUIDs after a certain time.
        self.acknowledgement_token_cache = {}
        self.acknowledgment_queue = Queue()

        self.rabbitmq = config.rabbitmq_class(config)
        self._basic_properties = pika.BasicProperties(
            # Make message persistent
            delivery_mode=2,
        )

    def save_raw_crash(self, raw_crash, dumps, crash_id):
        try:
            this_crash_should_be_queued = (
                not self.config.filter_on_legacy_processing or
                raw_crash.legacy_processing == 0
            )
        except KeyError:
            self.config.logger.debug(
                'RabbitMQCrashStorage legacy_processing key absent in crash '
                '%s', crash_id
            )
            return

        if this_crash_should_be_queued:
            self.config.logger.debug('RabbitMQCrashStorage saving crash %s', crash_id)
            retry(
                self.rabbitmq,
                self.quit_check,
                self._save_raw_crash,
                crash_id=crash_id
            )
            return True
        else:
            self.config.logger.debug(
                'RabbitMQCrashStorage not saving crash %s, legacy processing '
                'flag is %s', crash_id, raw_crash.legacy_processing
            )

    def _save_raw_crash(self, connection, crash_id):
        connection.channel.basic_publish(
            exchange='',
            routing_key=self.config.routing_key,
            body=crash_id,
            properties=self._basic_properties
        )

    def _basic_get(self, conn, queue):
        return conn.channel.basic_get(queue=queue)

    def new_crashes(self):
        """This generator fetches crash_ids from RabbitMQ."""

        # We've set up RabbitMQ to require acknowledgement of processing of a
        # crash_id from this generator.  It is the responsibility of the
        # consumer of the crash_id to tell this instance of the class when has
        # completed its work on the crash_id.  That is done with the call to
        # 'ack_crash' below.  Because RabbitMQ connections are not thread safe,
        # only the thread that read the crash may acknowledge it.  'ack_crash'
        # queues the crash_id. The '_consume_acknowledgement_queue' function
        # is run to send acknowledgments back to RabbitMQ
        self._consume_acknowledgement_queue()
        queues = [
            self.rabbitmq.config.priority_queue_name,
            self.rabbitmq.config.standard_queue_name,
            self.rabbitmq.config.reprocessing_queue_name,
            self.rabbitmq.config.priority_queue_name,
        ]
        while True:
            for queue in queues:
                method_frame, header_frame, body = retry(
                    self.rabbitmq,
                    self.quit_check,
                    self._basic_get,
                    queue=queue
                )
                # The body is always a string, so convert it to a string
                if body:
                    body = body.decode('utf-8')

                if method_frame and self._suppress_duplicate_jobs(body, method_frame):
                    continue
                if method_frame:
                    break
            # must consume ack queue before testing for end of iterator
            # or the last job won't get ack'd
            self._consume_acknowledgement_queue()
            if not method_frame:
                # there was nothing in the queue - leave the iterator
                return
            self.acknowledgement_token_cache[body] = method_frame
            yield body
            queues.reverse()

    def ack_crash(self, crash_id):
        self.acknowledgment_queue.put(crash_id)

    def _suppress_duplicate_jobs(self, crash_id, acknowledgement_token):
        """if this crash is in the cache, then it is already in progress
        and this is a duplicate.  Acknowledge it, then return to True
        to let the caller know to skip on to the next crash."""
        if crash_id in self.acknowledgement_token_cache:
            # reject this crash - it's already being processsed
            self.config.logger.info('duplicate job: %s is already in progress', crash_id)
            # ack this
            retry(
                self.rabbitmq,
                self.quit_check,
                self._ack_crash,
                crash_id=crash_id,
                acknowledgement_token=acknowledgement_token
            )
            return True
        return False

    def _consume_acknowledgement_queue(self):
        """The acknowledgement of the processing of each crash_id yielded
        from the 'new_crashes' method must take place on the same connection
        that the crash_id came from.  The crash_ids are queued in the
        'acknowledgment_queue'.  That queue is consumed by the QueuingThread"""
        try:
            while True:
                crash_id_to_be_acknowledged = self.acknowledgment_queue.get_nowait()

                try:
                    acknowledgement_token = self.acknowledgement_token_cache[
                        crash_id_to_be_acknowledged
                    ]
                    retry(
                        self.rabbitmq,
                        self.quit_check,
                        self._ack_crash,
                        crash_id=crash_id_to_be_acknowledged,
                        acknowledgement_token=acknowledgement_token
                    )
                    del self.acknowledgement_token_cache[crash_id_to_be_acknowledged]
                except KeyError:
                    self.config.logger.warning(
                        'RabbitMQCrashStorage tried to acknowledge crash %s'
                        ', which was not in the cache',
                        crash_id_to_be_acknowledged,
                        exc_info=True
                    )
                except Exception:
                    self.config.logger.error(
                        'RabbitMQCrashStorage unexpected failure on %s',
                        crash_id_to_be_acknowledged,
                        exc_info=True
                    )

        except Empty:
            pass  # nothing to do with an empty queue

    def _ack_crash(self, connection, crash_id, acknowledgement_token):
        connection.channel.basic_ack(delivery_tag=acknowledgement_token.delivery_tag)
        self.config.logger.debug(
            'RabbitMQCrashStorage acking %s with delivery_tag %s',
            crash_id,
            acknowledgement_token.delivery_tag
        )

class ReprocessingRabbitMQCrashStore(RabbitMQCrashStorage):
    required_config = Namespace()
    required_config.routing_key = change_default(
        RabbitMQCrashStorage,
        'routing_key',
        'socorro.reprocessing'
    )
    required_config.filter_on_legacy_processing = change_default(
        RabbitMQCrashStorage,
        'filter_on_legacy_processing',
        False
    )

class ReprocessingOneRabbitMQCrashStore(ReprocessingRabbitMQCrashStore):
    required_config = Namespace()
    required_config.rabbitmq_class = change_default(
        RabbitMQCrashStorage,
        'rabbitmq_class',
        ConnectionContext,
    )
    required_config.routing_key = change_default(
        RabbitMQCrashStorage,
        'routing_key',
        'socorro.reprocessing'
    )

    def reprocess(self, crash_ids):
        if not isinstance(crash_ids, (list, tuple)):
            crash_ids = [crash_ids]
        success = bool(crash_ids)
        for crash_id in crash_ids:
            if not self.save_raw_crash(
                DotDict({'legacy_processing': 0}),
                [],
                crash_id
            ):
                success = False
        return success

class PriorityjobRabbitMQCrashStore(RabbitMQCrashStorage):
    required_config = Namespace()
    required_config.rabbitmq_class = change_default(
        RabbitMQCrashStorage,
        'rabbitmq_class',
        ConnectionContext,
    )
    required_config.add_option(
        'routing_key',
        default='socorro.priority',
        doc='the name of the queue to receive crashes',
    )

    def process(self, crash_ids):
        if not isinstance(crash_ids, (list, tuple)):
            crash_ids = [crash_ids]
        success = bool(crash_ids)
        for crash_id in crash_ids:
            if not self.save_raw_crash(
                DotDict({'legacy_processing': 0}),
                [],
                crash_id
            ):
                success = False
        return success
<|endoftext|>


<commit_before>#!/usr/bin/python3

# This is the updater for the Wren blogging platform.
# Before running this script please familiarise your-
# self with the instructions in the readme.md file.

from datetime import datetime
from math import floor
from os import getcwd, makedirs, path, walk
from re import search, sub
from shutil import copy
from time import sleep

# These functions allow for timeout when the Wren
# updater is run via the "Run In Terminal" command

def sucess(message):
    print(message + "\n")
    sleep(3)
    exit(0)

def gerror(message):
    print("ERROR: {0}".format(message))
    sleep(3)
    exit(1)

# At various points it's needed to have a function
# that replaces all instances of a word in a file.
# The easy solution would be to os.call 'sed' but
# I wanted to avoid relying on outside programs.

def sed(find, replace, target):
    with open(target, "r", encoding='utf-8') as file:
        lines = file.readlines()
    with open(target, "w", encoding='utf-8') as file:
        for line in lines:
            file.write(sub(find, replace, line))

def supsed(find, replace):
    for root, dirs, files in walk("../", topdown=False):
        for name in files:
            target = path.join(root, name)
            if (".git" in target) or ("images" in target):
                pass
            else:
                sed(find, replace, target)

# Checks running from the .wren directory so that the
# relative locations of files can be known.

if ".wren" not in getcwd():
    gerror("please run from the .Wren directory")

# If the Wren has not been run before the following code
# will run. This sets up all the files with any variables
# that are user modified. WARNING: that this code does not
# currently check if the URL is correctly formatted, nor
# does it check if it actually exists.

if not path.isfile("catdat.csv"):
    print("Setting up Wren...")

    blogger = input("What is your full name? ")
    supsed("BLOGGER", blogger)

    blogurl = input("What is your website url? ")
    supsed("BLOGURL", blogurl)

    # Wren can integrate Disqus as a comments system if
    # the user so wants. In future versions I'm going to
    # have a look around for alternative systems.
    q = 0
    while q == 0:
        ans = input("Do you wish to use Disqus for comments? (y/n) ")
        if ans == "y":
            print("If you don't have a Disqus account go to")
            print("https://disqus.com/profile/signup/ now!")
            shortname = input("What is your Disqus shortname? ")
            sed("DSHORTNAME", shortname, "../php/comments.php")
            q = 1
            pass
        elif ans == "n":
            with open("../php/comments.php", "w+") as file:
                file.write('
\n')
            q = 1
            pass

    with open("catdat.csv", "w+") as file:
        file.write("Category,Count")

    sucess("Setup complete!")

# Extracts the blog body from the input file. It's fixed
# so that the level of indentation doesn't need changing.

with open("blog-new.html", 'r') as file:
    record, HTMLcontent = False, []
    for line in file:
        if "
" in line:
            record = True
        elif "
" in line:
            record = False
        elif record is True:
            HTMLcontent.append(line)

# Strips out the HTML tags and white space in order
# to calculate the approximate reading time.

TXTcontent = []
for line in HTMLcontent:
    raw = ''
    record = True
    for i in line:
        if i == "<":
            record = False
        elif i == ">":
            record = True
        elif record is True:
            raw += i
    TXTcontent.append(raw.strip())
text = " ".join(TXTcontent)

# This calculates the approximate reading time for the blog.
# The calculation is based upon an average screen-reading
# rate of ~200 words per minute.

if len(text)/200 < 1:
    secs = int(float(len(text))*0.3)
    time = "~{0} seconds".format(secs)
else:
    mins = int(floor(len(text)/float(200)))
    secs = int(((len(text)/float(200))-mins)*60)
    time = "~{0} minutes {1} seconds".format(mins, secs)

readtime = time + ' reading'

print("Please enter the following data as prompted")
title = input("\nTitle:\n")
catagories = input("\nCatagories:\n").split(" ")
summary = input("\nSummary:\n")

# The time of the blogs creation is needed in a variety of
# formats, not all provide by the datetime module.

# YYYY-MM-DD
datesmll = str(datetime.date(datetime.now()))
# YYYY/MM/DD
filedate = sub(r"-", "/", datesmll)
# Day DD Mon YYYY HH:MM:SS
datelong = str(datetime.ctime(datetime.now()))
# Day, DD Mon YYYY HH:MM:SS TMZ
datelong = ", ".join(datelong.split(" ", 1)) + " GMT"

# Data used in URLs has to be devoid of any special characters
# that could cause problems. Because of that, characters not
# in the range a-z or 0-1 are removed from certain values.

def clean(string):
    tmp = list(string.lower().strip())
    tmp = [i for i in tmp if search("[^a-z0-9]", i) is None]
    return "".join(tmp)

urltitle = clean(title)
url = "http://BLOGURL/blog/" + filedate + "/" + urltitle

tmp = []
for item in catagories:
    item = clean(item)
    if len(item) != 0 and item not in tmp:
        tmp.append(item)
catagories = tmp

print("\nGIVEN INFORMATION")
print("Title:", title)
print("URL:", url)
print("Time:", datelong)
print("Reading time:", time)
print("Catagories:", catagories)
print("Summary:", summary, "\n")

q = 0
while q == 0:
    ans = input("Are you happy with the above? (y/n) ")
    if ans == "y":
        q = 1
        pass
    elif ans == "n":
        gerror("meta data rejected")
print("Meta data accepted\n")

# Copies the blog data to the template file and then moves
# it into place. The formatting for blog-template.html is
# set for the destination rather than its current location

dest = "../blog/{0}/{1}.html".format(filedate, urltitle)
if not path.exists(path.dirname(dest)):
    makedirs(path.dirname(dest))
copy("blog-template.html", dest)
sed("BLOGDATE", datesmll, dest)
sed("BLOGTITLE", title, dest)
sed("READINGTIME", readtime, dest)
sed("BLOGCONTENT", "".join(HTMLcontent), dest)

# When a new catagory is created it needs all the files
# associated with it creating too, along with the existing
# files that need to refrence it updating.

with open("catdat.csv", "r") as file:
    catdat = [line.strip() for line in file]
catdat.pop(0)
catnames = [line.split(",")[0] for line in catdat]
catcount = [int(line.split(",")[1]) for line in catdat]

newcat = []
for item in catagories:
    if item not in catnames:
        newcat.append(item)

for cat in newcat:
    copy("feed-template.xml", "../blog/feed/" + cat + ".xml")
    copy("cat-template.html", "../blog/cat/" + cat + ".html")
    sed("TEMPLATE", cat, "../blog/feed/" + cat + ".xml")
    sed("TEMPLATE", cat, "../blog/cat/" + cat + ".html")
    catnames.append(cat)
    catcount.append(0)
    print("Created", cat, "catagory")

for cat in catagories:
    pos = catnames.index(cat)
    newcount = catcount[pos] + 1
    catcount.pop(pos)
    catcount.insert(pos, newcount)

combined = sorted(list(zip(catnames, catcount)))
catnames = [name for (name, count) in combined]
catcount = [count for (name, count) in combined]

with open("catdat.csv", "w") as file:
    file.write("Category,Count\n")
    for x in range(0, len(catnames)):
        file.write("{0},{1}\n".format(catnames[x], catcount[x]))

# This inserts a single line of HTML which links to the new
# blog into each of the catagory lists which it falls into.

newline = '
{1} : {2}
'
newline = newline.format(url, datesmll, title)

for cat in (catagories + ["all"]):
    with open("../blog/cat/" + cat + ".html", "r") as file:
        lines = [line for line in file]
    for line in lines:
        if "" in line:
            n = lines.index(line)
            lines.insert(n + 1, ' ' * 12 + newline + "\n")
    catfile = open("../blog/cat/" + cat + ".html", "w+")
    for line in lines:
        catfile.write(line)
    catfile.close()
    print("Added blog to", cat, "catagory list")

# Inserts a the nessisary lines into the XML files so that
# the new blog shows up in the RSS feeds it alls into. Note
# that this process doesn't currently remove any blogs from
# the RSS file. This is recommended as it prevents the file
# from becoming too big. I'll do some research and see what
# others consider to be an acceptable cutoff point.

newlines = [
    "<item>",
    "    <title>" + title + "</title>",
    "    <link>" + url + "</link>",
    "    <guid>" + url + "</guid>",
    "    <pubDate>" + datelong + "</pubDate>",
    "    <description>" + summary + "</description>",
    "</item>\n"
]

for cat in (catagories + ["all"]):
    with open("../blog/feed/" + cat + ".xml", "r") as file:
        lines = [line for line in file]
    for line in lines:
        if "" in line:
            n = lines.index(line)
            for x in range(0, 7):
                lines.insert(n + x + 1, " " * 8 + newlines[x] + "\n")
    catfeed = open("../blog/feed/" + cat + ".xml", "w+")
    for line in lines:
        catfeed.write(line)
    catfeed.close()
    print("Added blog to", cat, "RSS feed")

# The main blogs page lists the 5 most recent blogs posted so
# adding this new one means also removing the least recent.
# IMPORTANT: this code currently just removes the least recent
# post without checking if 5 blogs have actually been posted
# yet. I'll try to correct this in future versions.

newlines = [
    "
",
    '
{1} : {2}
'.format(url, title, datesmll),
    "
" + summary + "
",
    "
\n",
    '
\n'
]

with open("../blog/index.html", 'r') as file:
    lines, recentcount, record = [], 0, True
    for line in file:
        # Whether to write to file
        if record is True:
            lines.append(line)
        elif record is False:
            pass
        # Conditions
        if '
' in line:
            recentcount += 1
        if '' in line:
            record = True
            recentcount = 0
        if recentcount > 4:
            record = False

for line in lines:
    if "" in line:
        n = lines.index(line)
        for x in range(0, 5):
            lines.insert(n + x + 1, " " * 8 + newlines[x] + "\n")
blogfile = open("../blog/index.html", "w+")
for line in lines:
    blogfile.write(line)
blogfile.close()
print("Updated the main blogs page")

# The following code will update the catagory word cloud. The
# design is inspired by that used by TagCrowd.com but isn't
# disimilar to the ones commonly used on Wordpress. Note that
# this code does not yet emulate the colouring aspect of the
# word clouds as this requires a more sophisticated algorithm.
# The unit used for font size is 'em'

smallsize = 1.0
largesize = 5.0
sizerange = largesize - smallsize
countrange = max(catcount)-min(catcount)

try:
    ratio = sizerange/countrange
except:
    ratio = 1

catsizes = [count * ratio for count in catcount]

string = "
"
for cat in catnames:
    style = "font-size: {0}em;".format(catsizes[catnames.index(cat)])
    string += ''.format(cat, style)
    string += cat + " "
string += "
\n"

with open("../blog/cat/index.html", "r") as file:
    lines = [line for line in file]
for line in lines:
    if "" in line:
        index = lines.index(line) + 1
        lines.pop(index)
        lines.insert(index, string)
cloudfile = open("../blog/cat/index.html", "w+")
for line in lines:
    cloudfile.write(line)
cloudfile.close()
print("Updated the catagory wordcloud")

sucess("Done!")
<commit_msg>Refined recentblog code<commit_after>#!/usr/bin/python3

# This is the updater for the Wren blogging platform.
# Before running this script please familiarise your-
# self with the instructions in the readme.md file.

from datetime import datetime
from math import floor
from os import getcwd, makedirs, path, walk
from re import search, sub
from shutil import copy
from time import sleep

# These functions allow for timeout when the Wren
# updater is run via the "Run In Terminal" command

def sucess(message):
    print(message + "\n")
    sleep(3)
    exit(0)

def gerror(message):
    print("ERROR: {0}".format(message))
    sleep(3)
    exit(1)

# At various points it's needed to have a function
# that replaces all instances of a word in a file.
# The easy solution would be to os.call 'sed' but
# I wanted to avoid relying on outside programs.

def sed(find, replace, target):
    with open(target, "r", encoding='utf-8') as file:
        lines = file.readlines()
    with open(target, "w", encoding='utf-8') as file:
        for line in lines:
            file.write(sub(find, replace, line))

def supsed(find, replace):
    for root, dirs, files in walk("../", topdown=False):
        for name in files:
            target = path.join(root, name)
            if (".git" in target) or ("images" in target):
                pass
            else:
                sed(find, replace, target)

# Checks running from the .wren directory so that the
# relative locations of files can be known.

if ".wren" not in getcwd():
    gerror("please run from the .Wren directory")

# If the Wren has not been run before the following code
# will run. This sets up all the files with any variables
# that are user modified. WARNING: that this code does not
# currently check if the URL is correctly formatted, nor
# does it check if it actually exists.

if not path.isfile("catdat.csv"):
    print("Setting up Wren...")

    blogger = input("What is your full name? ")
    supsed("BLOGGER", blogger)

    blogurl = input("What is your website url? ")
    supsed("BLOGURL", blogurl)

    # Wren can integrate Disqus as a comments system if
    # the user so wants. In future versions I'm going to
    # have a look around for alternative systems.
    q = 0
    while q == 0:
        ans = input("Do you wish to use Disqus for comments? (y/n) ")
        if ans == "y":
            print("If you don't have a Disqus account go to")
            print("https://disqus.com/profile/signup/ now!")
            shortname = input("What is your Disqus shortname? ")
            sed("DSHORTNAME", shortname, "../php/comments.php")
            q = 1
            pass
        elif ans == "n":
            with open("../php/comments.php", "w+") as file:
                file.write('
\n')
            q = 1
            pass

    with open("catdat.csv", "w+") as file:
        file.write("Category,Count")

    sucess("Setup complete!")

# Extracts the blog body from the input file. It's fixed
# so that the level of indentation doesn't need changing.

with open("blog-new.html", 'r') as file:
    record, HTMLcontent = False, []
    for line in file:
        if "
" in line:
            record = True
        elif "
" in line:
            record = False
        elif record is True:
            HTMLcontent.append(line)

# Strips out the HTML tags and white space in order
# to calculate the approximate reading time.

TXTcontent = []
for line in HTMLcontent:
    raw = ''
    record = True
    for i in line:
        if i == "<":
            record = False
        elif i == ">":
            record = True
        elif record is True:
            raw += i
    TXTcontent.append(raw.strip())
text = " ".join(TXTcontent)

# This calculates the approximate reading time for the blog.
# The calculation is based upon an average screen-reading
# rate of ~200 words per minute.

if len(text)/200 < 1:
    secs = int(float(len(text))*0.3)
    time = "~{0} seconds".format(secs)
else:
    mins = int(floor(len(text)/float(200)))
    secs = int(((len(text)/float(200))-mins)*60)
    time = "~{0} minutes {1} seconds".format(mins, secs)

readtime = time + ' reading'

print("Please enter the following data as prompted")
title = input("\nTitle:\n")
catagories = input("\nCatagories:\n").split(" ")
summary = input("\nSummary:\n")

# The time of the blogs creation is needed in a variety of
# formats, not all provide by the datetime module.

# YYYY-MM-DD
datesmll = str(datetime.date(datetime.now()))
# YYYY/MM/DD
filedate = sub(r"-", "/", datesmll)
# Day DD Mon YYYY HH:MM:SS
datelong = str(datetime.ctime(datetime.now()))
# Day, DD Mon YYYY HH:MM:SS TMZ
datelong = ", ".join(datelong.split(" ", 1)) + " GMT"

# Data used in URLs has to be devoid of any special characters
# that could cause problems. Because of that, characters not
# in the range a-z or 0-1 are removed from certain values.

def clean(string):
    tmp = list(string.lower().strip())
    tmp = [i for i in tmp if search("[^a-z0-9]", i) is None]
    return "".join(tmp)

urltitle = clean(title)
url = "http://BLOGURL/blog/" + filedate + "/" + urltitle

tmp = []
for item in catagories:
    item = clean(item)
    if len(item) != 0 and item not in tmp:
        tmp.append(item)
catagories = tmp

print("\nGIVEN INFORMATION")
print("Title:", title)
print("URL:", url)
print("Time:", datelong)
print("Reading time:", time)
print("Catagories:", catagories)
print("Summary:", summary, "\n")

q = 0
while q == 0:
    ans = input("Are you happy with the above? (y/n) ")
    if ans == "y":
        q = 1
        pass
    elif ans == "n":
        gerror("meta data rejected")
print("Meta data accepted\n")

# Copies the blog data to the template file and then moves
# it into place. The formatting for blog-template.html is
# set for the destination rather than its current location

dest = "../blog/{0}/{1}.html".format(filedate, urltitle)
if not path.exists(path.dirname(dest)):
    makedirs(path.dirname(dest))
copy("blog-template.html", dest)
sed("BLOGDATE", datesmll, dest)
sed("BLOGTITLE", title, dest)
sed("READINGTIME", readtime, dest)
sed("BLOGCONTENT", "".join(HTMLcontent), dest)

# When a new catagory is created it needs all the files
# associated with it creating too, along with the existing
# files that need to refrence it updating.

with open("catdat.csv", "r") as file:
    catdat = [line.strip() for line in file]
catdat.pop(0)
catnames = [line.split(",")[0] for line in catdat]
catcount = [int(line.split(",")[1]) for line in catdat]

newcat = []
for item in catagories:
    if item not in catnames:
        newcat.append(item)

for cat in newcat:
    copy("feed-template.xml", "../blog/feed/" + cat + ".xml")
    copy("cat-template.html", "../blog/cat/" + cat + ".html")
    sed("TEMPLATE", cat, "../blog/feed/" + cat + ".xml")
    sed("TEMPLATE", cat, "../blog/cat/" + cat + ".html")
    catnames.append(cat)
    catcount.append(0)
    print("Created", cat, "catagory")

for cat in catagories:
    pos = catnames.index(cat)
    newcount = catcount[pos] + 1
    catcount.pop(pos)
    catcount.insert(pos, newcount)

combined = sorted(list(zip(catnames, catcount)))
catnames = [name for (name, count) in combined]
catcount = [count for (name, count) in combined]

with open("catdat.csv", "w") as file:
    file.write("Category,Count\n")
    for x in range(0, len(catnames)):
        file.write("{0},{1}\n".format(catnames[x], catcount[x]))

# This inserts a single line of HTML which links to the new
# blog into each of the catagory lists which it falls into.

newline = '
{1} : {2}
'
newline = newline.format(url, datesmll, title)

for cat in (catagories + ["all"]):
    with open("../blog/cat/" + cat + ".html", "r") as file:
        lines = [line for line in file]
    for line in lines:
        if "" in line:
            n = lines.index(line)
            lines.insert(n + 1, ' ' * 12 + newline + "\n")
    catfile = open("../blog/cat/" + cat + ".html", "w+")
    for line in lines:
        catfile.write(line)
    catfile.close()
    print("Added blog to", cat, "catagory list")

# Inserts a the nessisary lines into the XML files so that
# the new blog shows up in the RSS feeds it alls into. Note
# that this process doesn't currently remove any blogs from
# the RSS file. This is recommended as it prevents the file
# from becoming too big. I'll do some research and see what
# others consider to be an acceptable cutoff point.

newlines = [
    "<item>",
    "    <title>" + title + "</title>",
    "    <link>" + url + "</link>",
    "    <guid>" + url + "</guid>",
    "    <pubDate>" + datelong + "</pubDate>",
    "    <description>" + summary + "</description>",
    "</item>\n"
]

for cat in (catagories + ["all"]):
    with open("../blog/feed/" + cat + ".xml", "r") as file:
        lines = [line for line in file]
    for line in lines:
        if "" in line:
            n = lines.index(line)
            for x in range(0, 7):
                lines.insert(n + x + 1, " " * 8 + newlines[x] + "\n")
    catfeed = open("../blog/feed/" + cat + ".xml", "w+")
    for line in lines:
        catfeed.write(line)
    catfeed.close()
    print("Added blog to", cat, "RSS feed")

# The main blogs page lists the 5 most recent blogs posted so
# adding this new one means also removing the least recent.
# IMPORTANT: this code currently just removes the least recent
# post without checking if 5 blogs have actually been posted
# yet. I'll try to correct this in future versions.

newlines = [
    "
",
    '
{1} : {2}
'.format(url, title, datesmll),
    "
" + summary + "
",
    "
\n",
    '
\n'
]

with open("../blog/index.html", 'r') as file:
    lines, recent, record = [], 0, True
    for line in file:
        # Conditions
        if '
' in line:
            recent += 1
        if '' in line:
            record, recent = True, 0
        if recent > 5:
            record = False
        # Whether to write to file
        if record is True:
            lines.append(line)

for line in lines:
    if "" in line:
        n = lines.index(line)
        for x in range(0, 5):
            lines.insert(n + x + 1, " " * 8 + newlines[x] + "\n")
blogfile = open("../blog/index.html", "w+")
for line in lines:
    blogfile.write(line)
blogfile.close()
print("Updated the main blogs page")

# The following code will update the catagory word cloud. The
# design is inspired by that used by TagCrowd.com but isn't
# disimilar to the ones commonly used on Wordpress. Note that
# this code does not yet emulate the colouring aspect of the
# word clouds as this requires a more sophisticated algorithm.
# The unit used for font size is 'em'

smallsize = 1.0
largesize = 5.0
sizerange = largesize - smallsize
countrange = max(catcount)-min(catcount)

try:
    ratio = sizerange/countrange
except:
    ratio = 1

catsizes = [count * ratio for count in catcount]

string = "
"
for cat in catnames:
    style = "font-size: {0}em;".format(catsizes[catnames.index(cat)])
    string += ''.format(cat, style)
    string += cat + " "
string += "
\n"

with open("../blog/cat/index.html", "r") as file:
    lines = [line for line in file]
for line in lines:
    if "" in line:
        index = lines.index(line) + 1
        lines.pop(index)
        lines.insert(index, string)
cloudfile = open("../blog/cat/index.html", "w+")
for line in lines:
    cloudfile.write(line)
cloudfile.close()
print("Updated the catagory wordcloud")

sucess("Done!")
<|endoftext|>


<commit_before># handlers for the user-facing website

import hashlib
import json
import os
import urllib.parse
from base64 import b64encode

import github_collections
import json_api
from article_helpers import *
from base_handler import *
from user_account_helpers import *
from websockets import convert

class MainHandler(BaseHandler):
    """ The front page of Brainspell. """

    route = ""

    def get(self):
        try:  # handle failures in bulk_add
            submitted_bulk_add = int(self.get_argument("success", 0))
        except BaseException:
            submitted_bulk_add = 0
        try:
            failure_in_submitting_bulk_add = int(
                self.get_argument("failure", 0))
        except BaseException:
            failure_in_submitting_bulk_add = 0
        try:  # handle registration
            registered_right_now = int(self.get_argument("registered", 0))
        except BaseException:
            registered_right_now = 0

        custom_params = {
            "number_of_queries": get_number_of_articles(),
            "success": submitted_bulk_add,
            "failure": failure_in_submitting_bulk_add,
            # boolean that indicates if someone has just registered
            "registered": registered_right_now
        }

        self.render_with_user_info("static/html/index.html", custom_params)

class SearchHandler(BaseHandler):
    """ Search articles within Brainspell's database. """

    route = "search"

    def get(self):
        q = self.get_query_argument("q", "")
        start = self.get_query_argument("start", 0)
        req = self.get_query_argument("req", "t")
        custom_params = {
            "query": q,
            "start": start,
            "req": req,  # TODO: parameters like "req" and "title" need to be renamed to reflect what their values are)
        }
        self.render_with_user_info("static/html/search.html", custom_params)

class ViewArticleHandler(BaseHandler):
    """
    Display the contents of an article to the user, along with UI features for curation,
    visualization, statistics, etc.
    """

    route = "view-article"

    def get(self):
        article_id = -1
        try:
            article_id = self.get_query_argument("id")
        except BaseException:
            self.redirect("/")  # id wasn't passed; redirect to home page
        article_dict = {
            "article_id": article_id
        }

        redirect_uri = self.route + "?" + urllib.parse.urlencode({
            "id": self.get_query_argument("id")
        })

        self.render_with_user_info(
            "static/html/view-article.html",
            article_dict,
            logout_redir=redirect_uri)

class ContributionHandler(BaseHandler):
    """ Show the user how they can contribute to the Brainspell platform. """

    route = "contribute"

    def get(self):
        self.render_with_user_info('static/html/contribute.html')

class BulkAddHandler(BaseHandler):
    """
    Take a file in JSON format and add the articles to our database.
    (called from the /contribute page)
    This handler is deprecated; please use the JSON API instead.
    """

    route = "bulk-add"

    def post(self):
        file_body = self.request.files['articlesFile'][0]['body'].decode(
            'utf-8')
        contents = json.loads(file_body)
        if isinstance(contents, list):
            clean_articles = clean_bulk_add(contents)
            add_bulk(clean_articles)
            self.redirect("/?success=1")
        else:
            # data is malformed
            self.redirect("/?failure=1")

class CollectionsHandler(BaseHandler):
    """ Display the user's collections. """

    route = "collections"

    def get(self):
        if self.get_current_github_access_token():
            self.render_with_user_info("static/html/account.html")
        # if you're not authorized, redirect to OAuth
        else:
            self.redirect("/oauth?redirect_uri=/collections")

class SwaggerHandler(BaseHandler):
    """ Automatically generated swagger.json file, to document our API. """

    route = "swagger.json"

    swagger_info = {
        "swagger": "2.0",
        "info": {
            "title": "Brainspell",
            "description": "An open, human-curated repository of neuroimaging literature, with various statistical features.",
            "version": "1.0.0"},
        "host": "brainspell.herokuapp.com",
        "schemes": ["https"],
        "basePath": "/json/",
        "paths": {}}

    def parameter_object_to_swagger(name, p):
        """ Convert Brainspell parameters objects to Swagger. """

        def convert_type(t):
            """ Takes a type, and converts it to a human-readable string for Swagger. """
            if t == float:
                return "number"
            elif t == int:
                return "integer"
            # default value of string
            return "string"

        return {
            "name": name,
            "in": "query",
            "required": "default" in p,
            "type": convert_type(p["type"]),
            "default": "" if "default" not in p else p["default"]
        }

    # add the paths to the swagger.json file
    for name, func in [(convert(f.replace("EndpointHandler", "")), eval("json_api." + f))
                       for f in dir(json_api) if "EndpointHandler" in f] \
        + [(convert(f.replace("EndpointHandler", "")), eval("github_collections." + f))
            for f in dir(github_collections) if "EndpointHandler" in f]:

        parameters_object = []
        for p in func.parameters:
            parameters_object.append(
                parameter_object_to_swagger(
                    p, func.parameters[p]))

        operation = {
            "parameters": parameters_object,
            "produces": ["application/json"],
            "responses": {
                "default": {
                    "description": "The basic structure of a Brainspell response. Endpoint-specific JSON attributes not shown.",
                    "schema": {
                        "type": "object",
                        "required": ["success"],
                        "properties": {
                            "success": {
                                "type": "integer"}}}}}}

        swagger_info["paths"]["/" + name] = {
            "get": operation,
            "post": operation
        }

    def get(self):
        self.finish_async(self.swagger_info)
<commit_msg>added descriptions to swagger<commit_after># handlers for the user-facing website

import hashlib
import json
import os
import urllib.parse
from base64 import b64encode

import github_collections
import json_api
from article_helpers import *
from base_handler import *
from user_account_helpers import *
from websockets import convert

class MainHandler(BaseHandler):
    """ The front page of Brainspell. """

    route = ""

    def get(self):
        try:  # handle failures in bulk_add
            submitted_bulk_add = int(self.get_argument("success", 0))
        except BaseException:
            submitted_bulk_add = 0
        try:
            failure_in_submitting_bulk_add = int(
                self.get_argument("failure", 0))
        except BaseException:
            failure_in_submitting_bulk_add = 0
        try:  # handle registration
            registered_right_now = int(self.get_argument("registered", 0))
        except BaseException:
            registered_right_now = 0

        custom_params = {
            "number_of_queries": get_number_of_articles(),
            "success": submitted_bulk_add,
            "failure": failure_in_submitting_bulk_add,
            # boolean that indicates if someone has just registered
            "registered": registered_right_now
        }

        self.render_with_user_info("static/html/index.html", custom_params)

class SearchHandler(BaseHandler):
    """ Search articles within Brainspell's database. """

    route = "search"

    def get(self):
        q = self.get_query_argument("q", "")
        start = self.get_query_argument("start", 0)
        req = self.get_query_argument("req", "t")
        custom_params = {
            "query": q,
            "start": start,
            "req": req,  # TODO: parameters like "req" and "title" need to be renamed to reflect what their values are)
        }
        self.render_with_user_info("static/html/search.html", custom_params)

class ViewArticleHandler(BaseHandler):
    """
    Display the contents of an article to the user, along with UI features for curation,
    visualization, statistics, etc.
    """

    route = "view-article"

    def get(self):
        article_id = -1
        try:
            article_id = self.get_query_argument("id")
        except BaseException:
            self.redirect("/")  # id wasn't passed; redirect to home page
        article_dict = {
            "article_id": article_id
        }

        redirect_uri = self.route + "?" + urllib.parse.urlencode({
            "id": self.get_query_argument("id")
        })

        self.render_with_user_info(
            "static/html/view-article.html",
            article_dict,
            logout_redir=redirect_uri)

class ContributionHandler(BaseHandler):
    """ Show the user how they can contribute to the Brainspell platform. """

    route = "contribute"

    def get(self):
        self.render_with_user_info('static/html/contribute.html')

class BulkAddHandler(BaseHandler):
    """
    Take a file in JSON format and add the articles to our database.
    (called from the /contribute page)
    This handler is deprecated; please use the JSON API instead.
    """

    route = "bulk-add"

    def post(self):
        file_body = self.request.files['articlesFile'][0]['body'].decode(
            'utf-8')
        contents = json.loads(file_body)
        if isinstance(contents, list):
            clean_articles = clean_bulk_add(contents)
            add_bulk(clean_articles)
            self.redirect("/?success=1")
        else:
            # data is malformed
            self.redirect("/?failure=1")

class CollectionsHandler(BaseHandler):
    """ Display the user's collections. """

    route = "collections"

    def get(self):
        if self.get_current_github_access_token():
            self.render_with_user_info("static/html/account.html")
        # if you're not authorized, redirect to OAuth
        else:
            self.redirect("/oauth?redirect_uri=/collections")

class SwaggerHandler(BaseHandler):
    """ Automatically generated swagger.json file, to document our API. """

    route = "swagger.json"

    swagger_info = {
        "swagger": "2.0",
        "info": {
            "title": "Brainspell",
            "description": "An open, human-curated repository of neuroimaging literature, with various statistical features.",
            "version": "1.0.0"},
        "host": "brainspell.herokuapp.com",
        "schemes": ["https"],
        "basePath": "/json/",
        "paths": {}}

    def parameter_object_to_swagger(name, p):
        """ Convert Brainspell parameters objects to Swagger. """

        def convert_type(t):
            """ Takes a type, and converts it to a human-readable string for Swagger. """
            if t == float:
                return "number"
            elif t == int:
                return "integer"
            # default value of string
            return "string"

        parameter_obj = {
            "name": name,
            "in": "query",
            "required": "default" not in p,
            "type": convert_type(p["type"]),
            "default": "" if "default" not in p else p["default"]
        }

        if "default" in p:
            parameter_obj["default"] = p["default"]

        if "description" in p:
            parameter_obj["description"] = p["description"]

        return parameter_obj

    # add the paths to the swagger.json file
    for name, func in [(convert(f.replace("EndpointHandler", "")), eval("json_api." + f))
                       for f in dir(json_api) if "EndpointHandler" in f] \
        + [(convert(f.replace("EndpointHandler", "")), eval("github_collections." + f))
            for f in dir(github_collections) if "EndpointHandler" in f]:

        # add parameters from each endpoint
        parameters_object = []
        for p in func.parameters:
            parameters_object.append(
                parameter_object_to_swagger(
                    p, func.parameters[p]))

        # API key isn't explicitly listed for PUSH API endpoints
        if func.endpoint_type == Endpoint.PUSH_API:
            parameters_object.append(
                parameter_object_to_swagger(
                    "key", {
                        "type": str,
                        "description": "The user's Brainspell API key."
                    }))

        operation = {
            "parameters": parameters_object,
            "produces": ["application/json"],
            "responses": {
                "default": {
                    "description": "The basic structure of a Brainspell response. Endpoint-specific JSON attributes not shown.",
                    "schema": {
                        "type": "object",
                        "required": ["success"],
                        "properties": {
                            "success": {
                                "type": "integer"}}}}}}

        swagger_info["paths"]["/" + name] = {
            "get": operation,
            "post": operation
        }

    def get(self):
        self.finish_async(self.swagger_info)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon Dec  7 11:28:58 2015

@author: peterkroon
"""
from symfit import Variable, Parameter, exp, NumericalLeastSquares, Model
from symfit.contrib.interactive_guess import interactive_guess
import numpy as np

def distr(x, k, x0):
    kbT = 4.11
    return exp(-k*(x-x0)**2/kbT)

x = Variable()
y = Variable()
k = Parameter(900)
x0 = Parameter(1.5)

model = Model({y: distr(x, k, x0)})
x_data = np.linspace(0, 2.5, 50)
y_data = model(x=x_data, k=1000, x0=1).y

guess = interactive_guess.InteractiveGuess2D(model, x=x_data, y=y_data)

print("Guessed values: ")
for p in guess.model.params:
    print("{}: {}".format(p.name, p.value))

fit = ConstrainedNumericalLeastSquares(model, x=x_data, y=y_data)
fit_result = fit.execute(maxfev=1000)
print(fit_result)
<commit_msg>Fixed imports in simple_2D_example.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon Dec  7 11:28:58 2015

@author: peterkroon
"""
from symfit import Variable, Parameter, exp, Fit, Model
from symfit.contrib.interactive_guess import InteractiveGuess2D
import numpy as np

def distr(x, k, x0):
    kbT = 4.11
    return exp(-k*(x-x0)**2/kbT)

x = Variable()
y = Variable()
k = Parameter(900)
x0 = Parameter(1.5)

model = Model({y: distr(x, k, x0)})
x_data = np.linspace(0, 2.5, 50)
y_data = model(x=x_data, k=1000, x0=1).y

guess = InteractiveGuess2D(model, x=x_data, y=y_data)

print("Guessed values: ")
for p in guess.model.params:
    print("{}: {}".format(p.name, p.value))

fit = Fit(model, x=x_data, y=y_data)
fit_result = fit.execute(maxfev=1000)
print(fit_result)
<|endoftext|>


<commit_before>from .base import Message
from .decoder import register_to_decoder

class DeviceMessage(Message):
    """
    Abtract Class that holds common device information
    """
    (DIGITAL_PIN, TEMP_SENSOR,
     DS2413, DS2408VALVE) = range(1, 5)  # Hardware Type

    hardware_types = {0: 'Unknown hardware',
                      DIGITAL_PIN: 'Digital Pin',
                      TEMP_SENSOR: 'Temperature Sensor',
                      DS2413: 'DS2413 Expansion Board',
                      DS2408VALVE: 'DS2408 Valve Board'}

    (TEMP_SENSOR, SWITCH_SENSOR,
     SWITCH_ACTUATOR, PWM_ACTUATOR,
     MANUAL_ACTUATOR) = range(1, 6)  # Device type

    device_types = {0: 'Unknown device type',
                    TEMP_SENSOR: 'Temperature Sensor',
                    SWITCH_SENSOR: 'Switch Sensor',
                    SWITCH_ACTUATOR: 'Switch Actuator',
                    PWM_ACTUATOR: 'PWN Actuator',
                    MANUAL_ACTUATOR: 'Manual Actuator'}

    data_mapping = {
        'i': 'slot',
        't': 'device_type',
        'c': 'assigned_to_chamber',
        'b': 'assigned_to_beer',
        'f': 'function',
        'h': 'hardware_type',
        'd': 'd',
        'p': 'pin',
        'x': 'pin_inverted',
        'n': 'output_nr',
        'a': 'address',
        'j': 'value'
    }

    def __init__(self, slot, device_type, assigned_to_chamber,
                 assigned_to_beer, function, hardware_type, d, pin, pin_inverted=None,
                 output_nr=None, address=None, value=None):
        self.slot = slot
        self.device_type = device_type
        self.assigned_to_chamber = assigned_to_chamber
        self.assigned_to_beer = assigned_to_beer
        self.function = function
        self.hardware_type = hardware_type
        self.d = d
        self.pin = pin
        self.output_nr = output_nr
        self.pin_inverted = bool(pin_inverted)
        self.address = address
        self.value = value

    def hardware_type_str(self):
        """
        Represent hardware type as string
        """
        return self.hardware_types[self.hardware_type]

    def device_type_str(self):
        """
        Represent device type as string
        """
        return self.device_types[self.device_type]

    def __str__(self):
        return "Device <slot:{self.slot}, device_type:{device_type}, " \
            "chamber:{self.assigned_to_chamber}, beer:{self.assigned_to_beer}, " \
            "function:{self.function}, hardware_type:{hardware_type}, " \
            "d:{self.d}, pin:{self.pin}, pin_inverted?:{self.pin_inverted}, "\
            "output_nr:{self.output_nr}, " \
            "address:{self.address}, value:{self.value}>".format(self=self,
                                                                 device_type=self.device_type_str(),
                                                                 hardware_type=self.hardware_type_str())

@register_to_decoder()
class AvailableDeviceMessage(DeviceMessage):
    """
    Available devices (not installed) on the controller
    """
    cmd = 'h'

    def __str__(self):
        return "Available {0}".format(super().__str__())

@register_to_decoder()
class InstalledDeviceMessage(DeviceMessage):
    """
    Installed devices (installed) on the controller
    """
    cmd = 'd'

    def __str__(self):
        return "Installed {0}".format(super().__str__())
<commit_msg>fix value reading<commit_after>from .base import Message
from .decoder import register_to_decoder

class DeviceMessage(Message):
    """
    Abtract Class that holds common device information
    """
    (DIGITAL_PIN, TEMP_SENSOR,
     DS2413, DS2408VALVE) = range(1, 5)  # Hardware Type

    hardware_types = {0: 'Unknown hardware',
                      DIGITAL_PIN: 'Digital Pin',
                      TEMP_SENSOR: 'Temperature Sensor',
                      DS2413: 'DS2413 Expansion Board',
                      DS2408VALVE: 'DS2408 Valve Board'}

    (TEMP_SENSOR, SWITCH_SENSOR,
     SWITCH_ACTUATOR, PWM_ACTUATOR,
     MANUAL_ACTUATOR) = range(1, 6)  # Device type

    device_types = {0: 'Unknown device type',
                    TEMP_SENSOR: 'Temperature Sensor',
                    SWITCH_SENSOR: 'Switch Sensor',
                    SWITCH_ACTUATOR: 'Switch Actuator',
                    PWM_ACTUATOR: 'PWN Actuator',
                    MANUAL_ACTUATOR: 'Manual Actuator'}

    data_mapping = {
        'i': 'slot',
        't': 'device_type',
        'c': 'assigned_to_chamber',
        'b': 'assigned_to_beer',
        'f': 'function',
        'h': 'hardware_type',
        'd': 'd',
        'p': 'pin',
        'x': 'pin_inverted',
        'n': 'output_nr',
        'a': 'address',
        'v': 'value'
    }

    def __init__(self, slot, device_type, assigned_to_chamber,
                 assigned_to_beer, function, hardware_type, d, pin, pin_inverted=None,
                 output_nr=None, address=None, value=None):
        self.slot = slot
        self.device_type = device_type
        self.assigned_to_chamber = assigned_to_chamber
        self.assigned_to_beer = assigned_to_beer
        self.function = function
        self.hardware_type = hardware_type
        self.d = d
        self.pin = pin
        self.output_nr = output_nr
        self.pin_inverted = bool(pin_inverted)
        self.address = address
        self.value = value

    def hardware_type_str(self):
        """
        Represent hardware type as string
        """
        return self.hardware_types[self.hardware_type]

    def device_type_str(self):
        """
        Represent device type as string
        """
        return self.device_types[self.device_type]

    def __str__(self):
        return "Device <slot:{self.slot}, device_type:{device_type}, " \
            "chamber:{self.assigned_to_chamber}, beer:{self.assigned_to_beer}, " \
            "function:{self.function}, hardware_type:{hardware_type}, " \
            "d:{self.d}, pin:{self.pin}, pin_inverted?:{self.pin_inverted}, "\
            "output_nr:{self.output_nr}, " \
            "address:{self.address}, value:{self.value}>".format(self=self,
                                                                 device_type=self.device_type_str(),
                                                                 hardware_type=self.hardware_type_str())

@register_to_decoder()
class AvailableDeviceMessage(DeviceMessage):
    """
    Available devices (not installed) on the controller
    """
    cmd = 'h'

    def __str__(self):
        return "Available {0}".format(super().__str__())

@register_to_decoder()
class InstalledDeviceMessage(DeviceMessage):
    """
    Installed devices (installed) on the controller
    """
    cmd = 'd'

    def __str__(self):
        return "Installed {0}".format(super().__str__())
<|endoftext|>


<commit_before># Copyright (c) 2013.  Mark E. Madsen <mark@madsenlab.org>
#
# This work is licensed under the terms of the Apache Software License, Version 2.0.  See the file LICENSE for details.

from __future__ import print_function
import simuOpt, sys
simuOpt.setOptions(alleleType='long',optimized=False,quiet=True)
import simuPOP as sim
import uuid
import ctpy.sampling as sampling
import ctpy.utils as utils
import ctpy.math as cpm
import ctpy.innovation_models as innov
import ming
import logging
import pprint as pp

"""
This program simulates the Wright-Fisher model of genetic drift with infinite-alleles mutation in a single
population, and counts the number of alleles present in the population and in samples of specified size
"""

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s: %(message)s')

config = sampling.getMingConfiguration()
ming.configure(**config)

sim_id = uuid.uuid4().urn

options = [
    {
        'name':'popsize',
        'default':1000,
        'label':'Population Size',
        'type': 'integer',
        'validator': 'popsize > 0',
    },
    {
        'name':'stepsize',
        'default':100,
        'label':'Interval Between Data Samples',
        'type': 'integer',
        'validator': 'stepsize > 0',
    },
    {
        'name':'length',
        'default':10000,
        'label':'Length of simulation sample (in generations) after stationarity reached',
        'type': 'integer',
        'validator': 'length > 0',
    },
    {
        'name':'replications',
        'default':5,
        'label':'Number of populations to simulate in parallel',
        'type': 'integer',
        'validator': 'replications > 0',
    },
    {
        'name':'samplesize',
        'default':30,
        'label':'Size of sample to take each generation for allele counting',
        'type': 'integer',
        'validator': 'samplesize > 0',
    },
    {
        'name':'mutationrate',
        'default':0.001,
        'label':'Rate of individual innovations/mutations per generation',
        'type': 'number',
        'validator': 'mutationrate > 0.0',
    },
    {
        'name':'numloci',
        'default' : 2,
        'label' : 'Number of loci to model',
        'type' : 'integer',
        'validator' : 'numloci > 0',
    },
    {
        'name' : 'numalleles',
        'default' : 10,
        'label' : 'Number of initial alleles in population',
        'type' : 'integer',
        'validator' : 'numalleles > 0'
    }

]

# get all parameters
pars = simuOpt.Params(options, doc='This program simulates the Wright-Fisher model of genetic drift with infinite-alleles mutation in a single population, and counts the number of alleles present in the population and in samples of specified size')
# cancelled
if not pars.getParam():
    sys.exit(1)

logging.info("Beginning simulation run: %s", sim_id)

beginCollectingData = cpm.expectedIAQuasiStationarityTimeHaploid(pars.popsize,pars.mutationrate)
logging.info("Starting data collection at generation: %s", beginCollectingData)

totalSimulationLength = beginCollectingData + pars.length
logging.info("Simulation will sample %s generations after stationarity", pars.length)

sampling.storeSimulationData(pars.popsize,pars.mutationrate,sim_id,pars.samplesize,pars.replications,pars.numloci,__file__,pars.numalleles)

initial_distribution = utils.constructUniformAllelicDistribution(pars.numalleles)
logging.info("Initial allelic distribution: %s", initial_distribution)

lifetime_cache = sampling.TraitLifetimeCacheIAModels(pars.replications,pars.numloci,pars.numalleles)

pop = sim.Population(size=pars.popsize, ploidy=1, loci=pars.numloci)
simu = sim.Simulator(pop, rep=pars.replications)

simu.evolve(
	initOps = sim.InitGenotype(freq=initial_distribution),
    preOps = [
        sim.PyOperator(func=utils.logGenerationCount, param=(), step=1000, reps=0),
    ],
	matingScheme = sim.RandomSelection(),
	postOps = [sim.KAlleleMutator(k=100000000,rates=pars.mutationrate, loci=sim.ALL_AVAIL,output=lifetime_cache.allele_origin_tracker),
		#sim.PyEval(r"'%d, ' % gen", step=pars.stepsize,begin=beginCollectingData,reps=0),
        #sim.PyOperator(func=sampling.sampleNumAlleles, param=(pars.samplesize, pars.mutationrate, pars.popsize,sim_id,pars.numloci), step=pars.stepsize,begin=beginCollectingData),
        #sim.PyOperator(func=sampling.sampleTraitCounts, param=(pars.samplesize, pars.mutationrate, pars.popsize,sim_id,pars.numloci), step=pars.stepsize,begin=beginCollectingData),
        #sim.PyOperator(func=sampling.sampleIndividuals, param=(pars.samplesize, pars.mutationrate, pars.popsize, sim_id), step=pars.stepsize, begin=beginCollectingData),
        sim.PyOperator(func=lifetime_cache.allele_demise_tracker, param=(pars.samplesize, pars.mutationrate, pars.popsize, sim_id, pars.numloci), step=1),
        #sim.PyOutput('\n', reps=-1, step = pars.stepsize, begin=beginCollectingData),
		],
	gen = totalSimulationLength,
)

logging.info("Ending simulation run at generation %s", simu.population(0).dvars().gen)

<commit_msg>trait lifetime simulation not required<commit_after><|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Converts MNIST data to TFRecords file format with Example protos."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os
import sys

import tensorflow as tf

from tensorflow.contrib.learn.python.learn.datasets import mnist

FLAGS = None

def _int64_feature(value):
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def convert_to(data_set, name):
  """Converts a dataset to tfrecords."""
  images = data_set.images
  labels = data_set.labels
  num_examples = data_set.num_examples

  if images.shape[0] != num_examples:
    raise ValueError('Images size %d does not match label size %d.' %
                     (images.shape[0], num_examples))
  rows = images.shape[1]
  cols = images.shape[2]
  depth = images.shape[3]

  filename = os.path.join(FLAGS.directory, name + '.tfrecords')
  print('Writing', filename)
  writer = tf.python_io.TFRecordWriter(filename)
  for index in range(num_examples):
    image_raw = images[index].tostring()
    example = tf.train.Example(features=tf.train.Features(feature={
        'height': _int64_feature(rows),
        'width': _int64_feature(cols),
        'depth': _int64_feature(depth),
        'label': _int64_feature(int(labels[index])),
        'image_raw': _bytes_feature(image_raw)}))
    writer.write(example.SerializeToString())
  writer.close()

def main(unused_argv):
  # Get the data.
  data_sets = mnist.read_data_sets(FLAGS.directory,
                                   dtype=tf.uint8,
                                   reshape=False,
                                   validation_size=FLAGS.validation_size)

  # Convert to Examples and write the result to TFRecords.
  convert_to(data_sets.train, 'train')
  convert_to(data_sets.validation, 'validation')
  convert_to(data_sets.test, 'test')

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--directory',
      type=str,
      default='/tmp/data',
      help='Directory to download data files and write the converted result'
  )
  parser.add_argument(
      '--validation_size',
      type=int,
      default=5000,
      help="""\
      Number of examples to separate from the training data for the validation
      set.\
      """
  )
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
<commit_msg>use with when calling TFRecordWriter (#14497)<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Converts MNIST data to TFRecords file format with Example protos."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os
import sys

import tensorflow as tf

from tensorflow.contrib.learn.python.learn.datasets import mnist

FLAGS = None

def _int64_feature(value):
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def convert_to(data_set, name):
  """Converts a dataset to tfrecords."""
  images = data_set.images
  labels = data_set.labels
  num_examples = data_set.num_examples

  if images.shape[0] != num_examples:
    raise ValueError('Images size %d does not match label size %d.' %
                     (images.shape[0], num_examples))
  rows = images.shape[1]
  cols = images.shape[2]
  depth = images.shape[3]

  filename = os.path.join(FLAGS.directory, name + '.tfrecords')
  print('Writing', filename)
  with tf.python_io.TFRecordWriter(filename) as writer:
    for index in range(num_examples):
      image_raw = images[index].tostring()
      example = tf.train.Example(features=tf.train.Features(feature={
          'height': _int64_feature(rows),
          'width': _int64_feature(cols),
          'depth': _int64_feature(depth),
          'label': _int64_feature(int(labels[index])),
          'image_raw': _bytes_feature(image_raw)}))
      writer.write(example.SerializeToString())

def main(unused_argv):
  # Get the data.
  data_sets = mnist.read_data_sets(FLAGS.directory,
                                   dtype=tf.uint8,
                                   reshape=False,
                                   validation_size=FLAGS.validation_size)

  # Convert to Examples and write the result to TFRecords.
  convert_to(data_sets.train, 'train')
  convert_to(data_sets.validation, 'validation')
  convert_to(data_sets.test, 'test')

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--directory',
      type=str,
      default='/tmp/data',
      help='Directory to download data files and write the converted result'
  )
  parser.add_argument(
      '--validation_size',
      type=int,
      default=5000,
      help="""\
      Number of examples to separate from the training data for the validation
      set.\
      """
  )
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
<|endoftext|>