<commit_before>#!/usr/bin/python
#

# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""Script for testing ganeti.utils.retry"""

import unittest

from ganeti import constants
from ganeti import errors
from ganeti import utils

import testutils

class TestRetry(testutils.GanetiTestCase):
  def setUp(self):
    testutils.GanetiTestCase.setUp(self)
    self.retries = 0
    self.called = 0

  @staticmethod
  def _RaiseRetryAgain():
    raise utils.RetryAgain()

  @staticmethod
  def _RaiseRetryAgainWithArg(args):
    raise utils.RetryAgain(*args)

  def _WrongNestedLoop(self):
    return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)

  def _RetryAndSucceed(self, retries):
    if self.retries < retries:
      self.retries += 1
      raise utils.RetryAgain()
    else:
      return True

  def _SimpleRetryAndSucceed(self, retries):
    self.called += 1
    if self.retries < retries:
      self.retries += 1
      return False
    else:
      return True

  def testRaiseTimeout(self):
    self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
                          self._RaiseRetryAgain, 0.01, 0.02)
    self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
                          self._RetryAndSucceed, 0.01, 0, args=[1])
    self.failUnlessEqual(self.retries, 1)

  def testComplete(self):
    self.failUnlessEqual(utils.Retry(lambda: True, 0, 1), True)
    self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2]),
                         True)
    self.failUnlessEqual(self.retries, 2)

  def testNestedLoop(self):
    try:
      self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
                            self._WrongNestedLoop, 0, 1)
    except utils.RetryTimeout:
      self.fail("Didn't detect inner loop's exception")

  def testTimeoutArgument(self):
    retry_arg="my_important_debugging_message"
    try:
      utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]])
    except utils.RetryTimeout, err:
      self.failUnlessEqual(err.args, (retry_arg, ))
    else:
      self.fail("Expected timeout didn't happen")

  def testRaiseInnerWithExc(self):
    retry_arg="my_important_debugging_message"
    try:
      try:
        utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
                    args=[[errors.GenericError(retry_arg, retry_arg)]])
      except utils.RetryTimeout, err:
        err.RaiseInner()
      else:
        self.fail("Expected timeout didn't happen")
    except errors.GenericError, err:
      self.failUnlessEqual(err.args, (retry_arg, retry_arg))
    else:
      self.fail("Expected GenericError didn't happen")

  def testRaiseInnerWithMsg(self):
    retry_arg="my_important_debugging_message"
    try:
      try:
        utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
                    args=[[retry_arg, retry_arg]])
      except utils.RetryTimeout, err:
        err.RaiseInner()
      else:
        self.fail("Expected timeout didn't happen")
    except utils.RetryTimeout, err:
      self.failUnlessEqual(err.args, (retry_arg, retry_arg))
    else:
      self.fail("Expected RetryTimeout didn't happen")

  def testSimpleRetry(self):
    self.assertFalse(utils.SimpleRetry(True, lambda: False, 0.01, 0.02))
    self.assertFalse(utils.SimpleRetry(lambda x: x, lambda: False, 0.01, 0.02))
    self.assertTrue(utils.SimpleRetry(True, lambda: True, 0, 1))
    self.assertTrue(utils.SimpleRetry(lambda x: x, lambda: True, 0, 1))
    self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed,
                                      0, 1, args=[1]))
    self.assertEqual(self.retries, 1)
    self.assertEqual(self.called, 2)
    self.called = self.retries = 0
    self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed,
                                      0, 1, args=[2]))
    self.assertEqual(self.called, 3)

if __name__ == "__main__":
  testutils.GanetiTestProgram()
<commit_msg>Make retry tests independent of actual time<commit_after>#!/usr/bin/python
#

# Copyright (C) 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""Script for testing ganeti.utils.retry"""

import unittest

from ganeti import constants
from ganeti import errors
from ganeti import utils

import testutils

class TestRetry(testutils.GanetiTestCase):
  def setUp(self):
    testutils.GanetiTestCase.setUp(self)
    self.retries = 0
    self.called = 0
    self.time = 1379601882.0

  def _time_fn(self):
    return self.time

  def _wait_fn(self, delay):
    self.time += delay

  @staticmethod
  def _RaiseRetryAgain():
    raise utils.RetryAgain()

  @staticmethod
  def _RaiseRetryAgainWithArg(args):
    raise utils.RetryAgain(*args)

  def _WrongNestedLoop(self):
    return utils.Retry(self._RaiseRetryAgain, 0.01, 0.02)

  def _RetryAndSucceed(self, retries):
    if self.retries < retries:
      self.retries += 1
      raise utils.RetryAgain()
    else:
      return True

  def _SimpleRetryAndSucceed(self, retries):
    self.called += 1
    if self.retries < retries:
      self.retries += 1
      return False
    else:
      return True

  def testRaiseTimeout(self):
    self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
                          self._RaiseRetryAgain, 0.01, 0.02,
                          wait_fn = self._wait_fn, _time_fn = self._time_fn)
    self.failUnlessRaises(utils.RetryTimeout, utils.Retry,
                          self._RetryAndSucceed, 0.01, 0, args=[1],
                          wait_fn = self._wait_fn, _time_fn = self._time_fn)
    self.failUnlessEqual(self.retries, 1)

  def testComplete(self):
    self.failUnlessEqual(utils.Retry(lambda: True, 0, 1,
                                     wait_fn = self._wait_fn,
                                     _time_fn = self._time_fn),
                         True)
    self.failUnlessEqual(utils.Retry(self._RetryAndSucceed, 0, 1, args=[2],
                                     wait_fn = self._wait_fn,
                                     _time_fn = self._time_fn),
                         True)
    self.failUnlessEqual(self.retries, 2)

  def testNestedLoop(self):
    try:
      self.failUnlessRaises(errors.ProgrammerError, utils.Retry,
                            self._WrongNestedLoop, 0, 1,
                            wait_fn = self._wait_fn, _time_fn = self._time_fn)
    except utils.RetryTimeout:
      self.fail("Didn't detect inner loop's exception")

  def testTimeoutArgument(self):
    retry_arg="my_important_debugging_message"
    try:
      utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02, args=[[retry_arg]],
                  wait_fn = self._wait_fn, _time_fn = self._time_fn)
    except utils.RetryTimeout, err:
      self.failUnlessEqual(err.args, (retry_arg, ))
    else:
      self.fail("Expected timeout didn't happen")

  def testRaiseInnerWithExc(self):
    retry_arg="my_important_debugging_message"
    try:
      try:
        utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
                    args=[[errors.GenericError(retry_arg, retry_arg)]],
                    wait_fn = self._wait_fn, _time_fn = self._time_fn)
      except utils.RetryTimeout, err:
        err.RaiseInner()
      else:
        self.fail("Expected timeout didn't happen")
    except errors.GenericError, err:
      self.failUnlessEqual(err.args, (retry_arg, retry_arg))
    else:
      self.fail("Expected GenericError didn't happen")

  def testRaiseInnerWithMsg(self):
    retry_arg="my_important_debugging_message"
    try:
      try:
        utils.Retry(self._RaiseRetryAgainWithArg, 0.01, 0.02,
                    args=[[retry_arg, retry_arg]],
                    wait_fn = self._wait_fn, _time_fn = self._time_fn)
      except utils.RetryTimeout, err:
        err.RaiseInner()
      else:
        self.fail("Expected timeout didn't happen")
    except utils.RetryTimeout, err:
      self.failUnlessEqual(err.args, (retry_arg, retry_arg))
    else:
      self.fail("Expected RetryTimeout didn't happen")

  def testSimpleRetry(self):
    self.assertFalse(utils.SimpleRetry(True, lambda: False, 0.01, 0.02,
                                       wait_fn = self._wait_fn,
                                       _time_fn = self._time_fn))
    self.assertFalse(utils.SimpleRetry(lambda x: x, lambda: False, 0.01, 0.02,
                                       wait_fn = self._wait_fn,
                                       _time_fn = self._time_fn))
    self.assertTrue(utils.SimpleRetry(True, lambda: True, 0, 1,
                                      wait_fn = self._wait_fn,
                                      _time_fn = self._time_fn))
    self.assertTrue(utils.SimpleRetry(lambda x: x, lambda: True, 0, 1,
                                      wait_fn = self._wait_fn,
                                      _time_fn = self._time_fn))
    self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1,
                                      args=[1], wait_fn = self._wait_fn,
                                      _time_fn = self._time_fn))
    self.assertEqual(self.retries, 1)
    self.assertEqual(self.called, 2)
    self.called = self.retries = 0
    self.assertTrue(utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1,
                                      args=[2], wait_fn = self._wait_fn,
                                      _time_fn = self._time_fn))
    self.assertEqual(self.called, 3)

if __name__ == "__main__":
  testutils.GanetiTestProgram()
<|endoftext|>


<commit_before># Django
from django.db import models

# AstroBin
from astrobin.models import Image

class ThumbnailGroup(models.Model):
    image = models.ForeignKey(
        Image,
        related_name = 'thumbnails',
    )

    revision = models.CharField(
        max_length = 1,
        default = '0',
    )

    real             = models.CharField(max_length = 512, null = True, blank = True)
    real_inverted    = models.CharField(max_length = 512, null = True, blank = True)
    hd               = models.CharField(max_length = 512, null = True, blank = True)
    hd_inverted      = models.CharField(max_length = 512, null = True, blank = True)
    regular          = models.CharField(max_length = 512, null = True, blank = True)
    regular_inverted = models.CharField(max_length = 512, null = True, blank = True)
    gallery          = models.CharField(max_length = 512, null = True, blank = True)
    gallery_inverted = models.CharField(max_length = 512, null = True, blank = True)
    thumb            = models.CharField(max_length = 512, null = True, blank = True)
    revision         = models.CharField(max_length = 512, null = True, blank = True)
    histogram        = models.CharField(max_length = 512, null = True, blank = True)
    iotd             = models.CharField(max_length = 512, null = True, blank = True)
    act_target       = models.CharField(max_length = 512, null = True, blank = True)
    act_object       = models.CharField(max_length = 512, null = True, blank = True)

    def __unicode__(self):
        return "Thumbnails for image " % self.image.title

    class Meta:
        app_label = 'astrobin_apps_images'
        unique_together = ('image', 'revision',)

<commit_msg>Syntax<commit_after># Django
from django.db import models

# AstroBin
from astrobin.models import Image

class ThumbnailGroup(models.Model):
    image = models.ForeignKey(
        Image,
        related_name = 'thumbnails',
    )

    revision = models.CharField(
        max_length = 1,
        default = '0',
    )

    real             = models.CharField(max_length = 512, null = True, blank = True)
    real_inverted    = models.CharField(max_length = 512, null = True, blank = True)
    hd               = models.CharField(max_length = 512, null = True, blank = True)
    hd_inverted      = models.CharField(max_length = 512, null = True, blank = True)
    regular          = models.CharField(max_length = 512, null = True, blank = True)
    regular_inverted = models.CharField(max_length = 512, null = True, blank = True)
    gallery          = models.CharField(max_length = 512, null = True, blank = True)
    gallery_inverted = models.CharField(max_length = 512, null = True, blank = True)
    thumb            = models.CharField(max_length = 512, null = True, blank = True)
    revision         = models.CharField(max_length = 512, null = True, blank = True)
    histogram        = models.CharField(max_length = 512, null = True, blank = True)
    iotd             = models.CharField(max_length = 512, null = True, blank = True)
    act_target       = models.CharField(max_length = 512, null = True, blank = True)
    act_object       = models.CharField(max_length = 512, null = True, blank = True)

    def __unicode__(self):
        return "Thumbnails for image %s" % self.image.title

    class Meta:
        app_label = 'astrobin_apps_images'
        unique_together = ('image', 'revision',)

<|endoftext|>


<commit_before>import scrapy
from OnlineParticipationDataset import items
from datetime import datetime
import re

class Bonn2011Spider(scrapy.Spider):
    name = "bonn2011"
    start_urls = ['http://bonn-packts-an-2011.de/www.bonn-packts-an.de/dito/forumc0d2.html']

    def suggestion_id(self,response):
        return (re.search('([A-Z]\d+)',response.xpath('.//div[starts-with(@class,"vorschlag")]/h2/text()')
            .extract_first()))[0]

    def suggestion_author(self,response):
        return (re.search('(?:von\s)(\w+)',response.xpath('.//div[starts-with(@class,"vorschlag")]/h2/text()')
            .extract_first()))[1]

    def suggestion_title(self,response):
        return response.xpath('.//div[@class="col_01"]/h3/text()').extract_first()

    def suggestion_category(self,response):
        return response.xpath(
            './/div[starts-with(@class,"vorschlag")]/div[@class="image"]/img/@title').extract_first()

    def suggestion_type(self,response):
        return response.xpath('.//div[@class="col_01"]/strong/text()').extract_first()

    def suggestion_content(sef,response):
        return response.xpath('.//div[@class="col_01"]/p/text()').extract_first()

    def suggestion_summary_response(self,response):
        return response.xpath('.//div[@class="col_01"]/table')

    def suggestion_approval(self,response):
         return int(
             self.suggestion_summary_response(response).xpath('.//td[starts-with(@id,"votePro")]/text()')
            .extract_first())

    def suggestion_refusal(self,response):
        return int(
            self.suggestion_summary_response(response).xpath('.//td[starts-with(@id,"voteContra")]/text()')
            .extract_first())

    def suggestion_abstention(self,response):
        return int(
            self.suggestion_summary_response(response).xpath('.//td[starts-with(@id,"voteNeutral")]/text()')
            .extract_first())

    def suggestion_comment_count(self, response):
        return int(
            # float(response.xpath('count(//div[starts-with(@class,"kommentar")])')
            #       .extract_first()))
                self.suggestion_summary_response(response).xpath('.//td[starts-with(.,"Kommentare")]/../td[@class="r"]/text()')
                .extract_first())

    def parse_suggestion_tags(self, l):
        return [s.rsplit('(',1)[0].strip() for s in l]

    def suggestion_tags(self, response):
        return self.parse_suggestion_tags(response.xpath('.//div[@class="kasten cloud"]/div[@class="text"]//a/text()').extract())

    def parse_datetime(self, s):
        return datetime.strptime(
            re.sub(r"(\s|[a-z])+", "", s.lower(), flags=re.UNICODE),'%d.%m.%Y-%H:%M')

    def suggestion_datetime(self,response):
        return self.parse_datetime(
                response.xpath('.//div[@class="details"]/p/text()').extract_first())

    def parse_datetime_comment(self, s):
        s = s.split("|")
        s = "-".join(s[1:])
        return self.parse_datetime(s)

    def parse_comment_id(self, s):
        if s is not None:
            return re.search(r"ID\:\D*(\d+)",s)[1]

    def comment_id(self,response):
        return self.parse_comment_id(response.xpath('.//div[@class="col_01"]/comment()').extract_first())

    def parse_comment_id_official(self,s):
        return re.search(r"Nr\.\s*(\d+)",s)[1]

    def comment_id_official(self,response):
        return self.parse_comment_id_official(response.xpath('.//div[@class="user"]/text()').extract_first())

    def comment_content(self,response):
        return response.xpath('.//div[@class="col_01"]/p/text()').extract_first().strip()

    def comment_title(self,response):
        return response.xpath('.//div[@class="col_01"]/h2/text()').extract_first().strip()

    def parse_comment_author(self,s):
        return re.search(r"von\s+(\w+)\s+",s)[1]

    def comment_author(self,response):
        return self.parse_comment_author(response.xpath('.//div[@class="user"]/text()').extract_first())

    def comment_datetime(self,response):
        return self.parse_datetime_comment(response.xpath('.//div[@class="user"]/text()').extract_first())

    def create_comment_item(self, response):
        """
        Create a CommentItem, see :class:`~OnlineParticipationDataset.items.CommentItem`, from given response.

        :param response: scrapy response
        :return: scrapy item
        """
        comment_item = items.CommentItem()
        comment_class = response.xpath('@class').extract_first().replace('kommentar_','').split()

        comment_item['level'] = int(comment_class[0])
        comment_item['comment_id'] = self.comment_id(response)
        comment_item['suggestion_id'] = self.suggestion_id(response.xpath('..'))
        comment_item['content'] = self.comment_content(response)
        comment_item['author'] = self.comment_author(response)
        comment_item['date_time'] = self.comment_datetime(response)
        comment_item['title'] = self.comment_title(response)

        # If official the id is located elsewhere
        # if len < 2 its an answer on a comment with vote
        if(len(comment_class) == 2):
            if "ablehnung" == comment_class[1]:
                comment_item['vote'] = "refusal"
            elif "zustimmung" == comment_class[1]:
                comment_item['vote'] = "approval"
            elif "neutral" == comment_class[1]:
                comment_item['vote'] = "neutral"
            elif "stellungnahme" == comment_class[1] or "verwaltung" == comment_class[1]:

                comment_item['comment_id'] = self.comment_id_official(response)
                comment_item['vote'] = "official"
            else:
                comment_item['vote'] = "misc"
        else:
            comment_item['vote'] = "answer"

        return comment_item

    def parse_comment_tree(self, item_list):
        """
        Parse list with CommentItems, see :class:`~OnlineParticipationDataset.items.CommentItem`, to restore the comment tree. Write parent and children to items. Items need to have values for level, commment_id and suggestion_id.

        :param item_list: list with CommmentItems
        :return list with CommmentItems
        """
        stack = []
        for position,item in enumerate(item_list):
            item['children'] = []
            while(len(stack) >= item['level']):
                stack.pop()
            # Check if not first level, i.e. stack not is empty
            if stack:
                # (pos,id)
                parent_tuple = stack[-1]
                # Parent is the last seen item
                item['parent_id'] = parent_tuple[1]
                # Add current item as child to parent
                item_list[parent_tuple[0]]['children'].append(item)
            else:
                # Top-Level Comments
                item['parent_id'] = item['suggestion_id']
            stack.append(tuple((position,item['comment_id'])))
        return list(filter(lambda x: x['level'] <= 1, item_list))

    def create_comment_item_list(self, response):
        """
        Create a list of CommentItems, see :class:`~OnlineParticipationDataset.items.CommentItem`, from given response.

        :param response: scrapy response
        :return: list with CommentItems
        """
        comment_items = []
        for comment in response.xpath('//div[starts-with(@class,"kommentar")]'):
            comment_items.append(self.create_comment_item(comment))
        return self.parse_comment_tree(comment_items)

    def create_suggestion_item(self, response):
        """
        Create a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`, from given response.

        :param response: scrapy response
        :return: scrapy item
        """
        suggestion_item = items.SuggestionItem()
        suggestion_item['suggestion_id'] = self.suggestion_id(response)
        suggestion_item['author'] = self.suggestion_author(response)
        suggestion_item['title'] = self.suggestion_title(response)
        suggestion_item['category'] = self.suggestion_category(response)
        suggestion_item['suggestion_type'] = self.suggestion_type(response)
        suggestion_item['content'] = self.suggestion_content(response)
        suggestion_item['approval'] = self.suggestion_approval(response)
        suggestion_item['refusal'] = self.suggestion_refusal(response)
        suggestion_item['abstention'] = self.suggestion_abstention(response)
        suggestion_item['comment_count'] = self.suggestion_comment_count(response)
        suggestion_item['date_time'] = self.suggestion_datetime(response)
        suggestion_item['tags'] = self.suggestion_tags(response)
        return suggestion_item

    def parse(self, response):
        """
        Parse given response and yield items for suggestions and comments of all
        pages of bonn-packts-an-2011.

        :param response: scrapy response
        :return: generator
        """
        for thread in response.xpath('//div[starts-with(@class,"vorschlag")]'):
            thread_url = thread.xpath('.//div[@class="col_01"]/h3/a/@href').extract_first()
            yield response.follow(thread_url,self.parse_thread)

        # Here: Parse next Site
        next_page = response.xpath('.//div[@class="list_pages"]/a[.="vor"]/@href').extract_first()
        if next_page:
            yield response.follow(next_page,self.parse)

    def parse_thread(self, response):
        """
        Parse thread and yield a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`.

        :param response: scrapy response
        :return: generator
        """
        suggestion = self.create_suggestion_item(response)
        suggestion['comments'] = self.create_comment_item_list(response)
        yield suggestion
<commit_msg>Corrected some spelling<commit_after>import scrapy
from OnlineParticipationDataset import items
from datetime import datetime
import re

class Bonn2011Spider(scrapy.Spider):
    name = "bonn2011"
    start_urls = ['http://bonn-packts-an-2011.de/www.bonn-packts-an.de/dito/forumc0d2.html']

    def suggestion_id(self,response):
        return (re.search('([A-Z]\d+)',response.xpath('.//div[starts-with(@class,"vorschlag")]/h2/text()')
            .extract_first()))[0]

    def suggestion_author(self,response):
        return (re.search('(?:von\s)(\w+)',response.xpath('.//div[starts-with(@class,"vorschlag")]/h2/text()')
            .extract_first()))[1]

    def suggestion_title(self,response):
        return response.xpath('.//div[@class="col_01"]/h3/text()').extract_first()

    def suggestion_category(self,response):
        return response.xpath(
            './/div[starts-with(@class,"vorschlag")]/div[@class="image"]/img/@title').extract_first()

    def suggestion_type(self,response):
        return response.xpath('.//div[@class="col_01"]/strong/text()').extract_first()

    def suggestion_content(sef,response):
        return response.xpath('.//div[@class="col_01"]/p/text()').extract_first()

    def suggestion_summary_response(self,response):
        return response.xpath('.//div[@class="col_01"]/table')

    def suggestion_approval(self,response):
         return int(
             self.suggestion_summary_response(response).xpath('.//td[starts-with(@id,"votePro")]/text()')
            .extract_first())

    def suggestion_refusal(self,response):
        return int(
            self.suggestion_summary_response(response).xpath('.//td[starts-with(@id,"voteContra")]/text()')
            .extract_first())

    def suggestion_abstention(self,response):
        return int(
            self.suggestion_summary_response(response).xpath('.//td[starts-with(@id,"voteNeutral")]/text()')
            .extract_first())

    def suggestion_comment_count(self, response):
        return int(
            # float(response.xpath('count(//div[starts-with(@class,"kommentar")])')
            #       .extract_first()))
                self.suggestion_summary_response(response).xpath('.//td[starts-with(.,"Kommentare")]/../td[@class="r"]/text()')
                .extract_first())

    def parse_suggestion_tags(self, l):
        return [s.rsplit('(',1)[0].strip() for s in l]

    def suggestion_tags(self, response):
        return self.parse_suggestion_tags(response.xpath('.//div[@class="kasten cloud"]/div[@class="text"]//a/text()').extract())

    def parse_datetime(self, s):
        return datetime.strptime(
            re.sub(r"(\s|[a-z])+", "", s.lower(), flags=re.UNICODE),'%d.%m.%Y-%H:%M')

    def suggestion_datetime(self,response):
        return self.parse_datetime(
                response.xpath('.//div[@class="details"]/p/text()').extract_first())

    def parse_datetime_comment(self, s):
        s = s.split("|")
        s = "-".join(s[1:])
        return self.parse_datetime(s)

    def parse_comment_id(self, s):
        if s is not None:
            return re.search(r"ID\:\D*(\d+)",s)[1]

    def comment_id(self,response):
        return self.parse_comment_id(response.xpath('.//div[@class="col_01"]/comment()').extract_first())

    def parse_comment_id_official(self,s):
        return re.search(r"Nr\.\s*(\d+)",s)[1]

    def comment_id_official(self,response):
        return self.parse_comment_id_official(response.xpath('.//div[@class="user"]/text()').extract_first())

    def comment_content(self,response):
        return response.xpath('.//div[@class="col_01"]/p/text()').extract_first().strip()

    def comment_title(self,response):
        return response.xpath('.//div[@class="col_01"]/h2/text()').extract_first().strip()

    def parse_comment_author(self,s):
        return re.search(r"von\s+(\w+)\s+",s)[1]

    def comment_author(self,response):
        return self.parse_comment_author(response.xpath('.//div[@class="user"]/text()').extract_first())

    def comment_datetime(self,response):
        return self.parse_datetime_comment(response.xpath('.//div[@class="user"]/text()').extract_first())

    def create_comment_item(self, response):
        """
        Create a CommentItem, see :class:`~OnlineParticipationDataset.items.CommentItem`, from given response.

        :param response: scrapy response
        :return: scrapy item
        """
        comment_item = items.CommentItem()
        comment_class = response.xpath('@class').extract_first().replace('kommentar_','').split()

        comment_item['level'] = int(comment_class[0])
        comment_item['comment_id'] = self.comment_id(response)
        comment_item['suggestion_id'] = self.suggestion_id(response.xpath('..'))
        comment_item['content'] = self.comment_content(response)
        comment_item['author'] = self.comment_author(response)
        comment_item['date_time'] = self.comment_datetime(response)
        comment_item['title'] = self.comment_title(response)

        # If official the id is located elsewhere
        # if len < 2 its an answer on a comment with vote
        if(len(comment_class) == 2):
            if "ablehnung" == comment_class[1]:
                comment_item['vote'] = "refusal"
            elif "zustimmung" == comment_class[1]:
                comment_item['vote'] = "approval"
            elif "neutral" == comment_class[1]:
                comment_item['vote'] = "neutral"
            elif "stellungnahme" == comment_class[1] or "verwaltung" == comment_class[1]:

                comment_item['comment_id'] = self.comment_id_official(response)
                comment_item['vote'] = "official"
            else:
                comment_item['vote'] = "misc"
        else:
            comment_item['vote'] = "answer"

        return comment_item

    def parse_comment_tree(self, item_list):
        """
        Parse list with CommentItems, see :class:`~OnlineParticipationDataset.items.CommentItem`, to restore the comment tree. Write parent and children to items. Items need to have values for level, comment_id and suggestion_id.

        :param item_list: list with CommentItems
        :return list with CommentItems
        """
        stack = []
        for position,item in enumerate(item_list):
            item['children'] = []
            while(len(stack) >= item['level']):
                stack.pop()
            # Check if not first level, i.e. stack not is empty
            if stack:
                # (pos,id)
                parent_tuple = stack[-1]
                # Parent is the last seen item
                item['parent_id'] = parent_tuple[1]
                # Add current item as child to parent
                item_list[parent_tuple[0]]['children'].append(item)
            else:
                # Top-Level Comments
                item['parent_id'] = item['suggestion_id']
            stack.append(tuple((position,item['comment_id'])))
        return list(filter(lambda x: x['level'] <= 1, item_list))

    def create_comment_item_list(self, response):
        """
        Create a list of CommentItems, see :class:`~OnlineParticipationDataset.items.CommentItem`, from given response.

        :param response: scrapy response
        :return: list with CommentItems
        """
        comment_items = []
        for comment in response.xpath('//div[starts-with(@class,"kommentar")]'):
            comment_items.append(self.create_comment_item(comment))
        return self.parse_comment_tree(comment_items)

    def create_suggestion_item(self, response):
        """
        Create a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`, from given response.

        :param response: scrapy response
        :return: scrapy item
        """
        suggestion_item = items.SuggestionItem()
        suggestion_item['suggestion_id'] = self.suggestion_id(response)
        suggestion_item['author'] = self.suggestion_author(response)
        suggestion_item['title'] = self.suggestion_title(response)
        suggestion_item['category'] = self.suggestion_category(response)
        suggestion_item['suggestion_type'] = self.suggestion_type(response)
        suggestion_item['content'] = self.suggestion_content(response)
        suggestion_item['approval'] = self.suggestion_approval(response)
        suggestion_item['refusal'] = self.suggestion_refusal(response)
        suggestion_item['abstention'] = self.suggestion_abstention(response)
        suggestion_item['comment_count'] = self.suggestion_comment_count(response)
        suggestion_item['date_time'] = self.suggestion_datetime(response)
        suggestion_item['tags'] = self.suggestion_tags(response)
        return suggestion_item

    def parse(self, response):
        """
        Parse given response and yield items for suggestions and comments of all
        pages of bonn-packts-an-2011.

        :param response: scrapy response
        :return: generator
        """
        for thread in response.xpath('//div[starts-with(@class,"vorschlag")]'):
            thread_url = thread.xpath('.//div[@class="col_01"]/h3/a/@href').extract_first()
            yield response.follow(thread_url,self.parse_thread)

        # Here: Parse next Site
        next_page = response.xpath('.//div[@class="list_pages"]/a[.="vor"]/@href').extract_first()
        if next_page:
            yield response.follow(next_page,self.parse)

    def parse_thread(self, response):
        """
        Parse thread and yield a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`.

        :param response: scrapy response
        :return: generator
        """
        suggestion = self.create_suggestion_item(response)
        suggestion['comments'] = self.create_comment_item_list(response)
        yield suggestion
<|endoftext|>


<commit_before>import os
import re
import secrets
import shutil
import subprocess
import weakref
from functools import partial

from pyarrow.hdfs import HadoopFileSystem

from fsspec.spec import AbstractFileSystem
from fsspec.utils import infer_storage_options

def inherits(*methods):
    def client_getter(name, self):
        return getattr(self.client, name)

    def wrapper(cls):
        for method in methods:
            wrapped_method = partial(client_getter, method)
            setattr(cls, method, property(wrapped_method))
        return cls

    return wrapper

CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P<checksum>.*)")

@inherits(
    "chmod",
    "chown",
    "user",
    "df",
    "disk_usage",
    "download",
    "driver",
    "exists",
    "extra_conf",
    "get_capacity",
    "get_space_used",
    "host",
    "is_open",
    "kerb_ticket",
    "strip_protocol",
    "mkdir",
    "port",
    "get_capacity",
    "get_space_used",
    "df",
    "chmod",
    "chown",
    "disk_usage",
    "download",
    "upload",
    "read_parquet",
    "rm",
    "stat",
    "upload",
)
class PyArrowHDFS(AbstractFileSystem):
    """Adapted version of Arrow's HadoopFileSystem

    This is a very simple wrapper over the pyarrow.hdfs.HadoopFileSystem, which
    passes on all calls to the underlying class."""

    protocol = "hdfs"

    def __init__(
        self,
        host="default",
        port=0,
        user=None,
        kerb_ticket=None,
        driver="libhdfs",
        extra_conf=None,
        **kwargs,
    ):
        """

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        driver: 'libhdfs' or 'libhdfs3'
            Binary driver; libhdfs if the JNI library and default
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        """
        super().__init__(**kwargs)

        self.client = HadoopFileSystem(
            host=host,
            port=port,
            user=user,
            kerb_ticket=kerb_ticket,
            driver=driver,
            extra_conf=extra_conf,
        )
        weakref.finalize(self, lambda: self.client.close())

        self.pars = (host, port, user, kerb_ticket, driver, extra_conf)

    @staticmethod
    def _get_kwargs_from_urls(path):
        ops = infer_storage_options(path)
        out = {}
        if ops.get("host", None):
            out["host"] = ops["host"]
        if ops.get("username", None):
            out["user"] = ops["username"]
        if ops.get("port", None):
            out["port"] = ops["port"]
        return out

    @classmethod
    def _strip_protocol(cls, path):
        ops = infer_storage_options(path)
        return ops["path"]

    def __reduce_ex__(self, protocol):
        return PyArrowHDFS, self.pars

    def close(self):
        self.client.close()

    def ls(self, path, detail=True):
        listing = [
            self._adjust_entry(entry) for entry in self.client.ls(path, detail=True)
        ]

        if detail:
            return listing
        else:
            return [entry["name"] for entry in listing]

    def info(self, path):
        return self._adjust_entry(self.client.info(path))

    def _adjust_entry(self, original_entry):
        entry = original_entry.copy()
        entry["type"] = entry["kind"]
        entry["name"] = self._strip_protocol(entry["path"])
        return entry

    def cp_file(self, lpath, rpath):
        with self.open(lpath) as lstream:
            tmp_fname = "/".join([self._parent(rpath), f".tmp.{secrets.token_hex(16)}"])
            with open(tmp_fname, "w") as rstream:
                shutil.copyfileobj(lstream, rstream)
            self.move(tmp_fname, rpath)
            self.rm(tmp_fname)

    def makedirs(self, path):
        self.client.create_dir(path)

    def _open(
        self,
        path,
        mode="rb",
        block_size=None,
        autocommit=True,
        cache_options=None,
        **kwargs,
    ):
        """

        Parameters
        ----------
        path: str
            Location of file; should start with '/'
        mode: str
        block_size: int
            Hadoop block size, e.g., 2**26
        autocommit: True
            Transactions are not yet implemented for HDFS; errors if not True
        kwargs: dict or None
            Hadoop config parameters

        Returns
        -------
        HDFSFile file-like instance
        """

        return HDFSFile(
            self,
            path,
            mode,
            block_size=block_size,
            autocommit=autocommit,
            cache_options=cache_options,
            **kwargs,
        )

    def checksum(self, path, env=None, **kwargs):
        # PyArrow doesn't natively support retrieving the
        # checksum, so we have to use hadoop fs

        result = self._run_command(f"checksum {self._as_url(path)}", env=env)
        if result is None:
            return None

        match = CHECKSUM_REGEX.match(result)
        if match is None:
            return None

        return match.group("checksum")

    def _run_command(self, cmd, env=None):
        cmd = "hadoop fs -" + cmd
        if self.user:
            cmd = f"HADOOP_USER_NAME={self.user} " + cmd

        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
        # See https://github.com/iterative/dvc/issues/1197.
        close_fds = os.name != "nt"

        executable = os.getenv("SHELL") if os.name != "nt" else None
        p = subprocess.Popen(
            cmd,
            shell=True,
            close_fds=close_fds,
            executable=executable,
            env=env or os.environ,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        out, err = p.communicate()

        if p.returncode != 0:
            raise subprocess.CalledProcessError(p.returncode, cmd, out, err)
        else:
            return out.decode("utf-8")

    def _as_url(self, path):
        netloc = f"{self.host}:{self.port}"
        if self.user:
            netloc = f"{self.user}@{netloc}"
        return f"hdfs://{netloc}/{path.lstrip('/')}"

class HDFSFile(object):
    """Wrapper around arrow's HdfsFile

    Allows seek beyond EOF and (eventually) commit/discard
    """

    def __init__(
        self,
        fs,
        path,
        mode,
        block_size,
        autocommit=True,
        cache_type="readahead",
        cache_options=None,
        **kwargs,
    ):
        # TODO: Inherit from AbstractBufferedFile?
        if not autocommit:
            raise NotImplementedError(
                "HDFSFile cannot be opened with 'autocommit=False'."
            )

        self.fs = fs
        self.path = path
        self.mode = mode
        self.block_size = block_size
        self.fh = fs.client.open(path, mode, block_size, **kwargs)
        if self.fh.readable():
            self.seek_size = self.size()

    def seek(self, loc, whence=0):
        if whence == 0 and self.readable():
            loc = min(loc, self.seek_size)
        return self.fh.seek(loc, whence)

    def __getattr__(self, item):
        return getattr(self.fh, item)

    def __reduce_ex__(self, protocol):
        return HDFSFile, (self.fs, self.path, self.mode, self.block_size)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
<commit_msg>hdfs: support exist_ok on makedirs()<commit_after>import os
import re
import secrets
import shutil
import subprocess
import weakref
from functools import partial

from pyarrow.hdfs import HadoopFileSystem

from fsspec.spec import AbstractFileSystem
from fsspec.utils import infer_storage_options

def inherits(*methods):
    def client_getter(name, self):
        return getattr(self.client, name)

    def wrapper(cls):
        for method in methods:
            wrapped_method = partial(client_getter, method)
            setattr(cls, method, property(wrapped_method))
        return cls

    return wrapper

CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P<checksum>.*)")

@inherits(
    "chmod",
    "chown",
    "user",
    "df",
    "disk_usage",
    "download",
    "driver",
    "exists",
    "extra_conf",
    "get_capacity",
    "get_space_used",
    "host",
    "is_open",
    "kerb_ticket",
    "strip_protocol",
    "mkdir",
    "port",
    "get_capacity",
    "get_space_used",
    "df",
    "chmod",
    "chown",
    "disk_usage",
    "download",
    "upload",
    "read_parquet",
    "rm",
    "stat",
    "upload",
)
class PyArrowHDFS(AbstractFileSystem):
    """Adapted version of Arrow's HadoopFileSystem

    This is a very simple wrapper over the pyarrow.hdfs.HadoopFileSystem, which
    passes on all calls to the underlying class."""

    protocol = "hdfs"

    def __init__(
        self,
        host="default",
        port=0,
        user=None,
        kerb_ticket=None,
        driver="libhdfs",
        extra_conf=None,
        **kwargs,
    ):
        """

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        driver: 'libhdfs' or 'libhdfs3'
            Binary driver; libhdfs if the JNI library and default
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        """
        super().__init__(**kwargs)

        self.client = HadoopFileSystem(
            host=host,
            port=port,
            user=user,
            kerb_ticket=kerb_ticket,
            driver=driver,
            extra_conf=extra_conf,
        )
        weakref.finalize(self, lambda: self.client.close())

        self.pars = (host, port, user, kerb_ticket, driver, extra_conf)

    @staticmethod
    def _get_kwargs_from_urls(path):
        ops = infer_storage_options(path)
        out = {}
        if ops.get("host", None):
            out["host"] = ops["host"]
        if ops.get("username", None):
            out["user"] = ops["username"]
        if ops.get("port", None):
            out["port"] = ops["port"]
        return out

    @classmethod
    def _strip_protocol(cls, path):
        ops = infer_storage_options(path)
        return ops["path"]

    def __reduce_ex__(self, protocol):
        return PyArrowHDFS, self.pars

    def close(self):
        self.client.close()

    def ls(self, path, detail=True):
        listing = [
            self._adjust_entry(entry) for entry in self.client.ls(path, detail=True)
        ]

        if detail:
            return listing
        else:
            return [entry["name"] for entry in listing]

    def info(self, path):
        return self._adjust_entry(self.client.info(path))

    def _adjust_entry(self, original_entry):
        entry = original_entry.copy()
        entry["type"] = entry["kind"]
        entry["name"] = self._strip_protocol(entry["path"])
        return entry

    def cp_file(self, lpath, rpath):
        with self.open(lpath) as lstream:
            tmp_fname = "/".join([self._parent(rpath), f".tmp.{secrets.token_hex(16)}"])
            with open(tmp_fname, "w") as rstream:
                shutil.copyfileobj(lstream, rstream)
            self.move(tmp_fname, rpath)
            self.rm(tmp_fname)

    def makedirs(self, path, exist_ok=False):
        if not exist_ok and self.exists(path):
            raise FileExistsError(path)

        return self.client.mkdir(path, create_parents=True)

    def _open(
        self,
        path,
        mode="rb",
        block_size=None,
        autocommit=True,
        cache_options=None,
        **kwargs,
    ):
        """

        Parameters
        ----------
        path: str
            Location of file; should start with '/'
        mode: str
        block_size: int
            Hadoop block size, e.g., 2**26
        autocommit: True
            Transactions are not yet implemented for HDFS; errors if not True
        kwargs: dict or None
            Hadoop config parameters

        Returns
        -------
        HDFSFile file-like instance
        """

        return HDFSFile(
            self,
            path,
            mode,
            block_size=block_size,
            autocommit=autocommit,
            cache_options=cache_options,
            **kwargs,
        )

    def checksum(self, path, env=None, **kwargs):
        # PyArrow doesn't natively support retrieving the
        # checksum, so we have to use hadoop fs

        result = self._run_command(f"checksum {self._as_url(path)}", env=env)
        if result is None:
            return None

        match = CHECKSUM_REGEX.match(result)
        if match is None:
            return None

        return match.group("checksum")

    def _run_command(self, cmd, env=None):
        cmd = "hadoop fs -" + cmd
        if self.user:
            cmd = f"HADOOP_USER_NAME={self.user} " + cmd

        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
        # See https://github.com/iterative/dvc/issues/1197.
        close_fds = os.name != "nt"

        executable = os.getenv("SHELL") if os.name != "nt" else None
        p = subprocess.Popen(
            cmd,
            shell=True,
            close_fds=close_fds,
            executable=executable,
            env=env or os.environ,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        out, err = p.communicate()

        if p.returncode != 0:
            raise subprocess.CalledProcessError(p.returncode, cmd, out, err)
        else:
            return out.decode("utf-8")

    def _as_url(self, path):
        netloc = f"{self.host}:{self.port}"
        if self.user:
            netloc = f"{self.user}@{netloc}"
        return f"hdfs://{netloc}/{path.lstrip('/')}"

class HDFSFile(object):
    """Wrapper around arrow's HdfsFile

    Allows seek beyond EOF and (eventually) commit/discard
    """

    def __init__(
        self,
        fs,
        path,
        mode,
        block_size,
        autocommit=True,
        cache_type="readahead",
        cache_options=None,
        **kwargs,
    ):
        # TODO: Inherit from AbstractBufferedFile?
        if not autocommit:
            raise NotImplementedError(
                "HDFSFile cannot be opened with 'autocommit=False'."
            )

        self.fs = fs
        self.path = path
        self.mode = mode
        self.block_size = block_size
        self.fh = fs.client.open(path, mode, block_size, **kwargs)
        if self.fh.readable():
            self.seek_size = self.size()

    def seek(self, loc, whence=0):
        if whence == 0 and self.readable():
            loc = min(loc, self.seek_size)
        return self.fh.seek(loc, whence)

    def __getattr__(self, item):
        return getattr(self.fh, item)

    def __reduce_ex__(self, protocol):
        return HDFSFile, (self.fs, self.path, self.mode, self.block_size)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
"""Tests for the templateparser module."""
__author__ = 'janklopper@underdark.nl (Jan Klopper)'
__version__ = '0.3'

# Too many public methods
# pylint: disable-msg=R0904

# Standard modules
import unittest

# Unittest target
import templateparser as templateparser

class TemplateParserTags(unittest.TestCase):
  """Tests validity and parsing of simple tags."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testPlainTemplate(self):
    """Templates without tags get returned whole"""
    template = 'Template without any tags'
    parsed_template = self.parser.ParseString(template)
    self.assertEqual(template, parsed_template)
    self.assertTrue(isinstance(parsed_template, templateparser.SafeString))

  def testSingleTagTemplate(self):
    """Templates with basic tags get returned proper"""
    template = 'Template with [single] tag'
    output = 'Template with just one tag'
    self.assertEqual(output, self.parser.ParseString(
        template, single='just one'))

  def testCasedTag(self):
    """Template tags may contain uppercase and lowercase or a mix thereof"""
    template = 'The parser has no trouble with [cAsE] case.'
    output = 'The parser has no trouble with mixed case.'
    self.assertEqual(output, self.parser.ParseString(template, cAsE='mixed'))

  def testUnderscoredTag(self):
    """Template tags may contains underscores as part of their name"""
    template = 'The template may contain [under_scored] tags.'
    output = 'The template may contain underscored tags.'
    self.assertEqual(output, self.parser.ParseString(
        template, under_scored='underscored'))

  def testMultiTagTemplate(self):
    """Templates with multiple (repeating) tags get parsed properly"""
    template = '[adjective] [noun] are better than other [noun].'
    output = 'Beefy cows are better than other cows.'
    self.assertEqual(output, self.parser.ParseString(
        template, noun='cows', adjective='Beefy'))

  def testBrokenTags(self):
    """Empty tags or tags containing whitespace are not actual tags"""
    template = 'This [ is a ] broken [] template, ][, really'
    self.assertEqual(template, self.parser.ParseString(
        template, **{' is a ': 'HORRIBLY', '': ', NASTY'}))

  def testBadCharacterTags(self):
    """Tags with bad characters are not considered tags"""
    bad_chars = """ :~!@#$%^&*()+-={}\|;':",./<>? """
    template = ''.join('[%s] [check]' % char for char in bad_chars)
    output = ''.join('[%s] ..' % char for char in bad_chars)
    replaces = dict((char, 'FAIL') for char in bad_chars)
    replaces['check'] = '..'
    self.assertEqual(output, self.parser.ParseString(template, **replaces))

  def testUnreplacedTag(self):
    """Template tags for which there is no replacement still exist in output"""
    template = 'Template with an [undefined] tag.'
    self.assertEqual(template, self.parser.ParseString(template))

  def testBracketsInsideTag(self):
    """The last opening bracket and first closing bracket are the delimiters"""
    template = 'Template tags may not contain [[spam] [eggs]].'
    output = 'Template tags may not contain [opening or closing brackets].'
    self.assertEqual(output, self.parser.ParseString(
        template, **{'[spam': 'EPIC', 'eggs]': 'FAIL',
                     'spam': 'opening or', 'eggs': 'closing brackets'}))

class TemplateParserIndexedTags(unittest.TestCase):
  """Tests the handling of complex tags (those with attributes/keys/indexes)."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testTemplateMappingKey(self):
    """Template tags can address mappings properly"""
    template = 'This uses a [dictionary:key].'
    output = 'This uses a spoon.'
    self.assertEqual(output, self.parser.ParseString(
        template, dictionary={'key': 'spoon'}))

  def testTemplateIndexing(self):
    """Template tags can access indexed iterables"""
    template = 'Template that grabs the [obj:2] key from the given tuple/list.'
    output = 'Template that grabs the third key from the given tuple/list.'
    numbers = 'first', 'second', 'third'
    self.assertEqual(output, self.parser.ParseString(template, obj=numbers))
    numbers = list(numbers)
    self.assertEqual(output, self.parser.ParseString(template, obj=numbers))

  def testTemplateAttributes(self):
    """Template tags will do attribute lookups, but only if 'by key' fails"""
    class Mapping(dict):
      """A subclass of a dictionary, so we can define attributes on it."""
      NAME = 'attribute'

    template = 'Template used [tag:NAME] lookup.'
    lookup_attr = 'Template used attribute lookup.'
    lookup_dict = 'Template used key (mapping) lookup.'

    mapp = Mapping()
    self.assertEqual(lookup_attr, self.parser.ParseString(template, tag=mapp))
    mapp['NAME'] = 'key (mapping)'
    self.assertEqual(lookup_dict, self.parser.ParseString(template, tag=mapp))

  def testTemplateMissingIndexes(self):
    """Complex tags with missing indexes (:index) will NOT be replaced"""
    class Object(object):
      """A simple object to store an attribute on."""
      NAME = 'Freeman'

    template = 'Hello [titles:1] [names:NAME], how is [names:other] [date:now]?'
    output = 'Hello [titles:1] Freeman, how is [names:other] [date:now]?'
    self.assertEqual(output, self.parser.ParseString(
        template, titles=['Mr'], names=Object(), date={}))

#  def testPerformance(self):
#    """Basic performance test for 2 template replacements"""
#    for i in range(100):
#      parser = templateparser.Parser()
#      template = 'This [obj:foo] is just a quick [bar]'
#      parser['example'] = template
#      for n in xrange(10):
#        parser.Parse('example', obj={'foo':'text'}, bar='hack')

class TemplateParserFunctions(unittest.TestCase):
  """Tests the functions that are performed on replaced tags."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testPipedFunctionUse(self):
    """Piped functions do not break the parser"""
    template = 'This function does [none|raw].'
    output = 'This function does "nothing".'
    self.assertEqual(output, self.parser.ParseString(
        template, none='"nothing"'))

  def testDefaultHtmlEscapeFunction(self):
    """The default function escapes HTML entities, and works properly"""
    template_default = 'This function does [none].'
    template_escape = 'This function does [none|html].'
    output = 'This function does "nothing".'
    self.assertEqual(output, self.parser.ParseString(
        template_default, none='"nothing"'))
    self.assertEqual(output, self.parser.ParseString(
        template_escape, none='"nothing"'))

  def testNoDefaultForSafeString(self):
    first_template = 'Hello doctor [name]'
    second_template = '<assistant> [quote].'
    result = '<assistant> Hello doctor "Who".'
    output_first = self.parser.ParseString(first_template, name='"Who"')
    output_second = self.parser.ParseString(second_template, quote=output_first)
    self.assertEqual(result, output_second)

  def testCustomFunction(self):
    """Custom functions added to the parser work as expected"""
    self.parser.RegisterFunction('twice', lambda x: x + ' ' + x)
    template = 'The following will be stated [again|twice].'
    output = 'The following will be stated twice twice.'
    self.assertEqual(output, self.parser.ParseString(template, again='twice'))

  def testMultipleFunctions(self):
    """Multiple functions can be piped after one another"""
    self.parser.RegisterFunction('len', len)
    self.parser.RegisterFunction('count', lambda x: '%s characters' % x)
    template = 'A replacement processed by two functions: [spam|len|count].'
    output = 'A replacement processed by two functions: 8 characters.'
    self.assertEqual(output, self.parser.ParseString(template, spam='ham&eggs'))

  def testFunctionSeparation(self):
    """Template functions are only called for fragments that require them"""
    fragments_received = []
    def TemplateFunction(string):
      fragments_received.append(string)
      return string

    self.parser.RegisterFunction('x', TemplateFunction)
    template = 'X only has [num|x] call, else it\'s [expletive] [noun|raw].'
    output = 'X only has one call, else it\'s horribly broken.'
    self.assertEqual(output, self.parser.ParseString(
        template, num='one', expletive='horribly', noun='broken'))
    self.assertEqual(1, len(fragments_received))

class TemplateUnicodeBehavior(unittest.TestCase):
  """TemplateParser handles Unicode gracefully."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testUnicodeInput(self):
    """TemplateParser can handle unicode objects on input, converts to utf8"""
    template = 'Underdark Web framework, also known as [name].'
    output = u'Underdark Web framework, also known as \xb5Web.'.encode('utf8')
    name = u'\xb5Web'
    self.assertEqual(output, self.parser.ParseString(template, name=name))

  def testCreoleTemplateParsing(self):
    """The Creole module's return of <unicode> doesn't break the parser"""
    from underdark.libs import creole
    self.parser.RegisterFunction('creole', creole.CreoleToHtml)
    template = 'Creole [expr|creole]!'
    output = 'Creole
rocks \xc2\xb5Web
\n!'
    self.assertEqual(output, self.parser.ParseString(
        template, expr=u'**rocks** \xb5Web'))

  def testTemplateFunctionReturnUnicode(self):
    """Template functions may return unicode objects, they are later encoded"""
    function_result = u'No more \N{BLACK HEART SUIT}'
    def TemplateFunction(_unused):
      return function_result

    self.parser.RegisterFunction('nolove', TemplateFunction)
    template = '[love|nolove]'
    output = function_result.encode('utf8')
    self.assertEqual(output, self.parser.ParseString(template, love='love'))

if __name__ == '__main__':
  unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
<commit_msg>Updated docstrings and some function names for templateparser_test<commit_after>#!/usr/bin/python2.5
"""Tests for the templateparser module."""
__author__ = 'janklopper@underdark.nl (Jan Klopper)'
__version__ = '0.3'

# Too many public methods
# pylint: disable-msg=R0904

# Standard modules
import unittest

# Unittest target
import templateparser as templateparser

class TemplateParserTags(unittest.TestCase):
  """Tests validity and parsing of simple tags."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testPlainTemplate(self):
    """Templates without tags get returned whole"""
    template = 'Template without any tags'
    parsed_template = self.parser.ParseString(template)
    self.assertEqual(template, parsed_template)
    self.assertTrue(isinstance(parsed_template, templateparser.SafeString))

  def testSingleTagTemplate(self):
    """Templates with basic tags get returned proper"""
    template = 'Template with [single] tag'
    output = 'Template with just one tag'
    self.assertEqual(output, self.parser.ParseString(
        template, single='just one'))

  def testCasedTag(self):
    """Template tags may contain uppercase and lowercase or a mix thereof"""
    template = 'The parser has no trouble with [cAsE] case.'
    output = 'The parser has no trouble with mixed case.'
    self.assertEqual(output, self.parser.ParseString(template, cAsE='mixed'))

  def testUnderscoredTag(self):
    """Template tags may contains underscores as part of their name"""
    template = 'The template may contain [under_scored] tags.'
    output = 'The template may contain underscored tags.'
    self.assertEqual(output, self.parser.ParseString(
        template, under_scored='underscored'))

  def testMultiTagTemplate(self):
    """Templates with multiple (repeating) tags get parsed properly"""
    template = '[adjective] [noun] are better than other [noun].'
    output = 'Beefy cows are better than other cows.'
    self.assertEqual(output, self.parser.ParseString(
        template, noun='cows', adjective='Beefy'))

  def testBrokenTags(self):
    """Empty tags or tags containing whitespace are not actual tags"""
    template = 'This [ is a ] broken [] template, ][, really'
    self.assertEqual(template, self.parser.ParseString(
        template, **{' is a ': 'HORRIBLY', '': ', NASTY'}))

  def testBadCharacterTags(self):
    """Tags with bad characters are not considered tags"""
    bad_chars = """ :~!@#$%^&*()+-={}\|;':",./<>? """
    template = ''.join('[%s] [check]' % char for char in bad_chars)
    output = ''.join('[%s] ..' % char for char in bad_chars)
    replaces = dict((char, 'FAIL') for char in bad_chars)
    replaces['check'] = '..'
    self.assertEqual(output, self.parser.ParseString(template, **replaces))

  def testUnreplacedTag(self):
    """Template tags for which there is no replacement still exist in output"""
    template = 'Template with an [undefined] tag.'
    self.assertEqual(template, self.parser.ParseString(template))

  def testBracketsInsideTag(self):
    """The last opening bracket and first closing bracket are the delimiters"""
    template = 'Template tags may not contain [[spam] [eggs]].'
    output = 'Template tags may not contain [opening or closing brackets].'
    self.assertEqual(output, self.parser.ParseString(
        template, **{'[spam': 'EPIC', 'eggs]': 'FAIL',
                     'spam': 'opening or', 'eggs': 'closing brackets'}))

class TemplateParserIndexedTags(unittest.TestCase):
  """Tests the handling of complex tags (those with attributes/keys/indexes)."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testTemplateMappingKey(self):
    """Template tags can address mappings properly"""
    template = 'This uses a [dictionary:key].'
    output = 'This uses a spoon.'
    self.assertEqual(output, self.parser.ParseString(
        template, dictionary={'key': 'spoon'}))

  def testTemplateIndexing(self):
    """Template tags can access indexed iterables"""
    template = 'Template that grabs the [obj:2] key from the given tuple/list.'
    output = 'Template that grabs the third key from the given tuple/list.'
    numbers = 'first', 'second', 'third'
    self.assertEqual(output, self.parser.ParseString(template, obj=numbers))
    numbers = list(numbers)
    self.assertEqual(output, self.parser.ParseString(template, obj=numbers))

  def testTemplateAttributes(self):
    """Template tags will do attribute lookups, but only if 'by key' fails"""
    class Mapping(dict):
      """A subclass of a dictionary, so we can define attributes on it."""
      NAME = 'attribute'

    template = 'Template used [tag:NAME] lookup.'
    lookup_attr = 'Template used attribute lookup.'
    lookup_dict = 'Template used key (mapping) lookup.'

    mapp = Mapping()
    self.assertEqual(lookup_attr, self.parser.ParseString(template, tag=mapp))
    mapp['NAME'] = 'key (mapping)'
    self.assertEqual(lookup_dict, self.parser.ParseString(template, tag=mapp))

  def testTemplateMissingIndexes(self):
    """Complex tags with missing indexes (:index) will NOT be replaced"""
    class Object(object):
      """A simple object to store an attribute on."""
      NAME = 'Freeman'

    template = 'Hello [titles:1] [names:NAME], how is [names:other] [date:now]?'
    output = 'Hello [titles:1] Freeman, how is [names:other] [date:now]?'
    self.assertEqual(output, self.parser.ParseString(
        template, titles=['Mr'], names=Object(), date={}))

#  def testPerformance(self):
#    """Basic performance test for 2 template replacements"""
#    for i in range(100):
#      parser = templateparser.Parser()
#      template = 'This [obj:foo] is just a quick [bar]'
#      parser['example'] = template
#      for n in xrange(10):
#        parser.Parse('example', obj={'foo':'text'}, bar='hack')

class TemplateParserFunctions(unittest.TestCase):
  """Tests the functions that are performed on replaced tags."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testPipedFunctionUse(self):
    """Piped functions do not break the parser"""
    template = 'This function does [none|raw].'
    output = 'This function does "nothing".'
    self.assertEqual(output, self.parser.ParseString(
        template, none='"nothing"'))

  def testDefaultHtmlEscapeFunction(self):
    """The default function escapes HTML entities, and works properly"""
    template_default = 'This function does [none].'
    template_escape = 'This function does [none|html].'
    output = 'This function does "nothing".'
    self.assertEqual(output, self.parser.ParseString(
        template_default, none='"nothing"'))
    self.assertEqual(output, self.parser.ParseString(
        template_escape, none='"nothing"'))

  def testNoDefaultForSafeString(self):
    """SafeString objects are not fed through the default templating function"""
    first_template = 'Hello doctor [name]'
    second_template = '<assistant> [quote].'
    result = '<assistant> Hello doctor "Who".'
    output_first = self.parser.ParseString(first_template, name='"Who"')
    output_second = self.parser.ParseString(second_template, quote=output_first)
    self.assertEqual(result, output_second)

  def testCustomFunction(self):
    """Custom functions added to the parser work as expected"""
    self.parser.RegisterFunction('twice', lambda x: x + ' ' + x)
    template = 'The following will be stated [again|twice].'
    output = 'The following will be stated twice twice.'
    self.assertEqual(output, self.parser.ParseString(template, again='twice'))

  def testMultipleFunctions(self):
    """Multiple functions can be piped after one another"""
    self.parser.RegisterFunction('len', len)
    self.parser.RegisterFunction('count', lambda x: '%s characters' % x)
    template = 'A replacement processed by two functions: [spam|len|count].'
    output = 'A replacement processed by two functions: 8 characters.'
    self.assertEqual(output, self.parser.ParseString(template, spam='ham&eggs'))

  def testFunctionSeparation(self):
    """Template functions are only called for fragments that require them"""
    fragments_received = []
    def CountAndReturn(fragment):
      """Returns the given fragment after adding it to a counter list."""
      fragments_received.append(fragment)
      return fragment

    self.parser.RegisterFunction('x', CountAndReturn)
    template = 'X only has [num|x] call, else it\'s [expletive] [noun|raw].'
    output = 'X only has one call, else it\'s horribly broken.'
    self.assertEqual(output, self.parser.ParseString(
        template, num='one', expletive='horribly', noun='broken'))
    self.assertEqual(1, len(fragments_received))

class TemplateUnicodeBehavior(unittest.TestCase):
  """TemplateParser handles Unicode gracefully."""
  def setUp(self):
    """Sets up a parser instance, as it never changes."""
    self.parser = templateparser.Parser()

  def testUnicodeInput(self):
    """TemplateParser can handle unicode objects on input, converts to utf8"""
    template = 'Underdark Web framework, also known as [name].'
    output = u'Underdark Web framework, also known as \xb5Web.'.encode('utf8')
    name = u'\xb5Web'
    self.assertEqual(output, self.parser.ParseString(template, name=name))

  def testCreoleTemplateParsing(self):
    """The Creole module's return of <unicode> doesn't break the parser"""
    from underdark.libs import creole
    self.parser.RegisterFunction('creole', creole.CreoleToHtml)
    template = 'Creole [expr|creole]!'
    output = 'Creole
rocks \xc2\xb5Web
\n!'
    self.assertEqual(output, self.parser.ParseString(
        template, expr=u'**rocks** \xb5Web'))

  def testTemplateFunctionReturnUnicode(self):
    """Template functions may return unicode objects, they are later encoded"""
    function_result = u'No more \N{BLACK HEART SUIT}'
    def StaticReturn(_fragment):
      """Returns a static string, for any input fragment."""
      return function_result

    self.parser.RegisterFunction('nolove', StaticReturn)
    template = '[love|nolove]'
    output = function_result.encode('utf8')
    self.assertEqual(output, self.parser.ParseString(template, love='love'))

if __name__ == '__main__':
  unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))
<|endoftext|>


<commit_before>#!/usr/bin/python

import glob
import sys

# This script is run from Build/<target>_default.build/$(PX4_BASE)/Firmware/src/systemcmds/topic_listener

# argv[1] must be the full path of the top Firmware dir

raw_messages = glob.glob(sys.argv[1]+"/msg/*.msg")
messages = []
message_elements = []

for index,m in enumerate(raw_messages):
	temp_list_floats = []
	temp_list_uint64 = []
	temp_list_bool = []
	if("actuator_control" not in m and "pwm_input" not in m and "position_setpoint" not in m):
		temp_list = []
		f = open(m,'r')
		for line in f.readlines():
			if(line.split(' ')[0] == "float32"):
				temp_list.append(("float",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif(line.split(' ')[0] == "uint64"):
				temp_list.append(("uint64",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif (line.split(' ')[0] == "bool"):
				temp_list.append(("bool",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif (line.split(' ')[0] == "uint8") and len(line.split('=')) == 1:
				temp_list.append(("uint8",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif ('float32[' in line.split(' ')[0]):
				num_floats = int(line.split(" ")[0].split("[")[1].split("]")[0])
				temp_list.append(("float_array",line.split(' ')[1].split('\t')[0].split('\n')[0],num_floats))

		f.close()
		messages.append(m.split('/')[-1].split('.')[0])
		message_elements.append(temp_list)

num_messages = len(messages);

print
print """

/****************************************************************************
 *
 *   Copyright (c) 2015 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

/**
 * @file topic_listener.cpp, autogenerated by Tools/generate_listener.py
 *
 * Tool for listening to topics when running flight stack on linux.
 */
#include <px4_middleware.h>
#include <px4_app.h>
#include <px4_config.h>
#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include <vector>
#include <cstring>
#include <uORB/uORB.h>
#include <string.h>
#include <stdint.h>
"""
for m in messages:
	print "#include <uORB/topics/%s.h>" % m

print """
extern "C" __EXPORT int listener_main(int argc, char *argv[]);

int listener_main(int argc, char *argv[]) {
	int sub = -1;
	orb_id_t ID;
	if(argc < 3) {
		printf("need at least two arguments: topic name, number of messages to print\\n");
		return 1;
	}
"""
print "\tuint32_t num_msgs = (uint32_t)std::stoi(argv[2],NULL,10);"
print "\tif(strncmp(argv[1],\"%s\",50)== 0) {" % messages[0]
print "\t\tsub = orb_subscribe(ORB_ID(%s));" % messages[0]
print "\t\tID = ORB_ID(%s);" % messages[0]
print "\t\tstruct %s_s container;" % messages[0]
print "\t\tmemset(&container, 0, sizeof(container));"
for index,m in enumerate(messages[1:]):
	print "\t} else if(strncmp(argv[1],\"%s\",50) == 0) {" % m
	print "\t\tsub = orb_subscribe(ORB_ID(%s));" % m
	print "\t\tID = ORB_ID(%s);" % m
	print "\t\tstruct %s_s container;" % m
	print "\t\tmemset(&container, 0, sizeof(container));"
	print "\t\tbool updated;"
	print "\t\tfor(uint32_t i = 0;i<num_msgs;i++) {"
	print "\t\t\torb_check(sub,&updated);"
	print "\t\t\tupdated = true;"
	print "\t\t\tif(updated) {"
	print "\t\t\torb_copy(ID,sub,&container);"
	for item in message_elements[index+1]:
		if item[0] == "float":
			print "\t\t\tprintf(\"%s: %%f\\n \",(double)container.%s);" % (item[1], item[1])
		elif item[0] == "float_array":
			print "\t\t\tprintf(\"%s:\");" % item[1]
			print "\t\t\tfor (int j=0;j<%d;j++) {" % item[2]
			print "\t\t\t\tprintf(\"%%f \",(double)container.%s[j]);" % item[1]
			print "\t\t\t}"
			print "\t\t\tprintf(\"\\n\");"
		elif item[0] == "uint64":
			print "\t\t\tprintf(\"%s: %%lu\\n \",container.%s);" % (item[1], item[1])
		elif item[0] == "uint8":
			print "\t\t\tprintf(\"%s: %%u\\n \",container.%s);" % (item[1], item[1])
		elif item[0] == "bool":
			print "\t\t\tprintf(\"%s: %%s\\n \",container.%s ? \"True\" : \"False\");" % (item[1], item[1])
	print "\t\t\t}"
	print "\t\t}"
print "\t} else {"
print "\t\t printf(\" Topic did not match any known topics\\n\");"
print "\t}"
print("\t return 0;")

print "}"
<commit_msg>POSIX: Fixed uint64_t print in generated topic_listener.cpp<commit_after>#!/usr/bin/python

import glob
import sys

# This script is run from Build/<target>_default.build/$(PX4_BASE)/Firmware/src/systemcmds/topic_listener

# argv[1] must be the full path of the top Firmware dir

raw_messages = glob.glob(sys.argv[1]+"/msg/*.msg")
messages = []
message_elements = []

for index,m in enumerate(raw_messages):
	temp_list_floats = []
	temp_list_uint64 = []
	temp_list_bool = []
	if("actuator_control" not in m and "pwm_input" not in m and "position_setpoint" not in m):
		temp_list = []
		f = open(m,'r')
		for line in f.readlines():
			if(line.split(' ')[0] == "float32"):
				temp_list.append(("float",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif(line.split(' ')[0] == "uint64"):
				temp_list.append(("uint64",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif (line.split(' ')[0] == "bool"):
				temp_list.append(("bool",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif (line.split(' ')[0] == "uint8") and len(line.split('=')) == 1:
				temp_list.append(("uint8",line.split(' ')[1].split('\t')[0].split('\n')[0]))
			elif ('float32[' in line.split(' ')[0]):
				num_floats = int(line.split(" ")[0].split("[")[1].split("]")[0])
				temp_list.append(("float_array",line.split(' ')[1].split('\t')[0].split('\n')[0],num_floats))

		f.close()
		messages.append(m.split('/')[-1].split('.')[0])
		message_elements.append(temp_list)

num_messages = len(messages);

print
print """

/****************************************************************************
 *
 *   Copyright (c) 2015 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

/**
 * @file topic_listener.cpp, autogenerated by Tools/generate_listener.py
 *
 * Tool for listening to topics when running flight stack on linux.
 */
#include <px4_middleware.h>
#include <px4_app.h>
#include <px4_config.h>
#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include <vector>
#include <cstring>
#include <uORB/uORB.h>
#include <string.h>
#include <stdint.h>
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
"""
for m in messages:
	print "#include <uORB/topics/%s.h>" % m

print """
extern "C" __EXPORT int listener_main(int argc, char *argv[]);

int listener_main(int argc, char *argv[]) {
	int sub = -1;
	orb_id_t ID;
	if(argc < 3) {
		printf("need at least two arguments: topic name, number of messages to print\\n");
		return 1;
	}
"""
print "\tuint32_t num_msgs = (uint32_t)std::stoi(argv[2],NULL,10);"
print "\tif(strncmp(argv[1],\"%s\",50)== 0) {" % messages[0]
print "\t\tsub = orb_subscribe(ORB_ID(%s));" % messages[0]
print "\t\tID = ORB_ID(%s);" % messages[0]
print "\t\tstruct %s_s container;" % messages[0]
print "\t\tmemset(&container, 0, sizeof(container));"
for index,m in enumerate(messages[1:]):
	print "\t} else if(strncmp(argv[1],\"%s\",50) == 0) {" % m
	print "\t\tsub = orb_subscribe(ORB_ID(%s));" % m
	print "\t\tID = ORB_ID(%s);" % m
	print "\t\tstruct %s_s container;" % m
	print "\t\tmemset(&container, 0, sizeof(container));"
	print "\t\tbool updated;"
	print "\t\tfor(uint32_t i = 0;i<num_msgs;i++) {"
	print "\t\t\torb_check(sub,&updated);"
	print "\t\t\tupdated = true;"
	print "\t\t\tif(updated) {"
	print "\t\t\torb_copy(ID,sub,&container);"
	for item in message_elements[index+1]:
		if item[0] == "float":
			print "\t\t\tprintf(\"%s: %%f\\n \",(double)container.%s);" % (item[1], item[1])
		elif item[0] == "float_array":
			print "\t\t\tprintf(\"%s:\");" % item[1]
			print "\t\t\tfor (int j=0;j<%d;j++) {" % item[2]
			print "\t\t\t\tprintf(\"%%f \",(double)container.%s[j]);" % item[1]
			print "\t\t\t}"
			print "\t\t\tprintf(\"\\n\");"
		elif item[0] == "uint64":
			print "\t\t\tprintf(\"%s: %%\" PRIu64 \"\\n \",container.%s);" % (item[1], item[1])
		elif item[0] == "uint8":
			print "\t\t\tprintf(\"%s: %%u\\n \",container.%s);" % (item[1], item[1])
		elif item[0] == "bool":
			print "\t\t\tprintf(\"%s: %%s\\n \",container.%s ? \"True\" : \"False\");" % (item[1], item[1])
	print "\t\t\t}"
	print "\t\t}"
print "\t} else {"
print "\t\t printf(\" Topic did not match any known topics\\n\");"
print "\t}"
print("\t return 0;")

print "}"
<|endoftext|>


<commit_before>import os

from django.db import models
from django.core.files.base import ContentFile
from django.conf import settings
from django.db.models.signals import pre_delete
from django.dispatch.dispatcher import receiver
from django.contrib.auth.models import User

try:
    # Django 1.5
    from django.utils.text import slugify
except:
    # Older versions
    from django.template.defaultfilters import slugify

class Project(models.Model):
    name = models.CharField(max_length=200,
                            unique=True)
    admin = models.ForeignKey(User)
    notify_admin = models.BooleanField(default=False)

    def save(self, *args, **kwargs):
        self.name = slugify(self.name)
        super(Project, self).save(*args, **kwargs) # Call the "real" save() method.

    def __unicode__(self):
        return u'%s' % (self.name)

class Feedback(models.Model):
    project = models.ForeignKey(Project)
    page = models.CharField(max_length=2000)
    referrer = models.CharField(max_length=2000)
    user_agent = models.CharField(max_length=2000)
    comment = models.CharField(max_length=2000, null=True)
    html = models.TextField(null=True)
    image = models.TextField(null=True)
    created_date = models.DateTimeField(auto_now = True)

    def __unicode__(self):
        return u'Feedback #%s (%s)' % (self.id, self.project.name)

    class Meta:
        ordering = ('-created_date',)
        permissions = (
            ('readonly_feedback','Readonly Feedback'),
        )

@receiver(pre_delete, sender=Feedback)
def _feedback_delete(sender, instance, **kwargs):
    """ Delete auto-generated screenshot when model instance is deleted """
    screenshot = "%s/%s.jpg" % (os.path.join(settings.STATIC_ROOT, 'screenshots'),
                                 instance.id)
    try:
        os.remove(screenshot)
    except OSError:
        pass
<commit_msg>Enabling custom user model support<commit_after>import os

from django.db import models
from django.core.files.base import ContentFile
from django.conf import settings
from django.db.models.signals import pre_delete
from django.dispatch.dispatcher import receiver

try:
    # Django 1.5
    from django.utils.text import slugify
except:
    # Older versions
    from django.template.defaultfilters import slugify

class Project(models.Model):
    name = models.CharField(max_length=200,
                            unique=True)
    admin = models.ForeignKey(settings.AUTH_USER_MODEL)
    notify_admin = models.BooleanField(default=False)

    def save(self, *args, **kwargs):
        self.name = slugify(self.name)
        super(Project, self).save(*args, **kwargs) # Call the "real" save() method.

    def __unicode__(self):
        return u'%s' % (self.name)

class Feedback(models.Model):
    project = models.ForeignKey(Project)
    page = models.CharField(max_length=2000)
    referrer = models.CharField(max_length=2000)
    user_agent = models.CharField(max_length=2000)
    comment = models.CharField(max_length=2000, null=True)
    html = models.TextField(null=True)
    image = models.TextField(null=True)
    created_date = models.DateTimeField(auto_now = True)

    def __unicode__(self):
        return u'Feedback #%s (%s)' % (self.id, self.project.name)

    class Meta:
        ordering = ('-created_date',)
        permissions = (
            ('readonly_feedback','Readonly Feedback'),
        )

@receiver(pre_delete, sender=Feedback)
def _feedback_delete(sender, instance, **kwargs):
    """ Delete auto-generated screenshot when model instance is deleted """
    screenshot = "%s/%s.jpg" % (os.path.join(settings.STATIC_ROOT, 'screenshots'),
                                 instance.id)
    try:
        os.remove(screenshot)
    except OSError:
        pass
<|endoftext|>


<commit_before>"""
Serilizers for the Agency application API
"""

from rest_framework import serializers

from muckrock.agency.models import Agency

# pylint: disable=too-few-public-methods

class AgencySerializer(serializers.ModelSerializer):
    """Serializer for Agency model"""
    types = serializers.StringRelatedField(many=True)
    appeal_agency = serializers.PrimaryKeyRelatedField(
            queryset=Agency.objects.all(),
            style={'base_template': 'input.html'})
    parent = serializers.PrimaryKeyRelatedField(
            queryset=Agency.objects.all(),
            style={'base_template': 'input.html'})

    def __init__(self, *args, **kwargs):
        """After initializing the serializer,
        check that the current user has permission
        to view agency email data."""
        # pylint: disable=super-on-old-class
        super(AgencySerializer, self).__init__(*args, **kwargs)
        request = self.context.get('request', None)
        if request is None or not request.user.is_staff:
            self.fields.pop('email')
            self.fields.pop('other_emails')

    class Meta:
        model = Agency
        fields = (
            # describes agency
            'id',
            'name',
            'slug',
            'status',
            'stale',
            'exempt',
            'types',
            # location
            'jurisdiction',
            'address',
            'location',
            # contact info
            'email',
            'other_emails',
            'phone',
            'fax',
            'website',
            'twitter',
            'twitter_handles',
            # connects to other agencies
            'parent',
            'appeal_agency',
            'can_email_appeals',
            # describes agency foia process
            'url',
            'foia_logs',
            'foia_guide',
            # misc
            'public_notes',
        )

<commit_msg>adds location JSONField to agency serializer<commit_after>"""
Serilizers for the Agency application API
"""

from rest_framework import serializers

from muckrock.agency.models import Agency

# pylint: disable=too-few-public-methods

class AgencySerializer(serializers.ModelSerializer):
    """Serializer for Agency model"""
    types = serializers.StringRelatedField(many=True)
    appeal_agency = serializers.PrimaryKeyRelatedField(
            queryset=Agency.objects.all(),
            style={'base_template': 'input.html'})
    parent = serializers.PrimaryKeyRelatedField(
            queryset=Agency.objects.all(),
            style={'base_template': 'input.html'})
    location = serializers.JSONField()

    def __init__(self, *args, **kwargs):
        """After initializing the serializer,
        check that the current user has permission
        to view agency email data."""
        # pylint: disable=super-on-old-class
        super(AgencySerializer, self).__init__(*args, **kwargs)
        request = self.context.get('request', None)
        if request is None or not request.user.is_staff:
            self.fields.pop('email')
            self.fields.pop('other_emails')

    class Meta:
        model = Agency
        fields = (
            # describes agency
            'id',
            'name',
            'slug',
            'status',
            'stale',
            'exempt',
            'types',
            # location
            'jurisdiction',
            'address',
            'location',
            # contact info
            'email',
            'other_emails',
            'phone',
            'fax',
            'website',
            'twitter',
            'twitter_handles',
            # connects to other agencies
            'parent',
            'appeal_agency',
            'can_email_appeals',
            # describes agency foia process
            'url',
            'foia_logs',
            'foia_guide',
            # misc
            'public_notes',
        )
<|endoftext|>