<commit_before>#!/usr/bin/env python
"""Tests the scriptTree jobTree-script compiler.
"""

import unittest
import os
import random
from uuid import uuid4
import logging
import shutil
import tempfile
from jobTree.target import Target
from jobTree.lib.bioio import getLogLevelString
from jobTree.batchSystems.mesos.test import MesosTestSupport
from jobTree.common import parasolIsInstalled, gridEngineIsInstalled
from jobTree.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint
from jobTree.test import JobTreeTest

log = logging.getLogger(__name__)

class SortTest(JobTreeTest, MesosTestSupport):

    def setUp(self):
        super( SortTest, self).setUp()
        self.jobTreeDir = os.path.join(os.getcwd(), "testJobTree") #A directory for the job tree to be created in
        self.tempDir = tempfile.mkdtemp(prefix="tempDir", dir=os.getcwd())
        self.testNo = 5

    def tearDown(self):
        super( SortTest, self).tearDown( )
        if os.path.exists(self.jobTreeDir):
            shutil.rmtree(self.jobTreeDir)
        if os.path.exists(self.tempDir):
            shutil.rmtree(self.tempDir)

    def jobTreeSortTest(self, testNo=1, batchSystem="singleMachine", jobStore='file',
                lines=10000, maxLineLength=10, N=10000):
        """
        Tests jobTree by sorting a file in parallel.
        """
        for test in xrange(testNo):
            options = Target.Runner.getDefaultOptions()

            #Job tree
            if jobStore == 'file':
                options.jobTree = self.jobTreeDir
            else:
                options.jobTree = jobStore

            # Specify options
            options.logLevel = getLogLevelString()
            options.retryCount = 32

            #Make the file to sort
            tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
            makeFileToSort(tempSortFile, lines=lines, maxLineLength=maxLineLength)

            # First make our own sorted version
            with open(tempSortFile, 'r') as fileHandle:
                l = fileHandle.readlines()
                l.sort()

            # Run the jobTree
            firstTarget = Target.wrapTargetFn(setup, tempSortFile, N)
            i = Target.Runner.startJobTree(firstTarget, options)

            #There should be no failed jobs
            self.assertEquals(i, 0)

            # Now check the file is properly sorted..
            # Now get the sorted file
            with open(tempSortFile, 'r') as fileHandle:
                l2 = fileHandle.readlines()
                checkEqual(l, l2)

    def testJobTreeSortOnAWS(self):
        """Tests scriptTree/jobTree by sorting a file in parallel.
        """
        self.jobTreeSortTest(jobStore="aws:us-west-2:sort-test-%s" % uuid4(),
                      lines=100, N=100)

    def testScriptTree_SortSimpleOnAWSWithMesos(self):
        self._startMesos()
        try:
            self.scriptTree_SortTest(testNo=1,
                                     batchSystem="mesos",
                                     jobStore="aws:us-west-2:sort-test-%s" % uuid4(),
                                     lines=100,
                                     N=100)
        finally:
            self._stopMesos()

    def testJobTreeSort(self):
        """Tests scriptTree/jobTree by sorting a file in parallel.
        """
        self.jobTreeSortTest()

    # The following functions test the functions in the test!

    def testSort(self):
        for test in xrange(self.testNo):
            tempFile1 = os.path.join(self.tempDir, "fileToSort.txt")
            makeFileToSort(tempFile1)
            lines1 = loadFile(tempFile1)
            lines1.sort()
            sort(tempFile1)
            with open(tempFile1, 'r') as f:
                lines2 = f.readlines()
            checkEqual(lines1, lines2)

    def testMerge(self):
        for test in xrange(self.testNo):
            tempFile1 = os.path.join(self.tempDir, "fileToSort1.txt")
            tempFile2 = os.path.join(self.tempDir, "fileToSort2.txt")
            tempFile3 = os.path.join(self.tempDir, "mergedFile.txt")
            makeFileToSort(tempFile1)
            makeFileToSort(tempFile2)
            sort(tempFile1)
            sort(tempFile2)
            with open(tempFile3, 'w') as fileHandle:
                with open(tempFile1) as tempFileHandle1:
                    with open(tempFile2) as tempFileHandle2:
                        merge(tempFileHandle1, tempFileHandle2, fileHandle)
            lines1 = loadFile(tempFile1) + loadFile(tempFile2)
            lines1.sort()
            with open(tempFile3, 'r') as f:
                lines2 = f.readlines()
            checkEqual(lines1, lines2)

    def testCopySubRangeOfFile(self):
        for test in xrange(self.testNo):
            tempFile = os.path.join(self.tempDir, "fileToSort1.txt")
            outputFile = os.path.join(self.tempDir, "outputFileToSort1.txt")
            makeFileToSort(tempFile)
            fileSize = os.path.getsize(tempFile)
            assert fileSize > 0
            fileStart = random.choice(xrange(0, fileSize))
            fileEnd = random.choice(xrange(fileStart, fileSize))
            fileHandle = open(outputFile, 'w')
            copySubRangeOfFile(tempFile, fileStart, fileEnd, fileHandle)
            fileHandle.close()
            l = open(outputFile, 'r').read()
            l2 = open(tempFile, 'r').read()[fileStart:fileEnd]
            checkEqual(l, l2)

    def testGetMidPoint(self):
        for test in xrange(self.testNo):
            tempFile = os.path.join(self.tempDir, "fileToSort.txt")
            makeFileToSort(tempFile)
            l = open(tempFile, 'r').read()
            fileSize = os.path.getsize(tempFile)
            midPoint = getMidPoint(tempFile, 0, fileSize)
            print "the mid point is %i of a file of %i bytes" % (midPoint, fileSize)
            assert midPoint < fileSize
            assert l[midPoint] == '\n'
            assert midPoint >= 0

###########################################
#Functions to generate file store and check result is okay
###########################################

def checkEqual(i, j):
    if i != j:
        print "lengths", len(i), len(j)
        print "true", i
        print "false", j
    assert i == j

def loadFile(file):
    with open(file, 'r') as fileHandle:
        return fileHandle.readlines()

def getRandomLine(maxLineLength):
    return "".join([random.choice(['a', 'c', 't', 'g', "A", "C", "G", "T", "N", "X", "Y", "Z"]) for i in
                    xrange(maxLineLength)]) + "\n"

def makeFileToSort(fileName, lines=10, maxLineLength=10):
    with open(fileName, 'w') as fileHandle:
        for line in xrange(lines):
            fileHandle.write(getRandomLine(maxLineLength))

###########################################
#Target functions
###########################################

success_ratio = 0.5

def setup(target, inputFile, N):
    """Sets up the sort.
    """
    tempOutputFileStoreID = target.fileStore.getEmptyFileStoreID()
    target.addChildTargetFn(down, inputFile, 0, os.path.getsize(inputFile), N, tempOutputFileStoreID)
    target.addFollowOnTargetFn(cleanup, tempOutputFileStoreID, inputFile)

def down(target, inputFile, fileStart, fileEnd, N, outputFileStoreID):
    """Input is a file and a range into that file to sort and an output location in which
    to write the sorted file.
    If the range is larger than a threshold N the range is divided recursively and
    a follow on job is then created which merges back the results else
    the file is sorted and placed in the output.
    """
    if random.random() > success_ratio:
        raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
    length = fileEnd - fileStart
    assert length >= 0
    if length > N:
        target.fileStore.logToMaster( "Splitting range (%i..%i) of file: %s"
                                      % (fileStart, fileEnd, inputFile) )
        midPoint = getMidPoint(inputFile, fileStart, fileEnd)
        assert midPoint >= fileStart
        assert midPoint+1 < fileEnd
        #We will subdivide the file
        tempFileStoreID1 = target.fileStore.getEmptyFileStoreID()
        tempFileStoreID2 = target.fileStore.getEmptyFileStoreID()
        #The use of rv here is for testing purposes
        #The rv(0) of the first child target is tempFileStoreID1,
        #similarly rv(0) of the second child is tempFileStoreID2
        target.addFollowOnTargetFn(up,
                                   target.addChildTargetFn(down, inputFile, fileStart,
                                                           midPoint+1, N, tempFileStoreID1).rv(0),
                                   target.addChildTargetFn(down, inputFile, midPoint+1,
                                                           fileEnd, N, tempFileStoreID2).rv(0), #Add one to avoid the newline
                                   outputFileStoreID)
    else:
        #We can sort this bit of the file
        target.fileStore.logToMaster( "Sorting range (%i..%i) of file: %s"
                                      % (fileStart, fileEnd, inputFile) )
        with target.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
            copySubRangeOfFile(inputFile, fileStart, fileEnd, fileHandle)
        #Make a local copy and sort the file
        tempOutputFile = target.fileStore.readGlobalFile(outputFileStoreID)
        sort(tempOutputFile)
        target.fileStore.updateGlobalFile(outputFileStoreID, tempOutputFile)
    return outputFileStoreID

def up(target, inputFileID1, inputFileID2, outputFileStoreID):
    """Merges the two files and places them in the output.
    """
    if random.random() > success_ratio:
        raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
    with target.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
        with target.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
            with target.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
                merge(inputFileHandle1, inputFileHandle2, fileHandle)
    target.fileStore.logToMaster( "Merging %s and %s to %s"
                                  % (inputFileID1, inputFileID2, outputFileStoreID) )

def cleanup(target, tempOutputFileStoreID, outputFile):
    """Copies back the temporary file to input once we've successfully sorted the temporary file.
    """
    if random.random() > success_ratio:
        raise RuntimeError() #This is a test error and not a failure of the tests
    target.fileStore.readGlobalFile(tempOutputFileStoreID, outputFile)
    #sort(outputFile)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Small syntax changes, fixes sortTest on AWS with Mesos on local machine<commit_after>#!/usr/bin/env python
"""Tests the scriptTree jobTree-script compiler.
"""

import unittest
import os
import random
from uuid import uuid4
import logging
import shutil
import tempfile
from jobTree.target import Target
from jobTree.lib.bioio import getLogLevelString
from jobTree.batchSystems.mesos.test import MesosTestSupport
from jobTree.common import parasolIsInstalled, gridEngineIsInstalled
from jobTree.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint
from jobTree.test import JobTreeTest

log = logging.getLogger(__name__)

class SortTest(JobTreeTest, MesosTestSupport):

    def setUp(self):
        super( SortTest, self).setUp()
        self.jobTreeDir = os.path.join(os.getcwd(), "testJobTree") #A directory for the job tree to be created in
        self.tempDir = tempfile.mkdtemp(prefix="tempDir", dir=os.getcwd())
        self.testNo = 5

    def tearDown(self):
        super( SortTest, self).tearDown( )
        if os.path.exists(self.jobTreeDir):
            shutil.rmtree(self.jobTreeDir)
        if os.path.exists(self.tempDir):
            shutil.rmtree(self.tempDir)

    def jobTreeSortTest(self, testNo=1, batchSystem="singleMachine", jobStore='file',
                lines=10000, maxLineLength=10, N=10000):
        """
        Tests jobTree by sorting a file in parallel.
        """
        for test in xrange(testNo):
            options = Target.Runner.getDefaultOptions()

            #Job tree
            if jobStore == 'file':
                options.jobTree = self.jobTreeDir
            else:
                options.jobTree = jobStore

            # Specify options
            options.logLevel = getLogLevelString()
            options.retryCount = 32

            options.batchSystem = batchSystem

            #Make the file to sort
            tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
            makeFileToSort(tempSortFile, lines=lines, maxLineLength=maxLineLength)

            # First make our own sorted version
            with open(tempSortFile, 'r') as fileHandle:
                l = fileHandle.readlines()
                l.sort()

            # Run the jobTree
            firstTarget = Target.wrapTargetFn(setup, tempSortFile, N)
            i = Target.Runner.startJobTree(firstTarget, options)

            #There should be no failed jobs
            self.assertEquals(i, 0)

            # Now check the file is properly sorted..
            # Now get the sorted file
            with open(tempSortFile, 'r') as fileHandle:
                l2 = fileHandle.readlines()
                checkEqual(l, l2)

    def testJobTreeSortOnAWS(self):
        """Tests scriptTree/jobTree by sorting a file in parallel.
        """
        self.jobTreeSortTest(jobStore="aws:us-west-2:sort-test-%s" % uuid4(),
                      lines=100, N=100)

    def testScriptTree_SortSimpleOnAWSWithMesos(self):
        self._startMesos()
        try:
            self.jobTreeSortTest(testNo=1,
                                     batchSystem="mesos",
                                     jobStore="aws:us-west-2:sort-test-%s" % uuid4(),
                                     lines=100,
                                     N=100)
        finally:
            self._stopMesos()

    def testJobTreeSort(self):
        """Tests scriptTree/jobTree by sorting a file in parallel.
        """
        self.jobTreeSortTest()

    # The following functions test the functions in the test!

    def testSort(self):
        for test in xrange(self.testNo):
            tempFile1 = os.path.join(self.tempDir, "fileToSort.txt")
            makeFileToSort(tempFile1)
            lines1 = loadFile(tempFile1)
            lines1.sort()
            sort(tempFile1)
            with open(tempFile1, 'r') as f:
                lines2 = f.readlines()
            checkEqual(lines1, lines2)

    def testMerge(self):
        for test in xrange(self.testNo):
            tempFile1 = os.path.join(self.tempDir, "fileToSort1.txt")
            tempFile2 = os.path.join(self.tempDir, "fileToSort2.txt")
            tempFile3 = os.path.join(self.tempDir, "mergedFile.txt")
            makeFileToSort(tempFile1)
            makeFileToSort(tempFile2)
            sort(tempFile1)
            sort(tempFile2)
            with open(tempFile3, 'w') as fileHandle:
                with open(tempFile1) as tempFileHandle1:
                    with open(tempFile2) as tempFileHandle2:
                        merge(tempFileHandle1, tempFileHandle2, fileHandle)
            lines1 = loadFile(tempFile1) + loadFile(tempFile2)
            lines1.sort()
            with open(tempFile3, 'r') as f:
                lines2 = f.readlines()
            checkEqual(lines1, lines2)

    def testCopySubRangeOfFile(self):
        for test in xrange(self.testNo):
            tempFile = os.path.join(self.tempDir, "fileToSort1.txt")
            outputFile = os.path.join(self.tempDir, "outputFileToSort1.txt")
            makeFileToSort(tempFile)
            fileSize = os.path.getsize(tempFile)
            assert fileSize > 0
            fileStart = random.choice(xrange(0, fileSize))
            fileEnd = random.choice(xrange(fileStart, fileSize))
            fileHandle = open(outputFile, 'w')
            copySubRangeOfFile(tempFile, fileStart, fileEnd, fileHandle)
            fileHandle.close()
            l = open(outputFile, 'r').read()
            l2 = open(tempFile, 'r').read()[fileStart:fileEnd]
            checkEqual(l, l2)

    def testGetMidPoint(self):
        for test in xrange(self.testNo):
            tempFile = os.path.join(self.tempDir, "fileToSort.txt")
            makeFileToSort(tempFile)
            l = open(tempFile, 'r').read()
            fileSize = os.path.getsize(tempFile)
            midPoint = getMidPoint(tempFile, 0, fileSize)
            print "the mid point is %i of a file of %i bytes" % (midPoint, fileSize)
            assert midPoint < fileSize
            assert l[midPoint] == '\n'
            assert midPoint >= 0

###########################################
#Functions to generate file store and check result is okay
###########################################

def checkEqual(i, j):
    if i != j:
        print "lengths", len(i), len(j)
        print "true", i
        print "false", j
    assert i == j

def loadFile(file):
    with open(file, 'r') as fileHandle:
        return fileHandle.readlines()

def getRandomLine(maxLineLength):
    return "".join([random.choice(['a', 'c', 't', 'g', "A", "C", "G", "T", "N", "X", "Y", "Z"]) for i in
                    xrange(maxLineLength)]) + "\n"

def makeFileToSort(fileName, lines=10, maxLineLength=10):
    with open(fileName, 'w') as fileHandle:
        for line in xrange(lines):
            fileHandle.write(getRandomLine(maxLineLength))

###########################################
#Target functions
###########################################

success_ratio = 0.5

def setup(target, inputFile, N):
    """Sets up the sort.
    """
    tempOutputFileStoreID = target.fileStore.getEmptyFileStoreID()
    target.addChildTargetFn(down, inputFile, 0, os.path.getsize(inputFile), N, tempOutputFileStoreID)
    target.addFollowOnTargetFn(cleanup, tempOutputFileStoreID, inputFile)

def down(target, inputFile, fileStart, fileEnd, N, outputFileStoreID):
    """Input is a file and a range into that file to sort and an output location in which
    to write the sorted file.
    If the range is larger than a threshold N the range is divided recursively and
    a follow on job is then created which merges back the results else
    the file is sorted and placed in the output.
    """
    if random.random() > success_ratio:
        raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
    length = fileEnd - fileStart
    assert length >= 0
    if length > N:
        target.fileStore.logToMaster( "Splitting range (%i..%i) of file: %s"
                                      % (fileStart, fileEnd, inputFile) )
        midPoint = getMidPoint(inputFile, fileStart, fileEnd)
        assert midPoint >= fileStart
        assert midPoint+1 < fileEnd
        #We will subdivide the file
        tempFileStoreID1 = target.fileStore.getEmptyFileStoreID()
        tempFileStoreID2 = target.fileStore.getEmptyFileStoreID()
        #The use of rv here is for testing purposes
        #The rv(0) of the first child target is tempFileStoreID1,
        #similarly rv(0) of the second child is tempFileStoreID2
        target.addFollowOnTargetFn(up,
                                   target.addChildTargetFn(down, inputFile, fileStart,
                                                           midPoint+1, N, tempFileStoreID1).rv(0),
                                   target.addChildTargetFn(down, inputFile, midPoint+1,
                                                           fileEnd, N, tempFileStoreID2).rv(0), #Add one to avoid the newline
                                   outputFileStoreID)
    else:
        #We can sort this bit of the file
        target.fileStore.logToMaster( "Sorting range (%i..%i) of file: %s"
                                      % (fileStart, fileEnd, inputFile) )
        with target.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
            copySubRangeOfFile(inputFile, fileStart, fileEnd, fileHandle)
        #Make a local copy and sort the file
        tempOutputFile = target.fileStore.readGlobalFile(outputFileStoreID)
        sort(tempOutputFile)
        target.fileStore.updateGlobalFile(outputFileStoreID, tempOutputFile)
    return outputFileStoreID

def up(target, inputFileID1, inputFileID2, outputFileStoreID):
    """Merges the two files and places them in the output.
    """
    if random.random() > success_ratio:
        raise RuntimeError() #This error is a test error, it does not mean the tests have failed.
    with target.fileStore.updateGlobalFileStream(outputFileStoreID) as fileHandle:
        with target.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
            with target.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
                merge(inputFileHandle1, inputFileHandle2, fileHandle)
    target.fileStore.logToMaster( "Merging %s and %s to %s"
                                  % (inputFileID1, inputFileID2, outputFileStoreID) )

def cleanup(target, tempOutputFileStoreID, outputFile):
    """Copies back the temporary file to input once we've successfully sorted the temporary file.
    """
    if random.random() > success_ratio:
        raise RuntimeError() #This is a test error and not a failure of the tests
    target.fileStore.readGlobalFile(tempOutputFileStoreID, outputFile)
    #sort(outputFile)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from taggit.utils import edit_string_for_tags, parse_tags
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import Authorization
from django.template.defaultfilters import date
from tastypie.resources import ModelResource
from django.contrib.auth.models import User
from tastypie.validation import Validation
from tastypie.models import create_api_key
from snipts.models import Favorite, Snipt
from haystack.query import SearchQuerySet
from tastypie.cache import SimpleCache
from tastypie.fields import ListField
from taggit.models import Tag
from django.db import models
from tastypie import fields

import datetime, hashlib, time

import parsedatetime.parsedatetime as pdt
import parsedatetime.parsedatetime_consts as pdc

models.signals.post_save.connect(create_api_key, sender=User)

class FavoriteValidation(Validation):
    def is_valid(self, bundle, request=None):
        errors = {}
        snipt = bundle.data['snipt']

        if Favorite.objects.filter(user=request.user, snipt=snipt).count():
            errors['duplicate'] = 'User has already favorited this snipt.'

        return errors

class PublicUserResource(ModelResource):
    class Meta:
        queryset = User.objects.all()
        resource_name = 'user'
        fields = ['id', 'username',]
        include_absolute_url = True
        allowed_methods = ['get']
        filtering = { 'username': 'exact', }
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['snipts'] = '/api/public/snipt/?user=%d' % bundle.obj.id
        return bundle

class PublicTagResource(ModelResource):
    class Meta:
        queryset = Tag.objects.filter(snipt__public=True)
        queryset = queryset.annotate(count=models.Count('taggit_taggeditem_items__id'))
        queryset = queryset.order_by('-count', 'name')
        resource_name = 'tag'
        fields = ['id', 'name',]
        allowed_methods = ['get']
        max_limit = 200
        cache = SimpleCache()

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PublicTagResource, self).build_filters(filters)

        if 'q' in filters:
            orm_filters['slug'] = filters['q']

        return orm_filters

    def dehydrate(self, bundle):
        bundle.data['absolute_url'] = '/public/tag/%s/' % bundle.obj.slug
        bundle.data['snipts'] = '/api/public/snipt/?tag=%d' % bundle.obj.id
        bundle.data['count'] = bundle.obj.taggit_taggeditem_items.filter(
                               snipt__public=True).count()
        return bundle

class PublicSniptResource(ModelResource):
    user = fields.ForeignKey(PublicUserResource, 'user', full=True)
    tags = fields.ToManyField(PublicTagResource, 'tags', related_name='tag', full=True)

    class Meta:
        queryset = Snipt.objects.filter(public=True).order_by('-created')
        resource_name = 'snipt'
        fields = ['id', 'title', 'slug', 'lexer', 'code', 'line_count',
                  'stylized', 'created', 'modified',]
        include_absolute_url = True
        allowed_methods = ['get']
        filtering = { 'user': 'exact', }
        ordering = ['created', 'modified',]
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['embed_url'] = bundle.obj.get_embed_url()
        bundle.data['full_absolute_url'] = bundle.obj.get_full_absolute_url()
        return bundle

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PublicSniptResource, self).build_filters(filters)

        if 'tag' in filters:
            tag = Tag.objects.get(pk=filters['tag'])
            tagged_items = tag.taggit_taggeditem_items.all()
            orm_filters['pk__in'] = [i.object_id for i in tagged_items]

        if 'q' in filters:
            sqs = SearchQuerySet().auto_query(filters['q'])
            orm_filters['pk__in'] = [i.pk for i in sqs]

        return orm_filters

class PrivateUserResource(ModelResource):
    class Meta:
        queryset = User.objects.all()
        resource_name = 'user'
        fields = ['id', 'username', 'email',]
        include_absolute_url = True
        allowed_methods = ['get']
        list_allowed_methods = []
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def apply_authorization_limits(self, request, object_list):
        return object_list.filter(username=request.user.username)

    def dehydrate(self, bundle):
        bundle.data['email_md5'] = hashlib.md5(bundle.obj.email.lower()).hexdigest()
        bundle.data['profile'] = {
            'is_pro': bundle.obj.profile.is_pro
        }
        return bundle

class PrivateTagResource(ModelResource):
    class Meta:
        queryset = Tag.objects.all()
        resource_name = 'tag'
        fields = ['id', 'name',]
        allowed_methods = ['get']
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PrivateTagResource, self).build_filters(filters)

        if 'q' in filters:
            orm_filters['slug'] = filters['q']

        return orm_filters
    def dehydrate(self, bundle):
        bundle.data['absolute_url'] = '/%s/tag/%s/' % (bundle.request.user.username,
                                                       bundle.obj.slug)
        bundle.data['snipts'] = '/api/private/snipt/?tag=%d' % bundle.obj.id

        bundle.data['count'] = bundle.obj.taggit_taggeditem_items.filter(
                                       snipt__user=bundle.request.user
                                   ).count()

        return bundle

    def apply_authorization_limits(self, request, object_list):
        object_list = object_list.filter(snipt__user=request.user)
        object_list = object_list.annotate(count=models.Count('taggit_taggeditem_items__id'))
        object_list = object_list.order_by('-count', 'name')
        return object_list

class PrivateSniptResource(ModelResource):
    user = fields.ForeignKey(PrivateUserResource, 'user', full=True)
    tags = fields.ToManyField(PrivateTagResource, 'tags', related_name='tag', full=True)
    tags_list = ListField()

    class Meta:
        queryset = Snipt.objects.all().order_by('-created')
        resource_name = 'snipt'
        fields = ['id', 'title', 'slug', 'lexer', 'code', 'line_count', 'stylized',
                  'key', 'public', 'blog_post', 'created', 'modified', 'publish_date',]
        validation = Validation()
        include_absolute_url = True
        detail_allowed_methods = ['get', 'patch', 'put', 'delete']
        list_allowed_methods = ['get', 'post']
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        ordering = ['created', 'modified',]
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['embed_url'] = bundle.obj.get_embed_url()
        bundle.data['tags_list'] = edit_string_for_tags(bundle.obj.tags.all())
        bundle.data['full_absolute_url'] = bundle.obj.get_full_absolute_url()

        if bundle.data['user'].data['profile']['is_pro']:
            bundle.data['views'] = bundle.obj.views
            bundle.data['favs'] = bundle.obj.favs()

        if bundle.data['publish_date']:
            bundle.data['publish_date'] = date(bundle.data['publish_date'], 'M d, Y \\a\\t h:i A')

        return bundle

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.data['tags_list'] = bundle.data.get('tags')
        bundle.data['tags'] = ''

        if 'blog_post' in bundle.data:
            bundle = self._clean_publish_date(bundle)

        return super(PrivateSniptResource, self).obj_create(bundle, request,
                     user=request.user, **kwargs)

    def obj_update(self, bundle, request=None, **kwargs):
        bundle.data['user'] = request.user

        if type(bundle.data['tags']) in (str, unicode):
            bundle.data['tags_list'] = bundle.data['tags']
        else:
            bundle.data['tags_list'] = ''
        bundle.data['tags'] = ''

        if 'blog_post' in bundle.data:
            bundle = self._clean_publish_date(bundle)

        return super(PrivateSniptResource, self).obj_update(bundle, request,
                     user=request.user, **kwargs)

    def _clean_publish_date(self, bundle):
        if bundle.data['blog_post'] and 'publish_date' not in bundle.data:
            bundle.data['publish_date'] = datetime.datetime.now()
        elif bundle.data['publish_date'] == '':
            bundle.data['publish_date'] = datetime.datetime.now()
        elif bundle.data['blog_post']:
            c = pdc.Constants()
            p = pdt.Calendar(c)
            publish_date, result = p.parse(bundle.data['publish_date'])

            if result != 0:
                publish_date = time.strftime("%Y-%m-%d %H:%M:%S", publish_date)
            else:
                publish_date = datetime.datetime.now()

            bundle.data['publish_date'] = publish_date
        elif not bundle.data['blog_post']:
            bundle.data['publish_date'] = None

        return bundle

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PrivateSniptResource, self).build_filters(filters)

        if 'tag' in filters:
            tag = Tag.objects.get(pk=filters['tag'])
            tagged_items = tag.taggit_taggeditem_items.all()
            orm_filters['pk__in'] = [i.object_id for i in tagged_items]

        if 'q' in filters:
            sqs = SearchQuerySet().auto_query(filters['q'])
            orm_filters['pk__in'] = [i.pk for i in sqs]

        return orm_filters

    def apply_authorization_limits(self, request, object_list):
        return object_list.filter(user=request.user)

    def save_m2m(self, bundle):
        tags = bundle.data.get('tags_list', [])
        if tags != '':
            bundle.obj.tags.set(*parse_tags(tags))
        else:
            bundle.obj.tags.set()

class PrivateFavoriteResource(ModelResource):
    user = fields.ForeignKey(PrivateUserResource, 'user', full=True)
    snipt = fields.ForeignKey(PrivateSniptResource, 'snipt', full=False)

    class Meta:
        queryset = Favorite.objects.all().order_by('-created')
        resource_name = 'favorite'
        fields = ['id',]
        validation = FavoriteValidation()
        detail_allowed_methods = ['get', 'post', 'delete']
        list_allowed_methods = ['get', 'post',]
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        ordering = ['created',]
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.data['user'] = request.user
        bundle.data['snipt'] = Snipt.objects.get(pk=bundle.data['snipt'])
        return super(PrivateFavoriteResource, self).obj_create(bundle, request,
                     user=request.user, **kwargs)

    def apply_authorization_limits(self, request, object_list):
        return object_list.filter(user=request.user)
<commit_msg>Return a public snipt URL for favorited resources.<commit_after>from taggit.utils import edit_string_for_tags, parse_tags
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import Authorization
from django.template.defaultfilters import date
from tastypie.resources import ModelResource
from django.contrib.auth.models import User
from tastypie.validation import Validation
from tastypie.models import create_api_key
from snipts.models import Favorite, Snipt
from haystack.query import SearchQuerySet
from tastypie.cache import SimpleCache
from tastypie.fields import ListField
from taggit.models import Tag
from django.db import models
from tastypie import fields

import datetime, hashlib, time

import parsedatetime.parsedatetime as pdt
import parsedatetime.parsedatetime_consts as pdc

models.signals.post_save.connect(create_api_key, sender=User)

class FavoriteValidation(Validation):
    def is_valid(self, bundle, request=None):
        errors = {}
        snipt = bundle.data['snipt']

        if Favorite.objects.filter(user=request.user, snipt=snipt).count():
            errors['duplicate'] = 'User has already favorited this snipt.'

        return errors

class PublicUserResource(ModelResource):
    class Meta:
        queryset = User.objects.all()
        resource_name = 'user'
        fields = ['id', 'username',]
        include_absolute_url = True
        allowed_methods = ['get']
        filtering = { 'username': 'exact', }
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['snipts'] = '/api/public/snipt/?user=%d' % bundle.obj.id
        return bundle

class PublicTagResource(ModelResource):
    class Meta:
        queryset = Tag.objects.filter(snipt__public=True)
        queryset = queryset.annotate(count=models.Count('taggit_taggeditem_items__id'))
        queryset = queryset.order_by('-count', 'name')
        resource_name = 'tag'
        fields = ['id', 'name',]
        allowed_methods = ['get']
        max_limit = 200
        cache = SimpleCache()

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PublicTagResource, self).build_filters(filters)

        if 'q' in filters:
            orm_filters['slug'] = filters['q']

        return orm_filters

    def dehydrate(self, bundle):
        bundle.data['absolute_url'] = '/public/tag/%s/' % bundle.obj.slug
        bundle.data['snipts'] = '/api/public/snipt/?tag=%d' % bundle.obj.id
        bundle.data['count'] = bundle.obj.taggit_taggeditem_items.filter(
                               snipt__public=True).count()
        return bundle

class PublicSniptResource(ModelResource):
    user = fields.ForeignKey(PublicUserResource, 'user', full=True)
    tags = fields.ToManyField(PublicTagResource, 'tags', related_name='tag', full=True)

    class Meta:
        queryset = Snipt.objects.filter(public=True).order_by('-created')
        resource_name = 'snipt'
        fields = ['id', 'title', 'slug', 'lexer', 'code', 'line_count',
                  'stylized', 'created', 'modified',]
        include_absolute_url = True
        allowed_methods = ['get']
        filtering = { 'user': 'exact', }
        ordering = ['created', 'modified',]
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['embed_url'] = bundle.obj.get_embed_url()
        bundle.data['full_absolute_url'] = bundle.obj.get_full_absolute_url()
        return bundle

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PublicSniptResource, self).build_filters(filters)

        if 'tag' in filters:
            tag = Tag.objects.get(pk=filters['tag'])
            tagged_items = tag.taggit_taggeditem_items.all()
            orm_filters['pk__in'] = [i.object_id for i in tagged_items]

        if 'q' in filters:
            sqs = SearchQuerySet().auto_query(filters['q'])
            orm_filters['pk__in'] = [i.pk for i in sqs]

        return orm_filters

class PrivateUserResource(ModelResource):
    class Meta:
        queryset = User.objects.all()
        resource_name = 'user'
        fields = ['id', 'username', 'email',]
        include_absolute_url = True
        allowed_methods = ['get']
        list_allowed_methods = []
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def apply_authorization_limits(self, request, object_list):
        return object_list.filter(username=request.user.username)

    def dehydrate(self, bundle):
        bundle.data['email_md5'] = hashlib.md5(bundle.obj.email.lower()).hexdigest()
        bundle.data['profile'] = {
            'is_pro': bundle.obj.profile.is_pro
        }
        return bundle

class PrivateTagResource(ModelResource):
    class Meta:
        queryset = Tag.objects.all()
        resource_name = 'tag'
        fields = ['id', 'name',]
        allowed_methods = ['get']
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PrivateTagResource, self).build_filters(filters)

        if 'q' in filters:
            orm_filters['slug'] = filters['q']

        return orm_filters
    def dehydrate(self, bundle):
        bundle.data['absolute_url'] = '/%s/tag/%s/' % (bundle.request.user.username,
                                                       bundle.obj.slug)
        bundle.data['snipts'] = '/api/private/snipt/?tag=%d' % bundle.obj.id

        bundle.data['count'] = bundle.obj.taggit_taggeditem_items.filter(
                                       snipt__user=bundle.request.user
                                   ).count()

        return bundle

    def apply_authorization_limits(self, request, object_list):
        object_list = object_list.filter(snipt__user=request.user)
        object_list = object_list.annotate(count=models.Count('taggit_taggeditem_items__id'))
        object_list = object_list.order_by('-count', 'name')
        return object_list

class PrivateSniptResource(ModelResource):
    user = fields.ForeignKey(PrivateUserResource, 'user', full=True)
    tags = fields.ToManyField(PrivateTagResource, 'tags', related_name='tag', full=True)
    tags_list = ListField()

    class Meta:
        queryset = Snipt.objects.all().order_by('-created')
        resource_name = 'snipt'
        fields = ['id', 'title', 'slug', 'lexer', 'code', 'line_count', 'stylized',
                  'key', 'public', 'blog_post', 'created', 'modified', 'publish_date',]
        validation = Validation()
        include_absolute_url = True
        detail_allowed_methods = ['get', 'patch', 'put', 'delete']
        list_allowed_methods = ['get', 'post']
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        ordering = ['created', 'modified',]
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['embed_url'] = bundle.obj.get_embed_url()
        bundle.data['tags_list'] = edit_string_for_tags(bundle.obj.tags.all())
        bundle.data['full_absolute_url'] = bundle.obj.get_full_absolute_url()

        if bundle.data['user'].data['profile']['is_pro']:
            bundle.data['views'] = bundle.obj.views
            bundle.data['favs'] = bundle.obj.favs()

        if bundle.data['publish_date']:
            bundle.data['publish_date'] = date(bundle.data['publish_date'], 'M d, Y \\a\\t h:i A')

        return bundle

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.data['tags_list'] = bundle.data.get('tags')
        bundle.data['tags'] = ''

        if 'blog_post' in bundle.data:
            bundle = self._clean_publish_date(bundle)

        return super(PrivateSniptResource, self).obj_create(bundle, request,
                     user=request.user, **kwargs)

    def obj_update(self, bundle, request=None, **kwargs):
        bundle.data['user'] = request.user

        if type(bundle.data['tags']) in (str, unicode):
            bundle.data['tags_list'] = bundle.data['tags']
        else:
            bundle.data['tags_list'] = ''
        bundle.data['tags'] = ''

        if 'blog_post' in bundle.data:
            bundle = self._clean_publish_date(bundle)

        return super(PrivateSniptResource, self).obj_update(bundle, request,
                     user=request.user, **kwargs)

    def _clean_publish_date(self, bundle):
        if bundle.data['blog_post'] and 'publish_date' not in bundle.data:
            bundle.data['publish_date'] = datetime.datetime.now()
        elif bundle.data['publish_date'] == '':
            bundle.data['publish_date'] = datetime.datetime.now()
        elif bundle.data['blog_post']:
            c = pdc.Constants()
            p = pdt.Calendar(c)
            publish_date, result = p.parse(bundle.data['publish_date'])

            if result != 0:
                publish_date = time.strftime("%Y-%m-%d %H:%M:%S", publish_date)
            else:
                publish_date = datetime.datetime.now()

            bundle.data['publish_date'] = publish_date
        elif not bundle.data['blog_post']:
            bundle.data['publish_date'] = None

        return bundle

    def build_filters(self, filters=None):
        if filters is None:
            filters = {}

        orm_filters = super(PrivateSniptResource, self).build_filters(filters)

        if 'tag' in filters:
            tag = Tag.objects.get(pk=filters['tag'])
            tagged_items = tag.taggit_taggeditem_items.all()
            orm_filters['pk__in'] = [i.object_id for i in tagged_items]

        if 'q' in filters:
            sqs = SearchQuerySet().auto_query(filters['q'])
            orm_filters['pk__in'] = [i.pk for i in sqs]

        return orm_filters

    def apply_authorization_limits(self, request, object_list):
        return object_list.filter(user=request.user)

    def save_m2m(self, bundle):
        tags = bundle.data.get('tags_list', [])
        if tags != '':
            bundle.obj.tags.set(*parse_tags(tags))
        else:
            bundle.obj.tags.set()

class PrivateFavoriteResource(ModelResource):
    user = fields.ForeignKey(PrivateUserResource, 'user', full=True)
    snipt = fields.ForeignKey(PrivateSniptResource, 'snipt', full=False)

    class Meta:
        queryset = Favorite.objects.all().order_by('-created')
        resource_name = 'favorite'
        fields = ['id',]
        validation = FavoriteValidation()
        detail_allowed_methods = ['get', 'post', 'delete']
        list_allowed_methods = ['get', 'post',]
        authentication = ApiKeyAuthentication()
        authorization = Authorization()
        ordering = ['created',]
        always_return_data = True
        max_limit = 200
        cache = SimpleCache()

    def dehydrate(self, bundle):
        bundle.data['snipt'] = '/api/public/snipt/{}/'.format(
                bundle.obj.snipt.pk)
        return bundle

    def obj_create(self, bundle, request=None, **kwargs):
        bundle.data['user'] = request.user
        bundle.data['snipt'] = Snipt.objects.get(pk=bundle.data['snipt'])
        return super(PrivateFavoriteResource, self).obj_create(bundle, request,
                     user=request.user, **kwargs)

    def apply_authorization_limits(self, request, object_list):
        return object_list.filter(user=request.user)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# (c) 2015, Darren Worrall <darren@iweb.co.uk>
# (c) 2015, René Moser <mail@renemoser.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: cs_loadbalancer_rule
short_description: Manages load balancer rules on Apache CloudStack based clouds.
description:
    - Add, update and remove load balancer rules.
version_added: '2.0'
author:
    - "Darren Worrall (@dazworrall)"
    - "René Moser (@resmo)"
options:
  name:
    description:
      - The name of the load balancer rule.
    required: true
  description:
    description:
      - The description of the load balancer rule.
    required: false
    default: null
  algorithm:
    description:
      - Load balancer algorithm
      - Required when using C(state=present).
    required: false
    choices: [ 'source', 'roundrobin', 'leastconn' ]
    default: 'source'
  protocol:
    description:
      - Protocol for the load balancer rule.
    required: false
    default: null
  private_port:
    description:
      - The private port of the private ip address/virtual machine where the network traffic will be load balanced to.
      - Required when using C(state=present).
      - Can not be changed once the rule exists due API limitation.
    required: false
    default: null
  public_port:
    description:
      - The public port from where the network traffic will be load balanced from.
      - Required when using C(state=present).
      - Can not be changed once the rule exists due API limitation.
    required: true
    default: null
  ip_address:
    description:
      - Public IP address from where the network traffic will be load balanced from.
    required: true
    aliases: [ 'public_ip' ]
  open_firewall:
    description:
      - Whether the firewall rule for public port should be created, while creating the new rule.
      - Use M(cs_firewall) for managing firewall rules.
    required: false
    default: false
  cidr:
    description:
    required: false
    default: null
  protocol:
    description:
      - The protocol to be used on the load balancer
      - CIDR (full notation) to be used for firewall rule if required.
    required: false
    default: null
  project:
    description:
      - Name of the project the load balancer IP address is related to.
    required: false
    default: null
  state:
    description:
      - State of the rule.
    required: true
    default: 'present'
    choices: [ 'present', 'absent' ]
  domain:
    description:
      - Domain the rule is related to.
    required: false
    default: null
  account:
    description:
      - Account the rule is related to.
    required: false
    default: null
  zone:
    description:
      - Name of the zone in which the rule shoud be created.
      - If not set, default zone is used.
    required: false
    default: null
extends_documentation_fragment: cloudstack
'''

EXAMPLES = '''
# Create a load balancer rule
- local_action:
    module: cs_loadbalancer_rule
    name: balance_http
    public_ip: 1.2.3.4
    algorithm: leastconn
    public_port: 80
    private_port: 8080

# update algorithm of an existing load balancer rule
- local_action:
    module: cs_loadbalancer_rule
    name: balance_http
    public_ip: 1.2.3.4
    algorithm: roundrobin
    public_port: 80
    private_port: 8080

# Delete a load balancer rule
- local_action:
    module: cs_loadbalancer_rule
    name: balance_http
    public_ip: 1.2.3.4
    state: absent
'''

RETURN = '''
---
id:
  description: UUID of the rule.
  returned: success
  type: string
  sample: a6f7a5fc-43f8-11e5-a151-feff819cdc9f
zone:
  description: Name of zone the rule is related to.
  returned: success
  type: string
  sample: ch-gva-2
project:
  description: Name of project the rule is related to.
  returned: success
  type: string
  sample: Production
account:
  description: Account the rule is related to.
  returned: success
  type: string
  sample: example account
domain:
  description: Domain the rule is related to.
  returned: success
  type: string
  sample: example domain
algorithm:
  description: Load balancer algorithm used.
  returned: success
  type: string
  sample: "source"
cidr:
  description: CIDR to forward traffic from.
  returned: success
  type: string
  sample: ""
name:
  description: Name of the rule.
  returned: success
  type: string
  sample: "http-lb"
description:
  description: Description of the rule.
  returned: success
  type: string
  sample: "http load balancer rule"
protocol:
  description: Protocol of the rule.
  returned: success
  type: string
  sample: "tcp"
public_port:
  description: Public port.
  returned: success
  type: string
  sample: 80
private_port:
  description: Private IP address.
  returned: success
  type: string
  sample: 80
public_ip:
  description: Public IP address.
  returned: success
  type: string
  sample: "1.2.3.4"
tags:
  description: List of resource tags associated with the rule.
  returned: success
  type: dict
  sample: '[ { "key": "foo", "value": "bar" } ]'
state:
  description: State of the rule.
  returned: success
  type: string
  sample: "Add"
'''

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

# import cloudstack common
from ansible.module_utils.cloudstack import *

class AnsibleCloudStackLBRule(AnsibleCloudStack):

    def __init__(self, module):
        super(AnsibleCloudStackLBRule, self).__init__(module)
        self.returns = {
            'publicip': 'public_ip',
            'algorithm': 'algorithm',
            'cidrlist': 'cidr',
            'protocol': 'protocol',
        }
        # these values will be casted to int
        self.returns_to_int = {
            'publicport': 'public_port',
            'privateport': 'private_port',
        }

    def get_rule(self, **kwargs):
        rules = self.cs.listLoadBalancerRules(**kwargs)
        if rules:
            return rules['loadbalancerrule'][0]

    def _get_common_args(self):
        return {
            'account': self.get_account(key='name'),
            'domainid': self.get_domain(key='id'),
            'projectid': self.get_project(key='id'),
            'zoneid': self.get_zone(key='id'),
            'publicipid': self.get_ip_address(key='id'),
            'name': self.module.params.get('name'),
        }

    def present_lb_rule(self):
        missing_params = []
        for required_params in [
            'algorithm',
            'private_port',
            'public_port',
        ]:
            if not self.module.params.get(required_params):
                missing_params.append(required_params)
        if missing_params:
            self.module.fail_json(msg="missing required arguments: %s" % ','.join(missing_params))

        args = self._get_common_args()
        rule = self.get_rule(**args)
        if rule:
            rule = self._update_lb_rule(rule)
        else:
            rule = self._create_lb_rule(rule)

        if rule:
            rule = self.ensure_tags(resource=rule, resource_type='LoadBalancer')
        return rule

    def _create_lb_rule(self, rule):
        self.result['changed'] = True
        if not self.module.check_mode:
            args = self._get_common_args()
            args['algorithm']   = self.module.params.get('algorithm')
            args['privateport'] = self.module.params.get('private_port')
            args['publicport']  = self.module.params.get('public_port')
            args['cidrlist']    = self.module.params.get('cidr')
            args['description'] = self.module.params.get('description')
            args['protocol']    = self.module.params.get('protocol')
            res = self.cs.createLoadBalancerRule(**args)
            if 'errortext' in res:
                self.module.fail_json(msg="Failed: '%s'" % res['errortext'])

            poll_async = self.module.params.get('poll_async')
            if poll_async:
                rule = self.poll_job(res, 'loadbalancer')
        return rule

    def _update_lb_rule(self, rule):
        args                = {}
        args['id']          = rule['id']
        args['algorithm']   = self.module.params.get('algorithm')
        args['description'] = self.module.params.get('description')
        if self.has_changed(args, rule):
            self.result['changed'] = True
            if not self.module.check_mode:
                res = self.cs.updateLoadBalancerRule(**args)
                if 'errortext' in res:
                    self.module.fail_json(msg="Failed: '%s'" % res['errortext'])

                poll_async = self.module.params.get('poll_async')
                if poll_async:
                    rule = self.poll_job(res, 'loadbalancer')
        return rule

    def absent_lb_rule(self):
        args = self._get_common_args()
        rule = self.get_rule(**args)
        if rule:
            self.result['changed'] = True
        if rule and not self.module.check_mode:
            res = self.cs.deleteLoadBalancerRule(id=rule['id'])
            if 'errortext' in res:
                self.module.fail_json(msg="Failed: '%s'" % res['errortext'])
            poll_async = self.module.params.get('poll_async')
            if poll_async:
                res = self._poll_job(res, 'loadbalancer')
        return rule

def main():
    argument_spec = cs_argument_spec()
    argument_spec.update(dict(
        name = dict(required=True),
        description = dict(default=None),
        algorithm = dict(choices=['source', 'roundrobin', 'leastconn'], default='source'),
        private_port = dict(type='int', default=None),
        public_port = dict(type='int', default=None),
        protocol = dict(default=None),
        state = dict(choices=['present', 'absent'], default='present'),
        ip_address = dict(required=True, aliases=['public_ip']),
        cidr = dict(default=None),
        project = dict(default=None),
        open_firewall = dict(choices=BOOLEANS, default=False),
        tags = dict(type='list', aliases=['tag'], default=None),
        zone = dict(default=None),
        domain = dict(default=None),
        account = dict(default=None),
        poll_async = dict(choices=BOOLEANS, default=True),
    ))

    module = AnsibleModule(
        argument_spec=argument_spec,
        required_together=cs_required_together(),
        supports_check_mode=True
    )

    if not has_lib_cs:
        module.fail_json(msg="python library cs required: pip install cs")

    try:
        acs_lb_rule = AnsibleCloudStackLBRule(module)

        state = module.params.get('state')
        if state in ['absent']:
            rule = acs_lb_rule.absent_lb_rule()
        else:
            rule = acs_lb_rule.present_lb_rule()

        result = acs_lb_rule.get_result(rule)

    except CloudStackException, e:
        module.fail_json(msg='CloudStackException: %s' % str(e))

    module.exit_json(**result)

# import module snippets
from ansible.module_utils.basic import *
if __name__ == '__main__':
    main()
<commit_msg>Fix doc build by moving misplaced CIDR documentation<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# (c) 2015, Darren Worrall <darren@iweb.co.uk>
# (c) 2015, René Moser <mail@renemoser.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: cs_loadbalancer_rule
short_description: Manages load balancer rules on Apache CloudStack based clouds.
description:
    - Add, update and remove load balancer rules.
version_added: '2.0'
author:
    - "Darren Worrall (@dazworrall)"
    - "René Moser (@resmo)"
options:
  name:
    description:
      - The name of the load balancer rule.
    required: true
  description:
    description:
      - The description of the load balancer rule.
    required: false
    default: null
  algorithm:
    description:
      - Load balancer algorithm
      - Required when using C(state=present).
    required: false
    choices: [ 'source', 'roundrobin', 'leastconn' ]
    default: 'source'
  protocol:
    description:
      - Protocol for the load balancer rule.
    required: false
    default: null
  private_port:
    description:
      - The private port of the private ip address/virtual machine where the network traffic will be load balanced to.
      - Required when using C(state=present).
      - Can not be changed once the rule exists due API limitation.
    required: false
    default: null
  public_port:
    description:
      - The public port from where the network traffic will be load balanced from.
      - Required when using C(state=present).
      - Can not be changed once the rule exists due API limitation.
    required: true
    default: null
  ip_address:
    description:
      - Public IP address from where the network traffic will be load balanced from.
    required: true
    aliases: [ 'public_ip' ]
  open_firewall:
    description:
      - Whether the firewall rule for public port should be created, while creating the new rule.
      - Use M(cs_firewall) for managing firewall rules.
    required: false
    default: false
  cidr:
    description:
      - CIDR (full notation) to be used for firewall rule if required.
    required: false
    default: null
  protocol:
    description:
      - The protocol to be used on the load balancer
    required: false
    default: null
  project:
    description:
      - Name of the project the load balancer IP address is related to.
    required: false
    default: null
  state:
    description:
      - State of the rule.
    required: true
    default: 'present'
    choices: [ 'present', 'absent' ]
  domain:
    description:
      - Domain the rule is related to.
    required: false
    default: null
  account:
    description:
      - Account the rule is related to.
    required: false
    default: null
  zone:
    description:
      - Name of the zone in which the rule shoud be created.
      - If not set, default zone is used.
    required: false
    default: null
extends_documentation_fragment: cloudstack
'''

EXAMPLES = '''
# Create a load balancer rule
- local_action:
    module: cs_loadbalancer_rule
    name: balance_http
    public_ip: 1.2.3.4
    algorithm: leastconn
    public_port: 80
    private_port: 8080

# update algorithm of an existing load balancer rule
- local_action:
    module: cs_loadbalancer_rule
    name: balance_http
    public_ip: 1.2.3.4
    algorithm: roundrobin
    public_port: 80
    private_port: 8080

# Delete a load balancer rule
- local_action:
    module: cs_loadbalancer_rule
    name: balance_http
    public_ip: 1.2.3.4
    state: absent
'''

RETURN = '''
---
id:
  description: UUID of the rule.
  returned: success
  type: string
  sample: a6f7a5fc-43f8-11e5-a151-feff819cdc9f
zone:
  description: Name of zone the rule is related to.
  returned: success
  type: string
  sample: ch-gva-2
project:
  description: Name of project the rule is related to.
  returned: success
  type: string
  sample: Production
account:
  description: Account the rule is related to.
  returned: success
  type: string
  sample: example account
domain:
  description: Domain the rule is related to.
  returned: success
  type: string
  sample: example domain
algorithm:
  description: Load balancer algorithm used.
  returned: success
  type: string
  sample: "source"
cidr:
  description: CIDR to forward traffic from.
  returned: success
  type: string
  sample: ""
name:
  description: Name of the rule.
  returned: success
  type: string
  sample: "http-lb"
description:
  description: Description of the rule.
  returned: success
  type: string
  sample: "http load balancer rule"
protocol:
  description: Protocol of the rule.
  returned: success
  type: string
  sample: "tcp"
public_port:
  description: Public port.
  returned: success
  type: string
  sample: 80
private_port:
  description: Private IP address.
  returned: success
  type: string
  sample: 80
public_ip:
  description: Public IP address.
  returned: success
  type: string
  sample: "1.2.3.4"
tags:
  description: List of resource tags associated with the rule.
  returned: success
  type: dict
  sample: '[ { "key": "foo", "value": "bar" } ]'
state:
  description: State of the rule.
  returned: success
  type: string
  sample: "Add"
'''

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

# import cloudstack common
from ansible.module_utils.cloudstack import *

class AnsibleCloudStackLBRule(AnsibleCloudStack):

    def __init__(self, module):
        super(AnsibleCloudStackLBRule, self).__init__(module)
        self.returns = {
            'publicip': 'public_ip',
            'algorithm': 'algorithm',
            'cidrlist': 'cidr',
            'protocol': 'protocol',
        }
        # these values will be casted to int
        self.returns_to_int = {
            'publicport': 'public_port',
            'privateport': 'private_port',
        }

    def get_rule(self, **kwargs):
        rules = self.cs.listLoadBalancerRules(**kwargs)
        if rules:
            return rules['loadbalancerrule'][0]

    def _get_common_args(self):
        return {
            'account': self.get_account(key='name'),
            'domainid': self.get_domain(key='id'),
            'projectid': self.get_project(key='id'),
            'zoneid': self.get_zone(key='id'),
            'publicipid': self.get_ip_address(key='id'),
            'name': self.module.params.get('name'),
        }

    def present_lb_rule(self):
        missing_params = []
        for required_params in [
            'algorithm',
            'private_port',
            'public_port',
        ]:
            if not self.module.params.get(required_params):
                missing_params.append(required_params)
        if missing_params:
            self.module.fail_json(msg="missing required arguments: %s" % ','.join(missing_params))

        args = self._get_common_args()
        rule = self.get_rule(**args)
        if rule:
            rule = self._update_lb_rule(rule)
        else:
            rule = self._create_lb_rule(rule)

        if rule:
            rule = self.ensure_tags(resource=rule, resource_type='LoadBalancer')
        return rule

    def _create_lb_rule(self, rule):
        self.result['changed'] = True
        if not self.module.check_mode:
            args = self._get_common_args()
            args['algorithm']   = self.module.params.get('algorithm')
            args['privateport'] = self.module.params.get('private_port')
            args['publicport']  = self.module.params.get('public_port')
            args['cidrlist']    = self.module.params.get('cidr')
            args['description'] = self.module.params.get('description')
            args['protocol']    = self.module.params.get('protocol')
            res = self.cs.createLoadBalancerRule(**args)
            if 'errortext' in res:
                self.module.fail_json(msg="Failed: '%s'" % res['errortext'])

            poll_async = self.module.params.get('poll_async')
            if poll_async:
                rule = self.poll_job(res, 'loadbalancer')
        return rule

    def _update_lb_rule(self, rule):
        args                = {}
        args['id']          = rule['id']
        args['algorithm']   = self.module.params.get('algorithm')
        args['description'] = self.module.params.get('description')
        if self.has_changed(args, rule):
            self.result['changed'] = True
            if not self.module.check_mode:
                res = self.cs.updateLoadBalancerRule(**args)
                if 'errortext' in res:
                    self.module.fail_json(msg="Failed: '%s'" % res['errortext'])

                poll_async = self.module.params.get('poll_async')
                if poll_async:
                    rule = self.poll_job(res, 'loadbalancer')
        return rule

    def absent_lb_rule(self):
        args = self._get_common_args()
        rule = self.get_rule(**args)
        if rule:
            self.result['changed'] = True
        if rule and not self.module.check_mode:
            res = self.cs.deleteLoadBalancerRule(id=rule['id'])
            if 'errortext' in res:
                self.module.fail_json(msg="Failed: '%s'" % res['errortext'])
            poll_async = self.module.params.get('poll_async')
            if poll_async:
                res = self._poll_job(res, 'loadbalancer')
        return rule

def main():
    argument_spec = cs_argument_spec()
    argument_spec.update(dict(
        name = dict(required=True),
        description = dict(default=None),
        algorithm = dict(choices=['source', 'roundrobin', 'leastconn'], default='source'),
        private_port = dict(type='int', default=None),
        public_port = dict(type='int', default=None),
        protocol = dict(default=None),
        state = dict(choices=['present', 'absent'], default='present'),
        ip_address = dict(required=True, aliases=['public_ip']),
        cidr = dict(default=None),
        project = dict(default=None),
        open_firewall = dict(choices=BOOLEANS, default=False),
        tags = dict(type='list', aliases=['tag'], default=None),
        zone = dict(default=None),
        domain = dict(default=None),
        account = dict(default=None),
        poll_async = dict(choices=BOOLEANS, default=True),
    ))

    module = AnsibleModule(
        argument_spec=argument_spec,
        required_together=cs_required_together(),
        supports_check_mode=True
    )

    if not has_lib_cs:
        module.fail_json(msg="python library cs required: pip install cs")

    try:
        acs_lb_rule = AnsibleCloudStackLBRule(module)

        state = module.params.get('state')
        if state in ['absent']:
            rule = acs_lb_rule.absent_lb_rule()
        else:
            rule = acs_lb_rule.present_lb_rule()

        result = acs_lb_rule.get_result(rule)

    except CloudStackException, e:
        module.fail_json(msg='CloudStackException: %s' % str(e))

    module.exit_json(**result)

# import module snippets
from ansible.module_utils.basic import *
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import re
from bs4 import BeautifulSoup, UnicodeDammit
import lxml.html
import lxml.html.clean
from lxml import etree

def _extract_links(html):
    tree = etree.fromstring(html, parser=etree.HTMLParser())

    url_index = []
    i = 0
    for link in tree.findall('.//a'):
        url = link.get('href')
        if url:
            i += 1
            index_data = {'title': link.text, 'url': url, 'id': i}

            link.tag = 'span'
            link.text = '{title} [{id}]'.format(**index_data)
            url_index.append(index_data)

    html = etree.tostring(tree, encoding="UTF-8", method='html')
    html = html.decode('utf-8')

    html += '
\n
\n'
    for index_data in url_index:
        html += '[{id}] {title}: \t {url}
\n'.format(**index_data)

    return html

def _create_lxml_html_cleaner():
    # Consult http://lxml.de/3.4/api/lxml.html.clean.Cleaner-class.html
    cleaner = lxml.html.clean.Cleaner()

    # The idea here is to make it explicit to copy/paste a link, as all of our
    # links will be available as a text-only index.
    # Also remove any tags that would make inline printing bad, like ,
    #  and body.
    cleaner.remove_tags = ['a', 'img', 'head', 'html', 'body']

    # remove the shady stuff
    cleaner.javascript = True
    cleaner.scripts = True
    cleaner.links = True
    cleaner.embedded = True
    cleaner.frames = True

    # may contain exploits for IE6
    cleaner.comments = True
    cleaner.processing_instructions = True
    cleaner.meta = True
    cleaner.forms = True
    cleaner.remove_unknown_tags = True
    cleaner.annoying_tags = True

    # remove


<commit_before>import xmpp
import threading
import Agent
import Envelope
import FIPAMessage
import AID
import Behaviour

class SpadePlatform(Agent.PlatformAgent):
	class RouteBehaviour(Behaviour.Behaviour):
		def __init__(self):
			Behaviour.Behaviour.__init__(self)
		def _process(self):
			msg = self._receive(True)
			if (msg != None):
				to_list = msg.getReceivers()
				d = {}
				for to in to_list:
					if not to.getAddresses()[0] in d: d[to.getAddresses()[0]]=list()
					d[to.getAddresses()[0]].append(to)
				for k,v in d.items():
					newmsg = msg
					newmsg.to = v
					#print "############# k != plat?"
					#print " k>>"+k[7:] + "    plat>>"+self.myAgent.getSpadePlatformJID()
					if k[7:] != self.myAgent.getSpadePlatformJID():
						self.myAgent._sendTo(newmsg, k[7:])
					else:
						for recv in v:
							self.myAgent._sendTo(newmsg, recv.getName())
					# Reenviamos el msg a todos los destinatarios
					# Deberiamos de comprobar si el destinatario estaen otro servidor y reenviarselo a la plataforma en vez de a el.
					# Tambien deberiamos comprobar el protocolo y usar una pasarela en el caso de que sea necesario.
					#print "Message to", to.getName(), "... Posting!"
			else:
				print "ACC::dying... it shouldn't happen"

	def __init__(self, node, password, server, port):
		Agent.PlatformAgent.__init__(self, node, password, server, port, debug=[])

	def _setup(self):
		self.setDefaultBehaviour(self.RouteBehaviour())

"""
class FipperPlatform(threading.Thread):
    def __init__(self, params=dict()):
        threading.Thread.__init__(self)
        self.__params = params
        self.__mts = MTS(params)

        try:
            mtp_list = params["mtps"]
            for mtp in mtp_list:
                self.__mts.addMTP(mtp)
        except KeyError:
            pass

    def getMTS(self):
        return self.__mts

    def run(self):
        pass

class MTS:

    def __init__(self, params=dict()):
        self.__platformagents = dict()

    def addAgent(self, agent):
        aid = agent.getAID()
        self.__platformagents[aid] = agent

    def addMTP(self, mtp):
        pass

    def dispatch(self, msg):
        print "Dispatching msg..."
        envelope = msg.getEnvelope()
        to_list = envelope.getTo()
        #for to in to_list:
	#	print to

        for to in to_list:
            if to in self.__platformagents:
                print "Agend finded! Posting..."
                self.__platformagents[to].postMessage(msg)

class MTP:
    def getName(self):
        raise NotImplemented

    def getSupportedProtocols(self):
        raise NotImplemented

    def activate(self, dispatcher, params):
        raise NotImplemented

    def deactivate(self):
        raise NotImplemented

    def deliver(self, addr, envelope, payload):
        raise NotImplemented

"""

<commit_msg>packaging<commit_after>import xmpp
import threading
import Agent
import Envelope
import FIPAMessage
import AID
import Behaviour

class SpadePlatform(Agent.PlatformAgent):
	class RouteBehaviour(Behaviour.Behaviour):
		def __init__(self):
			Behaviour.Behaviour.__init__(self)
		def _process(self):
			msg = self._receive(True)
			if (msg != None):
				to_list = msg.getReceivers()
				d = {}
				for to in to_list:
					if not to.getAddresses()[0] in d: d[to.getAddresses()[0]]=list()
					d[to.getAddresses()[0]].append(to)
				for k,v in d.items():
					newmsg = msg
					newmsg.to = v
					#print "############# k != plat?"
					#print " k>>"+k[7:] + "    plat>>"+self.myAgent.getSpadePlatformJID()
					if k[7:] != self.myAgent.getSpadePlatformJID():
						self.myAgent._sendTo(newmsg, k[7:])
					else:
						for recv in v:
							self.myAgent._sendTo(newmsg, recv.getName())
					# Reenviamos el msg a todos los destinatarios
					# Deberiamos de comprobar si el destinatario estaen otro servidor y reenviarselo a la plataforma en vez de a el.
					# Tambien deberiamos comprobar el protocolo y usar una pasarela en el caso de que sea necesario.
					#print "Message to", to.getName(), "... Posting!"
			else:
				print "ACC::dying... it shouldn't happen"

	def __init__(self, node, password, server, port):
		Agent.PlatformAgent.__init__(self, node, password, server, port, debug=[])

	def _setup(self):
		print ">>>>>>>>>>>>>>SETUP PLATFORM"
		self.setDefaultBehaviour(self.RouteBehaviour())

"""
class FipperPlatform(threading.Thread):
    def __init__(self, params=dict()):
        threading.Thread.__init__(self)
        self.__params = params
        self.__mts = MTS(params)

        try:
            mtp_list = params["mtps"]
            for mtp in mtp_list:
                self.__mts.addMTP(mtp)
        except KeyError:
            pass

    def getMTS(self):
        return self.__mts

    def run(self):
        pass

class MTS:

    def __init__(self, params=dict()):
        self.__platformagents = dict()

    def addAgent(self, agent):
        aid = agent.getAID()
        self.__platformagents[aid] = agent

    def addMTP(self, mtp):
        pass

    def dispatch(self, msg):
        print "Dispatching msg..."
        envelope = msg.getEnvelope()
        to_list = envelope.getTo()
        #for to in to_list:
	#	print to

        for to in to_list:
            if to in self.__platformagents:
                print "Agend finded! Posting..."
                self.__platformagents[to].postMessage(msg)

class MTP:
    def getName(self):
        raise NotImplemented

    def getSupportedProtocols(self):
        raise NotImplemented

    def activate(self, dispatcher, params):
        raise NotImplemented

    def deactivate(self):
        raise NotImplemented

    def deliver(self, addr, envelope, payload):
        raise NotImplemented

"""

<|endoftext|>


<commit_before>"""Miscellaneous tools and shortcuts"""
import inspect
import types
import typing as t
from dataclasses import Field, dataclass, field
from datetime import datetime
from functools import wraps

T = t.TypeVar('T')

def apply(func, args=(), kwargs=None):
    """apply args and kwargs to a function"""
    return func(*args, **kwargs or {})

def notnone(obj):
    """return whether an object is not None"""
    return obj is not None

class StrRepr():
    """mixin which adds a ``__repr__`` based on ``__str__``"""

    def __str__(self):
        return '{0.__class__.__name__} object'.format(self)

    def __repr__(self):
        return '<{0.__class__.__name__}: {0}>'.format(self)

class NO_DEFAULT:
    """sentinel for no default"""

@dataclass(frozen=True)
class lookup_defaults(t.Callable[[t.Any], T]):
    """wrap a lookup function with a default if lookup fails"""
    lookup: t.Callable[[t.Any], T]
    default: T

    def __call__(self, obj):
        try:
            return self.lookup(obj)
        except LookupError:
            return self.default

def skipnone(func):
    """wrap a function so that it returns None when getting None as input"""
    @wraps(func)
    def wrapper(arg):
        return None if arg is None else func(arg)

    return wrapper

def parse_iso8601(dtstring: str) -> datetime:
    """naive parser for ISO8061 datetime strings,

    Parameters
    ----------
    dtstring
        the datetime as string in one of two formats:

        * ``2017-11-20T07:16:29+0000``
        * ``2017-11-20T07:16:29Z``

    """
    return datetime.strptime(
        dtstring,
        '%Y-%m-%dT%H:%M:%SZ' if len(dtstring) == 20 else '%Y-%m-%dT%H:%M:%S%z')

def genresult(gen, value):
    """retrieve the return value from a generator"""
    try:
        gen.send(value)
    except StopIteration as e:
        return e.value
    else:
        raise TypeError('generator did not return as expected')

def func_to_fields(func: types.FunctionType) -> t.List[Field]:
    """get dataclass fields from a function signature

    Parameters
    ----------
    func
        a python function

    Notes
    -----
    * keyword-only, varargs, and varkeywords are not supported
    """
    spec = inspect.getfullargspec(func)
    defaults = dict(zip(reversed(spec.args), reversed(spec.defaults or ())))
    if spec.kwonlyargs:
        raise TypeError('keyword-only args not supported')
    elif spec.varargs:
        raise TypeError('varargs not supported')
    elif spec.varkw:
        raise TypeError('varkw not supported')
    return [
        (name,
         spec.annotations.get(name, t.Any),
         field(default=defaults[name]) if name in defaults else field())
        for name in spec.args
    ]

def identity(obj):
    """identity function, returns input unmodified"""
    return obj

@dataclass(frozen=True)
class flip:
    """create a function with flipped arguments"""
    func: t.Callable[[t.Any, t.Any], t.Any]

    def __call__(self, a, b):
        return self.func(b, a)

@dataclass(init=False)
class compose:
    """compose a function from a chain of functions"""
    funcs: t.List[t.Callable]

    def __init__(self, *funcs):
        self.funcs = funcs

    def __call__(self, *args, **kwargs):
        if not self.funcs:
            return identity(*args, **kwargs)
        *tail, head = self.funcs
        value = head(*args, **kwargs)
        for func in reversed(tail):
            value = func(value)
        return value

def valmap(func: t.Callable, mapping: t.Mapping) -> t.Mapping:
    """map() for values of a mapping"""
    return {k: func(v) for k, v in mapping.items()}

def valfilter(predicate: t.Callable, mapping: t.Mapping) -> t.Mapping:
    """filter() for values of a mapping"""
    return {k: v for k, v in mapping.items() if predicate(v)}

def pipe(value, *funcs):
    """pipe a value through a sequence of functions"""
    for func in funcs:
        value = func(value)
    return value

def as_tuple(dclass):
    """like dataclasses.astuple(), but without recursing into fields"""
    return tuple(getattr(dclass, name) for name in dclass.__dataclass_fields__)

JSONType = t.Union[str, int, float, bool, None,
                   t.Dict[str, t.Any],
                   t.List[t.Any]]
<commit_msg>fix type annotation<commit_after>"""Miscellaneous tools and shortcuts"""
import inspect
import types
import typing as t
from dataclasses import Field, dataclass, field
from datetime import datetime
from functools import wraps

T = t.TypeVar('T')

def apply(func, args=(), kwargs=None):
    """apply args and kwargs to a function"""
    return func(*args, **kwargs or {})

def notnone(obj):
    """return whether an object is not None"""
    return obj is not None

class StrRepr():
    """mixin which adds a ``__repr__`` based on ``__str__``"""

    def __str__(self):
        return '{0.__class__.__name__} object'.format(self)

    def __repr__(self):
        return '<{0.__class__.__name__}: {0}>'.format(self)

class NO_DEFAULT:
    """sentinel for no default"""

@dataclass(frozen=True)
class lookup_defaults(t.Callable[[t.Any], T]):
    """wrap a lookup function with a default if lookup fails"""
    lookup: t.Callable[[t.Any], T]
    default: T

    def __call__(self, obj):
        try:
            return self.lookup(obj)
        except LookupError:
            return self.default

def skipnone(func):
    """wrap a function so that it returns None when getting None as input"""
    @wraps(func)
    def wrapper(arg):
        return None if arg is None else func(arg)

    return wrapper

def parse_iso8601(dtstring: str) -> datetime:
    """naive parser for ISO8061 datetime strings,

    Parameters
    ----------
    dtstring
        the datetime as string in one of two formats:

        * ``2017-11-20T07:16:29+0000``
        * ``2017-11-20T07:16:29Z``

    """
    return datetime.strptime(
        dtstring,
        '%Y-%m-%dT%H:%M:%SZ' if len(dtstring) == 20 else '%Y-%m-%dT%H:%M:%S%z')

def genresult(gen, value):
    """retrieve the return value from a generator"""
    try:
        gen.send(value)
    except StopIteration as e:
        return e.value
    else:
        raise TypeError('generator did not return as expected')

def func_to_fields(func: types.FunctionType) -> t.List[Field]:
    """get dataclass fields from a function signature

    Parameters
    ----------
    func
        a python function

    Notes
    -----
    * keyword-only, varargs, and varkeywords are not supported
    """
    spec = inspect.getfullargspec(func)
    defaults = dict(zip(reversed(spec.args), reversed(spec.defaults or ())))
    if spec.kwonlyargs:
        raise TypeError('keyword-only args not supported')
    elif spec.varargs:
        raise TypeError('varargs not supported')
    elif spec.varkw:
        raise TypeError('varkw not supported')
    return [
        (name,
         spec.annotations.get(name, t.Any),
         field(default=defaults[name]) if name in defaults else field())
        for name in spec.args
    ]

def identity(obj):
    """identity function, returns input unmodified"""
    return obj

@dataclass(frozen=True)
class flip:
    """create a function with flipped arguments"""
    func: t.Callable[[t.Any, t.Any], t.Any]

    def __call__(self, a, b):
        return self.func(b, a)

@dataclass(init=False)
class compose:
    """compose a function from a chain of functions"""
    funcs: t.Tuple[t.Callable, ...]

    def __init__(self, *funcs):
        self.funcs = funcs

    def __call__(self, *args, **kwargs):
        if not self.funcs:
            return identity(*args, **kwargs)
        *tail, head = self.funcs
        value = head(*args, **kwargs)
        for func in reversed(tail):
            value = func(value)
        return value

def valmap(func: t.Callable, mapping: t.Mapping) -> t.Mapping:
    """map() for values of a mapping"""
    return {k: func(v) for k, v in mapping.items()}

def valfilter(predicate: t.Callable, mapping: t.Mapping) -> t.Mapping:
    """filter() for values of a mapping"""
    return {k: v for k, v in mapping.items() if predicate(v)}

def pipe(value, *funcs):
    """pipe a value through a sequence of functions"""
    for func in funcs:
        value = func(value)
    return value

def as_tuple(dclass):
    """like dataclasses.astuple(), but without recursing into fields"""
    return tuple(getattr(dclass, name) for name in dclass.__dataclass_fields__)

JSONType = t.Union[str, int, float, bool, None,
                   t.Dict[str, t.Any],
                   t.List[t.Any]]
<|endoftext|>


<commit_before>""" Test a live app connection.
"""

import os
import time
import sys

from flexx import app, event, webruntime

from flexx.util.testing import run_tests_if_main, raises, skip

ON_TRAVIS = os.getenv('TRAVIS', '') == 'true'
ON_PYPY = '__pypy__' in sys.builtin_module_names

TIMEOUT1 = 5.0
TIMEOUT2 = 1.0

def runner(cls):
    t = app.launch(cls, 'firefox')  # fails somehow with XUL
    t.test_init()
    app.call_later(TIMEOUT1, app.stop)
    app.run()
    if not (ON_TRAVIS and ON_PYPY):  # has intermittent fails on pypy3
        t.test_check()
    t.session.close()

class ModelA(app.Model):

    @event.prop
    def foo1(self, v=0):
        return float(v+1)

    @event.prop
    def foo2(self, v=0):
        return float(v+1)

    @event.prop
    def result(self, v=None):
        if v:
            #app.stop()
            print('stopping by ourselves', v)
            app.call_later(TIMEOUT2, app.stop)
        return v

    def test_init(self):

        assert self.foo1 == 1
        assert self.bar1 is None  # not yet initialized

        self.call_js('set_result()')

    def test_check(self):
        assert self.foo1 == 1
        assert self.foo2 == 1
        #
        assert self.bar1 == 1
        assert self.bar2 == 1
        #
        assert self.result == '1 1 - 1 1'
        print('A ok')

    class JS:

        @event.prop
        def bar1(self, v=0):
            return int(v+1)

        @event.prop
        def bar2(self, v=0):
            return int(v+1)

        def set_result(self):
            self.result = ' '.join([self.foo1, self.foo2, '-',
                                    self.bar1, self.bar2])

class ModelB(ModelA):

    @event.prop
    def foo2(self, v=0):
        return int(v+2)

    @event.prop
    def foo3(self, v=0):
        return int(v+2)

    def test_check(self):
        assert self.foo1 == 1
        assert self.foo2 == 2
        assert self.foo3 == 2
        #
        assert self.bar1 == 1
        assert self.bar2 == 2
        assert self.bar3 == 2
        #
        assert self.result == '1 2 2 - 1 2 2'
        print('B ok')

    class JS:

        @event.prop
        def bar2(self, v=0):
            return int(v+2)

        @event.prop
        def bar3(self, v=0):
            return int(v+2)

        def set_result(self):
            self.result = ' '.join([self.foo1, self.foo2, self.foo3, '-',
                                    self.bar1, self.bar2, self.bar3])

class ModelC(ModelB):
    # Test properties and proxy properties, no duplicates etc.

    def test_check(self):
        py_result = ' '.join(self.__properties__) + ' - ' + ' '.join(self.__proxy_properties__)
        js_result = self.result
        assert py_result == 'bar1 bar2 bar3 foo1 foo2 foo3 result - bar1 bar2 bar3'
        assert js_result == 'bar1 bar2 bar3 foo1 foo2 foo3 result - foo1 foo2 foo3 result'
        print('C ok')

    class JS:

        def set_result(self):
            self.result = ' '.join(self.__properties__) + ' - ' + ' '.join(self.__proxy_properties__)

class ModelD(ModelB):
    # Test setting properties

    def test_init(self):

        assert self.foo2 == 2
        self.foo2 = 10
        assert self.foo2 == 12

        assert self.bar2 is None
        self.bar2 = 10
        assert self.bar2 is None

        self.call_js('set_result()')

    def test_check(self):

        assert self.result == 'ok'

        assert self.foo2 == 12
        assert self.foo3 == 12
        assert self.bar2 == 12
        assert self.bar3 == 12

    class JS:

        def init(self):
            super().init()
            print(self.foo3, self.bar3)

            assert self.foo3 is None
            self.foo3 = 10
            assert self.foo3 is None

            assert self.bar3 == 2
            self.bar3 = 10
            assert self.bar3 == 12

        def set_result(self):
            assert self.foo2 == 12
            # assert self.foo3 == 12  # this takes more cycles, hard to test
            assert self.bar2 == 12
            assert self.bar3 == 12
            self.result = 'ok'

class ModelE(ModelA):

    def init(self):
        self.res1 = []
        self.res2 = []

    @event.connect('foo')
    def foo_handler(self, *events):
        self.res1.append(len(events))
        print('Py saw %i foo events' % len(events))

    @event.connect('bar')
    def bar_handler(self, *events):
        self.res2.append(len(events))
        print('Py saw %i bar events' % len(events))

    def test_init(self):
        app.call_later(0.2, self._emit_foo)
        app.call_later(0.9, lambda:self.call_js('set_result()'))

    def _emit_foo(self):
        self.emit('foo', {})
        self.emit('foo', {})

    def test_check(self):
        result_py = self.res1 + [''] + self.res2
        result_js = self.result
        print(result_py)
        print(result_js)
        assert result_py == [2, '', 2]
        assert result_js == [2, '', 2]

    class JS:

        def init(self):
            self.res3 = []
            self.res4 = []

            self.emit('bar', {})
            self.emit('bar', {})

        @event.connect('foo')
        def foo_handler(self, *events):
            self.res3.append(len(events))
            print('JS saw %i foo events' % len(events))

        @event.connect('bar')
        def bar_handler(self, *events):
            self.res4.append(len(events))
            print('JS saw %i bar events' % len(events))

        def set_result(self):
            self.result = self.res3 + [''] + self.res4

##

def test_generated_javascript():
    # Test that there are no diplicate funcs etc.

    codeA, codeB = ModelA.JS.CODE, ModelB.JS.CODE

    assert codeA.count('_foo1_func = function') == 1
    assert codeA.count('_foo2_func = function') == 1
    assert codeA.count('_foo3_func = function') == 0
    assert codeA.count('_bar1_func = function') == 1
    assert codeA.count('_bar2_func = function') == 1
    assert codeA.count('_bar3_func = function') == 0

    assert codeB.count('_foo1_func = function') == 0
    assert codeB.count('_foo2_func = function') == 0  # proxy needs no new func
    assert codeB.count('_foo3_func = function') == 1
    assert codeB.count('_bar1_func = function') == 0
    assert codeB.count('_bar2_func = function') == 1  # but real prop does
    assert codeB.count('_bar3_func = function') == 1

def test_apps():

    if not webruntime.has_firefox():
        skip('This live test needs firefox.')

    runner(ModelA)
    runner(ModelB)
    runner(ModelC)
    runner(ModelD)
    runner(ModelE)

# NOTE: beware future self: if running this in Pyzo, turn off GUI integration!

# runner(ModelD)
run_tests_if_main()
<commit_msg>Maybe Travis needs more time?<commit_after>""" Test a live app connection.
"""

import os
import time
import sys

from flexx import app, event, webruntime

from flexx.util.testing import run_tests_if_main, raises, skip

ON_TRAVIS = os.getenv('TRAVIS', '') == 'true'
ON_PYPY = '__pypy__' in sys.builtin_module_names

TIMEOUT1 = 10.0 if ON_TRAVIS else 3.0
TIMEOUT2 = 1.0

def runner(cls):
    t = app.launch(cls, 'firefox')  # fails somehow with XUL
    t.test_init()
    app.call_later(TIMEOUT1, app.stop)
    app.run()
    if not (ON_TRAVIS and ON_PYPY):  # has intermittent fails on pypy3
        t.test_check()
    t.session.close()

class ModelA(app.Model):

    @event.prop
    def foo1(self, v=0):
        return float(v+1)

    @event.prop
    def foo2(self, v=0):
        return float(v+1)

    @event.prop
    def result(self, v=None):
        if v:
            #app.stop()
            print('stopping by ourselves', v)
            app.call_later(TIMEOUT2, app.stop)
        return v

    def test_init(self):

        assert self.foo1 == 1
        assert self.bar1 is None  # not yet initialized

        self.call_js('set_result()')

    def test_check(self):
        assert self.foo1 == 1
        assert self.foo2 == 1
        #
        assert self.bar1 == 1
        assert self.bar2 == 1
        #
        assert self.result == '1 1 - 1 1'
        print('A ok')

    class JS:

        @event.prop
        def bar1(self, v=0):
            return int(v+1)

        @event.prop
        def bar2(self, v=0):
            return int(v+1)

        def set_result(self):
            self.result = ' '.join([self.foo1, self.foo2, '-',
                                    self.bar1, self.bar2])

class ModelB(ModelA):

    @event.prop
    def foo2(self, v=0):
        return int(v+2)

    @event.prop
    def foo3(self, v=0):
        return int(v+2)

    def test_check(self):
        assert self.foo1 == 1
        assert self.foo2 == 2
        assert self.foo3 == 2
        #
        assert self.bar1 == 1
        assert self.bar2 == 2
        assert self.bar3 == 2
        #
        assert self.result == '1 2 2 - 1 2 2'
        print('B ok')

    class JS:

        @event.prop
        def bar2(self, v=0):
            return int(v+2)

        @event.prop
        def bar3(self, v=0):
            return int(v+2)

        def set_result(self):
            self.result = ' '.join([self.foo1, self.foo2, self.foo3, '-',
                                    self.bar1, self.bar2, self.bar3])

class ModelC(ModelB):
    # Test properties and proxy properties, no duplicates etc.

    def test_check(self):
        py_result = ' '.join(self.__properties__) + ' - ' + ' '.join(self.__proxy_properties__)
        js_result = self.result
        assert py_result == 'bar1 bar2 bar3 foo1 foo2 foo3 result - bar1 bar2 bar3'
        assert js_result == 'bar1 bar2 bar3 foo1 foo2 foo3 result - foo1 foo2 foo3 result'
        print('C ok')

    class JS:

        def set_result(self):
            self.result = ' '.join(self.__properties__) + ' - ' + ' '.join(self.__proxy_properties__)

class ModelD(ModelB):
    # Test setting properties

    def test_init(self):

        assert self.foo2 == 2
        self.foo2 = 10
        assert self.foo2 == 12

        assert self.bar2 is None
        self.bar2 = 10
        assert self.bar2 is None

        self.call_js('set_result()')

    def test_check(self):

        assert self.result == 'ok'

        assert self.foo2 == 12
        assert self.foo3 == 12
        assert self.bar2 == 12
        assert self.bar3 == 12

    class JS:

        def init(self):
            super().init()
            print(self.foo3, self.bar3)

            assert self.foo3 is None
            self.foo3 = 10
            assert self.foo3 is None

            assert self.bar3 == 2
            self.bar3 = 10
            assert self.bar3 == 12

        def set_result(self):
            assert self.foo2 == 12
            # assert self.foo3 == 12  # this takes more cycles, hard to test
            assert self.bar2 == 12
            assert self.bar3 == 12
            self.result = 'ok'

class ModelE(ModelA):

    def init(self):
        self.res1 = []
        self.res2 = []

    @event.connect('foo')
    def foo_handler(self, *events):
        self.res1.append(len(events))
        print('Py saw %i foo events' % len(events))

    @event.connect('bar')
    def bar_handler(self, *events):
        self.res2.append(len(events))
        print('Py saw %i bar events' % len(events))

    def test_init(self):
        app.call_later(0.2, self._emit_foo)
        app.call_later(0.9, lambda:self.call_js('set_result()'))

    def _emit_foo(self):
        self.emit('foo', {})
        self.emit('foo', {})

    def test_check(self):
        result_py = self.res1 + [''] + self.res2
        result_js = self.result
        print(result_py)
        print(result_js)
        assert result_py == [2, '', 2]
        assert result_js == [2, '', 2]

    class JS:

        def init(self):
            self.res3 = []
            self.res4 = []

            self.emit('bar', {})
            self.emit('bar', {})

        @event.connect('foo')
        def foo_handler(self, *events):
            self.res3.append(len(events))
            print('JS saw %i foo events' % len(events))

        @event.connect('bar')
        def bar_handler(self, *events):
            self.res4.append(len(events))
            print('JS saw %i bar events' % len(events))

        def set_result(self):
            self.result = self.res3 + [''] + self.res4

##

def test_generated_javascript():
    # Test that there are no diplicate funcs etc.

    codeA, codeB = ModelA.JS.CODE, ModelB.JS.CODE

    assert codeA.count('_foo1_func = function') == 1
    assert codeA.count('_foo2_func = function') == 1
    assert codeA.count('_foo3_func = function') == 0
    assert codeA.count('_bar1_func = function') == 1
    assert codeA.count('_bar2_func = function') == 1
    assert codeA.count('_bar3_func = function') == 0

    assert codeB.count('_foo1_func = function') == 0
    assert codeB.count('_foo2_func = function') == 0  # proxy needs no new func
    assert codeB.count('_foo3_func = function') == 1
    assert codeB.count('_bar1_func = function') == 0
    assert codeB.count('_bar2_func = function') == 1  # but real prop does
    assert codeB.count('_bar3_func = function') == 1

def test_apps():

    if not webruntime.has_firefox():
        skip('This live test needs firefox.')

    runner(ModelA)
    runner(ModelB)
    runner(ModelC)
    runner(ModelD)
    runner(ModelE)

# NOTE: beware future self: if running this in Pyzo, turn off GUI integration!

# runner(ModelD)
run_tests_if_main()
<|endoftext|>


<commit_before>import ACLMessage
import Behaviour
import Envelope
import MessageReceiver
import SL0Parser
import XMLCodec
import ACLParser
import AID
import BasicFipaDateTime
import DF
import FIPAMessage
import Platform
import Agent
import AMS
import spade_backend
import SpadeConfigParser

<commit_msg>packaging<commit_after>import ACLMessage
import Behaviour
import Envelope
import MessageReceiver
import SL0Parser
import XMLCodec
import ACLParser
import AID
import BasicFipaDateTime
import DF
import FIPAMessage
import Platform
import Agent
import AMS
import spade_backend
import SpadeConfigParser
import stanza_queue
<|endoftext|>