<commit_before># author: G. Alomar
from hecuba.datastore import *
from cassandra.cluster import Cluster
from hecuba.dict import *
from app.qbeastiface import *
from conf.hecuba_params import execution_name, ranges_per_block
import time

class Block(object):
    def __init__(self, peer, keynames, tablename, blockkeyspace):
        self.node = peer[0]
        self.token_ranges = ''
        for ind, position in enumerate(peer):
            if ind == 0:
                self.node = position
            else:
                self.token_ranges = position
        self.key_names = keynames
        self.table_name = tablename
        self.keyspace = blockkeyspace
        self.storageobj = ""
        self.cntxt = ""

    def __iter__(self):
        return BlockIter(self)

    def __getitem__(self, key):
        keys = self.storageobj.keyList[self.storageobj.__class__.__name__]
        exec ("persistentdict = self.storageobj." + str(keys[0]))
        return persistentdict[key]

    def __setitem__(self, key, val):
        keys = self.storageobj.keyList[self.storageobj.__class__.__name__]
        exec ("persistentdict = self.storageobj." + str(keys[0]))
        persistentdict[key] = val

    def getID(self):
        self.key_names = str(self.key_names).replace('\'', '')
        self.key_names = str(self.key_names).replace('(', '')
        self.key_names = str(self.key_names).replace(')', '')
        self.key_names = str(self.key_names).replace(' ', '')
        identifier = "%s_%s_%s_%s" % (self.keyspace, self.key_names, self.table_name, self.token_ranges)
        identifier = identifier.replace(' ', '')
        return identifier

    def iteritems(self):
        return BlockItemsIter(self)

    def itervalues(self):
        return BlockValuesIter(self)

    def iterkeys(self):
        return BlockIter(self)

class IxBlock(Block):
    def __init__(self, peer, keynames, tablename, blockkeyspace, queryLocations):
       super(IxBlock, self).__init__(peer, keynames, tablename, blockkeyspace)
       self.queryLocations = queryLocations
       print "sorprendentemente tambien hemos llegado hasta aqui"

    def getID(self):
       id = super(IxBlock, self).getID()
       #id = id + str(self.queryLocations)
       return id

class BlockIter(object):
    def __init__(self, iterable):
        self.pos = 0
        self.keys = []
        self.num_keys = 0
        self.iterable = iterable
        self.end = False

        keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
        exec ("self.iterable.storageobj." + str(keys[0]) + ".prefetchManager.pipeq_write[0].send(['query'])")

    def next(self):
        if self.pos == self.num_keys:
            if self.end:
                raise StopIteration
            else:
                keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
                exec ("self.iterable.storageobj." + str(keys[0]) + ".prefetchManager.pipeq_write[0].send(['continue'])")
                exec ("usedpipe = self.iterable.storageobj." + str(keys[0]) + ".prefetchManager.piper_read[0]")
                results = usedpipe.recv()
                if len(results) == 0:
                    raise StopIteration
                else:
                    self.keys = []
                    exec ("currpersistentdict = self.iterable.storageobj." + str(keys[0]))
                    for entry in results:
                        currpersistentdict.dictCache[entry[0]] = [entry[1], 'Sync']
                        self.keys.append(entry[0])
                    self.num_keys = len(self.keys)
                    self.pos = 0
                    if len(results) < 100:
                        self.end = True

        if len(self.keys) == 0:
            print "Error obtaining block_keys in iter.py"

        key = self.keys[self.pos]
        self.pos += 1
        return key

class BlockItemsIter(object):
    def __iter__(self):
        return self

    def __init__(self, iterable):
        self.pos = 0
        self.keys = []
        self.num_keys = 0
        self.iterable = iterable
        self.end = False
        self.persistentDict = ""

        keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
        exec ("self.persistentDict = self.iterable.storageobj." + str(keys[0]))
        self.persistentDict.prefetchManager.pipeq_write[0].send(['query'])

    def next(self):
        if self.pos == self.num_keys:
            if self.end:
                raise StopIteration
            else:
                self.persistentDict.prefetchManager.pipeq_write[0].send(['continue'])
                usedpipe = self.persistentDict.prefetchManager.piper_read[0]
                results = usedpipe.recv()
                if len(results) == 0:
                    raise StopIteration
                else:
                    self.keys = []
                    for entry in results:
                        self.persistentDict.dictCache[entry[0]] = [entry[1], 'Sync']
                        self.keys.append(entry[0])
                    self.num_keys = len(self.keys)
                    self.pos = 0
                    if len(results) < 100:
                        self.end = True

        if len(self.keys) == 0:
            print "Error obtaining block_keys in iter.py"

        key = self.keys[self.pos]
        value = self.persistentDict.dictCache[key]
        self.pos += 1
        return key, value[0]

class BlockValuesIter(object):
    def __iter__(self):
        return self

    def __init__(self, iterable):
        self.pos = 0
        self.keys = []
        self.num_keys = 0
        self.iterable = iterable
        self.end = False
        self.persistentDict = ""

        keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
        exec ("self.persistentDict = self.iterable.storageobj." + str(keys[0]))
        self.persistentDict.prefetchManager.pipeq_write[0].send(['query'])

    def next(self):
        self.persistentDict.reads += 1                                                        #STATISTICS
        if self.pos == self.num_keys:
            if self.end:
                raise StopIteration
            else:
                self.persistentDict.prefetchManager.pipeq_write[0].send(['continue'])
                usedpipe = self.persistentDict.prefetchManager.piper_read[0]
                self.persistentDict.cache_prefetchs += 1                                      #STATISTICS
                self.persistentDict.cache_hits_graph += 'P'                                   #STATISTICS
                start = time.time()                                                           #STATISTICS
                results = usedpipe.recv()
                self.persistentDict.pending_requests_time += time.time() - start              #STATISTICS
                if len(results) == 0:
                    raise StopIteration
                else:
                    self.keys = []
                    for entry in results:
                        self.persistentDict.cache_hits += 1                                   #STATISTICS
                        self.persistentDict.cache_hits_graph += 'X'                           #STATISTICS
                        start = time.time()                                                   #STATISTICS
                        self.persistentDict.dictCache[entry[0]] = [entry[1], 'Sync']
                        self.persistentDict.cache_hits_time += time.time() - start            #STATISTICS
                        self.keys.append(entry[0])
                    self.num_keys = len(self.keys)
                    self.pos = 0
                    if len(results) < 100:
                        self.end = True

        if len(self.keys) == 0:
            self.persistentDict.miss += 1                                                     #STATISTICS
            self.persistentDict.cache_hits_graph += '_'                                       #STATISTICS
            print "Error obtaining block_keys in iter.py"

        value = self.persistentDict.dictCache[self.keys[self.pos]]
        self.pos += 1
        return value[0]

class KeyIter(object):
    blockKeySpace = ''

    def __init__(self, iterable):
        self.pos = 0
        self.ring = []
        self.mypdict = iterable.mypdict
        cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
        session = cluster.connect()
        metadata = cluster.metadata
        ringtokens = metadata.token_map
        tokentohosts = ringtokens.token_to_host_owner
        token_ranges = ''
        starttok = 0
        self.tokenList = []
        for i, token in enumerate(ringtokens.ring):
            self.tokenList.append(int(token.value))
            if ranges_per_block == 1:
                if i == 0:
                    starttok = token
                else:
                    if i < (len(ringtokens.ring)):
                        endtok = token
                        host = str(tokentohosts[starttok])
                        self.ring.append((host, str(i - 1)))
                        starttok = endtok
                    if i == (len(ringtokens.ring) - 1):
                        host = str(tokentohosts[starttok])
                        self.ring.append((host, str(i)))
            else:
                if not (i + 1) % ranges_per_block == 0:
                    if i == 0:
                        starttok = token
                    if (i + 1) % ranges_per_block == 1:
                        starttok = token
                        token_ranges += str(i)
                    else:
                        if i < (len(ringtokens.ring)):
                            endtok = token
                            token_ranges = token_ranges + '_' + str(i)
                            starttok = endtok
                        if i == (len(ringtokens.ring) - 1):
                            token_ranges = token_ranges + '_' + str(i)
                else:
                    if i == 0:
                        starttok = token
                    else:
                        if i < (len(ringtokens.ring)):
                            endtok = token
                            host = str(tokentohosts[starttok])
                            token_ranges = token_ranges + '_' + str(i)
                            self.ring.append((host, token_ranges))
                            token_ranges = ''
                            starttok = endtok

        self.num_peers = len(self.ring)

        session.shutdown()
        cluster.shutdown()

    def next(self):
        start = self.pos
        if start == self.num_peers:
            raise StopIteration
        b = Block(self.ring[self.pos], self.mypdict.dict_keynames, self.mypdict.mypo.name, self.blockkeyspace)
        self.pos += 1
        return b

class IxKeyIter(KeyIter):

    blockKeySpace = ''

    def __init__(self, iterable):
        print "iterable.indexarguments:", iterable.indexarguments
        #super(IxKeyIter, self).__init__(iterable.IxPKeyList)
        super(IxKeyIter, self).__init__(iterable)
        #print "self.ring:", self.ring
        print "self.tokenList:", self.tokenList
        print "InitQuery"
        selects = 'partind'
        keyspace = 'qbeast'
        table = 'MyObj'
        minarguments = {}
        maxarguments = {}
        for argument in iterable.indexarguments:
            if '<' in str(argument):
                val = str(str(argument).split('<')[0])
                maxarguments[val].append(int(str(argument).split('<')[1]))
            if '>' in str(argument):
                val = str(str(argument).split('<')[0])
                minarguments[val].append(int(str(argument).split('<')[1]))
        print "minarguments:", minarguments
        print "maxarguments:", maxarguments
        area = [(0,0,0),(10,10,10)]
        precision = 90
        maxResults = 5
        tokens = self.tokenList # [1]
        qbeastInterface= QbeastIface() # this will be moved to __init__
        qbeastInterface.initQuery(selects, keyspace, table, area, precision, maxResults, tokens)

        self.queryLoc = 'queryLocation'

    def next(self):
        start = self.pos
        if start == self.num_peers:
            raise StopIteration
        b = IxBlock(self.ring[self.pos], self.mypdict.dict_keynames, self.mypdict.mypo.name, self.blockkeyspace, self.queryLoc)
        self.pos += 1
        return b

<commit_msg>still on it<commit_after># author: G. Alomar
from hecuba.datastore import *
from cassandra.cluster import Cluster
from hecuba.dict import *
from app.qbeastiface import *
from conf.hecuba_params import execution_name, ranges_per_block
import time

class Block(object):
    def __init__(self, peer, keynames, tablename, blockkeyspace):
        self.node = peer[0]
        self.token_ranges = ''
        for ind, position in enumerate(peer):
            if ind == 0:
                self.node = position
            else:
                self.token_ranges = position
        self.key_names = keynames
        self.table_name = tablename
        self.keyspace = blockkeyspace
        self.storageobj = ""
        self.cntxt = ""

    def __iter__(self):
        return BlockIter(self)

    def __getitem__(self, key):
        keys = self.storageobj.keyList[self.storageobj.__class__.__name__]
        exec ("persistentdict = self.storageobj." + str(keys[0]))
        return persistentdict[key]

    def __setitem__(self, key, val):
        keys = self.storageobj.keyList[self.storageobj.__class__.__name__]
        exec ("persistentdict = self.storageobj." + str(keys[0]))
        persistentdict[key] = val

    def getID(self):
        self.key_names = str(self.key_names).replace('\'', '')
        self.key_names = str(self.key_names).replace('(', '')
        self.key_names = str(self.key_names).replace(')', '')
        self.key_names = str(self.key_names).replace(' ', '')
        identifier = "%s_%s_%s_%s" % (self.keyspace, self.key_names, self.table_name, self.token_ranges)
        identifier = identifier.replace(' ', '')
        return identifier

    def iteritems(self):
        return BlockItemsIter(self)

    def itervalues(self):
        return BlockValuesIter(self)

    def iterkeys(self):
        return BlockIter(self)

class IxBlock(Block):
    def __init__(self, peer, keynames, tablename, blockkeyspace, queryLocations):
       super(IxBlock, self).__init__(peer, keynames, tablename, blockkeyspace)
       self.queryLocations = queryLocations
       print "sorprendentemente tambien hemos llegado hasta aqui"

    def getID(self):
       id = super(IxBlock, self).getID()
       #id = id + str(self.queryLocations)
       return id

class BlockIter(object):
    def __init__(self, iterable):
        self.pos = 0
        self.keys = []
        self.num_keys = 0
        self.iterable = iterable
        self.end = False

        keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
        exec ("self.iterable.storageobj." + str(keys[0]) + ".prefetchManager.pipeq_write[0].send(['query'])")

    def next(self):
        if self.pos == self.num_keys:
            if self.end:
                raise StopIteration
            else:
                keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
                exec ("self.iterable.storageobj." + str(keys[0]) + ".prefetchManager.pipeq_write[0].send(['continue'])")
                exec ("usedpipe = self.iterable.storageobj." + str(keys[0]) + ".prefetchManager.piper_read[0]")
                results = usedpipe.recv()
                if len(results) == 0:
                    raise StopIteration
                else:
                    self.keys = []
                    exec ("currpersistentdict = self.iterable.storageobj." + str(keys[0]))
                    for entry in results:
                        currpersistentdict.dictCache[entry[0]] = [entry[1], 'Sync']
                        self.keys.append(entry[0])
                    self.num_keys = len(self.keys)
                    self.pos = 0
                    if len(results) < 100:
                        self.end = True

        if len(self.keys) == 0:
            print "Error obtaining block_keys in iter.py"

        key = self.keys[self.pos]
        self.pos += 1
        return key

class BlockItemsIter(object):
    def __iter__(self):
        return self

    def __init__(self, iterable):
        self.pos = 0
        self.keys = []
        self.num_keys = 0
        self.iterable = iterable
        self.end = False
        self.persistentDict = ""

        keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
        exec ("self.persistentDict = self.iterable.storageobj." + str(keys[0]))
        self.persistentDict.prefetchManager.pipeq_write[0].send(['query'])

    def next(self):
        if self.pos == self.num_keys:
            if self.end:
                raise StopIteration
            else:
                self.persistentDict.prefetchManager.pipeq_write[0].send(['continue'])
                usedpipe = self.persistentDict.prefetchManager.piper_read[0]
                results = usedpipe.recv()
                if len(results) == 0:
                    raise StopIteration
                else:
                    self.keys = []
                    for entry in results:
                        self.persistentDict.dictCache[entry[0]] = [entry[1], 'Sync']
                        self.keys.append(entry[0])
                    self.num_keys = len(self.keys)
                    self.pos = 0
                    if len(results) < 100:
                        self.end = True

        if len(self.keys) == 0:
            print "Error obtaining block_keys in iter.py"

        key = self.keys[self.pos]
        value = self.persistentDict.dictCache[key]
        self.pos += 1
        return key, value[0]

class BlockValuesIter(object):
    def __iter__(self):
        return self

    def __init__(self, iterable):
        self.pos = 0
        self.keys = []
        self.num_keys = 0
        self.iterable = iterable
        self.end = False
        self.persistentDict = ""

        keys = self.iterable.storageobj.keyList[self.iterable.storageobj.__class__.__name__]
        exec ("self.persistentDict = self.iterable.storageobj." + str(keys[0]))
        self.persistentDict.prefetchManager.pipeq_write[0].send(['query'])

    def next(self):
        self.persistentDict.reads += 1                                                        #STATISTICS
        if self.pos == self.num_keys:
            if self.end:
                raise StopIteration
            else:
                self.persistentDict.prefetchManager.pipeq_write[0].send(['continue'])
                usedpipe = self.persistentDict.prefetchManager.piper_read[0]
                self.persistentDict.cache_prefetchs += 1                                      #STATISTICS
                self.persistentDict.cache_hits_graph += 'P'                                   #STATISTICS
                start = time.time()                                                           #STATISTICS
                results = usedpipe.recv()
                self.persistentDict.pending_requests_time += time.time() - start              #STATISTICS
                if len(results) == 0:
                    raise StopIteration
                else:
                    self.keys = []
                    for entry in results:
                        self.persistentDict.cache_hits += 1                                   #STATISTICS
                        self.persistentDict.cache_hits_graph += 'X'                           #STATISTICS
                        start = time.time()                                                   #STATISTICS
                        self.persistentDict.dictCache[entry[0]] = [entry[1], 'Sync']
                        self.persistentDict.cache_hits_time += time.time() - start            #STATISTICS
                        self.keys.append(entry[0])
                    self.num_keys = len(self.keys)
                    self.pos = 0
                    if len(results) < 100:
                        self.end = True

        if len(self.keys) == 0:
            self.persistentDict.miss += 1                                                     #STATISTICS
            self.persistentDict.cache_hits_graph += '_'                                       #STATISTICS
            print "Error obtaining block_keys in iter.py"

        value = self.persistentDict.dictCache[self.keys[self.pos]]
        self.pos += 1
        return value[0]

class KeyIter(object):
    blockKeySpace = ''

    def __init__(self, iterable):
        self.pos = 0
        self.ring = []
        self.mypdict = iterable.mypdict
        cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
        session = cluster.connect()
        metadata = cluster.metadata
        ringtokens = metadata.token_map
        tokentohosts = ringtokens.token_to_host_owner
        token_ranges = ''
        starttok = 0
        self.tokenList = []
        for i, token in enumerate(ringtokens.ring):
            self.tokenList.append(int(token.value))
            if ranges_per_block == 1:
                if i == 0:
                    starttok = token
                else:
                    if i < (len(ringtokens.ring)):
                        endtok = token
                        host = str(tokentohosts[starttok])
                        self.ring.append((host, str(i - 1)))
                        starttok = endtok
                    if i == (len(ringtokens.ring) - 1):
                        host = str(tokentohosts[starttok])
                        self.ring.append((host, str(i)))
            else:
                if not (i + 1) % ranges_per_block == 0:
                    if i == 0:
                        starttok = token
                    if (i + 1) % ranges_per_block == 1:
                        starttok = token
                        token_ranges += str(i)
                    else:
                        if i < (len(ringtokens.ring)):
                            endtok = token
                            token_ranges = token_ranges + '_' + str(i)
                            starttok = endtok
                        if i == (len(ringtokens.ring) - 1):
                            token_ranges = token_ranges + '_' + str(i)
                else:
                    if i == 0:
                        starttok = token
                    else:
                        if i < (len(ringtokens.ring)):
                            endtok = token
                            host = str(tokentohosts[starttok])
                            token_ranges = token_ranges + '_' + str(i)
                            self.ring.append((host, token_ranges))
                            token_ranges = ''
                            starttok = endtok

        self.num_peers = len(self.ring)

        session.shutdown()
        cluster.shutdown()

    def next(self):
        start = self.pos
        if start == self.num_peers:
            raise StopIteration
        b = Block(self.ring[self.pos], self.mypdict.dict_keynames, self.mypdict.mypo.name, self.blockkeyspace)
        self.pos += 1
        return b

class IxKeyIter(KeyIter):

    blockKeySpace = ''

    def __init__(self, iterable):
        print "iterable.indexarguments:", iterable.indexarguments
        #super(IxKeyIter, self).__init__(iterable.IxPKeyList)
        super(IxKeyIter, self).__init__(iterable)
        #print "self.ring:", self.ring
        print "self.tokenList:", self.tokenList
        print "InitQuery"
        selects = 'partind'
        keyspace = 'qbeast'
        table = 'MyObj'
        minarguments = {}
        maxarguments = {}
        for argument in iterable.indexarguments:
            if '<' in str(argument):
                val = str(str(argument).split('<')[0])
                maxarguments[val].append(int(str(argument).split('<')[1]))
            if '>' in str(argument):
                val = str(str(argument).split('>')[0])
                minarguments[val].append(int(str(argument).split('>')[1]))
        print "minarguments:", minarguments
        print "maxarguments:", maxarguments
        area = [(0,0,0),(10,10,10)]
        precision = 90
        maxResults = 5
        tokens = self.tokenList # [1]
        qbeastInterface= QbeastIface() # this will be moved to __init__
        qbeastInterface.initQuery(selects, keyspace, table, area, precision, maxResults, tokens)

        self.queryLoc = 'queryLocation'

    def next(self):
        start = self.pos
        if start == self.num_peers:
            raise StopIteration
        b = IxBlock(self.ring[self.pos], self.mypdict.dict_keynames, self.mypdict.mypo.name, self.blockkeyspace, self.queryLoc)
        self.pos += 1
        return b

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

"""
Verify that we can import and use the contents of Platform and Tool
modules directly.
"""

import os
import re
import sys

# must do this here, since TestSCons will chdir
tooldir = os.path.join(os.getcwd(), 'src', 'engine', 'SCons', 'Tool')

import TestSCons
test = TestSCons.TestSCons()

# Before executing the any of the platform or tool modules, add some
# null entries to the environment $PATH variable to make sure there's
# no code that tries to index elements from the list before making sure
# they're non-null.
# (This was a problem in checkpoint release 0.97.d020070809.)
os.environ['PATH'] = os.pathsep + os.environ['PATH'] + \
                     os.pathsep + os.pathsep + '/no/such/dir' + os.pathsep

SConstruct_path = test.workpath('SConstruct')

platforms = [
    'aix',
    'cygwin',
    'darwin',
    'hpux',
    'irix',
    'os2',
    'posix',
    'sunos',
    'win32'
]

for platform in platforms:
    test.write('SConstruct', """
print "Platform %(platform)s"
env = Environment(platform = '%(platform)s')
import SCons.Platform.%(platform)s
x = SCons.Platform.%(platform)s.generate
""" % locals())
    test.run()

ignore = ('__init__.py',
        # Can't import these everywhere due to Windows registry dependency.
        '386asm.py', 'linkloc.py',
        # Directory of common stuff for MSVC and MSVS
        'MSCommon',
        # Sun pkgchk and pkginfo common stuff
        'sun_pkg.py',
        )
tools = []
for name in os.listdir(tooldir):
    if name in ignore: continue
    if name[-3:] == '.py':
        if name[-8:] not in ('Tests.py', 'ommon.py'):
            tools.append(name[:-3])
    elif os.path.exists(os.path.join(tooldir,name,'__init__.py')):
        tools.append(name)
tools.sort()    # why not?

#if sys.platform == 'win32':
# Just comment out (for now?) due to registry dependency.
#    tools.extend([
#        '386asm',
#        'linkloc',
#    ])

# Intel no compiler warning..
intel_no_compiler_warning = """
scons: warning: Failed to find Intel compiler for version='None', abi='[^']*'
""" + TestSCons.file_expr

# Intel no top dir warning.
intel_no_top_dir_warning = """
scons: warning: Can't find Intel compiler top dir for version='None', abi='[^']*'
""" + TestSCons.file_expr

# Intel no license directory warning
intel_license_warning = re.escape("""
scons: warning: Intel license dir was not found.  Tried using the INTEL_LICENSE_FILE environment variable (), the registry () and the default path (C:\Program Files\Common Files\Intel\Licenses).  Using the default path as a last resort.
""") + TestSCons.file_expr

intel_warnings = [
    re.compile(intel_license_warning),
    re.compile(intel_no_compiler_warning),
    re.compile(intel_no_compiler_warning + intel_license_warning),
    re.compile(intel_no_top_dir_warning),
    re.compile(intel_no_top_dir_warning + intel_license_warning),
]

moc = test.where_is('moc')
if moc:
    import os.path

    qtdir = os.path.dirname(os.path.dirname(moc))

    qt_err = r"""
scons: warning: Could not detect qt, using moc executable as a hint \(QTDIR=%(qtdir)s\)
""" % locals()

else:

    qt_err = """
scons: warning: Could not detect qt, using empty QTDIR
"""

qt_warnings = [ re.compile(qt_err + TestSCons.file_expr) ]

error_output = {
    'icl' : intel_warnings,
    'intelc' : intel_warnings,
    'qt' : qt_warnings,
}

# An SConstruct for importing Tool names that have illegal characters
# for Python variable names.
indirect_import = """\
print "Tool %(tool)s (indirect)"
env = Environment(tools = ['%(tool)s'])

SCons = __import__('SCons.Tool.%(tool)s', globals(), locals(), [])
m = getattr(SCons.Tool, '%(tool)s')
env = Environment()
m.generate(env)
"""

# An SConstruct for importing Tool names "normally."
direct_import = """\
print "Tool %(tool)s (direct)"
env = Environment(tools = ['%(tool)s'])

import SCons.Tool.%(tool)s
env = Environment()
SCons.Tool.%(tool)s.exists(env)
SCons.Tool.%(tool)s.generate(env)
"""

failures = []
for tool in tools:
    if tool[0] in '0123456789' or '+' in tool or tool in ('as',):
        test.write('SConstruct', indirect_import % locals())
    else:
        test.write('SConstruct', direct_import % locals())
    test.run(stderr=None)
    stderr = test.stderr()
    if stderr:
        matched = None
        for expression in error_output.get(tool, []):
            if expression.match(stderr):
                matched = 1
                break
        if not matched:
            print "Failed importing '%s', stderr:" % tool
            print stderr
            failures.append(tool)

test.fail_test(len(failures))

test.pass_test()

# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=4 shiftwidth=4:
<commit_msg>Filter out more bogus files from import.py test. It was trying to use emacs and other junk files which were left in the source tree and failing<commit_after>#!/usr/bin/env python
#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

"""
Verify that we can import and use the contents of Platform and Tool
modules directly.
"""

import os
import re
import sys

# must do this here, since TestSCons will chdir
tooldir = os.path.join(os.getcwd(), 'src', 'engine', 'SCons', 'Tool')

import TestSCons
test = TestSCons.TestSCons()

# Before executing the any of the platform or tool modules, add some
# null entries to the environment $PATH variable to make sure there's
# no code that tries to index elements from the list before making sure
# they're non-null.
# (This was a problem in checkpoint release 0.97.d020070809.)
os.environ['PATH'] = os.pathsep + os.environ['PATH'] + \
                     os.pathsep + os.pathsep + '/no/such/dir' + os.pathsep

SConstruct_path = test.workpath('SConstruct')

platforms = [
    'aix',
    'cygwin',
    'darwin',
    'hpux',
    'irix',
    'os2',
    'posix',
    'sunos',
    'win32'
]

for platform in platforms:
    test.write('SConstruct', """
print "Platform %(platform)s"
env = Environment(platform = '%(platform)s')
import SCons.Platform.%(platform)s
x = SCons.Platform.%(platform)s.generate
""" % locals())
    test.run()

ignore = ('__init__.py',
        # Can't import these everywhere due to Windows registry dependency.
        '386asm.py', 'linkloc.py',
        # Directory of common stuff for MSVC and MSVS
        'MSCommon',
        # Sun pkgchk and pkginfo common stuff
        'sun_pkg.py',
        )
tools = []
for name in os.listdir(tooldir):
    if name in ignore: continue
    if name[0] == '#': continue
    if name[0:1] == '.': continue
    if name[-1]  == '~' : continue
    if name[-3:] == '.py':
        if name[-8:] not in ('Tests.py', 'ommon.py'):
            tools.append(name[:-3])
    elif os.path.exists(os.path.join(tooldir,name,'__init__.py')):
        tools.append(name)
tools.sort()    # why not?

#if sys.platform == 'win32':
# Just comment out (for now?) due to registry dependency.
#    tools.extend([
#        '386asm',
#        'linkloc',
#    ])

# Intel no compiler warning..
intel_no_compiler_warning = """
scons: warning: Failed to find Intel compiler for version='None', abi='[^']*'
""" + TestSCons.file_expr

# Intel no top dir warning.
intel_no_top_dir_warning = """
scons: warning: Can't find Intel compiler top dir for version='None', abi='[^']*'
""" + TestSCons.file_expr

# Intel no license directory warning
intel_license_warning = re.escape("""
scons: warning: Intel license dir was not found.  Tried using the INTEL_LICENSE_FILE environment variable (), the registry () and the default path (C:\Program Files\Common Files\Intel\Licenses).  Using the default path as a last resort.
""") + TestSCons.file_expr

intel_warnings = [
    re.compile(intel_license_warning),
    re.compile(intel_no_compiler_warning),
    re.compile(intel_no_compiler_warning + intel_license_warning),
    re.compile(intel_no_top_dir_warning),
    re.compile(intel_no_top_dir_warning + intel_license_warning),
]

moc = test.where_is('moc')
if moc:
    import os.path

    qtdir = os.path.dirname(os.path.dirname(moc))

    qt_err = r"""
scons: warning: Could not detect qt, using moc executable as a hint \(QTDIR=%(qtdir)s\)
""" % locals()

else:

    qt_err = """
scons: warning: Could not detect qt, using empty QTDIR
"""

qt_warnings = [ re.compile(qt_err + TestSCons.file_expr) ]

error_output = {
    'icl' : intel_warnings,
    'intelc' : intel_warnings,
    'qt' : qt_warnings,
}

# An SConstruct for importing Tool names that have illegal characters
# for Python variable names.
indirect_import = """\
print "Tool %(tool)s (indirect)"
env = Environment(tools = ['%(tool)s'])

SCons = __import__('SCons.Tool.%(tool)s', globals(), locals(), [])
m = getattr(SCons.Tool, '%(tool)s')
env = Environment()
m.generate(env)
"""

# An SConstruct for importing Tool names "normally."
direct_import = """\
print "Tool %(tool)s (direct)"
env = Environment(tools = ['%(tool)s'])

import SCons.Tool.%(tool)s
env = Environment()
SCons.Tool.%(tool)s.exists(env)
SCons.Tool.%(tool)s.generate(env)
"""

failures = []
for tool in tools:
    if tool[0] in '0123456789' or '+' in tool or tool in ('as',):
        test.write('SConstruct', indirect_import % locals())
    else:
        test.write('SConstruct', direct_import % locals())
    test.run(stderr=None)
    stderr = test.stderr()
    if stderr:
        matched = None
        for expression in error_output.get(tool, []):
            if expression.match(stderr):
                matched = 1
                break
        if not matched:
            print "Failed importing '%s', stderr:" % tool
            print stderr
            failures.append(tool)

test.fail_test(len(failures))

test.pass_test()

# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=4 shiftwidth=4:
<|endoftext|>


<commit_before>"""
hellofriend.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""
print= input("Tell me your name!")
print=("Hi {0}!".format(user))
user=input("how old are you {0}?".format(user))
print=("Well {0}, python is (24-{1}) years older than you!")<commit_msg>Updated from Brython Server: 9/10/2015 2:45:42 PM<commit_after>"""
hellofriend.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""
print= input("Tell me your name!")
user=("Hi {0}!".format(user))
user=input("how old are you {0}?".format(user))
print=("Well {0}, python is (24-{1}) years older than you!")<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
from urllib2 import urlopen

def find_food(data):
    """Give me a file like object or a URL"""
    if type(data) == str or type(data) == unicode:
        data = urlopen(data)
    soup = BeautifulSoup(data)

    for p in soup.findAll('p'):
        if "food" in str(p) or "Food" in str(p):
            return True

    return False
<commit_msg>Helper function that filters for food events<commit_after>from bs4 import BeautifulSoup
from urllib2 import urlopen

def find_food(data):
    """
    Give me a file like object or a URL, returns True or False based on if it finds
    food
    """

    if type(data) == str or type(data) == unicode:
        data = urlopen(data)
    soup = BeautifulSoup(data)

    for p in soup.findAll('p'):
        if "food" in str(p) or "Food" in str(p):
            return True

    return False

def give_food(events):
    """Go through a list of events and return the ones that have food"""

    output = []

    for event in events:
        if find_food(event.url):
            output.append(event)

    return output
<|endoftext|>


<commit_before>"""
Base notification classes
"""

import json
import six

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template import TemplateDoesNotExist
from django.template.loader import render_to_string
from django.utils import timezone

from .models import SentNotification

class NotificationBase(object):
    """
    base class for sending notifications
    """

    render_types = []
    template_name = None
    context = None

    def get_context_data(self):
        """
        :return: the context data for rendering the email or text template
        """

        context = self.context or {}

        if settings.DEBUG:
            context['base_url'] = ''
        else:
            site = Site.objects.get_current()
            context['base_url'] = 'http://' + site.domain

        return context

    def send(self, raise_exception=False):
        """
        Handles the preparing the notification for sending. Called to trigger the send from code.
        If raise_exception is True, it will raise any exceptions rather than simply logging them.
        returns boolean whether or not the notification was sent successfully
        """
        context = self.get_context_data()

        recipients = self.get_recipients()
        text_content = self.render('text', context)
        html_content = self.render('html', context)
        sent_from = self.get_sent_from()
        subject = self.get_subject()
        extra_data = self.get_extra_data()

        sent_notification = SentNotification(
            recipients=','.join(recipients),
            text_content=text_content,
            html_content=html_content,
            sent_from=sent_from,
            subject=subject,
            extra_data=json.dumps(extra_data) if extra_data else None,
            notification_class='{}.{}'.format(self.__class__.__module__, self.__class__.__name__)
        )

        return self.resend(sent_notification, raise_exception=raise_exception)

    def get_recipients(self):
        """
        Returns a list of recipients. However the subclass defines that. (emails, phone numbers, etc)
        """

        raise NotImplementedError('Must implement get_recipients.')

    def get_extra_data(self):
        """
        Returns a dictionary of extra data to be stored, and used for sending.
        MUST BE JSON SERIALIZABLE
        """

        return {}

    def get_sent_from(self):
        """
        Returns a "sent from" string. However the subclass defines that. (email, phone number, etc)
        """

        raise NotImplementedError('Must implement get_recipients.')

    def get_subject(self):
        """
        Returns a subject string. Optional.
        """

        return None

    def render(self, render_type, context):
        """
        Renders the template

        :param render_type: the content type to render
        :param context: context data dictionary
        :return: the rendered content
        """

        if render_type not in self.render_types:
            return None

        try:
            content = render_to_string('herald/{}/{}.{}'.format(
                render_type,
                self.template_name,
                'txt' if render_type == 'text' else render_type
            ), context)

            return content
        except TemplateDoesNotExist:
            return None

    @staticmethod
    def get_demo_args():
        """
        Returns iterable of arguments needed to initialize notification for demo purposes
        Usually you want to generate dummy data here for testing
        """
        return []

    @classmethod
    def resend(cls, sent_notification, raise_exception=False):
        """
        Takes a saved sent_notification and sends it again.
        returns boolean whether or not the notification was sent successfully
        """

        try:
            cls._send(
                sent_notification.get_recipients(),
                sent_notification.text_content,
                sent_notification.html_content,
                sent_notification.sent_from,
                sent_notification.subject,
                sent_notification.get_extra_data()
            )
            sent_notification.status = sent_notification.STATUS_SUCCESS
        except Exception as exc:  # pylint: disable=W0703
            # we want to handle any exception whatsoever
            sent_notification.status = sent_notification.STATUS_FAILED
            sent_notification.error_message = six.text_type(exc)

            if raise_exception:
                raise exc

        sent_notification.date_sent = timezone.now()
        sent_notification.save()

        return sent_notification.status == sent_notification.STATUS_SUCCESS

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None):
        """
        Handles the actual sending of the notification. Sub classes must override this
        """

        raise NotImplementedError('Must implement send.')

class EmailNotification(NotificationBase):
    """
    Base class for email notifications
    """

    render_types = ['text', 'html']
    from_email = None
    subject = None
    to_emails = None
    bcc = None
    cc = None  # pylint: disable=C0103
    headers = None
    reply_to = None

    def get_context_data(self):
        context = super(EmailNotification, self).get_context_data()
        context['subject'] = self.subject
        return context

    def get_recipients(self):
        return self.to_emails

    def get_sent_from(self):
        from_email = self.from_email
        if not from_email:
            from_email = settings.DEFAULT_FROM_EMAIL

        return from_email

    def get_subject(self):
        return self.subject

    def get_extra_data(self):
        extra_data = {}

        if self.bcc:
            extra_data['bcc'] = self.bcc

        if self.cc:
            extra_data['cc'] = self.cc

        if self.headers:
            extra_data['headers'] = self.headers

        if self.reply_to:
            extra_data['reply_to'] = self.reply_to

        return extra_data

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None):
        if extra_data is None:
            extra_data = {}

        mail = EmailMultiAlternatives(
            subject=subject,
            body=text_content,
            from_email=sent_from,
            to=recipients,
            bcc=extra_data.get('bcc', None),
            headers=extra_data.get('headers', None),
            cc=extra_data.get('cc', None),
            reply_to=extra_data.get('reply_to', None)
        )

        if html_content:
            mail.attach_alternative(html_content, 'text/html')

        mail.send()

class TwilioTextNotification(NotificationBase):
    """
    Base class for text notifications.
    Uses twilio
    """

    render_types = ['text']
    from_number = None
    to_number = None

    def get_recipients(self):
        return [self.to_number]

    def get_sent_from(self):
        from_number = self.from_number
        if not from_number:
            try:
                from_number = settings.TWILIO_DEFAULT_FROM_NUMBER
            except AttributeError:
                raise Exception(
                    'TWILIO_DEFAULT_FROM_NUMBER setting is required for sending a TwilioTextNotification'
                )

        return from_number

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None):
        try:
            import twilio
        except ImportError:
            raise Exception('Twilio is required for sending a TwilioTextNotification.')

        try:
            account_sid = settings.TWILIO_ACCOUNT_SID
            auth_token = settings.TWILIO_AUTH_TOKEN
        except AttributeError:
            raise Exception(
                'TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN settings are required for sending a TwilioTextNotification'
            )

        client = twilio.rest.TwilioRestClient(account_sid, auth_token)

        client.messages.create(
            body=text_content,
            to=recipients[0],
            from_=sent_from
        )
<commit_msg>It was difficult to tell why it was returning 'None'.  We would never want to send an email with None as the content, and in test mode, None is not helpful.  Better to let the exception raise and get handled by sentry or a program crash.<commit_after>"""
Base notification classes
"""

import json
import six

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template import TemplateDoesNotExist
from django.template.loader import render_to_string
from django.utils import timezone

from .models import SentNotification

class NotificationBase(object):
    """
    base class for sending notifications
    """

    render_types = []
    template_name = None
    context = None

    def get_context_data(self):
        """
        :return: the context data for rendering the email or text template
        """

        context = self.context or {}

        if settings.DEBUG:
            context['base_url'] = ''
        else:
            site = Site.objects.get_current()
            context['base_url'] = 'http://' + site.domain

        return context

    def send(self, raise_exception=False):
        """
        Handles the preparing the notification for sending. Called to trigger the send from code.
        If raise_exception is True, it will raise any exceptions rather than simply logging them.
        returns boolean whether or not the notification was sent successfully
        """
        context = self.get_context_data()

        recipients = self.get_recipients()
        text_content = self.render('text', context)
        html_content = self.render('html', context)
        sent_from = self.get_sent_from()
        subject = self.get_subject()
        extra_data = self.get_extra_data()

        sent_notification = SentNotification(
            recipients=','.join(recipients),
            text_content=text_content,
            html_content=html_content,
            sent_from=sent_from,
            subject=subject,
            extra_data=json.dumps(extra_data) if extra_data else None,
            notification_class='{}.{}'.format(self.__class__.__module__, self.__class__.__name__)
        )

        return self.resend(sent_notification, raise_exception=raise_exception)

    def get_recipients(self):
        """
        Returns a list of recipients. However the subclass defines that. (emails, phone numbers, etc)
        """

        raise NotImplementedError('Must implement get_recipients.')

    def get_extra_data(self):
        """
        Returns a dictionary of extra data to be stored, and used for sending.
        MUST BE JSON SERIALIZABLE
        """

        return {}

    def get_sent_from(self):
        """
        Returns a "sent from" string. However the subclass defines that. (email, phone number, etc)
        """

        raise NotImplementedError('Must implement get_recipients.')

    def get_subject(self):
        """
        Returns a subject string. Optional.
        """

        return None

    def render(self, render_type, context):
        """
        Renders the template

        :param render_type: the content type to render
        :param context: context data dictionary
        :return: the rendered content
        """

        assert(render_type in self.render_types, 'Invalid Render Type')

        content = render_to_string('herald/{}/{}.{}'.format(
            render_type,
            self.template_name,
            'txt' if render_type == 'text' else render_type
        ), context)

        return content

    @staticmethod
    def get_demo_args():
        """
        Returns iterable of arguments needed to initialize notification for demo purposes
        Usually you want to generate dummy data here for testing
        """
        return []

    @classmethod
    def resend(cls, sent_notification, raise_exception=False):
        """
        Takes a saved sent_notification and sends it again.
        returns boolean whether or not the notification was sent successfully
        """

        try:
            cls._send(
                sent_notification.get_recipients(),
                sent_notification.text_content,
                sent_notification.html_content,
                sent_notification.sent_from,
                sent_notification.subject,
                sent_notification.get_extra_data()
            )
            sent_notification.status = sent_notification.STATUS_SUCCESS
        except Exception as exc:  # pylint: disable=W0703
            # we want to handle any exception whatsoever
            sent_notification.status = sent_notification.STATUS_FAILED
            sent_notification.error_message = six.text_type(exc)

            if raise_exception:
                raise exc

        sent_notification.date_sent = timezone.now()
        sent_notification.save()

        return sent_notification.status == sent_notification.STATUS_SUCCESS

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None):
        """
        Handles the actual sending of the notification. Sub classes must override this
        """

        raise NotImplementedError('Must implement send.')

class EmailNotification(NotificationBase):
    """
    Base class for email notifications
    """

    render_types = ['text', 'html']
    from_email = None
    subject = None
    to_emails = None
    bcc = None
    cc = None  # pylint: disable=C0103
    headers = None
    reply_to = None

    def get_context_data(self):
        context = super(EmailNotification, self).get_context_data()
        context['subject'] = self.subject
        return context

    def get_recipients(self):
        return self.to_emails

    def get_sent_from(self):
        from_email = self.from_email
        if not from_email:
            from_email = settings.DEFAULT_FROM_EMAIL

        return from_email

    def get_subject(self):
        return self.subject

    def get_extra_data(self):
        extra_data = {}

        if self.bcc:
            extra_data['bcc'] = self.bcc

        if self.cc:
            extra_data['cc'] = self.cc

        if self.headers:
            extra_data['headers'] = self.headers

        if self.reply_to:
            extra_data['reply_to'] = self.reply_to

        return extra_data

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None):
        if extra_data is None:
            extra_data = {}

        mail = EmailMultiAlternatives(
            subject=subject,
            body=text_content,
            from_email=sent_from,
            to=recipients,
            bcc=extra_data.get('bcc', None),
            headers=extra_data.get('headers', None),
            cc=extra_data.get('cc', None),
            reply_to=extra_data.get('reply_to', None)
        )

        if html_content:
            mail.attach_alternative(html_content, 'text/html')

        mail.send()

class TwilioTextNotification(NotificationBase):
    """
    Base class for text notifications.
    Uses twilio
    """

    render_types = ['text']
    from_number = None
    to_number = None

    def get_recipients(self):
        return [self.to_number]

    def get_sent_from(self):
        from_number = self.from_number
        if not from_number:
            try:
                from_number = settings.TWILIO_DEFAULT_FROM_NUMBER
            except AttributeError:
                raise Exception(
                    'TWILIO_DEFAULT_FROM_NUMBER setting is required for sending a TwilioTextNotification'
                )

        return from_number

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None):
        try:
            import twilio
        except ImportError:
            raise Exception('Twilio is required for sending a TwilioTextNotification.')

        try:
            account_sid = settings.TWILIO_ACCOUNT_SID
            auth_token = settings.TWILIO_AUTH_TOKEN
        except AttributeError:
            raise Exception(
                'TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN settings are required for sending a TwilioTextNotification'
            )

        client = twilio.rest.TwilioRestClient(account_sid, auth_token)

        client.messages.create(
            body=text_content,
            to=recipients[0],
            from_=sent_from
        )
<|endoftext|>


<commit_before>"""
Base notification classes
"""

import json
from email.mime.base import MIMEBase
from mimetypes import guess_type

import jsonpickle
import re
import six

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template import TemplateDoesNotExist
from django.template.loader import render_to_string
from django.utils import timezone
from django.core.files import File

from .models import SentNotification

class NotificationBase(object):
    """
    base class for sending notifications
    """

    render_types = []
    template_name = None
    context = None
    user = None
    can_disable = True
    verbose_name = None

    def get_context_data(self):
        """
        :return: the context data for rendering the email or text template
        """

        context = self.context or {}

        site = Site.objects.get_current()
        context['base_url'] = 'http://' + site.domain

        return context

    @classmethod
    def get_verbose_name(cls):
        if cls.verbose_name:
            return cls.verbose_name
        else:
            return re.sub(
                r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))',
                r' \1',
                cls.__name__
            )

    @classmethod
    def get_class_path(cls):
        return '{}.{}'.format(cls.__module__, cls.__name__)

    def send(self, raise_exception=False, user=None):
        """
        Handles the preparing the notification for sending. Called to trigger the send from code.
        If raise_exception is True, it will raise any exceptions rather than simply logging them.
        returns boolean whether or not the notification was sent successfully
        """
        context = self.get_context_data()

        recipients = self.get_recipients()

        if 'text' in self.render_types:
            text_content = self.render('text', context)
        else:
            text_content = None

        if 'html' in self.render_types:
            html_content = self.render('html', context)
        else:
            html_content = None

        sent_from = self.get_sent_from()
        subject = self.get_subject()
        extra_data = self.get_extra_data()

        sent_notification = SentNotification(
            recipients=','.join(recipients),
            text_content=text_content,
            html_content=html_content,
            sent_from=sent_from,
            subject=subject,
            extra_data=json.dumps(extra_data) if extra_data else None,
            notification_class=self.get_class_path(),
            attachments=self._get_encoded_attachments(),
            user=user,
        )

        return self.resend(sent_notification, raise_exception=raise_exception)

    def _get_encoded_attachments(self):
        attachments = self.get_attachments()

        new_attachments = []

        for attachment in attachments or []:
            if isinstance(attachment, File):
                # cannot do with attachment.open() since django 1.11 doesn't support that
                attachment.open()
                new_attachments.append((attachment.name, attachment.read(), guess_type(attachment.name)[0]))
                attachment.close()
            else:
                new_attachments.append(attachment)

        return jsonpickle.dumps(new_attachments)

    @staticmethod
    def _delete_expired_notifications():
        """
        This deletes any notifications that have passed the retention time setting
        """
        retention_time = getattr(settings, 'HERALD_NOTIFICATION_RETENTION_TIME', None)

        if not retention_time:
            return

        cutoff_date = timezone.now() - retention_time

        notifications = SentNotification.objects.filter(date_sent__lt=cutoff_date)
        count = notifications.delete()
        print('Deleted {} expired notifications.'.format(count))

    def get_recipients(self):
        """
        Returns a list of recipients. However the subclass defines that. (emails, phone numbers, etc)
        """

        raise NotImplementedError('Must implement get_recipients.')

    def get_extra_data(self):
        """
        Returns a dictionary of extra data to be stored, and used for sending.
        MUST BE JSON SERIALIZABLE
        """

        return {}

    def get_sent_from(self):
        """
        Returns a "sent from" string. However the subclass defines that. (email, phone number, etc)
        """

        raise NotImplementedError('Must implement get_sent_from.')

    def get_subject(self):
        """
        Returns a subject string. Optional.
        """

        return None

    def get_attachments(self):
        """
        Return a list of attachments or None.

        This only works with email.
        """
        return None

    def render(self, render_type, context):
        """
        Renders the template

        :param render_type: the content type to render
        :param context: context data dictionary
        :return: the rendered content
        """

        assert render_type in self.render_types, 'Invalid Render Type'

        try:
            content = render_to_string('herald/{}/{}.{}'.format(
                render_type,
                self.template_name,
                'txt' if render_type == 'text' else render_type
            ), context)
        except TemplateDoesNotExist:
            content = None

            if settings.DEBUG:
                raise

        return content

    @staticmethod
    def get_demo_args():
        """
        Returns iterable of arguments needed to initialize notification for demo purposes
        Usually you want to generate dummy data here for testing
        """
        return []

    @classmethod
    def resend(cls, sent_notification, raise_exception=False):
        """
        Takes a saved sent_notification and sends it again.
        returns boolean whether or not the notification was sent successfully
        """

        # handle skipping a notification based on user preference
        if hasattr(sent_notification.user, 'usernotification'):
            notifications = sent_notification.user.usernotification
            if notifications.disabled_notifications.filter(notification_class=cls.get_class_path()).exists():
                sent_notification.date_sent = timezone.now()
                sent_notification.status = sent_notification.STATUS_USER_DISABLED
                sent_notification.save()
                return True

        try:
            cls._send(
                sent_notification.get_recipients(),
                sent_notification.text_content,
                sent_notification.html_content,
                sent_notification.sent_from,
                sent_notification.subject,
                sent_notification.get_extra_data(),
                sent_notification.get_attachments(),
            )
            sent_notification.status = sent_notification.STATUS_SUCCESS
        except Exception as exc:  # pylint: disable=W0703
            # we want to handle any exception whatsoever
            sent_notification.status = sent_notification.STATUS_FAILED
            sent_notification.error_message = six.text_type(exc)

            if raise_exception:
                raise exc

        sent_notification.date_sent = timezone.now()
        sent_notification.save()

        cls._delete_expired_notifications()

        return sent_notification.status == sent_notification.STATUS_SUCCESS

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None,
              attachments=None):
        """
        Handles the actual sending of the notification. Sub classes must override this
        """

        raise NotImplementedError('Must implement send.')

class EmailNotification(NotificationBase):
    """
    Base class for email notifications
    """

    render_types = ['text', 'html']
    from_email = None
    subject = None
    to_emails = None
    bcc = None
    cc = None  # pylint: disable=C0103
    headers = None
    reply_to = None
    attachments = None

    def get_context_data(self):
        context = super(EmailNotification, self).get_context_data()
        context['subject'] = self.subject
        return context

    def get_recipients(self):
        return self.to_emails

    def get_sent_from(self):
        from_email = self.from_email

        if not from_email:
            from_email = settings.DEFAULT_FROM_EMAIL

        return from_email

    def get_subject(self):
        return self.subject

    def get_extra_data(self):
        extra_data = {}

        if self.bcc:
            extra_data['bcc'] = self.bcc

        if self.cc:
            extra_data['cc'] = self.cc

        if self.headers:
            extra_data['headers'] = self.headers

        if self.reply_to:
            extra_data['reply_to'] = self.reply_to

        return extra_data

    def get_attachments(self):
        """
        Return a list of attachments or None.

        This only works with email.
        """
        return self.attachments

    def render(self, render_type, context):
        if render_type == 'text' and getattr(settings, 'HERALD_HTML2TEXT_ENABLED', False):
            try:
                content = super(EmailNotification, self).render('text', context)

            # Render plain text version from HTML
            except TemplateDoesNotExist:
                content = None

            if content is None:
                content = self.get_html2text_converter().handle(super(EmailNotification, self).render('html', context))
        else:
            content = super(EmailNotification, self).render(render_type, context)

        return content

    @staticmethod
    def get_html2text_converter():
        try:
            import html2text
        except ImportError:
            raise Exception(
                "HTML2Text is required for sending an EmailNotification with auto HTML to text conversion."
            )

        h = html2text.HTML2Text()

        if hasattr(settings, 'HERALD_HTML2TEXT_CONFIG'):
            for k, v in settings.HERALD_HTML2TEXT_CONFIG.items():
                setattr(h, k, v)

        return h

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None,
              attachments=None):

        extra_data = extra_data or {}

        mail = EmailMultiAlternatives(
            subject=subject,
            body=text_content,
            from_email=sent_from,
            to=recipients,
            bcc=extra_data.get('bcc', None),
            headers=extra_data.get('headers', None),
            cc=extra_data.get('cc', None),
            reply_to=extra_data.get('reply_to', None),
        )

        if html_content:
            mail.attach_alternative(html_content, 'text/html')

        for attachment in (attachments or []):
            # All mimebase attachments must have a Content-ID or Content-Disposition header
            # or they will show up as unnamed attachments"
            if isinstance(attachment, MIMEBase):
                if attachment.get('Content-ID', False):
                    # if you are sending attachment with content id,
                    # subtype must be 'related'.
                    mail.mixed_subtype = 'related'

                mail.attach(attachment)
            else:
                mail.attach(*attachment)

        mail.send()

class TwilioTextNotification(NotificationBase):
    """
    Base class for text notifications.
    Uses twilio
    """

    render_types = ['text']
    from_number = None
    to_number = None

    def get_recipients(self):
        return [self.to_number]

    def get_sent_from(self):
        from_number = self.from_number
        if not from_number:
            try:
                from_number = settings.TWILIO_DEFAULT_FROM_NUMBER
            except AttributeError:
                raise Exception(
                    'TWILIO_DEFAULT_FROM_NUMBER setting is required for sending a TwilioTextNotification'
                )

        return from_number

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None,
              attachments=None):
        try:
            # twilio version 6
            from twilio.rest import Client
        except ImportError:
            try:
                # twillio version < 6
                from twilio.rest import TwilioRestClient as Client
            except ImportError:
                raise Exception(
                    "Twilio is required for sending a TwilioTextNotification."
                )

        try:
            account_sid = settings.TWILIO_ACCOUNT_SID
            auth_token = settings.TWILIO_AUTH_TOKEN
        except AttributeError:
            raise Exception(
                "TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN settings are required for sending a TwilioTextNotification"
            )

        client = Client(account_sid, auth_token)

        for recipient in recipients:
            client.messages.create(body=text_content, to=recipient, from_=sent_from)
<commit_msg>Configurable raise exception on missing templates (#74)<commit_after>"""
Base notification classes
"""

import json
from email.mime.base import MIMEBase
from mimetypes import guess_type

import jsonpickle
import re
import six

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMultiAlternatives
from django.template import TemplateDoesNotExist
from django.template.loader import render_to_string
from django.utils import timezone
from django.core.files import File

from .models import SentNotification

class NotificationBase(object):
    """
    base class for sending notifications
    """

    render_types = []
    template_name = None
    context = None
    user = None
    can_disable = True
    verbose_name = None

    def get_context_data(self):
        """
        :return: the context data for rendering the email or text template
        """

        context = self.context or {}

        site = Site.objects.get_current()
        context['base_url'] = 'http://' + site.domain

        return context

    @classmethod
    def get_verbose_name(cls):
        if cls.verbose_name:
            return cls.verbose_name
        else:
            return re.sub(
                r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))',
                r' \1',
                cls.__name__
            )

    @classmethod
    def get_class_path(cls):
        return '{}.{}'.format(cls.__module__, cls.__name__)

    def send(self, raise_exception=False, user=None):
        """
        Handles the preparing the notification for sending. Called to trigger the send from code.
        If raise_exception is True, it will raise any exceptions rather than simply logging them.
        returns boolean whether or not the notification was sent successfully
        """
        context = self.get_context_data()

        recipients = self.get_recipients()

        if 'text' in self.render_types:
            text_content = self.render('text', context)
        else:
            text_content = None

        if 'html' in self.render_types:
            html_content = self.render('html', context)
        else:
            html_content = None

        sent_from = self.get_sent_from()
        subject = self.get_subject()
        extra_data = self.get_extra_data()

        sent_notification = SentNotification(
            recipients=','.join(recipients),
            text_content=text_content,
            html_content=html_content,
            sent_from=sent_from,
            subject=subject,
            extra_data=json.dumps(extra_data) if extra_data else None,
            notification_class=self.get_class_path(),
            attachments=self._get_encoded_attachments(),
            user=user,
        )

        return self.resend(sent_notification, raise_exception=raise_exception)

    def _get_encoded_attachments(self):
        attachments = self.get_attachments()

        new_attachments = []

        for attachment in attachments or []:
            if isinstance(attachment, File):
                # cannot do with attachment.open() since django 1.11 doesn't support that
                attachment.open()
                new_attachments.append((attachment.name, attachment.read(), guess_type(attachment.name)[0]))
                attachment.close()
            else:
                new_attachments.append(attachment)

        return jsonpickle.dumps(new_attachments)

    @staticmethod
    def _delete_expired_notifications():
        """
        This deletes any notifications that have passed the retention time setting
        """
        retention_time = getattr(settings, 'HERALD_NOTIFICATION_RETENTION_TIME', None)

        if not retention_time:
            return

        cutoff_date = timezone.now() - retention_time

        notifications = SentNotification.objects.filter(date_sent__lt=cutoff_date)
        count = notifications.delete()
        print('Deleted {} expired notifications.'.format(count))

    def get_recipients(self):
        """
        Returns a list of recipients. However the subclass defines that. (emails, phone numbers, etc)
        """

        raise NotImplementedError('Must implement get_recipients.')

    def get_extra_data(self):
        """
        Returns a dictionary of extra data to be stored, and used for sending.
        MUST BE JSON SERIALIZABLE
        """

        return {}

    def get_sent_from(self):
        """
        Returns a "sent from" string. However the subclass defines that. (email, phone number, etc)
        """

        raise NotImplementedError('Must implement get_sent_from.')

    def get_subject(self):
        """
        Returns a subject string. Optional.
        """

        return None

    def get_attachments(self):
        """
        Return a list of attachments or None.

        This only works with email.
        """
        return None

    def render(self, render_type, context):
        """
        Renders the template

        :param render_type: the content type to render
        :param context: context data dictionary
        :return: the rendered content
        """

        assert render_type in self.render_types, 'Invalid Render Type'

        try:
            content = render_to_string('herald/{}/{}.{}'.format(
                render_type,
                self.template_name,
                'txt' if render_type == 'text' else render_type
            ), context)
        except TemplateDoesNotExist:
            content = None

            if settings.DEBUG or getattr(settings, 'HERALD_RAISE_MISSING_TEMPLATES', True):
                raise

        return content

    @staticmethod
    def get_demo_args():
        """
        Returns iterable of arguments needed to initialize notification for demo purposes
        Usually you want to generate dummy data here for testing
        """
        return []

    @classmethod
    def resend(cls, sent_notification, raise_exception=False):
        """
        Takes a saved sent_notification and sends it again.
        returns boolean whether or not the notification was sent successfully
        """

        # handle skipping a notification based on user preference
        if hasattr(sent_notification.user, 'usernotification'):
            notifications = sent_notification.user.usernotification
            if notifications.disabled_notifications.filter(notification_class=cls.get_class_path()).exists():
                sent_notification.date_sent = timezone.now()
                sent_notification.status = sent_notification.STATUS_USER_DISABLED
                sent_notification.save()
                return True

        try:
            cls._send(
                sent_notification.get_recipients(),
                sent_notification.text_content,
                sent_notification.html_content,
                sent_notification.sent_from,
                sent_notification.subject,
                sent_notification.get_extra_data(),
                sent_notification.get_attachments(),
            )
            sent_notification.status = sent_notification.STATUS_SUCCESS
        except Exception as exc:  # pylint: disable=W0703
            # we want to handle any exception whatsoever
            sent_notification.status = sent_notification.STATUS_FAILED
            sent_notification.error_message = six.text_type(exc)

            if raise_exception:
                raise exc

        sent_notification.date_sent = timezone.now()
        sent_notification.save()

        cls._delete_expired_notifications()

        return sent_notification.status == sent_notification.STATUS_SUCCESS

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None,
              attachments=None):
        """
        Handles the actual sending of the notification. Sub classes must override this
        """

        raise NotImplementedError('Must implement send.')

class EmailNotification(NotificationBase):
    """
    Base class for email notifications
    """

    render_types = ['text', 'html']
    from_email = None
    subject = None
    to_emails = None
    bcc = None
    cc = None  # pylint: disable=C0103
    headers = None
    reply_to = None
    attachments = None

    def get_context_data(self):
        context = super(EmailNotification, self).get_context_data()
        context['subject'] = self.subject
        return context

    def get_recipients(self):
        return self.to_emails

    def get_sent_from(self):
        from_email = self.from_email

        if not from_email:
            from_email = settings.DEFAULT_FROM_EMAIL

        return from_email

    def get_subject(self):
        return self.subject

    def get_extra_data(self):
        extra_data = {}

        if self.bcc:
            extra_data['bcc'] = self.bcc

        if self.cc:
            extra_data['cc'] = self.cc

        if self.headers:
            extra_data['headers'] = self.headers

        if self.reply_to:
            extra_data['reply_to'] = self.reply_to

        return extra_data

    def get_attachments(self):
        """
        Return a list of attachments or None.

        This only works with email.
        """
        return self.attachments

    def render(self, render_type, context):
        if render_type == 'text' and getattr(settings, 'HERALD_HTML2TEXT_ENABLED', False):
            try:
                content = super(EmailNotification, self).render('text', context)

            # Render plain text version from HTML
            except TemplateDoesNotExist:
                content = None

            if content is None:
                content = self.get_html2text_converter().handle(super(EmailNotification, self).render('html', context))
        else:
            content = super(EmailNotification, self).render(render_type, context)

        return content

    @staticmethod
    def get_html2text_converter():
        try:
            import html2text
        except ImportError:
            raise Exception(
                "HTML2Text is required for sending an EmailNotification with auto HTML to text conversion."
            )

        h = html2text.HTML2Text()

        if hasattr(settings, 'HERALD_HTML2TEXT_CONFIG'):
            for k, v in settings.HERALD_HTML2TEXT_CONFIG.items():
                setattr(h, k, v)

        return h

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None,
              attachments=None):

        extra_data = extra_data or {}

        mail = EmailMultiAlternatives(
            subject=subject,
            body=text_content,
            from_email=sent_from,
            to=recipients,
            bcc=extra_data.get('bcc', None),
            headers=extra_data.get('headers', None),
            cc=extra_data.get('cc', None),
            reply_to=extra_data.get('reply_to', None),
        )

        if html_content:
            mail.attach_alternative(html_content, 'text/html')

        for attachment in (attachments or []):
            # All mimebase attachments must have a Content-ID or Content-Disposition header
            # or they will show up as unnamed attachments"
            if isinstance(attachment, MIMEBase):
                if attachment.get('Content-ID', False):
                    # if you are sending attachment with content id,
                    # subtype must be 'related'.
                    mail.mixed_subtype = 'related'

                mail.attach(attachment)
            else:
                mail.attach(*attachment)

        mail.send()

class TwilioTextNotification(NotificationBase):
    """
    Base class for text notifications.
    Uses twilio
    """

    render_types = ['text']
    from_number = None
    to_number = None

    def get_recipients(self):
        return [self.to_number]

    def get_sent_from(self):
        from_number = self.from_number
        if not from_number:
            try:
                from_number = settings.TWILIO_DEFAULT_FROM_NUMBER
            except AttributeError:
                raise Exception(
                    'TWILIO_DEFAULT_FROM_NUMBER setting is required for sending a TwilioTextNotification'
                )

        return from_number

    @staticmethod
    def _send(recipients, text_content=None, html_content=None, sent_from=None, subject=None, extra_data=None,
              attachments=None):
        try:
            # twilio version 6
            from twilio.rest import Client
        except ImportError:
            try:
                # twillio version < 6
                from twilio.rest import TwilioRestClient as Client
            except ImportError:
                raise Exception(
                    "Twilio is required for sending a TwilioTextNotification."
                )

        try:
            account_sid = settings.TWILIO_ACCOUNT_SID
            auth_token = settings.TWILIO_AUTH_TOKEN
        except AttributeError:
            raise Exception(
                "TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN settings are required for sending a TwilioTextNotification"
            )

        client = Client(account_sid, auth_token)

        for recipient in recipients:
            client.messages.create(body=text_content, to=recipient, from_=sent_from)
<|endoftext|>


<commit_before>max_len = 200

def shorten_text(msg):
    if len(msg.text) < max_len:
        return msg.text
    else:
        return msg.text[:max_len] + '... /msg' + msg.short_id

def escape_html(text):
    if text is None:
        return None
    return text.replace("&", "&").replace("<", "<").replace(">", ">")

def entities_to_html(text, entities):
    if entities is None:
        return text
    new_text = ''
    p = 0
    for entity in entities:
        l = entity.offset
        r = l + entity.length
        if r > max_len:
            break
        new_text += escape_html(text[p:l])
        p = r
        m = escape_html(text[l:r])
        if entity.type == 'bold' or entity.type == 'italic':
            new_text += '<{t}>{m}</{t}>'.format(t=entity.type[0], m=m)
        elif entity.type == 'code' or entity.type == 'pre':
            new_text += '<{t}>{m}</{t}>'.format(t=entity.type, m=m)
        elif entity.type == 'text_link':
            new_text += '{m}'.format(url=entity.url, m=m)
        else:
            new_text += m
    new_text += escape_html(text[p:])
    return new_text

def entities_to_md(text, entities):
    if entities is None:
        return text
    new_text = ''
    p = 0
    for entity in entities:
        l = entity.offset
        r = l + entity.length
        new_text += text[p:l]
        p = r
        if entity.type == 'bold':
            new_text += '*{}*'.format(text[l:r].replace('*', '\*'))
        elif entity.type == 'italic':
            new_text += '_{}_'.format(text[l:r].replace('_', '\_'))
        elif entity.type == 'code':
            new_text += '`{}`'.format(text[l:r].replace('`', '\`'))
        elif entity.type == 'pre':
            new_text += '```{}```'.format(text[l:r])
        elif entity.type == 'text_link':
            new_text += '[{}]({})'.format(text[l:r], entity.url)
        else:
            new_text += text[l:r]
    new_text += text[p:]
    return new_text

def to_html(msg):
    if msg.content_type == 'text':
        text = shorten_text(msg)
        return entities_to_html(text, msg.entities)
    else:
        text = '[{}] '.format(msg.content_type)
        if msg.caption:
            text += escape_html(msg.caption)
        text += ' /msg' + msg.short_id
        return text
<commit_msg>Fix bug with unescaped html<commit_after>max_len = 200

def shorten_text(msg):
    if len(msg.text) < max_len:
        return msg.text
    else:
        return msg.text[:max_len] + '... /msg' + msg.short_id

def escape_html(text):
    if text is None:
        return None
    return text.replace("&", "&").replace("<", "<").replace(">", ">")

def entities_to_html(text, entities):
    if entities is None:
        return escape_html(text)
    new_text = ''
    p = 0
    for entity in entities:
        l = entity.offset
        r = l + entity.length
        if r > max_len:
            break
        new_text += escape_html(text[p:l])
        p = r
        m = escape_html(text[l:r])
        if entity.type == 'bold' or entity.type == 'italic':
            new_text += '<{t}>{m}</{t}>'.format(t=entity.type[0], m=m)
        elif entity.type == 'code' or entity.type == 'pre':
            new_text += '<{t}>{m}</{t}>'.format(t=entity.type, m=m)
        elif entity.type == 'text_link':
            new_text += '{m}'.format(url=entity.url, m=m)
        else:
            new_text += m
    new_text += escape_html(text[p:])
    return new_text

def entities_to_md(text, entities):
    if entities is None:
        return text
    new_text = ''
    p = 0
    for entity in entities:
        l = entity.offset
        r = l + entity.length
        new_text += text[p:l]
        p = r
        if entity.type == 'bold':
            new_text += '*{}*'.format(text[l:r].replace('*', '\*'))
        elif entity.type == 'italic':
            new_text += '_{}_'.format(text[l:r].replace('_', '\_'))
        elif entity.type == 'code':
            new_text += '`{}`'.format(text[l:r].replace('`', '\`'))
        elif entity.type == 'pre':
            new_text += '```{}```'.format(text[l:r])
        elif entity.type == 'text_link':
            new_text += '[{}]({})'.format(text[l:r], entity.url)
        else:
            new_text += text[l:r]
    new_text += text[p:]
    return new_text

def to_html(msg):
    if msg.content_type == 'text':
        text = shorten_text(msg)
        return entities_to_html(text, msg.entities)
    else:
        text = '[{}] '.format(msg.content_type)
        if msg.caption:
            text += escape_html(msg.caption)
        text += ' /msg' + msg.short_id
        return text
<|endoftext|>


<commit_before>'''
A simple, plain HTTP client which mixes httplib with gevent and PayPal protecteds.

API is currently a single function:

http_client.request("get", "http://example.com/foo")
'''
import httplib
from urlparse import urlparse, urlunparse
import functools
import urllib2

import context

from gevent import socket

# TODO: make and use a better HTTP library instead of wrapping httplib.
# hopefully this is at least a pretty stable abstraction that can migrate over
# ... if nothing else, much better than shrugging our shoulders when someone
# asks how to make an http request

class _GHTTPConnection(httplib.HTTPConnection):

    def __init__(self, host, port=None, protected=None, strict=None,
                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
        httplib.HTTPConnection.__init__(self, host, port, strict, timeout)
        self.protected = protected

    def connect(self):
        ctx = context.get_context()
        self.sock = ctx.connection_mgr.get_connection((self.host, self.port),
                                                      self.protected)
        if self._tunnel_host:
            self._tunnel()

    def release_sock(self):
        if self._HTTPConnection__state == "Idle" and self.sock:
            context.get_context().connection_mgr.release_connection(self.sock)
            self.sock = None

    def __del__(self):
        self.release_sock()

def urllib2_request(u2req, timeout=None):
    """\
    Translate a urllib2.Request to something we can pass to our
    request() function, and translate our Response to a
    urllib2.addinfourl object
    """
    # TODO: proxy support?
    method = u2req.get_method()
    url = u2req._Request__original
    body = u2req.get_data()
    headers = dict(u2req.unredirected_hdrs)
    headers.update((k, v) for k, v in u2req.headers.items()
                   if k not in headers)
    try:
        kwargs = {}
        if timeout is not None:
            kwargs['timeout'] = timeout
        resp = request(method, url, body, headers, **kwargs)
        hr = resp.http_response
        hr.recv = hr.read
        fp = socket._fileobject(hr, close=True)
        aiu = urllib2.addinfourl(fp=fp,
                                 headers=hr.msg,
                                 url=resp.request.url)
        aiu.code = hr.status
        aiu.msg = hr.reason
        return aiu
    except ValueError as e:
        raise urllib2.URLError(e.msg)

def request(method, url, body=None, headers={},
            literal=False, use_protected=False,
            timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
    if method not in _HTTP_METHODS:
        raise ValueError("invalid http method {0}".format(method))

    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        raise ValueError('unknown protocol %s' % parsed.scheme)
    domain, _, port = parsed.netloc.partition(':')
    try:
        port = int(port)
    except ValueError:
        port = 80 if parsed.scheme == 'http' else 443

    protected = (parsed.scheme == 'https') and (True if use_protected
                                                else "PLAIN_SSL")
    conn = _GHTTPConnection(domain, port, protected=protected, timeout=timeout)

    selector = urlunparse(parsed._replace(scheme='', netloc=''))

    skips = {'skip_host': True,
             'skip_accept_encoding': True} if literal else {}

    if not literal:
        headers.setdefault('User-Agent', 'python')

    conn.putrequest(method, selector, **skips)
    # OMD!
    if not literal and body is not None and 'Content-Length' not in headers:
        conn._set_content_length(body)

    for header, value in headers.items():
        if type(value) is list:
            for subvalue in value:
                conn.putheader(header, subvalue)
        else:
            conn.putheader(header, value)

    conn.endheaders()

    if body is not None:
        conn.send(body)
    raw = conn.getresponse()    # does NOT hold a reference to the
                                # HTTPConnection
    raw._connection = conn      # so the finalizer doesn't get called
                                # until the request has died
    return Response(
        Request(method, url, headers, body),
        raw.status, raw.getheaders(), raw)

class Request(object):
    def __init__(self, method, url, headers, body):
        self.method = method
        self.url = url
        self.headers = headers
        self.body = body

    def __repr__(self):
        return "<http_client.Request {0} {1}>".format(self.method, self.url)

class Response(object):
    def __init__(self, request, status, headers, http_response):
        self.request = request
        self.status = status
        self.headers = headers
        self.http_response = http_response

    @property
    def body(self):
        return self.http_response.read()

    def __repr__(self):
        return "<http_client.Response ({0}) {1} {2}>".format(
            self.status, self.request.method, self.request.url)

#http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods
_HTTP_METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'OPTION',
                 'CONNECT', 'PATCH')

def _init_methods():
    g = globals()
    for m in _HTTP_METHODS:
        g[m.lower()] = functools.partial(request, m)

_init_methods()
<commit_msg>fixing http_client.request(headers={})  ==>  http_client.request(headers=None)<commit_after>'''
A simple, plain HTTP client which mixes httplib with gevent and PayPal protecteds.

API is currently a single function:

http_client.request("get", "http://example.com/foo")
'''
import httplib
from urlparse import urlparse, urlunparse
import functools
import urllib2

import context

from gevent import socket

# TODO: make and use a better HTTP library instead of wrapping httplib.
# hopefully this is at least a pretty stable abstraction that can migrate over
# ... if nothing else, much better than shrugging our shoulders when someone
# asks how to make an http request

class _GHTTPConnection(httplib.HTTPConnection):

    def __init__(self, host, port=None, protected=None, strict=None,
                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
        httplib.HTTPConnection.__init__(self, host, port, strict, timeout)
        self.protected = protected

    def connect(self):
        ctx = context.get_context()
        self.sock = ctx.connection_mgr.get_connection((self.host, self.port),
                                                      self.protected)
        if self._tunnel_host:
            self._tunnel()

    def release_sock(self):
        if self._HTTPConnection__state == "Idle" and self.sock:
            context.get_context().connection_mgr.release_connection(self.sock)
            self.sock = None

    def __del__(self):
        self.release_sock()

def urllib2_request(u2req, timeout=None):
    """\
    Translate a urllib2.Request to something we can pass to our
    request() function, and translate our Response to a
    urllib2.addinfourl object
    """
    # TODO: proxy support?
    method = u2req.get_method()
    url = u2req._Request__original
    body = u2req.get_data()
    headers = dict(u2req.unredirected_hdrs)
    headers.update((k, v) for k, v in u2req.headers.items()
                   if k not in headers)
    try:
        kwargs = {}
        if timeout is not None:
            kwargs['timeout'] = timeout
        resp = request(method, url, body, headers, **kwargs)
        hr = resp.http_response
        hr.recv = hr.read
        fp = socket._fileobject(hr, close=True)
        aiu = urllib2.addinfourl(fp=fp,
                                 headers=hr.msg,
                                 url=resp.request.url)
        aiu.code = hr.status
        aiu.msg = hr.reason
        return aiu
    except ValueError as e:
        raise urllib2.URLError(e.msg)

def request(method, url, body=None, headers=None,
            literal=False, use_protected=False,
            timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
    if method not in _HTTP_METHODS:
        raise ValueError("invalid http method {0}".format(method))

    parsed = urlparse(url)
    if parsed.scheme not in ('http', 'https'):
        raise ValueError('unknown protocol %s' % parsed.scheme)
    domain, _, port = parsed.netloc.partition(':')
    try:
        port = int(port)
    except ValueError:
        port = 80 if parsed.scheme == 'http' else 443

    protected = (parsed.scheme == 'https') and (True if use_protected
                                                else "PLAIN_SSL")
    conn = _GHTTPConnection(domain, port, protected=protected, timeout=timeout)

    selector = urlunparse(parsed._replace(scheme='', netloc=''))

    skips = {'skip_host': True,
             'skip_accept_encoding': True} if literal else {}

    if not literal:
        headers.setdefault('User-Agent', 'python')

    conn.putrequest(method, selector, **skips)
    # OMD!
    if headers is None:
        headers = {}
    if not literal and body is not None and 'Content-Length' not in headers:
        conn._set_content_length(body)

    for header, value in headers.items():
        if type(value) is list:
            for subvalue in value:
                conn.putheader(header, subvalue)
        else:
            conn.putheader(header, value)

    conn.endheaders()

    if body is not None:
        conn.send(body)
    raw = conn.getresponse()    # does NOT hold a reference to the
                                # HTTPConnection
    raw._connection = conn      # so the finalizer doesn't get called
                                # until the request has died
    return Response(
        Request(method, url, headers, body),
        raw.status, raw.getheaders(), raw)

class Request(object):
    def __init__(self, method, url, headers, body):
        self.method = method
        self.url = url
        self.headers = headers
        self.body = body

    def __repr__(self):
        return "<http_client.Request {0} {1}>".format(self.method, self.url)

class Response(object):
    def __init__(self, request, status, headers, http_response):
        self.request = request
        self.status = status
        self.headers = headers
        self.http_response = http_response

    @property
    def body(self):
        return self.http_response.read()

    def __repr__(self):
        return "<http_client.Response ({0}) {1} {2}>".format(
            self.status, self.request.method, self.request.url)

#http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods
_HTTP_METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'OPTION',
                 'CONNECT', 'PATCH')

def _init_methods():
    g = globals()
    for m in _HTTP_METHODS:
        g[m.lower()] = functools.partial(request, m)

_init_methods()
<|endoftext|>