<commit_before>#!/usr/bin/python
#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

from irclib import SimpleIRCClient
import random
import re
import eventlet
from eventlet.green import time

class LpBot(SimpleIRCClient):
    def __init__(self, bitHopper):
        SimpleIRCClient.__init__(self)
        self.bitHopper = bitHopper
        self.nick = 'lp' + str(random.randint(1,9999999999))
        self.chan_list=[]
        self.newblock_re = re.compile('\*\*\* New Block \{(?P<server>.+)\} - (?P<hash>.*)')
        self.hashes = ['']
        self.hashinfo = {'':''}
        self.server=''
        self.current_block=''
        eventlet.spawn_n(self.run)
        eventlet.spawn_n(self.process_forever)

    def process_forever(self):
        while True:
            self.ircobj.process_once(0.2)

    def run(self):
        while True:
                if self.is_connected():
                        self.join()
                else:
                        self.chan_list = []
                        self._connect()
                        print "Connect returned"
                eventlet.sleep(15)

    def _connect(self):
        print "Connecting..."
        try:
            self.connect('irc.bithopper.org', 6667, self.nick)
        except Exception, e:
            print e

    def is_connected(self):
        return self.connection.is_connected();

    def on_disconnect(self, c, e):
        print "Disconnected..."
        self.chan_list=[]
        self.connection.execute_delayed(10, self._connect)

    def on_connect(self, c, e):
        self.join()

    def on_pubmsg(self, c, e):
        bl_match = self.newblock_re.match(e.arguments()[0])
        if bl_match != None:
            last_hash = bl_match.group('hash')
            server = bl_match.group('server')
            self.decider(server, last_hash)

    def get_last_block(self):
        return self.hashes[-1]

    def decider(self, server, block):
        last_server = self.server
        last_block = self.get_last_block()
        votes = 0
        total_votes = 0

        if block not in self.hashes:
            ### New block I know nothing about
            print "New Block: {" + str(server) + "} - " + block
            self.hashes.append(block)
            if len(self.hashes) > 50:
                del self.hashes[0];
            self.hashinfo[block] = [server]
            # Am I working on this now?
            if self.current_block == block:
                print "Server selected: " + server
                self.server = server
                votes = 1
                total_votes = 1
            else:
                # Info added, I have nothing else to do
                #print "Unknown work - " + block
                return
        else:
            # Add a vote
            self.hashinfo[block].append(server)
            # Talley the votes based on who we have selected so far
            for v in self.hashinfo[block]:
                if v == self.server:
                    votes += 1
                total_votes += 1

            # If I haven't received the new work yet, I don't want to decide anything, just store it
            if self.current_block != block:
                #print "Old  work - " + block
                return

            print "Total Votes: " + str(total_votes)
            # Enough votes for a quarum?
            if total_votes > 5:
                # Enought compelling evidence to switch?
                # Loop through unique servers for the block
                for test_server in set(self.hashinfo[block]):
                    test_votes = 0
                    test_total_votes = 0
                    print "Tallying votes for " + test_server
                    ## Talley up the votes for that server
                    for test_vote in self.hashinfo[block]:
                        test_total_votes += 1
                        if test_vote == test_server:
                            test_votes += 1
                    print str(test_votes) + " out of " + str(test_total_votes) + " votes."
                    if float(test_votes) / test_total_votes > .5
                        if self.server != test_server:
                            print "In the minority, updating to  " + test_server + ": " + str(test_votes) + "/" + str(test_total_votes)
                            self.server = test_server
                            votes = test_votes
                            total_votes = test_total_votes
                        else:
                            print "In the majority, keeping server"
                    else:
                        print "Not enough votes in one direction to make a decision"
            else: # Not enough for quarum, select first
                self.server = self.hashinfo[block][0]
                votes = 0
                total_votes = 0
                for vote_server in self.hashinfo[block]:
                    if vote_server == self.server:
                        votes += 1
                    total_votes += 1

        if self.server != last_server:
            self.say("Best Guess: {" + self.server + "} with " + str(votes) + " of " + str(total_votes) + " votes - " + self.current_block)
            self.bitHopper.lp.set_owner(self.server, self.current_block)

        # Cleanup
        # Delete any orbaned blocks out of blockinfo
        print "Clean Up..."
        for clean_block, clean_val in self.hashinfo.items():
            if clean_block not in self.hashes:
                print "Deleting old work... " + clean_block
                del self.hashinfo[clean_block]

    def say(self, text):
        self.connection.privmsg("#bithopper-lp", text)

    def announce(self, server, last_hash):
        try:
            if self.is_connected():
                self.server=''
                self.current_block = last_hash
                print "Announcing: *** New Block {" + str(server) + "} - " + last_hash
                self.say("*** New Block {" + str(server) + "} - " + last_hash)
                self.decider(server, last_hash)
            else:
                print "Not connected to IRC..."
        except Exception, e:
            print "********************************"
            print "*****  ERROR IN ANNOUCE  *******"
            print "********************************"
            print str(e)

    def join(self):
        if '#bithopper-lp' not in self.chan_list:
            self.connection.join('#bithopper-lp')
            self.chan_list.append('#bithopper-lp')

#class test_eventargs():
#    def __init__(self, message):
#        self.arguments = [message]

#if __name__ == "__main__":
#    bot = LpBot()
#    while not bot.is_connected:
#        thread.sleep(3)
#
#    print 'Testing me first, everyone agrees'
#    bot.announce('test', '1')
#    bot.on_pubmsg('', test_eventargs('*** New Block {test} - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#
#    print 'Testing someone first, me later with wrong server'

<commit_msg>Fix potential p2pLP threading issues<commit_after>#!/usr/bin/python
#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

from irclib import SimpleIRCClient
import random
import re
import eventlet
from eventlet.green import time, threading

class LpBot(SimpleIRCClient):
    def __init__(self, bitHopper):
        SimpleIRCClient.__init__(self)
        self.bitHopper = bitHopper
        self.nick = 'lp' + str(random.randint(1,9999999999))
        self.chan_list=[]
        self.newblock_re = re.compile('\*\*\* New Block \{(?P<server>.+)\} - (?P<hash>.*)')
        self.hashes = ['']
        self.hashinfo = {'':''}
        self.server=''
        self.current_block=''
        eventlet.spawn_n(self.run)
        eventlet.spawn_n(self.process_forever)
        self.lock = threading.RLock()

    def process_forever(self):
        while True:
            self.ircobj.process_once(0.2)

    def run(self):
        while True:
                if self.is_connected():
                        self.join()
                else:
                        self.chan_list = []
                        self._connect()
                        print "Connect returned"
                eventlet.sleep(15)

    def _connect(self):
        print "Connecting..."
        try:
            self.connect('irc.bithopper.org', 6667, self.nick)
        except Exception, e:
            print e

    def is_connected(self):
        return self.connection.is_connected();

    def on_disconnect(self, c, e):
        print "Disconnected..."
        self.chan_list=[]
        self.connection.execute_delayed(10, self._connect)

    def on_connect(self, c, e):
        self.join()

    def on_pubmsg(self, c, e):
        with self.lock:
            bl_match = self.newblock_re.match(e.arguments()[0])
            if bl_match != None:
                last_hash = bl_match.group('hash')
                server = bl_match.group('server')
                self.decider(server, last_hash)

    def get_last_block(self):
        return self.hashes[-1]

    def decider(self, server, block):
        with self.lock:
            last_server = self.server
            last_block = self.get_last_block()
            votes = 0
            total_votes = 0

            if block not in self.hashes:
                ### New block I know nothing about
                print "New Block: {" + str(server) + "} - " + block
                self.hashes.append(block)
                if len(self.hashes) > 50:
                    del self.hashes[0];
                self.hashinfo[block] = [server]
                # Am I working on this now?
                if self.current_block == block:
                    print "Server selected: " + server
                    self.server = server
                    votes = 1
                    total_votes = 1
                else:
                    # Info added, I have nothing else to do
                    #print "Unknown work - " + block
                    return
            else:
                # Add a vote
                self.hashinfo[block].append(server)
                # Talley the votes based on who we have selected so far
                for v in self.hashinfo[block]:
                    if v == self.server:
                        votes += 1
                    total_votes += 1

                # If I haven't received the new work yet, I don't want to decide anything, just store it
                if self.current_block != block:
                    #print "Old  work - " + block
                    return

                print "Total Votes: " + str(total_votes)
                # Enough votes for a quarum?
                if total_votes > 5:
                    # Enought compelling evidence to switch?
                    # Loop through unique servers for the block
                    for test_server in set(self.hashinfo[block]):
                        test_votes = 0
                        test_total_votes = 0
                        print "Tallying votes for " + test_server
                        ## Talley up the votes for that server
                        for test_vote in self.hashinfo[block]:
                            test_total_votes += 1
                            if test_vote == test_server:
                                test_votes += 1
                        print str(test_votes) + " out of " + str(test_total_votes) + " votes."
                        if float(test_votes) / test_total_votes > .5:
                            if self.server != test_server:
                                print "In the minority, updating to  " + test_server + ": " + str(test_votes) + "/" + str(test_total_votes)
                                self.server = test_server
                                votes = test_votes
                                total_votes = test_total_votes
                            else:
                                print "In the majority, keeping server"
                        else:
                            print "Not enough votes in one direction to make a decision"
                else: # Not enough for quarum, select first
                    self.server = self.hashinfo[block][0]
                    votes = 0
                    total_votes = 0
                    for vote_server in self.hashinfo[block]:
                        if vote_server == self.server:
                            votes += 1
                        total_votes += 1

            if self.server != last_server:
                self.say("Best Guess: {" + self.server + "} with " + str(votes) + " of " + str(total_votes) + " votes - " + self.current_block)
                self.bitHopper.lp.set_owner(self.server, self.current_block)

            # Cleanup
            # Delete any orbaned blocks out of blockinfo
            print "Clean Up..."
            for clean_block, clean_val in self.hashinfo.items():
                if clean_block not in self.hashes:
                    print "Deleting old work... " + clean_block
                    del self.hashinfo[clean_block]

    def say(self, text):
        self.connection.privmsg("#bithopper-lp", text)

    def announce(self, server, last_hash):
        with self.lock:
            try:
                if self.is_connected():
                    self.server=''
                    self.current_block = last_hash
                    print "Announcing: *** New Block {" + str(server) + "} - " + last_hash
                    self.say("*** New Block {" + str(server) + "} - " + last_hash)
                    self.decider(server, last_hash)
                else:
                    print "Not connected to IRC..."
            except Exception, e:
                print "********************************"
                print "*****  ERROR IN ANNOUCE  *******"
                print "********************************"
                print str(e)

    def join(self):
        if '#bithopper-lp' not in self.chan_list:
            self.connection.join('#bithopper-lp')
            self.chan_list.append('#bithopper-lp')

#class test_eventargs():
#    def __init__(self, message):
#        self.arguments = [message]

#if __name__ == "__main__":
#    bot = LpBot()
#    while not bot.is_connected:
#        thread.sleep(3)
#
#    print 'Testing me first, everyone agrees'
#    bot.announce('test', '1')
#    bot.on_pubmsg('', test_eventargs('*** New Block {test} - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#    bot.on_pubmsg('', test_eventargs('*** New Block (test) - 1'))
#
#    print 'Testing someone first, me later with wrong server'

<|endoftext|>


<commit_before># util.py - common utility functions
#
# Copyright (C) 2012, 2013 Arthur de Jong
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""Common utility functions for other stdnum modules.

This module is meant for internal use by stdnum modules and is not
guaranteed to remain stable and as such not part of the public API of
stdnum.
"""

import pkgutil
import pydoc
import re
import sys

from stdnum.exceptions import *

_strip_doctest_re = re.compile('^>>> .*\Z', re.DOTALL | re.MULTILINE)

def clean(number, deletechars):
    """Remove the specified characters from the supplied number.

    >>> clean('123-456:78 9', ' -:')
    '123456789'
    """
    try:
        return ''.join(x for x in number if x not in deletechars)
    except:
        raise InvalidFormat()

def get_number_modules(base='stdnum'):
    """Yield all the module and package names under the specified module."""
    __import__(base)
    module = sys.modules[base]
    for loader, name, is_pkg in pkgutil.walk_packages(
                    module.__path__, module.__name__ + '.',
                    onerror=lambda x: None
                ):
        __import__(name)
        module = sys.modules[name]
        if hasattr(module, 'validate'):
            yield module

def get_module_name(module):
    """Return the short description of the number."""
    return pydoc.splitdoc(pydoc.getdoc(module))[0].strip('.')

def get_module_description(module):
    """Return a description of the number."""
    doc = pydoc.splitdoc(pydoc.getdoc(module))[1]
    # remove the doctests
    return _strip_doctest_re.sub('', doc[1]).strip(),

def get_module_list():
    for module in get_number_modules():
        yield ' * %s: %s' % (
            module.__name__.replace('stdnum.', ''),
            get_module_name(module),
        )
<commit_msg>Try to replace Unicode characters with ASCII<commit_after># util.py - common utility functions
# coding: utf-8
#
# Copyright (C) 2012, 2013 Arthur de Jong
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""Common utility functions for other stdnum modules.

This module is meant for internal use by stdnum modules and is not
guaranteed to remain stable and as such not part of the public API of
stdnum.
"""

import pkgutil
import pydoc
import re
import sys
import unicodedata

from stdnum.exceptions import *

_strip_doctest_re = re.compile('^>>> .*\Z', re.DOTALL | re.MULTILINE)

def _mk_char_map(mapping):
    """Transform a dictionary with comma separated uniode chracter names
    to tuples with unicode characters as key."""
    for key, value in mapping.items():
        for char in key.split(','):
            try:
                yield (unicodedata.lookup(char), value)
            except KeyError:
                pass

# build mapping of Unicode characters to equivalent ASCII characters
_char_map = dict(_mk_char_map({
    'HYPHEN-MINUS,ARMENIAN HYPHEN,HEBREW PUNCTUATION MAQAF,HYPHEN,'
    'NON-BREAKING HYPHEN,FIGURE DASH,EN DASH,EM DASH,HORIZONTAL BAR,'
    'SMALL HYPHEN-MINUS,FULLWIDTH HYPHEN-MINUS,MONGOLIAN NIRUGU,OVERLINE,'
    'HYPHEN BULLET,MACRON,MODIFIER LETTER MINUS SIGN,FULLWIDTH MACRON,'
    'OGHAM SPACE MARK,SUPERSCRIPT MINUS,SUBSCRIPT MINUS,MINUS SIGN,'
    'HORIZONTAL LINE EXTENSION,HORIZONTAL SCAN LINE-1,HORIZONTAL SCAN LINE-3,'
    'HORIZONTAL SCAN LINE-7,HORIZONTAL SCAN LINE-9,STRAIGHTNESS': '-',
    'ASTERISK,ARABIC FIVE POINTED STAR,SYRIAC HARKLEAN ASTERISCUS,'
    'FLOWER PUNCTUATION MARK,VAI FULL STOP,SMALL ASTERISK,FULLWIDTH ASTERISK,'
    'ASTERISK OPERATOR,STAR OPERATOR,HEAVY ASTERISK,LOW ASTERISK,'
    'OPEN CENTRE ASTERISK,EIGHT SPOKED ASTERISK,SIXTEEN POINTED ASTERISK,'
    'TEARDROP-SPOKED ASTERISK,OPEN CENTRE TEARDROP-SPOKED ASTERISK,'
    'HEAVY TEARDROP-SPOKED ASTERISK,EIGHT TEARDROP-SPOKED PROPELLER ASTERISK,'
    'HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK,'
    'ARABIC FIVE POINTED STAR': '*',
    'COMMA,ARABIC COMMA,SINGLE LOW-9 QUOTATION MARK,IDEOGRAPHIC COMMA,'
    'ARABIC DECIMAL SEPARATOR,ARABIC THOUSANDS SEPARATOR,PRIME,RAISED COMMA,'
    'PRESENTATION FORM FOR VERTICAL COMMA,SMALL COMMA,'
    'SMALL IDEOGRAPHIC COMMA,FULLWIDTH COMMA,CEDILLA': ',',
    'FULL STOP,MIDDLE DOT,GREEK ANO TELEIA,ARABIC FULL STOP,'
    'IDEOGRAPHIC FULL STOP,SYRIAC SUPRALINEAR FULL STOP,'
    'SYRIAC SUBLINEAR FULL STOP,SAMARITAN PUNCTUATION NEQUDAA,'
    'TIBETAN MARK INTERSYLLABIC TSHEG,TIBETAN MARK DELIMITER TSHEG BSTAR,'
    'RUNIC SINGLE PUNCTUATION,BULLET,ONE DOT LEADER,HYPHENATION POINT,'
    'WORD SEPARATOR MIDDLE DOT,RAISED DOT,KATAKANA MIDDLE DOT,'
    'SMALL FULL STOP,FULLWIDTH FULL STOP,HALFWIDTH KATAKANA MIDDLE DOT,'
    'AEGEAN WORD SEPARATOR DOT,PHOENICIAN WORD SEPARATOR,'
    'KHAROSHTHI PUNCTUATION DOT,DOT ABOVE,ARABIC SYMBOL DOT ABOVE,'
    'ARABIC SYMBOL DOT BELOW,BULLET OPERATOR,DOT OPERATOR': '.',
    'SOLIDUS,SAMARITAN PUNCTUATION ARKAANU,FULLWIDTH SOLIDUS,DIVISION SLASH,'
    'MATHEMATICAL RISING DIAGONAL,BIG SOLIDUS,FRACTION SLASH': '/',
    'COLON,ETHIOPIC WORDSPACE,RUNIC MULTIPLE PUNCTUATION,MONGOLIAN COLON,'
    'PRESENTATION FORM FOR VERTICAL COLON,FULLWIDTH COLON,'
    'PRESENTATION FORM FOR VERTICAL TWO DOT LEADER,SMALL COLON': ':',
    'SPACE,NO-BREAK SPACE,EN QUAD,EM QUAD,EN SPACE,EM SPACE,'
    'THREE-PER-EM SPACE,FOUR-PER-EM SPACE,SIX-PER-EM SPACE,FIGURE SPACE,'
    'PUNCTUATION SPACE,THIN SPACE,HAIR SPACE,NARROW NO-BREAK SPACE,'
    'MEDIUM MATHEMATICAL SPACE,IDEOGRAPHIC SPACE': ' ',
    }))

def _clean_chars(number):
    """Replace various Unicode characters with their ASCII counterpart."""
    return ''.join(_char_map.get(x, x) for x in number)

def clean(number, deletechars=''):
    """Remove the specified characters from the supplied number.

    >>> clean('123-456:78 9', ' -:')
    '123456789'
    >>> clean('1–2—3―4')
    '1-2-3-4'
    """
    try:
        number = ''.join(x for x in number)
    except:
        raise InvalidFormat()
    if sys.version < '3' and isinstance(number, str):  # pragma: no cover (Python 2/3 specific code)
        try:
            number = _clean_chars(number.decode()).encode()
        except UnicodeError:
            try:
                number = _clean_chars(number.decode('utf-8')).encode('utf-8')
            except UnicodeError:
                pass
    else:  # pragma: no cover (Python 2/3 specific code)
        number = _clean_chars(number)
    return ''.join(x for x in number if x not in deletechars)

def get_number_modules(base='stdnum'):
    """Yield all the module and package names under the specified module."""
    __import__(base)
    module = sys.modules[base]
    for loader, name, is_pkg in pkgutil.walk_packages(
                    module.__path__, module.__name__ + '.',
                    onerror=lambda x: None
                ):
        __import__(name)
        module = sys.modules[name]
        if hasattr(module, 'validate'):
            yield module

def get_module_name(module):
    """Return the short description of the number."""
    return pydoc.splitdoc(pydoc.getdoc(module))[0].strip('.')

def get_module_description(module):
    """Return a description of the number."""
    doc = pydoc.splitdoc(pydoc.getdoc(module))[1]
    # remove the doctests
    return _strip_doctest_re.sub('', doc[1]).strip(),

def get_module_list():
    for module in get_number_modules():
        yield ' * %s: %s' % (
            module.__name__.replace('stdnum.', ''),
            get_module_name(module),
        )
<|endoftext|>


<commit_before># author: G. Alomar
from cassandra.cluster import Cluster
from hecuba.datastore import *
from hecuba.iter import Block
from hecuba.dict import *
from hecuba.storageobj import *
from hecuba.storageobjix import *
import collections

from conf.hecuba_params import *
from conf.apppath import apppath

def start_task(params):
    indexed = True
    if indexed == False:
		if not 'prefetch_activated' in globals():
		    global prefetch_activated
		    prefetch_activated = True # modified for qbeast
		if not 'batch_activated' in globals():
		    global batch_activated
		    batch_activated = True
		if prefetch_activated or batch_activated:
		    path = apppath + '/conf/imports.py'
		    file = open(path, 'r')
		    for line in file:
		        exec line

		if batch_activated:
		    for param in params:
		        if isinstance(param, StorageObj) or isinstance(param, StorageObjIx) or isinstance(param, Block):
		            param.cntxt = context(param)
		            param.cntxt.__enter__()

def end_task(params):
    indexed = True
    if indexed == False:
		if not 'prefetch_activated' in globals():
		    global prefetch_activated
		    prefetch_activated = True # modified for qbeast
		if not 'batch_activated' in globals():
		    global batch_activated
		    batch_activated = True
		if prefetch_activated or batch_activated:
		    path = apppath + '/conf/imports.py'
		    file = open(path, 'r')
		    for line in file:
		        exec line

		if batch_activated:
		    for param in params:
		        if isinstance(param, Block) or isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
		            try:
		                param.cntxt.__exit__()
		            except Exception as e:
		                print "error trying to exit context:", e

		if prefetch_activated:
		    for param in params:
		        if isinstance(param, Block):
		            keys = param.storageobj.keyList[param.storageobj.__class__.__name__]
		            exec("persistentdict = param.storageobj." + str(keys[0]))
		            if persistentdict.prefetch == True:
		                try:
		                    persistentdict.end_prefetch()
		                except Exception as e:
		                    print "error trying to prefetch:", e

		if not 'statistics_activated' in globals():
		    global statistics_activated
		    statistics_activated = True
		if statistics_activated:
		    for param in params:
		        if isinstance(param, Block):
		            param.storageobj.statistics()
		        if isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
		            param.statistics()

def getByID(objid):
    path = apppath + '/conf/imports.py'
    file = open(path, 'r')
    for line in file:
        exec(line)
    objidsplit = objid.split("_")
    print "objidsplit:", objidsplit

    if len(objidsplit) == 2:
        result = ''
        exec('result = ' + objidsplit[0] + '()')
        print "str(objidsplit[0]):", str(objidsplit[0])
        result.getByName(str(objidsplit[0]))
        return result

    else:
        cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
        session = cluster.connect()
        #(ksp,tab,dict_name,obj_type,entryPoint,port,tokens) = session.execute("SELECT ksp,tab,dict_name,obj_type,entrypoint,port,tkns FROM hecuba.blocks WHERE blockid = %s",(objid))[0]
        ksp =        session.execute("SELECT ksp FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].ksp
        tab =        session.execute("SELECT tab FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].tab
        dict_name =  session.execute("SELECT dict_name FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].dict_name
        obj_type =   session.execute("SELECT obj_type FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].obj_type
        entryPoint = session.execute("SELECT entrypoint FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].entrypoint
        port =       session.execute("SELECT port FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].port
        tokens =     session.execute("SELECT tkns FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].tkns
        if str(obj_type) == 'qbeast':
            print "indexed storageobj"
            blockid = objid
            metadata = cluster.metadata
            tokenmap = metadata.token_map.token_to_host_owner
            odtokenmap = collections.OrderedDict(sorted(tokenmap.items()))
            if not 'prefetch_activated' in globals():
                global prefetch_activated
                prefetch_activated = False # this should be false, right?
            for position in tokens:
                for key, val in odtokenmap.iteritems():
                    # (self, peer,        keynames,           tablename,     blockkeyspace, myuuid)
                    b = IxBlock(str(val), tab,                tab,           ksp,           objid) # the keynames vale is wrong, needs to be corrected
                    exec("b.storageobj = " + str(dict_name) + "('" + str(dict_name) + "')")
                    if prefetch_activated:
                        b.storageobj.init_prefetch(b)
                    session.shutdown()
                    cluster.shutdown()
                    return b
        else:
            print "normal storageobj"
            # blockKeyspace = objidsplit[0]
            # blockKeyNames = str(objidsplit[1])
            # blockTableName = objidsplit[2]
            # blockRange = objidsplit[3]
            blockranges = objidsplit[3:len(objidsplit)]
            blockrangesf = ''
            for ind, val in enumerate(blockranges):
                if ind < (len(blockranges)-1):
                    blockrangesf += str(val) + '_'
                else:
                    blockrangesf += str(val)
            cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
            session = cluster.connect()
            metadata = cluster.metadata
            tokenmap = metadata.token_map.token_to_host_owner
            odtokenmap = collections.OrderedDict(sorted(tokenmap.items()))
            position = 0
            if not 'prefetch_activated' in globals():
                global prefetch_activated
            prefetch_activated = True
            for key, val in odtokenmap.iteritems():
                if str(position) == objidsplit[3]:
                    b = Block((str(val), blockrangesf), str(objidsplit[1]), objidsplit[2], objidsplit[0])
                    exec("b.storageobj = " + str(objidsplit[2]) + "('" + str(objidsplit[2]) + "')")
                    if prefetch_activated:
                        b.storageobj.init_prefetch(b)
                    session.shutdown()
                    cluster.shutdown()
                    return b
                position += 1
<commit_msg>improving the system<commit_after># author: G. Alomar
from cassandra.cluster import Cluster
from hecuba.datastore import *
from hecuba.iter import Block
from hecuba.dict import *
from hecuba.storageobj import *
from hecuba.storageobjix import *
import collections

from conf.hecuba_params import *
from conf.apppath import apppath

def start_task(params):
    cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
    session = cluster.connect()
    obj_type = session.execute("SELECT obj_type FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].obj_type
    session.shutdown()
    cluster.shutdown()
    if str(obj_type) == 'qbeast':
		if not 'prefetch_activated' in globals():
		    global prefetch_activated
		    prefetch_activated = True # modified for qbeast
		if not 'batch_activated' in globals():
		    global batch_activated
		    batch_activated = True
		if prefetch_activated or batch_activated:
		    path = apppath + '/conf/imports.py'
		    file = open(path, 'r')
		    for line in file:
		        exec line

		if batch_activated:
		    for param in params:
		        if isinstance(param, StorageObj) or isinstance(param, StorageObjIx) or isinstance(param, Block):
		            param.cntxt = context(param)
		            param.cntxt.__enter__()

def end_task(params):
    cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
    session = cluster.connect()
    obj_type = session.execute("SELECT obj_type FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].obj_type
    session.shutdown()
    cluster.shutdown()
    if str(obj_type) == 'qbeast':
		if not 'prefetch_activated' in globals():
		    global prefetch_activated
		    prefetch_activated = True # modified for qbeast
		if not 'batch_activated' in globals():
		    global batch_activated
		    batch_activated = True
		if prefetch_activated or batch_activated:
		    path = apppath + '/conf/imports.py'
		    file = open(path, 'r')
		    for line in file:
		        exec line

		if batch_activated:
		    for param in params:
		        if isinstance(param, Block) or isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
		            try:
		                param.cntxt.__exit__()
		            except Exception as e:
		                print "error trying to exit context:", e

		if prefetch_activated:
		    for param in params:
		        if isinstance(param, Block):
		            keys = param.storageobj.keyList[param.storageobj.__class__.__name__]
		            exec("persistentdict = param.storageobj." + str(keys[0]))
		            if persistentdict.prefetch == True:
		                try:
		                    persistentdict.end_prefetch()
		                except Exception as e:
		                    print "error trying to prefetch:", e

		if not 'statistics_activated' in globals():
		    global statistics_activated
		    statistics_activated = True
		if statistics_activated:
		    for param in params:
		        if isinstance(param, Block):
		            param.storageobj.statistics()
		        if isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
		            param.statistics()

def getByID(objid):
    path = apppath + '/conf/imports.py'
    file = open(path, 'r')
    for line in file:
        exec(line)
    objidsplit = objid.split("_")
    print "objidsplit:", objidsplit

    if len(objidsplit) == 2:
        result = ''
        exec('result = ' + objidsplit[0] + '()')
        print "str(objidsplit[0]):", str(objidsplit[0])
        result.getByName(str(objidsplit[0]))
        return result

    else:
        cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
        session = cluster.connect()
        #(ksp,tab,dict_name,obj_type,entryPoint,port,tokens) = session.execute("SELECT ksp,tab,dict_name,obj_type,entrypoint,port,tkns FROM hecuba.blocks WHERE blockid = %s",(objid))[0]
        ksp =        session.execute("SELECT ksp FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].ksp
        tab =        session.execute("SELECT tab FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].tab
        dict_name =  session.execute("SELECT dict_name FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].dict_name
        obj_type =   session.execute("SELECT obj_type FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].obj_type
        entryPoint = session.execute("SELECT entrypoint FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].entrypoint
        port =       session.execute("SELECT port FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].port
        tokens =     session.execute("SELECT tkns FROM hecuba.blocks WHERE blockid = %s",(objid,))[0].tkns
        if str(obj_type) == 'qbeast':
            print "indexed storageobj"
            blockid = objid
            metadata = cluster.metadata
            tokenmap = metadata.token_map.token_to_host_owner
            odtokenmap = collections.OrderedDict(sorted(tokenmap.items()))
            if not 'prefetch_activated' in globals():
                global prefetch_activated
                prefetch_activated = False # this should be false, right?
            for position in tokens:
                for key, val in odtokenmap.iteritems():
                    # (self, peer,        keynames,           tablename,     blockkeyspace, myuuid)
                    b = IxBlock(str(val), tab,                tab,           ksp,           objid) # the keynames vale is wrong, needs to be corrected
                    exec("b.storageobj = " + str(dict_name) + "('" + str(dict_name) + "')")
                    if prefetch_activated:
                        b.storageobj.init_prefetch(b)
                    session.shutdown()
                    cluster.shutdown()
                    return b
        else:
            print "normal storageobj"
            # blockKeyspace = objidsplit[0]
            # blockKeyNames = str(objidsplit[1])
            # blockTableName = objidsplit[2]
            # blockRange = objidsplit[3]
            blockranges = objidsplit[3:len(objidsplit)]
            blockrangesf = ''
            for ind, val in enumerate(blockranges):
                if ind < (len(blockranges)-1):
                    blockrangesf += str(val) + '_'
                else:
                    blockrangesf += str(val)
            cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
            session = cluster.connect()
            metadata = cluster.metadata
            tokenmap = metadata.token_map.token_to_host_owner
            odtokenmap = collections.OrderedDict(sorted(tokenmap.items()))
            position = 0
            if not 'prefetch_activated' in globals():
                global prefetch_activated
            prefetch_activated = True
            for key, val in odtokenmap.iteritems():
                if str(position) == objidsplit[3]:
                    b = Block((str(val), blockrangesf), str(objidsplit[1]), objidsplit[2], objidsplit[0])
                    exec("b.storageobj = " + str(objidsplit[2]) + "('" + str(objidsplit[2]) + "')")
                    if prefetch_activated:
                        b.storageobj.init_prefetch(b)
                    session.shutdown()
                    cluster.shutdown()
                    return b
                position += 1
<|endoftext|>


<commit_before>from stream.utils import validate_feed_id, validate_user_id, validate_feed_slug

class Feed(object):

    def __init__(self, client, feed_slug, user_id, token):
        '''
        Initializes the Feed class

        :param client: the api client
        :param slug: the slug of the feed, ie user, flat, notification
        :param user_id: the id of the user
        :param token: the token
        '''
        self.client = client
        self.slug = feed_slug
        self.user_id = str(user_id)
        self.id = '%s:%s' % (feed_slug, user_id)
        self.token = token

        self.feed_url = 'feed/%s/' % self.id.replace(':', '/')
        self.feed_targets_url = 'feed_targets/%s/' % self.id.replace(':', '/')
        self.feed_together = self.id.replace(':', '')
        self.signature = self.feed_together + ' ' + self.token

    def create_scope_token(self, resource, action):
        '''
        creates the JWT token to perform an action on a owned resource
        '''
        return self.client.create_jwt_token(resource, action, feed_id=self.feed_together)

    def get_readonly_token(self):
        '''
        creates the JWT token to perform readonly operations
        '''
        return self.create_scope_token('*', 'read')

    def add_activity(self, activity_data):
        '''
        Adds an activity to the feed, this will also trigger an update
        to all the feeds which follow this feed

        :param activity_data: a dict with the activity data

        **Example**::

            activity_data = {'actor': 1, 'verb': 'tweet', 'object': 1}
            activity_id = feed.add_activity(activity_data)
        '''
        if activity_data.get('to') and not isinstance(activity_data.get('to'), (list, tuple, set)):
            raise TypeError('please provide the activity\'s to field as a list not a string')

        if activity_data.get('to'):
            activity_data = activity_data.copy()
            activity_data['to'] = self.add_to_signature(activity_data['to'])

        token = self.create_scope_token('feed', 'write')
        result = self.client.post(
            self.feed_url, data=activity_data, signature=token)
        return result

    def add_activities(self, activity_list):
        '''
        Adds a list of activities to the feed

        :param activity_list: a list with the activity data dicts

        **Example**::

            activity_data = [
                {'actor': 1, 'verb': 'tweet', 'object': 1},
                {'actor': 2, 'verb': 'watch', 'object': 2},
            ]
            result = feed.add_activities(activity_data)
        '''
        activities = []
        for activity_data in activity_list:
            activity_data = activity_data.copy()
            activities.append(activity_data)
            if activity_data.get('to'):
                activity_data['to'] = self.add_to_signature(
                    activity_data['to'])
        token = self.create_scope_token('feed', 'write')
        data = dict(activities=activities)
        if activities:
            result = self.client.post(
                self.feed_url, data=data, signature=token)
            return result

    def remove_activity(self, activity_id=None, foreign_id=None):
        '''
        Removes an activity from the feed

        :param activity_id: the activity id to remove from this feed
        (note this will also remove the activity from feeds which follow this feed)
        :param foreign_id: the foreign id you provided when adding the activity
        '''
        identifier = activity_id or foreign_id
        if not identifier:
            raise ValueError('please either provide activity_id or foreign_id')
        url = self.feed_url + '%s/' % identifier
        params = dict()
        token = self.create_scope_token('feed', 'delete')
        if foreign_id is not None:
            params['foreign_id'] = '1'
        result = self.client.delete(
            url, signature=token, params=params)
        return result

    def get(self, **params):
        '''
        Get the activities in this feed

        **Example**::

            # fast pagination using id filtering
            feed.get(limit=10, id_lte=100292310)

            # slow pagination using offset
            feed.get(limit=10, offset=10)
        '''
        for field in ['mark_read', 'mark_seen']:
            value = params.get(field)
            if isinstance(value, (list, tuple)):
                params[field] = ','.join(value)
        token = self.create_scope_token('feed', 'read')
        response = self.client.get(
            self.feed_url, params=params, signature=token)
        return response

    def follow(self, target_feed_slug, target_user_id, activity_copy_limit=None, **extra_data):
        '''
        Follows the given feed

        :param target_feed_slug: the slug of the target feed
        :param target_user_id: the user id
        '''
        target_feed_slug = validate_feed_slug(target_feed_slug)
        target_user_id = validate_user_id(target_user_id)
        target_feed_id = '%s:%s' % (target_feed_slug, target_user_id)
        url = self.feed_url + 'follows/'
        data = {
            'target': target_feed_id,
            'target_token': self.client.feed(target_feed_slug, target_user_id).token
        }
        if activity_copy_limit != None:
            data['activity_copy_limit'] = activity_copy_limit
        token = self.create_scope_token('follower', 'write')
        data.update(extra_data)
        response = self.client.post(
            url, data=data, signature=token)
        return response

    def unfollow(self, target_feed_slug, target_user_id, keep_history=False):
        '''
        Unfollow the given feed
        '''
        target_feed_slug = validate_feed_slug(target_feed_slug)
        target_user_id = validate_user_id(target_user_id)
        target_feed_id = '%s:%s' % (target_feed_slug, target_user_id)
        token = self.create_scope_token('follower', 'delete')
        url = self.feed_url + 'follows/%s/' % target_feed_id
        params = {}
        if keep_history:
            params['keep_history'] = True
        response = self.client.delete(url, signature=token, params=params)
        return response

    def followers(self, offset=0, limit=25, feeds=None):
        '''
        Lists the followers for the given feed
        '''
        feeds = feeds is not None and ','.join(feeds) or ''
        params = {
            'limit': limit,
            'offset': offset,
            'filter': feeds
        }
        url = self.feed_url + 'followers/'
        token = self.create_scope_token('follower', 'read')
        response = self.client.get(
            url, params=params, signature=token)
        return response

    def following(self, offset=0, limit=25, feeds=None, filter=None):
        '''
        List the feeds which this feed is following
        '''
        if feeds is not None:
            feeds = feeds is not None and ','.join(feeds) or ''
        params = {
            'offset': offset,
            'limit': limit,
            'filter': feeds
        }
        url = self.feed_url + 'follows/'
        token = self.create_scope_token('follower', 'read')
        response = self.client.get(
            url, params=params, signature=token)
        return response

    def add_to_signature(self, recipients):
        '''
        Takes a list of recipients such as ['user:1', 'user:2']
        and turns it into a list with the tokens included
        ['user:1 token', 'user:2 token']
        '''
        data = []
        for recipient in recipients:
            validate_feed_id(recipient)
            feed_slug, user_id = recipient.split(':')
            feed = self.client.feed(feed_slug, user_id)
            data.append("%s %s" % (recipient, feed.token))
        return data

    def update_activity_to_targets(self, foreign_id, time,
                                   replace=None, add=None, remove=None):
        data = {
            'foreign_id': foreign_id,
            'time': time,
        }

        if replace is not None:
            data['replace'] = replace
        if add is not None:
            data['add'] = add
        if remove is not None:
            data['remove'] = remove

        url = self.feed_targets_url + 'update_activity_to_targets/'

        token = self.create_scope_token('feed_targets', 'write')
        return self.client.post(url, data=data, signature=token)
<commit_msg>Change url a bit again<commit_after>from stream.utils import validate_feed_id, validate_user_id, validate_feed_slug

class Feed(object):

    def __init__(self, client, feed_slug, user_id, token):
        '''
        Initializes the Feed class

        :param client: the api client
        :param slug: the slug of the feed, ie user, flat, notification
        :param user_id: the id of the user
        :param token: the token
        '''
        self.client = client
        self.slug = feed_slug
        self.user_id = str(user_id)
        self.id = '%s:%s' % (feed_slug, user_id)
        self.token = token

        self.feed_url = 'feed/%s/' % self.id.replace(':', '/')
        self.feed_targets_url = 'feed_targets/%s/' % self.id.replace(':', '/')
        self.feed_together = self.id.replace(':', '')
        self.signature = self.feed_together + ' ' + self.token

    def create_scope_token(self, resource, action):
        '''
        creates the JWT token to perform an action on a owned resource
        '''
        return self.client.create_jwt_token(resource, action, feed_id=self.feed_together)

    def get_readonly_token(self):
        '''
        creates the JWT token to perform readonly operations
        '''
        return self.create_scope_token('*', 'read')

    def add_activity(self, activity_data):
        '''
        Adds an activity to the feed, this will also trigger an update
        to all the feeds which follow this feed

        :param activity_data: a dict with the activity data

        **Example**::

            activity_data = {'actor': 1, 'verb': 'tweet', 'object': 1}
            activity_id = feed.add_activity(activity_data)
        '''
        if activity_data.get('to') and not isinstance(activity_data.get('to'), (list, tuple, set)):
            raise TypeError('please provide the activity\'s to field as a list not a string')

        if activity_data.get('to'):
            activity_data = activity_data.copy()
            activity_data['to'] = self.add_to_signature(activity_data['to'])

        token = self.create_scope_token('feed', 'write')
        result = self.client.post(
            self.feed_url, data=activity_data, signature=token)
        return result

    def add_activities(self, activity_list):
        '''
        Adds a list of activities to the feed

        :param activity_list: a list with the activity data dicts

        **Example**::

            activity_data = [
                {'actor': 1, 'verb': 'tweet', 'object': 1},
                {'actor': 2, 'verb': 'watch', 'object': 2},
            ]
            result = feed.add_activities(activity_data)
        '''
        activities = []
        for activity_data in activity_list:
            activity_data = activity_data.copy()
            activities.append(activity_data)
            if activity_data.get('to'):
                activity_data['to'] = self.add_to_signature(
                    activity_data['to'])
        token = self.create_scope_token('feed', 'write')
        data = dict(activities=activities)
        if activities:
            result = self.client.post(
                self.feed_url, data=data, signature=token)
            return result

    def remove_activity(self, activity_id=None, foreign_id=None):
        '''
        Removes an activity from the feed

        :param activity_id: the activity id to remove from this feed
        (note this will also remove the activity from feeds which follow this feed)
        :param foreign_id: the foreign id you provided when adding the activity
        '''
        identifier = activity_id or foreign_id
        if not identifier:
            raise ValueError('please either provide activity_id or foreign_id')
        url = self.feed_url + '%s/' % identifier
        params = dict()
        token = self.create_scope_token('feed', 'delete')
        if foreign_id is not None:
            params['foreign_id'] = '1'
        result = self.client.delete(
            url, signature=token, params=params)
        return result

    def get(self, **params):
        '''
        Get the activities in this feed

        **Example**::

            # fast pagination using id filtering
            feed.get(limit=10, id_lte=100292310)

            # slow pagination using offset
            feed.get(limit=10, offset=10)
        '''
        for field in ['mark_read', 'mark_seen']:
            value = params.get(field)
            if isinstance(value, (list, tuple)):
                params[field] = ','.join(value)
        token = self.create_scope_token('feed', 'read')
        response = self.client.get(
            self.feed_url, params=params, signature=token)
        return response

    def follow(self, target_feed_slug, target_user_id, activity_copy_limit=None, **extra_data):
        '''
        Follows the given feed

        :param target_feed_slug: the slug of the target feed
        :param target_user_id: the user id
        '''
        target_feed_slug = validate_feed_slug(target_feed_slug)
        target_user_id = validate_user_id(target_user_id)
        target_feed_id = '%s:%s' % (target_feed_slug, target_user_id)
        url = self.feed_url + 'follows/'
        data = {
            'target': target_feed_id,
            'target_token': self.client.feed(target_feed_slug, target_user_id).token
        }
        if activity_copy_limit != None:
            data['activity_copy_limit'] = activity_copy_limit
        token = self.create_scope_token('follower', 'write')
        data.update(extra_data)
        response = self.client.post(
            url, data=data, signature=token)
        return response

    def unfollow(self, target_feed_slug, target_user_id, keep_history=False):
        '''
        Unfollow the given feed
        '''
        target_feed_slug = validate_feed_slug(target_feed_slug)
        target_user_id = validate_user_id(target_user_id)
        target_feed_id = '%s:%s' % (target_feed_slug, target_user_id)
        token = self.create_scope_token('follower', 'delete')
        url = self.feed_url + 'follows/%s/' % target_feed_id
        params = {}
        if keep_history:
            params['keep_history'] = True
        response = self.client.delete(url, signature=token, params=params)
        return response

    def followers(self, offset=0, limit=25, feeds=None):
        '''
        Lists the followers for the given feed
        '''
        feeds = feeds is not None and ','.join(feeds) or ''
        params = {
            'limit': limit,
            'offset': offset,
            'filter': feeds
        }
        url = self.feed_url + 'followers/'
        token = self.create_scope_token('follower', 'read')
        response = self.client.get(
            url, params=params, signature=token)
        return response

    def following(self, offset=0, limit=25, feeds=None, filter=None):
        '''
        List the feeds which this feed is following
        '''
        if feeds is not None:
            feeds = feeds is not None and ','.join(feeds) or ''
        params = {
            'offset': offset,
            'limit': limit,
            'filter': feeds
        }
        url = self.feed_url + 'follows/'
        token = self.create_scope_token('follower', 'read')
        response = self.client.get(
            url, params=params, signature=token)
        return response

    def add_to_signature(self, recipients):
        '''
        Takes a list of recipients such as ['user:1', 'user:2']
        and turns it into a list with the tokens included
        ['user:1 token', 'user:2 token']
        '''
        data = []
        for recipient in recipients:
            validate_feed_id(recipient)
            feed_slug, user_id = recipient.split(':')
            feed = self.client.feed(feed_slug, user_id)
            data.append("%s %s" % (recipient, feed.token))
        return data

    def update_activity_to_targets(self, foreign_id, time,
                                   replace=None, add=None, remove=None):
        data = {
            'foreign_id': foreign_id,
            'time': time,
        }

        if replace is not None:
            data['replace'] = replace
        if add is not None:
            data['add'] = add
        if remove is not None:
            data['remove'] = remove

        url = self.feed_targets_url + 'activity_to_targets/'

        token = self.create_scope_token('feed_targets', 'write')
        return self.client.post(url, data=data, signature=token)
<|endoftext|>


<commit_before>import hashlib
from io import StringIO
from datetime import timedelta
import re
import random
import string
import struct
import time
import sys
import shutil
import subprocess
import os
import numpy as np
import requests
import six

from tensorflow.core.util import event_pb2

import boto3

DAY = 86400
HOUR = 3600
MINUTE = 60

def remove_backspaces(line):
    splitline = re.split('(\x08+)', line)
    try:
        splitline = [unicode(s, 'utf-8') for s in splitline]
    except NameError:
        splitline = [str(s) for s in splitline]

    buf = StringIO()
    for i in range(0, len(splitline) - 1, 2):
        buf.write(splitline[i][:-len(splitline[i + 1])])

    if len(splitline) % 2 == 1:
        buf.write(splitline[-1])

    return buf.getvalue()

def sha256_checksum(filename, block_size=65536):
    sha256 = hashlib.sha256()
    with open(filename, 'rb') as f:
        for block in iter(lambda: f.read(block_size), b''):
            sha256.update(block)
    return sha256.hexdigest()

def rand_string(length):
    return "".join([random.choice(string.ascii_letters + string.digits)
                    for n in range(length)])

def event_reader(fileobj):

    if isinstance(fileobj, str):
        fileobj = open(fileobj, 'rb')

    header_len = 12
    footer_len = 4
    size_len = 8

    while True:
        try:
            data_len = struct.unpack('Q', fileobj.read(size_len))[0]
            fileobj.read(header_len - size_len)

            data = fileobj.read(data_len)

            event = None
            event = event_pb2.Event()
            event.ParseFromString(data)

            fileobj.read(footer_len)
            yield event
        except BaseException:
            break

    fileobj.close()

def rsync_cp(source, dest, ignore_arg='', logger=None):
    if os.path.exists(dest):
        shutil.rmtree(dest) if os.path.isdir(dest) else os.remove(dest)
    os.makedirs(dest)

    if ignore_arg != '':
        source += "/"
        tool = 'rsync'
        args = [tool, ignore_arg, '-aHAXE', source, dest]
    else:
        os.rmdir(dest)
        tool = 'cp'
        args = [tool, '-pR', source, dest]

    pcp = subprocess.Popen(args, stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
    cpout, _ = pcp.communicate()
    if pcp.returncode != 0 and logger is not None:
        logger.info('%s returned non-zero exit code. Output:' % tool)
        logger.info(cpout)

class Progbar(object):
    """Displays a progress bar.

    # Arguments
        target: Total number of steps expected, None if unknown.
        interval: Minimum visual progress update interval (in seconds).
    """

    def __init__(self, target, width=30, verbose=1, interval=0.05):
        self.width = width
        if target is None:
            target = -1
        self.target = target
        self.sum_values = {}
        self.unique_values = []
        self.start = time.time()
        self.last_update = 0
        self.interval = interval
        self.total_width = 0
        self.seen_so_far = 0
        self.verbose = verbose

    def update(self, current, values=None, force=False):
        """Updates the progress bar.

        # Arguments
            current: Index of current step.
            values: List of tuples (name, value_for_last_step).
                The progress bar will display averages for these values.
            force: Whether to force visual progress update.
        """
        values = values or []
        for k, v in values:
            if k not in self.sum_values:
                self.sum_values[k] = [v * (current - self.seen_so_far),
                                      current - self.seen_so_far]
                self.unique_values.append(k)
            else:
                self.sum_values[k][0] += v * (current - self.seen_so_far)
                self.sum_values[k][1] += (current - self.seen_so_far)
        self.seen_so_far = current

        now = time.time()
        if self.verbose == 1:
            if not force and (now - self.last_update) < self.interval:
                return

            prev_total_width = self.total_width
            sys.stdout.write('\b' * prev_total_width)
            sys.stdout.write('\r')

            if self.target is not -1:
                numdigits = int(np.floor(np.log10(self.target))) + 1
                barstr = '%%%dd/%%%dd [' % (numdigits, numdigits)
                bar = barstr % (current, self.target)
                prog = float(current) / self.target
                prog_width = int(self.width * prog)
                if prog_width > 0:
                    bar += ('=' * (prog_width - 1))
                    if current < self.target:
                        bar += '>'
                    else:
                        bar += '='
                bar += ('.' * (self.width - prog_width))
                bar += ']'
                sys.stdout.write(bar)
                self.total_width = len(bar)

            if current:
                time_per_unit = (now - self.start) / current
            else:
                time_per_unit = 0
            eta = time_per_unit * (self.target - current)
            info = ''
            if current < self.target and self.target is not -1:
                info += ' - ETA: %ds' % eta
            else:
                info += ' - %ds' % (now - self.start)
            for k in self.unique_values:
                info += ' - %s:' % k
                if isinstance(self.sum_values[k], list):
                    avg = np.mean(
                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                    if abs(avg) > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                else:
                    info += ' %s' % self.sum_values[k]

            self.total_width += len(info)
            if prev_total_width > self.total_width:
                info += ((prev_total_width - self.total_width) * ' ')

            sys.stdout.write(info)
            sys.stdout.flush()

            if current >= self.target:
                sys.stdout.write('\n')

        if self.verbose == 2:
            if current >= self.target:
                info = '%ds' % (now - self.start)
                for k in self.unique_values:
                    info += ' - %s:' % k
                    avg = np.mean(
                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                    if avg > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                sys.stdout.write(info + "\n")

        self.last_update = now

    def add(self, n, values=None):
        self.update(self.seen_so_far + n, values)

def download_file(url, local_path, logger=None):
    response = requests.get(
        url,
        stream=True)
    if logger:
        logger.info(("Trying to download file at url {} to " +
                     "local path {}").format(url, local_path))

    if response.status_code == 200:
        with open(local_path, 'wb') as f:
            for chunk in response:
                f.write(chunk)
    elif logger:
        logger.info("Response error with code {}"
                    .format(response.status_code))

    return response

def upload_file(url, local_path, logger=None):
    if logger:
        logger.info(("Trying to upload file {} to " +
                     "url {}").format(local_path, url))
    tic = time.time()
    with open(local_path, 'rb') as f:
        resp = requests.put(url, data=f)

    if resp.status_code != 200 and logger:
        logger.error(str(resp.reason))

    if logger:
        logger.debug('File upload done in {} s'
            .format(time.time() - tic))

def download_file_from_qualified(qualified, local_path, logger=None):
    assert qualified.startswith('s3://') or \
        qualified.startswith('gs://')

    bucket = qualified.split('/')[2]
    key = '/'.join(qualified.split('/')[3:])

    if logger is not None:
        logger.debug(('Downloading file from bucket {} ' +
                      ' and key {} to local path {}')
                     .format(bucket, key, local_path))

    if qualified.startswith('s3://'):
        boto3.client('s3').download_file(bucket, key, local_path)
    else:
        raise NotImplementedError

def has_aws_credentials():
    return boto3.client('s3')._request_signer._credentials is not None

def retry(f,
          no_retries=5, sleeptime=1,
          exception_class=BaseException, logger=None):
    for i in range(no_retries):
        try:
            return f()
        except exception_class as e:
            if logger:
                logger.info(
                    ('Exception {} is caught, ' +
                     'sleeping {}s and retrying (attempt {} of {})')
                    .format(e, sleeptime, i, no_retries))
            time.sleep(sleeptime)

def compression_to_extension(compression):
    return _compression_to_extension_taropt(compression)[0]

def compression_to_taropt(compression):
    return _compression_to_extension_taropt(compression)[1]

def _compression_to_extension_taropt(compression):
    default_compression = 'none'
    if compression is None:
        compression = default_compression

    compression = compression.lower()

    if compression == 'bzip2':
        return '.bz2', '--bzip2'

    elif compression == 'gzip':
        return '.gz', '--gzip'

    elif compression == 'xz':
        return '.xz', '--xz'

    elif compression == 'lzma':
        return '.lzma', '--lzma'

    elif compression == 'lzop':
        return '.lzop', '--lzop'

    elif compression == 'none':
        return '', ''

    raise ValueError('Unknown compression method {}'
                     .format(compression))

def timeit(method):

    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()

        line = '%r (%r, %r) %2.2f sec' % \
            (method.__name__, args, kw, te - ts)

        try:
            logger = args[0].logger
            logger.info(line)
        except BaseException:
            print(line)

        return result

    return timed

def sixdecode(s):
    if isinstance(s, six.string_types):
        return s
    if isinstance(s, six.binary_type):
        return s.decode('utf8')

    raise TypeError("Unknown type of " + str(s))

regex = re.compile(
    r'((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')

# parse_duration parses strings into time delta values that python can
# deal with.  Examples include 12h, 11h60m, 719m60s, 11h3600s
#
def parse_duration(duration_str):
    parts = regex.match(duration_str)
    if not parts:
        return
    parts = parts.groupdict()
    time_params = {}
    for (name, param) in six.iteritems(parts):
        if param:
            time_params[name] = int(param)
    return timedelta(**time_params)

def str2duration(s):
    return parse_duration(s.lower())
<commit_msg>autopep8<commit_after>import hashlib
from io import StringIO
from datetime import timedelta
import re
import random
import string
import struct
import time
import sys
import shutil
import subprocess
import os
import numpy as np
import requests
import six

from tensorflow.core.util import event_pb2

import boto3

DAY = 86400
HOUR = 3600
MINUTE = 60

def remove_backspaces(line):
    splitline = re.split('(\x08+)', line)
    try:
        splitline = [unicode(s, 'utf-8') for s in splitline]
    except NameError:
        splitline = [str(s) for s in splitline]

    buf = StringIO()
    for i in range(0, len(splitline) - 1, 2):
        buf.write(splitline[i][:-len(splitline[i + 1])])

    if len(splitline) % 2 == 1:
        buf.write(splitline[-1])

    return buf.getvalue()

def sha256_checksum(filename, block_size=65536):
    sha256 = hashlib.sha256()
    with open(filename, 'rb') as f:
        for block in iter(lambda: f.read(block_size), b''):
            sha256.update(block)
    return sha256.hexdigest()

def rand_string(length):
    return "".join([random.choice(string.ascii_letters + string.digits)
                    for n in range(length)])

def event_reader(fileobj):

    if isinstance(fileobj, str):
        fileobj = open(fileobj, 'rb')

    header_len = 12
    footer_len = 4
    size_len = 8

    while True:
        try:
            data_len = struct.unpack('Q', fileobj.read(size_len))[0]
            fileobj.read(header_len - size_len)

            data = fileobj.read(data_len)

            event = None
            event = event_pb2.Event()
            event.ParseFromString(data)

            fileobj.read(footer_len)
            yield event
        except BaseException:
            break

    fileobj.close()

def rsync_cp(source, dest, ignore_arg='', logger=None):
    if os.path.exists(dest):
        shutil.rmtree(dest) if os.path.isdir(dest) else os.remove(dest)
    os.makedirs(dest)

    if ignore_arg != '':
        source += "/"
        tool = 'rsync'
        args = [tool, ignore_arg, '-aHAXE', source, dest]
    else:
        os.rmdir(dest)
        tool = 'cp'
        args = [tool, '-pR', source, dest]

    pcp = subprocess.Popen(args, stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
    cpout, _ = pcp.communicate()
    if pcp.returncode != 0 and logger is not None:
        logger.info('%s returned non-zero exit code. Output:' % tool)
        logger.info(cpout)

class Progbar(object):
    """Displays a progress bar.

    # Arguments
        target: Total number of steps expected, None if unknown.
        interval: Minimum visual progress update interval (in seconds).
    """

    def __init__(self, target, width=30, verbose=1, interval=0.05):
        self.width = width
        if target is None:
            target = -1
        self.target = target
        self.sum_values = {}
        self.unique_values = []
        self.start = time.time()
        self.last_update = 0
        self.interval = interval
        self.total_width = 0
        self.seen_so_far = 0
        self.verbose = verbose

    def update(self, current, values=None, force=False):
        """Updates the progress bar.

        # Arguments
            current: Index of current step.
            values: List of tuples (name, value_for_last_step).
                The progress bar will display averages for these values.
            force: Whether to force visual progress update.
        """
        values = values or []
        for k, v in values:
            if k not in self.sum_values:
                self.sum_values[k] = [v * (current - self.seen_so_far),
                                      current - self.seen_so_far]
                self.unique_values.append(k)
            else:
                self.sum_values[k][0] += v * (current - self.seen_so_far)
                self.sum_values[k][1] += (current - self.seen_so_far)
        self.seen_so_far = current

        now = time.time()
        if self.verbose == 1:
            if not force and (now - self.last_update) < self.interval:
                return

            prev_total_width = self.total_width
            sys.stdout.write('\b' * prev_total_width)
            sys.stdout.write('\r')

            if self.target is not -1:
                numdigits = int(np.floor(np.log10(self.target))) + 1
                barstr = '%%%dd/%%%dd [' % (numdigits, numdigits)
                bar = barstr % (current, self.target)
                prog = float(current) / self.target
                prog_width = int(self.width * prog)
                if prog_width > 0:
                    bar += ('=' * (prog_width - 1))
                    if current < self.target:
                        bar += '>'
                    else:
                        bar += '='
                bar += ('.' * (self.width - prog_width))
                bar += ']'
                sys.stdout.write(bar)
                self.total_width = len(bar)

            if current:
                time_per_unit = (now - self.start) / current
            else:
                time_per_unit = 0
            eta = time_per_unit * (self.target - current)
            info = ''
            if current < self.target and self.target is not -1:
                info += ' - ETA: %ds' % eta
            else:
                info += ' - %ds' % (now - self.start)
            for k in self.unique_values:
                info += ' - %s:' % k
                if isinstance(self.sum_values[k], list):
                    avg = np.mean(
                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                    if abs(avg) > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                else:
                    info += ' %s' % self.sum_values[k]

            self.total_width += len(info)
            if prev_total_width > self.total_width:
                info += ((prev_total_width - self.total_width) * ' ')

            sys.stdout.write(info)
            sys.stdout.flush()

            if current >= self.target:
                sys.stdout.write('\n')

        if self.verbose == 2:
            if current >= self.target:
                info = '%ds' % (now - self.start)
                for k in self.unique_values:
                    info += ' - %s:' % k
                    avg = np.mean(
                        self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                    if avg > 1e-3:
                        info += ' %.4f' % avg
                    else:
                        info += ' %.4e' % avg
                sys.stdout.write(info + "\n")

        self.last_update = now

    def add(self, n, values=None):
        self.update(self.seen_so_far + n, values)

def download_file(url, local_path, logger=None):
    response = requests.get(
        url,
        stream=True)
    if logger:
        logger.info(("Trying to download file at url {} to " +
                     "local path {}").format(url, local_path))

    if response.status_code == 200:
        with open(local_path, 'wb') as f:
            for chunk in response:
                f.write(chunk)
    elif logger:
        logger.info("Response error with code {}"
                    .format(response.status_code))

    return response

def upload_file(url, local_path, logger=None):
    if logger:
        logger.info(("Trying to upload file {} to " +
                     "url {}").format(local_path, url))
    tic = time.time()
    with open(local_path, 'rb') as f:
        resp = requests.put(url, data=f)

    if resp.status_code != 200 and logger:
        logger.error(str(resp.reason))

    if logger:
        logger.debug('File upload done in {} s'
                     .format(time.time() - tic))

def download_file_from_qualified(qualified, local_path, logger=None):
    assert qualified.startswith('s3://') or \
        qualified.startswith('gs://')

    bucket = qualified.split('/')[2]
    key = '/'.join(qualified.split('/')[3:])

    if logger is not None:
        logger.debug(('Downloading file from bucket {} ' +
                      ' and key {} to local path {}')
                     .format(bucket, key, local_path))

    if qualified.startswith('s3://'):
        boto3.client('s3').download_file(bucket, key, local_path)
    else:
        raise NotImplementedError

def has_aws_credentials():
    return boto3.client('s3')._request_signer._credentials is not None

def retry(f,
          no_retries=5, sleeptime=1,
          exception_class=BaseException, logger=None):
    for i in range(no_retries):
        try:
            return f()
        except exception_class as e:
            if logger:
                logger.info(
                    ('Exception {} is caught, ' +
                     'sleeping {}s and retrying (attempt {} of {})')
                    .format(e, sleeptime, i, no_retries))
            time.sleep(sleeptime)

def compression_to_extension(compression):
    return _compression_to_extension_taropt(compression)[0]

def compression_to_taropt(compression):
    return _compression_to_extension_taropt(compression)[1]

def _compression_to_extension_taropt(compression):
    default_compression = 'none'
    if compression is None:
        compression = default_compression

    compression = compression.lower()

    if compression == 'bzip2':
        return '.bz2', '--bzip2'

    elif compression == 'gzip':
        return '.gz', '--gzip'

    elif compression == 'xz':
        return '.xz', '--xz'

    elif compression == 'lzma':
        return '.lzma', '--lzma'

    elif compression == 'lzop':
        return '.lzop', '--lzop'

    elif compression == 'none':
        return '', ''

    raise ValueError('Unknown compression method {}'
                     .format(compression))

def timeit(method):

    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()

        line = '%r (%r, %r) %2.2f sec' % \
            (method.__name__, args, kw, te - ts)

        try:
            logger = args[0].logger
            logger.info(line)
        except BaseException:
            print(line)

        return result

    return timed

def sixdecode(s):
    if isinstance(s, six.string_types):
        return s
    if isinstance(s, six.binary_type):
        return s.decode('utf8')

    raise TypeError("Unknown type of " + str(s))

regex = re.compile(
    r'((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')

# parse_duration parses strings into time delta values that python can
# deal with.  Examples include 12h, 11h60m, 719m60s, 11h3600s
#
def parse_duration(duration_str):
    parts = regex.match(duration_str)
    if not parts:
        return
    parts = parts.groupdict()
    time_params = {}
    for (name, param) in six.iteritems(parts):
        if param:
            time_params[name] = int(param)
    return timedelta(**time_params)

def str2duration(s):
    return parse_duration(s.lower())
<|endoftext|>


<commit_before>"""
The MIT License

Copyright (c) 2014 Giovanni Damiola

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from flask import Flask, jsonify, render_template, url_for
from flask.ext.restful import reqparse, abort, Api, Resource
from flask_bootstrap import Bootstrap

import re
import sys
import getopt
import json
from bson import json_util

from tools import connector
from tools import scraper
from tools import clusterizer

VERSION = "0.1"

app = Flask(__name__)
Bootstrap(app)
api = Api(app)

parser = reqparse.RequestParser()
parser.add_argument('url', type=str)

## fucntion to check if the entry already exists
def it_exists(url):
	try:
		url_norm = scraper.normalize_url(url)
		print url
		print url_norm
		if len(connector.mongo_find(url_norm)) > 0:
			return 1
		else:
			return 0
	except:
		print "...this is the first entry?"
		return 3

## check if the string is a valid url
def it_is_url(url):
	regex = re.compile(
        r'^https?://' # http:// or https://
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
        r'localhost|' #localhost...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
        r'(?::\d+)?' # optional port
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
	if re.match(regex,url):
		return True
	else:
		return True

##### Main page
#####
@app.route('/')
def index():
	url_for('static', filename='style.css')
	url_for('static', filename='favicon.ico')
	url_for('static', filename='jquery.tagcloud.js')
	return render_template('main.html')

##### Sumo REST API initialization

##	show a list of documents and lets you POST to add new documents
class Sumo(Resource):

	# GET : retrieve the documents index from mongodb
    def get(self):
        return connector.mongo_all(1000), 200

    # POST : analyze the submitted document and save the results on mongodb
    def post(self):
		args = parser.parse_args()
		url = args['url']
		if it_is_url(url):
			if it_exists(url) == 1:
				abort(409, message = "CONFLICT The document {} already exists".format(url))
			else:
				page_title, text, img = scraper.extract(url)
				result = scraper.get_scraped_data(page_title,text,img,url)
				connector.mongo_insert(result)
				return {"CREATED":url},201
		else:
			return {"MALFORMED URL":url},415

## resource for retrieve the documents
##
class SumoDocument(Resource):

	# GET : retrieve the results for the document requested
	def get(self,url_norm):
		if it_exists(url_norm) == 1:
			result = connector.mongo_find(url_norm)
			return result, 200
		else:
			abort(404, message = "NOT FOUND: The document {} doesn't exists".format(url_norm))
		return url

	# DELETE : remove the document requested from mongodb
	def delete(self,url_norm):
		if it_exists(url_norm) == 1:
			result = connector.mongo_remove(url_norm)
			return result, 204
		else:
			abort(404, message = "NOT FOUND: The document {} doesn't exists".format(url_norm))
		return url

# resource to retrieve the document cluster
class SumoCluster(Resource):

	# GET : retrieve the cluster for the document requested
	def get(self,url_norm):
		if it_exists(url_norm) == 1:
			cluster = clusterizer.get_cluster(url_norm)
			print "WARNING: clustering in real time, consider a solution usin a db for partial results"
			return cluster, 200
		else:
			abort(404, message = "NOT FOUND: The document {} doesn't exists".format(url_norm))
		return url

# Initializing the api resources

api.add_resource(Sumo, '/sumo')
api.add_resource(SumoDocument, '/sumo/<string:url_norm>')
api.add_resource(SumoCluster, '/sumo/cluster/<string:url_norm>')

def main(argv):
	server_port = 5000
	server_host = "0.0.0.0"
	try:
		opts, args = getopt.getopt(argv,"hvs:p:",["help","version","server","port"])
		if not opts:
			usage()
			sys.exit(2)
	except getopt.GetoptError as e:
			usage()
			sys.exit(2)
	for opt, arg in opts:
		if opt in ('-h','--help'):
			usage()
			sys.exit()
		elif opt in ('-v','--version'):
			print __file__+" "+VERSION
			sys.exit()
		elif opt in ('-s','--server'):
			server_host = str(arg)
		elif opt in ('-p','--port'):
			server_port = int(arg)
	app.run(debug=True, host=server_host, port=server_port)

def usage():
	print "\nUsage: "+__file__+" -s <IP> [OPTION]"
	print "A tool for semantic analysis of web articles\n"
	print "-h, --help	 give this help"
	print "-v, --version    display version number"
	print "-s, --server     IP where the server is listening"
	print "-p, --port       port where the server is listening, default 5000"
	print "\n\n"

if __name__ == '__main__':
	main(sys.argv[1:])
<commit_msg>minor fixed<commit_after>"""
The MIT License

Copyright (c) 2014 Giovanni Damiola

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

from flask import Flask, jsonify, render_template, url_for
from flask.ext.restful import reqparse, abort, Api, Resource
from flask_bootstrap import Bootstrap

import re
import sys
import getopt
import json
from bson import json_util

from tools import connector
from tools import scraper
from tools import clusterizer

VERSION = "0.1"

app = Flask(__name__)
Bootstrap(app)
api = Api(app)

parser = reqparse.RequestParser()
parser.add_argument('url', type=str)

## fucntion to check if the entry already exists
def it_exists(url):
	try:
		url_norm = scraper.normalize_url(url)
		print url
		print url_norm
		if len(connector.mongo_find(url_norm)) > 0:
			return 1
		else:
			return 0
	except:
		print "...this is the first entry?"
		return 3

## check if the string is a valid url
def it_is_url(url):
	regex = re.compile(
        r'^https?://' # http:// or https://
        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
        r'localhost|' #localhost...
        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
        r'(?::\d+)?' # optional port
        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
	if re.match(regex,url):
		return True
	else:
		return True

# static files
@app.route('/')
def index():
	url_for('static', filename='style.css')
	url_for('static', filename='favicon.ico')
	url_for('static', filename='jquery.tagcloud.js')
	return render_template('main.html')

##### Sumo REST API initialization

## show a list of documents and lets you POST to add new documents
class Sumo(Resource):

    # GET : retrieve the documents index from mongodb
    def get(self):
        return connector.mongo_all(1000), 200

    # POST : analyze the submitted document and save the results on mongodb
    def post(self):
		args = parser.parse_args()
		url = args['url']
		if it_is_url(url):
			if it_exists(url) == 1:
				abort(409, message = "CONFLICT The document {} already exists".format(url))
			else:
				page_title, text, img = scraper.extract(url)
				result = scraper.get_scraped_data(page_title,text,img,url)
				connector.mongo_insert(result)
				return {"CREATED":url},201
		else:
			return {"MALFORMED URL":url},415

## resource for retrieve the documents
##
class SumoDocument(Resource):

	# GET : retrieve the results for the requested document
	def get(self,url_norm):
		if it_exists(url_norm) == 1:
			result = connector.mongo_find(url_norm)
			return result, 200
		else:
			abort(404, message = "NOT FOUND: The document {} doesn't exists".format(url_norm))
		return url

	# DELETE : remove the requested document from mongodb
	def delete(self,url_norm):
		if it_exists(url_norm) == 1:
			result = connector.mongo_remove(url_norm)
			return result, 204
		else:
			abort(404, message = "NOT FOUND: The document {} doesn't exists".format(url_norm))
		return url

# resource to retrieve the document cluster
class SumoCluster(Resource):

	# GET : retrieve the cluster for the requested document
	def get(self,url_norm):
		if it_exists(url_norm) == 1:
			cluster = clusterizer.get_cluster(url_norm)
			print "WARNING: clustering in real time, consider a solution usin a db for partial results"
			return cluster, 200
		else:
			abort(404, message = "NOT FOUND: The document {} doesn't exists".format(url_norm))
		return url

# Initializing the api resources

api.add_resource(Sumo, '/sumo')
api.add_resource(SumoDocument, '/sumo/<string:url_norm>')
api.add_resource(SumoCluster, '/sumo/cluster/<string:url_norm>')

# main
def main(argv):
	server_port = 5000
	server_host = "0.0.0.0"
	try:
		opts, args = getopt.getopt(argv,"hvs:p:",["help","version","server","port"])
		if not opts:
			usage()
			sys.exit(2)
	except getopt.GetoptError as e:
			usage()
			sys.exit(2)
	for opt, arg in opts:
		if opt in ('-h','--help'):
			usage()
			sys.exit()
		elif opt in ('-v','--version'):
			print __file__+" "+VERSION
			sys.exit()
		elif opt in ('-s','--server'):
			server_host = str(arg)
		elif opt in ('-p','--port'):
			server_port = int(arg)
	app.run(debug=True, host=server_host, port=server_port)

# to print the usage help string
def usage():
	print "\nUsage: "+__file__+" -s <IP> [OPTION]"
	print "A tool for semantic analysis of web articles\n"
	print "-h, --help	 give this help"
	print "-v, --version    display version number"
	print "-s, --server     IP where the server is listening"
	print "-p, --port       port where the server is listening, default 5000"
	print "\n\n"

if __name__ == '__main__':
	main(sys.argv[1:])
<|endoftext|>