<commit_before>#!/usr/bin/env python
'''Web-based file differ.

For usage, see README.md.
'''

import argparse
import logging
import mimetypes
import os
import socket
import sys
from threading import Timer
import webbrowser

from flask import (Flask, render_template, send_from_directory,
                   request, jsonify, Response)
from werkzeug.serving import WSGIRequestHandler

import util

def determine_path():
    """Borrowed from wxglade.py"""
    try:
        root = __file__
        if os.path.islink (root):
            root = os.path.realpath (root)
        return os.path.dirname (os.path.abspath (root))
    except:
        print "I'm sorry, but something is wrong."
        print "There is no __file__ variable. Please contact the author."
        sys.exit()

# This is essential when run from distutils and does no harm otherwise.
ORIGINAL_DIR = os.getcwd()
path = determine_path()
os.chdir(path)

parser = argparse.ArgumentParser(description='Run webdiff.')
parser.add_argument('a', type=str, help="'Left' file to diff")
parser.add_argument('b', type=str, help="'Right' file to diff")
args = parser.parse_args()

class Config:
    pass
    #TESTING=True  # not exactly sure what this does...

app = Flask(__name__)
app.config.from_object(Config)
app.config.from_envvar('WEBDIFF_CONFIG', silent=True)

A_DIR = None
B_DIR = None
DIFF = None
PORT = None

if app.config['TESTING'] or app.config['DEBUG']:
    handler = logging.StreamHandler()
    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)

    app.logger.addHandler(handler)
    for logname in ['github', '']:
        log = logging.getLogger(logname)
        log.setLevel(logging.DEBUG)
        log.addHandler(handler)
else:
    # quiet down werkzeug -- no need to log every request.
    logging.getLogger('werkzeug').setLevel(logging.ERROR)

@app.route("/<side>/get_contents", methods=['POST'])
def get_contents(side):
    assert side in ('a', 'b')
    path = request.form.get('path', '')
    if not path:
        e = {"code": "incomplete",
             "message": "Incomplete request (need path)"}
        response = jsonify(e)
        response.status_code = 400
        return response

    try:
        abs_path = os.path.join(A_DIR if side == 'a' else B_DIR, path)
        is_binary = util.is_binary_file(abs_path)
        if is_binary:
            size = os.path.getsize(abs_path)
            contents = "Binary file (%d bytes)" % size
        else:
            contents = open(abs_path).read()
        return Response(contents, mimetype='text/plain')
    except Exception:
        e = {"code": "read-error",
             "message": "Unable to read %s" % abs_path}
        response = jsonify(e)
        response.status_code = 400
        return response

@app.route("/<side>/image/<path:path>")
def get_image(side, path):
    assert side in ('a', 'b')
    if not path:
        e = {"code": "incomplete",
             "message": "Incomplete request (need path)"}
        response = jsonify(e)
        response.status_code = 400
        return response

    mime_type, enc = mimetypes.guess_type(path)
    if not mime_type.startswith('image/') or enc is not None:
        e = {"code": "wrongtype",
             "message": "Requested file of type (%s, %s) as image" % (
                 mime_type, enc)}
        response = jsonify(e)
        response.status_code = 400
        return response

    try:
        abs_path = os.path.join(A_DIR if side == 'a' else B_DIR, path)
        contents = open(abs_path).read()
        return Response(contents, mimetype=mime_type)
    except Exception:
        e = {"code": "read-error",
             "message": "Unable to read %s" % abs_path}
        response = jsonify(e)
        response.status_code = 400
        return response

# Show the first diff by default
@app.route("/")
def index():
    if not 'NO_FRAME' in app.config:
        return render_template('heartbeat.html', src='/0')
    else:
        return file_diff('0')

@app.route("/<idx>")
def file_diff(idx):
    idx = int(idx)
    path = DIFF[idx]['a'] or DIFF[idx]['b']
    return render_template('file_diff.html',
                           idx=idx,
                           pairs=DIFF,
                           this_pair=DIFF[idx],
                           is_image_diff=util.is_image_diff(DIFF[idx]),
                           num_pairs=len(DIFF))

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static/img'),
                               'favicon.ico',
                               mimetype='image/vnd.microsoft.icon')

@app.route('/kill', methods=['POST'])
def kill():
    if 'STAY_RUNNING' in app.config: return
    func = request.environ.get('werkzeug.server.shutdown')
    if func is None:
        raise RuntimeError('Not running with the Werkzeug Server')
    func()
    return "Shutting down..."

def open_browser():
    global PORT
    if not 'NO_OPEN_BROWSER' in app.config:
        webbrowser.open_new_tab('http://localhost:%s' % PORT)

def usage_and_die():
    sys.stderr.write(
'''Usage: webdiff <left_dir> <right_dir>
       webdiff <left_file> <right_file>

Or run "git webdiff" from a git repository.
''')
    sys.exit(1)

def pick_a_port():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(('localhost', 0))
    port = sock.getsockname()[1]
    sock.close()
    return port

def adjust_path(path):
    '''Changes relative paths to be abs w/r/t/ the original cwd.'''
    global ORIGINAL_DIR
    if os.path.isabs(path):
        return path
    else:
        return os.path.join(ORIGINAL_DIR, path)

def shim_for_file_diff(a_file, b_file):
    '''Sets A_DIR, B_DIR and DIFF to do a one-file diff.'''
    global A_DIR, B_DIR, DIFF
    A_DIR = os.path.dirname(a_file)
    a_file = os.path.basename(a_file)
    B_DIR = os.path.dirname(b_file)
    b_file = os.path.basename(b_file)
    DIFF = [{'a': a_file,
             'b': b_file,
             'idx': 0,
             'path': a_file,
             'type': 'change'}]  # 'change' is the only likely case.

def run():
    global A_DIR, B_DIR, DIFF, PORT
    A_DIR = adjust_path(args.a)
    B_DIR = adjust_path(args.b)
    if os.path.isdir(A_DIR) != os.path.isdir(B_DIR):
        usage_and_die()

    PORT = pick_a_port()

    if app.config['TESTING'] or app.config['DEBUG']:
        sys.stderr.write('Diffing:\nA: %s\nB: %s\n\n' % (A_DIR, B_DIR))

    sys.stderr.write('''Serving diffs on http://localhost:%s
Close the browser tab or hit Ctrl-C when you're done.
''' % PORT)
    if os.path.isdir(A_DIR):
        DIFF = util.find_diff(A_DIR, B_DIR)
    else:
        shim_for_file_diff(A_DIR, B_DIR)
    logging.info('Diff: %r', DIFF)
    Timer(0.1, open_browser).start()
    app.run(host='0.0.0.0', port=PORT)

if __name__ == '__main__':
    run()
<commit_msg>add --port option<commit_after>#!/usr/bin/env python
'''Web-based file differ.

For usage, see README.md.
'''

import argparse
import logging
import mimetypes
import os
import socket
import sys
from threading import Timer
import webbrowser

from flask import (Flask, render_template, send_from_directory,
                   request, jsonify, Response)
from werkzeug.serving import WSGIRequestHandler

import util

def determine_path():
    """Borrowed from wxglade.py"""
    try:
        root = __file__
        if os.path.islink (root):
            root = os.path.realpath (root)
        return os.path.dirname (os.path.abspath (root))
    except:
        print "I'm sorry, but something is wrong."
        print "There is no __file__ variable. Please contact the author."
        sys.exit()

# This is essential when run from distutils and does no harm otherwise.
ORIGINAL_DIR = os.getcwd()
path = determine_path()
os.chdir(path)

parser = argparse.ArgumentParser(description='Run webdiff.')
parser.add_argument('--port', '-p', type=int, help="Port to run webdiff on.", default=8080)
parser.add_argument('a', type=str, help="'Left' file to diff")
parser.add_argument('b', type=str, help="'Right' file to diff")
args = parser.parse_args()

class Config:
    pass
    #TESTING=True  # not exactly sure what this does...

app = Flask(__name__)
app.config.from_object(Config)
app.config.from_envvar('WEBDIFF_CONFIG', silent=True)

A_DIR = None
B_DIR = None
DIFF = None
PORT = args.port

if app.config['TESTING'] or app.config['DEBUG']:
    handler = logging.StreamHandler()
    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)

    app.logger.addHandler(handler)
    for logname in ['github', '']:
        log = logging.getLogger(logname)
        log.setLevel(logging.DEBUG)
        log.addHandler(handler)
else:
    # quiet down werkzeug -- no need to log every request.
    logging.getLogger('werkzeug').setLevel(logging.ERROR)

@app.route("/<side>/get_contents", methods=['POST'])
def get_contents(side):
    assert side in ('a', 'b')
    path = request.form.get('path', '')
    if not path:
        e = {"code": "incomplete",
             "message": "Incomplete request (need path)"}
        response = jsonify(e)
        response.status_code = 400
        return response

    try:
        abs_path = os.path.join(A_DIR if side == 'a' else B_DIR, path)
        is_binary = util.is_binary_file(abs_path)
        if is_binary:
            size = os.path.getsize(abs_path)
            contents = "Binary file (%d bytes)" % size
        else:
            contents = open(abs_path).read()
        return Response(contents, mimetype='text/plain')
    except Exception:
        e = {"code": "read-error",
             "message": "Unable to read %s" % abs_path}
        response = jsonify(e)
        response.status_code = 400
        return response

@app.route("/<side>/image/<path:path>")
def get_image(side, path):
    assert side in ('a', 'b')
    if not path:
        e = {"code": "incomplete",
             "message": "Incomplete request (need path)"}
        response = jsonify(e)
        response.status_code = 400
        return response

    mime_type, enc = mimetypes.guess_type(path)
    if not mime_type.startswith('image/') or enc is not None:
        e = {"code": "wrongtype",
             "message": "Requested file of type (%s, %s) as image" % (
                 mime_type, enc)}
        response = jsonify(e)
        response.status_code = 400
        return response

    try:
        abs_path = os.path.join(A_DIR if side == 'a' else B_DIR, path)
        contents = open(abs_path).read()
        return Response(contents, mimetype=mime_type)
    except Exception:
        e = {"code": "read-error",
             "message": "Unable to read %s" % abs_path}
        response = jsonify(e)
        response.status_code = 400
        return response

# Show the first diff by default
@app.route("/")
def index():
    if not 'NO_FRAME' in app.config:
        return render_template('heartbeat.html', src='/0')
    else:
        return file_diff('0')

@app.route("/<idx>")
def file_diff(idx):
    idx = int(idx)
    path = DIFF[idx]['a'] or DIFF[idx]['b']
    return render_template('file_diff.html',
                           idx=idx,
                           pairs=DIFF,
                           this_pair=DIFF[idx],
                           is_image_diff=util.is_image_diff(DIFF[idx]),
                           num_pairs=len(DIFF))

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static/img'),
                               'favicon.ico',
                               mimetype='image/vnd.microsoft.icon')

@app.route('/kill', methods=['POST'])
def kill():
    if 'STAY_RUNNING' in app.config: return
    func = request.environ.get('werkzeug.server.shutdown')
    if func is None:
        raise RuntimeError('Not running with the Werkzeug Server')
    func()
    return "Shutting down..."

def open_browser():
    global PORT
    if not 'NO_OPEN_BROWSER' in app.config:
        webbrowser.open_new_tab('http://localhost:%s' % PORT)

def usage_and_die():
    sys.stderr.write(
'''Usage: webdiff <left_dir> <right_dir>
       webdiff <left_file> <right_file>

Or run "git webdiff" from a git repository.
''')
    sys.exit(1)

def pick_a_port():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(('localhost', 0))
    port = sock.getsockname()[1]
    sock.close()
    return port

def adjust_path(path):
    '''Changes relative paths to be abs w/r/t/ the original cwd.'''
    global ORIGINAL_DIR
    if os.path.isabs(path):
        return path
    else:
        return os.path.join(ORIGINAL_DIR, path)

def shim_for_file_diff(a_file, b_file):
    '''Sets A_DIR, B_DIR and DIFF to do a one-file diff.'''
    global A_DIR, B_DIR, DIFF
    A_DIR = os.path.dirname(a_file)
    a_file = os.path.basename(a_file)
    B_DIR = os.path.dirname(b_file)
    b_file = os.path.basename(b_file)
    DIFF = [{'a': a_file,
             'b': b_file,
             'idx': 0,
             'path': a_file,
             'type': 'change'}]  # 'change' is the only likely case.

def run():
    global A_DIR, B_DIR, DIFF, PORT
    A_DIR = adjust_path(args.a)
    B_DIR = adjust_path(args.b)
    if os.path.isdir(A_DIR) != os.path.isdir(B_DIR):
        usage_and_die()

    if app.config['TESTING'] or app.config['DEBUG']:
        sys.stderr.write('Diffing:\nA: %s\nB: %s\n\n' % (A_DIR, B_DIR))

    sys.stderr.write('''Serving diffs on http://localhost:%s
Close the browser tab or hit Ctrl-C when you're done.
''' % PORT)
    if os.path.isdir(A_DIR):
        DIFF = util.find_diff(A_DIR, B_DIR)
    else:
        shim_for_file_diff(A_DIR, B_DIR)
    logging.info('Diff: %r', DIFF)
    Timer(0.1, open_browser).start()
    app.run(host='0.0.0.0', port=PORT)

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before>from common import *
from collections import MutableMapping, MutableSequence

is_string = lambda x:isinstance(x,basestring)

__all__ = ['Node']
class Node(MutableMapping, MutableSequence):
	'''An object with a name, children and attributes, used to represent components
	of a document.

	>>> rep = lambda x:sorted(vars(x).items())
	>>> Node('x')
	Node('x')
	>>> Node('y', 1, 2, 3, a=5)
	Node('y', 1, 2, 3, _a=5)
	>>> n = Node('n', r='s')
	>>> n.append('o')
	>>> n.extend(['p', 'z'])
	>>> n[-1] = 'q'
	>>> n['r'] = 'r'
	>>> n
	Node('n', 'o', 'p', 'q', _r='r')
	>>> n.update(_s='t', r=0)
	>>> del n['r']
	>>> n.pop(1)
	'p'
	>>> n
	Node('n', 'o', 'q', _s='t')

	An underscore '_' is prepended to all attribute names to allow for attribute
	names that are also python keywords such as 'class', which is a valid concern e.g. in
	xml. To set set a literal attribute '_name', use two underscores: '__name'.
	repr() is unaffected so that eval(repr(Node(...))) evaluates to the original
	Node value.

	>>> a = Node('a', _class='123')
	>>> a
	Node('a', _class='123')
	>>> a['class'] = '234'
	>>> a
	Node('a', _class='234')
	>>> print a
	Node('a', class='234')

	If the first argument is another node, that node's values are copied. The
	other arguments are ignored.

	>>> b = Node.copy(a)
	>>> b
	Node('a', _class='234')
	>>> b.children is a.children
	False
	>>> b.attributes is a.attributes
	False
	>>> is_sequence(Node())
	False

	Use Node.walk to iterate over its entire tree.

	>>> doc = Node('a', Node('b', 'c', Node('d', e='f')))
	>>> for depth, element in doc.walk():
	...   if isinstance(element, Node):
	...     print '  '*depth, element.name, element.attributes
	...   else:
	...     print '  '*depth, `element`
	 a {}
	   b {}
	     'c'
	     d {'_e': 'f'}
	'''
	__sequence__ = False
	def __init__(self, name=None, *children, **attributes):
		self.name = name
		self.children = list(children)
		self.attributes = dict((self._attr_key(k),v) for k,v in attributes.items())

	@classmethod
	def copy(cls, node):
		return cls(node.name, *node.children, **node.attributes)

	@staticmethod
	def _attr_key(key):
		return key if key[0]=='_' else '_'+key

	def __getitem__(self, key):
		if is_string(key):
			return self.__dict__['attributes'][self._attr_key(key)]
		else:
			return self.__dict__['children'][key]

	def __setitem__(self, key, value):
		if is_string(key):
			self.__dict__['attributes'][self._attr_key(key)] = value
		else:
			self.__dict__['children'][key] = value

	def __delitem__(self, key):
		if is_string(key):
			del self.attributes[self._attr_key(key)]
		else:
			del self.children[key]

	def __len__(self):
		return len(self.children)

	def insert(self, place, value):
		self.children.insert(place, value)

	def _real_attrs(self):
		return dict((k[1:],v) if k[0]=='_' else (k,v) for k,v in self.attributes.iteritems())

	def __repr__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self.attributes.items()]))

	def __str__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self._real_attrs().items()]))

	def __nonzero__(self):
		return True

	def walk(self, filter=None, depth=0):
		yield depth, self
		for element in self.children:
			if isinstance(element, Node):
				for d,sub in element.walk(depth=depth+1):
					yield d,sub
			else:
				yield depth+1, element

__all__.append('NoChildrenMixin')
class NoChildrenMixin(object):
	'''Node mixin which ignores getting and setting children

	This mixin is useful for node types which are always leaf nodes, for example

 tags in html.'''
	children = property(lambda s:[], lambda s,v:None)

__all__.append('NoAttributesMixin')
class NoAttributesMixin(object):
	'''Node mixin which ignores getting and setting attributes.

	This mixin is useful for node types which never need attributes set, for
	example raw text nodes.'''
	attributes = property(lambda s:{}, lambda s,v:None)

__all__.append('NoNameMixin')
class NoNameMixin(Node):
	'''Node mixin which reroutes name access to children.

	This mixin is useful for node types, for which it doesn't make sense to
	have a name, for example comment nodes.'''
	def __init__(self, *children, **attributes):
		Node.__init__(self, None, *children, **attributes)
	#@property
	#def children(self):
		#return self._children
	#@children.setter
	#def children(self, value):
		#self._children = [None] + list(value)

	#@property
	#def name(self):
		#return None
	#@name.setter
	#def name(self, value):
		## Node.__init__ always sets children before name, so this shouldn't cause problems
		#self._children[0] = value

if __name__=='__main__':
	import doctest
	doctest.testmod()
<commit_msg>Added __getattr__... behavior to nodes, duck typing in 'walk'<commit_after>from common import *
from collections import MutableMapping, MutableSequence

is_string = lambda x:isinstance(x,basestring)

__all__ = ['Node']
class Node(MutableMapping, MutableSequence):
	'''An object with a name, children and attributes, used to represent components
	of a document.

	>>> rep = lambda x:sorted(vars(x).items())
	>>> Node('x')
	Node('x')
	>>> Node('y', 1, 2, 3, a=5)
	Node('y', 1, 2, 3, _a=5)
	>>> n = Node('n', r='s')
	>>> n.append('o')
	>>> n.extend(['p', 'z'])
	>>> n[-1] = 'q'
	>>> n['r'] = 'r'
	>>> n
	Node('n', 'o', 'p', 'q', _r='r')
	>>> n.update(_s='t', r=0)
	>>> del n['r']
	>>> n.pop(1)
	'p'
	>>> n
	Node('n', 'o', 'q', _s='t')

	An underscore '_' is prepended to all attribute names to allow for attribute
	names that are also python keywords such as 'class', which is a valid concern e.g. in
	xml. To set set a literal attribute '_name', use two underscores: '__name'.
	repr() is unaffected so that eval(repr(Node(...))) evaluates to the original
	Node value.

	>>> a = Node('a', _class='123')
	>>> a
	Node('a', _class='123')
	>>> a['class'] = '234'
	>>> a
	Node('a', _class='234')
	>>> print a
	Node('a', class='234')

	Set and retrieve attributes on the object that start with an underscore. If
	that attribute doesn't exist, None is returned (similar to dict.get()).

	>>> a._class = '345'
	>>> print a
	Node('a', class='345')
	>>> print a._class
	345
	>>> print a._milk
	None

	Normal attribute access isn't so lucky

	>>> a.milk
	Traceback (most recent call last):
	 ...
	AttributeError: 'Node' object has no attribute 'milk'

	If the first argument is another node, that node's values are copied. The
	other arguments are ignored.

	>>> b = Node.copy(a)
	>>> b
	Node('a', _class='345')
	>>> b.children is a.children
	False
	>>> b.attributes is a.attributes
	False
	>>> is_sequence(Node())
	False

	Use Node.walk to iterate over its entire tree.

	>>> doc = Node('a', Node('b', 'c', Node('d', e='f')))
	>>> for depth, element in doc.walk():
	...   if isinstance(element, Node):
	...     print '  '*depth, element.name, element.attributes
	...   else:
	...     print '  '*depth, `element`
	 a {}
	   b {}
	     'c'
	     d {'_e': 'f'}
	'''
	__sequence__ = False
	def __init__(self, name=None, *children, **attributes):
		self.name = name
		self.children = list(children)
		self.attributes = dict((self._attr_key(k),v) for k,v in attributes.items())

	@classmethod
	def copy(cls, node):
		return cls(node.name, *node.children, **node.attributes)

	@staticmethod
	def _attr_key(key):
		return key if key[0]=='_' else '_'+key

	def __getitem__(self, key):
		if is_string(key):
			return self.__dict__['attributes'][self._attr_key(key)]
		else:
			return self.__dict__['children'][key]

	def __getattr__(self, key):
		if key[0] == '_':
			return self.get(key)
		else:
			return super(Node, self).__getattribute__(key)

	def __setitem__(self, key, value):
		if is_string(key):
			self.__dict__['attributes'][self._attr_key(key)] = value
		else:
			self.__dict__['children'][key] = value

	def __setattr__(self, key, value):
		if key[0] == '_':
			self.__setitem__(key, value)
		else:
			super(Node, self).__setattr__(key, value)

	def __delitem__(self, key):
		if is_string(key):
			del self.attributes[self._attr_key(key)]
		else:
			del self.children[key]

	def __len__(self):
		return len(self.children)

	def insert(self, place, value):
		self.children.insert(place, value)

	def _real_attrs(self):
		return dict((k[1:],v) if k[0]=='_' else (k,v) for k,v in self.attributes.iteritems())

	def __repr__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self.attributes.items()]))

	def __str__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self._real_attrs().items()]))

	def __nonzero__(self):
		return True

	def walk(self, filter=None, depth=0):
		yield depth, self
		for element in self.children:
			if hasattr(element, 'walk'):
				for d,sub in element.walk(depth=depth+1):
					yield d,sub
			else:
				yield depth+1, element

__all__.append('NoChildrenMixin')
class NoChildrenMixin(object):
	'''Node mixin which ignores getting and setting children

	This mixin is useful for node types which are always leaf nodes, for example

 tags in html.'''
	children = property(lambda s:[], lambda s,v:None)

__all__.append('NoAttributesMixin')
class NoAttributesMixin(object):
	'''Node mixin which ignores getting and setting attributes.

	This mixin is useful for node types which never need attributes set, for
	example raw text nodes.'''
	attributes = property(lambda s:{}, lambda s,v:None)

__all__.append('NoNameMixin')
class NoNameMixin(Node):
	'''Node mixin which reroutes name access to children.

	This mixin is useful for node types, for which it doesn't make sense to
	have a name, for example comment nodes.'''
	def __init__(self, *children, **attributes):
		Node.__init__(self, None, *children, **attributes)
	#@property
	#def children(self):
		#return self._children
	#@children.setter
	#def children(self, value):
		#self._children = [None] + list(value)

	#@property
	#def name(self):
		#return None
	#@name.setter
	#def name(self, value):
		## Node.__init__ always sets children before name, so this shouldn't cause problems
		#self._children[0] = value

if __name__=='__main__':
	import doctest
	doctest.testmod()
<|endoftext|>


<commit_before>from PyQt5 import QtCore, QtNetwork, QtWebSockets

class Server():
    def __init__(self, port):
        self.server = QtWebSockets.QWebSocketServer("Socket1", QtWebSockets.QWebSocketServer.NonSecureMode)

        if self.server.listen(QtNetwork.QHostAddress.Any, port):
            print("listening on port" + str(port))
        return<commit_msg>Ajout des événements de connection<commit_after>from PyQt5 import QtCore, QtNetwork, QtWebSockets

class Server():
    def __init__(self, port):
        self.clients = []
        self.connection = None
        self.server = QtWebSockets.QWebSocketServer("Socket1", QtWebSockets.QWebSocketServer.NonSecureMode)

        if self.server.listen(QtNetwork.QHostAddress.Any, port):
            print("listening on port" + str(port))
        else:
            print('error')

        self.server.newConnection.connect(self.onNewConnection)
        return

    def onNewConnection(self):
        print("new user")
        self.connection = self.server.nextPendingConnection()
        self.connection.textMessageReceived.connect(self.processTextMessage)
        self.connection.disconnected.connect(self.socketDisconnected)

        self.clients.append(self.connection)

    def processTextMessage(self,  message):
        if (self.connection):
            self.connection.sendTextMessage(message)

    def processBinaryMessage(self,  message):
        if (self.connection):
            self.connection.sendBinaryMessage(message)

    def socketDisconnected(self):
        if (self.connection):
            self.clients.remove(self.connection)
            self.connection.deleteLater()<|endoftext|>


<commit_before>import string
from chatterbot import languages
from chatterbot.utils import treebank_to_wordnet
from nltk import pos_tag
from nltk.data import load as load_data
from nltk.corpus import wordnet, stopwords

class PosHypernymTagger(object):
    """
    For each non-stopword in a string, return a string where each word is a
    hypernym preceded by the part of speech of the word before it.
    """

    def __init__(self, language=None):
        self.language = language or languages.ENG

        self.sentence_detector = None

        self.stopwords = None

    def initialize_nltk_stopwords(self):
        """
        Download required NLTK stopwords corpus if it has not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('stopwords')

    def initialize_nltk_wordnet(self):
        """
        Download required NLTK corpora if they have not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('corpora/wordnet')

    def initialize_nltk_punkt(self):
        """
        Download required NLTK punkt corpus if it has not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('punkt')

    def initialize_nltk_averaged_perceptron_tagger(self):
        """
        Download the NLTK averaged perceptron tagger that is required for this algorithm
        to run only if the corpora has not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('averaged_perceptron_tagger')

    def get_stopwords(self):
        """
        Get the list of stopwords from the NLTK corpus.
        """
        if self.stopwords is None:
            self.stopwords = stopwords.words(self.language.ENGLISH_NAME.lower())

        return self.stopwords

    def get_sentence_detector(self):
        """
        Get the initialized sentence detector.
        """
        if self.sentence_detector is None:
            self.sentence_detector = load_data('tokenizers/punkt/english.pickle')

        return self.sentence_detector

    def get_hypernyms(self, pos_tags):
        """
        Return the hypernyms for each word in a list of POS tagged words.
        """
        results = []

        for word, pos in pos_tags:
            synsets = wordnet.synsets(word, treebank_to_wordnet(pos))

            if synsets:
                synset = synsets[0]
                hypernyms = synset.hypernyms()

                if hypernyms:
                    results.append(hypernyms[0].name().split('.')[0])
                else:
                    results.append(word)
            else:
                results.append(word)

        return results

    def get_bigram_pair_string(self, text):
        """
        For example:
        What a beautiful swamp

        becomes:

        DT:beautiful JJ:wetland
        """
        WORD_INDEX = 0
        POS_INDEX = 1

        pos_tags = []

        sentence_detector = self.get_sentence_detector()

        for sentence in sentence_detector.tokenize(text.strip()):

            # Remove punctuation
            if sentence and sentence[-1] in string.punctuation:
                sentence_with_punctuation_removed = sentence[:-1]

                if sentence_with_punctuation_removed:
                    sentence = sentence_with_punctuation_removed

            words = sentence.split()

            pos_tags.extend(pos_tag(words))

        hypernyms = self.get_hypernyms(pos_tags)

        high_quality_bigrams = []
        all_bigrams = []

        word_count = len(pos_tags)

        if word_count == 1:
            all_bigrams.append(
                pos_tags[0][WORD_INDEX].lower()
            )

        for index in range(1, word_count):
            word = pos_tags[index][WORD_INDEX].lower()
            previous_word_pos = pos_tags[index - 1][POS_INDEX]
            if word not in self.get_stopwords() and len(word) > 1:
                bigram = previous_word_pos + ':' + hypernyms[index].lower()
                high_quality_bigrams.append(bigram)
                all_bigrams.append(bigram)
            else:
                bigram = previous_word_pos + ':' + word
                all_bigrams.append(bigram)

        if high_quality_bigrams:
            all_bigrams = high_quality_bigrams

        return ' '.join(all_bigrams)
<commit_msg>Use language parameter to get sentence tokenizer<commit_after>import string
from chatterbot import languages
from chatterbot.utils import treebank_to_wordnet
from nltk import pos_tag
from nltk.data import load as load_data
from nltk.corpus import wordnet, stopwords

class PosHypernymTagger(object):
    """
    For each non-stopword in a string, return a string where each word is a
    hypernym preceded by the part of speech of the word before it.
    """

    def __init__(self, language=None):
        self.language = language or languages.ENG

        self.sentence_tokenizer = None

        self.stopwords = None

    def initialize_nltk_stopwords(self):
        """
        Download required NLTK stopwords corpus if it has not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('stopwords')

    def initialize_nltk_wordnet(self):
        """
        Download required NLTK corpora if they have not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('corpora/wordnet')

    def initialize_nltk_punkt(self):
        """
        Download required NLTK punkt corpus if it has not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('punkt')

    def initialize_nltk_averaged_perceptron_tagger(self):
        """
        Download the NLTK averaged perceptron tagger that is required for this algorithm
        to run only if the corpora has not already been downloaded.
        """
        from chatterbot.utils import nltk_download_corpus

        nltk_download_corpus('averaged_perceptron_tagger')

    def get_stopwords(self):
        """
        Get the list of stopwords from the NLTK corpus.
        """
        if self.stopwords is None:
            self.stopwords = stopwords.words(self.language.ENGLISH_NAME.lower())

        return self.stopwords

    def get_sentence_tokenizer(self):
        """
        Get the initialized sentence detector.
        """
        if self.sentence_tokenizer is None:
            self.sentence_tokenizer = load_data('tokenizers/punkt/{language}.pickle'.format(
                language=self.language.ENGLISH_NAME.lower()
            ))

        return self.sentence_tokenizer

    def get_hypernyms(self, pos_tags):
        """
        Return the hypernyms for each word in a list of POS tagged words.
        """
        results = []

        for word, pos in pos_tags:
            synsets = wordnet.synsets(word, treebank_to_wordnet(pos))

            if synsets:
                synset = synsets[0]
                hypernyms = synset.hypernyms()

                if hypernyms:
                    results.append(hypernyms[0].name().split('.')[0])
                else:
                    results.append(word)
            else:
                results.append(word)

        return results

    def get_bigram_pair_string(self, text):
        """
        For example:
        What a beautiful swamp

        becomes:

        DT:beautiful JJ:wetland
        """
        WORD_INDEX = 0
        POS_INDEX = 1

        pos_tags = []

        sentence_tokenizer = self.get_sentence_tokenizer()

        for sentence in sentence_tokenizer.tokenize(text.strip()):

            # Remove punctuation
            if sentence and sentence[-1] in string.punctuation:
                sentence_with_punctuation_removed = sentence[:-1]

                if sentence_with_punctuation_removed:
                    sentence = sentence_with_punctuation_removed

            words = sentence.split()

            pos_tags.extend(pos_tag(words))

        hypernyms = self.get_hypernyms(pos_tags)

        high_quality_bigrams = []
        all_bigrams = []

        word_count = len(pos_tags)

        if word_count == 1:
            all_bigrams.append(
                pos_tags[0][WORD_INDEX].lower()
            )

        for index in range(1, word_count):
            word = pos_tags[index][WORD_INDEX].lower()
            previous_word_pos = pos_tags[index - 1][POS_INDEX]
            if word not in self.get_stopwords() and len(word) > 1:
                bigram = previous_word_pos + ':' + hypernyms[index].lower()
                high_quality_bigrams.append(bigram)
                all_bigrams.append(bigram)
            else:
                bigram = previous_word_pos + ':' + word
                all_bigrams.append(bigram)

        if high_quality_bigrams:
            all_bigrams = high_quality_bigrams

        return ' '.join(all_bigrams)
<|endoftext|>


<commit_before>"""
Configuration system for CherryPy.

Configuration in CherryPy is implemented via dictionaries. Keys are strings
which name the mapped value, which may be of any type.

Architecture
------------

CherryPy Requests are part of an Application, which runs in a global context,
and configuration data may apply to any of those three scopes:

Global
    Configuration entries which apply everywhere are stored in
    cherrypy.config.

Application
    Entries which apply to each mounted application are stored
    on the Application object itself, as 'app.config'. This is a two-level
    dict where each key is a path, or "relative URL" (for example, "/" or
    "/path/to/my/page"), and each value is a config dict. Usually, this
    data is provided in the call to tree.mount(root(), config=conf),
    although you may also use app.merge(conf).

Request
    Each Request object possesses a single 'Request.config' dict.
    Early in the request process, this dict is populated by merging global
    config entries, Application entries (whose path equals or is a parent
    of Request.path_info), and any config acquired while looking up the
    page handler (see next).

Declaration
-----------

Configuration data may be supplied as a Python dictionary, as a filename,
or as an open file object. When you supply a filename or file, CherryPy
uses Python's builtin ConfigParser; you declare Application config by
writing each path as a section header::

    [/path/to/my/page]
    request.stream = True

To declare global configuration entries, place them in a [global] section.

You may also declare config entries directly on the classes and methods
(page handlers) that make up your CherryPy application via the ``_cp_config``
attribute. For example::

    class Demo:
        _cp_config = {'tools.gzip.on': True}

        @cherrypy.expose
        def index(self):
            return "Hello world"
        index._cp_config = {'request.show_tracebacks': False}

.. note::

    This behavior is only guaranteed for the default dispatcher.
    Other dispatchers may have different restrictions on where
    you can attach _cp_config attributes.

Namespaces
----------

Configuration keys are separated into namespaces by the first "." in the key.
Current namespaces:

engine
    Controls the 'application engine', including autoreload.
    These can only be declared in the global config.

tree
    Grafts cherrypy.Application objects onto cherrypy.tree.
    These can only be declared in the global config.

hooks
    Declares additional request-processing functions.

log
    Configures the logging for each application.
    These can only be declared in the global or / config.

request
    Adds attributes to each Request.

response
    Adds attributes to each Response.

server
    Controls the default HTTP server via cherrypy.server.
    These can only be declared in the global config.

tools
    Runs and configures additional request-processing packages.

wsgi
    Adds WSGI middleware to an Application's "pipeline".
    These can only be declared in the app's root config ("/").

checker
    Controls the 'checker', which looks for common errors in
    app state (including config) when the engine starts.
    Global config only.

The only key that does not exist in a namespace is the "environment" entry.
This special entry 'imports' other config entries from a template stored in
cherrypy._cpconfig.environments[environment]. It only applies to the global
config, and only when you use cherrypy.config.update.

You can define your own namespaces to be called at the Global, Application,
or Request level, by adding a named handler to cherrypy.config.namespaces,
app.namespaces, or app.request_class.namespaces. The name can
be any string, and the handler must be either a callable or a (Python 2.5
style) context manager.
"""

import cherrypy
from cherrypy._cpcompat import basestring
from cherrypy.lib import reprconf

# Deprecated in  CherryPy 3.2--remove in 3.3
NamespaceSet = reprconf.NamespaceSet

def merge(base, other):
    """Merge one app config (from a dict, file, or filename) into another.

    If the given config is a filename, it will be appended to
    the list of files to monitor for "autoreload" changes.
    """
    if isinstance(other, basestring):
        cherrypy.engine.autoreload.files.add(other)

    # Load other into base
    for section, value_map in reprconf.as_dict(other).items():
        if not isinstance(value_map, dict):
            raise ValueError(
                "Application config must include section headers, but the "
                "config you tried to merge doesn't have any sections. "
                "Wrap your config in another dict with paths as section "
                "headers, for example: {'/': config}.")
        base.setdefault(section, {}).update(value_map)

class Config(reprconf.Config):

    """The 'global' configuration data for the entire CherryPy process."""

    def update(self, config):
        """Update self from a dict, file or filename."""
        if isinstance(config, basestring):
            # Filename
            cherrypy.engine.autoreload.files.add(config)
        reprconf.Config.update(self, config)

    def _apply(self, config):
        """Update self from a dict."""
        if isinstance(config.get("global"), dict):
            if len(config) > 1:
                cherrypy.checker.global_config_contained_paths = True
            config = config["global"]
        if 'tools.staticdir.dir' in config:
            config['tools.staticdir.section'] = "global"
        reprconf.Config._apply(self, config)

    def __call__(self, *args, **kwargs):
        """Decorator for page handlers to set _cp_config."""
        if args:
            raise TypeError(
                "The cherrypy.config decorator does not accept positional "
                "arguments; you must use keyword arguments.")

        def tool_decorator(f):
            if not hasattr(f, "_cp_config"):
                f._cp_config = {}
            for k, v in kwargs.items():
                f._cp_config[k] = v
            return f
        return tool_decorator

# Sphinx begin config.environments
Config.environments = environments = {
    "staging": {
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': False,
        'request.show_mismatched_params': False,
    },
    "production": {
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': False,
        'request.show_mismatched_params': False,
        'log.screen': False,
    },
    "embedded": {
        # For use with CherryPy embedded in another deployment stack.
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': False,
        'request.show_mismatched_params': False,
        'log.screen': False,
        'engine.SIGHUP': None,
        'engine.SIGTERM': None,
    },
    "test_suite": {
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': True,
        'request.show_mismatched_params': True,
        'log.screen': False,
    },
}
# Sphinx end config.environments

def _server_namespace_handler(k, v):
    """Config handler for the "server" namespace."""
    atoms = k.split(".", 1)
    if len(atoms) > 1:
        # Special-case config keys of the form 'server.servername.socket_port'
        # to configure additional HTTP servers.
        if not hasattr(cherrypy, "servers"):
            cherrypy.servers = {}

        servername, k = atoms
        if servername not in cherrypy.servers:
            from cherrypy import _cpserver
            cherrypy.servers[servername] = _cpserver.Server()
            # On by default, but 'on = False' can unsubscribe it (see below).
            cherrypy.servers[servername].subscribe()

        if k == 'on':
            if v:
                cherrypy.servers[servername].subscribe()
            else:
                cherrypy.servers[servername].unsubscribe()
        else:
            setattr(cherrypy.servers[servername], k, v)
    else:
        setattr(cherrypy.server, k, v)
Config.namespaces["server"] = _server_namespace_handler

def _engine_namespace_handler(k, v):
    """Backward compatibility handler for the "engine" namespace."""
    engine = cherrypy.engine

    deprecated = {
        'autoreload_on': 'autoreload.on',
        'autoreload_frequency': 'autoreload.frequency',
        'autoreload_match': 'autoreload.match',
        'reload_files': 'autoreload.files',
        'deadlock_poll_freq': 'timeout_monitor.frequency'
    }

    if k in deprecated:
        engine.log(
            'WARNING: Use of engine.%s is deprecated and will be removed in a '
            'future version. Use engine.%s instead.' % (k, deprecated[k]))

    if k == 'autoreload_on':
        if v:
            engine.autoreload.subscribe()
        else:
            engine.autoreload.unsubscribe()
    elif k == 'autoreload_frequency':
        engine.autoreload.frequency = v
    elif k == 'autoreload_match':
        engine.autoreload.match = v
    elif k == 'reload_files':
        engine.autoreload.files = set(v)
    elif k == 'deadlock_poll_freq':
        engine.timeout_monitor.frequency = v
    elif k == 'SIGHUP':
        engine.listeners['SIGHUP'] = set([v])
    elif k == 'SIGTERM':
        engine.listeners['SIGTERM'] = set([v])
    elif "." in k:
        plugin, attrname = k.split(".", 1)
        plugin = getattr(engine, plugin)
        if attrname == 'on':
            if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'):
                plugin.subscribe()
                return
            elif (
                (not v) and
                hasattr(getattr(plugin, 'unsubscribe', None), '__call__')
            ):
                plugin.unsubscribe()
                return
        setattr(plugin, attrname, v)
    else:
        setattr(engine, k, v)
Config.namespaces["engine"] = _engine_namespace_handler

def _tree_namespace_handler(k, v):
    """Namespace handler for the 'tree' config namespace."""
    if isinstance(v, dict):
        for script_name, app in v.items():
            cherrypy.tree.graft(app, script_name)
            msg = "Mounted: %s on %s" % (app, script_name or "/")
            cherrypy.engine.log(msg)
    else:
        cherrypy.tree.graft(v, v.script_name)
        cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/"))
Config.namespaces["tree"] = _tree_namespace_handler
<commit_msg>Rewrite @cherrypy.config decorator using setdefault and update<commit_after>"""
Configuration system for CherryPy.

Configuration in CherryPy is implemented via dictionaries. Keys are strings
which name the mapped value, which may be of any type.

Architecture
------------

CherryPy Requests are part of an Application, which runs in a global context,
and configuration data may apply to any of those three scopes:

Global
    Configuration entries which apply everywhere are stored in
    cherrypy.config.

Application
    Entries which apply to each mounted application are stored
    on the Application object itself, as 'app.config'. This is a two-level
    dict where each key is a path, or "relative URL" (for example, "/" or
    "/path/to/my/page"), and each value is a config dict. Usually, this
    data is provided in the call to tree.mount(root(), config=conf),
    although you may also use app.merge(conf).

Request
    Each Request object possesses a single 'Request.config' dict.
    Early in the request process, this dict is populated by merging global
    config entries, Application entries (whose path equals or is a parent
    of Request.path_info), and any config acquired while looking up the
    page handler (see next).

Declaration
-----------

Configuration data may be supplied as a Python dictionary, as a filename,
or as an open file object. When you supply a filename or file, CherryPy
uses Python's builtin ConfigParser; you declare Application config by
writing each path as a section header::

    [/path/to/my/page]
    request.stream = True

To declare global configuration entries, place them in a [global] section.

You may also declare config entries directly on the classes and methods
(page handlers) that make up your CherryPy application via the ``_cp_config``
attribute. For example::

    class Demo:
        _cp_config = {'tools.gzip.on': True}

        @cherrypy.expose
        def index(self):
            return "Hello world"
        index._cp_config = {'request.show_tracebacks': False}

.. note::

    This behavior is only guaranteed for the default dispatcher.
    Other dispatchers may have different restrictions on where
    you can attach _cp_config attributes.

Namespaces
----------

Configuration keys are separated into namespaces by the first "." in the key.
Current namespaces:

engine
    Controls the 'application engine', including autoreload.
    These can only be declared in the global config.

tree
    Grafts cherrypy.Application objects onto cherrypy.tree.
    These can only be declared in the global config.

hooks
    Declares additional request-processing functions.

log
    Configures the logging for each application.
    These can only be declared in the global or / config.

request
    Adds attributes to each Request.

response
    Adds attributes to each Response.

server
    Controls the default HTTP server via cherrypy.server.
    These can only be declared in the global config.

tools
    Runs and configures additional request-processing packages.

wsgi
    Adds WSGI middleware to an Application's "pipeline".
    These can only be declared in the app's root config ("/").

checker
    Controls the 'checker', which looks for common errors in
    app state (including config) when the engine starts.
    Global config only.

The only key that does not exist in a namespace is the "environment" entry.
This special entry 'imports' other config entries from a template stored in
cherrypy._cpconfig.environments[environment]. It only applies to the global
config, and only when you use cherrypy.config.update.

You can define your own namespaces to be called at the Global, Application,
or Request level, by adding a named handler to cherrypy.config.namespaces,
app.namespaces, or app.request_class.namespaces. The name can
be any string, and the handler must be either a callable or a (Python 2.5
style) context manager.
"""

import cherrypy
from cherrypy._cpcompat import basestring
from cherrypy.lib import reprconf

# Deprecated in  CherryPy 3.2--remove in 3.3
NamespaceSet = reprconf.NamespaceSet

def merge(base, other):
    """Merge one app config (from a dict, file, or filename) into another.

    If the given config is a filename, it will be appended to
    the list of files to monitor for "autoreload" changes.
    """
    if isinstance(other, basestring):
        cherrypy.engine.autoreload.files.add(other)

    # Load other into base
    for section, value_map in reprconf.as_dict(other).items():
        if not isinstance(value_map, dict):
            raise ValueError(
                "Application config must include section headers, but the "
                "config you tried to merge doesn't have any sections. "
                "Wrap your config in another dict with paths as section "
                "headers, for example: {'/': config}.")
        base.setdefault(section, {}).update(value_map)

class Config(reprconf.Config):

    """The 'global' configuration data for the entire CherryPy process."""

    def update(self, config):
        """Update self from a dict, file or filename."""
        if isinstance(config, basestring):
            # Filename
            cherrypy.engine.autoreload.files.add(config)
        reprconf.Config.update(self, config)

    def _apply(self, config):
        """Update self from a dict."""
        if isinstance(config.get("global"), dict):
            if len(config) > 1:
                cherrypy.checker.global_config_contained_paths = True
            config = config["global"]
        if 'tools.staticdir.dir' in config:
            config['tools.staticdir.section'] = "global"
        reprconf.Config._apply(self, config)

    @staticmethod
    def __call__(*args, **kwargs):
        """Decorator for page handlers to set _cp_config."""
        if args:
            raise TypeError(
                "The cherrypy.config decorator does not accept positional "
                "arguments; you must use keyword arguments.")

        def tool_decorator(f):
            vars(f).setdefault('_cp_config', {}).update(kwargs)
            return f
        return tool_decorator

# Sphinx begin config.environments
Config.environments = environments = {
    "staging": {
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': False,
        'request.show_mismatched_params': False,
    },
    "production": {
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': False,
        'request.show_mismatched_params': False,
        'log.screen': False,
    },
    "embedded": {
        # For use with CherryPy embedded in another deployment stack.
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': False,
        'request.show_mismatched_params': False,
        'log.screen': False,
        'engine.SIGHUP': None,
        'engine.SIGTERM': None,
    },
    "test_suite": {
        'engine.autoreload.on': False,
        'checker.on': False,
        'tools.log_headers.on': False,
        'request.show_tracebacks': True,
        'request.show_mismatched_params': True,
        'log.screen': False,
    },
}
# Sphinx end config.environments

def _server_namespace_handler(k, v):
    """Config handler for the "server" namespace."""
    atoms = k.split(".", 1)
    if len(atoms) > 1:
        # Special-case config keys of the form 'server.servername.socket_port'
        # to configure additional HTTP servers.
        if not hasattr(cherrypy, "servers"):
            cherrypy.servers = {}

        servername, k = atoms
        if servername not in cherrypy.servers:
            from cherrypy import _cpserver
            cherrypy.servers[servername] = _cpserver.Server()
            # On by default, but 'on = False' can unsubscribe it (see below).
            cherrypy.servers[servername].subscribe()

        if k == 'on':
            if v:
                cherrypy.servers[servername].subscribe()
            else:
                cherrypy.servers[servername].unsubscribe()
        else:
            setattr(cherrypy.servers[servername], k, v)
    else:
        setattr(cherrypy.server, k, v)
Config.namespaces["server"] = _server_namespace_handler

def _engine_namespace_handler(k, v):
    """Backward compatibility handler for the "engine" namespace."""
    engine = cherrypy.engine

    deprecated = {
        'autoreload_on': 'autoreload.on',
        'autoreload_frequency': 'autoreload.frequency',
        'autoreload_match': 'autoreload.match',
        'reload_files': 'autoreload.files',
        'deadlock_poll_freq': 'timeout_monitor.frequency'
    }

    if k in deprecated:
        engine.log(
            'WARNING: Use of engine.%s is deprecated and will be removed in a '
            'future version. Use engine.%s instead.' % (k, deprecated[k]))

    if k == 'autoreload_on':
        if v:
            engine.autoreload.subscribe()
        else:
            engine.autoreload.unsubscribe()
    elif k == 'autoreload_frequency':
        engine.autoreload.frequency = v
    elif k == 'autoreload_match':
        engine.autoreload.match = v
    elif k == 'reload_files':
        engine.autoreload.files = set(v)
    elif k == 'deadlock_poll_freq':
        engine.timeout_monitor.frequency = v
    elif k == 'SIGHUP':
        engine.listeners['SIGHUP'] = set([v])
    elif k == 'SIGTERM':
        engine.listeners['SIGTERM'] = set([v])
    elif "." in k:
        plugin, attrname = k.split(".", 1)
        plugin = getattr(engine, plugin)
        if attrname == 'on':
            if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'):
                plugin.subscribe()
                return
            elif (
                (not v) and
                hasattr(getattr(plugin, 'unsubscribe', None), '__call__')
            ):
                plugin.unsubscribe()
                return
        setattr(plugin, attrname, v)
    else:
        setattr(engine, k, v)
Config.namespaces["engine"] = _engine_namespace_handler

def _tree_namespace_handler(k, v):
    """Namespace handler for the 'tree' config namespace."""
    if isinstance(v, dict):
        for script_name, app in v.items():
            cherrypy.tree.graft(app, script_name)
            msg = "Mounted: %s on %s" % (app, script_name or "/")
            cherrypy.engine.log(msg)
    else:
        cherrypy.tree.graft(v, v.script_name)
        cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/"))
Config.namespaces["tree"] = _tree_namespace_handler
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import struct
try:
    import usocket as socket
except ImportError:
    import socket
import websocket_helper

# Define to 1 to use builtin "websocket" module of MicroPython
USE_BUILTIN_WEBSOCKET = 0
# Treat this remote directory as a root for file transfers
SANDBOX = "."
#SANDBOX = "/tmp/webrepl/"

WEBREPL_FILE = "<3sBBBHQH64s"

if USE_BUILTIN_WEBSOCKET:
    from websocket import websocket
else:
    class websocket:

        def __init__(self, s):
            self.s = s
            self.buf = b""

        def write(self, data):
            l = len(data)
            if l < 126:
                # TODO: hardcoded "binary" type
                hdr = struct.pack(">BB", 0x82, l)
            else:
                hdr = struct.pack(">BBH", 0x82, 126, l)
            self.s.send(hdr)
            self.s.send(data)

        def recvexactly(self, sz):
            res = b""
            while sz:
                data = self.s.recv(sz)
                if not data:
                    break
                res += data
                sz -= len(data)
            return res

        def read(self, size):
            if not self.buf:
                while True:
                    hdr = self.recvexactly(2)
                    assert len(hdr) == 2
                    fl, sz = struct.unpack(">BB", hdr)
                    if sz == 126:
                        hdr = self.recvexactly(2)
                        assert len(hdr) == 2
                        (sz,) = struct.unpack(">H", hdr)
                    if fl == 0x82:
                        break
                    print("Got unexpected websocket record of type %x, skipping it" % fl)
                    while sz:
                        skip = self.s.recv(sz)
                        print("Skip data:", skip)
                        sz -= len(skip)
                data = self.recvexactly(sz)
                assert len(data) == sz
                self.buf = data

            d = self.buf[:size]
            self.buf = self.buf[size:]
            assert len(d) == size, len(d)
            return d

        def ioctl(self, req, val):
            assert req == 9 and val == 2

def read_resp(ws):
    data = ws.read(4)
    sig, code = struct.unpack("<2sH", data)
    assert sig == b"WB"
    return code

def put_file(ws, local_file, remote_file):
    sz = os.stat(local_file)[6]
    dest_fname = bytes(SANDBOX + remote_file, "utf-8")
    rec = struct.pack(WEBREPL_FILE, b"WRA", 1, 0, 0, sz, 0, len(dest_fname), dest_fname)
    print(rec, len(rec))
    ws.write(rec[:10])
    ws.write(rec[10:])
    assert read_resp(ws) == 0
    with open(local_file, "rb") as f:
        while True:
            buf = f.read(256)
            if not buf:
                break
            ws.write(buf)
    assert read_resp(ws) == 0

def get_file(ws, local_file, remote_file):
    src_fname = bytes(SANDBOX + remote_file, "utf-8")
    rec = struct.pack(WEBREPL_FILE, b"WRA", 2, 0, 0, 0, 0, len(src_fname), src_fname)
    print(rec, len(rec))
    ws.write(rec)
    assert read_resp(ws) == 0
    with open(local_file, "wb") as f:
        while True:
            (sz,) = struct.unpack("<H", ws.read(2))
            if sz == 0:
                break
            while sz:
                buf = ws.read(sz)
                if not buf:
                    raise OSError()
                f.write(buf)
                sz -= len(buf)
    assert read_resp(ws) == 0

def help(rc=0):
    exename = sys.argv[0].rsplit("/", 1)[1]
    print("%s - Perform remote file operations using MicroPython WebREPL protocol" % exename)
    print("Arguments:")
    print("  <host>:<remote_file> <local_file> - Copy remote file to local file")
    print("  <local_file> <host>:<remote_file> - Copy local file to remote file")
    print("Examples:")
    print("  %s script.py 192.168.4.1:/another_name.py" % exename)
    print("  %s script.py 192.168.4.1:/app/" % exename)
    print("  %s 192.168.4.1:/app/script.py ." % exename)
    sys.exit(rc)

def error(msg):
    print(msg)
    sys.exit(1)

def parse_remote(remote):
    host, fname = remote.rsplit(":", 1)
    port = 8266
    if ":" in host:
        host, port = remote.split(":")
        port = int(port)
    return (host, port, fname)

def main():

    if len(sys.argv) != 3:
        help(1)

    if ":" in sys.argv[1] and ":" in sys.argv[2]:
        error("Operations on 2 remote files are not supported")
    if ":" not in sys.argv[1] and ":" not in sys.argv[2]:
        error("One remote file is required")

    if ":" in sys.argv[1]:
        op = "get"
        host, port, src_file = parse_remote(sys.argv[1])
        dst_file = sys.argv[2]
        if os.path.isdir(dst_file):
            basename = src_file.rsplit("/", 1)[-1]
            dst_file += "/" + basename
    else:
        op = "put"
        host, port, dst_file = parse_remote(sys.argv[2])
        src_file = sys.argv[1]
        if dst_file[-1] == "/":
            basename = src_file.rsplit("/", 1)[-1]
            dst_file += basename

    if 1:
        print(op, host, port)
        print(src_file, "->", dst_file)

    s = socket.socket()

    ai = socket.getaddrinfo(host, port)
    addr = ai[0][4]

    s.connect(addr)
    #s = s.makefile("rwb")
    websocket_helper.client_handshake(s)

    ws = websocket(s)
    # Set websocket to send data marked as "binary"
    ws.ioctl(9, 2)

    if op == "get":
        get_file(ws, dst_file, src_file)
    elif op == "put":
        put_file(ws, src_file, dst_file)

    s.close()

if __name__ == "__main__":
    main()
<commit_msg>webrepl_cli.py: Fix rsplit usage.<commit_after>#!/usr/bin/env python
import sys
import os
import struct
try:
    import usocket as socket
except ImportError:
    import socket
import websocket_helper

# Define to 1 to use builtin "websocket" module of MicroPython
USE_BUILTIN_WEBSOCKET = 0
# Treat this remote directory as a root for file transfers
SANDBOX = "."
#SANDBOX = "/tmp/webrepl/"

WEBREPL_FILE = "<3sBBBHQH64s"

if USE_BUILTIN_WEBSOCKET:
    from websocket import websocket
else:
    class websocket:

        def __init__(self, s):
            self.s = s
            self.buf = b""

        def write(self, data):
            l = len(data)
            if l < 126:
                # TODO: hardcoded "binary" type
                hdr = struct.pack(">BB", 0x82, l)
            else:
                hdr = struct.pack(">BBH", 0x82, 126, l)
            self.s.send(hdr)
            self.s.send(data)

        def recvexactly(self, sz):
            res = b""
            while sz:
                data = self.s.recv(sz)
                if not data:
                    break
                res += data
                sz -= len(data)
            return res

        def read(self, size):
            if not self.buf:
                while True:
                    hdr = self.recvexactly(2)
                    assert len(hdr) == 2
                    fl, sz = struct.unpack(">BB", hdr)
                    if sz == 126:
                        hdr = self.recvexactly(2)
                        assert len(hdr) == 2
                        (sz,) = struct.unpack(">H", hdr)
                    if fl == 0x82:
                        break
                    print("Got unexpected websocket record of type %x, skipping it" % fl)
                    while sz:
                        skip = self.s.recv(sz)
                        print("Skip data:", skip)
                        sz -= len(skip)
                data = self.recvexactly(sz)
                assert len(data) == sz
                self.buf = data

            d = self.buf[:size]
            self.buf = self.buf[size:]
            assert len(d) == size, len(d)
            return d

        def ioctl(self, req, val):
            assert req == 9 and val == 2

def read_resp(ws):
    data = ws.read(4)
    sig, code = struct.unpack("<2sH", data)
    assert sig == b"WB"
    return code

def put_file(ws, local_file, remote_file):
    sz = os.stat(local_file)[6]
    dest_fname = bytes(SANDBOX + remote_file, "utf-8")
    rec = struct.pack(WEBREPL_FILE, b"WRA", 1, 0, 0, sz, 0, len(dest_fname), dest_fname)
    print(rec, len(rec))
    ws.write(rec[:10])
    ws.write(rec[10:])
    assert read_resp(ws) == 0
    with open(local_file, "rb") as f:
        while True:
            buf = f.read(256)
            if not buf:
                break
            ws.write(buf)
    assert read_resp(ws) == 0

def get_file(ws, local_file, remote_file):
    src_fname = bytes(SANDBOX + remote_file, "utf-8")
    rec = struct.pack(WEBREPL_FILE, b"WRA", 2, 0, 0, 0, 0, len(src_fname), src_fname)
    print(rec, len(rec))
    ws.write(rec)
    assert read_resp(ws) == 0
    with open(local_file, "wb") as f:
        while True:
            (sz,) = struct.unpack("<H", ws.read(2))
            if sz == 0:
                break
            while sz:
                buf = ws.read(sz)
                if not buf:
                    raise OSError()
                f.write(buf)
                sz -= len(buf)
    assert read_resp(ws) == 0

def help(rc=0):
    exename = sys.argv[0].rsplit("/", 1)[-1]
    print("%s - Perform remote file operations using MicroPython WebREPL protocol" % exename)
    print("Arguments:")
    print("  <host>:<remote_file> <local_file> - Copy remote file to local file")
    print("  <local_file> <host>:<remote_file> - Copy local file to remote file")
    print("Examples:")
    print("  %s script.py 192.168.4.1:/another_name.py" % exename)
    print("  %s script.py 192.168.4.1:/app/" % exename)
    print("  %s 192.168.4.1:/app/script.py ." % exename)
    sys.exit(rc)

def error(msg):
    print(msg)
    sys.exit(1)

def parse_remote(remote):
    host, fname = remote.rsplit(":", 1)
    port = 8266
    if ":" in host:
        host, port = remote.split(":")
        port = int(port)
    return (host, port, fname)

def main():

    if len(sys.argv) != 3:
        help(1)

    if ":" in sys.argv[1] and ":" in sys.argv[2]:
        error("Operations on 2 remote files are not supported")
    if ":" not in sys.argv[1] and ":" not in sys.argv[2]:
        error("One remote file is required")

    if ":" in sys.argv[1]:
        op = "get"
        host, port, src_file = parse_remote(sys.argv[1])
        dst_file = sys.argv[2]
        if os.path.isdir(dst_file):
            basename = src_file.rsplit("/", 1)[-1]
            dst_file += "/" + basename
    else:
        op = "put"
        host, port, dst_file = parse_remote(sys.argv[2])
        src_file = sys.argv[1]
        if dst_file[-1] == "/":
            basename = src_file.rsplit("/", 1)[-1]
            dst_file += basename

    if 1:
        print(op, host, port)
        print(src_file, "->", dst_file)

    s = socket.socket()

    ai = socket.getaddrinfo(host, port)
    addr = ai[0][4]

    s.connect(addr)
    #s = s.makefile("rwb")
    websocket_helper.client_handshake(s)

    ws = websocket(s)
    # Set websocket to send data marked as "binary"
    ws.ioctl(9, 2)

    if op == "get":
        get_file(ws, dst_file, src_file)
    elif op == "put":
        put_file(ws, src_file, dst_file)

    s.close()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division
from collections import Counter
from datetime import datetime, timedelta
from time import sleep
from os import environ

from flask import Blueprint, Response, render_template, request, render_template_string
from flask_mail import Mail
from flask_user import login_required, UserManager, SQLAlchemyAdapter
from redis import StrictRedis
import requests
import ujson as json

from website import app, db
from models import User
from hortiradar import tokenizeRawTweetText, Tweety, TOKEN

bp = Blueprint("horti", __name__, template_folder="templates", static_folder="static")

mail = Mail(app)                # initialize flask-mail

# Setup Flask-User
db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model
user_manager = UserManager(db_adapter, app)     # Initialize Flask-User

if environ.get("VERSION") == "2":
    tweety = Tweety("http://127.0.0.1:8888", TOKEN)
    redis_namespace = "2:"
else:
    tweety = Tweety("http://bigtu.q-ray.nl", TOKEN)
    redis_namespace = ""

r = StrictRedis()

CACHE_TIME = 60 * 60

def get_cache_key(func, *args, **kwargs):
    k = (
        func.__name__,
        str(args),
        str(sorted(kwargs.items(), key=lambda x: x[0]))
    )
    return redis_namespace + json.dumps(":".join(k))

# tweety methods return json string
# internal app functions return python dicts/lists
def cache(func, *args, **kwargs):
    force_refresh = kwargs.pop("force_refresh", None) or False
    cache_time = kwargs.pop("cache_time", None) or CACHE_TIME
    key = get_cache_key(func, *args, **kwargs)
    v = r.get(key)
    if v == "loading" and not force_refresh:
        sleep(0.7)
        kwargs["force_refresh"] = force_refresh
        kwargs["cache_time"] = cache_time
        return cache(func, *args, **kwargs)
    elif v is not None and not force_refresh:
        return json.loads(v) if type(v) == str else v
    else:
        if not force_refresh:
            r.set(key, "loading", ex=60)
        response = func(*args, force_refresh=force_refresh, cache_time=cache_time, **kwargs)
        v = json.dumps(response) if type(response) != str else response
        r.set(key, v, ex=cache_time)
        return response if type(response) != str else json.loads(response)

def jsonify(**kwargs):
    return Response(json.dumps(kwargs), status=200, mimetype="application/json")

def round_time(dt):
    return dt + timedelta(minutes=-dt.minute, seconds=-dt.second, microseconds=-dt.microsecond)

@bp.route("/")
def home():
    sync_time = r.get(redis_namespace + "sync_time")
    return render_template("home.html", title=make_title("BigTU research project"), sync_time=sync_time)

@bp.route("/widget/<group>")
def top_widget(group):
    """A small widget showing the top 5 in the group."""
    max_amount = request.args.get("k", 10, type=int)  # this is 10, so we re-use the cached data from the top 10
    data = cache(process_top, group, max_amount)[:5]
    data = [d["label"] for d in data]
    return render_template("widget.html", data=data)

@bp.route("/_add_top_k/<group>")
def show_top(group):
    """Visualize a top k result file"""
    max_amount = request.args.get('k', 10, type=int)
    data = cache(process_top, group, max_amount)
    return jsonify(result=data)

@bp.route("/details")
def details():
    return render_template("details.html")

@bp.route("/_get_details")
def show_details():
    """
    Visualize the details of a top k product
    product:    Product for which the details page should be constructed
    interval:   Interval in seconds for which tweets should be extracted through API
    """
    prod = request.args.get("product", u"", type=unicode)
    interval = request.args.get("interval", 60 * 60 * 24 * 7, type=int)
    end = request.args.get("end", None, type=unicode)
    if end:
        end = datetime.strptime(end, "%Y-%m-%d %H:%M") + timedelta(hours=1)
    else:
        end = round_time(datetime.utcnow())
    start = end + timedelta(seconds=-interval)
    params = {"start": start.strftime(API_time_format), "end": end.strftime(API_time_format)}
    details = cache(process_details, prod, params)
    return jsonify(result=details)

@bp.route("/members")
@login_required
def members_page():
    return render_template_string("""
    {% extends "base.html" %}
    {% block content %}

Members page

This page can only be accessed by authenticated users.

Home page (anyone)

Members page (login required)

    {% endblock %}
    """)

def process_top(group, max_amount, force_refresh=False, cache_time=CACHE_TIME):
    end = round_time(datetime.utcnow())
    start = end + timedelta(days=-1)
    params = {
        "start": start.strftime(API_time_format), "end": end.strftime(API_time_format),
        "group": group
    }
    counts = cache(tweety.get_keywords, force_refresh=force_refresh, cache_time=cache_time, **params)

    total = sum([entry["count"] for entry in counts])

    # we tag these, but filter them out of the top 10
    BLACKLIST = [u'veiling', u'naaldwijk', u'fhgt', u'fhtf', u'community', u'fhalv', u'varen corso westland', u'fhglazentulp', u'fhgt2014', u'fhgt2015', u'aalsmeer', u'floraholland 2020 strategie', u'floraholland community', u'kom in de kas', u'westland', u'fh2020', u'klok', u'bloemistenklok', u'morgenvoordeklok', u'fhstf', u'floraholland magazine', u'floraholland', u'aanvoertijden', u'fhmagazine', u'floranext']
    topkArray = []
    for entry in counts:
        if len(topkArray) < max_amount:
            if entry["keyword"] not in BLACKLIST:
                topkArray.append({"label": entry["keyword"], "y": entry["count"] / total})
        else:
            break

    return topkArray

def process_details(prod, params, force_refresh=False, cache_time=CACHE_TIME):
    tweets = cache(tweety.get_keyword, prod, force_refresh=force_refresh, cache_time=CACHE_TIME, **params)

    tweetList = []
    imagesList = []
    URLList = []
    wordCloudDict = Counter()
    tsDict = Counter()
    mapLocations = []

    for tweet in tweets:
        tweetList.append(tweet["id_str"])

        tokens = tokenizeRawTweetText(tweet["text"].lower())
        wordCloudDict.update(tokens)

        dt = datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S +0000 %Y")
        tsDict.update([(dt.year, dt.month, dt.day, dt.hour)])

        try:
            for obj in tweet["entities"]["media"]:
                imagesList.append(obj["media_url"])
        except KeyError:
            pass

        try:
            for obj in tweet["entities"]["urls"]:
                # using "expand" here synchronously will slow everything down tremendously
                url = obj["expanded_url"]
                if url is not None:
                    URLList.append(url)
        except KeyError:
            pass

        try:
            if tweet["coordinates"] is not None:
                if tweet["coordinates"]["type"] == "Point":
                    coords = tweet["coordinates"]["coordinates"]
                    mapLocations.append({"lng": coords[0], "lat": coords[1]})
        except KeyError:
            pass

    wordCloud = []
    for token in wordCloudDict:
        if token not in stop_words and "http" not in token and len(token) > 1:
            wordCloud.append({"text": token, "weight": wordCloudDict[token]})

    ts = []
    tsStart = sorted(tsDict)[0]
    tsEnd = sorted(tsDict)[-1]
    temp = datetime(tsStart[0], tsStart[1], tsStart[2], tsStart[3], 0, 0)
    while temp < datetime(tsEnd[0], tsEnd[1], tsEnd[2], tsEnd[3], 0, 0):
        if (temp.year, temp.month, temp.day, temp.hour) in tsDict:
            ts.append({"year": temp.year, "month": temp.month, "day": temp.day, "hour": temp.hour, "value": tsDict[(temp.year, temp.month, temp.day, temp.hour)]})
        else:
            ts.append({"year": temp.year, "month": temp.month, "day": temp.day, "hour": temp.hour, "value": 0})

        temp += timedelta(hours=1)

    lng = 0
    lat = 0
    if mapLocations:
        for loc in mapLocations:
            lng += loc["lng"]
            lat += loc["lat"]
            avLoc = {"lng": lng / len(mapLocations), "lat": lat / len(mapLocations)}
    else:
        avLoc = {"lng": 5, "lat": 52}

    images = []
    for (url, count) in Counter(imagesList).most_common():
        images.append({"link": url, "occ": count})

    urls = []
    for (url, count) in Counter(URLList).most_common():
        urls.append({"link": url, "occ": count})

    data = {
        "tweets": tweetList[::-1],
        "timeSeries": ts,
        "URLs": urls,
        "photos": images,
        "tagCloud": wordCloud,
        "locations": mapLocations,
        "centerloc": avLoc
    }
    return data

def expand(url):
    """Expands URLs from URL shorteners."""
    try:
        r = requests.head(url)
        while r.is_redirect and r.headers.get("location") is not None:
            url = r.headers["location"]
            r = requests.head(url)
        return r.url
    except:
        return url

def make_title(page):
    return u"Hortiradar — " + page

with open("../database/data/stoplist-nl.txt", "rb") as f:
    stop_words = [w.decode("utf-8").strip() for w in f]
    stop_words = {w: 1 for w in stop_words}  # stop words to filter out in word cloud

API_time_format = "%Y-%m-%d-%H-%M-%S"

app.register_blueprint(bp, url_prefix="/hortiradar")

if __name__ == "__main__":
    app.run(debug=True, port=8000)
<commit_msg>app: load twitter images over TLS<commit_after># -*- coding: utf-8 -*-
from __future__ import division
from collections import Counter
from datetime import datetime, timedelta
from time import sleep
from os import environ

from flask import Blueprint, Response, render_template, request, render_template_string
from flask_mail import Mail
from flask_user import login_required, UserManager, SQLAlchemyAdapter
from redis import StrictRedis
import requests
import ujson as json

from website import app, db
from models import User
from hortiradar import tokenizeRawTweetText, Tweety, TOKEN

bp = Blueprint("horti", __name__, template_folder="templates", static_folder="static")

mail = Mail(app)                # initialize flask-mail

# Setup Flask-User
db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model
user_manager = UserManager(db_adapter, app)     # Initialize Flask-User

if environ.get("VERSION") == "2":
    tweety = Tweety("http://127.0.0.1:8888", TOKEN)
    redis_namespace = "2:"
else:
    tweety = Tweety("http://bigtu.q-ray.nl", TOKEN)
    redis_namespace = ""

r = StrictRedis()

CACHE_TIME = 60 * 60

def get_cache_key(func, *args, **kwargs):
    k = (
        func.__name__,
        str(args),
        str(sorted(kwargs.items(), key=lambda x: x[0]))
    )
    return redis_namespace + json.dumps(":".join(k))

# tweety methods return json string
# internal app functions return python dicts/lists
def cache(func, *args, **kwargs):
    force_refresh = kwargs.pop("force_refresh", None) or False
    cache_time = kwargs.pop("cache_time", None) or CACHE_TIME
    key = get_cache_key(func, *args, **kwargs)
    v = r.get(key)
    if v == "loading" and not force_refresh:
        sleep(0.7)
        kwargs["force_refresh"] = force_refresh
        kwargs["cache_time"] = cache_time
        return cache(func, *args, **kwargs)
    elif v is not None and not force_refresh:
        return json.loads(v) if type(v) == str else v
    else:
        if not force_refresh:
            r.set(key, "loading", ex=60)
        response = func(*args, force_refresh=force_refresh, cache_time=cache_time, **kwargs)
        v = json.dumps(response) if type(response) != str else response
        r.set(key, v, ex=cache_time)
        return response if type(response) != str else json.loads(response)

def jsonify(**kwargs):
    return Response(json.dumps(kwargs), status=200, mimetype="application/json")

def round_time(dt):
    return dt + timedelta(minutes=-dt.minute, seconds=-dt.second, microseconds=-dt.microsecond)

@bp.route("/")
def home():
    sync_time = r.get(redis_namespace + "sync_time")
    return render_template("home.html", title=make_title("BigTU research project"), sync_time=sync_time)

@bp.route("/widget/<group>")
def top_widget(group):
    """A small widget showing the top 5 in the group."""
    max_amount = request.args.get("k", 10, type=int)  # this is 10, so we re-use the cached data from the top 10
    data = cache(process_top, group, max_amount)[:5]
    data = [d["label"] for d in data]
    return render_template("widget.html", data=data)

@bp.route("/_add_top_k/<group>")
def show_top(group):
    """Visualize a top k result file"""
    max_amount = request.args.get('k', 10, type=int)
    data = cache(process_top, group, max_amount)
    return jsonify(result=data)

@bp.route("/details")
def details():
    return render_template("details.html")

@bp.route("/_get_details")
def show_details():
    """
    Visualize the details of a top k product
    product:    Product for which the details page should be constructed
    interval:   Interval in seconds for which tweets should be extracted through API
    """
    prod = request.args.get("product", u"", type=unicode)
    interval = request.args.get("interval", 60 * 60 * 24 * 7, type=int)
    end = request.args.get("end", None, type=unicode)
    if end:
        end = datetime.strptime(end, "%Y-%m-%d %H:%M") + timedelta(hours=1)
    else:
        end = round_time(datetime.utcnow())
    start = end + timedelta(seconds=-interval)
    params = {"start": start.strftime(API_time_format), "end": end.strftime(API_time_format)}
    details = cache(process_details, prod, params)
    return jsonify(result=details)

@bp.route("/members")
@login_required
def members_page():
    return render_template_string("""
    {% extends "base.html" %}
    {% block content %}

Members page

This page can only be accessed by authenticated users.

Home page (anyone)

Members page (login required)

    {% endblock %}
    """)

def process_top(group, max_amount, force_refresh=False, cache_time=CACHE_TIME):
    end = round_time(datetime.utcnow())
    start = end + timedelta(days=-1)
    params = {
        "start": start.strftime(API_time_format), "end": end.strftime(API_time_format),
        "group": group
    }
    counts = cache(tweety.get_keywords, force_refresh=force_refresh, cache_time=cache_time, **params)

    total = sum([entry["count"] for entry in counts])

    # we tag these, but filter them out of the top 10
    BLACKLIST = [u'veiling', u'naaldwijk', u'fhgt', u'fhtf', u'community', u'fhalv', u'varen corso westland', u'fhglazentulp', u'fhgt2014', u'fhgt2015', u'aalsmeer', u'floraholland 2020 strategie', u'floraholland community', u'kom in de kas', u'westland', u'fh2020', u'klok', u'bloemistenklok', u'morgenvoordeklok', u'fhstf', u'floraholland magazine', u'floraholland', u'aanvoertijden', u'fhmagazine', u'floranext']
    topkArray = []
    for entry in counts:
        if len(topkArray) < max_amount:
            if entry["keyword"] not in BLACKLIST:
                topkArray.append({"label": entry["keyword"], "y": entry["count"] / total})
        else:
            break

    return topkArray

def process_details(prod, params, force_refresh=False, cache_time=CACHE_TIME):
    tweets = cache(tweety.get_keyword, prod, force_refresh=force_refresh, cache_time=CACHE_TIME, **params)

    tweetList = []
    imagesList = []
    URLList = []
    wordCloudDict = Counter()
    tsDict = Counter()
    mapLocations = []

    for tweet in tweets:
        tweetList.append(tweet["id_str"])

        tokens = tokenizeRawTweetText(tweet["text"].lower())
        wordCloudDict.update(tokens)

        dt = datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S +0000 %Y")
        tsDict.update([(dt.year, dt.month, dt.day, dt.hour)])

        try:
            for obj in tweet["entities"]["media"]:
                imagesList.append(obj["media_url_https"])
        except KeyError:
            pass

        try:
            for obj in tweet["entities"]["urls"]:
                # using "expand" here synchronously will slow everything down tremendously
                url = obj["expanded_url"]
                if url is not None:
                    URLList.append(url)
        except KeyError:
            pass

        try:
            if tweet["coordinates"] is not None:
                if tweet["coordinates"]["type"] == "Point":
                    coords = tweet["coordinates"]["coordinates"]
                    mapLocations.append({"lng": coords[0], "lat": coords[1]})
        except KeyError:
            pass

    wordCloud = []
    for token in wordCloudDict:
        if token not in stop_words and "http" not in token and len(token) > 1:
            wordCloud.append({"text": token, "weight": wordCloudDict[token]})

    ts = []
    tsStart = sorted(tsDict)[0]
    tsEnd = sorted(tsDict)[-1]
    temp = datetime(tsStart[0], tsStart[1], tsStart[2], tsStart[3], 0, 0)
    while temp < datetime(tsEnd[0], tsEnd[1], tsEnd[2], tsEnd[3], 0, 0):
        if (temp.year, temp.month, temp.day, temp.hour) in tsDict:
            ts.append({"year": temp.year, "month": temp.month, "day": temp.day, "hour": temp.hour, "value": tsDict[(temp.year, temp.month, temp.day, temp.hour)]})
        else:
            ts.append({"year": temp.year, "month": temp.month, "day": temp.day, "hour": temp.hour, "value": 0})

        temp += timedelta(hours=1)

    lng = 0
    lat = 0
    if mapLocations:
        for loc in mapLocations:
            lng += loc["lng"]
            lat += loc["lat"]
            avLoc = {"lng": lng / len(mapLocations), "lat": lat / len(mapLocations)}
    else:
        avLoc = {"lng": 5, "lat": 52}

    images = []
    for (url, count) in Counter(imagesList).most_common():
        images.append({"link": url, "occ": count})

    urls = []
    for (url, count) in Counter(URLList).most_common():
        urls.append({"link": url, "occ": count})

    data = {
        "tweets": tweetList[::-1],
        "timeSeries": ts,
        "URLs": urls,
        "photos": images,
        "tagCloud": wordCloud,
        "locations": mapLocations,
        "centerloc": avLoc
    }
    return data

def expand(url):
    """Expands URLs from URL shorteners."""
    try:
        r = requests.head(url)
        while r.is_redirect and r.headers.get("location") is not None:
            url = r.headers["location"]
            r = requests.head(url)
        return r.url
    except:
        return url

def make_title(page):
    return u"Hortiradar — " + page

with open("../database/data/stoplist-nl.txt", "rb") as f:
    stop_words = [w.decode("utf-8").strip() for w in f]
    stop_words = {w: 1 for w in stop_words}  # stop words to filter out in word cloud

API_time_format = "%Y-%m-%d-%H-%M-%S"

app.register_blueprint(bp, url_prefix="/hortiradar")

if __name__ == "__main__":
    app.run(debug=True, port=8000)
<|endoftext|>


<commit_before>from matexpr import MatrixExpr
from sympy import Basic

class Transpose(MatrixExpr):
    """Matrix Transpose

    Represents the transpose of a matrix expression.

    Use .T as shorthand

    >>> from sympy import MatrixSymbol, Transpose
    >>> A = MatrixSymbol('A', 3, 5)
    >>> B = MatrixSymbol('B', 5, 3)
    >>> Transpose(A)
    A'
    >>> A.T
    A'
    >>> Transpose(A*B)
    B'*A'
    """
    is_Transpose = True
    def __new__(cls, mat):

        if not mat.is_Matrix:
            return mat

        if hasattr(mat, '_eval_transpose'):
            return mat._eval_transpose()

        return Basic.__new__(cls, mat)

    @property
    def arg(self):
        return self.args[0]

    @property
    def shape(self):
        return self.arg.shape[::-1]

    def _entry(self, i, j):
        return self.arg._entry(j, i)

    def _eval_transpose(self):
        return self.arg

from matmul import MatMul
from matadd import MatAdd
<commit_msg>Remove unnecessary MatAdd and MatMul imports<commit_after>from matexpr import MatrixExpr
from sympy import Basic

class Transpose(MatrixExpr):
    """Matrix Transpose

    Represents the transpose of a matrix expression.

    Use .T as shorthand

    >>> from sympy import MatrixSymbol, Transpose
    >>> A = MatrixSymbol('A', 3, 5)
    >>> B = MatrixSymbol('B', 5, 3)
    >>> Transpose(A)
    A'
    >>> A.T
    A'
    >>> Transpose(A*B)
    B'*A'
    """
    is_Transpose = True
    def __new__(cls, mat):

        if not mat.is_Matrix:
            return mat

        if hasattr(mat, '_eval_transpose'):
            return mat._eval_transpose()

        return Basic.__new__(cls, mat)

    @property
    def arg(self):
        return self.args[0]

    @property
    def shape(self):
        return self.arg.shape[::-1]

    def _entry(self, i, j):
        return self.arg._entry(j, i)

    def _eval_transpose(self):
        return self.arg
<|endoftext|>


<commit_before>import cv2
import cv2.cv as cv
import numpy as np
import json

img_name = 'calibration/images/calibrationcross.jpg'
#img_name = 'calibration/images/glasses001.jpg'
#img_name = 'laserfrontcross.jpg'
img = cv2.imread(img_name,0)
window_img = cv2.resize(img, (int(img.shape[1] * (1080.0 / img.shape[0])), 1080))

centers = []
# mouse callback function
def draw_circle(event,x,y,flags,param):
	global centers
	if event == cv2.EVENT_LBUTTONDOWN:
#		cv2.circle(window_img,(x,y),20,(255,0,0),-1)
		centers.append((y,x))

# Display image, a window and bind the function to window
window_title = 'Click 4 circle centers'
cv2.namedWindow(window_title)
cv2.setMouseCallback(window_title,draw_circle)

while(len(centers) < 4):
	cv2.imshow(window_title,window_img)
	k = cv2.waitKey(20) & 0xFF
	if k == 27:
		break

window_img = cv2.medianBlur(window_img,5)
cimg = cv2.cvtColor(window_img,cv2.COLOR_GRAY2BGR)

circ_json = []
for c in range(len(centers)):
	pt = centers[c]
	# TODO: check bounds of pt
	max_radius = 100
	#   cv::Canny(gray, canny, 200,20);
	circles = cv2.HoughCircles(window_img[pt[0]-max_radius:pt[0]+max_radius, pt[1]-max_radius:pt[1]+max_radius],cv.CV_HOUGH_GRADIENT,1,20,param1=50,param2=30,minRadius=0,maxRadius=0)
	circles = np.uint16(np.around(circles))
	for i in circles[0,:]:
		if i[2] < max_radius:
			# draw the outer circle
			cv2.circle(cimg[pt[0]-max_radius:pt[0]+max_radius, pt[1]-max_radius:pt[1]+max_radius],(i[0],i[1]),i[2],(0,255,0),2)
			# draw the center of the circle
			cv2.circle(cimg[pt[0]-max_radius:pt[0]+max_radius, pt[1]-max_radius:pt[1]+max_radius],(i[0],i[1]),2,(0,0,255),3)
			print "adding to json:", i[0] + pt[0], i[1] + pt[1]
			circ_json.append({'x':float(i[0] + pt[0]), 'y':float(i[1] + pt[1]), 'radius':float(i[2])})
			# TODO: rescale from window image coords to original
			# draw the outer circle
			#pt_rescaled = [int(pt[0] / float(cimg.shape[0]) * float(img.shape[0])), int(pt[1] / float(cimg.shape[1]) * float(img.shape[1]))]
			#max_radius_rescaled = [int(max_radius / float(cimg.shape[0] * img.shape[0])), int(max_radius / float(cimg.shape[1] * img.shape[1]))]
			#i_rescaled = [int(i[0] / float(cimg.shape[0]) * float(img.shape[0])), int(i[1] / float(cimg.shape[1]) * float(img.shape[1]))]
			#print i_rescaled
			#cv2.circle(img[pt_rescaled[0]-max_radius_rescaled[0]:pt_rescaled[0]+max_radius_rescaled[0], pt_rescaled[1]-max_radius_rescaled[1]:pt_rescaled[1]+max_radius_rescaled[1]],(i_rescaled[0],i_rescaled[1]),i[2],(0,255,0),2)
			# draw the center of the circle
			#cv2.circle(img[pt_rescaled[0]-max_radius_rescaled[0]:pt_rescaled[0]+max_radius_rescaled[0], pt_rescaled[1]-max_radius_rescaled[1]:pt_rescaled[1]+max_radius_rescaled[1]],(i_rescaled[0],i_rescaled[1]),2,(0,0,255),3)

json.dump(circ_json, open("circles.json", 'w'))
cv2.imwrite(img_name + '_circles.jpg', cimg)
cv2.destroyAllWindows()
<commit_msg>Added width scaling by cli for circle_point_input.<commit_after>import cv2
import cv2.cv as cv
import numpy as np
import json
import getopt
import sys

centers = []
# mouse callback function
def draw_circle(event,x,y,flags,param):
	global centers
	if event == cv2.EVENT_LBUTTONDOWN:
#		cv2.circle(window_img,(x,y),20,(255,0,0),-1)
		centers.append((y,x))

if __name__ == "__main__":
	# Parse input width and image name
	width = 0.0
	opts, args = getopt.getopt(sys.argv[1:], 'w:i:h', ['width', 'image'])
	for o, a in opts:
		if o in ('-w'):
			width = float(a)
		elif o in ('-i'):
			img_name = a
			print img_name
		else:
			#img_name = 'calibration/images/glasses001.jpg'
			#img_name = 'laserfrontcross.jpg'
			print "usage: python circle_point_input.py -w .375 -i 'calibration/images/calibrationcross.jpg'"

	print img_name
	img = cv2.imread(img_name,0)
	window_img = cv2.resize(img, (int(img.shape[1] * (1080.0 / img.shape[0])), 1080))

	# Display image, a window and bind the function to window
	window_title = 'Click 4 circle centers'
	cv2.namedWindow(window_title)
	cv2.setMouseCallback(window_title,draw_circle)

	while(len(centers) < 4):
		cv2.imshow(window_title,window_img)
		k = cv2.waitKey(20) & 0xFF
		if k == 27:
			break

	window_img = cv2.medianBlur(window_img,5)
	cimg = cv2.cvtColor(window_img,cv2.COLOR_GRAY2BGR)
	circ_json = []
	avg_radius = 0.0
	n = 0.0

	# Detect circle per clicked circle center:
	for c in range(len(centers)):
		pt = centers[c]
		# TODO: check bounds of pt
		max_radius = 100
		circles = cv2.HoughCircles(window_img[pt[0]-max_radius:pt[0]+max_radius, pt[1]-max_radius:pt[1]+max_radius],cv.CV_HOUGH_GRADIENT,1,20,param1=50,param2=30,minRadius=0,maxRadius=0)
		circles = np.uint16(np.around(circles))
		for i in circles[0,:]:
			if i[2] < max_radius:
				# draw the outer circle
				cv2.circle(cimg[pt[0]-max_radius:pt[0]+max_radius, pt[1]-max_radius:pt[1]+max_radius],(i[0],i[1]),i[2],(0,255,0),2)
				# draw the center of the circle
				cv2.circle(cimg[pt[0]-max_radius:pt[0]+max_radius, pt[1]-max_radius:pt[1]+max_radius],(i[0],i[1]),2,(0,0,255),3)
				print "adding to json:", i[0] + pt[0], i[1] + pt[1]
				circ_json.append({'x':float(i[0] + pt[0]), 'y':float(i[1] + pt[1]), 'radius':float(i[2])})
				avg_radius += float(i[2])
				n += 1

	# Now scale from pixels to width passed in:
	avg_radius /= n
	scale = width / (2 * avg_radius)
	for i in circ_json:
		i['x'] *= scale
		i['y'] *= scale
		i['radius'] *= scale

	json.dump(circ_json, open("circles.json", 'w'))
	cv2.imwrite(img_name + '_circles.jpg', cimg)
	cv2.destroyAllWindows()
<|endoftext|>