<commit_before># Copyright 2014 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import os
import re
import json
import sys

import webapp2
from google.appengine.ext.webapp import template
from google.appengine.api import images
from google.appengine.api import urlfetch

sys.path.append(os.path.join(os.path.dirname(__file__),'../lib'))
from bs4 import BeautifulSoup
import cloudstorage as gcs

from handlers.common import *
from models import FeaturedArtwork

class ServiceListHandler(BaseHandler):
  def get(self):
    self.response.headers['Content-Type'] = 'application/json'
    self.response.out.write(self.render())

  def render(self):
    queue = (FeaturedArtwork.all()
        .filter('publish_date >=', datetime.date.today() - datetime.timedelta(days=30))
        .order('publish_date')
        .fetch(1000))
    return json.dumps([dict(
        id=a.key().id(),
        title=a.title,
        byline=a.byline,
        imageUri=a.image_url,
        thumbUri=a.thumb_url,
        detailsUri=a.details_url,
        publishDate=date_to_timestamp(a.publish_date),)
        for a in queue])

def maybe_process_image(image_url, base_name):
  if CLOUD_STORAGE_ROOT_URL in image_url:
    return (image_url, None)

  image_result = urlfetch.fetch(image_url)
  if image_result.status_code < 200 or image_result.status_code >= 300:
    raise IOError('Error downloading image: HTTP %d.' % image_result.status_code)

  base_filename = re.sub(r'[^\w]+', '-', base_name.strip().lower())

  # main image
  image_gcs_path = '/muzeifeaturedart/' + base_filename + '.jpg'
  # upload with default ACLs set on the bucket  # or use options={'x-goog-acl': 'public-read'})
  gcs_file = gcs.open(image_gcs_path, 'w', content_type='image/jpeg')
  gcs_file.write(image_result.content)
  gcs_file.close()

  # thumb
  thumb_gcs_path = '/muzeifeaturedart/' + base_filename + '_thumb.jpg'
  thumb = images.Image(image_result.content)
  thumb.resize(width=(thumb.width * 600 / thumb.height), height=600)
  thumb_contents = thumb.execute_transforms(output_encoding=images.JPEG, quality=40)
  gcs_file = gcs.open(thumb_gcs_path, 'w', content_type='image/jpeg')
  gcs_file.write(thumb_contents)
  gcs_file.close()

  return (CLOUD_STORAGE_ROOT_URL + image_gcs_path,
          CLOUD_STORAGE_ROOT_URL + thumb_gcs_path)

CLOUD_STORAGE_ROOT_URL = 'http://storage.googleapis.com'

class ServiceAddHandler(BaseHandler):
  def post(self):
    artwork_json = json.loads(self.request.get('json'))
    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        artwork_json['title'] + ' ' + artwork_json['byline'])
    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']
    new_artwork = FeaturedArtwork(
        title=artwork_json['title'],
        byline=artwork_json['byline'],
        image_url=new_image_url,
        thumb_url=new_thumb_url,
        details_url=artwork_json['detailsUri'],
        publish_date=datetime.datetime
            .utcfromtimestamp(artwork_json['publishDate'] / 1000)
            .date())
    new_artwork.save()
    self.response.set_status(200)

class ServiceAddFromWikiPaintingsHandler(BaseHandler):
  def post(self):
    wikipaintings_url = self.request.get('wikiPaintingsUrl')
    result = urlfetch.fetch(wikipaintings_url)
    if result.status_code < 200 or result.status_code >= 300:
      self.response.out.write('Error processing URL: HTTP %d. Content: %s'
          % (result.status_code, result.content))
      self.response.set_status(500)
      return

    self.process_html(wikipaintings_url, result.content)

  def process_html(self, url, html):
    soup = BeautifulSoup(html)

    details_url = re.sub(r'#.+', '', url, re.I | re.S) + '?utm_source=Muzei&utm_campaign=Muzei'
    title = soup.find(itemprop='name').get_text()
    author = soup.find(itemprop='author').get_text()
    completion_year_el = soup.find(itemprop='dateCreated')
    byline = author + ((', ' + completion_year_el.get_text()) if completion_year_el else '')
    image_url = soup.find(id='paintingImage')['href']

    if not title or not author or not image_url:
      self.response.out.write('Could not parse HTML')
      self.response.set_status(500)
      return

    image_url, thumb_url = maybe_process_image(image_url, title + ' ' + byline)

    # create the artwork entry
    new_artwork = FeaturedArtwork(
        title=title,
        byline=byline,
        image_url=image_url,
        thumb_url=thumb_url,
        details_url=details_url,
        publish_date=datetime.datetime
            .utcfromtimestamp(int(self.request.get('publishDate')) / 1000)
            .date())
    new_artwork.save()
    self.response.set_status(200)

class ServiceEditHandler(BaseHandler):
  def post(self):
    id = long(self.request.get('id'))
    artwork_json = json.loads(self.request.get('json'))
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return

    target_artwork.title = artwork_json['title']
    target_artwork.byline = artwork_json['byline']

    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        artwork_json['title'] + ' ' + artwork_json['byline'])
    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']

    target_artwork.image_url = new_image_url
    target_artwork.thumb_url = new_thumb_url
    target_artwork.details_url = artwork_json['detailsUri']
    target_artwork.save()
    self.response.set_status(200)

class ServiceMoveHandler(BaseHandler):
  def post(self):
    id = long(self.request.get('id'))
    publish_date = (datetime.datetime
        .utcfromtimestamp(long(self.request.get('publishDate')) / 1000)
        .date())
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return

    # shift other artworks over
    self.move_artwork(target_artwork, publish_date, target_artwork.key().id())
    self.response.set_status(200)

  def move_artwork(self, artwork, publish_date, initial_artwork_id):
    # cascade moves
    current_artwork_at_date = FeaturedArtwork.all().filter('publish_date =', publish_date).get()
    if current_artwork_at_date and current_artwork_at_date.key().id() != initial_artwork_id:
      self.move_artwork(current_artwork_at_date, publish_date + datetime.timedelta(hours=24),
          initial_artwork_id)
    artwork.publish_date = publish_date
    artwork.save()

class ServiceRemoveHandler(BaseHandler):
  def post(self):
    id = long(self.request.get('id'))
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return
    target_artwork.delete()
    self.response.set_status(200)

class ScheduleHandler(BaseHandler):
  def get(self):
    self.response.out.write(self.render())

  def render(self):
    return template.render(
        os.path.join(os.path.dirname(__file__), '../templates/backroom_schedule.html'),
        values_with_defaults(dict(
            title='Schedule',
            )))

app = webapp2.WSGIApplication([
    ('/backroom/s/list', ServiceListHandler),
    ('/backroom/s/add', ServiceAddHandler),
    ('/backroom/s/addfromwikipaintings', ServiceAddFromWikiPaintingsHandler),
    ('/backroom/s/edit', ServiceEditHandler),
    ('/backroom/s/remove', ServiceRemoveHandler),
    ('/backroom/s/move', ServiceMoveHandler),
    ('/backroom/schedule', ScheduleHandler),
    ],
    debug=IS_DEVELOPMENT)

def main():
  app.run()

if __name__ == '__main__':
  main()<commit_msg>limit image size on backend<commit_after># Copyright 2014 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import os
import re
import json
import sys

import webapp2
from google.appengine.ext.webapp import template
from google.appengine.api import images
from google.appengine.api import urlfetch

sys.path.append(os.path.join(os.path.dirname(__file__),'../lib'))
from bs4 import BeautifulSoup
import cloudstorage as gcs

from handlers.common import *
from models import FeaturedArtwork

class ServiceListHandler(BaseHandler):
  def get(self):
    self.response.headers['Content-Type'] = 'application/json'
    self.response.out.write(self.render())

  def render(self):
    queue = (FeaturedArtwork.all()
        .filter('publish_date >=', datetime.date.today() - datetime.timedelta(days=30))
        .order('publish_date')
        .fetch(1000))
    return json.dumps([dict(
        id=a.key().id(),
        title=a.title,
        byline=a.byline,
        imageUri=a.image_url,
        thumbUri=a.thumb_url,
        detailsUri=a.details_url,
        publishDate=date_to_timestamp(a.publish_date),)
        for a in queue])

def maybe_process_image(image_url, base_name):
  if CLOUD_STORAGE_ROOT_URL in image_url:
    return (image_url, None)

  image_result = urlfetch.fetch(image_url)
  if image_result.status_code < 200 or image_result.status_code >= 300:
    raise IOError('Error downloading image: HTTP %d.' % image_result.status_code)

  base_filename = re.sub(r'[^\w]+', '-', base_name.strip().lower())

  # main image
  image_gcs_path = '/muzeifeaturedart/' + base_filename + '.jpg'
  # resize to max width 4000 or max height 2000
  image_contents = image_result.content
  image = images.Image(image_contents)
  if image.height > 2000:
    image.resize(width=(thumb.width * 2000 / thumb.height), height=2000)
    image_contents = image.execute_transforms(output_encoding=images.JPEG, quality=80)
  elif image.width > 4000:
    image.resize(width=4000, height=(thumb.height * 4000 / thumb.thumb.width))
    image_contents = image.execute_transforms(output_encoding=images.JPEG, quality=80)
  # upload with default ACLs set on the bucket  # or use options={'x-goog-acl': 'public-read'})
  gcs_file = gcs.open(image_gcs_path, 'w', content_type='image/jpeg')
  gcs_file.write(image_contents)
  gcs_file.close()

  # thumb
  thumb_gcs_path = '/muzeifeaturedart/' + base_filename + '_thumb.jpg'
  thumb = images.Image(image_result.content)
  thumb.resize(width=(thumb.width * 600 / thumb.height), height=600)
  thumb_contents = thumb.execute_transforms(output_encoding=images.JPEG, quality=40)
  gcs_file = gcs.open(thumb_gcs_path, 'w', content_type='image/jpeg')
  gcs_file.write(thumb_contents)
  gcs_file.close()

  return (CLOUD_STORAGE_ROOT_URL + image_gcs_path,
          CLOUD_STORAGE_ROOT_URL + thumb_gcs_path)

CLOUD_STORAGE_ROOT_URL = 'http://storage.googleapis.com'

class ServiceAddHandler(BaseHandler):
  def post(self):
    artwork_json = json.loads(self.request.get('json'))
    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        artwork_json['title'] + ' ' + artwork_json['byline'])
    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']
    new_artwork = FeaturedArtwork(
        title=artwork_json['title'],
        byline=artwork_json['byline'],
        image_url=new_image_url,
        thumb_url=new_thumb_url,
        details_url=artwork_json['detailsUri'],
        publish_date=datetime.datetime
            .utcfromtimestamp(artwork_json['publishDate'] / 1000)
            .date())
    new_artwork.save()
    self.response.set_status(200)

class ServiceAddFromWikiPaintingsHandler(BaseHandler):
  def post(self):
    wikipaintings_url = self.request.get('wikiPaintingsUrl')
    result = urlfetch.fetch(wikipaintings_url)
    if result.status_code < 200 or result.status_code >= 300:
      self.response.out.write('Error processing URL: HTTP %d. Content: %s'
          % (result.status_code, result.content))
      self.response.set_status(500)
      return

    self.process_html(wikipaintings_url, result.content)

  def process_html(self, url, html):
    soup = BeautifulSoup(html)

    details_url = re.sub(r'#.+', '', url, re.I | re.S) + '?utm_source=Muzei&utm_campaign=Muzei'
    title = soup.find(itemprop='name').get_text()
    author = soup.find(itemprop='author').get_text()
    completion_year_el = soup.find(itemprop='dateCreated')
    byline = author + ((', ' + completion_year_el.get_text()) if completion_year_el else '')
    image_url = soup.find(id='paintingImage')['href']

    if not title or not author or not image_url:
      self.response.out.write('Could not parse HTML')
      self.response.set_status(500)
      return

    image_url, thumb_url = maybe_process_image(image_url, title + ' ' + byline)

    # create the artwork entry
    new_artwork = FeaturedArtwork(
        title=title,
        byline=byline,
        image_url=image_url,
        thumb_url=thumb_url,
        details_url=details_url,
        publish_date=datetime.datetime
            .utcfromtimestamp(int(self.request.get('publishDate')) / 1000)
            .date())
    new_artwork.save()
    self.response.set_status(200)

class ServiceEditHandler(BaseHandler):
  def post(self):
    id = long(self.request.get('id'))
    artwork_json = json.loads(self.request.get('json'))
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return

    target_artwork.title = artwork_json['title']
    target_artwork.byline = artwork_json['byline']

    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        artwork_json['title'] + ' ' + artwork_json['byline'])
    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']

    target_artwork.image_url = new_image_url
    target_artwork.thumb_url = new_thumb_url
    target_artwork.details_url = artwork_json['detailsUri']
    target_artwork.save()
    self.response.set_status(200)

class ServiceMoveHandler(BaseHandler):
  def post(self):
    id = long(self.request.get('id'))
    publish_date = (datetime.datetime
        .utcfromtimestamp(long(self.request.get('publishDate')) / 1000)
        .date())
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return

    # shift other artworks over
    self.move_artwork(target_artwork, publish_date, target_artwork.key().id())
    self.response.set_status(200)

  def move_artwork(self, artwork, publish_date, initial_artwork_id):
    # cascade moves
    current_artwork_at_date = FeaturedArtwork.all().filter('publish_date =', publish_date).get()
    if current_artwork_at_date and current_artwork_at_date.key().id() != initial_artwork_id:
      self.move_artwork(current_artwork_at_date, publish_date + datetime.timedelta(hours=24),
          initial_artwork_id)
    artwork.publish_date = publish_date
    artwork.save()

class ServiceRemoveHandler(BaseHandler):
  def post(self):
    id = long(self.request.get('id'))
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return
    target_artwork.delete()
    self.response.set_status(200)

class ScheduleHandler(BaseHandler):
  def get(self):
    self.response.out.write(self.render())

  def render(self):
    return template.render(
        os.path.join(os.path.dirname(__file__), '../templates/backroom_schedule.html'),
        values_with_defaults(dict(
            title='Schedule',
            )))

app = webapp2.WSGIApplication([
    ('/backroom/s/list', ServiceListHandler),
    ('/backroom/s/add', ServiceAddHandler),
    ('/backroom/s/addfromwikipaintings', ServiceAddFromWikiPaintingsHandler),
    ('/backroom/s/edit', ServiceEditHandler),
    ('/backroom/s/remove', ServiceRemoveHandler),
    ('/backroom/s/move', ServiceMoveHandler),
    ('/backroom/schedule', ScheduleHandler),
    ],
    debug=IS_DEVELOPMENT)

def main():
  app.run()

if __name__ == '__main__':
  main()<|endoftext|>


<commit_before>#!/usr/bin/env python3
import select
import socket
import threading
from collections import namedtuple

Server = namedtuple('Server', ['host', 'port'])

# ServerThread(server, control_socket)
# Creates a new server main loop thread
class ServerThread(threading.Thread):
	def __init__(self, server, control_socket):
		self.server = server
		self.control_socket = control_socket

		threading.Thread.__init__(self)

	def send_line_raw(self, line):
		# Sanitize line just in case
		line = line.replace(b'\r', b'').replace(b'\n', b'')[:510]

		self.server_socket.sendall(line + b'\r\n')

		# FIXME: print is not thread safe
		print('>' + line.decode(encoding = 'utf-8', errors = 'replace'))

	def handle_line(self, line):
		# TODO: implement line handling
		# FIXME: print is not thread safe
		print('<' + line.decode(encoding = 'utf-8', errors = 'replace'))

	def mainloop(self):
		# Register both the server and the control socket to our polling object
		poll = select.poll()
		poll.register(self.server_socket, select.POLLIN)
		poll.register(self.control_socket, select.POLLIN)

		# Keep buffers for input and output
		server_input_buffer = bytearray()
		server_output_buffer = bytearray()
		control_input_buffer = bytearray()

		quitting = False
		writing = False
		while not quitting:
			# Wait until we can do something
			for fd, event in poll.poll():
				# Server
				if fd == self.server_socket.fileno():
					# Ready to receive, read into buffer and handle full messages
					if event | select.POLLIN:
						data = self.server_socket.recv(1024)
						server_input_buffer.extend(data)

						# Try to see if we have a full line ending with \r\n in the buffer
						# If yes, handle it
						if b'\r\n' in server_input_buffer:
							# Newline was found, split buffer
							line, _, server_input_buffer = server_input_buffer.partition(b'\r\n')

							self.handle_line(line)

					# Ready to send, send buffered output as far as possible
					if event | select.POLLOUT:
						sent = self.server_socket.send(server_output_buffer)
						server_output_buffer = server_output_buffer[sent:]

				# Control
				elif fd == self.control_socket.fileno():
					# Read into buffer and handle full commands
					data = self.control_socket.recv(1024)
					control_input_buffer.extend(data)

					# TODO: implement command handling
					if len(control_input_buffer) > 1:
						quitting = True

				else:
					assert False #unreachable

			# See if we have to change what we're listening for
			if not writing and len(server_output_buffer) > 0:
				# Mark we're listening to socket becoming writable, and start listening
				writing = True
				poll.modify(self.server_socket, select.POLLIN | select.POLLOUT)

			elif writing and len(server_output_buffer) == 0:
				# Mark we're not listening to socket becoming writable, and stop listening
				writing = False
				poll.modify(self.server_socket, select.POLLIN)

	def run(self):
		# Connect to given server
		address = (self.server.host, self.server.port)
		try:
			self.server_socket = socket.create_connection(address)
		except ConnectionRefusedError:
			# Tell controller we failed
			self.control_socket.sendall(b'F')
			self.control_socket.close()

		# Run initialization
		# TODO: read nick/username/etc. from a config
		self.send_line_raw(b'NICK HynneFlip')
		self.send_line_raw(b'USER HynneFlip a a :HynneFlip IRC bot')

		# Run mainloop
		self.mainloop()

		# Tell the server we're quiting
		self.send_line_raw(b'QUIT :HynneFlip exiting normally')
		self.server_socket.close()

		# Tell controller we're quiting
		self.control_socket.sendall(b'Q' + b'\n')
		self.control_socket.close()

# spawn_serverthread(server) → control_socket
# Creates a ServerThread for given server and returns the socket for controlling it
def spawn_serverthread(server):
	thread_control_socket, spawner_control_socket = socket.socketpair()
	ServerThread(server, thread_control_socket).start()
	return spawner_control_socket

if __name__ == '__main__':
	control_socket = spawn_serverthread(Server('irc.freenode.net', 6667))

	while True:
		cmd = input(': ')
		if cmd == '':
			control_messages = bytearray()
			while True:
				data = control_socket.recv(1024)

				if not data:
					break

				control_messages.extend(data)

			print(control_messages.decode(encoding = 'utf-8', errors = 'replace'))

		elif cmd == 'q':
			control_socket.close()
			break

		else:
			control_socket.sendall(cmd.encode('utf-8') + b'\n')
<commit_msg>Use a lock in send_line_raw and don't thread sending of data through the mainloop<commit_after>#!/usr/bin/env python3
import select
import socket
import threading
from collections import namedtuple

Server = namedtuple('Server', ['host', 'port'])

# ServerThread(server, control_socket)
# Creates a new server main loop thread
class ServerThread(threading.Thread):
	def __init__(self, server, control_socket):
		self.server = server
		self.control_socket = control_socket

		self.server_socket_write_lock = threading.Lock()

		threading.Thread.__init__(self)

	def send_line_raw(self, line):
		# Sanitize line just in case
		line = line.replace(b'\r', b'').replace(b'\n', b'')[:510]

		with self.server_socket_write_lock:
			self.server_socket.sendall(line + b'\r\n')

		# FIXME: print is not thread safe
		print('>' + line.decode(encoding = 'utf-8', errors = 'replace'))

	def handle_line(self, line):
		# TODO: implement line handling
		# FIXME: print is not thread safe
		print('<' + line.decode(encoding = 'utf-8', errors = 'replace'))

	def mainloop(self):
		# Register both the server and the control socket to our polling object
		poll = select.poll()
		poll.register(self.server_socket, select.POLLIN)
		poll.register(self.control_socket, select.POLLIN)

		# Keep buffers for input and output
		server_input_buffer = bytearray()
		control_input_buffer = bytearray()

		quitting = False
		while not quitting:
			# Wait until we can do something
			for fd, event in poll.poll():
				# Server
				if fd == self.server_socket.fileno():
					# Ready to receive, read into buffer and handle full messages
					if event | select.POLLIN:
						data = self.server_socket.recv(1024)
						server_input_buffer.extend(data)

						# Try to see if we have a full line ending with \r\n in the buffer
						# If yes, handle it
						if b'\r\n' in server_input_buffer:
							# Newline was found, split buffer
							line, _, server_input_buffer = server_input_buffer.partition(b'\r\n')

							self.handle_line(line)

				# Control
				elif fd == self.control_socket.fileno():
					# Read into buffer and handle full commands
					data = self.control_socket.recv(1024)
					control_input_buffer.extend(data)

					# TODO: implement command handling
					if len(control_input_buffer) > 1:
						quitting = True

				else:
					assert False #unreachable

	def run(self):
		# Connect to given server
		address = (self.server.host, self.server.port)
		try:
			self.server_socket = socket.create_connection(address)
		except ConnectionRefusedError:
			# Tell controller we failed
			self.control_socket.sendall(b'F')
			self.control_socket.close()

		# Run initialization
		# TODO: read nick/username/etc. from a config
		self.send_line_raw(b'NICK HynneFlip')
		self.send_line_raw(b'USER HynneFlip a a :HynneFlip IRC bot')

		# Run mainloop
		self.mainloop()

		# Tell the server we're quiting
		self.send_line_raw(b'QUIT :HynneFlip exiting normally')
		self.server_socket.close()

		# Tell controller we're quiting
		self.control_socket.sendall(b'Q' + b'\n')
		self.control_socket.close()

# spawn_serverthread(server) → control_socket
# Creates a ServerThread for given server and returns the socket for controlling it
def spawn_serverthread(server):
	thread_control_socket, spawner_control_socket = socket.socketpair()
	ServerThread(server, thread_control_socket).start()
	return spawner_control_socket

if __name__ == '__main__':
	control_socket = spawn_serverthread(Server('irc.freenode.net', 6667))

	while True:
		cmd = input(': ')
		if cmd == '':
			control_messages = bytearray()
			while True:
				data = control_socket.recv(1024)

				if not data:
					break

				control_messages.extend(data)

			print(control_messages.decode(encoding = 'utf-8', errors = 'replace'))

		elif cmd == 'q':
			control_socket.sendall(b'Q\n')
			control_socket.close()
			break

		else:
			control_socket.sendall(cmd.encode('utf-8') + b'\n')
<|endoftext|>


<commit_before>import datetime
import json
import logging
import urllib

from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy.dialects.mysql import BIGINT
from sqlalchemy.orm import relationship

from pajbot import utils
from pajbot.managers.db import Base
from pajbot.managers.db import DBManager
from pajbot.managers.handler import HandlerManager
from pajbot.managers.redis import RedisManager

log = logging.getLogger("pajbot")

def parse_twitch_datetime(datetime_str):
    return datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc)

class Stream(Base):
    __tablename__ = "tb_stream"

    id = Column(Integer, primary_key=True)
    title = Column(String(256), nullable=False)
    stream_start = Column(DateTime, nullable=False)
    stream_end = Column(DateTime, nullable=True)
    ended = Column(Boolean, nullable=False, default=False)

    stream_chunks = relationship("StreamChunk", backref="stream", cascade="save-update, merge, expunge", lazy="joined")

    def __init__(self, created_at, **options):
        self.id = None
        self.title = options.get("title", "NO TITLE")
        self.stream_start = parse_twitch_datetime(created_at)
        self.stream_end = None
        self.ended = False

    @property
    def uptime(self):
        """
        Returns a TimeDelta for how long the stream was online, or is online.
        """

        if self.ended is False:
            return utils.now() - self.stream_start

        return self.stream_end - self.stream_start

class StreamChunk(Base):
    __tablename__ = "tb_stream_chunk"

    id = Column(Integer, primary_key=True)
    stream_id = Column(Integer, ForeignKey("tb_stream.id"), nullable=False)
    broadcast_id = Column(BIGINT(unsigned=True), nullable=False)
    video_url = Column(String(128), nullable=True)
    video_preview_image_url = Column(String(256), nullable=True)
    chunk_start = Column(DateTime, nullable=False)
    chunk_end = Column(DateTime, nullable=True)

    def __init__(self, stream, broadcast_id, created_at, **options):
        self.id = None
        self.stream_id = stream.id
        self.broadcast_id = broadcast_id
        self.video_url = None
        self.video_preview_image_url = None
        self.chunk_start = parse_twitch_datetime(created_at)
        self.chunk_end = None

        self.stream = stream

class StreamManager:
    NUM_OFFLINES_REQUIRED = 10
    STATUS_CHECK_INTERVAL = 20  # seconds
    VIDEO_URL_CHECK_INTERVAL = 60 * 5  # seconds

    def fetch_video_url_stage1(self):
        if self.online is False:
            return

        try:
            data = self.bot.twitchapi.get(
                ["channels", self.bot.streamer, "videos"],
                parameters={"broadcasts": "true"},
                base="http://127.0.0.1:7221/kraken/",
            )

            self.execute_now(lambda: self.refresh_video_url_stage2(data))
        except urllib.error.HTTPError as e:
            raw_data = e.read().decode("utf-8")
            log.exception("OMGScoots")
            log.info(raw_data)
        except:
            log.exception("Uncaught exception in fetch_video_url")

    def fetch_video_url_stage2(self, data):
        stream_chunk = self.current_stream_chunk if self.current_stream_chunk.video_url is None else None
        try:
            for video in data["videos"]:
                if video["broadcast_type"] == "archive":
                    recorded_at = parse_twitch_datetime(video["recorded_at"])
                    if stream_chunk is not None:
                        time_diff = stream_chunk.chunk_start - recorded_at
                        if abs(time_diff.total_seconds()) < 60 * 5:
                            # we found the relevant video!
                            return (video["url"], video["preview"]["large"], video["recorded_at"])
                    else:
                        if video["status"] == "recording":
                            return (video["url"], video["preview"]["large"], video["recorded_at"])
        except urllib.error.HTTPError as e:
            raw_data = e.read().decode("utf-8")
            log.exception("OMGScoots")
            log.info(raw_data)
        except:
            log.exception("Uncaught exception in fetch_video_url")

        return None, None, None

    def __init__(self, bot):
        self.bot = bot

        self.current_stream_chunk = None  # should this even exist?

        self.num_offlines = 0
        self.first_offline = None

        self.num_viewers = 0

        self.game = "Loading..."
        self.title = "Loading..."

        self.bot.execute_every(
            self.STATUS_CHECK_INTERVAL, self.bot.action_queue.add, (self.refresh_stream_status_stage1,)
        )
        self.bot.execute_every(
            self.VIDEO_URL_CHECK_INTERVAL, self.bot.action_queue.add, (self.refresh_video_url_stage1,)
        )

        # This will load the latest stream so we can post an accurate "time since last online" figure.
        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            self.current_stream = (
                db_session.query(Stream).filter_by(ended=False).order_by(Stream.stream_start.desc()).first()
            )
            self.last_stream = db_session.query(Stream).filter_by(ended=True).order_by(Stream.stream_end.desc()).first()
            if self.current_stream:
                self.current_stream_chunk = (
                    db_session.query(StreamChunk)
                    .filter_by(stream_id=self.current_stream.id)
                    .order_by(StreamChunk.chunk_start.desc())
                    .first()
                )
                log.info("Set current stream chunk here to {0}".format(self.current_stream_chunk))
            db_session.expunge_all()

    def get_viewer_data(self, redis=None):
        if self.offline:
            return False

        if not redis:
            redis = RedisManager.get()

        data = redis.hget("{streamer}:viewer_data".format(streamer=self.bot.streamer), self.current_stream.id)

        if data is None:
            data = {}
        else:
            data = json.loads(data)

        return data

    def update_chatters(self, chatters, minutes):
        """
        chatters is a list of usernames
        """

        if self.offline:
            return False

        redis = RedisManager.get()

        data = self.get_viewer_data(redis=redis)

        for chatter in chatters:
            if chatter in data:
                data[chatter] += minutes
            else:
                data[chatter] = minutes

        redis.hset(
            "{streamer}:viewer_data".format(streamer=self.bot.streamer),
            self.current_stream.id,
            json.dumps(data, separators=(",", ":")),
        )

    @property
    def online(self):
        return self.current_stream is not None

    @property
    def offline(self):
        return self.current_stream is None

    @staticmethod
    def commit():
        log.info("commiting something?")

    def create_stream_chunk(self, status):
        if self.current_stream_chunk is not None:
            # There's already a stream chunk started!
            self.current_stream_chunk.chunk_end = utils.now()
            DBManager.session_add_expunge(self.current_stream_chunk)

        stream_chunk = None

        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            stream_chunk = db_session.query(StreamChunk).filter_by(broadcast_id=status["broadcast_id"]).one_or_none()
            if stream_chunk is None:
                log.info("Creating stream chunk, from create_stream_chunk")
                stream_chunk = StreamChunk(self.current_stream, status["broadcast_id"], status["created_at"])
                self.current_stream_chunk = stream_chunk
                db_session.add(stream_chunk)
                db_session.commit()
            else:
                log.info("We already have a stream chunk!")
                self.current_stream_chunk = stream_chunk
                stream_chunk = None
            db_session.expunge_all()

        if stream_chunk:
            self.current_stream.stream_chunks.append(stream_chunk)

    def create_stream(self, status):
        log.info("Attempting to create a stream!")
        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            stream_chunk = db_session.query(StreamChunk).filter_by(broadcast_id=status["broadcast_id"]).one_or_none()
            new_stream = False
            if stream_chunk is not None:
                stream = stream_chunk.stream
            else:
                log.info("checking if there is an active stream already")
                stream = db_session.query(Stream).filter_by(ended=False).order_by(Stream.stream_start.desc()).first()
                new_stream = stream is None

                if new_stream:
                    log.info("No active stream, create new!")
                    stream = Stream(status["created_at"], title=status["title"])
                    db_session.add(stream)
                    db_session.commit()
                    log.info("Successfully added stream!")
                stream_chunk = StreamChunk(stream, status["broadcast_id"], status["created_at"])
                db_session.add(stream_chunk)
                db_session.commit()
                stream.stream_chunks.append(stream_chunk)
                log.info("Created stream chunk")

            self.current_stream = stream
            self.current_stream_chunk = stream_chunk
            db_session.expunge_all()

            if new_stream:
                HandlerManager.trigger("on_stream_start", stop_on_false=False)

            log.info("Successfully created a stream")

    def go_offline(self):
        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            self.current_stream.ended = True
            self.current_stream.stream_end = self.first_offline
            self.current_stream_chunk.chunk_end = self.first_offline

            db_session.add(self.current_stream)
            db_session.add(self.current_stream_chunk)

            db_session.commit()

            db_session.expunge_all()

        self.last_stream = self.current_stream
        self.current_stream = None
        self.current_stream_chunk = None

        HandlerManager.trigger("on_stream_stop", stop_on_false=False)

    def refresh_stream_status_stage1(self):
        try:
            status = self.bot.twitchapi.get_status(self.bot.streamer)
            if status["error"] is True:
                # log.error('An error occured while fetching stream status')
                # I'll comment this out since all errors are posted live anyway
                return

            self.bot.execute_now(lambda: self.refresh_stream_status_stage2(status))
        except:
            log.exception("Uncaught exception while refreshing stream status (Stage 1)")

    def refresh_stream_status_stage2(self, status):
        try:
            redis = RedisManager.get()
            key_prefix = self.bot.streamer + ":"

            stream_data = {}
            stream_data[key_prefix + "online"] = str(status["online"])
            stream_data[key_prefix + "viewers"] = status["viewers"]
            if status["game"]:
                stream_data[key_prefix + "game"] = status["game"]
            else:
                stream_data[key_prefix + "game"] = ""

            redis.hmset("stream_data", stream_data)

            self.num_viewers = status["viewers"]
            self.game = status["game"]
            self.title = status["title"]

            if status["online"]:
                if self.current_stream is None:
                    self.create_stream(status)
                if self.current_stream_chunk is None:
                    self.create_stream_chunk(status)
                if self.current_stream_chunk.broadcast_id != status["broadcast_id"]:
                    log.debug("Detected a new chunk!")
                    self.create_stream_chunk(status)

                self.num_offlines = 0
                self.first_offline = None
            else:
                if self.online is True:
                    log.info("Offline. {0}".format(self.num_offlines))
                    if self.first_offline is None:
                        self.first_offline = utils.now()

                    if self.num_offlines >= self.NUM_OFFLINES_REQUIRED:
                        log.info("Switching to offline state!")
                        self.go_offline()
                    self.num_offlines += 1
        except:
            log.exception("Uncaught exception while refreshing stream status (Stage 2)")

    def refresh_video_url_stage1(self):
        self.fetch_video_url_stage1()

    def refresh_video_url_stage2(self, data):
        if self.online is False:
            return

        if self.current_stream_chunk is None or self.current_stream is None:
            return

        log.info("Attempting to fetch video url for broadcast {0}".format(self.current_stream_chunk.broadcast_id))
        stream_chunk = self.current_stream_chunk if self.current_stream_chunk.video_url is None else None
        video_url, video_preview_image_url, video_recorded_at = self.fetch_video_url_stage2(data)
        if video_url is not None:
            log.info("Successfully fetched a video url: {0}".format(video_url))
            if self.current_stream_chunk is None or self.current_stream_chunk.video_url is None:
                with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                    self.current_stream_chunk.video_url = video_url
                    self.current_stream_chunk.video_preview_image_url = video_preview_image_url

                    db_session.add(self.current_stream_chunk)

                    db_session.commit()

                    db_session.expunge_all()
                log.info("Successfully commited video url data.")
            elif self.current_stream_chunk.video_url != video_url:
                # End current stream chunk
                self.current_stream_chunk.chunk_end = utils.now()
                DBManager.session_add_expunge(self.current_stream_chunk)

                with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                    stream_chunk = StreamChunk(
                        self.current_stream, self.current_stream_chunk.broadcast_id, video_recorded_at
                    )
                    self.current_stream_chunk = stream_chunk
                    self.current_stream_chunk.video_url = video_url
                    self.current_stream_chunk.video_preview_image_url = video_preview_image_url

                    db_session.add(self.current_stream_chunk)

                    db_session.commit()

                    db_session.expunge_all()
                log.info("Successfully commited video url data in a new chunk.")
        else:
            log.info("Not video for broadcast found")

    def get_stream_value(self, key, extra={}):
        return getattr(self, key, None)

    def get_current_stream_value(self, key, extra={}):
        if self.current_stream is not None:
            return getattr(self.current_stream, key, None)

        return None

    def get_last_stream_value(self, key, extra={}):
        if self.last_stream is not None:
            return getattr(self.last_stream, key, None)

        return None
<commit_msg>Missed a self.bot instead of self<commit_after>import datetime
import json
import logging
import urllib

from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy.dialects.mysql import BIGINT
from sqlalchemy.orm import relationship

from pajbot import utils
from pajbot.managers.db import Base
from pajbot.managers.db import DBManager
from pajbot.managers.handler import HandlerManager
from pajbot.managers.redis import RedisManager

log = logging.getLogger("pajbot")

def parse_twitch_datetime(datetime_str):
    return datetime.datetime.strptime(datetime_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=datetime.timezone.utc)

class Stream(Base):
    __tablename__ = "tb_stream"

    id = Column(Integer, primary_key=True)
    title = Column(String(256), nullable=False)
    stream_start = Column(DateTime, nullable=False)
    stream_end = Column(DateTime, nullable=True)
    ended = Column(Boolean, nullable=False, default=False)

    stream_chunks = relationship("StreamChunk", backref="stream", cascade="save-update, merge, expunge", lazy="joined")

    def __init__(self, created_at, **options):
        self.id = None
        self.title = options.get("title", "NO TITLE")
        self.stream_start = parse_twitch_datetime(created_at)
        self.stream_end = None
        self.ended = False

    @property
    def uptime(self):
        """
        Returns a TimeDelta for how long the stream was online, or is online.
        """

        if self.ended is False:
            return utils.now() - self.stream_start

        return self.stream_end - self.stream_start

class StreamChunk(Base):
    __tablename__ = "tb_stream_chunk"

    id = Column(Integer, primary_key=True)
    stream_id = Column(Integer, ForeignKey("tb_stream.id"), nullable=False)
    broadcast_id = Column(BIGINT(unsigned=True), nullable=False)
    video_url = Column(String(128), nullable=True)
    video_preview_image_url = Column(String(256), nullable=True)
    chunk_start = Column(DateTime, nullable=False)
    chunk_end = Column(DateTime, nullable=True)

    def __init__(self, stream, broadcast_id, created_at, **options):
        self.id = None
        self.stream_id = stream.id
        self.broadcast_id = broadcast_id
        self.video_url = None
        self.video_preview_image_url = None
        self.chunk_start = parse_twitch_datetime(created_at)
        self.chunk_end = None

        self.stream = stream

class StreamManager:
    NUM_OFFLINES_REQUIRED = 10
    STATUS_CHECK_INTERVAL = 20  # seconds
    VIDEO_URL_CHECK_INTERVAL = 60 * 5  # seconds

    def fetch_video_url_stage1(self):
        if self.online is False:
            return

        try:
            data = self.bot.twitchapi.get(
                ["channels", self.bot.streamer, "videos"],
                parameters={"broadcasts": "true"},
                base="http://127.0.0.1:7221/kraken/",
            )

            self.bot.execute_now(lambda: self.refresh_video_url_stage2(data))
        except urllib.error.HTTPError as e:
            raw_data = e.read().decode("utf-8")
            log.exception("OMGScoots")
            log.info(raw_data)
        except:
            log.exception("Uncaught exception in fetch_video_url")

    def fetch_video_url_stage2(self, data):
        stream_chunk = self.current_stream_chunk if self.current_stream_chunk.video_url is None else None
        try:
            for video in data["videos"]:
                if video["broadcast_type"] == "archive":
                    recorded_at = parse_twitch_datetime(video["recorded_at"])
                    if stream_chunk is not None:
                        time_diff = stream_chunk.chunk_start - recorded_at
                        if abs(time_diff.total_seconds()) < 60 * 5:
                            # we found the relevant video!
                            return (video["url"], video["preview"]["large"], video["recorded_at"])
                    else:
                        if video["status"] == "recording":
                            return (video["url"], video["preview"]["large"], video["recorded_at"])
        except urllib.error.HTTPError as e:
            raw_data = e.read().decode("utf-8")
            log.exception("OMGScoots")
            log.info(raw_data)
        except:
            log.exception("Uncaught exception in fetch_video_url")

        return None, None, None

    def __init__(self, bot):
        self.bot = bot

        self.current_stream_chunk = None  # should this even exist?

        self.num_offlines = 0
        self.first_offline = None

        self.num_viewers = 0

        self.game = "Loading..."
        self.title = "Loading..."

        self.bot.execute_every(
            self.STATUS_CHECK_INTERVAL, self.bot.action_queue.add, (self.refresh_stream_status_stage1,)
        )
        self.bot.execute_every(
            self.VIDEO_URL_CHECK_INTERVAL, self.bot.action_queue.add, (self.refresh_video_url_stage1,)
        )

        # This will load the latest stream so we can post an accurate "time since last online" figure.
        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            self.current_stream = (
                db_session.query(Stream).filter_by(ended=False).order_by(Stream.stream_start.desc()).first()
            )
            self.last_stream = db_session.query(Stream).filter_by(ended=True).order_by(Stream.stream_end.desc()).first()
            if self.current_stream:
                self.current_stream_chunk = (
                    db_session.query(StreamChunk)
                    .filter_by(stream_id=self.current_stream.id)
                    .order_by(StreamChunk.chunk_start.desc())
                    .first()
                )
                log.info("Set current stream chunk here to {0}".format(self.current_stream_chunk))
            db_session.expunge_all()

    def get_viewer_data(self, redis=None):
        if self.offline:
            return False

        if not redis:
            redis = RedisManager.get()

        data = redis.hget("{streamer}:viewer_data".format(streamer=self.bot.streamer), self.current_stream.id)

        if data is None:
            data = {}
        else:
            data = json.loads(data)

        return data

    def update_chatters(self, chatters, minutes):
        """
        chatters is a list of usernames
        """

        if self.offline:
            return False

        redis = RedisManager.get()

        data = self.get_viewer_data(redis=redis)

        for chatter in chatters:
            if chatter in data:
                data[chatter] += minutes
            else:
                data[chatter] = minutes

        redis.hset(
            "{streamer}:viewer_data".format(streamer=self.bot.streamer),
            self.current_stream.id,
            json.dumps(data, separators=(",", ":")),
        )

    @property
    def online(self):
        return self.current_stream is not None

    @property
    def offline(self):
        return self.current_stream is None

    @staticmethod
    def commit():
        log.info("commiting something?")

    def create_stream_chunk(self, status):
        if self.current_stream_chunk is not None:
            # There's already a stream chunk started!
            self.current_stream_chunk.chunk_end = utils.now()
            DBManager.session_add_expunge(self.current_stream_chunk)

        stream_chunk = None

        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            stream_chunk = db_session.query(StreamChunk).filter_by(broadcast_id=status["broadcast_id"]).one_or_none()
            if stream_chunk is None:
                log.info("Creating stream chunk, from create_stream_chunk")
                stream_chunk = StreamChunk(self.current_stream, status["broadcast_id"], status["created_at"])
                self.current_stream_chunk = stream_chunk
                db_session.add(stream_chunk)
                db_session.commit()
            else:
                log.info("We already have a stream chunk!")
                self.current_stream_chunk = stream_chunk
                stream_chunk = None
            db_session.expunge_all()

        if stream_chunk:
            self.current_stream.stream_chunks.append(stream_chunk)

    def create_stream(self, status):
        log.info("Attempting to create a stream!")
        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            stream_chunk = db_session.query(StreamChunk).filter_by(broadcast_id=status["broadcast_id"]).one_or_none()
            new_stream = False
            if stream_chunk is not None:
                stream = stream_chunk.stream
            else:
                log.info("checking if there is an active stream already")
                stream = db_session.query(Stream).filter_by(ended=False).order_by(Stream.stream_start.desc()).first()
                new_stream = stream is None

                if new_stream:
                    log.info("No active stream, create new!")
                    stream = Stream(status["created_at"], title=status["title"])
                    db_session.add(stream)
                    db_session.commit()
                    log.info("Successfully added stream!")
                stream_chunk = StreamChunk(stream, status["broadcast_id"], status["created_at"])
                db_session.add(stream_chunk)
                db_session.commit()
                stream.stream_chunks.append(stream_chunk)
                log.info("Created stream chunk")

            self.current_stream = stream
            self.current_stream_chunk = stream_chunk
            db_session.expunge_all()

            if new_stream:
                HandlerManager.trigger("on_stream_start", stop_on_false=False)

            log.info("Successfully created a stream")

    def go_offline(self):
        with DBManager.create_session_scope(expire_on_commit=False) as db_session:
            self.current_stream.ended = True
            self.current_stream.stream_end = self.first_offline
            self.current_stream_chunk.chunk_end = self.first_offline

            db_session.add(self.current_stream)
            db_session.add(self.current_stream_chunk)

            db_session.commit()

            db_session.expunge_all()

        self.last_stream = self.current_stream
        self.current_stream = None
        self.current_stream_chunk = None

        HandlerManager.trigger("on_stream_stop", stop_on_false=False)

    def refresh_stream_status_stage1(self):
        try:
            status = self.bot.twitchapi.get_status(self.bot.streamer)
            if status["error"] is True:
                # log.error('An error occured while fetching stream status')
                # I'll comment this out since all errors are posted live anyway
                return

            self.bot.execute_now(lambda: self.refresh_stream_status_stage2(status))
        except:
            log.exception("Uncaught exception while refreshing stream status (Stage 1)")

    def refresh_stream_status_stage2(self, status):
        try:
            redis = RedisManager.get()
            key_prefix = self.bot.streamer + ":"

            stream_data = {}
            stream_data[key_prefix + "online"] = str(status["online"])
            stream_data[key_prefix + "viewers"] = status["viewers"]
            if status["game"]:
                stream_data[key_prefix + "game"] = status["game"]
            else:
                stream_data[key_prefix + "game"] = ""

            redis.hmset("stream_data", stream_data)

            self.num_viewers = status["viewers"]
            self.game = status["game"]
            self.title = status["title"]

            if status["online"]:
                if self.current_stream is None:
                    self.create_stream(status)
                if self.current_stream_chunk is None:
                    self.create_stream_chunk(status)
                if self.current_stream_chunk.broadcast_id != status["broadcast_id"]:
                    log.debug("Detected a new chunk!")
                    self.create_stream_chunk(status)

                self.num_offlines = 0
                self.first_offline = None
            else:
                if self.online is True:
                    log.info("Offline. {0}".format(self.num_offlines))
                    if self.first_offline is None:
                        self.first_offline = utils.now()

                    if self.num_offlines >= self.NUM_OFFLINES_REQUIRED:
                        log.info("Switching to offline state!")
                        self.go_offline()
                    self.num_offlines += 1
        except:
            log.exception("Uncaught exception while refreshing stream status (Stage 2)")

    def refresh_video_url_stage1(self):
        self.fetch_video_url_stage1()

    def refresh_video_url_stage2(self, data):
        if self.online is False:
            return

        if self.current_stream_chunk is None or self.current_stream is None:
            return

        log.info("Attempting to fetch video url for broadcast {0}".format(self.current_stream_chunk.broadcast_id))
        stream_chunk = self.current_stream_chunk if self.current_stream_chunk.video_url is None else None
        video_url, video_preview_image_url, video_recorded_at = self.fetch_video_url_stage2(data)
        if video_url is not None:
            log.info("Successfully fetched a video url: {0}".format(video_url))
            if self.current_stream_chunk is None or self.current_stream_chunk.video_url is None:
                with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                    self.current_stream_chunk.video_url = video_url
                    self.current_stream_chunk.video_preview_image_url = video_preview_image_url

                    db_session.add(self.current_stream_chunk)

                    db_session.commit()

                    db_session.expunge_all()
                log.info("Successfully commited video url data.")
            elif self.current_stream_chunk.video_url != video_url:
                # End current stream chunk
                self.current_stream_chunk.chunk_end = utils.now()
                DBManager.session_add_expunge(self.current_stream_chunk)

                with DBManager.create_session_scope(expire_on_commit=False) as db_session:
                    stream_chunk = StreamChunk(
                        self.current_stream, self.current_stream_chunk.broadcast_id, video_recorded_at
                    )
                    self.current_stream_chunk = stream_chunk
                    self.current_stream_chunk.video_url = video_url
                    self.current_stream_chunk.video_preview_image_url = video_preview_image_url

                    db_session.add(self.current_stream_chunk)

                    db_session.commit()

                    db_session.expunge_all()
                log.info("Successfully commited video url data in a new chunk.")
        else:
            log.info("Not video for broadcast found")

    def get_stream_value(self, key, extra={}):
        return getattr(self, key, None)

    def get_current_stream_value(self, key, extra={}):
        if self.current_stream is not None:
            return getattr(self.current_stream, key, None)

        return None

    def get_last_stream_value(self, key, extra={}):
        if self.last_stream is not None:
            return getattr(self.last_stream, key, None)

        return None
<|endoftext|>


<commit_before>import config
import cv2
import cv2.cv as cv
import numpy as np
from pylab import array

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original.copy()
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        img = (255.0/0.5) * (img/(255.0/0.5))**2
        img = array(img,dtype=np.uint8)

        # cv2.imshow('4', img)
        cv2.imshow('frame', img)

        # find edges
        # values 50, 130 work well
        edges = cv2.Canny(img, 40, 100)

        # kernel = np.ones((3,3), np.uint8)
        # edges = cv2.dilate(edges, kernel, iterations = 1)

        edges = cv2.Canny(img, 50, 120)
        cv2.imshow('2', edges)

        edges = cv2.Canny(img, 40, 130)
        cv2.imshow('3', edges)

        edges = cv2.Canny(img, 70, 100)
        cv2.imshow('4', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            # find only largest contour
            if area < largest_area:
                continue

            # discard contours that are not in the center of the image
            # discard contours which bounding circle is too large
            (x,y),radius = cv2.minEnclosingCircle(i)
            if y < 50 or y > 250 or radius > 50 or radius < 15:
                continue

            contour = i
            largest_area = area

        # if not self.printed:
        #     print e
        # self.printed = True

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.processed, center, radius,(0,255,0),2)
            img = cv2.circle(self.processed, center, 2,(0,0,255),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.processed, 'Largest contour area: ' + str(round(largest_area, 2)), (10,10), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Center of the contour: (' + str(round(x, 2)) + ', ' + str(round(y, 2)) + ')', (10,30), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Radius of the contour: ' + str(round(radius, 2)), (10,60), font, 0.5, (255,255,255), 2)

        # cv2.imshow('3', self.processed)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<commit_msg>Update<commit_after>import config
import cv2
import cv2.cv as cv
import numpy as np
from pylab import array

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original.copy()
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        img = (255.0/2) * (img/(255.0/2))**2
        img = array(img,dtype=np.uint8)

        # cv2.imshow('4', img)
        cv2.imshow('frame', img)

        # find edges
        # values 50, 130 work well
        edges = cv2.Canny(img, 40, 100)

        # kernel = np.ones((3,3), np.uint8)
        # edges = cv2.dilate(edges, kernel, iterations = 1)

        edges = cv2.Canny(img, 50, 120)
        cv2.imshow('2', edges)

        edges = cv2.Canny(img, 40, 130)
        cv2.imshow('3', edges)

        edges = cv2.Canny(img, 70, 100)
        cv2.imshow('4', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            # find only largest contour
            if area < largest_area:
                continue

            # discard contours that are not in the center of the image
            # discard contours which bounding circle is too large
            (x,y),radius = cv2.minEnclosingCircle(i)
            if y < 50 or y > 250 or radius > 50 or radius < 15:
                continue

            contour = i
            largest_area = area

        # if not self.printed:
        #     print e
        # self.printed = True

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.processed, center, radius,(0,255,0),2)
            img = cv2.circle(self.processed, center, 2,(0,0,255),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.processed, 'Largest contour area: ' + str(round(largest_area, 2)), (10,10), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Center of the contour: (' + str(round(x, 2)) + ', ' + str(round(y, 2)) + ')', (10,30), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Radius of the contour: ' + str(round(radius, 2)), (10,60), font, 0.5, (255,255,255), 2)

        # cv2.imshow('3', self.processed)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import logging
import sqlite3

class Event:
    """Class representing a single Event."""
    def __init__(self, db_id, title, datum, url):
        self.data = dict()

        # Get Information for this event from the sqlite DB
        try:
            conn = sqlite3.connect(os.path.join(os.path.dirname(__file__), "../locations.sqlite"))
            conn.row_factory = sqlite3.Row
            c = conn.cursor()
        except:
            logging.error("Cant connect to database!")

        c.execute('select * from locations where id=?', [db_id])
        r = c.fetchone()

        for key in list(r.keys()):  # Populate list with data from database
            self.data[key] = r[key]

        # Populate list with data from web
        self.data["title"] = title
        self.data["datetime"] = datum
        self.data["datum"] = datum.strftime("%a, %d. %b")
        uhrzeit = datum.strftime("%H:%M")
        if uhrzeit == "00:00":  # "probably" no specific time set for this event...'
            uhrzeit = ""
        self.data["uhrzeit"] = uhrzeit  # uhrzeit = german time
        self.data["event_url"] = url

        #logging.info(" Found Event: " + str(self.data["title"]) + " at " + str(self.data["name"]))

    def get(self, param):
        return self.data[param]

    def get_all(self):
        return self.data

class EventVault:
    """Central Storage for events"""
    def __init__(self):
        self.events = []

    def add(self, e):
        self.events.append(e)

    def get_events_for_date(self, date):
        e_subset = []
        for e in self.events:
            if e.get("datetime").date() == date:
                logging.info("&s: %s" % (date, e.get("title")))
                e_subset.append(e)

        if len(e_subset) > 0:
            e_subset = sorted(e_subset, key=lambda e: e.data["uhrzeit"])  # Sort by uhrzeit
            return e_subset
        else:
            return False

    def get_all_events(self):
        if len(self.events) > 0:
            return self.events
        else:
            return False<commit_msg>Typo...<commit_after># -*- coding: utf-8 -*-

import os
import logging
import sqlite3

class Event:
    """Class representing a single Event."""
    def __init__(self, db_id, title, datum, url):
        self.data = dict()

        # Get Information for this event from the sqlite DB
        try:
            conn = sqlite3.connect(os.path.join(os.path.dirname(__file__), "../locations.sqlite"))
            conn.row_factory = sqlite3.Row
            c = conn.cursor()
        except:
            logging.error("Cant connect to database!")

        c.execute('select * from locations where id=?', [db_id])
        r = c.fetchone()

        for key in list(r.keys()):  # Populate list with data from database
            self.data[key] = r[key]

        # Populate list with data from web
        self.data["title"] = title
        self.data["datetime"] = datum
        self.data["datum"] = datum.strftime("%a, %d. %b")
        uhrzeit = datum.strftime("%H:%M")
        if uhrzeit == "00:00":  # "probably" no specific time set for this event...'
            uhrzeit = ""
        self.data["uhrzeit"] = uhrzeit  # uhrzeit = german time
        self.data["event_url"] = url

        #logging.info(" Found Event: " + str(self.data["title"]) + " at " + str(self.data["name"]))

    def get(self, param):
        return self.data[param]

    def get_all(self):
        return self.data

class EventVault:
    """Central Storage for events"""
    def __init__(self):
        self.events = []

    def add(self, e):
        self.events.append(e)

    def get_events_for_date(self, date):
        e_subset = []
        for e in self.events:
            if e.get("datetime").date() == date:
                logging.info("%s: %s" % (date, e.get("title")))
                e_subset.append(e)

        if len(e_subset) > 0:
            e_subset = sorted(e_subset, key=lambda e: e.data["uhrzeit"])  # Sort by uhrzeit
            return e_subset
        else:
            return False

    def get_all_events(self):
        if len(self.events) > 0:
            return self.events
        else:
            return False<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from cloudbrain.settings import RABBITMQ_ADDRESS

import pika
import json
import logging

from sockjs.tornado.conn import SockJSConnection
from sockjs.tornado import SockJSRouter
from tornado.ioloop import PeriodicCallback, IOLoop
from tornado.web import RequestHandler, Application

from cloudbrain.subscribers.SubscriberInterface import Subscriber

SERVER_PORT = 31415
logging.getLogger().setLevel(logging.INFO)

class RtStreamConnection(SockJSConnection):
    """RtStreamConnection connection implementation"""
    # Class level variable
    clients = set()

    def __init__(self, session):
        super(self.__class__, self).__init__(session)
        self.subscribers = {}

    def send_probe_factory(self, metric_name):

        def send_probe(body):
            logging.debug("GOT: " + body)
            buffer_content = json.loads(body)

            for record in buffer_content:
                record["metric"] = metric_name
                self.send(json.dumps(record))

        return send_probe

    def on_open(self, info):
        logging.info("Got a new connection...")
        self.clients.add(self)
        # self.timeout = PeriodicCallback(self.send_heartbeat, 1000)
        # self.timeout.start()

    # This will receive instructions from the client to change the
    # stream. After the connection is established we expect to receive a JSON
    # with deviceName, deviceId, metric; then we subscribe to RabbitMQ and
    # start streaming the data.
    #
    # NOTE: it's not possible to open multiple connections from the same client.
    #       so in case we need to stream different devices/metrics/etc. at the
    #       same time, we need to use a solution that is like the multiplexing
    #       in the sockjs-tornado examples folder.
    def on_message(self, message):
        logging.info("Got a new message: " + message)

        msg_dict = json.loads(message)
        if msg_dict['type'] == 'subscription':
            self.handle_channel_subscription(msg_dict)
        elif msg_dict['type'] == 'unsubscription':
            self.handle_channel_unsubscription(msg_dict)

    def handle_channel_subscription(self, stream_configuration):
        device_name = stream_configuration['deviceName']
        device_id = stream_configuration['deviceId']
        metric = stream_configuration['metric']

        if metric not in self.subscribers:
            self.subscribers[metric] = TornadoSubscriber(callback=self.send_probe_factory(metric),
                                       device_name=device_name,
                                       device_id=device_id,
                                       rabbitmq_address=RABBITMQ_ADDRESS,
                                       metric_name=metric)

            self.subscribers[metric].connect()

    def handle_channel_unsubscription(self, unsubscription_msg):
        logging.info('Unsubscription received for ' + unsubscription_msg['metric'])
        if unsubscription_msg['metric'] in self.subscribers:
            self.subscribers[unsubscription_msg['metric']].disconnect()

    def on_close(self):
        logging.info('Disconnecting client, unsubscribing from queues...')
        for (metric, subscriber) in self.subscribers.keys():
          if subscriber is not None:
              subscriber.disconnect()

        self.subscribers = {}
        #self.timeout.stop()
        self.clients.remove(self)

    def send_heartbeat(self):
        self.broadcast(self.clients, 'message')

class MockHandler(RequestHandler):
    """Just a mock page to test it out..."""
    def get(self):
        self.render('mock.html')

class RtDataStreamHandler(RequestHandler):
    """
    Just a custom handler for the rt-data-stream.js file :)
    """
    def get(self):
        self.render('rt-data-stream.js')

# Based on: https://pika.readthedocs.org/en/0.9.14/examples/tornado_consumer.html
class TornadoSubscriber(object):

    def __init__(self, callback, device_name, device_id, rabbitmq_address, metric_name):
        self.callback = callback
        self.device_name = device_name
        self.device_id = device_id
        self.metric_name = metric_name

        self.connection = None
        self.channel = None

        self.host = RABBITMQ_ADDRESS
        self.queue_name = None

    def connect(self):
        credentials = pika.PlainCredentials('cloudbrain', 'cloudbrain')
        self.connection = pika.adapters.tornado_connection.TornadoConnection(pika.ConnectionParameters(
                                        host=self.host, credentials=credentials),
                                        self.on_connected,
                                        stop_ioloop_on_close=False,
                                        custom_ioloop=IOLoop.instance())

    def disconnect(self):
        if self.connection is not None:
            self.connection.close()

    def on_connected(self, connection):
        self.connection = connection
        self.connection.add_on_close_callback(self.on_connection_closed)
        self.connection.add_backpressure_callback(self.on_backpressure_callback)
        self.open_channel()

    def on_connection_closed(self, connection, reply_code, reply_text):
        self.connection = None
        self.channel = None

    def on_backpressure_callback(self, connection):
        logging.info('******** Backpressure detected for ' + self.get_key())

    def open_channel(self):
        self.connection.channel(self.on_channel_open)

    def on_channel_open(self, channel):
        self.channel = channel
        self.channel.add_on_close_callback(self.on_channel_closed)
        # self.setup_exchange(self.EXCHANGE)
        # self.channel.confirm_delivery(self.on_delivery_confirmation)
        logging.info("Declaring exchange: " + self.get_key())
        self.channel.exchange_declare(self.on_exchange_declareok,
                                      exchange=self.get_key(),
                                      type='direct',
                                      passive=True)
                                      #type='direct')
        # self.queue_name = self.channel.queue_declare(exclusive=True).method.queue

    def on_channel_closed(self, channel, reply_code, reply_text):
        self.connection.close()

    def on_exchange_declareok(self, unused_frame):
        self.channel.queue_declare(self.on_queue_declareok, self.get_key())

    def on_queue_declareok(self, unused_frame):
        logging.info("Binding queue: " + self.get_key())
        self.channel.queue_bind(
                       self.on_bindok,
                       exchange=self.get_key(),
                       queue=self.get_key(),
                       routing_key=self.get_key())

    def on_bindok(self, unused_frame):
        self.channel.add_on_cancel_callback(self.on_consumer_cancelled)
        self.consumer_tag = self.channel.basic_consume(self.on_message, self.get_key())

    def on_consumer_cancelled(self, method_frame):
        if self.channel:
            self.channel.close()

    def on_message(self, unused_channel, basic_deliver, properties, body):
        self.acknowledge_message(basic_deliver.delivery_tag)
        self.callback(body)

    def acknowledge_message(self, delivery_tag):
        self.channel.basic_ack(delivery_tag)

    def get_key(self):
        key = "%s:%s:%s" %(self.device_id,self.device_name, self.metric_name)
        return key

if __name__ == "__main__":

    # 1. Create chat router
    RtStreamRouter = SockJSRouter(RtStreamConnection, '/rt-stream')

    # 2. Create Tornado application
    app = Application(
            [(r"/", MockHandler), (r"/rt-data-stream.js", RtDataStreamHandler)] + RtStreamRouter.urls
    )

    # 3. Make Tornado app listen on Pi
    app.listen(SERVER_PORT)

    print "Real-time data server running at http://localhost:%s" %SERVER_PORT

    # 4. Start IOLoop
    IOLoop.instance().start()
<commit_msg>RT-SRV-OPT Bugfixes in disconnection/unsubscribing logic<commit_after># -*- coding: utf-8 -*-

from cloudbrain.settings import RABBITMQ_ADDRESS

import pika
import json
import logging

from sockjs.tornado.conn import SockJSConnection
from sockjs.tornado import SockJSRouter
from tornado.ioloop import PeriodicCallback, IOLoop
from tornado.web import RequestHandler, Application

from cloudbrain.subscribers.SubscriberInterface import Subscriber

SERVER_PORT = 31415
logging.getLogger().setLevel(logging.INFO)

class RtStreamConnection(SockJSConnection):
    """RtStreamConnection connection implementation"""
    # Class level variable
    clients = set()

    def __init__(self, session):
        super(self.__class__, self).__init__(session)
        self.subscribers = {}

    def send_probe_factory(self, metric_name):

        def send_probe(body):
            logging.debug("GOT: " + body)
            buffer_content = json.loads(body)

            for record in buffer_content:
                record["metric"] = metric_name
                self.send(json.dumps(record))

        return send_probe

    def on_open(self, info):
        logging.info("Got a new connection...")
        self.clients.add(self)
        # self.timeout = PeriodicCallback(self.send_heartbeat, 1000)
        # self.timeout.start()

    # This will receive instructions from the client to change the
    # stream. After the connection is established we expect to receive a JSON
    # with deviceName, deviceId, metric; then we subscribe to RabbitMQ and
    # start streaming the data.
    #
    # NOTE: it's not possible to open multiple connections from the same client.
    #       so in case we need to stream different devices/metrics/etc. at the
    #       same time, we need to use a solution that is like the multiplexing
    #       in the sockjs-tornado examples folder.
    def on_message(self, message):
        logging.info("Got a new message: " + message)

        msg_dict = json.loads(message)
        if msg_dict['type'] == 'subscription':
            self.handle_channel_subscription(msg_dict)
        elif msg_dict['type'] == 'unsubscription':
            self.handle_channel_unsubscription(msg_dict)

    def handle_channel_subscription(self, stream_configuration):
        device_name = stream_configuration['deviceName']
        device_id = stream_configuration['deviceId']
        metric = stream_configuration['metric']

        if metric not in self.subscribers:
            self.subscribers[metric] = TornadoSubscriber(callback=self.send_probe_factory(metric),
                                       device_name=device_name,
                                       device_id=device_id,
                                       rabbitmq_address=RABBITMQ_ADDRESS,
                                       metric_name=metric)

            self.subscribers[metric].connect()

    def handle_channel_unsubscription(self, unsubscription_msg):
        logging.info('Unsubscription received for ' + unsubscription_msg['metric'])
        if unsubscription_msg['metric'] in self.subscribers:
            self.subscribers[unsubscription_msg['metric']].disconnect()

    def on_close(self):
        logging.info('Disconnecting client...')
        for metric in self.subscribers.keys():
            subscriber = self.subscribers[metric]
            if subscriber is not None:
                logging.info('Disconnecting subscriber for metric: ' + metric)
                subscriber.disconnect()

        self.subscribers = {}
        #self.timeout.stop()
        self.clients.remove(self)
        logging.info('Client disconnection complete!')

    def send_heartbeat(self):
        self.broadcast(self.clients, 'message')

class MockHandler(RequestHandler):
    """Just a mock page to test it out..."""
    def get(self):
        self.render('mock.html')

class RtDataStreamHandler(RequestHandler):
    """
    Just a custom handler for the rt-data-stream.js file :)
    """
    def get(self):
        self.render('rt-data-stream.js')

# Based on: https://pika.readthedocs.org/en/0.9.14/examples/tornado_consumer.html
class TornadoSubscriber(object):

    def __init__(self, callback, device_name, device_id, rabbitmq_address, metric_name):
        self.callback = callback
        self.device_name = device_name
        self.device_id = device_id
        self.metric_name = metric_name

        self.connection = None
        self.channel = None

        self.host = RABBITMQ_ADDRESS
        self.queue_name = None

    def connect(self):
        credentials = pika.PlainCredentials('cloudbrain', 'cloudbrain')
        self.connection = pika.adapters.tornado_connection.TornadoConnection(pika.ConnectionParameters(
                                        host=self.host, credentials=credentials),
                                        self.on_connected,
                                        stop_ioloop_on_close=False,
                                        custom_ioloop=IOLoop.instance())

    def disconnect(self):
        if self.connection is not None:
            self.connection.close()

    def on_connected(self, connection):
        self.connection = connection
        self.connection.add_on_close_callback(self.on_connection_closed)
        self.connection.add_backpressure_callback(self.on_backpressure_callback)
        self.open_channel()

    def on_connection_closed(self, connection, reply_code, reply_text):
        self.connection = None
        self.channel = None

    def on_backpressure_callback(self, connection):
        logging.info('******** Backpressure detected for ' + self.get_key())

    def open_channel(self):
        self.connection.channel(self.on_channel_open)

    def on_channel_open(self, channel):
        self.channel = channel
        self.channel.add_on_close_callback(self.on_channel_closed)
        # self.setup_exchange(self.EXCHANGE)
        # self.channel.confirm_delivery(self.on_delivery_confirmation)
        logging.info("Declaring exchange: " + self.get_key())
        self.channel.exchange_declare(self.on_exchange_declareok,
                                      exchange=self.get_key(),
                                      type='direct',
                                      passive=True)
                                      #type='direct')
        # self.queue_name = self.channel.queue_declare(exclusive=True).method.queue

    def on_channel_closed(self, channel, reply_code, reply_text):
        self.connection.close()

    def on_exchange_declareok(self, unused_frame):
        self.channel.queue_declare(self.on_queue_declareok, self.get_key())

    def on_queue_declareok(self, unused_frame):
        logging.info("Binding queue: " + self.get_key())
        self.channel.queue_bind(
                       self.on_bindok,
                       exchange=self.get_key(),
                       queue=self.get_key(),
                       routing_key=self.get_key())

    def on_bindok(self, unused_frame):
        self.channel.add_on_cancel_callback(self.on_consumer_cancelled)
        self.consumer_tag = self.channel.basic_consume(self.on_message, self.get_key())

    def on_consumer_cancelled(self, method_frame):
        if self.channel:
            self.channel.close()

    def on_message(self, unused_channel, basic_deliver, properties, body):
        self.acknowledge_message(basic_deliver.delivery_tag)
        self.callback(body)

    def acknowledge_message(self, delivery_tag):
        self.channel.basic_ack(delivery_tag)

    def get_key(self):
        key = "%s:%s:%s" %(self.device_id,self.device_name, self.metric_name)
        return key

if __name__ == "__main__":

    # 1. Create chat router
    RtStreamRouter = SockJSRouter(RtStreamConnection, '/rt-stream')

    # 2. Create Tornado application
    app = Application(
            [(r"/", MockHandler), (r"/rt-data-stream.js", RtDataStreamHandler)] + RtStreamRouter.urls
    )

    # 3. Make Tornado app listen on Pi
    app.listen(SERVER_PORT)

    print "Real-time data server running at http://localhost:%s" %SERVER_PORT

    # 4. Start IOLoop
    IOLoop.instance().start()
<|endoftext|>