<commit_before>import sys
import time
import math
import random
import logbook
import colorsys
import zmq
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import MoveTargetOutOfBoundsException

# Server related constants.
DEFAULT_LISTENING_PORT = '5556'
STOP_MESSAGE = 'STOP'
OK_REPLY = 'OK'
COLORS_DELIMITER = ','

# A JS script which removes a given element.
REMOVE_SCRIPT = 'var element = document.querySelector("{0}");' \
                'if (element) element.parentNode.removeChild(element);'

# A JS script which modifies a specific value in all elements of a given class.
MODIFY_SCRIPT = 'var list = document.getElementsByClassName("{0}");' \
                'for (var i = 0; i < list.length; i++) {{ ' \
                'list[i].{1} = "{2}" }};'

# The size of one part of the color square.
COLOR_SQUARE_SIZE = 68
# Sliders settings.
MIN_OPACITY = 30
MAX_OPACITY = 118
MIN_SIZE = 5
MAX_SIZE = 22

log = logbook.Logger('DeviantArt')

def remove_class(driver, class_name):
    """
    Removes the element with the given class.
    """
    driver.execute_script(REMOVE_SCRIPT.format('.' + class_name))

def remove_id(driver, id_name):
    """
    Removes the element with the given ID.
    """
    driver.execute_script(REMOVE_SCRIPT.format('#' + id_name))

def change_canvas_size(driver, class_name, width, height, only_style=False):
    """
    Changes the size of a canvas, to the desired width and height.

    :param class_name: The canvas' class name.
    :param width: The desired width.
    :param height: The desired height.
    :param only_style: If True, the width and height will be changed in the style attribute only.
    """
    driver.execute_script(MODIFY_SCRIPT.format(class_name, 'style.width', str(width) + 'px'))
    driver.execute_script(MODIFY_SCRIPT.format(class_name, 'style.height', str(height) + 'px'))
    if not only_style:
        driver.execute_script(MODIFY_SCRIPT.format(class_name, 'width', str(width)))
        driver.execute_script(MODIFY_SCRIPT.format(class_name, 'height', str(height)))

def change_slider(driver, slider_element, value):
    """
    Changes one of the 3 option sliders.

    :param slider_element: The slider element to change.
    :param value: The desired value.
    """
    action = webdriver.common.action_chains.ActionChains(driver)
    action.move_to_element_with_offset(slider_element, value, 0)
    action.click()
    action.perform()

def change_color(driver, r, g, b):
    """
    Changes the color according to the given RGB values.
    """
    h, s, v = colorsys.rgb_to_hsv(float(r), float(g), float(b))
    # Change hue.
    r = COLOR_SQUARE_SIZE / 2
    # The hue angle has an offset of 45 degrees.
    teta = math.radians(h * 360 - 45)
    # Add the center coordinates.
    x = r * math.cos(teta) + COLOR_SQUARE_SIZE / 2
    y = -r * math.sin(teta) + COLOR_SQUARE_SIZE / 2
    hue_element = driver.find_element_by_class_name('hueClick')
    action = webdriver.common.action_chains.ActionChains(driver)
    action.move_to_element_with_offset(hue_element, x, y)
    action.click()
    action.perform()
    # Change saturation and value.
    x = s * COLOR_SQUARE_SIZE
    y = COLOR_SQUARE_SIZE - (v / 255.0) * COLOR_SQUARE_SIZE
    saturation_element = driver.find_element_by_class_name('satval_area')
    action = webdriver.common.action_chains.ActionChains(driver)
    action.move_to_element_with_offset(saturation_element, x, y)
    action.click()
    action.perform()

def paint(driver, canvas_element, x, y):
    """
    Paint in the given coordinates.
    """
    try:
        action = webdriver.common.action_chains.ActionChains(driver)
        action.move_to_element_with_offset(canvas_element, x, y)
        action.click_and_hold()
        for j in xrange(70):
            action.move_to_element_with_offset(canvas_element, x + (2 if j % 2 == 0 else -2), y)
        action.release()
        action.perform()
    except MoveTargetOutOfBoundsException:
        # If we accidentaly stepped out of bounds,
        # stop the current paint action and release the mouse button.
        action = webdriver.common.action_chains.ActionChains(driver)
        action.release()
        action.perform()

def setup():
	"""
	Setup the browser window and painting canvas.

	:return: The newly created web driver.
	"""
	# Set up the window.
	log.info('Loading the original website...')
	driver = webdriver.Firefox()
	driver.get('http://sta.sh/muro')
	driver.maximize_window()
	#driver.find_element_by_tag_name('html').send_keys(Keys.F11)
	time.sleep(5)
	# Pick the right tool.
	log.info('Picking the splatter brush...')
	driver.find_element_by_class_name('iconSplatter').click()
	# Remove all unwanted side bars.
	log.info('Removing side bars...')
	remove_class(driver, 'sideBar')
	remove_class(driver, 'topArea')
	remove_id(driver, 'overhead-collect')
	# Set the density slider.
	log.info('Setting density value to 100%')
	sliders = driver.find_elements_by_class_name('sliderProgress')
	# Density.
	change_slider(driver, sliders[2], 120)
	# Maximize the canvas.
	log.info('Maximizing canvas size...')
	window_size = driver.get_window_size()
	canvas_width = window_size['width']
	canvas_height = window_size['height']
	change_canvas_size(driver, 'selectionCanvas', canvas_width, canvas_height)
	change_canvas_size(driver, 'drawPlzCanvas', canvas_width, canvas_height)
	change_canvas_size(driver, 'canvasBuffer', canvas_width, canvas_height)
	change_canvas_size(driver, 'stagingBuffer', canvas_width, canvas_height)
	change_canvas_size(driver, 'marchingAnts', canvas_width, canvas_height)
	change_canvas_size(driver, 'canvasPaint', canvas_width, canvas_height, only_style=True)
	# Modify specific styles and positions.
	driver.execute_script(MODIFY_SCRIPT.format('middleArea', 'style.paddingRight', '0px'))
	driver.execute_script(MODIFY_SCRIPT.format('middleArea', 'style.height', '100%'))
	driver.execute_script(MODIFY_SCRIPT.format('drawPlzCanvas', 'style.left', '0px'))
	driver.execute_script(MODIFY_SCRIPT.format('drawPlzCanvas', 'style.top', '0px'))
	driver.execute_script(MODIFY_SCRIPT.format('drawPlzCanvas', 'width', canvas_width))
	driver.execute_script('document.getElementsByClassName("canvasPaint")[1].width = %d;' % canvas_width)
	driver.execute_script('document.getElementsByClassName("canvasPaint")[1].height = %d;' % canvas_height)
	# Hide the mouse cursor.
	log.info('Hiding mouse reticule...')
	driver.execute_script(MODIFY_SCRIPT.format('cursorPreview', 'style.visibility', 'hidden'))
	return driver

def main():
	"""
	Paints based on colors received from the server.
	Usage: deviantArt.py [PORT]
	"""
	# Validate input parameters.
	port = DEFAULT_LISTENING_PORT if len(sys.argv) == 1 else sys.argv[1]
	# Connect to the server.
	context = zmq.Context()
	log.info('Waiting for painting requests (port: {0})...'.format(port))
	socket = context.socket(zmq.REP)
	socket.bind ('tcp://*:{0}'.format(port))
	# Initialize the website.
	driver = setup()
	# Find the canvas width and height.
	window_size = driver.get_window_size()
	canvas_width = window_size['width']
	canvas_height = window_size['height']
	# Find the main canvas element.
	canvas_element = driver.find_element_by_class_name('drawPlzCanvas')
	# Start painting!
	log.info('Let\'s paint!')
	while True:
		message = socket.recv()
		log.info('Received request: {0}'.format(message))
		socket.send(OK_REPLY)
		log.info('Sent {0} back'.format(OK_REPLY))
		if message == STOP_MESSAGE:
			log.info('Stopping!')
			break
		# Pick the requested color.
		r, g, b = message.split(COLORS_DELIMITER)
		# Don't get to close to the borders.
		x = random.randint(0, canvas_width - 50)
		y = random.randint(0, canvas_height - 50)
		log.info('Current RGB color - ({0}, {1}, {2})'.format(r, g, b))
		log.info('Current coordinates - ({0}, {1})'.format(x, y))
		change_color(driver, r, g, b)
		# Change opacity and size randomly.
		sliders = driver.find_elements_by_class_name('sliderProgress')
		# Opacity.
		opacity = random.randint(MIN_OPACITY, MAX_OPACITY)
		change_slider(driver, sliders[0], opacity)
		# Size.
		size = random.randint(MIN_SIZE, MAX_SIZE)
		change_slider(driver, sliders[1], size)
		log.info('Opacity: {0}, Size: {1}'.format(opacity, size))
		# Paint!
		paint(driver, canvas_element, x, y)

if __name__ == '__main__':
	main()<commit_msg>Forgot some debug notes.<commit_after>import sys
import time
import math
import random
import logbook
import colorsys
import zmq
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import MoveTargetOutOfBoundsException

# Server related constants.
DEFAULT_LISTENING_PORT = '5556'
STOP_MESSAGE = 'STOP'
OK_REPLY = 'OK'
COLORS_DELIMITER = ','

# A JS script which removes a given element.
REMOVE_SCRIPT = 'var element = document.querySelector("{0}");' \
                'if (element) element.parentNode.removeChild(element);'

# A JS script which modifies a specific value in all elements of a given class.
MODIFY_SCRIPT = 'var list = document.getElementsByClassName("{0}");' \
                'for (var i = 0; i < list.length; i++) {{ ' \
                'list[i].{1} = "{2}" }};'

# The size of one part of the color square.
COLOR_SQUARE_SIZE = 68
# Sliders settings.
MIN_OPACITY = 30
MAX_OPACITY = 118
MIN_SIZE = 5
MAX_SIZE = 22

log = logbook.Logger('DeviantArt')

def remove_class(driver, class_name):
    """
    Removes the element with the given class.
    """
    driver.execute_script(REMOVE_SCRIPT.format('.' + class_name))

def remove_id(driver, id_name):
    """
    Removes the element with the given ID.
    """
    driver.execute_script(REMOVE_SCRIPT.format('#' + id_name))

def change_canvas_size(driver, class_name, width, height, only_style=False):
    """
    Changes the size of a canvas, to the desired width and height.

    :param class_name: The canvas' class name.
    :param width: The desired width.
    :param height: The desired height.
    :param only_style: If True, the width and height will be changed in the style attribute only.
    """
    driver.execute_script(MODIFY_SCRIPT.format(class_name, 'style.width', str(width) + 'px'))
    driver.execute_script(MODIFY_SCRIPT.format(class_name, 'style.height', str(height) + 'px'))
    if not only_style:
        driver.execute_script(MODIFY_SCRIPT.format(class_name, 'width', str(width)))
        driver.execute_script(MODIFY_SCRIPT.format(class_name, 'height', str(height)))

def change_slider(driver, slider_element, value):
    """
    Changes one of the 3 option sliders.

    :param slider_element: The slider element to change.
    :param value: The desired value.
    """
    action = webdriver.common.action_chains.ActionChains(driver)
    action.move_to_element_with_offset(slider_element, value, 0)
    action.click()
    action.perform()

def change_color(driver, r, g, b):
    """
    Changes the color according to the given RGB values.
    """
    h, s, v = colorsys.rgb_to_hsv(float(r), float(g), float(b))
    # Change hue.
    r = COLOR_SQUARE_SIZE / 2
    # The hue angle has an offset of 45 degrees.
    teta = math.radians(h * 360 - 45)
    # Add the center coordinates.
    x = r * math.cos(teta) + COLOR_SQUARE_SIZE / 2
    y = -r * math.sin(teta) + COLOR_SQUARE_SIZE / 2
    hue_element = driver.find_element_by_class_name('hueClick')
    action = webdriver.common.action_chains.ActionChains(driver)
    action.move_to_element_with_offset(hue_element, x, y)
    action.click()
    action.perform()
    # Change saturation and value.
    x = s * COLOR_SQUARE_SIZE
    y = COLOR_SQUARE_SIZE - (v / 255.0) * COLOR_SQUARE_SIZE
    saturation_element = driver.find_element_by_class_name('satval_area')
    action = webdriver.common.action_chains.ActionChains(driver)
    action.move_to_element_with_offset(saturation_element, x, y)
    action.click()
    action.perform()

def paint(driver, canvas_element, x, y):
    """
    Paint in the given coordinates.
    """
    try:
        action = webdriver.common.action_chains.ActionChains(driver)
        action.move_to_element_with_offset(canvas_element, x, y)
        action.click_and_hold()
        for j in xrange(70):
            action.move_to_element_with_offset(canvas_element, x + (2 if j % 2 == 0 else -2), y)
        action.release()
        action.perform()
    except MoveTargetOutOfBoundsException:
        # If we accidentaly stepped out of bounds,
        # stop the current paint action and release the mouse button.
        action = webdriver.common.action_chains.ActionChains(driver)
        action.release()
        action.perform()

def setup():
	"""
	Setup the browser window and painting canvas.

	:return: The newly created web driver.
	"""
	# Set up the window.
	log.info('Loading the original website...')
	driver = webdriver.Firefox()
	driver.get('http://sta.sh/muro')
	driver.maximize_window()
	driver.find_element_by_tag_name('html').send_keys(Keys.F11)
	time.sleep(5)
	# Pick the right tool.
	log.info('Picking the splatter brush...')
	driver.find_element_by_class_name('iconSplatter').click()
	# Remove all unwanted side bars.
	log.info('Removing side bars...')
	remove_class(driver, 'sideBar')
	remove_class(driver, 'topArea')
	remove_id(driver, 'overhead-collect')
	# Set the density slider.
	log.info('Setting density value to 100%')
	sliders = driver.find_elements_by_class_name('sliderProgress')
	# Density.
	change_slider(driver, sliders[2], 120)
	# Maximize the canvas.
	log.info('Maximizing canvas size...')
	window_size = driver.get_window_size()
	canvas_width = window_size['width']
	canvas_height = window_size['height']
	change_canvas_size(driver, 'selectionCanvas', canvas_width, canvas_height)
	change_canvas_size(driver, 'drawPlzCanvas', canvas_width, canvas_height)
	change_canvas_size(driver, 'canvasBuffer', canvas_width, canvas_height)
	change_canvas_size(driver, 'stagingBuffer', canvas_width, canvas_height)
	change_canvas_size(driver, 'marchingAnts', canvas_width, canvas_height)
	change_canvas_size(driver, 'canvasPaint', canvas_width, canvas_height, only_style=True)
	# Modify specific styles and positions.
	driver.execute_script(MODIFY_SCRIPT.format('middleArea', 'style.paddingRight', '0px'))
	driver.execute_script(MODIFY_SCRIPT.format('middleArea', 'style.height', '100%'))
	driver.execute_script(MODIFY_SCRIPT.format('drawPlzCanvas', 'style.left', '0px'))
	driver.execute_script(MODIFY_SCRIPT.format('drawPlzCanvas', 'style.top', '0px'))
	driver.execute_script(MODIFY_SCRIPT.format('drawPlzCanvas', 'width', canvas_width))
	driver.execute_script('document.getElementsByClassName("canvasPaint")[1].width = %d;' % canvas_width)
	driver.execute_script('document.getElementsByClassName("canvasPaint")[1].height = %d;' % canvas_height)
	# Hide the mouse cursor.
	log.info('Hiding mouse reticule...')
	driver.execute_script(MODIFY_SCRIPT.format('cursorPreview', 'style.visibility', 'hidden'))
	return driver

def main():
	"""
	Paints based on colors received from the server.
	Usage: deviantArt.py [PORT]
	"""
	# Validate input parameters.
	port = DEFAULT_LISTENING_PORT if len(sys.argv) == 1 else sys.argv[1]
	# Connect to the server.
	context = zmq.Context()
	log.info('Waiting for painting requests (port: {0})...'.format(port))
	socket = context.socket(zmq.REP)
	socket.bind ('tcp://*:{0}'.format(port))
	# Initialize the website.
	driver = setup()
	# Find the canvas width and height.
	window_size = driver.get_window_size()
	canvas_width = window_size['width']
	canvas_height = window_size['height']
	# Find the main canvas element.
	canvas_element = driver.find_element_by_class_name('drawPlzCanvas')
	# Start painting!
	log.info('Let\'s paint!')
	while True:
		message = socket.recv()
		log.info('Received request: {0}'.format(message))
		socket.send(OK_REPLY)
		log.info('Sent {0} back'.format(OK_REPLY))
		if message == STOP_MESSAGE:
			log.info('Stopping!')
			break
		# Pick the requested color.
		r, g, b = message.split(COLORS_DELIMITER)
		# Don't get to close to the borders.
		x = random.randint(0, canvas_width - 50)
		y = random.randint(0, canvas_height - 50)
		log.info('Current RGB color - ({0}, {1}, {2})'.format(r, g, b))
		log.info('Current coordinates - ({0}, {1})'.format(x, y))
		change_color(driver, r, g, b)
		# Change opacity and size randomly.
		sliders = driver.find_elements_by_class_name('sliderProgress')
		# Opacity.
		opacity = random.randint(MIN_OPACITY, MAX_OPACITY)
		change_slider(driver, sliders[0], opacity)
		# Size.
		size = random.randint(MIN_SIZE, MAX_SIZE)
		change_slider(driver, sliders[1], size)
		log.info('Opacity: {0}, Size: {1}'.format(opacity, size))
		# Paint!
		paint(driver, canvas_element, x, y)

if __name__ == '__main__':
	main()<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2009-2020 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

# Note that this module may not have a docstring because any
# global variable defined here will override the global
# namespace of lino/__init__.py who includes it with execfile.

# This module is part of the Lino test suite.
# To test only this module:
#
#   $ python setup.py test -s tests.PackagesTests

import sys

SETUP_INFO = dict(
    name='lino',
    version='21.1.1',
    description="A framework for writing desktop-like web applications "
                "using Django and ExtJS or React",
    license='BSD-2-Clause',
    # obsoletes=['djangosite', 'north'],
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://www.lino-framework.org",
    # ~ test_suite = 'lino.test_apps',
    test_suite='tests')

install_requires = [
    # 'Sphinx',
    'Django',
    # 'atelier',
    'unipath', 'python_dateutil', 'dateparser',
    'etgen', # 'docutils',
    'Babel', 'lxml',
    'odfpy',
    'jinja2', 'pytidylib', 'PyYAML',
    'clint',  # lino.modlib.checkdata.management.commands
    'django-localflavor',  # lino.modlib.sepa
    'django-click',  # for passwd command
    'openpyxl',  # removed version requirement 'openpyxl==3.0.1',
    'html2text',
    'schedule']

install_requires.append('Sphinx')  # lino.utils.restify
install_requires.append('docutils')  # lino.modlib.memo.mixins
install_requires.append('beautifulsoup4')  # lino.modlib.memo.mixins
# install_requires.append('reportlab')
install_requires.append('weasyprint')
# install_requires.append('zodb')
install_requires.append('DateTime')
# install_requires.append('appy@svn+https://svn.forge.pallavi.be/appy-dev/dev1')
# 'django-mailbox@git+https://github.com/cylonoven/django-mailbox#egg=django-mailbox'
# SETUP_INFO.update(dependency_links=[
#     "svn+https://svn.forge.pallavi.be/appy-dev/dev1#egg=appy-dev"
    # "git+https://github.com/lino-framework/appypod.git@dbf123584cd9c5ef4a35e8efb9f489eaa54e26f2#egg=appy-dev"
# ])

SETUP_INFO.update(install_requires=install_requires)

# .. raw:: html
#
#

SETUP_INFO.update(long_description="""

Lino is a high-level framework for writing desktop-like customized database
applications based on `Django <https://www.djangoproject.com/>`_ and `Sencha
ExtJS <http://www.sencha.com/products/extjs/>`_. Lino applications are Django
projects where the application developer does not need to write any URLconf,
HTML nor CSS (`more <http://lino-framework.org/about/what.html>`__).

- This repository is considered an integral part of the Lino framework, which is
  documented as a whole in the `Lino Book
  <http://www.lino-framework.org/about/overview.html>`__.

- Your feedback is welcome.  Our `community page
  <http://www.lino-framework.org/community>`__ explains how to contact us.

- API changes to this repository are logged at
  http://www.lino-framework.org/changes/

""")

SETUP_INFO.update(classifiers="""\
Programming Language :: Python
Programming Language :: Python :: 3
Development Status :: 5 - Production/Stable
Environment :: Web Environment
Framework :: Django
Intended Audience :: Developers
Intended Audience :: System Administrators
License :: OSI Approved :: BSD License
Natural Language :: English
Natural Language :: French
Natural Language :: German
Operating System :: OS Independent
Topic :: Database :: Front-Ends
Topic :: Office/Business
Topic :: Software Development :: Libraries :: Application Frameworks""".splitlines())

SETUP_INFO.update(packages=[str(n) for n in """
lino
lino.api
lino.core
lino.core.auth
lino.fake_migrations
lino.mixins
lino.modlib
lino.modlib.about
lino.modlib.awesomeuploader
lino.modlib.blacklist
lino.modlib.bootstrap3
lino.modlib.changes
lino.modlib.comments
lino.modlib.comments.fixtures
lino.modlib.dashboard
lino.modlib.dupable
lino.modlib.export_excel
lino.modlib.extjs
lino.modlib.forms
lino.modlib.gfks
lino.modlib.gfks.fixtures
lino.modlib.ipdict
lino.modlib.jinja
lino.modlib.jinja.management
lino.modlib.jinja.management.commands
lino.modlib.importfilters
lino.modlib.languages
lino.modlib.languages.fixtures
lino.management
lino.management.commands
lino.modlib.odata
lino.modlib.memo
lino.modlib.office
lino.modlib.chat
lino.modlib.chat.fixtures
lino.modlib.checkdata
lino.modlib.checkdata.fixtures
lino.modlib.checkdata.management
lino.modlib.checkdata.management.commands
lino.modlib.publisher
lino.modlib.printing
lino.modlib.restful
lino.modlib.smtpd
lino.modlib.smtpd.management
lino.modlib.smtpd.management.commands
lino.modlib.notify
lino.modlib.notify.fixtures
lino.modlib.summaries
lino.modlib.summaries.fixtures
lino.modlib.summaries.management
lino.modlib.summaries.management.commands
lino.modlib.system
lino.modlib.tinymce
lino.modlib.tinymce.fixtures
lino.modlib.uploads
lino.modlib.users
lino.modlib.users.fixtures
lino.modlib.weasyprint
lino.modlib.wkhtmltopdf
lino.projects
lino.projects.std
lino.sphinxcontrib
lino.sphinxcontrib.logo
lino.utils
lino.utils.demonames
lino.utils.mldbc
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino': [
        ('**/sandbox/**', 'ignore', None),
        ('**/cache/**', 'ignore', None),
        ('**.py', 'python', None),
        ('**/linoweb.js', 'jinja2', None),
        # ~ ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
        # ~ ('**/templates/**.txt',  'genshi', {
        # ~ 'template_class': 'genshi.template:TextTemplate'
        # ~ })
    ],
})

# SETUP_INFO.update(package_data=dict())

SETUP_INFO.update(include_package_data=True)

# def add_package_data(package, *patterns):
#     package = str(package)
#     l = SETUP_INFO['package_data'].setdefault(package, [])
#     l += [str(x) for x in patterns]
#     l.extend(patterns)
#     return l

# add_package_data('lino.modlib.davlink', '*.jnlp')
# add_package_data('lino.utils', '*.ods')
# add_package_data('lino.config', '*.html')
# add_package_data('lino.config', '*.tmpl')
# add_package_data('lino.config', 'about/*.tmpl')
# add_package_data('lino.config', 'makedocs/*.tmpl')
# add_package_data('lino.modlib.bootstrap3', 'config/bootstrap3/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.js')
# add_package_data('lino.modlib.languages.fixtures', '*.tab')
# add_package_data('lino.modlib.notify', 'config/notify/*.eml')

# l = add_package_data('lino.locale')
# for lng in 'de es fr et nl pt pt-br'.split():
#     l.append('lino/locale/%s/LC_MESSAGES/*.mo' % lng)
<commit_msg>release to pypi<commit_after># -*- coding: UTF-8 -*-
# Copyright 2009-2020 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

# Note that this module may not have a docstring because any
# global variable defined here will override the global
# namespace of lino/__init__.py who includes it with execfile.

# This module is part of the Lino test suite.
# To test only this module:
#
#   $ python setup.py test -s tests.PackagesTests

import sys

SETUP_INFO = dict(
    name='lino',
    version='21.1.2',
    description="A framework for writing desktop-like web applications "
                "using Django and ExtJS or React",
    license='BSD-2-Clause',
    # obsoletes=['djangosite', 'north'],
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://www.lino-framework.org",
    # ~ test_suite = 'lino.test_apps',
    test_suite='tests')

install_requires = [
    # 'Sphinx',
    'Django',
    # 'atelier',
    'unipath', 'python_dateutil', 'dateparser',
    'etgen', # 'docutils',
    'Babel', 'lxml',
    'odfpy',
    'jinja2', 'pytidylib', 'PyYAML',
    'clint',  # lino.modlib.checkdata.management.commands
    'django-localflavor',  # lino.modlib.sepa
    'django-click',  # for passwd command
    'openpyxl',  # removed version requirement 'openpyxl==3.0.1',
    'html2text',
    'schedule']

install_requires.append('Sphinx')  # lino.utils.restify
install_requires.append('docutils')  # lino.modlib.memo.mixins
install_requires.append('beautifulsoup4')  # lino.modlib.memo.mixins
# install_requires.append('reportlab')
install_requires.append('weasyprint')
# install_requires.append('zodb')
install_requires.append('DateTime')
# install_requires.append('appy@svn+https://svn.forge.pallavi.be/appy-dev/dev1')
# 'django-mailbox@git+https://github.com/cylonoven/django-mailbox#egg=django-mailbox'
# SETUP_INFO.update(dependency_links=[
#     "svn+https://svn.forge.pallavi.be/appy-dev/dev1#egg=appy-dev"
    # "git+https://github.com/lino-framework/appypod.git@dbf123584cd9c5ef4a35e8efb9f489eaa54e26f2#egg=appy-dev"
# ])

SETUP_INFO.update(install_requires=install_requires)

# .. raw:: html
#
#

SETUP_INFO.update(long_description="""

Lino is a high-level framework for writing desktop-like customized database
applications based on `Django <https://www.djangoproject.com/>`_ and `Sencha
ExtJS <http://www.sencha.com/products/extjs/>`_. Lino applications are Django
projects where the application developer does not need to write any URLconf,
HTML nor CSS (`more <http://lino-framework.org/about/what.html>`__).

- This repository is considered an integral part of the Lino framework, which is
  documented as a whole in the `Lino Book
  <http://www.lino-framework.org/about/overview.html>`__.

- Your feedback is welcome.  Our `community page
  <http://www.lino-framework.org/community>`__ explains how to contact us.

- API changes to this repository are logged at
  http://www.lino-framework.org/changes/

""")

SETUP_INFO.update(classifiers="""\
Programming Language :: Python
Programming Language :: Python :: 3
Development Status :: 5 - Production/Stable
Environment :: Web Environment
Framework :: Django
Intended Audience :: Developers
Intended Audience :: System Administrators
License :: OSI Approved :: BSD License
Natural Language :: English
Natural Language :: French
Natural Language :: German
Operating System :: OS Independent
Topic :: Database :: Front-Ends
Topic :: Office/Business
Topic :: Software Development :: Libraries :: Application Frameworks""".splitlines())

SETUP_INFO.update(packages=[str(n) for n in """
lino
lino.api
lino.core
lino.core.auth
lino.fake_migrations
lino.mixins
lino.modlib
lino.modlib.about
lino.modlib.awesomeuploader
lino.modlib.blacklist
lino.modlib.bootstrap3
lino.modlib.changes
lino.modlib.comments
lino.modlib.comments.fixtures
lino.modlib.dashboard
lino.modlib.dupable
lino.modlib.export_excel
lino.modlib.extjs
lino.modlib.forms
lino.modlib.gfks
lino.modlib.gfks.fixtures
lino.modlib.ipdict
lino.modlib.jinja
lino.modlib.jinja.management
lino.modlib.jinja.management.commands
lino.modlib.importfilters
lino.modlib.languages
lino.modlib.languages.fixtures
lino.management
lino.management.commands
lino.modlib.odata
lino.modlib.memo
lino.modlib.office
lino.modlib.chat
lino.modlib.chat.fixtures
lino.modlib.checkdata
lino.modlib.checkdata.fixtures
lino.modlib.checkdata.management
lino.modlib.checkdata.management.commands
lino.modlib.publisher
lino.modlib.printing
lino.modlib.restful
lino.modlib.smtpd
lino.modlib.smtpd.management
lino.modlib.smtpd.management.commands
lino.modlib.notify
lino.modlib.notify.fixtures
lino.modlib.summaries
lino.modlib.summaries.fixtures
lino.modlib.summaries.management
lino.modlib.summaries.management.commands
lino.modlib.system
lino.modlib.tinymce
lino.modlib.tinymce.fixtures
lino.modlib.uploads
lino.modlib.users
lino.modlib.users.fixtures
lino.modlib.weasyprint
lino.modlib.wkhtmltopdf
lino.projects
lino.projects.std
lino.sphinxcontrib
lino.sphinxcontrib.logo
lino.utils
lino.utils.demonames
lino.utils.mldbc
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino': [
        ('**/sandbox/**', 'ignore', None),
        ('**/cache/**', 'ignore', None),
        ('**.py', 'python', None),
        ('**/linoweb.js', 'jinja2', None),
        # ~ ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
        # ~ ('**/templates/**.txt',  'genshi', {
        # ~ 'template_class': 'genshi.template:TextTemplate'
        # ~ })
    ],
})

# SETUP_INFO.update(package_data=dict())

SETUP_INFO.update(include_package_data=True)

# def add_package_data(package, *patterns):
#     package = str(package)
#     l = SETUP_INFO['package_data'].setdefault(package, [])
#     l += [str(x) for x in patterns]
#     l.extend(patterns)
#     return l

# add_package_data('lino.modlib.davlink', '*.jnlp')
# add_package_data('lino.utils', '*.ods')
# add_package_data('lino.config', '*.html')
# add_package_data('lino.config', '*.tmpl')
# add_package_data('lino.config', 'about/*.tmpl')
# add_package_data('lino.config', 'makedocs/*.tmpl')
# add_package_data('lino.modlib.bootstrap3', 'config/bootstrap3/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.js')
# add_package_data('lino.modlib.languages.fixtures', '*.tab')
# add_package_data('lino.modlib.notify', 'config/notify/*.eml')

# l = add_package_data('lino.locale')
# for lng in 'de es fr et nl pt pt-br'.split():
#     l.append('lino/locale/%s/LC_MESSAGES/*.mo' % lng)
<|endoftext|>


<commit_before>
import time
import os

from devops.model import Environment, Node, Network, Interface
from devops.controller import Controller
from devops.driver.libvirt import Libvirt

from devops import yaml_config_loader

INSTALLATION_ISO = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'install.iso'))
INSTALLATION_ISO_URL = "http://mc0n1-srt.srt.mirantis.net/nailgun-ubuntu-12.04-amd64.last.iso"

MASTER_AND_SLAVE_CONFIG = """
name: 'Sample environment'
networks:
  - network: internal
  - network: external
    type: bridged
nodes:
  - node: gateway
    disk: '5Gb'
    cdrom: '%s'
    networks: ['external', 'internal']
  - node: slug
    networks: ['internal']
""" % (INSTALLATION_ISO,)

def download_iso():
    if not os.path.exists(INSTALLATION_ISO):
        print("Downloading installation iso")
        os.system("wget -O '%s' -c '%s'" % (INSTALLATION_ISO, INSTALLATION_ISO_URL))
        print("Finished downloading installation iso")

def main():
    download_iso()

    environment = yaml_config_loader.load(MASTER_AND_SLAVE_CONFIG)

    external_network = environment.network['external']

    gateway = environment.node['gateway']
    gateway.vnc = True

    print("Creating environment")

    controller = Controller(Libvirt())
    controller.build_environment(environment)

    print("Environment created")

    print("Starting node")
    gateway.start()

    print("VNC is available on vnc://localhost:%d" % gateway.vnc_port)

    print("Waiting node to boot")
    time.sleep(15)

    print("Sending user input")

    ip = external_network.ip_addresses

    gateway.send_keys("""<Esc><Enter>
<Wait>
/install/vmlinuz initrd=/install/initrd.gz
 priority=critical
 locale=en_US
 file=/cdrom/preseed/manual.seed
 vga=788
 netcfg/get_ipaddress=%s
 netcfg/get_netmask=%s
 netcfg/get_gateway=%s
 netcfg/get_nameservers=%s
 netcfg/confirm_static=true
 <Enter>""" % (ip[2], ip.netmask, ip[1], ip[1]))
    print("Finished sending user input")

if __name__ == '__main__':
    main()

<commit_msg>[devops] Updated run.py sample script to start second node and wait for ipaddress<commit_after>import time

import devops
from devops.helpers import wait, tcp_ping

import logging

MASTER_AND_SLAVE_CONFIG = """
    name: 'Sample environment'
    networks:
      - network: internal
      - network: external
    nodes:
      - node: master
        disk: '5Gb'
        cdrom: http://mc0n1-srt.srt.mirantis.net/nailgun-ubuntu-12.04-amd64.last.iso
        networks: ['external', 'internal']
        vnc: True
      - node: slave
        networks: ['internal']
        vnc: True
"""

def main():
    logging.basicConfig(level=logging.WARN)
    logger = logging.getLogger('test.integration')
    logger.setLevel(logging.INFO)

    environment = devops.load(MASTER_AND_SLAVE_CONFIG)

    logger.info("Building environment")

    devops.build(environment)

    logger.info("Environment ready")

    external_network = environment.network['external']

    master_node = environment.node['master']
    slave_node  = environment.node['slave']

    logger.info("Starting master node")
    master_node.start()

    logger.info("VNC to master is available on %d" % master_node.vnc_port)

    logger.info("Waiting master node to boot")
    time.sleep(15)

    logger.info("Sending user input")

    ip = external_network.ip_addresses
    host_ip   = ip[1]
    master_ip = ip[2]
    netmask   = ip.netmask

    master_node.send_keys("""<Esc><Enter>
<Wait>
/install/vmlinuz initrd=/install/initrd.gz
 priority=critical
 locale=en_US
 file=/cdrom/preseed/manual.seed
 vga=788
 netcfg/get_ipaddress=%s
 netcfg/get_netmask=%s
 netcfg/get_gateway=%s
 netcfg/get_nameservers=%s
 netcfg/confirm_static=true
 <Enter>""" % (master_ip, netmask, host_ip, host_ip))
    logger.info("Finished sending user input")

    logger.info("Waiting master node to install")
    wait(lambda: tcp_ping(master_ip, 22))

    logger.info("Starting slave node")

    slave_node.start()

    logger.info("VNC to slave node at port %d" % slave_node.vnc_port)

    logger.info("Waiting slave node to configure network")

    wait(lambda: len(slave_node.ip_addresses) > 0, timeout=120)

    logger.info("Slave node has IP address %s" % slave_node.ip_addresses[0])

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2009-2021 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

# Note that this module may not have a docstring because any
# global variable defined here will override the global
# namespace of lino/__init__.py who includes it with execfile.

# This module is part of the Lino test suite.
# To test only this module:
#
#   $ python setup.py test -s tests.PackagesTests

import sys

SETUP_INFO = dict(
    name='lino',
    version='21.2.3',
    description="A framework for writing desktop-like web applications "
                "using Django and ExtJS or React",
    license='BSD-2-Clause',
    # obsoletes=['djangosite', 'north'],
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://www.lino-framework.org",
    # ~ test_suite = 'lino.test_apps',
    test_suite='tests')

install_requires = [
    # 'Sphinx',
    'Django',
    # 'atelier',
    'unipath', 'python_dateutil', 'dateparser',
    'etgen', # 'docutils',
    'Babel', 'lxml',
    'odfpy',
    'jinja2', 'pytidylib', 'PyYAML',
    'clint',  # lino.modlib.checkdata.management.commands
    'django-localflavor',  # lino.modlib.sepa
    'django-click',  # for passwd command
    'openpyxl',  # removed version requirement 'openpyxl==3.0.1',
    'html2text',
    'schedule']

install_requires.append('Sphinx')  # lino.utils.restify
install_requires.append('docutils')  # lino.modlib.memo.mixins
install_requires.append('beautifulsoup4')  # lino.modlib.memo.mixins
# install_requires.append('reportlab')
install_requires.append('weasyprint')
# install_requires.append('zodb')
install_requires.append('DateTime')
# install_requires.append('appy@svn+https://svn.forge.pallavi.be/appy-dev/dev1')
# 'django-mailbox@git+https://github.com/cylonoven/django-mailbox#egg=django-mailbox'
# SETUP_INFO.update(dependency_links=[
#     "svn+https://svn.forge.pallavi.be/appy-dev/dev1#egg=appy-dev"
    # "git+https://github.com/lino-framework/appypod.git@dbf123584cd9c5ef4a35e8efb9f489eaa54e26f2#egg=appy-dev"
# ])

SETUP_INFO.update(install_requires=install_requires)

# .. raw:: html
#
#

SETUP_INFO.update(long_description="""

Lino is a high-level framework for writing desktop-like customized database
applications based on `Django <https://www.djangoproject.com/>`_ and `Sencha
ExtJS <http://www.sencha.com/products/extjs/>`_. Lino applications are Django
projects where the application developer does not need to write any URLconf,
HTML nor CSS (`more <http://lino-framework.org/about/what.html>`__).

- This repository is considered an integral part of the Lino framework, which is
  documented as a whole in the `Lino Book
  <http://www.lino-framework.org/about/overview.html>`__.

- Your feedback is welcome.  Our `community page
  <http://www.lino-framework.org/community>`__ explains how to contact us.

- API changes to this repository are logged at
  http://www.lino-framework.org/changes/

""")

SETUP_INFO.update(classifiers="""\
Programming Language :: Python
Programming Language :: Python :: 3
Development Status :: 5 - Production/Stable
Environment :: Web Environment
Framework :: Django
Intended Audience :: Developers
Intended Audience :: System Administrators
License :: OSI Approved :: BSD License
Natural Language :: English
Natural Language :: French
Natural Language :: German
Operating System :: OS Independent
Topic :: Database :: Front-Ends
Topic :: Office/Business
Topic :: Software Development :: Libraries :: Application Frameworks""".splitlines())

SETUP_INFO.update(packages=[str(n) for n in """
lino
lino.api
lino.core
lino.core.auth
lino.fake_migrations
lino.mixins
lino.modlib
lino.modlib.about
lino.modlib.awesomeuploader
lino.modlib.blacklist
lino.modlib.bootstrap3
lino.modlib.changes
lino.modlib.comments
lino.modlib.comments.fixtures
lino.modlib.dashboard
lino.modlib.dupable
lino.modlib.export_excel
lino.modlib.extjs
lino.modlib.forms
lino.modlib.gfks
lino.modlib.gfks.fixtures
lino.modlib.ipdict
lino.modlib.jinja
lino.modlib.jinja.management
lino.modlib.jinja.management.commands
lino.modlib.importfilters
lino.modlib.languages
lino.modlib.languages.fixtures
lino.management
lino.management.commands
lino.modlib.odata
lino.modlib.memo
lino.modlib.office
lino.modlib.chat
lino.modlib.chat.fixtures
lino.modlib.checkdata
lino.modlib.checkdata.fixtures
lino.modlib.checkdata.management
lino.modlib.checkdata.management.commands
lino.modlib.publisher
lino.modlib.printing
lino.modlib.restful
lino.modlib.smtpd
lino.modlib.smtpd.management
lino.modlib.smtpd.management.commands
lino.modlib.notify
lino.modlib.notify.fixtures
lino.modlib.summaries
lino.modlib.summaries.fixtures
lino.modlib.summaries.management
lino.modlib.summaries.management.commands
lino.modlib.system
lino.modlib.tinymce
lino.modlib.tinymce.fixtures
lino.modlib.uploads
lino.modlib.uploads.fixtures
lino.modlib.users
lino.modlib.users.fixtures
lino.modlib.weasyprint
lino.modlib.wkhtmltopdf
lino.projects
lino.projects.std
lino.sphinxcontrib
lino.sphinxcontrib.logo
lino.utils
lino.utils.demonames
lino.utils.mldbc
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino': [
        ('**/sandbox/**', 'ignore', None),
        ('**/cache/**', 'ignore', None),
        ('**.py', 'python', None),
        ('**/linoweb.js', 'jinja2', None),
        # ~ ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
        # ~ ('**/templates/**.txt',  'genshi', {
        # ~ 'template_class': 'genshi.template:TextTemplate'
        # ~ })
    ],
})

# SETUP_INFO.update(package_data=dict())

SETUP_INFO.update(include_package_data=True)

# def add_package_data(package, *patterns):
#     package = str(package)
#     l = SETUP_INFO['package_data'].setdefault(package, [])
#     l += [str(x) for x in patterns]
#     l.extend(patterns)
#     return l

# add_package_data('lino.modlib.davlink', '*.jnlp')
# add_package_data('lino.utils', '*.ods')
# add_package_data('lino.config', '*.html')
# add_package_data('lino.config', '*.tmpl')
# add_package_data('lino.config', 'about/*.tmpl')
# add_package_data('lino.config', 'makedocs/*.tmpl')
# add_package_data('lino.modlib.bootstrap3', 'config/bootstrap3/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.js')
# add_package_data('lino.modlib.languages.fixtures', '*.tab')
# add_package_data('lino.modlib.notify', 'config/notify/*.eml')

# l = add_package_data('lino.locale')
# for lng in 'de es fr et nl pt pt-br'.split():
#     l.append('lino/locale/%s/LC_MESSAGES/*.mo' % lng)
<commit_msg>release to pypi<commit_after># -*- coding: UTF-8 -*-
# Copyright 2009-2021 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

# Note that this module may not have a docstring because any
# global variable defined here will override the global
# namespace of lino/__init__.py who includes it with execfile.

# This module is part of the Lino test suite.
# To test only this module:
#
#   $ python setup.py test -s tests.PackagesTests

import sys

SETUP_INFO = dict(
    name='lino',
    version='21.2.4',
    description="A framework for writing desktop-like web applications "
                "using Django and ExtJS or React",
    license='BSD-2-Clause',
    # obsoletes=['djangosite', 'north'],
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://www.lino-framework.org",
    # ~ test_suite = 'lino.test_apps',
    test_suite='tests')

install_requires = [
    # 'Sphinx',
    'Django',
    # 'atelier',
    'unipath', 'python_dateutil', 'dateparser',
    'etgen', # 'docutils',
    'Babel', 'lxml',
    'odfpy',
    'jinja2', 'pytidylib', 'PyYAML',
    'clint',  # lino.modlib.checkdata.management.commands
    'django-localflavor',  # lino.modlib.sepa
    'django-click',  # for passwd command
    'openpyxl',  # removed version requirement 'openpyxl==3.0.1',
    'html2text',
    'schedule']

install_requires.append('Sphinx')  # lino.utils.restify
install_requires.append('docutils')  # lino.modlib.memo.mixins
install_requires.append('beautifulsoup4')  # lino.modlib.memo.mixins
# install_requires.append('reportlab')
install_requires.append('weasyprint')
# install_requires.append('zodb')
install_requires.append('DateTime')
# install_requires.append('appy@svn+https://svn.forge.pallavi.be/appy-dev/dev1')
# 'django-mailbox@git+https://github.com/cylonoven/django-mailbox#egg=django-mailbox'
# SETUP_INFO.update(dependency_links=[
#     "svn+https://svn.forge.pallavi.be/appy-dev/dev1#egg=appy-dev"
    # "git+https://github.com/lino-framework/appypod.git@dbf123584cd9c5ef4a35e8efb9f489eaa54e26f2#egg=appy-dev"
# ])

SETUP_INFO.update(install_requires=install_requires)

# .. raw:: html
#
#

SETUP_INFO.update(long_description="""

Lino is a high-level framework for writing desktop-like customized database
applications based on `Django <https://www.djangoproject.com/>`_ and `Sencha
ExtJS <http://www.sencha.com/products/extjs/>`_. Lino applications are Django
projects where the application developer does not need to write any URLconf,
HTML nor CSS (`more <http://lino-framework.org/about/what.html>`__).

- This repository is considered an integral part of the Lino framework, which is
  documented as a whole in the `Lino Book
  <http://www.lino-framework.org/about/overview.html>`__.

- Your feedback is welcome.  Our `community page
  <http://www.lino-framework.org/community>`__ explains how to contact us.

- API changes to this repository are logged at
  http://www.lino-framework.org/changes/

""")

SETUP_INFO.update(classifiers="""\
Programming Language :: Python
Programming Language :: Python :: 3
Development Status :: 5 - Production/Stable
Environment :: Web Environment
Framework :: Django
Intended Audience :: Developers
Intended Audience :: System Administrators
License :: OSI Approved :: BSD License
Natural Language :: English
Natural Language :: French
Natural Language :: German
Operating System :: OS Independent
Topic :: Database :: Front-Ends
Topic :: Office/Business
Topic :: Software Development :: Libraries :: Application Frameworks""".splitlines())

SETUP_INFO.update(packages=[str(n) for n in """
lino
lino.api
lino.core
lino.core.auth
lino.fake_migrations
lino.mixins
lino.modlib
lino.modlib.about
lino.modlib.awesomeuploader
lino.modlib.blacklist
lino.modlib.bootstrap3
lino.modlib.changes
lino.modlib.comments
lino.modlib.comments.fixtures
lino.modlib.dashboard
lino.modlib.dupable
lino.modlib.export_excel
lino.modlib.extjs
lino.modlib.forms
lino.modlib.gfks
lino.modlib.gfks.fixtures
lino.modlib.ipdict
lino.modlib.jinja
lino.modlib.jinja.management
lino.modlib.jinja.management.commands
lino.modlib.importfilters
lino.modlib.languages
lino.modlib.languages.fixtures
lino.management
lino.management.commands
lino.modlib.odata
lino.modlib.memo
lino.modlib.office
lino.modlib.chat
lino.modlib.chat.fixtures
lino.modlib.checkdata
lino.modlib.checkdata.fixtures
lino.modlib.checkdata.management
lino.modlib.checkdata.management.commands
lino.modlib.publisher
lino.modlib.printing
lino.modlib.restful
lino.modlib.smtpd
lino.modlib.smtpd.management
lino.modlib.smtpd.management.commands
lino.modlib.notify
lino.modlib.notify.fixtures
lino.modlib.summaries
lino.modlib.summaries.fixtures
lino.modlib.summaries.management
lino.modlib.summaries.management.commands
lino.modlib.system
lino.modlib.tinymce
lino.modlib.tinymce.fixtures
lino.modlib.uploads
lino.modlib.uploads.fixtures
lino.modlib.users
lino.modlib.users.fixtures
lino.modlib.weasyprint
lino.modlib.wkhtmltopdf
lino.projects
lino.projects.std
lino.sphinxcontrib
lino.sphinxcontrib.logo
lino.utils
lino.utils.demonames
lino.utils.mldbc
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino': [
        ('**/sandbox/**', 'ignore', None),
        ('**/cache/**', 'ignore', None),
        ('**.py', 'python', None),
        ('**/linoweb.js', 'jinja2', None),
        # ~ ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
        # ~ ('**/templates/**.txt',  'genshi', {
        # ~ 'template_class': 'genshi.template:TextTemplate'
        # ~ })
    ],
})

# SETUP_INFO.update(package_data=dict())

SETUP_INFO.update(include_package_data=True)

# def add_package_data(package, *patterns):
#     package = str(package)
#     l = SETUP_INFO['package_data'].setdefault(package, [])
#     l += [str(x) for x in patterns]
#     l.extend(patterns)
#     return l

# add_package_data('lino.modlib.davlink', '*.jnlp')
# add_package_data('lino.utils', '*.ods')
# add_package_data('lino.config', '*.html')
# add_package_data('lino.config', '*.tmpl')
# add_package_data('lino.config', 'about/*.tmpl')
# add_package_data('lino.config', 'makedocs/*.tmpl')
# add_package_data('lino.modlib.bootstrap3', 'config/bootstrap3/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.html')
# add_package_data('lino.modlib.extjs', 'config/extjs/*.js')
# add_package_data('lino.modlib.languages.fixtures', '*.tab')
# add_package_data('lino.modlib.notify', 'config/notify/*.eml')

# l = add_package_data('lino.locale')
# for lng in 'de es fr et nl pt pt-br'.split():
#     l.append('lino/locale/%s/LC_MESSAGES/*.mo' % lng)
<|endoftext|>


<commit_before>import logging
from datetime import datetime, timedelta
from dateutil.parser import parse
log = logging.getLogger(__name__)

from flask import request, url_for, redirect, jsonify
from flask.ext.login import login_required
from flask.ext import htauth
from sqlalchemy.orm import joinedload, lazyload
from sqlalchemy.sql import func

from .app import app
from .models import db, Author, Person, Entity, Document, DocumentSource
from .processing import BiasCalculator

@app.route('/api/authors')
@login_required
def api_authors():
    authors = Author.query.order_by(Author.name).all()
    return jsonify({'authors': [a.json() for a in authors]})

@app.route('/api/people')
@login_required
def api_people():
    people = Person.query\
        .options(joinedload(Person.affiliation))\
        .order_by(Person.name)\
        .all()
    return jsonify({'people': [p.json() for p in people]})

@app.route('/api/entities')
@login_required
def api_entities():
    query = Entity.query.order_by(Entity.name)
    q = request.args.get('q', '').strip()
    if q:
        q = '%' + q.replace('%', '%%').replace(' ', '%') + '%'
        query = query.filter(Entity.name.like(q))

    entities = query.all()

    return jsonify({'entities': [e.json() for e in entities]})

@app.route('/api/entities/<string:group>')
@login_required
def api_group_entities(group):
    query = Entity.query.filter(Entity.group == group).order_by(Entity.name)
    q = request.args.get('q', '').strip()
    if q:
        q = '%' + q.replace('%', '%%').replace(' ', '%') + '%'
        query = query.filter(Entity.name.like(q))

    entities = query.all()

    return jsonify({'entities': [e.json() for e in entities]})

@app.route('/api/feeds/sources/political-parties')
@htauth.authenticated
def api_feed_sources():
    from dexter.models.views import DocumentSourcesView, DocumentsView

    start_date, end_date = api_date_range(request)

    query = db.session.query(
                func.count(DocumentSourcesView.c.document_id).label("record_count"),
                DocumentSourcesView.c.affiliation.label("affiliation"),
                DocumentSourcesView.c.source_name.label("source_name"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .join(DocumentsView, DocumentSourcesView.c.document_id == DocumentsView.c.document_id)\
            .group_by(
                DocumentSourcesView.c.affiliation.label("affiliation"),
                DocumentSourcesView.c.source_name.label("source_name"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .filter(DocumentSourcesView.c.affiliation_code.like('4.%'))\
            .filter(DocumentsView.c.published_at >= start_date)\
            .filter(DocumentsView.c.published_at <= end_date)

    # {
    #   "date-start":"2014-04-03",
    #   "date-end":"2014-04-05",
    #   "cells": [
    #     {
    #       "affiliation": null,
    #       "record_count": 1,
    #       "source_name": "Narend Singh",
    #       "medium_group": "Beeld"
    #     },
    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": [r._asdict() for r in query.all()]
    }

    return jsonify(results)

@app.route('/api/feeds/topics')
@htauth.authenticated
def api_feed_topics():
    from dexter.models.views import DocumentSourcesView, DocumentsView

    start_date, end_date = api_date_range(request)

    query = db.session.query(
                func.count(DocumentsView.c.document_id).label("record_count"),
                DocumentsView.c.topic.label("topic"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .group_by(
                DocumentsView.c.topic.label("topic"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .filter(DocumentsView.c.published_at >= start_date)\
            .filter(DocumentsView.c.published_at <= end_date)

    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": [r._asdict() for r in query.all()]
    }

    return jsonify(results)

@app.route('/api/feeds/origins')
@htauth.authenticated
def api_feed_origins():
    from dexter.models.views import DocumentSourcesView, DocumentsView

    start_date, end_date = api_date_range(request)

    query = db.session.query(
                func.count(DocumentsView.c.document_id).label("record_count"),
                DocumentsView.c.origin.label("origin"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .group_by(
                DocumentsView.c.origin.label("origin"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .filter(DocumentsView.c.published_at >= start_date)\
            .filter(DocumentsView.c.published_at <= end_date)

    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": [r._asdict() for r in query.all()]
    }

    return jsonify(results)

@app.route('/api/feeds/bias')
@htauth.authenticated
def api_feed_bias():
    start_date, end_date = api_date_range(request)

    log.info("Loading documents for bias calculation")
    documents = Document.query\
            .options(
                joinedload(Document.sources),
                joinedload(Document.fairness),
                joinedload(Document.medium),
                lazyload('sources.person'),
                lazyload('sources.unnamed_gender'),
                lazyload('sources.unnamed_race'))\
            .filter(Document.published_at >= start_date)\
            .filter(Document.published_at <= end_date).all()
    log.info("Loaded %d docs" % len(documents))

    scores = BiasCalculator().calculate_bias_scores(documents, key=lambda d: (d.medium.group_name(), d.medium.medium_type))

    cells = []
    for score in scores:
        cell = score.asdict()
        cell['medium_group'] = score.group[0]
        cell['medium_type'] = score.group[1]
        cells.append(cell)

    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": cells,
    }

    return jsonify(results)

def api_date_range(request):
    end_date = datetime.utcnow()
    try:
        end_date = parse(request.args.get('end-date', ''), default=end_date, yearfirst=True)
    except ValueError:
        pass

    start_date = end_date - timedelta(days=7)
    try:
        start_date = parse(request.args.get('start-date', ''), default=start_date, yearfirst=True)
    except ValueError:
        pass

    end_date = end_date.strftime("%Y/%m/%d")
    start_date = start_date.strftime("%Y/%m/%d")

    return (start_date, end_date)
<commit_msg>Add /api/feeds/metadata JSON feed.<commit_after>import logging
import urlparse
from datetime import datetime, timedelta
from dateutil.parser import parse
log = logging.getLogger(__name__)

from flask import request, url_for, redirect, jsonify
from flask.ext.login import login_required
from flask.ext import htauth
from sqlalchemy.orm import joinedload, lazyload
from sqlalchemy.sql import func

from .app import app
from .models import db, Author, Person, Entity, Document, DocumentSource, Medium, Location, Topic, Affiliation
from .processing import BiasCalculator

@app.route('/api/authors')
@login_required
def api_authors():
    authors = Author.query.order_by(Author.name).all()
    return jsonify({'authors': [a.json() for a in authors]})

@app.route('/api/people')
@login_required
def api_people():
    people = Person.query\
        .options(joinedload(Person.affiliation))\
        .order_by(Person.name)\
        .all()
    return jsonify({'people': [p.json() for p in people]})

@app.route('/api/entities')
@login_required
def api_entities():
    query = Entity.query.order_by(Entity.name)
    q = request.args.get('q', '').strip()
    if q:
        q = '%' + q.replace('%', '%%').replace(' ', '%') + '%'
        query = query.filter(Entity.name.like(q))

    entities = query.all()

    return jsonify({'entities': [e.json() for e in entities]})

@app.route('/api/entities/<string:group>')
@login_required
def api_group_entities(group):
    query = Entity.query.filter(Entity.group == group).order_by(Entity.name)
    q = request.args.get('q', '').strip()
    if q:
        q = '%' + q.replace('%', '%%').replace(' ', '%') + '%'
        query = query.filter(Entity.name.like(q))

    entities = query.all()

    return jsonify({'entities': [e.json() for e in entities]})

@app.route('/api/feeds/sources/political-parties')
@htauth.authenticated
def api_feed_parties():
    from dexter.models.views import DocumentSourcesView, DocumentsView

    start_date, end_date = api_date_range(request)

    query = db.session.query(
                func.count(DocumentSourcesView.c.document_id).label("record_count"),
                DocumentSourcesView.c.affiliation.label("affiliation"),
                DocumentSourcesView.c.source_name.label("source_name"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .join(DocumentsView, DocumentSourcesView.c.document_id == DocumentsView.c.document_id)\
            .group_by(
                DocumentSourcesView.c.affiliation.label("affiliation"),
                DocumentSourcesView.c.source_name.label("source_name"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .filter(DocumentSourcesView.c.affiliation_code.like('4.%'))\
            .filter(DocumentsView.c.published_at >= start_date)\
            .filter(DocumentsView.c.published_at <= end_date)

    # {
    #   "date-start":"2014-04-03",
    #   "date-end":"2014-04-05",
    #   "cells": [
    #     {
    #       "affiliation": null,
    #       "record_count": 1,
    #       "source_name": "Narend Singh",
    #       "medium_group": "Beeld"
    #     },
    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": [r._asdict() for r in query.all()]
    }

    return jsonify(results)

@app.route('/api/feeds/topics')
@htauth.authenticated
def api_feed_topics():
    from dexter.models.views import DocumentSourcesView, DocumentsView

    start_date, end_date = api_date_range(request)

    query = db.session.query(
                func.count(DocumentsView.c.document_id).label("record_count"),
                DocumentsView.c.topic.label("topic"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .group_by(
                DocumentsView.c.topic.label("topic"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .filter(DocumentsView.c.published_at >= start_date)\
            .filter(DocumentsView.c.published_at <= end_date)

    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": [r._asdict() for r in query.all()]
    }

    return jsonify(results)

@app.route('/api/feeds/origins')
@htauth.authenticated
def api_feed_origins():
    from dexter.models.views import DocumentSourcesView, DocumentsView

    start_date, end_date = api_date_range(request)

    query = db.session.query(
                func.count(DocumentsView.c.document_id).label("record_count"),
                DocumentsView.c.origin.label("origin"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .group_by(
                DocumentsView.c.origin.label("origin"),
                DocumentsView.c.medium_group.label("medium_group"),
                DocumentsView.c.medium_type.label("medium_type"),
            )\
            .filter(DocumentsView.c.published_at >= start_date)\
            .filter(DocumentsView.c.published_at <= end_date)

    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": [r._asdict() for r in query.all()]
    }

    return jsonify(results)

@app.route('/api/feeds/bias')
@htauth.authenticated
def api_feed_bias():
    start_date, end_date = api_date_range(request)

    log.info("Loading documents for bias calculation")
    documents = Document.query\
            .options(
                joinedload(Document.sources),
                joinedload(Document.fairness),
                joinedload(Document.medium),
                lazyload('sources.person'),
                lazyload('sources.unnamed_gender'),
                lazyload('sources.unnamed_race'))\
            .filter(Document.published_at >= start_date)\
            .filter(Document.published_at <= end_date).all()
    log.info("Loaded %d docs" % len(documents))

    scores = BiasCalculator().calculate_bias_scores(documents, key=lambda d: (d.medium.group_name(), d.medium.medium_type))

    cells = []
    for score in scores:
        cell = score.asdict()
        cell['medium_group'] = score.group[0]
        cell['medium_type'] = score.group[1]
        cells.append(cell)

    results = {
        "date-start": start_date,
        "date-end": end_date,
        "cells": cells,
    }

    return jsonify(results)

@app.route('/api/feeds/metadata')
@htauth.authenticated
def api_feed_metadata():
    data = {}

    media = Medium.query.all()
    data['media'] = {
        "bias_feed_url": "%s" % urlparse.urljoin(request.url_root, url_for('api_feed_bias')),
        "names": [m.name for m in media],
        "groups": list(set(m.group_name() for m in media)),
    }

    data['origins'] = {
        "names": [x.name for x in Location.query.all()],
        "feed_url": "%s" % urlparse.urljoin(request.url_root, url_for('api_feed_origins')),
    }

    data['topics'] = {
        "names": [x.name for x in Topic.query.all()],
        "feed_url": "%s" % urlparse.urljoin(request.url_root, url_for('api_feed_topics')),
    }

    data['affiliations'] = {
        "names": [x.name for x in Affiliation.query.all()],
    }

    data['political-parties'] = {
        "names": [x.name for x in Affiliation.query.filter(Affiliation.code.like("4.%")).all()],
        "feed_url": "%s" % urlparse.urljoin(request.url_root, url_for('api_feed_parties')),
    }

    return jsonify({"metadata": data})

def api_date_range(request):
    end_date = datetime.utcnow()
    try:
        end_date = parse(request.args.get('end-date', ''), default=end_date, yearfirst=True)
    except ValueError:
        pass

    start_date = end_date - timedelta(days=7)
    try:
        start_date = parse(request.args.get('start-date', ''), default=start_date, yearfirst=True)
    except ValueError:
        pass

    end_date = end_date.strftime("%Y/%m/%d")
    start_date = start_date.strftime("%Y/%m/%d")

    return (start_date, end_date)
<|endoftext|>


<commit_before>from otm1_migrator.migration_rules.standard_otm1 import MIGRATION_RULES

SORT_ORDER_INDEX = {
    'Bucks': 3,
    'Burlington': 4,
    'Camden': 5,
    'Chester': 6,
    'Delaware': 7,
    'Gloucester': 8,
    'Kent': 9,
    'Mercer': 10,
    'Montgomery': 11,
    'New Castle': 12,
    'Salem': 13,
    'Sussex': 14,
}

def mutate_boundary(boundary_obj, otm1_fields):
    if ((boundary_obj.name.find('County') != -1
         or boundary_obj.name == 'Philadelphia')):
        boundary_obj.category = 'County'
        boundary_obj.sort_order = 1
    elif otm1_fields['county'] == 'Philadelphia':
        boundary_obj.category = 'Philadelphia Neighborhood'
        boundary_obj.sort_order = 2
    else:
        county = otm1_fields['county']
        boundary_obj.category = county + ' Township'
        boundary_obj.sort_order = SORT_ORDER_INDEX[county]

MIGRATION_RULES['boundary']['record_mutators'] = [mutate_boundary]
<commit_msg>Fix lint<commit_after>from otm1_migrator.migration_rules.standard_otm1 import MIGRATION_RULES

SORT_ORDER_INDEX = {
    'Bucks': 3,
    'Burlington': 4,
    'Camden': 5,
    'Chester': 6,
    'Delaware': 7,
    'Gloucester': 8,
    'Kent': 9,
    'Mercer': 10,
    'Montgomery': 11,
    'New Castle': 12,
    'Salem': 13,
    'Sussex': 14,
}

def mutate_boundary(boundary_obj, otm1_fields):
    if ((boundary_obj.name.find('County') != -1
         or boundary_obj.name == 'Philadelphia')):
        boundary_obj.category = 'County'
        boundary_obj.sort_order = 1
    elif otm1_fields['county'] == 'Philadelphia':
        boundary_obj.category = 'Philadelphia Neighborhood'
        boundary_obj.sort_order = 2
    else:
        county = otm1_fields['county']
        boundary_obj.category = county + ' Township'
        boundary_obj.sort_order = SORT_ORDER_INDEX[county]

MIGRATION_RULES['boundary']['record_mutators'] = [mutate_boundary]
<|endoftext|>


<commit_before>"""
This module provides functions to render placeholder content manually.
Contents is cached in memcache whenever possible, only the remaining items are queried.
The templatetags also use these functions to render the :class:`~fluent_contents.models.ContentItem` objects.
"""
import logging, os, six
from future.builtins import str
from django.conf import settings
from django.core.cache import cache
from django.forms import Media
from django.template.context import RequestContext
from django.template.loader import render_to_string, select_template
from django.utils.html import conditional_escape, escape
from django.utils.safestring import mark_safe
from django.utils.translation import get_language
from fluent_contents import appsettings
from fluent_contents.cache import get_rendering_cache_key
from fluent_contents.extensions import PluginNotFound, ContentPlugin
from fluent_contents.models import ContentItemOutput, ImmutableMedia

# This code is separate from the templatetags,
# so it can be called outside the templates as well.
from parler.utils.context import smart_override

logger = logging.getLogger(__name__)

def render_placeholder(request, placeholder, parent_object=None, template_name=None, limit_parent_language=True, fallback_language=None):
    """
    Render a :class:`~fluent_contents.models.Placeholder` object.
    Returns a :class:`~fluent_contents.models.ContentItemOutput` object
    which contains the HTML output and :class:`~django.forms.Media` object.

    :param request: The current request object.
    :type request: :class:`~django.http.HttpRequest`
    :param placeholder: The placeholder object.
    :type placeholder: :class:`~fluent_contents.models.Placeholder`
    :param parent_object: Optional, the parent object of the placeholder (already implied by the placeholder)
    :param template_name: Optional template name used to concatenate the placeholder output.
    :type template_name: str
    :param limit_parent_language: Whether the items should be limited to the parent language.
    :type limit_parent_language: bool
    :param fallback_language: The fallback language to use if there are no items in the current language. Passing ``True`` uses the default :ref:`FLUENT_CONTENTS_DEFAULT_LANGUAGE_CODE`.
    :type fallback_language: bool/str
    :rtype: :class:`~fluent_contents.models.ContentItemOutput`
    """
    # Get the items
    items = placeholder.get_content_items(parent_object, limit_parent_language=limit_parent_language).non_polymorphic()
    if fallback_language and not items:  # NOTES: performs query, so hence the .non_polymorphic() above
        language_code = appsettings.FLUENT_CONTENTS_DEFAULT_LANGUAGE_CODE if fallback_language is True else fallback_language
        items = placeholder.get_content_items(parent_object, limit_parent_language=False).translated(language_code).non_polymorphic()

    output = _render_items(request, placeholder, items, parent_object=parent_object, template_name=template_name)

    if is_edit_mode(request):
        output.html = _wrap_placeholder_output(output.html, placeholder)

    return output

def render_content_items(request, items, template_name=None):
    """
    Render a list of :class:`~fluent_contents.models.ContentItem` objects as HTML string.
    This is a variation of the :func:`render_placeholder` function.

    Note that the items are not filtered in any way by parent or language.
    The items are rendered as-is.

    :param request: The current request object.
    :type request: :class:`~django.http.HttpRequest`
    :param items: The list or queryset of objects to render. Passing a queryset is preferred.
    :type items: list or queryset of :class:`~fluent_contents.models.ContentItem`.
    :param template_name: Optional template name used to concatenate the placeholder output.
    :type template_name: str
    :rtype: :class:`~fluent_contents.models.ContentItemOutput`
    """
    if not items:
        output = ContentItemOutput(mark_safe(u""))
    else:
        output = _render_items(request, None, items, parent_object=None, template_name=template_name)

    if is_edit_mode(request):
        output.html = _wrap_anonymous_output(output.html)

    return output

def set_edit_mode(request, state):
    """
    Enable the edit mode; placeholders and plugins will be wrapped in a ``
`` that exposes metadata for frontend editing.
    """
    setattr(request, '_fluent_contents_edit_mode', bool(state))

def is_edit_mode(request):
    """
    Return whether edit mode is enabled; output is wrapped in ``
`` elements with metadata for frontend editing.
    """
    return getattr(request, '_fluent_contents_edit_mode', False)

def register_frontend_media(request, media):
    """
    Add a :class:`~django.forms.Media` class to the current request.
    This will be rendered by the ``render_plugin_media`` template tag.
    """
    if not hasattr(request, '_fluent_contents_frontend_media'):
        request._fluent_contents_frontend_media = Media()

    _add_media(request._fluent_contents_frontend_media, media)

def get_frontend_media(request):
    """
    Return the media that was registered in the request object.
    """
    return getattr(request, '_fluent_contents_frontend_media', None) or ImmutableMedia.empty_instance

def _render_items(request, placeholder, items, parent_object=None, template_name=None):
    edit_mode = is_edit_mode(request)
    item_output = {}
    output_ordering = []
    placeholder_cache_name = '@global@' if placeholder is None else placeholder.slot

    if not hasattr(items, "non_polymorphic"):
        # The items is either a list of manually created items, or it's a QuerySet.
        # Can't prevent reading the subclasses only, so don't bother with caching here.
        remaining_items = items
        output_ordering = [item.pk or id(item) for item in items]
    else:
        # Unless it was done before, disable polymorphic effects.
        if not items.polymorphic_disabled:
            items = items.non_polymorphic()

        # First try to fetch all items non-polymorphic from memcache
        # If these are found, there is no need to query the derived data from the database.
        remaining_items = []
        for i, contentitem in enumerate(items):
            output_ordering.append(contentitem.pk)
            output = None
            try:
                # Respect the cache output setting of the plugin
                if appsettings.FLUENT_CONTENTS_CACHE_OUTPUT and contentitem.plugin.cache_output and contentitem.pk:
                    output = contentitem.plugin.get_cached_output(placeholder_cache_name, contentitem)

                    # Support transition to new output format.
                    if not isinstance(output, ContentItemOutput):
                        output = None
                        logger.debug("Flushed cached output of {0}#{1} to store new format (key: {2}) ".format(contentitem.plugin.type_name, contentitem.pk, placeholder_cache_name))
            except PluginNotFound:
                pass

            # For debugging, ignore cached values when the template is updated.
            if output and settings.DEBUG:
                cachekey = get_rendering_cache_key(placeholder_cache_name, contentitem)
                if _is_template_updated(request, contentitem, cachekey):
                    output = None

            if output:
                item_output[contentitem.pk] = output
            else:
                remaining_items.append(contentitem)

        # Fetch derived table data for all objects not found in memcached
        if remaining_items:
            remaining_items = items.get_real_instances(remaining_items)

    # See if the queryset contained anything.
    # This test is moved here, to prevent earlier query execution.
    if not items:
        return ContentItemOutput(mark_safe(u"".format(escape(_get_placeholder_name(placeholder)))))
    elif remaining_items:
        # Render remaining items
        for contentitem in remaining_items:
            try:
                plugin = contentitem.plugin
            except PluginNotFound as e:
                output = ContentItemOutput(mark_safe(u'\n'.format(str(e))))
            else:
                if plugin.render_ignore_item_language \
                or (plugin.cache_output and plugin.cache_output_per_language):
                    # Render the template in the current language.
                    # The cache also stores the output under the current language code.
                    #
                    # It would make sense to apply this for fallback content too,
                    # but that would be ambiguous however because the parent_object could also be a fallback,
                    # and that case can't be detected here. Hence, better be explicit when desiring multi-lingual content.
                    render_language = get_language()  # Avoid switching the content,
                else:
                    # Render the template in the ContentItem language.
                    # This makes sure that {% trans %} tag output matches the language of the model field data.
                    render_language = contentitem.language_code

                with smart_override(render_language):
                    # Plugin output is likely HTML, but it should be placed in mark_safe() to raise awareness about escaping.
                    # This is just like Django's Input.render() and unlike Node.render().
                    output = plugin._render_contentitem(request, contentitem)

                if appsettings.FLUENT_CONTENTS_CACHE_OUTPUT and plugin.cache_output and contentitem.pk:
                    contentitem.plugin.set_cached_output(placeholder_cache_name, contentitem, output)

                if edit_mode:
                    output.html = _wrap_contentitem_output(output.html, contentitem)

            item_id = contentitem.pk or id(contentitem)
            item_output[item_id] = output

    # Order all rendered items in the correct sequence.  The derived tables could be truncated/reset,
    # so the base class model indexes don't necessary match with the derived indexes. Hence the dict + KeyError handling.
    #
    # The media is also collected in the same ordering, in case it's handled by django-compressor for example.
    output_ordered = []
    merged_media = Media()
    for pk in output_ordering:
        try:
            output = item_output[pk]
            output_ordered.append(output.html)
            _add_media(merged_media, output.media)
        except KeyError:
            # The get_real_instances() didn't return an item for the derived table. This happens when either:
            # - that table is truncated/reset, while there is still an entry in the base ContentItem table.
            #   A query at the derived table happens every time the page is being rendered.
            # - the model was completely removed which means there is also a stale ContentType object.
            item = next(item for item in items if item.pk == pk)
            try:
                class_name = item.plugin.type_name
            except PluginNotFound:
                # Derived table isn't there because the model has been removed.
                # There is a stale ContentType object, no plugin associated or loaded.
                class_name = 'content type is stale'

            output_ordered.append(u"\n".format(id=pk, cls=class_name))
            logger.warning("Missing derived model for ContentItem #{id}: {cls}.".format(id=pk, cls=class_name))
            pass

    # Combine all rendered items. Allow rendering the items with a template,
    # to inserting separators or nice start/end code.
    if not template_name:
        merged_output = mark_safe(''.join(output_ordered))
    else:
        context = {
            'contentitems': list(zip(items, output_ordered)),
            'parent_object': parent_object,  # Can be None
            'edit_mode': edit_mode,
        }
        merged_output = render_to_string(template_name, context, context_instance=RequestContext(request))

    return ContentItemOutput(merged_output, merged_media)

def _wrap_placeholder_output(html, placeholder):
    return mark_safe('
' \
           '{html}' \
           '
\n'.format(
        html=conditional_escape(html),
        id=placeholder.id,
        slot=placeholder.slot,
    ))

def _wrap_anonymous_output(html):
    return mark_safe('
' \
           '{html}' \
           '
\n'.format(
        html=conditional_escape(html),
    ))

def _wrap_contentitem_output(html, contentitem):
    return mark_safe('
' \
           '{html}' \
           '
\n'.format(
        html=conditional_escape(html),
        itemtype=contentitem.__class__.__name__,  # Same as ContentPlugin.type_name
        id=contentitem.id,
    ))

def _get_placeholder_name(placeholder):
    # TODO: Cheating here with knowledge of "fluent_contents.plugins.sharedcontent" package:
    #       prevent unclear message in  debug output.
    if placeholder.slot == 'shared_content':
        sharedcontent = placeholder.parent
        return "shared_content:{0}".format(sharedcontent.slug)

    return placeholder.slot

def _add_media(dest, media):
    # Do what django.forms.Media.__add__() does without creating a new object.
    dest.add_css(media._css)
    dest.add_js(media._js)

def _is_template_updated(request, contentitem, cachekey):
    if not settings.DEBUG:
        return False
    # For debugging only: tell whether the template is updated,
    # so the cached values can be ignored.

    plugin = contentitem.plugin
    if plugin.get_render_template.__func__ is not ContentPlugin.get_render_template.__func__:
        # oh oh, really need to fetch the real object.
        # Won't be needed that often.
        contentitem = contentitem.get_real_instance()  # This is only needed with DEBUG=True
        template_names = plugin.get_render_template(request, contentitem)
    else:
        template_names = plugin.render_template

    if not template_names:
        return False
    if isinstance(template_names, six.string_types):
        template_names = [template_names]

    # With TEMPLATE_DEBUG = True, each node tracks it's origin.
    node0 = select_template(template_names).nodelist[0]
    attr = 'source' if hasattr(node0, 'source') else 'origin'  # attribute depends on object type
    try:
        template_filename = getattr(node0, attr)[0].name
    except (AttributeError, IndexError):
        return False

    cache_stat = cache.get(cachekey + ".debug-stat")
    current_stat = os.path.getmtime(template_filename)
    if cache_stat != current_stat:
        cache.set(cachekey + ".debug-stat", current_stat)
        return True
<commit_msg>Fix ContentItemOutput flush debug message for expired cache entries.<commit_after>"""
This module provides functions to render placeholder content manually.
Contents is cached in memcache whenever possible, only the remaining items are queried.
The templatetags also use these functions to render the :class:`~fluent_contents.models.ContentItem` objects.
"""
import logging, os, six
from future.builtins import str
from django.conf import settings
from django.core.cache import cache
from django.forms import Media
from django.template.context import RequestContext
from django.template.loader import render_to_string, select_template
from django.utils.html import conditional_escape, escape
from django.utils.safestring import mark_safe
from django.utils.translation import get_language
from fluent_contents import appsettings
from fluent_contents.cache import get_rendering_cache_key
from fluent_contents.extensions import PluginNotFound, ContentPlugin
from fluent_contents.models import ContentItemOutput, ImmutableMedia

# This code is separate from the templatetags,
# so it can be called outside the templates as well.
from parler.utils.context import smart_override

logger = logging.getLogger(__name__)

def render_placeholder(request, placeholder, parent_object=None, template_name=None, limit_parent_language=True, fallback_language=None):
    """
    Render a :class:`~fluent_contents.models.Placeholder` object.
    Returns a :class:`~fluent_contents.models.ContentItemOutput` object
    which contains the HTML output and :class:`~django.forms.Media` object.

    :param request: The current request object.
    :type request: :class:`~django.http.HttpRequest`
    :param placeholder: The placeholder object.
    :type placeholder: :class:`~fluent_contents.models.Placeholder`
    :param parent_object: Optional, the parent object of the placeholder (already implied by the placeholder)
    :param template_name: Optional template name used to concatenate the placeholder output.
    :type template_name: str
    :param limit_parent_language: Whether the items should be limited to the parent language.
    :type limit_parent_language: bool
    :param fallback_language: The fallback language to use if there are no items in the current language. Passing ``True`` uses the default :ref:`FLUENT_CONTENTS_DEFAULT_LANGUAGE_CODE`.
    :type fallback_language: bool/str
    :rtype: :class:`~fluent_contents.models.ContentItemOutput`
    """
    # Get the items
    items = placeholder.get_content_items(parent_object, limit_parent_language=limit_parent_language).non_polymorphic()
    if fallback_language and not items:  # NOTES: performs query, so hence the .non_polymorphic() above
        language_code = appsettings.FLUENT_CONTENTS_DEFAULT_LANGUAGE_CODE if fallback_language is True else fallback_language
        items = placeholder.get_content_items(parent_object, limit_parent_language=False).translated(language_code).non_polymorphic()

    output = _render_items(request, placeholder, items, parent_object=parent_object, template_name=template_name)

    if is_edit_mode(request):
        output.html = _wrap_placeholder_output(output.html, placeholder)

    return output

def render_content_items(request, items, template_name=None):
    """
    Render a list of :class:`~fluent_contents.models.ContentItem` objects as HTML string.
    This is a variation of the :func:`render_placeholder` function.

    Note that the items are not filtered in any way by parent or language.
    The items are rendered as-is.

    :param request: The current request object.
    :type request: :class:`~django.http.HttpRequest`
    :param items: The list or queryset of objects to render. Passing a queryset is preferred.
    :type items: list or queryset of :class:`~fluent_contents.models.ContentItem`.
    :param template_name: Optional template name used to concatenate the placeholder output.
    :type template_name: str
    :rtype: :class:`~fluent_contents.models.ContentItemOutput`
    """
    if not items:
        output = ContentItemOutput(mark_safe(u""))
    else:
        output = _render_items(request, None, items, parent_object=None, template_name=template_name)

    if is_edit_mode(request):
        output.html = _wrap_anonymous_output(output.html)

    return output

def set_edit_mode(request, state):
    """
    Enable the edit mode; placeholders and plugins will be wrapped in a ``
`` that exposes metadata for frontend editing.
    """
    setattr(request, '_fluent_contents_edit_mode', bool(state))

def is_edit_mode(request):
    """
    Return whether edit mode is enabled; output is wrapped in ``
`` elements with metadata for frontend editing.
    """
    return getattr(request, '_fluent_contents_edit_mode', False)

def register_frontend_media(request, media):
    """
    Add a :class:`~django.forms.Media` class to the current request.
    This will be rendered by the ``render_plugin_media`` template tag.
    """
    if not hasattr(request, '_fluent_contents_frontend_media'):
        request._fluent_contents_frontend_media = Media()

    _add_media(request._fluent_contents_frontend_media, media)

def get_frontend_media(request):
    """
    Return the media that was registered in the request object.
    """
    return getattr(request, '_fluent_contents_frontend_media', None) or ImmutableMedia.empty_instance

def _render_items(request, placeholder, items, parent_object=None, template_name=None):
    edit_mode = is_edit_mode(request)
    item_output = {}
    output_ordering = []
    placeholder_cache_name = '@global@' if placeholder is None else placeholder.slot

    if not hasattr(items, "non_polymorphic"):
        # The items is either a list of manually created items, or it's a QuerySet.
        # Can't prevent reading the subclasses only, so don't bother with caching here.
        remaining_items = items
        output_ordering = [item.pk or id(item) for item in items]
    else:
        # Unless it was done before, disable polymorphic effects.
        if not items.polymorphic_disabled:
            items = items.non_polymorphic()

        # First try to fetch all items non-polymorphic from memcache
        # If these are found, there is no need to query the derived data from the database.
        remaining_items = []
        for i, contentitem in enumerate(items):
            output_ordering.append(contentitem.pk)
            output = None
            try:
                # Respect the cache output setting of the plugin
                if appsettings.FLUENT_CONTENTS_CACHE_OUTPUT and contentitem.plugin.cache_output and contentitem.pk:
                    output = contentitem.plugin.get_cached_output(placeholder_cache_name, contentitem)

                    # Support transition to new output format.
                    if output is not None and not isinstance(output, ContentItemOutput):
                        output = None
                        logger.debug("Flushed cached output of {0}#{1} to store new ContentItemOutput format (key: {2})".format(contentitem.plugin.type_name, contentitem.pk, placeholder_cache_name))
            except PluginNotFound:
                pass

            # For debugging, ignore cached values when the template is updated.
            if output and settings.DEBUG:
                cachekey = get_rendering_cache_key(placeholder_cache_name, contentitem)
                if _is_template_updated(request, contentitem, cachekey):
                    output = None

            if output:
                item_output[contentitem.pk] = output
            else:
                remaining_items.append(contentitem)

        # Fetch derived table data for all objects not found in memcached
        if remaining_items:
            remaining_items = items.get_real_instances(remaining_items)

    # See if the queryset contained anything.
    # This test is moved here, to prevent earlier query execution.
    if not items:
        return ContentItemOutput(mark_safe(u"".format(escape(_get_placeholder_name(placeholder)))))
    elif remaining_items:
        # Render remaining items
        for contentitem in remaining_items:
            try:
                plugin = contentitem.plugin
            except PluginNotFound as e:
                output = ContentItemOutput(mark_safe(u'\n'.format(str(e))))
            else:
                if plugin.render_ignore_item_language \
                or (plugin.cache_output and plugin.cache_output_per_language):
                    # Render the template in the current language.
                    # The cache also stores the output under the current language code.
                    #
                    # It would make sense to apply this for fallback content too,
                    # but that would be ambiguous however because the parent_object could also be a fallback,
                    # and that case can't be detected here. Hence, better be explicit when desiring multi-lingual content.
                    render_language = get_language()  # Avoid switching the content,
                else:
                    # Render the template in the ContentItem language.
                    # This makes sure that {% trans %} tag output matches the language of the model field data.
                    render_language = contentitem.language_code

                with smart_override(render_language):
                    # Plugin output is likely HTML, but it should be placed in mark_safe() to raise awareness about escaping.
                    # This is just like Django's Input.render() and unlike Node.render().
                    output = plugin._render_contentitem(request, contentitem)

                if appsettings.FLUENT_CONTENTS_CACHE_OUTPUT and plugin.cache_output and contentitem.pk:
                    contentitem.plugin.set_cached_output(placeholder_cache_name, contentitem, output)

                if edit_mode:
                    output.html = _wrap_contentitem_output(output.html, contentitem)

            item_id = contentitem.pk or id(contentitem)
            item_output[item_id] = output

    # Order all rendered items in the correct sequence.  The derived tables could be truncated/reset,
    # so the base class model indexes don't necessary match with the derived indexes. Hence the dict + KeyError handling.
    #
    # The media is also collected in the same ordering, in case it's handled by django-compressor for example.
    output_ordered = []
    merged_media = Media()
    for pk in output_ordering:
        try:
            output = item_output[pk]
            output_ordered.append(output.html)
            _add_media(merged_media, output.media)
        except KeyError:
            # The get_real_instances() didn't return an item for the derived table. This happens when either:
            # - that table is truncated/reset, while there is still an entry in the base ContentItem table.
            #   A query at the derived table happens every time the page is being rendered.
            # - the model was completely removed which means there is also a stale ContentType object.
            item = next(item for item in items if item.pk == pk)
            try:
                class_name = item.plugin.type_name
            except PluginNotFound:
                # Derived table isn't there because the model has been removed.
                # There is a stale ContentType object, no plugin associated or loaded.
                class_name = 'content type is stale'

            output_ordered.append(u"\n".format(id=pk, cls=class_name))
            logger.warning("Missing derived model for ContentItem #{id}: {cls}.".format(id=pk, cls=class_name))
            pass

    # Combine all rendered items. Allow rendering the items with a template,
    # to inserting separators or nice start/end code.
    if not template_name:
        merged_output = mark_safe(''.join(output_ordered))
    else:
        context = {
            'contentitems': list(zip(items, output_ordered)),
            'parent_object': parent_object,  # Can be None
            'edit_mode': edit_mode,
        }
        merged_output = render_to_string(template_name, context, context_instance=RequestContext(request))

    return ContentItemOutput(merged_output, merged_media)

def _wrap_placeholder_output(html, placeholder):
    return mark_safe('
' \
           '{html}' \
           '
\n'.format(
        html=conditional_escape(html),
        id=placeholder.id,
        slot=placeholder.slot,
    ))

def _wrap_anonymous_output(html):
    return mark_safe('
' \
           '{html}' \
           '
\n'.format(
        html=conditional_escape(html),
    ))

def _wrap_contentitem_output(html, contentitem):
    return mark_safe('
' \
           '{html}' \
           '
\n'.format(
        html=conditional_escape(html),
        itemtype=contentitem.__class__.__name__,  # Same as ContentPlugin.type_name
        id=contentitem.id,
    ))

def _get_placeholder_name(placeholder):
    # TODO: Cheating here with knowledge of "fluent_contents.plugins.sharedcontent" package:
    #       prevent unclear message in  debug output.
    if placeholder.slot == 'shared_content':
        sharedcontent = placeholder.parent
        return "shared_content:{0}".format(sharedcontent.slug)

    return placeholder.slot

def _add_media(dest, media):
    # Do what django.forms.Media.__add__() does without creating a new object.
    dest.add_css(media._css)
    dest.add_js(media._js)

def _is_template_updated(request, contentitem, cachekey):
    if not settings.DEBUG:
        return False
    # For debugging only: tell whether the template is updated,
    # so the cached values can be ignored.

    plugin = contentitem.plugin
    if plugin.get_render_template.__func__ is not ContentPlugin.get_render_template.__func__:
        # oh oh, really need to fetch the real object.
        # Won't be needed that often.
        contentitem = contentitem.get_real_instance()  # This is only needed with DEBUG=True
        template_names = plugin.get_render_template(request, contentitem)
    else:
        template_names = plugin.render_template

    if not template_names:
        return False
    if isinstance(template_names, six.string_types):
        template_names = [template_names]

    # With TEMPLATE_DEBUG = True, each node tracks it's origin.
    node0 = select_template(template_names).nodelist[0]
    attr = 'source' if hasattr(node0, 'source') else 'origin'  # attribute depends on object type
    try:
        template_filename = getattr(node0, attr)[0].name
    except (AttributeError, IndexError):
        return False

    cache_stat = cache.get(cachekey + ".debug-stat")
    current_stat = os.path.getmtime(template_filename)
    if cache_stat != current_stat:
        cache.set(cachekey + ".debug-stat", current_stat)
        return True
<|endoftext|>