<commit_before>#  coding=utf-8
import json
import threading
import time
import traceback
from datetime import datetime
from logging import getLogger
from threading import Thread
from typing import List, Tuple, Callable

from .enums import PluginType, API
from .plugin import Plugin
from .plugin.utils import willExecute

logger = getLogger(__name__)

class ThreadManager:
    def __init__(self, core) -> None:
        self.core = core
        self.threads: List[Tuple[Thread, Callable[[], None]]] = []

        # noinspection PyTypeChecker
        self.startThread(self.watchThreads)

    def startThread(self, target: Callable, name: str=None, args: List=None, keepalive: bool=True) -> Thread:
        name: str = name or target.__name__
        args: Tuple = tuple(args) if args else ()

        t: Thread = Thread(target=target, name=name, args=args, daemon=True)
        t.start()

        if keepalive:
            self.threads.append((t, target))

        return t

    @staticmethod
    def wrapPlugin(plugin: Plugin) -> None:
        try:
            plugin.module.do()
            logger.info(f"{plugin.meta.type.name} plugin \"{plugin.meta.name}\" was executed successfully.")
        except Exception:
            logger.warning(
                f"{plugin.meta.type.name} plugin \"{plugin.meta.name}\" "
                f"could not be executed. Error Detail:\n{traceback.format_exc()}"
            )

    def executePluginSafely(self, plugin: Plugin, args: List=None) -> None:
        # noinspection PyTypeChecker
        self.startThread(self.wrapPlugin(plugin), name=plugin.meta.name, args=args or [])

    def destroyThread(self, name: str) -> bool:
        for i, thread, func in enumerate(self.threads):
            if thread.name == name:
                del self.threads[i]
                return True
        return False

    def watchThreads(self) -> None:
        while True:
            try:
                workingThreads: List[Thread] = threading.enumerate()
                with open(f"{self.core.config.directory.api}/{API.Thread.value}", "w") as f:
                    json.dump([x.name for x in workingThreads], f, sort_keys=True, indent=4)

                for i, thread, func in enumerate(self.threads):
                    if not thread.is_alive() or thread not in workingThreads:
                        # noinspection PyTypeChecker
                        self.threads[i] = self.startThread(func, name=thread.name)

            except:
                pass
            finally:
                time.sleep(10)

    def startSchedulePlugins(self):
        while True:
            willExecutePlugins: List[Plugin] = [
                schedulePlugin
                for schedulePlugin in self.core.PM.plugins[PluginType.Schedule.name]
                if willExecute(schedulePlugin.meta.ratio)
            ]

            now: datetime = datetime.now()
            time.sleep(60 - now.second - now.microsecond / 1000000)

            now: datetime = datetime.now()
            [
                self.executePluginSafely(schedulePlugin)
                for schedulePlugin in willExecutePlugins
                if now.hour in schedulePlugin.meta.hours
                and now.minute in schedulePlugin.meta.minutes
            ]
            time.sleep(1)
<commit_msg>Fix: not starting schedule plugins<commit_after>#  coding=utf-8
import json
import threading
import time
import traceback
from datetime import datetime
from logging import getLogger
from threading import Thread
from typing import List, Tuple, Callable

from .enums import PluginType, API
from .plugin import Plugin
from .plugin.utils import willExecute

logger = getLogger(__name__)

class ThreadManager:
    def __init__(self, core) -> None:
        self.core = core
        self.threads: List[Tuple[Thread, Callable[[], None]]] = []

        # noinspection PyTypeChecker
        self.startThread(self.watchThreads)

    def startThread(self, target: Callable, name: str=None, args: List=None, keepalive: bool=True) -> Thread:
        name: str = name or target.__name__
        args: Tuple = tuple(args) if args else ()

        t: Thread = Thread(target=target, name=name, args=args, daemon=True)
        t.start()

        if keepalive:
            self.threads.append((t, target))

        return t

    @staticmethod
    def wrapPlugin(plugin: Plugin) -> None:
        try:
            plugin.module.do()
            logger.info(f"{plugin.meta.type.name} plugin \"{plugin.meta.name}\" was executed successfully.")
        except Exception:
            logger.warning(
                f"{plugin.meta.type.name} plugin \"{plugin.meta.name}\" "
                f"could not be executed. Error Detail:\n{traceback.format_exc()}"
            )

    def executePluginSafely(self, plugin: Plugin, args: List=None) -> None:
        # noinspection PyTypeChecker
        self.startThread(self.wrapPlugin(plugin), name=plugin.meta.name, args=args or [])

    def destroyThread(self, name: str) -> bool:
        for i, thread, func in enumerate(self.threads):
            if thread.name == name:
                del self.threads[i]
                return True
        return False

    def watchThreads(self) -> None:
        while True:
            try:
                workingThreads: List[Thread] = threading.enumerate()
                with open(f"{self.core.config.directory.api}/{API.Thread.value}", "w") as f:
                    json.dump([x.name for x in workingThreads], f, sort_keys=True, indent=4)

                for i, thread, func in enumerate(self.threads):
                    if not thread.is_alive() or thread not in workingThreads:
                        # noinspection PyTypeChecker
                        self.threads[i] = self.startThread(func, name=thread.name)

            except:
                pass
            finally:
                time.sleep(10)

    def startSchedulePlugins(self):
        while True:
            willExecutePlugins: List[Plugin] = [
                schedulePlugin
                for schedulePlugin in self.core.PM.plugins[PluginType.Schedule.name]
                if willExecute(schedulePlugin.meta.ratio)
            ]

            now: datetime = datetime.now()
            time.sleep(60 - now.second - now.microsecond / 1000000)

            now: datetime = datetime.now()
            [
                self.executePluginSafely(schedulePlugin)
                for schedulePlugin in willExecutePlugins
                if now.hour in schedulePlugin.meta.combinedHours
                and now.minute in schedulePlugin.meta.combinedMinutes
            ]
            time.sleep(1)
<|endoftext|>


<commit_before>"""
Name: Paul Briant
Date: 11/15/16
Class: Introduction to Python
Session: 07
Assignment: HTML Renderer

Description:
Tests for HTML Renderer
"""

# -------------------------------Imports----------------------------------------
import html_render as hr
import io

# -------------------------------Functions--------------------------------------

def test_init():
    e = hr.Element()
    e = hr.Element("Please remain calm, this is a test.")

def test_content():
    """
    Checks whether content is a str.
    """
    e = hr.Element("Testing, 1 2 3..")
    assert "Testing, 1 2 3.." in e.content

def test_append():
    """
    Verifies additional content is successfully added to rest of content
    """
    e = hr.Element("Please remain calm, this is a test.")
    e.append("DON'T PANIC!")
    assert "DON'T PANIC!" in e.content

def test_two_instances():
    e = hr.Element("Testing, 1 2 3..")
    e2 = hr.Element("Testing, 1 2 3..")
    e.append("More tests! We need more tests!")
    assert "More tests! We need more tests!" not in e2.content

def test_render():
    """
    Tests whether render successfully writes content to file by creating
    instance of element, adding content to it using the append function and
    calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.Element("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert("Testing, 1 2 3..") in file_cont
    assert("DON'T PANIC") in file_cont
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")

def test_html_render():
    """
    Tests whether render successfully writes content to file for the html tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.Html("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")

def test_body_render():
    """
    Tests whether render successfully writes content to file for the body tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.Body("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")

def test_p_render():
    """
    Tests whether render successfully writes content to file for the p tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.P("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("
")
    assert file_cont.strip().endswith("
")

def test_append_instances():
    """
    Test to verify an instance of an html element can be appended into another.
    """
    output = io.StringIO()
    e_body = hr.Body()
    e_p = hr.P("Testing, 1 2 3..")
    e_body.append(e_p)
    e_body.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.endswith("")
    print(file_cont)
    assert "Testing, 1 2 3.." in file_cont
    # check p tag is in body
    assert "
" in file_cont and "
" in file_cont
    # additional tests?

def test_head_render():
    """
    Tests whether render successfully writes content to file for the head tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    output = io.StringIO()
    e = hr.Head("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")
<commit_msg>Added strip function to test for closing tag to remove whitespace at end of string.<commit_after>"""
Name: Paul Briant
Date: 11/15/16
Class: Introduction to Python
Session: 07
Assignment: HTML Renderer

Description:
Tests for HTML Renderer
"""

# -------------------------------Imports----------------------------------------
import html_render as hr
import io

# -------------------------------Functions--------------------------------------

def test_init():
    e = hr.Element()
    e = hr.Element("Please remain calm, this is a test.")

def test_content():
    """
    Checks whether content is a str.
    """
    e = hr.Element("Testing, 1 2 3..")
    assert "Testing, 1 2 3.." in e.content

def test_append():
    """
    Verifies additional content is successfully added to rest of content
    """
    e = hr.Element("Please remain calm, this is a test.")
    e.append("DON'T PANIC!")
    assert "DON'T PANIC!" in e.content

def test_two_instances():
    e = hr.Element("Testing, 1 2 3..")
    e2 = hr.Element("Testing, 1 2 3..")
    e.append("More tests! We need more tests!")
    assert "More tests! We need more tests!" not in e2.content

def test_render():
    """
    Tests whether render successfully writes content to file by creating
    instance of element, adding content to it using the append function and
    calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.Element("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert("Testing, 1 2 3..") in file_cont
    assert("DON'T PANIC") in file_cont
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")

def test_html_render():
    """
    Tests whether render successfully writes content to file for the html tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.Html("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")

def test_body_render():
    """
    Tests whether render successfully writes content to file for the body tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.Body("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")

def test_p_render():
    """
    Tests whether render successfully writes content to file for the p tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    # Creates in memory version of content to write to file
    output = io.StringIO()
    e = hr.P("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("
")
    assert file_cont.strip().endswith("
")

def test_append_instances():
    """
    Test to verify an instance of an html element can be appended into another.
    """
    output = io.StringIO()
    e_body = hr.Body()
    e_p = hr.P("Testing, 1 2 3..")
    e_body.append(e_p)
    e_body.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")
    print(file_cont)
    assert "Testing, 1 2 3.." in file_cont
    # check p tag is in body
    assert "
" in file_cont and "
" in file_cont
    # additional tests?

def test_head_render():
    """
    Tests whether render successfully writes content to file for the head tag
    by creating instance of element, adding content to it using the append
    function and calling the render funtion on it.
    """
    output = io.StringIO()
    e = hr.Head("Testing, 1 2 3..")
    e.append("DON'T PANIC")
    e.render(output, "")
    file_cont = output.getvalue()
    assert file_cont.startswith("")
    assert file_cont.strip().endswith("")
<|endoftext|>


<commit_before>"""Utility methods for data handling."""

import json
import unicodedata
import yaml

def remove_accents(s):
    s_NFKD = unicodedata.normalize('NFKD', s)
    return ''.join([c for c in s_NFKD if not unicodedata.combining(c)])

def json_load(path):
    with open(path, 'rt', encoding='utf-8') as f:
        data_json = json.load(f)
    return data_json

def json_dump_utf8(var, path, indent=4, flatten_level=None):
    with open(path, 'w', encoding='utf-8') as f:
        data = json.dumps(var, indent=indent, sort_keys=True, separators=(',', ': '), ensure_ascii=False)
        if (indent is not None) and (flatten_level is not None):
            flatten_string = '\n' + ' ' * (indent * flatten_level)
            data = data.replace(flatten_string, ' ')
            data = data.replace('\n' + ' ' * (indent * (flatten_level - 1)) + ']', ']')
        f.write(str(data))

def yaml_load(path):
    with open(path, 'r') as f:
        # data_yaml = yaml.load(f, Loader=yaml.FullLoader)
        data_yaml = yaml.load(f)
    return data_yaml

def execute_script(db, path):
    """Executes SQL script at `path` on database `db`."""
    with open(path, 'r') as f:
        db.execute(f.read())
<commit_msg>Small fix to utils.py<commit_after>"""Utility methods for data handling."""

import json
import unicodedata
import yaml

def remove_accents(s):
    s_NFKD = unicodedata.normalize('NFKD', s)
    return ''.join([c for c in s_NFKD if not unicodedata.combining(c)])

def json_load(path):
    with open(path, 'rt', encoding='utf-8') as f:
        data_json = json.load(f)
    return data_json

def json_dump_utf8(var, path, indent=4, flatten_level=None):
    with open(path, 'w', encoding='utf-8') as f:
        data = json.dumps(var, indent=indent, sort_keys=True, separators=(',', ': '), ensure_ascii=False)
        if (indent is not None) and (flatten_level is not None):
            flatten_string = '\n' + ' ' * (indent * flatten_level)
            data = data.replace(flatten_string, ' ')
            data = data.replace('\n' + ' ' * (indent * (flatten_level - 1)) + ']', ']')
        f.write(str(data))

def yaml_load(path):
    with open(path, 'rt', encoding='utf-8') as f:
        # data_yaml = yaml.load(f, Loader=yaml.FullLoader)
        data_yaml = yaml.load(f)
    return data_yaml

def execute_script(db, path):
    """Executes SQL script at `path` on database `db`."""
    with open(path, 'r', encoding='utf-8') as f:
        db.execute(f.read())
<|endoftext|>


<commit_before>import os

from django.shortcuts import render, redirect
from django.template import Context
from django.http import HttpResponse
from django.core.servers.basehttp import FileWrapper

from chemtools.utils import CLUSTERS, CLUSTER_TUPLES

def frag_index(request):
    data = (
        ["Cores", ("CON", "TON", "TSN", "CSN", "TNN", "CNN", "CCC", "TCC")],
        ["X Groups", (["A", "H"], ["B", "Cl"], ["C", "Br"], ["D", "CN"], ["E", "CCH"],
            ["F", "OH"], ["G", "SH"], ["H", "NH_2"], ["I", "CH_3"], ["J", "phenyl"], ["K", "TMS"],
             ["L", "OCH_3"])],
        ["Aryl Groups", (["2", "double"], ["3", "triple"], ["4", "phenyl"], ["5", "thiophene"],
            ["6", "pyridine"], ["7", "carbazole"], ["8", "TZ"], ["9", "EDOT"])],
        ["R Groups", (["a", "H"], ["b", "Cl"], ["c", "Br"], ["d", "CN"], ["e", "CCH"],
            ["f", "OH"], ["g", "SH"], ["h", "NH_2"], ["i", "CH_3"], ["j", "phenyl"], ["k", "TMS"],
             ["l", "OCH_3"])],
    )
    c = Context({"usable_parts": data})
    return render(request, "data/frag_index.html", c)

def get_frag(request, frag):
    if frag in os.listdir("chemtools/data/"):
        f = open("chemtools/data/" + frag, "r")
        response = HttpResponse(FileWrapper(f), content_type="text/plain")
        return response
    else:
        return redirect(frag)

def template_index(request):
    data = [CLUSTERS[x] for x in CLUSTERS.keys()]
    c = Context({"templates": data})
    return render(request, "data/template_index.html", c)

def get_template(request, template):
    template = template.lower()
    if template in [x.lower() for x in sum(CLUSTER_TUPLES, ())]:
        letter = template[0]
        f = open("chemtools/templates/chemtools/%sjob.txt" % letter)
        response = HttpResponse(FileWrapper(f), content_type="text/plain")
    else:
        response = HttpResponse('', content_type="text/plain")
    return response
<commit_msg>Fix data.get_frag view<commit_after>import os

from django.shortcuts import render, redirect
from django.template import Context
from django.http import HttpResponse
from django.core.servers.basehttp import FileWrapper

from chemtools.utils import CLUSTERS, CLUSTER_TUPLES

def frag_index(request):
    data = (
        ["Cores", ("CON", "TON", "TSN", "CSN", "TNN", "CNN", "CCC", "TCC")],
        ["X Groups", (["A", "H"], ["B", "Cl"], ["C", "Br"], ["D", "CN"], ["E", "CCH"],
            ["F", "OH"], ["G", "SH"], ["H", "NH_2"], ["I", "CH_3"], ["J", "phenyl"], ["K", "TMS"],
             ["L", "OCH_3"])],
        ["Aryl Groups", (["2", "double"], ["3", "triple"], ["4", "phenyl"], ["5", "thiophene"],
            ["6", "pyridine"], ["7", "carbazole"], ["8", "TZ"], ["9", "EDOT"])],
        ["R Groups", (["a", "H"], ["b", "Cl"], ["c", "Br"], ["d", "CN"], ["e", "CCH"],
            ["f", "OH"], ["g", "SH"], ["h", "NH_2"], ["i", "CH_3"], ["j", "phenyl"], ["k", "TMS"],
             ["l", "OCH_3"])],
    )
    c = Context({"usable_parts": data})
    return render(request, "data/frag_index.html", c)

def get_frag(request, frag):
    if len(frag) == 1:
        frag = frag.lower()
    if frag in os.listdir("chemtools/data/"):
        f = open("chemtools/data/" + frag, "r")
        response = HttpResponse(FileWrapper(f), content_type="text/plain")
        return response
    else:
        return redirect(frag_index)

def template_index(request):
    data = [CLUSTERS[x] for x in CLUSTERS.keys()]
    c = Context({"templates": data})
    return render(request, "data/template_index.html", c)

def get_template(request, template):
    template = template.lower()
    if template in [x.lower() for x in sum(CLUSTER_TUPLES, ())]:
        letter = template[0]
        f = open("chemtools/templates/chemtools/%sjob.txt" % letter)
        response = HttpResponse(FileWrapper(f), content_type="text/plain")
    else:
        response = HttpResponse('', content_type="text/plain")
    return response
<|endoftext|>


<commit_before>'''
Dataset source: https://nlds.soe.ucsc.edu/sarcasm2

By Shereen Oraby, Vrindavan Harrison, Lena Reed, Ernesto Hernandez, Ellen Riloff and Marilyn Walker. "Creating and Characterizing a Diverse Corpus of Sarcasm in Dialogue." In The 17th Annual SIGdial Meeting on Discourse and Dialogue (SIGDIAL), Los Angeles, California, USA, 2016.
'''
import numpy as np
import pandas
import re
import os

def main():
	data = clean_data()

def clean_data():
	'''
	Steps:
	1. Load data from csv file
	2. Remove unnecessary quotes
	3. Remove emoticons
	3. Fix all typos using Microsoft's Bing Spell Check API: https://www.microsoft.com/cognitive-services/en-us/bing-spell-check-api
	'''
	# step 1
	data = pandas.read_csv('sarcasm_v2.csv').as_matrix()
	# print(data.shape)
	data[:, 0] = np.array([find_category(x) for x in data[:, 0]])
	data[:, 1] = np.array([sarcasm(x) for x in data[:, 1]])
	# print(data[0,1]) # should be 1 for sarcasm

	# step 2
	data[:, 3] = np.array([clean_quotes(x) for x in data[:, 3]])
	data[:, 4] = np.array([clean_quotes(x) for x in data[:, 4]])

	# step 3
	data[:, 3] = np.array([remove_emoticons(x) for x in data[:, 3]])
	data[:, 4] = np.array([remove_emoticons(x) for x in data[:, 4]])

	return data

def find_category(category):
	return {
        'GEN': 0, # general
        'HYP': 1, # hyperbole
		'RQ': 2 # rhetorical question
    }[category]

def sarcasm(sarcasm_value):
	return {
		'sarc': 1, # true for sarcasm
		'notsarc': 0 # false for sarcasm
	}[sarcasm_value]

def get_data_index(ID):
	'''find the index of the data point. Corresponds to 1234 in GEN_sarc_1234 under ID in data.'''
	result = ''
	for i in ID:
		if i.isdigit():
			result = result + i
	return int(result)

def clean_quotes(string):
	'''
	clean_quotes a string and removes beginning and closing quotation marks. Also replaces all duplicated double quotation marks with just one set of double quotation marks, e.g. ""hello"" with "hello"
	'''
	if string[0] == '"':
		string = string[1:] # first character
	if string[-1] == '"':
		string = string[:-1] # last character
	string = re.sub('\"\"', "\"", string)
	# triple quotes should not exist after removing first and last quotes from beginning and end, respectively, of the original string.
	# quadruple quotes will also be removed by replacing all duplicated double quotes "" with a single double quote ". This way, back-to-back duplicated double quotes (e.g. ""hello""""world"" will become "hello""world") will be taken care of as well.
	return string

def remove_emoticons(string):
	''' removes emoticons from the strings.'''
	return re.sub('emoticons\S*', '', string)

def get_api_key_path():
	'''gets path to api key in a text file stored locally, outside of git repository.'''
	parent_path = os.path.pardir
	return parent_path + "/api_key.txt"

def get_api_key():
	'''gets api key.'''
	handle = open(get_api_key_path(), 'r')
	data = handle.readline().strip() # primary API key is on the first line, secondary API key is the second line.
	handle.close()
	return data

if __name__ == '__main__':
	main()
<commit_msg>Modify data_clean slightly<commit_after>'''
Dataset source: https://nlds.soe.ucsc.edu/sarcasm2

By Shereen Oraby, Vrindavan Harrison, Lena Reed, Ernesto Hernandez, Ellen Riloff and Marilyn Walker. "Creating and Characterizing a Diverse Corpus of Sarcasm in Dialogue." In The 17th Annual SIGdial Meeting on Discourse and Dialogue (SIGDIAL), Los Angeles, California, USA, 2016.
'''
import numpy as np
import pandas
import re
import os

def main():
	data = clean_data()
	# print(data[0, 1]) # should be 1 for sarcasm

def clean_data():
	'''
	Steps:
	1. Load data from csv file
	2. Remove unnecessary quotes
	3. Remove emoticons
	3. Fix all typos using Microsoft's Bing Spell Check API: https://www.microsoft.com/cognitive-services/en-us/bing-spell-check-api
	'''
	# step 1
	data = pandas.read_csv('sarcasm_v2.csv').as_matrix()
	# print(data.shape)
	data[:, 0] = np.array([find_category(x) for x in data[:, 0]])
	data[:, 1] = np.array([sarcasm(x) for x in data[:, 1]])

	# step 2
	data[:, 3] = np.array([clean_quotes(x) for x in data[:, 3]])
	data[:, 4] = np.array([clean_quotes(x) for x in data[:, 4]])

	# step 3
	data[:, 3] = np.array([remove_emoticons(x) for x in data[:, 3]])
	data[:, 4] = np.array([remove_emoticons(x) for x in data[:, 4]])

	return data

def find_category(category):
	return {
        'GEN': 0, # general
        'HYP': 1, # hyperbole
		'RQ': 2 # rhetorical question
    }[category]

def sarcasm(sarcasm_value):
	return {
		'sarc': 1, # true for sarcasm
		'notsarc': 0 # false for sarcasm
	}[sarcasm_value]

def get_data_index(ID):
	'''find the index of the data point. Corresponds to 1234 in GEN_sarc_1234 under ID in data.'''
	result = ''
	for i in ID:
		if i.isdigit():
			result = result + i
	return int(result)

def clean_quotes(string):
	'''
	clean_quotes a string and removes beginning and closing quotation marks. Also replaces all duplicated double quotation marks with just one set of double quotation marks, e.g. ""hello"" with "hello"
	'''
	if string[0] == '"':
		string = string[1:] # first character
	if string[-1] == '"':
		string = string[:-1] # last character
	string = re.sub('\"\"', "\"", string)
	# triple quotes should not exist after removing first and last quotes from beginning and end, respectively, of the original string.
	# quadruple quotes will also be removed by replacing all duplicated double quotes "" with a single double quote ". This way, back-to-back duplicated double quotes (e.g. ""hello""""world"" will become "hello""world") will be taken care of as well.
	return string

def remove_emoticons(string):
	''' removes emoticons from the strings.'''
	return re.sub('emoticons\S*', '', string)

def get_api_key_path():
	'''gets path to api key in a text file stored locally, outside of git repository.'''
	parent_path = os.path.pardir
	return parent_path + "/api_key.txt"

def get_api_key():
	'''gets api key.'''
	handle = open(get_api_key_path(), 'r')
	data = handle.readline().strip() # primary API key is on the first line, secondary API key is the second line.
	handle.close()
	return data

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>
from __future__ import division

import sys
import time

import numpy as np

from memn2n.memory import MemoryL, MemoryBoW
from memn2n.nn import AddTable, CrossEntropyLoss, Duplicate, ElemMult, LinearNB
from memn2n.nn import Identity, ReLU, Sequential, LookupTable, Sum, Parallel, Softmax

def parse_babi_task(data_files, dictionary, include_question):
    """ Parse bAbI data.
    Args:
       data_files (list): a list of data file's paths.
       dictionary (dict): word's dictionary
       include_question (bool): whether count question toward input sentence.
    Returns:
        A tuple of (story, questions, qstory):
            story (3-D array)
                [position of word in sentence, sentence index, story index] = index of word in dictionary
            questions (2-D array)
                [0-9, question index], in which the first component is encoded as follows:
                    0 - story index
                    1 - index of the last sentence before the question
                    2 - index of the answer word in dictionary
                    3 to 13 - indices of supporting sentence
                    14 - line index
            qstory (2-D array) question's indices within a story
                [index of word in question, question index] = index of word in dictionary
    """
    # Try to reserve spaces beforehand (large matrices for both 1k and 10k data sets)
    # maximum number of words in sentence = 20
    story     = np.zeros((20, 500, len(data_files) * 3500), np.int16)
    questions = np.zeros((14, len(data_files) * 10000), np.int16)
    qstory    = np.zeros((20, len(data_files) * 10000), np.int16)

    # NOTE: question's indices are not reset when going through a new story
    story_idx, question_idx, sentence_idx, max_words, max_sentences = -1, -1, -1, 0, 0

    # Mapping line number (within a story) to sentence's index (to support the flag include_question)
    mapping = None

    for fp in data_files:
        with open(fp) as f:
            for line_idx, line in enumerate(f):
                line = line.rstrip().lower()
                words = line.split()

                # Story begins
                if words[0] == '1':
                    story_idx += 1
                    sentence_idx = -1
                    mapping = []

                # FIXME: This condition makes the code more fragile!
                if '?' not in line:
                    is_question = False
                    sentence_idx += 1
                else:
                    is_question = True
                    question_idx += 1
                    questions[0, question_idx] = story_idx
                    questions[1, question_idx] = sentence_idx
                    if include_question:
                        sentence_idx += 1

                mapping.append(sentence_idx)

                # Skip substory index
                for k in range(1, len(words)):
                    w = words[k]

                    if w.endswith('.') or w.endswith('?'):
                        w = w[:-1]

                    if w not in dictionary:
                        dictionary[w] = len(dictionary)

                    if max_words < k:
                        max_words = k

                    if not is_question:
                        story[k - 1, sentence_idx, story_idx] = dictionary[w]
                    else:
                        qstory[k - 1, question_idx] = dictionary[w]
                        if include_question:
                            story[k - 1, sentence_idx, story_idx] = dictionary[w]

                        # NOTE: Punctuation is already removed from w
                        if words[k].endswith('?'):
                            answer = words[k + 1]
                            if answer not in dictionary:
                                dictionary[answer] = len(dictionary)

                            questions[2, question_idx] = dictionary[answer]

                            # Indices of supporting sentences
                            for h in range(k + 2, len(words)):
                                questions[1 + h - k, question_idx] = mapping[int(words[h]) - 1]

                            questions[-1, question_idx] = line_idx
                            break

                if max_sentences < sentence_idx + 1:
                    max_sentences = sentence_idx + 1

    story     = story[:max_words, :max_sentences, :(story_idx + 1)]
    questions = questions[:, :(question_idx + 1)]
    qstory    = qstory[:max_words, :(question_idx + 1)]

    return story, questions, qstory

class Progress(object):
    """
    Progress bar
    """

    def __init__(self, iterable, bar_length=50):
        self.iterable = iterable
        self.bar_length = bar_length
        self.total_length = len(iterable)
        self.start_time = time.time()
        self.count = 0

    def __iter__(self):
        for obj in self.iterable:
            yield obj
            self.count += 1
            percent = self.count / self.total_length
            print_length = int(percent * self.bar_length)
            progress = "=" * print_length + " " * (self.bar_length - print_length)
            elapsed_time = time.time() - self.start_time
            print_msg = "\r|%s| %.0f%% %.1fs" % (progress, percent * 100, elapsed_time)
            sys.stdout.write(print_msg)
            if self.count == self.total_length:
                sys.stdout.write("\r" + " " * len(print_msg) + "\r")
            sys.stdout.flush()<commit_msg>fixed comma problem that xxx,xxx sticks together, which is wrong<commit_after>
from __future__ import division

import re
import sys
import time
import glob
import numpy as np

_WORD_SPLIT = re.compile("([, !?\"':;)(])")

def parse_babi_task(data_files, dictionary, include_question):
    """ Parse bAbI data.
    Args:
       data_files (list): a list of data file's paths.
       dictionary (dict): word's dictionary
       include_question (bool): whether count question toward input sentence.
    Returns:
        A tuple of (story, questions, qstory):
            story (3-D array)
                [position of word in sentence, sentence index, story index] = index of word in dictionary
            questions (2-D array)
                [0-9, question index], in which the first component is encoded as follows:
                    0 - story index
                    1 - index of the last sentence before the question
                    2 to 3 - index of the answer word in dictionary
                    4 to 13 - indices of supporting sentence
                    14 - line index
            qstory (2-D array) question's indices within a story
                [index of word in question, question index] = index of word in dictionary
    """
    # Try to reserve spaces beforehand (large matrices for both 1k and 10k data sets)
    # maximum number of words in sentence = 20
    story     = np.zeros((20, 500, len(data_files) * 3500), np.int16)
    questions = np.zeros((14, len(data_files) * 10000), np.int16)
    qstory    = np.zeros((20, len(data_files) * 10000), np.int16)

    # NOTE: question's indices are not reset when going through a new story
    story_idx, question_idx, sentence_idx, max_words, max_sentences = -1, -1, -1, 0, 0

    # Mapping line number (within a story) to sentence's index (to support the flag include_question)
    mapping = None

    for fp in data_files:
        with open(fp) as f:
            for line_idx, line in enumerate(f):
                line = line.replace(',',' ')
                line = line.rstrip().lower()
                words = line.split()
                print words

                # Story begins
                if words[0] == '1':
                    story_idx += 1
                    sentence_idx = -1
                    mapping = []

                # FIXME: This condition makes the code more fragile!
                if '?' not in line:
                    is_question = False
                    sentence_idx += 1
                else:
                    is_question = True
                    question_idx += 1
                    questions[0, question_idx] = story_idx
                    questions[1, question_idx] = sentence_idx
                    if include_question:
                        sentence_idx += 1

                mapping.append(sentence_idx)

                # Skip substory index
                for k in range(1, len(words)):
                    w = words[k]

                    if w.endswith('.') or w.endswith('?'):
                        w = w[:-1]

                    if w not in dictionary:
                        dictionary[w] = len(dictionary)

                    if max_words < k:
                        max_words = k

                    if not is_question:
                        story[k - 1, sentence_idx, story_idx] = dictionary[w]
                    else:
                        qstory[k - 1, question_idx] = dictionary[w]
                        if include_question:
                            story[k - 1, sentence_idx, story_idx] = dictionary[w]

                        # NOTE: Punctuation is already removed from w
                        if words[k].endswith('?'):
                            answer = words[k + 1]
                            flag = 2
                            while not words[k+flag].isdigit():
                                answer += ' '
                                answer += words[k+flag]
                                flag += 1
                            if answer not in dictionary:
                                dictionary[answer] = len(dictionary)

                            questions[2, question_idx] = dictionary[answer]

                            # Indices of supporting sentences
                            for h in range(k + flag, len(words)):
                                questions[1 + h - k, question_idx] = mapping[int(words[h]) - 1]

                            questions[-1, question_idx] = line_idx
                            break

                if max_sentences < sentence_idx + 1:
                    max_sentences = sentence_idx + 1

    story     = story[:max_words, :max_sentences, :(story_idx + 1)]
    questions = questions[:, :(question_idx + 1)]
    qstory    = qstory[:max_words, :(question_idx + 1)]

    return story, questions, qstory

class Progress(object):
    """
    Progress bar
    """

    def __init__(self, iterable, bar_length=50):
        self.iterable = iterable
        self.bar_length = bar_length
        self.total_length = len(iterable)
        self.start_time = time.time()
        self.count = 0

    def __iter__(self):
        for obj in self.iterable:
            yield obj
            self.count += 1
            percent = self.count / self.total_length
            print_length = int(percent * self.bar_length)
            progress = "=" * print_length + " " * (self.bar_length - print_length)
            elapsed_time = time.time() - self.start_time
            print_msg = "\r|%s| %.0f%% %.1fs" % (progress, percent * 100, elapsed_time)
            sys.stdout.write(print_msg)
            if self.count == self.total_length:
                sys.stdout.write("\r" + " " * len(print_msg) + "\r")
            sys.stdout.flush()

# data_dir = "bAbI_data/en"
# train_data_path = glob.glob('%s/qa*_*_train.txt' % data_dir)
# dictionary = {"nil": 0}
# story, qustions, qstroy = parse_babi_task(train_data_path, dictionary, False)
# # for e in story:
#     for ee in e:
#         print ('story:', ee)

# print ('qustions:', qustions)
# print ('qstroy:', qstroy)<|endoftext|>


<commit_before>import codecs
import glob
import json
import random

import numpy as np

class Vocabulary(object):

    def __init__(self):
        self._token_to_id = {}
        self._token_to_count = {}
        self._id_to_token = []
        self._num_tokens = 0
        self._s_id = None
        self._unk_id = None

    @property
    def num_tokens(self):
        return self._num_tokens

    @property
    def unk(self):
        return "<UNK>"

    @property
    def unk_id(self):
        return self._unk_id

    @property
    def s(self):
        return "<S>"

    @property
    def s_id(self):
        return self._s_id

    def add(self, token, count):
        self._token_to_id[token] = self._num_tokens
        self._token_to_count[token] = count
        self._id_to_token.append(token)
        self._num_tokens += 1

    def finalize(self):
        self._s_id = self.get_id(self.s)
        self._unk_id = self.get_id(self.unk)

    def get_id(self, token):
        return self._token_to_id.get(token, self.unk_id)

    def get_token(self, id_):
        return self._id_to_token[id_]

    @staticmethod
    def from_file(filename):
        vocab = Vocabulary()
        with codecs.open(filename, "r", "utf-8") as f:
            for line in f:
                word, count = line.strip().split()
                vocab.add(word, int(count))
        vocab.finalize()
        return vocab

class Dataset(object):

    def __init__(self, vocab, file_pattern, deterministic=False):
        self._vocab = vocab
        self._file_pattern = file_pattern
        self._deterministic = deterministic

    def _parse_sentence(self, line):
        s_id = self._vocab.s_id
        return [s_id] + [self._vocab.get_id(word) for word in line.strip().split()] + [s_id]

    def _parse_file(self, file_name):
        print("Processing file: %s" % file_name)
        with codecs.open(file_name, "r", "utf-8") as f:
            lines = [line.strip() for line in f]
            if not self._deterministic:
                random.shuffle(lines)
            print("Finished processing!")
            for line in lines:
                yield self._parse_sentence(line)

    def _sentence_stream(self, file_stream):
        for file_name in file_stream:
            for sentence in self._parse_file(file_name):
                yield sentence

    def _iterate(self, sentences, batch_size, num_steps):

        streams = [None] * batch_size
        x = np.zeros([batch_size, num_steps], np.int32)
        y = np.zeros([batch_size, num_steps], np.int32)
        #w = np.zeros([batch_size, num_steps], np.uint8)
        do_iters = True
        while do_iters:
            x[:] = 0
            y[:] = 0
            #w[:] = 0
            for i in range(batch_size):
                tokens_filled = 0
                try:
                    while tokens_filled < num_steps:
                        if streams[i] is None or len(streams[i]) <= 1:
                            streams[i] = next(sentences)
                        print('\x1b[6;30;42m' + '~~~~~>>Almog&Dor debug: streams at ',i,'position: ', streams[i], '\x1b[0m')
                        num_tokens = min(len(streams[i]) - 1, num_steps - tokens_filled)
                        x[i, tokens_filled:tokens_filled+num_tokens] = streams[i][:num_tokens]
                        y[i, tokens_filled:tokens_filled + num_tokens] = streams[i][1:num_tokens+1]
                        #w[i, tokens_filled:tokens_filled + num_tokens] = 1
                        streams[i] = streams[i][num_tokens:]
                        tokens_filled += num_tokens
                except StopIteration:
                    do_iters = False
            #if not np.any(w):
            #    return

            #yield x, y, w
            yield x, y

    def iterate_once(self, batch_size, num_steps):
        def file_stream():
            for file_name in glob.glob(self._file_pattern):
                yield file_name
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value

    def iterate_forever(self, batch_size, num_steps):
        def file_stream():
            epoch_num = 0
            while True:
                file_patterns = glob.glob(self._file_pattern)
                if not self._deterministic:
                    random.shuffle(file_patterns)
                for file_name in file_patterns:
                    yield file_name
                print('Done epoch: %s' % epoch_num)
                epoch_num += 1
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value
<commit_msg>removed unnecessary debug print<commit_after>import codecs
import glob
import json
import random

import numpy as np

class Vocabulary(object):

    def __init__(self):
        self._token_to_id = {}
        self._token_to_count = {}
        self._id_to_token = []
        self._num_tokens = 0
        self._s_id = None
        self._unk_id = None

    @property
    def num_tokens(self):
        return self._num_tokens

    @property
    def unk(self):
        return "<UNK>"

    @property
    def unk_id(self):
        return self._unk_id

    @property
    def s(self):
        return "<S>"

    @property
    def s_id(self):
        return self._s_id

    def add(self, token, count):
        self._token_to_id[token] = self._num_tokens
        self._token_to_count[token] = count
        self._id_to_token.append(token)
        self._num_tokens += 1

    def finalize(self):
        self._s_id = self.get_id(self.s)
        self._unk_id = self.get_id(self.unk)

    def get_id(self, token):
        return self._token_to_id.get(token, self.unk_id)

    def get_token(self, id_):
        return self._id_to_token[id_]

    @staticmethod
    def from_file(filename):
        vocab = Vocabulary()
        with codecs.open(filename, "r", "utf-8") as f:
            for line in f:
                word, count = line.strip().split()
                vocab.add(word, int(count))
        vocab.finalize()
        return vocab

class Dataset(object):

    def __init__(self, vocab, file_pattern, deterministic=False):
        self._vocab = vocab
        self._file_pattern = file_pattern
        self._deterministic = deterministic

    def _parse_sentence(self, line):
        s_id = self._vocab.s_id
        return [s_id] + [self._vocab.get_id(word) for word in line.strip().split()] + [s_id]

    def _parse_file(self, file_name):
        print("Processing file: %s" % file_name)
        with codecs.open(file_name, "r", "utf-8") as f:
            lines = [line.strip() for line in f]
            if not self._deterministic:
                random.shuffle(lines)
            print("Finished processing!")
            for line in lines:
                yield self._parse_sentence(line)

    def _sentence_stream(self, file_stream):
        for file_name in file_stream:
            for sentence in self._parse_file(file_name):
                yield sentence

    def _iterate(self, sentences, batch_size, num_steps):

        streams = [None] * batch_size
        x = np.zeros([batch_size, num_steps], np.int32)
        y = np.zeros([batch_size, num_steps], np.int32)
        #w = np.zeros([batch_size, num_steps], np.uint8)
        do_iters = True
        while do_iters:
            x[:] = 0
            y[:] = 0
            #w[:] = 0
            for i in range(batch_size):
                tokens_filled = 0
                try:
                    while tokens_filled < num_steps:
                        if streams[i] is None or len(streams[i]) <= 1:
                            streams[i] = next(sentences)
                        num_tokens = min(len(streams[i]) - 1, num_steps - tokens_filled)
                        x[i, tokens_filled:tokens_filled+num_tokens] = streams[i][:num_tokens]
                        y[i, tokens_filled:tokens_filled + num_tokens] = streams[i][1:num_tokens+1]
                        #w[i, tokens_filled:tokens_filled + num_tokens] = 1
                        streams[i] = streams[i][num_tokens:]
                        tokens_filled += num_tokens
                except StopIteration:
                    do_iters = False
            #if not np.any(w):
            #    return

            #yield x, y, w
            yield x, y

    def iterate_once(self, batch_size, num_steps):
        def file_stream():
            for file_name in glob.glob(self._file_pattern):
                yield file_name
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value

    def iterate_forever(self, batch_size, num_steps):
        def file_stream():
            epoch_num = 0
            while True:
                file_patterns = glob.glob(self._file_pattern)
                if not self._deterministic:
                    random.shuffle(file_patterns)
                for file_name in file_patterns:
                    yield file_name
                print('Done epoch: %s' % epoch_num)
                epoch_num += 1
        for value in self._iterate(self._sentence_stream(file_stream()), batch_size, num_steps):
            yield value
<|endoftext|>


<commit_before>"""
Instrumentation for FastAPI
https://fastapi.tiangolo.com/
"""
try:
    import fastapi
    import wrapt
    import signal
    import os

    from ..log import logger
    from ..util import running_in_gunicorn
    from .asgi import InstanaASGIMiddleware
    from starlette.middleware import Middleware

    @wrapt.patch_function_wrapper('fastapi.applications', 'FastAPI.__init__')
    def init_with_instana(wrapped, instance, args, kwargs):
        middleware = kwargs.get('middleware')
        if middleware is None:
            kwargs['middleware'] = [Middleware(InstanaASGIMiddleware)]
        elif isinstance(middleware, list):
            middleware.append(Middleware(InstanaASGIMiddleware))

        return wrapped(*args, **kwargs)

    logger.debug("Instrumenting FastAPI")

    # Reload GUnicorn when we are instrumenting an already running application
    if "INSTANA_MAGIC" in os.environ and running_in_gunicorn():
        os.kill(os.getpid(), signal.SIGHUP)

except ImportError:
    pass<commit_msg>FastAPI: Load only when supported version in use (#284)<commit_after>"""
Instrumentation for FastAPI
https://fastapi.tiangolo.com/
"""
try:
    import fastapi
    import os
    import wrapt
    import signal
    from distutils.version import LooseVersion

    from ..log import logger
    from ..util import running_in_gunicorn
    from .asgi import InstanaASGIMiddleware
    from starlette.middleware import Middleware

    if hasattr(fastapi, '__version__') and \
        (LooseVersion(fastapi.__version__) >= LooseVersion('0.51.0')):

        @wrapt.patch_function_wrapper('fastapi.applications', 'FastAPI.__init__')
        def init_with_instana(wrapped, instance, args, kwargs):
            middleware = kwargs.get('middleware')
            if middleware is None:
                kwargs['middleware'] = [Middleware(InstanaASGIMiddleware)]
            elif isinstance(middleware, list):
                middleware.append(Middleware(InstanaASGIMiddleware))

            return wrapped(*args, **kwargs)

        logger.debug("Instrumenting FastAPI")

        # Reload GUnicorn when we are instrumenting an already running application
        if "INSTANA_MAGIC" in os.environ and running_in_gunicorn():
            os.kill(os.getpid(), signal.SIGHUP)
    else:
        logger.debug("Instana supports FastAPI package versions 0.51.0 and newer.  Skipping.")

except ImportError:
    pass
<|endoftext|>


<commit_before>"""
This is the default renderer.  It tries to match the behavior of the
blosxom renderer.
"""

from Pyblosxom import tools
from Pyblosxom.renderers.base import RendererBase
import re, os, sys, codecs
try:
    from xml.sax.saxutils import escape
except ImportError:
    from cgi import escape

# Ugly default templates, have to though :(
HTML = {'content_type' : 'text/html',
        'head' : """<link rel="alternate" type="application/rss+xml" title="RSS" href="$url/?flav=rss" /><title>$blog_title $pi_da $pi_mo $pi_yr</title>$blog_title
$pi_da $pi_mo $pi_yr
""",
        'date_head' : '
\n$date',
        'story' : """
$title
$body

posted at: $ti | path: /$absolute_path | permanent link to this entry
\n""",
        'date_foot' : '
',
        'foot' : """
"""}

RSS = {'content_type' : 'text/xml',
       'head' : """<?xml version="1.0"?>\n\n<!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" "http://my.netscape.com/publish/formats/rss-0.91.dtd">\n\n<rss version="0.91">\n  <channel>\n    <title>$blog_title $pi_da $pi_mo $pi_yr</title>\n    <link>$url</link>\n    <description>$blog_description</description>\n    <language>$blog_language</language>\n""",
       'story' : """<item>\n    <title>$title</title>\n    <link>$base_url/$file_path.html</link>\n    <description>$body</description>\n  </item>\n""",
       'foot' : '   </channel>\n</rss>'}

RSS3 = {'content_type' : 'text/plain',
        'head' : """title: $blog_title\ndescription: $blog_description\nlink: $url\ncreator: $blog_author\nerrorsTo: $blog_author\nlang: $blog_language\n\n\n""",
        'story' :  """title: $title\nlink: $base_url/$file_path.html\ncreated: $w3cdate\nsubject: $path\nguid: $file_path\n\n""",
        'foot' : ''}

ESF = {'content_type' : 'text/plain',
        'head' : """title: $blog_title\ncontact: contact@example.com (The Contact Person)\nlink: $url\n\n\n""",
        'story' :  """$mtime\t$title\t$base_url/$file_path\n""",
        'foot' : ''}

ERROR = {'content_type' : 'text/plain',
         'head' : """ Error: I'm afraid this is the first I've heard of a "$flavour" flavoured pyblosxom.\n Try dropping the "?flav=$flavour" bit from the end of the URL.\n\n"""}

DEFAULT_FLAVOURS = {'html' : HTML,
                    'rss' : RSS,
                    'error' : ERROR,
                    'rss3' : RSS3,
                    'esf' : ESF}

class BlosxomRenderer(RendererBase):
    def __init__(self, request, stdoutput = sys.stdout):
        RendererBase.__init__(self, request, stdoutput)
        config = request.getConfiguration()
        (e, d, sr, sw) = codecs.lookup(config.get('blog_encoding',
                'iso-8859-1'))
        self._out = sw(self._out)
        self.dayFlag = 1
        self._request = request
        self._encoding = config.get("blog_encoding", "iso-8859-1")

    def _getFlavour(self, taste='html'):
        """
        Flavours, or views, or templates, as some may call it, defaults are
        given, but can be overidden with files on the datadir. Don't like the
        default html templates, add your own, head.html, story.html etc.
        """
        data = self._request.getData()
        config = self._request.getConfiguration()

        pattern = re.compile(r'.+?\.' + taste + '$')

        datadir = config["datadir"]

        dirname = data["root_datadir"]
        if os.path.isfile(dirname):
            dirname = os.path.dirname(dirname)

        template_files = None
        while len(dirname) >= len(datadir):
            template_files = tools.Walk(self._request, dirname, 1, pattern)
            if template_files: break
            dirname = os.path.split(dirname)[0]

        if not template_files:
            template_files = tools.Walk(self._request, config['datadir'], 1, pattern)

        # we grab a copy of the templates for the taste we want
        flavour = {}
        flavour.update(DEFAULT_FLAVOURS.get(taste, {}))

        # we update the flavours dict with what we found
        for filename in template_files:
            flavouring = os.path.basename(filename).split('.')
            flav_template = unicode(open(filename).read(),
                    config.get('blog_encoding', 'iso-8859-1'))

            flavour[flavouring[0]] = flav_template

        if not flavour:
            return DEFAULT_FLAVOURS["error"]

        return flavour

    def _printTemplate(self, entry, template):
        """
        @param entry: either a dict or the Entry object
        @type  entry: dict or Entry

        @param template: the template string
        @type  template: string

        @returns: the content string
        @rtype: string
        """
        if template:
            template = unicode(template)
            finaltext = tools.parse(self._request, self._encoding, entry, template)
            return finaltext.replace(r'\$', '$')
        return ""

    def _processEntry(self, entry, current_date):
        """
        Main workhorse of pyblosxom stories, comments and other miscelany goes
        here

        @param entry: either a dict or an Entry object
        @type  entry: dict or Entry object
        """
        data = self._request.getData()
        config = self._request.getConfiguration()

        output = []

        if re.search(r'\Wxml', data['content-type']):
            quotes = {"'":"'", '"':"""}
            entry['title'] = escape(entry['title'],quotes)
            entry.setData(escape(entry.getData(),quotes))

        elif data['content-type'] == 'text/plain':
            s = tools.Stripper()
            s.feed(entry.getData())
            s.close()
            p = ['  ' + line for line in s.gettext().split('\n')]
            entry.setData('\n'.join(p))

        entry.update(data)
        entry.update(config)

        if entry['date'] != current_date:
            current_date = entry['date']
            if not self.dayFlag:
                self.outputTemplate(output, entry, 'date_foot')
            self.dayFlag = 0
            self.outputTemplate(output, entry, 'date_head')

        self.outputTemplate(output, entry, 'story', override=1)

        template = u""
        args = self._run_callback("story_end", { "entry": entry, "template": template })

        return "".join(output) + args['template'], current_date

    def _processContent(self):
        """
        Processes the content for the story portion of a page.

        @returns: the content string
        @rtype: string
        """
        config = self._request.getConfiguration()
        data = self._request.getData()

        outputbuffer = []

        content_type = type(self._content)
        if callable(self._content):
            # if the content is a callable function, then we just spit out
            # whatever it returns as a string
            outputbuffer.append(self._content())

        elif content_type is type({}):
            # if the content is a dict, then we parse it as if it were an
            # entry--except it's distinctly not an EntryBase derivative
            self._content.update(data)
            output = tools.parse(self._request, self._encoding, self._content, self.flavour['story'])
            outputbuffer.append(output)

        elif content_type is type([]):
            current_date = ''

            maxcount = config['num_entries']
            if maxcount and len(self._content) > maxcount:
                self._content = self._content[:maxcount]

            for entry in self._content:
                # FIXME - commented this next line out
                # data.update(entry)
                output, current_date = self._processEntry(entry, current_date)
                outputbuffer.append(output)

        return self.write(u"".join(outputbuffer))

    def render(self, header = 1):
        """
        Figures out flavours and such and then renders the content according
        to which flavour we're using.

        @param header: whether (1) or not (0) to render the HTTP headers
        @type  header: boolean
        """
        data = self._request.getData()
        config = self._request.getConfiguration()

        parsevars = {}
        for mem in config.keys():
            parsevars[mem] = config[mem]

        for mem in data.keys():
            parsevars[mem] = data[mem]

        self.flavour = self._getFlavour(data.get('flavour', 'html'))
        data['content-type'] = self.flavour['content_type'].strip()
        if header:
            if self._needs_content_type and data['content-type'] !="":
                self.addHeader('Content-type', '%(content-type)s' % data)

            self.showHeaders()

        if self._content:
            if self.flavour.has_key('head'):
                self._outputFlavour(parsevars,'head')
            if self.flavour.has_key('story'):
                self._processContent()
            if self.flavour.has_key('date_foot'):
                self._outputFlavour(parsevars,'date_foot')
            if self.flavour.has_key('foot'):
                self._outputFlavour(parsevars,'foot')

        self.rendered = 1

        # FIXME - we might want to do this at a later point?
        cache = tools.get_cache(self._request)
        if cache:
            cache.close()

    def _outputFlavour(self, entry, template_name):
        """
        Find the flavour template for template_name, run any blosxom callbacks,
        substitute vars into it and write the template to the output

        @param entry: the EntryBase object
        @type entry: L{Pyblosxom.entries.base.EntryBase}

        @param template_name: - name of the flavour template
        @type template_name: string
        """
        template = self.flavour[template_name]

        args = self._run_callback(template_name, { "entry": entry, "template": template })
        template = args["template"]
        entry = args["entry"]

        self.write(tools.parse(self._request, self._encoding, entry, template))

    def outputTemplate(self, output, entry, template_name, override=0):
        """
        Find the flavour template for template_name, run any blosxom callbacks,
        substitute entry into it and append the template to the output.

        If the entry has a "template_name" property and override is 1
        (this happens in the story template), then we'll use that
        template instead.

        @param output: list of strings of the output
        @type output: list

        @param entry: the entry to render with this flavour template
        @type entry: L{Pyblosxom.entries.base.EntryBase}

        @param template_name: name of the flavour template to use
        @type template_name: string

        @param override: whether (1) or not (0) this template can
            be overriden with the "template_name" property of the entry
        @type  override: boolean
        """
        template = ""
        if override == 1:
            # here we do a quick override...  if the entry has a template
            # field we use that instead of the template_name argument
            # passed in.
            actual_template_name = entry.get("template_name", template_name)
            template = self.flavour.get(actual_template_name, '')

        if not template:
            template = self.flavour.get(template_name, '')

        # we run this through the regular callbacks
        args = self._run_callback(template_name, { "entry": entry, "template": template })

        template = args["template"]
        entry = args["entry"]

        output.append(self._printTemplate(entry, template))

    def _run_callback(self, chain, input):
        """
        Makes calling blosxom callbacks a bit easier since they all have the
        same mechanics.  This function merely calls run_callback with
        the arguments given and a mappingfunc.

        The mappingfunc copies the "template" value from the output to the
        input for the next function.

        Refer to run_callback for more details.
        """
        input.update( { "renderer": self } )
        input.update( { "request": self._request } )

        return tools.run_callback(chain, input,
                            mappingfunc=lambda x,y: x,
                            defaultfunc=lambda x:x)

    def getContent(self):
        """
        Return the content field

        This is exposed for blosxom callbacks.

        @returns: content
        """
        return self._content

class Renderer(BlosxomRenderer):
    pass

# vim: shiftwidth=4 tabstop=4 expandtab
<commit_msg>Removed RSS3 and ESF default flavours.  Adjusted the HTML flavour so that it validates as valid HTML 4.01 transitional (though it's missing the doctype and things of that nature).<commit_after>"""
This is the default renderer.  It tries to match the behavior of the
blosxom renderer.
"""

from Pyblosxom import tools
from Pyblosxom.renderers.base import RendererBase
import re, os, sys, codecs
try:
    from xml.sax.saxutils import escape
except ImportError:
    from cgi import escape

# Ugly default templates, have to though :(
HTML = {'content_type' : 'text/html',
        'head' : """<title>$blog_title $pi_da $pi_mo $pi_yr</title>
$blog_title

$pi_da $pi_mo $pi_yr
""",
        'date_head' : '
\n$date',
        'story' : """
$title
$body

posted at: $ti | path: /$absolute_path | permanent link to this entry
\n""",
        'date_foot' : '
',
        'foot' : """

"""}

RSS = {'content_type' : 'text/xml',
       'head' : """<?xml version="1.0"?>\n\n<!DOCTYPE rss PUBLIC "-//Netscape Communications//DTD RSS 0.91//EN" "http://my.netscape.com/publish/formats/rss-0.91.dtd">\n\n<rss version="0.91">\n  <channel>\n    <title>$blog_title $pi_da $pi_mo $pi_yr</title>\n    <link>$url</link>\n    <description>$blog_description</description>\n    <language>$blog_language</language>\n""",
       'story' : """<item>\n    <title>$title</title>\n    <link>$base_url/$file_path.html</link>\n    <description>$body</description>\n  </item>\n""",
       'foot' : '   </channel>\n</rss>'}

ERROR = {'content_type' : 'text/plain',
         'head' : """Error: I'm afraid this is the first I've heard of a "$flavour" flavoured pyblosxom.\n Try dropping the "?flav=$flavour" bit from the end of the URL.\n\n"""}

DEFAULT_FLAVOURS = {'html' : HTML,
                    'rss' : RSS,
                    'error' : ERROR}

class BlosxomRenderer(RendererBase):
    def __init__(self, request, stdoutput = sys.stdout):
        RendererBase.__init__(self, request, stdoutput)
        config = request.getConfiguration()
        (e, d, sr, sw) = codecs.lookup(config.get('blog_encoding',
                'iso-8859-1'))
        self._out = sw(self._out)
        self.dayFlag = 1
        self._request = request
        self._encoding = config.get("blog_encoding", "iso-8859-1")

    def _getFlavour(self, taste='html'):
        """
        Flavours, or views, or templates, as some may call it, defaults are
        given, but can be overidden with files on the datadir. Don't like the
        default html templates, add your own, head.html, story.html etc.
        """
        data = self._request.getData()
        config = self._request.getConfiguration()

        pattern = re.compile(r'.+?\.' + taste + '$')

        datadir = config["datadir"]

        dirname = data["root_datadir"]
        if os.path.isfile(dirname):
            dirname = os.path.dirname(dirname)

        template_files = None
        while len(dirname) >= len(datadir):
            template_files = tools.Walk(self._request, dirname, 1, pattern)
            if template_files: break
            dirname = os.path.split(dirname)[0]

        if not template_files:
            template_files = tools.Walk(self._request, config['datadir'], 1, pattern)

        # we grab a copy of the templates for the taste we want
        flavour = {}
        flavour.update(DEFAULT_FLAVOURS.get(taste, {}))

        # we update the flavours dict with what we found
        for filename in template_files:
            flavouring = os.path.basename(filename).split('.')
            flav_template = unicode(open(filename).read(),
                    config.get('blog_encoding', 'iso-8859-1'))

            flavour[flavouring[0]] = flav_template

        if not flavour:
            return DEFAULT_FLAVOURS["error"]

        return flavour

    def _printTemplate(self, entry, template):
        """
        @param entry: either a dict or the Entry object
        @type  entry: dict or Entry

        @param template: the template string
        @type  template: string

        @returns: the content string
        @rtype: string
        """
        if template:
            template = unicode(template)
            finaltext = tools.parse(self._request, self._encoding, entry, template)
            return finaltext.replace(r'\$', '$')
        return ""

    def _processEntry(self, entry, current_date):
        """
        Main workhorse of pyblosxom stories, comments and other miscelany goes
        here

        @param entry: either a dict or an Entry object
        @type  entry: dict or Entry object
        """
        data = self._request.getData()
        config = self._request.getConfiguration()

        output = []

        if re.search(r'\Wxml', data['content-type']):
            quotes = {"'":"'", '"':"""}
            entry['title'] = escape(entry['title'],quotes)
            entry.setData(escape(entry.getData(),quotes))

        elif data['content-type'] == 'text/plain':
            s = tools.Stripper()
            s.feed(entry.getData())
            s.close()
            p = ['  ' + line for line in s.gettext().split('\n')]
            entry.setData('\n'.join(p))

        entry.update(data)
        entry.update(config)

        if entry['date'] != current_date:
            current_date = entry['date']
            if not self.dayFlag:
                self.outputTemplate(output, entry, 'date_foot')
            self.dayFlag = 0
            self.outputTemplate(output, entry, 'date_head')

        self.outputTemplate(output, entry, 'story', override=1)

        template = u""
        args = self._run_callback("story_end", { "entry": entry, "template": template })

        return "".join(output) + args['template'], current_date

    def _processContent(self):
        """
        Processes the content for the story portion of a page.

        @returns: the content string
        @rtype: string
        """
        config = self._request.getConfiguration()
        data = self._request.getData()

        outputbuffer = []

        content_type = type(self._content)
        if callable(self._content):
            # if the content is a callable function, then we just spit out
            # whatever it returns as a string
            outputbuffer.append(self._content())

        elif content_type is type({}):
            # if the content is a dict, then we parse it as if it were an
            # entry--except it's distinctly not an EntryBase derivative
            self._content.update(data)
            output = tools.parse(self._request, self._encoding, self._content, self.flavour['story'])
            outputbuffer.append(output)

        elif content_type is type([]):
            current_date = ''

            maxcount = config['num_entries']
            if maxcount and len(self._content) > maxcount:
                self._content = self._content[:maxcount]

            for entry in self._content:
                # FIXME - commented this next line out
                # data.update(entry)
                output, current_date = self._processEntry(entry, current_date)
                outputbuffer.append(output)

        return self.write(u"".join(outputbuffer))

    def render(self, header = 1):
        """
        Figures out flavours and such and then renders the content according
        to which flavour we're using.

        @param header: whether (1) or not (0) to render the HTTP headers
        @type  header: boolean
        """
        data = self._request.getData()
        config = self._request.getConfiguration()

        parsevars = {}
        for mem in config.keys():
            parsevars[mem] = config[mem]

        for mem in data.keys():
            parsevars[mem] = data[mem]

        self.flavour = self._getFlavour(data.get('flavour', 'html'))
        data['content-type'] = self.flavour['content_type'].strip()
        if header:
            if self._needs_content_type and data['content-type'] !="":
                self.addHeader('Content-type', '%(content-type)s' % data)

            self.showHeaders()

        if self._content:
            if self.flavour.has_key('head'):
                self._outputFlavour(parsevars,'head')
            if self.flavour.has_key('story'):
                self._processContent()
            if self.flavour.has_key('date_foot'):
                self._outputFlavour(parsevars,'date_foot')
            if self.flavour.has_key('foot'):
                self._outputFlavour(parsevars,'foot')

        self.rendered = 1

        # FIXME - we might want to do this at a later point?
        cache = tools.get_cache(self._request)
        if cache:
            cache.close()

    def _outputFlavour(self, entry, template_name):
        """
        Find the flavour template for template_name, run any blosxom callbacks,
        substitute vars into it and write the template to the output

        @param entry: the EntryBase object
        @type entry: L{Pyblosxom.entries.base.EntryBase}

        @param template_name: - name of the flavour template
        @type template_name: string
        """
        template = self.flavour[template_name]

        args = self._run_callback(template_name, { "entry": entry, "template": template })
        template = args["template"]
        entry = args["entry"]

        self.write(tools.parse(self._request, self._encoding, entry, template))

    def outputTemplate(self, output, entry, template_name, override=0):
        """
        Find the flavour template for template_name, run any blosxom callbacks,
        substitute entry into it and append the template to the output.

        If the entry has a "template_name" property and override is 1
        (this happens in the story template), then we'll use that
        template instead.

        @param output: list of strings of the output
        @type output: list

        @param entry: the entry to render with this flavour template
        @type entry: L{Pyblosxom.entries.base.EntryBase}

        @param template_name: name of the flavour template to use
        @type template_name: string

        @param override: whether (1) or not (0) this template can
            be overriden with the "template_name" property of the entry
        @type  override: boolean
        """
        template = ""
        if override == 1:
            # here we do a quick override...  if the entry has a template
            # field we use that instead of the template_name argument
            # passed in.
            actual_template_name = entry.get("template_name", template_name)
            template = self.flavour.get(actual_template_name, '')

        if not template:
            template = self.flavour.get(template_name, '')

        # we run this through the regular callbacks
        args = self._run_callback(template_name, { "entry": entry, "template": template })

        template = args["template"]
        entry = args["entry"]

        output.append(self._printTemplate(entry, template))

    def _run_callback(self, chain, input):
        """
        Makes calling blosxom callbacks a bit easier since they all have the
        same mechanics.  This function merely calls run_callback with
        the arguments given and a mappingfunc.

        The mappingfunc copies the "template" value from the output to the
        input for the next function.

        Refer to run_callback for more details.
        """
        input.update( { "renderer": self } )
        input.update( { "request": self._request } )

        return tools.run_callback(chain, input,
                            mappingfunc=lambda x,y: x,
                            defaultfunc=lambda x:x)

    def getContent(self):
        """
        Return the content field

        This is exposed for blosxom callbacks.

        @returns: content
        """
        return self._content

class Renderer(BlosxomRenderer):
    pass

# vim: shiftwidth=4 tabstop=4 expandtab
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, pydenticon, core, time, global_variables, requests, cloudinary
from PIL import Image, ImageChops, ImageEnhance, ImageFilter, ImageOps

is_sam_plugin = 1
name = "Images"
keywords = ["identicon", "schedule_h", "wallpaper"]
has_toggle = 0
has_set = 0

DEBUG = 1

def initialize():
    core.log(name, "Startup")
    core.log(name, "Hello World!")

def stop():
    core.log(name, "I'm not even running anymore!")

def get_wallpaper():
    core.log(name, "    Downloading the baseimage.")
    path = "/data/wallpaper_background.png"
    response = requests.get("https://earthview.withgoogle.com/")
    html = response.text
    regex = r"background-image: url\((?P<link>.+)\);"
    m = re.search(regex, html, re.IGNORECASE)
    if m:
        if m.group("link"):
            image_url = m.group("link")
        else:
            image_url = None
    else:
        image_url = None
    core.log(name, "    The URL is {}. Downloading to {}.".format(image_url, global_variables.folder_base + path))

    if image_url:
        f = open(global_variables.folder_base + path, "wb")
        f.write(requests.get(image_url).content)
        f.close()
    else:
        core.log(name, "    Error: Download not completed.")
    core.log(name, "    Download completed to {}.".format(global_variables.folder_base + path))
    return path

def resize(im, size, offset=(0,0)):
    bg = Image.new(im.mode, size)
    if im.size[0] < size[0]:
        factor_width = -1
    else:
        factor_width = 1
    if im.size[1] < size[1]:
        factor_height = -1
    else:
        factor_height = 1
    shift_left = factor_width * (((size[0] - im.size[0]) / 2) - offset[0])
    shift_top = factor_height * (((size[1] - im.size[1]) / 2) - offset[1])
    shift_right = size[0] - (factor_width * shift_left)
    shift_bottom = size[1] - (factor_height * shift_top)
    return im.crop((shift_left, shift_top, shift_right, shift_bottom))

def generate_wallpaper(background_path, mask_path, destination_path="/data/wallpaper.png"):

    core.log(name, "    Creating the final image")

    #generate the background

    core.log(name, "      Creating the background")
    bg_layer = Image.open(global_variables.folder_base + background_path)    #the background in color
    bg_layer = bg_layer.convert("RGBA")
    converter_color = ImageEnhance.Color(bg_layer)
    bg_layer = converter_color.enhance(0.05)
    converter_brightness = ImageEnhance.Brightness(bg_layer)
    bg_layer = converter_brightness.enhance(0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer.png")

    #generate the masks

    core.log(name, "      Creating the normal masks")
    size = bg_layer.size
    mask_BoW = Image.open(global_variables.folder_base + mask_path)     #"BoW" means black icon on white background.
    mask_WoB = ImageOps.invert(mask_BoW)                                #"WoB" is a white icon on black bg.
    if not mask_BoW.size == size:
        mask_WoB = resize(mask_WoB, size)
        mask_BoW = ImageOps.invert(mask_WoB)
    mask_WoB = mask_WoB.convert("1")
    mask_BoW = mask_BoW.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the normal masks")
        mask_WoB.save(global_variables.folder_base + "/data/mask_WoB.png")
        mask_BoW.save(global_variables.folder_base + "/data/mask_BoW.png")

    core.log(name, "      Creating the big masks")
    mask_BoT_big = mask_BoW.convert("RGBA")
    mask_BoT_big.putalpha(mask_WoB)
    core.log(name, "        Adding the Offset")
    offset_layers = []
    offsets = [(10,10),(-10,10),(10,-10),(-10,-10)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(mask_BoT_big, x, y))
    for offset_layer in offset_layers:
        mask_BoT_big.paste(offset_layer, None, offset_layer)
    core.log(name, "        Creating mask_BoT_big")
    mask_BoW_big =  Image.new("RGB", size, "white")
    mask_BoW_big.paste(mask_BoT_big, None, mask_BoT_big)
    mask_WoB_big = ImageOps.invert(mask_BoW_big)
    mask_WoB_big = mask_WoB_big.convert("1")
    mask_BoW_big = mask_BoW_big.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the big masks")
        mask_WoB_big.save(global_variables.folder_base + "/data/mask_WoB_big.png")
        mask_BoW_big.save(global_variables.folder_base + "/data/mask_BoW_big.png")

    #generate the colored overlay

    core.log(name, "      Creating the colored overlay")
    overlay_layer = Image.open(global_variables.folder_base + background_path)
    overlay_layer.putalpha(mask_WoB)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the colored overlay")
        overlay_layer.save(global_variables.folder_base + "/data/overlay_layer.png")

    #generate the frame around the colored overlay

    core.log(name, "      Creating the frame around the overlay")
    frame_layer = Image.open(global_variables.folder_base + background_path)
    frame_layer = Image.blend(Image.new("RGB", size, "white"), frame_layer, 0.8)
    frame_layer.putalpha(mask_WoB_big)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the frame")
        frame_layer.save(global_variables.folder_base + "/data/frame_layer.png")

    #generate the shadow

    core.log(name, "      Creating the dropshadow")
    shadow_layer =  mask_BoW.convert("RGBA")
    shadow_layer.putalpha(mask_WoB)
    """
    A copy if the mask is pasted ontop of itself with a 2px offset into the 4 diagonal directions each to increase it's size into each direction. (The shadow would otherwise disappear behind the colored overlay.)
    The whole layer is then blurred multiple times to increase the blur's effect.
    Finally, the shadow_layer is blended with a completely transparent layer to add some transparency to it.
    """

    offset_layers = []
    offsets = [(11,11),(-11,11),(11,-11),(-11,-11)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(shadow_layer, x, y))
    for offset_layer in offset_layers:
        shadow_layer.paste(offset_layer, None, offset_layer)

    core.log(name, "      Blurring the shadow")
    for n in range(3):      #as noted above, the Blur is applied multiple times for a stronger effect. 3 has proven as a good compromise between a good effect and not too much necessary calculation.
        core.log(name, "        Step {}/{}".format(n + 1, 3))
        shadow_layer = shadow_layer.filter(ImageFilter.BLUR)
    core.log(name, "      Adding transparency")
    shadow_layer = Image.blend(Image.new("RGBA", size), shadow_layer, 0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the dropshadow")
        shadow_layer.save(global_variables.folder_base + "/data/shadow_layer.png")

    #merge the shadow with the background

    """
    This will add the shadow manually to the background image. Usign PIL's merge() or composite_alpha() was causing the image underneath the shadow to turn greyish which looked pretty ugly on dark parts of the image.
    To prevent this behaviour, this tool iterates over the pixels of the whole image and changes the colorvalues according to the alphavalue of the corresponding image in the mask.
    If the pixel in the mask is completely transparent (a_mask == 0) the calculation will be skipped and the original pixel won't be changed. If the pixel in the mask is completely opaque (a_mask == 255) the pixel in the original image will be turned black. For every other alpha value the new color is calculated via new = old * (1 - (a_mask / 255)).
    """
    core.log(name, "      Merging the shadow with the background")
    try:
        pixels_changed_count = 0
        pixels_bg = bg_layer.load()
        pixels_mask = shadow_layer.load()
        for x in range(bg_layer.size[0]):
            for y in range(bg_layer.size[1]):
                r_mask, g_mask, b_mask, a_mask = pixels_mask[x, y]
                if 0 < a_mask < 255:
                    r_bg, g_bg, b_bg, a_bg = pixels_bg[x, y]
                    factor = (1.0 - (float(a_mask) / 255.0))
                    r_bg = int(float(r_bg) * factor)
                    g_bg = int(float(g_bg) * factor)
                    b_bg = int(float(b_bg) * factor)
                    pixels_bg[x, y] = (r_bg, g_bg, b_bg, a_bg)
                    pixels_changed_count += 1
                elif a_mask == 255:
                    a_bg = pixels_bg[x, y][-1]
                    pixels_bg[x, y] = (0, 0, 0, a_bg)
                    pixels_changed_count += 1
    except Exception as e:
        core.log(name, "Error: {}".format(e))
    finally:
        core.log(name, "      {} out of {} pixels processed.".format(pixels_changed_count, bg_layer.size[0] * bg_layer.size[1]))
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background+shadow")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer_shadow.png")

    #merge the layers

    core.log(name, "      Merging the layers")
    final = Image.new("RGBA", size)
    final.paste(bg_layer)
    final.paste(frame_layer, None, frame_layer)
    final.paste(overlay_layer, None, overlay_layer)
    final.save(global_variables.folder_base + destination_path)
    core.log(name, "    Created the wallpaper at {}.".format(global_variables.folder_base + destination_path))
    return destination_path

def generate_identicon(data="I'm Samantha", path="/data/identicon.png"):
    """
    generates an identicon and sends it to the G2
    possibly via an AutoRemote Plugin?
    """
    core.log(name, "    Generating the Identicon.")
    generator = pydenticon.Generator(5, 5)
    identicon = generator.generate(data, 300, 300)
    core.log(name, "    Generated the Identicon. Saving at {}.".format(global_variables.folder_base + path))
    f = open(global_variables.folder_base + path, "wb")
    f.write(identicon)
    f.close()
    core.log(name, "    Generated the Identicon at {}.".format(global_variables.folder_base + path))
    return path

def process(key, param, comm):
    try:
        if key == "schedule_h":
            if param == "0":
                core.log(name, "  Generating the daily wallpaper.")
                wallpaper_bg = get_wallpaper()
                identicon = generate_identicon(str(time.time()))
                wallpaper = generate_wallpaper(wallpaper_bg, identicon)
            else:
                core.log(name, "  Warning: parameter {} not in use.".format(param))
        elif key == "wallpaper":
            core.log(name, "  Generating a new wallpaper.")
            wallpaper_bg = get_wallpaper()
            identicon = generate_identicon(str(time.time()))
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
        elif key == "identicon":
            core.log(name, "  Generating an Identicon with the data '{}'.".format(param))
            if param:
                generate_identicon(param)
            else:
                generate_identicon()
        else:
            core.log(name, "  Warning: parameter {} not in use.".format(param))
    except Exception as e:
        core.log(name, "Error: {}".format(e))<commit_msg>Woohoo \*-*/<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, pydenticon, core, time, global_variables, requests, cloudinary
from PIL import Image, ImageChops, ImageEnhance, ImageFilter, ImageOps

is_sam_plugin = 1
name = "Images"
keywords = ["identicon", "schedule_h", "wallpaper"]
has_toggle = 0
has_set = 0

DEBUG = 1

def initialize():
    core.log(name, "Startup")
    core.log(name, "Hello World!")

def stop():
    core.log(name, "I'm not even running anymore!")

def get_wallpaper():
    core.log(name, "    Downloading the baseimage.")
    path = "/data/wallpaper_background.png"
    response = requests.get("https://earthview.withgoogle.com/")
    html = response.text
    regex = r"background-image: url\((?P<link>.+)\);"
    m = re.search(regex, html, re.IGNORECASE)
    if m:
        if m.group("link"):
            image_url = m.group("link")
        else:
            image_url = None
    else:
        image_url = None
    core.log(name, "    The URL is {}. Downloading to {}.".format(image_url, global_variables.folder_base + path))

    if image_url:
        f = open(global_variables.folder_base + path, "wb")
        f.write(requests.get(image_url).content)
        f.close()
    else:
        core.log(name, "    Error: Download not completed.")
    core.log(name, "    Download completed to {}.".format(global_variables.folder_base + path))
    return path

def resize(im, size, offset=(0,0)):
    bg = Image.new(im.mode, size)
    if im.size[0] < size[0]:
        factor_width = -1
    else:
        factor_width = 1
    if im.size[1] < size[1]:
        factor_height = -1
    else:
        factor_height = 1
    shift_left = factor_width * (((size[0] - im.size[0]) / 2) - offset[0])
    shift_top = factor_height * (((size[1] - im.size[1]) / 2) - offset[1])
    shift_right = size[0] - (factor_width * shift_left)
    shift_bottom = size[1] - (factor_height * shift_top)
    return im.crop((shift_left, shift_top, shift_right, shift_bottom))

def generate_wallpaper(background_path, mask_path, destination_path="/data/wallpaper.png"):

    core.log(name, "    Creating the final image")

    #generate the background

    core.log(name, "      Creating the background")
    bg_layer = Image.open(global_variables.folder_base + background_path)    #the background in color
    bg_layer = bg_layer.convert("RGBA")
    converter_color = ImageEnhance.Color(bg_layer)
    bg_layer = converter_color.enhance(0.05)
    converter_brightness = ImageEnhance.Brightness(bg_layer)
    bg_layer = converter_brightness.enhance(0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer.png")

    #generate the masks

    core.log(name, "      Creating the normal masks")
    size = bg_layer.size
    mask_BoW = Image.open(global_variables.folder_base + mask_path)     #"BoW" means black icon on white background.
    mask_WoB = ImageOps.invert(mask_BoW)                                #"WoB" is a white icon on black bg.
    if not mask_BoW.size == size:
        mask_WoB = resize(mask_WoB, size)
        mask_BoW = ImageOps.invert(mask_WoB)
    mask_WoB = mask_WoB.convert("1")
    mask_BoW = mask_BoW.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the normal masks")
        mask_WoB.save(global_variables.folder_base + "/data/mask_WoB.png")
        mask_BoW.save(global_variables.folder_base + "/data/mask_BoW.png")

    core.log(name, "      Creating the big masks")
    mask_BoT_big = mask_BoW.convert("RGBA")
    mask_BoT_big.putalpha(mask_WoB)
    core.log(name, "        Adding the Offset")
    offset_layers = []
    offsets = [(10,10),(-10,10),(10,-10),(-10,-10)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(mask_BoT_big, x, y))
    for offset_layer in offset_layers:
        mask_BoT_big.paste(offset_layer, None, offset_layer)
    core.log(name, "        Creating mask_BoT_big")
    mask_BoW_big =  Image.new("RGB", size, "white")
    mask_BoW_big.paste(mask_BoT_big, None, mask_BoT_big)
    mask_WoB_big = ImageOps.invert(mask_BoW_big)
    mask_WoB_big = mask_WoB_big.convert("1")
    mask_BoW_big = mask_BoW_big.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the big masks")
        mask_WoB_big.save(global_variables.folder_base + "/data/mask_WoB_big.png")
        mask_BoW_big.save(global_variables.folder_base + "/data/mask_BoW_big.png")

    #generate the colored overlay

    core.log(name, "      Creating the colored overlay")
    overlay_layer = Image.open(global_variables.folder_base + background_path)
    overlay_layer.putalpha(mask_WoB)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the colored overlay")
        overlay_layer.save(global_variables.folder_base + "/data/overlay_layer.png")

    #generate the frame around the colored overlay

    core.log(name, "      Creating the frame around the overlay")
    frame_layer = Image.open(global_variables.folder_base + background_path)
    frame_layer = Image.blend(Image.new("RGB", size, "white"), frame_layer, 0.5)
    frame_layer.putalpha(mask_WoB_big)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the frame")
        frame_layer.save(global_variables.folder_base + "/data/frame_layer.png")

    #generate the shadow

    core.log(name, "      Creating the dropshadow")
    shadow_layer =  mask_BoW.convert("RGBA")
    shadow_layer.putalpha(mask_WoB)
    """
    A copy if the mask is pasted ontop of itself with a 2px offset into the 4 diagonal directions each to increase it's size into each direction. (The shadow would otherwise disappear behind the colored overlay.)
    The whole layer is then blurred multiple times to increase the blur's effect.
    Finally, the shadow_layer is blended with a completely transparent layer to add some transparency to it.
    """

    offset_layers = []
    offsets = [(11,11),(-11,11),(11,-11),(-11,-11)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(shadow_layer, x, y))
    for offset_layer in offset_layers:
        shadow_layer.paste(offset_layer, None, offset_layer)

    core.log(name, "      Blurring the shadow")
    for n in range(3):      #as noted above, the Blur is applied multiple times for a stronger effect. 3 has proven as a good compromise between a good effect and not too much necessary calculation.
        core.log(name, "        Step {}/{}".format(n + 1, 3))
        shadow_layer = shadow_layer.filter(ImageFilter.BLUR)
    core.log(name, "      Adding transparency")
    shadow_layer = Image.blend(Image.new("RGBA", size), shadow_layer, 0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the dropshadow")
        shadow_layer.save(global_variables.folder_base + "/data/shadow_layer.png")

    #merge the shadow with the background

    """
    This will add the shadow manually to the background image. Usign PIL's merge() or composite_alpha() was causing the image underneath the shadow to turn greyish which looked pretty ugly on dark parts of the image.
    To prevent this behaviour, this tool iterates over the pixels of the whole image and changes the colorvalues according to the alphavalue of the corresponding image in the mask.
    If the pixel in the mask is completely transparent (a_mask == 0) the calculation will be skipped and the original pixel won't be changed. If the pixel in the mask is completely opaque (a_mask == 255) the pixel in the original image will be turned black. For every other alpha value the new color is calculated via new = old * (1 - (a_mask / 255)).
    """
    core.log(name, "      Merging the shadow with the background")
    try:
        pixels_changed_count = 0
        pixels_bg = bg_layer.load()
        pixels_mask = shadow_layer.load()
        for x in range(bg_layer.size[0]):
            for y in range(bg_layer.size[1]):
                r_mask, g_mask, b_mask, a_mask = pixels_mask[x, y]
                if 0 < a_mask < 255:
                    r_bg, g_bg, b_bg, a_bg = pixels_bg[x, y]
                    factor = (1.0 - (float(a_mask) / 255.0))
                    r_bg = int(float(r_bg) * factor)
                    g_bg = int(float(g_bg) * factor)
                    b_bg = int(float(b_bg) * factor)
                    pixels_bg[x, y] = (r_bg, g_bg, b_bg, a_bg)
                    pixels_changed_count += 1
                elif a_mask == 255:
                    a_bg = pixels_bg[x, y][-1]
                    pixels_bg[x, y] = (0, 0, 0, a_bg)
                    pixels_changed_count += 1
    except Exception as e:
        core.log(name, "Error: {}".format(e))
    finally:
        core.log(name, "      {} out of {} pixels processed.".format(pixels_changed_count, bg_layer.size[0] * bg_layer.size[1]))
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background+shadow")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer_shadow.png")

    #merge the layers

    core.log(name, "      Merging the layers")
    final = Image.new("RGBA", size)
    final.paste(bg_layer)
    final.paste(frame_layer, None, frame_layer)
    final.paste(overlay_layer, None, overlay_layer)
    final.save(global_variables.folder_base + destination_path)
    core.log(name, "    Created the wallpaper at {}.".format(global_variables.folder_base + destination_path))
    return destination_path

def generate_identicon(data="I'm Samantha", path="/data/identicon.png"):
    """
    generates an identicon and sends it to the G2
    possibly via an AutoRemote Plugin?
    """
    core.log(name, "    Generating the Identicon.")
    generator = pydenticon.Generator(5, 5)
    identicon = generator.generate(data, 300, 300)
    core.log(name, "    Generated the Identicon. Saving at {}.".format(global_variables.folder_base + path))
    f = open(global_variables.folder_base + path, "wb")
    f.write(identicon)
    f.close()
    core.log(name, "    Generated the Identicon at {}.".format(global_variables.folder_base + path))
    return path

def process(key, param, comm):
    try:
        if key == "schedule_h":
            if param == "0":
                core.log(name, "  Generating the daily wallpaper.")
                wallpaper_bg = get_wallpaper()
                identicon = generate_identicon(str(time.time()))
                wallpaper = generate_wallpaper(wallpaper_bg, identicon)
            else:
                core.log(name, "  Warning: parameter {} not in use.".format(param))
        elif key == "wallpaper":
            core.log(name, "  Generating a new wallpaper.")
            wallpaper_bg = get_wallpaper()
            identicon = generate_identicon(str(time.time()))
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
        elif key == "identicon":
            core.log(name, "  Generating an Identicon with the data '{}'.".format(param))
            if param:
                generate_identicon(param)
            else:
                generate_identicon()
        else:
            core.log(name, "  Warning: parameter {} not in use.".format(param))
    except Exception as e:
        core.log(name, "Error: {}".format(e))<|endoftext|>