<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, pydenticon, core, time, global_variables, requests, cloudinary
from PIL import Image, ImageChops, ImageEnhance, ImageFilter, ImageOps

is_sam_plugin = 1
name = "Images"
keywords = ["identicon", "schedule_h", "wallpaper"]
has_toggle = 0
has_set = 0

DEBUG = 1

def initialize():
    core.log(name, "Startup")
    core.log(name, "Hello World!")

def stop():
    core.log(name, "I'm not even running anymore!")

def get_wallpaper():
    core.log(name, "    Downloading the baseimage.")
    path = "/data/wallpaper_background.png"
    response = requests.get("https://earthview.withgoogle.com/")
    html = response.text
    regex = r"background-image: url\((?P<link>.+)\);"
    m = re.search(regex, html, re.IGNORECASE)
    if m:
        if m.group("link"):
            image_url = m.group("link")
        else:
            image_url = None
    else:
        image_url = None
    core.log(name, "    The URL is {}. Downloading to {}.".format(image_url, global_variables.folder_base + path))

    if image_url:
        f = open(global_variables.folder_base + path, "wb")
        f.write(requests.get(image_url).content)
        f.close()
    else:
        core.log(name, "    Error: Download not completed.")
    core.log(name, "    Download completed to {}.".format(global_variables.folder_base + path))
    return path

def resize(im, size, offset=(0,0)):
    bg = Image.new(im.mode, size)
    if im.size[0] < size[0]:
        factor_width = -1
    else:
        factor_width = 1
    if im.size[1] < size[1]:
        factor_height = -1
    else:
        factor_height = 1
    shift_left = factor_width * (((size[0] - im.size[0]) / 2) - offset[0])
    shift_top = factor_height * (((size[1] - im.size[1]) / 2) - offset[1])
    shift_right = size[0] - (factor_width * shift_left)
    shift_bottom = size[1] - (factor_height * shift_top)
    return im.crop((shift_left, shift_top, shift_right, shift_bottom))

def generate_wallpaper(background_path, mask_path, destination_path="/data/wallpaper.png"):

    core.log(name, "    Creating the final image")

    #generate the background

    core.log(name, "      Creating the background")
    bg_layer = Image.open(global_variables.folder_base + background_path)    #the background in color
    bg_layer = bg_layer.convert("RGBA")
    converter_color = ImageEnhance.Color(bg_layer)
    bg_layer = converter_color.enhance(0.05)
    converter_brightness = ImageEnhance.Brightness(bg_layer)
    bg_layer = converter_brightness.enhance(0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer.png")

    #generate the masks

    core.log(name, "      Creating the normal masks")
    size = bg_layer.size
    mask_BoW = Image.open(global_variables.folder_base + mask_path)     #"BoW" means black icon on white background.
    mask_WoB = ImageOps.invert(mask_BoW)                                #"WoB" is a white icon on black bg.
    if not mask_BoW.size == size:
        mask_WoB = resize(mask_WoB, size)
        mask_BoW = ImageOps.invert(mask_WoB)
    mask_WoB = mask_WoB.convert("1")
    mask_BoW = mask_BoW.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the normal masks")
        mask_WoB.save(global_variables.folder_base + "/data/mask_WoB.png")
        mask_BoW.save(global_variables.folder_base + "/data/mask_BoW.png")

    core.log(name, "      Creating the big masks")
    mask_BoW_big = mask_BoW.convert("RGBA")
    mask_BoW_big.putalpha(mask_WoB)
    core.log(name, "        Adding the Offset")
    offset_layers = []
    offsets = [(2,2),(-2,2),(2,-2),(-2,-2)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(mask_BoW_big, x, y))
    for offset_layer in offset_layers:
        mask_BoW_big.paste(offset_layer, None, offset_layer)
    core.log(name, "        Creating mask_WoB_big")
    #mask_WoB_big = ImageOps.invert(mask_BoW_big)
    #mask_WoB_big = mask_WoB_big.convert("1")
    #mask_BoW_big = mask_BoW_big.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the big masks")
        #mask_WoB_big.save(global_variables.folder_base + "/data/mask_WoB_big.png")
        mask_BoW_big.save(global_variables.folder_base + "/data/mask_BoW_big.png")

    #generate the colored overlay

    core.log(name, "      Creating the colored overlay")
    overlay_layer = Image.open(global_variables.folder_base + background_path)
    overlay_layer.putalpha(mask_WoB)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the colored overlay")
        overlay_layer.save(global_variables.folder_base + "/data/overlay_layer.png")

    #generate the frame around the colored overlay

    core.log(name, "      Creating the frame around the overlay")
    frame_layer = Image.open(global_variables.folder_base + background_path)
    frame_layer = Image.blend(Image.new("RGBA", size, "white"), frame_layer, 0.8)
    frame_layer.putalpha(mask_BoW_big)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the frame")
        frame_layer.save(global_variables.folder_base + "/data/frame_layer.png")

    #generate the shadow

    core.log(name, "      Creating the dropshadow")
    shadow_layer =  mask_BoW.convert("RGBA")
    shadow_layer.putalpha(mask_WoB)
    """
    A copy if the mask is pasted ontop of itself with a 2px offset into the 4 diagonal directions each to increase it's size into each direction. (The shadow would otherwise disappear behind the colored overlay.)
    The whole layer is then blurred multiple times to increase the blur's effect.
    Finally, the shadow_layer is blended with a completely transparent layer to add some transparency to it.
    """

    offset_layers = []
    offsets = [(1,1),(-1,1),(1,-1),(-1,-1)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(shadow_layer, x, y))
    for offset_layer in offset_layers:
        shadow_layer.paste(offset_layer, None, offset_layer)

    core.log(name, "      Blurring the shadow")
    for n in range(3):      #as noted above, the Blur is applied multiple times for a stronger effect. 3 has proven as a good compromise between a good effect and not too much necessary calculation.
        core.log(name, "        Step {}/{}".format(n + 1, 3))
        shadow_layer = shadow_layer.filter(ImageFilter.BLUR)
    core.log(name, "      Adding transparency")
    shadow_layer = Image.blend(Image.new("RGBA", size), shadow_layer, 0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the dropshadow")
        shadow_layer.save(global_variables.folder_base + "/data/shadow_layer.png")

    #merge the shadow with the background

    """
    This will add the shadow manually to the background image. Usign PIL's merge() or composite_alpha() was causing the image underneath the shadow to turn greyish which looked pretty ugly on dark parts of the image.
    To prevent this behaviour, this tool iterates over the pixels of the whole image and changes the colorvalues according to the alphavalue of the corresponding image in the mask.
    If the pixel in the mask is completely transparent (a_mask == 0) the calculation will be skipped and the original pixel won't be changed. If the pixel in the mask is completely opaque (a_mask == 255) the pixel in the original image will be turned black. For every other alpha value the new color is calculated via new = old * (1 - (a_mask / 255)).
    """
    core.log(name, "      Merging the shadow with the background")
    try:
        pixels_changed_count = 0
        pixels_bg = bg_layer.load()
        pixels_mask = shadow_layer.load()
        for x in range(bg_layer.size[0]):
            for y in range(bg_layer.size[1]):
                r_mask, g_mask, b_mask, a_mask = pixels_mask[x, y]
                if 0 < a_mask < 255:
                    r_bg, g_bg, b_bg, a_bg = pixels_bg[x, y]
                    factor = (1.0 - (float(a_mask) / 255.0))
                    r_bg = int(float(r_bg) * factor)
                    g_bg = int(float(g_bg) * factor)
                    b_bg = int(float(b_bg) * factor)
                    pixels_bg[x, y] = (r_bg, g_bg, b_bg, a_bg)
                    pixels_changed_count += 1
                elif a_mask == 255:
                    a_bg = pixels_bg[x, y][-1]
                    pixels_bg[x, y] = (0, 0, 0, a_bg)
                    pixels_changed_count += 1
    except Exception as e:
        core.log(name, "Error: {}".format(e))
    finally:
        core.log(name, "      {} out of {} pixels processed.".format(pixels_changed_count, bg_layer.size[0] * bg_layer.size[1]))
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background+shadow")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer_shadow.png")

    #merge the layers

    core.log(name, "      Merging the layers")
    final = Image.new("RGBA", size)
    final.paste(bg_layer)
#    final.paste(frame_layer, None, frame_layer)
    final.paste(overlay_layer, None, overlay_layer)
    final.save(global_variables.folder_base + destination_path)
    core.log(name, "    Created the wallpaper at {}.".format(global_variables.folder_base + destination_path))
    return destination_path

def generate_identicon(data="I'm Samantha", path="/data/identicon.png"):
    """
    generates an identicon and sends it to the G2
    possibly via an AutoRemote Plugin?
    """
    core.log(name, "    Generating the Identicon.")
    generator = pydenticon.Generator(5, 5)
    identicon = generator.generate(data, 300, 300)
    core.log(name, "    Generated the Identicon. Saving at {}.".format(global_variables.folder_base + path))
    f = open(global_variables.folder_base + path, "wb")
    f.write(identicon)
    f.close()
    core.log(name, "    Generated the Identicon at {}.".format(global_variables.folder_base + path))
    return path

def process(key, param, comm):
    try:
        if key == "schedule_h":
            if param == "0":
                core.log(name, "  Generating the daily wallpaper.")
                wallpaper_bg = get_wallpaper()
                identicon = generate_identicon(str(time.time()))
                wallpaper = generate_wallpaper(wallpaper_bg, identicon)
            else:
                core.log(name, "  Warning: parameter {} not in use.".format(param))
        elif key == "wallpaper":
            core.log(name, "  Generating a new wallpaper.")
            wallpaper_bg = get_wallpaper()
            identicon = generate_identicon(str(time.time()))
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
        elif key == "identicon":
            core.log(name, "  Generating an Identicon with the data '{}'.".format(param))
            if param:
                generate_identicon(param)
            else:
                generate_identicon()
        else:
            core.log(name, "  Warning: parameter {} not in use.".format(param))
    except Exception as e:
        core.log(name, "Error: {}".format(e))<commit_msg>creating the big masks transparent at first, then pasign it onto a white layer<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, pydenticon, core, time, global_variables, requests, cloudinary
from PIL import Image, ImageChops, ImageEnhance, ImageFilter, ImageOps

is_sam_plugin = 1
name = "Images"
keywords = ["identicon", "schedule_h", "wallpaper"]
has_toggle = 0
has_set = 0

DEBUG = 1

def initialize():
    core.log(name, "Startup")
    core.log(name, "Hello World!")

def stop():
    core.log(name, "I'm not even running anymore!")

def get_wallpaper():
    core.log(name, "    Downloading the baseimage.")
    path = "/data/wallpaper_background.png"
    response = requests.get("https://earthview.withgoogle.com/")
    html = response.text
    regex = r"background-image: url\((?P<link>.+)\);"
    m = re.search(regex, html, re.IGNORECASE)
    if m:
        if m.group("link"):
            image_url = m.group("link")
        else:
            image_url = None
    else:
        image_url = None
    core.log(name, "    The URL is {}. Downloading to {}.".format(image_url, global_variables.folder_base + path))

    if image_url:
        f = open(global_variables.folder_base + path, "wb")
        f.write(requests.get(image_url).content)
        f.close()
    else:
        core.log(name, "    Error: Download not completed.")
    core.log(name, "    Download completed to {}.".format(global_variables.folder_base + path))
    return path

def resize(im, size, offset=(0,0)):
    bg = Image.new(im.mode, size)
    if im.size[0] < size[0]:
        factor_width = -1
    else:
        factor_width = 1
    if im.size[1] < size[1]:
        factor_height = -1
    else:
        factor_height = 1
    shift_left = factor_width * (((size[0] - im.size[0]) / 2) - offset[0])
    shift_top = factor_height * (((size[1] - im.size[1]) / 2) - offset[1])
    shift_right = size[0] - (factor_width * shift_left)
    shift_bottom = size[1] - (factor_height * shift_top)
    return im.crop((shift_left, shift_top, shift_right, shift_bottom))

def generate_wallpaper(background_path, mask_path, destination_path="/data/wallpaper.png"):

    core.log(name, "    Creating the final image")

    #generate the background

    core.log(name, "      Creating the background")
    bg_layer = Image.open(global_variables.folder_base + background_path)    #the background in color
    bg_layer = bg_layer.convert("RGBA")
    converter_color = ImageEnhance.Color(bg_layer)
    bg_layer = converter_color.enhance(0.05)
    converter_brightness = ImageEnhance.Brightness(bg_layer)
    bg_layer = converter_brightness.enhance(0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer.png")

    #generate the masks

    core.log(name, "      Creating the normal masks")
    size = bg_layer.size
    mask_BoW = Image.open(global_variables.folder_base + mask_path)     #"BoW" means black icon on white background.
    mask_WoB = ImageOps.invert(mask_BoW)                                #"WoB" is a white icon on black bg.
    if not mask_BoW.size == size:
        mask_WoB = resize(mask_WoB, size)
        mask_BoW = ImageOps.invert(mask_WoB)
    mask_WoB = mask_WoB.convert("1")
    mask_BoW = mask_BoW.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the normal masks")
        mask_WoB.save(global_variables.folder_base + "/data/mask_WoB.png")
        mask_BoW.save(global_variables.folder_base + "/data/mask_BoW.png")

    core.log(name, "      Creating the big masks")
    mask_BoT_big = mask_BoT.convert("RGBA")
    mask_BoT_big.putalpha(mask_WoB)
    core.log(name, "        Adding the Offset")
    offset_layers = []
    offsets = [(2,2),(-2,2),(2,-2),(-2,-2)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(mask_BoT_big, x, y))
    for offset_layer in offset_layers:
        mask_BoT_big.paste(offset_layer, None, offset_layer)
    core.log(name, "        Creating mask_BoT_big")
    mask_BoW_big =  Image.new("RGBA", size, "white")
    mask_BoW_big.paste(mask_BoT_big, None, mask_BoT_big)
    mask_WoB_big = ImageOps.invert(mask_BoW_big)
    mask_WoB_big = mask_WoB_big.convert("1")
    mask_BoW_big = mask_BoW_big.convert("1")
    if DEBUG:
        core.log(name, "      DEBUG: Saving the big masks")
        #mask_WoB_big.save(global_variables.folder_base + "/data/mask_WoB_big.png")
        mask_BoW_big.save(global_variables.folder_base + "/data/mask_BoW_big.png")

    #generate the colored overlay

    core.log(name, "      Creating the colored overlay")
    overlay_layer = Image.open(global_variables.folder_base + background_path)
    overlay_layer.putalpha(mask_WoB)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the colored overlay")
        overlay_layer.save(global_variables.folder_base + "/data/overlay_layer.png")

    #generate the frame around the colored overlay

    core.log(name, "      Creating the frame around the overlay")
    frame_layer = Image.open(global_variables.folder_base + background_path)
    frame_layer = Image.blend(Image.new("RGBA", size, "white"), frame_layer, 0.8)
    frame_layer.putalpha(mask_BoW_big)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the frame")
        frame_layer.save(global_variables.folder_base + "/data/frame_layer.png")

    #generate the shadow

    core.log(name, "      Creating the dropshadow")
    shadow_layer =  mask_BoW.convert("RGBA")
    shadow_layer.putalpha(mask_WoB)
    """
    A copy if the mask is pasted ontop of itself with a 2px offset into the 4 diagonal directions each to increase it's size into each direction. (The shadow would otherwise disappear behind the colored overlay.)
    The whole layer is then blurred multiple times to increase the blur's effect.
    Finally, the shadow_layer is blended with a completely transparent layer to add some transparency to it.
    """

    offset_layers = []
    offsets = [(1,1),(-1,1),(1,-1),(-1,-1)]
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(shadow_layer, x, y))
    for offset_layer in offset_layers:
        shadow_layer.paste(offset_layer, None, offset_layer)

    core.log(name, "      Blurring the shadow")
    for n in range(3):      #as noted above, the Blur is applied multiple times for a stronger effect. 3 has proven as a good compromise between a good effect and not too much necessary calculation.
        core.log(name, "        Step {}/{}".format(n + 1, 3))
        shadow_layer = shadow_layer.filter(ImageFilter.BLUR)
    core.log(name, "      Adding transparency")
    shadow_layer = Image.blend(Image.new("RGBA", size), shadow_layer, 0.8)
    if DEBUG:
        core.log(name, "      DEBUG: Saving the dropshadow")
        shadow_layer.save(global_variables.folder_base + "/data/shadow_layer.png")

    #merge the shadow with the background

    """
    This will add the shadow manually to the background image. Usign PIL's merge() or composite_alpha() was causing the image underneath the shadow to turn greyish which looked pretty ugly on dark parts of the image.
    To prevent this behaviour, this tool iterates over the pixels of the whole image and changes the colorvalues according to the alphavalue of the corresponding image in the mask.
    If the pixel in the mask is completely transparent (a_mask == 0) the calculation will be skipped and the original pixel won't be changed. If the pixel in the mask is completely opaque (a_mask == 255) the pixel in the original image will be turned black. For every other alpha value the new color is calculated via new = old * (1 - (a_mask / 255)).
    """
    core.log(name, "      Merging the shadow with the background")
    try:
        pixels_changed_count = 0
        pixels_bg = bg_layer.load()
        pixels_mask = shadow_layer.load()
        for x in range(bg_layer.size[0]):
            for y in range(bg_layer.size[1]):
                r_mask, g_mask, b_mask, a_mask = pixels_mask[x, y]
                if 0 < a_mask < 255:
                    r_bg, g_bg, b_bg, a_bg = pixels_bg[x, y]
                    factor = (1.0 - (float(a_mask) / 255.0))
                    r_bg = int(float(r_bg) * factor)
                    g_bg = int(float(g_bg) * factor)
                    b_bg = int(float(b_bg) * factor)
                    pixels_bg[x, y] = (r_bg, g_bg, b_bg, a_bg)
                    pixels_changed_count += 1
                elif a_mask == 255:
                    a_bg = pixels_bg[x, y][-1]
                    pixels_bg[x, y] = (0, 0, 0, a_bg)
                    pixels_changed_count += 1
    except Exception as e:
        core.log(name, "Error: {}".format(e))
    finally:
        core.log(name, "      {} out of {} pixels processed.".format(pixels_changed_count, bg_layer.size[0] * bg_layer.size[1]))
    if DEBUG:
        core.log(name, "      DEBUG: Saving the background+shadow")
        bg_layer.save(global_variables.folder_base + "/data/bg_layer_shadow.png")

    #merge the layers

    core.log(name, "      Merging the layers")
    final = Image.new("RGBA", size)
    final.paste(bg_layer)
#    final.paste(frame_layer, None, frame_layer)
    final.paste(overlay_layer, None, overlay_layer)
    final.save(global_variables.folder_base + destination_path)
    core.log(name, "    Created the wallpaper at {}.".format(global_variables.folder_base + destination_path))
    return destination_path

def generate_identicon(data="I'm Samantha", path="/data/identicon.png"):
    """
    generates an identicon and sends it to the G2
    possibly via an AutoRemote Plugin?
    """
    core.log(name, "    Generating the Identicon.")
    generator = pydenticon.Generator(5, 5)
    identicon = generator.generate(data, 300, 300)
    core.log(name, "    Generated the Identicon. Saving at {}.".format(global_variables.folder_base + path))
    f = open(global_variables.folder_base + path, "wb")
    f.write(identicon)
    f.close()
    core.log(name, "    Generated the Identicon at {}.".format(global_variables.folder_base + path))
    return path

def process(key, param, comm):
    try:
        if key == "schedule_h":
            if param == "0":
                core.log(name, "  Generating the daily wallpaper.")
                wallpaper_bg = get_wallpaper()
                identicon = generate_identicon(str(time.time()))
                wallpaper = generate_wallpaper(wallpaper_bg, identicon)
            else:
                core.log(name, "  Warning: parameter {} not in use.".format(param))
        elif key == "wallpaper":
            core.log(name, "  Generating a new wallpaper.")
            wallpaper_bg = get_wallpaper()
            identicon = generate_identicon(str(time.time()))
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
        elif key == "identicon":
            core.log(name, "  Generating an Identicon with the data '{}'.".format(param))
            if param:
                generate_identicon(param)
            else:
                generate_identicon()
        else:
            core.log(name, "  Warning: parameter {} not in use.".format(param))
    except Exception as e:
        core.log(name, "Error: {}".format(e))<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, pydenticon, core, time, global_variables, requests, cloudinary
from PIL import Image, ImageChops, ImageEnhance, ImageFilter, ImageOps

is_sam_plugin = 1
name = "Images"
keywords = ["identicon", "schedule_h", "wallpaper"]
has_toggle = 0
has_set = 0

DEBUG = 0

def initialize():
    core.log(name, "Startup")
    core.log(name, "Hello World!")

def stop():
    core.log(name, "I'm not even running anymore!")

def get_wallpaper():
    core.log(name, "    Downloading the baseimage.")
    path = "/data/wallpaper_background.png"
    response = requests.get("https://earthview.withgoogle.com/")
    html = response.text
    regex = r"background-image: url\((?P<link>.+)\);"
    m = re.search(regex, html, re.IGNORECASE)
    if m:
        if m.group("link"):
            image_url = m.group("link")
        else:
            image_url = None
    else:
        image_url = None
    core.log(name, "    The URL is {}. Downloading to {}.".format(image_url, global_variables.folder_base + path))

    if image_url:
        f = open(global_variables.folder_base + path, "wb")
        f.write(requests.get(image_url).content)
        f.close()
    else:
        core.log(name, "    Error: Download not completed.")
    core.log(name, "    Download completed to {}.".format(global_variables.folder_base + path))
    return path

def resize(im, size, offset=(0,0)):
    bg = Image.new(im.mode, size)
    if im.size[0] < size[0]:
        factor_width = -1
    else:
        factor_width = 1
    if im.size[1] < size[1]:
        factor_height = -1
    else:
        factor_height = 1
    shift_left = factor_width * (((size[0] - im.size[0]) / 2) - offset[0])
    shift_top = factor_height * (((size[1] - im.size[1]) / 2) - offset[1])
    shift_right = size[0] - (factor_width * shift_left)
    shift_bottom = size[1] - (factor_height * shift_top)
    return im.crop((shift_left, shift_top, shift_right, shift_bottom))

def generate_wallpaper(background_path, mask_path, destination_path="/data/wallpaper.png", offset=2):

    core.log(name, "    Creating the final image")

    #generate the background

    core.log(name, "      Creating the background")
    bg_layer = Image.open(global_variables.folder_base + background_path)    #the background in color
    bg_layer = bg_layer.convert("RGBA")
    converter_color = ImageEnhance.Color(bg_layer)
    bg_layer = converter_color.enhance(0.05)
    converter_brightness = ImageEnhance.Brightness(bg_layer)
    bg_layer = converter_brightness.enhance(0.8)
    if DEBUG:
        bg_layer.save(global_variables.folder_base + "/data/bg_layer.png")

    #generate the mask

    core.log(name, "      Creating the mask")
    size = bg_layer.size
    mask_BoW = Image.open(global_variables.folder_base + mask_path)     #"BoW" means black icon on white background.
    mask_WoB = ImageOps.invert(mask_BoW)                                #"WoB" is a white icon on black bg.
    if not mask_BoW.size == size:
        mask_WoB = resize(mask_WoB, size)
        mask_BoW = ImageOps.invert(mask_WoB)
    mask_WoB = mask_WoB.convert("1")
    mask_BoW = mask_BoW.convert("1")
    if DEBUG:
        mask_WoB.save(global_variables.folder_base + "/data/mask_WoB.png")
        mask_BoW.save(global_variables.folder_base + "/data/mask_BoW.png")

    #generate the colored overlay

    core.log(name, "      Creating the colored overlay")
    overlay_layer = Image.open(global_variables.folder_base + background_path)
    overlay_layer.putalpha(mask_WoB)
    if DEBUG:
        overlay_layer.save(global_variables.folder_base + "/data/overlay_layer.png")

    #generate the shadow

    core.log(name, "      Creating the dropshadow")
    shadow_layer =  mask_BoW.convert("RGBA")
    shadow_layer.putalpha(mask_WoB)
    """
    A copy if the mask is pasted ontop of itself with a 2px offset into the 4 diagonal directions each to increase it's size into each direction. (The shadow would otherwise disappear behind the colored overlay.)
    The whole layer is then blurred multiple times to increase the blur's effect.
    Finally, the shadow_layer is blended with a completely transparent layer to add some transparency to it.
    """

    offset_layers = []
    offsets = [(offset,offset),(-offset,offset),(offset,-offset),(-offset,-offset)]     #offset is defined in the definition of the function. Default is 2.
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(shadow_layer, x, y))
    for offset_layer in offset_layers:
        shadow_layer.paste(offset_layer, None, offset_layer)

    core.log(name, "      Blurring the shadow")
    n = 0
    while n < repetitions:      #as noted above, the Blur is applied multiple times for a stronger effect.
        n += 1
        core.log(name, "        Step {}/{}".format(n, repetitions))
        shadow_layer = shadow_layer.filter(ImageFilter.BLUR)
    core.log(name, "      Adding transparency")
    shadow_layer = Image.blend(Image.new("RGBA", size), shadow_layer, 0.7)
    if DEBUG:
        shadow_layer.save(global_variables.folder_base + "/data/shadow_layer.png")

    #merge the shadow with the background

    """
    This will add the shadow manually to the background image. Usign PIL's merge() or composite_alpha() was causing the image underneath the shadow to turn greyish which looked pretty ugly on dark parts of the image.
    To prevent this behaviour, this tool iterates over the pixels of the whole image and changes the colorvalues according to the alphavalue of the corresponding image in the mask.
    If the pixel in the mask is completely transparent (a_mask == 0) the calculation will be skipped and the original pixel won't be changed. If the pixel in the mask is completely opaque (a_mask == 255) the pixel in the original image will be turned black. For every other alpha value the new color is calculated via new = old * (1 - (a_mask / 255)).
    """
    core.log(name, "      Merging the shadow with the background")
    try:
        pixels_changed_count = 0
        pixels_bg = bg_layer.load()
        pixels_mask = shadow_layer.load()
        for x in range(bg_layer.size[0]):
            for y in range(bg_layer.size[1]):
                r_mask, g_mask, b_mask, a_mask = pixels_mask[x, y]
                if 0 < a_mask < 255:
                    r_bg, g_bg, b_bg, a_bg = pixels_bg[x, y]
                    factor = (1.0 - (float(a_mask) / 255.0))
                    r_bg = int(float(r_bg) * factor)
                    g_bg = int(float(g_bg) * factor)
                    b_bg = int(float(b_bg) * factor)
                    pixels_bg[x, y] = (r_bg, g_bg, b_bg, a_bg)
                    pixels_changed_count += 1
                elif a_mask == 255:
                    a_bg = pixels_bg[x, y][-1]
                    pixels_bg[x, y] = (0, 0, 0, a_bg)
                    pixels_changed_count += 1
    except Exception as e:
        core.log(name, "Error: {}".format(e))
    finally:
        core.log(name, "      {} out of {} pixels processed.".format(pixels_changed_count, bg_layer.size[0] * bg_layer.size[1]))
    if DEBUG:
        bg_layer.save(global_variables.folder_base + "/data/bg_layer_shadow.png")
    '''
    #generate the light frame
    frame_layer = overlay_layer.filter(ImageFilter.FIND_EDGES)
    frame_layer.save("./frame_layer.png")
    '''
    #merge the layers

    core.log(name, "      Merging the layers")
    final = Image.new("RGBA", size)
    final.paste(bg_layer)
    #final.paste(shadow_layer, None, shadow_layer)
    final.paste(overlay_layer, None, overlay_layer)
    final.save(global_variables.folder_base + destination_path)
    core.log(name, "    Created the wallpaper at {}.".format(global_variables.folder_base + destination_path))
    return destination_path

def generate_identicon(data="I'm Samantha", path="/data/identicon.png"):
    """
    generates an identicon and sends it to the G2
    possibly via an AutoRemote Plugin?
    """
    core.log(name, "    Generating the Identicon.")
    generator = pydenticon.Generator(5, 5)
    identicon = generator.generate(data, 300, 300)
    core.log(name, "    Generated the Identicon. Saving at {}.".format(global_variables.folder_base + path))
    f = open(global_variables.folder_base + path, "wb")
    f.write(identicon)
    f.close()
    core.log(name, "    Generated the Identicon at {}.".format(global_variables.folder_base + path))
    return path

def process(key, param, comm):
    if key == "schedule_h":
        if param == "0":
            core.log(name, "  Generating the daily wallpaper.")
            wallpaper_bg = get_wallpaper()
            identicon = generate_identicon(str(time.time()))
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
        else:
            core.log(name, "  Error: illegal parameter: {}.".format(param))
    elif key == "wallpaper":
        core.log(name, "  Generating a new wallpaper.")
        wallpaper_bg = get_wallpaper()
        identicon = generate_identicon(str(time.time()))
        if param == "test":
            wallpaper = generate_wallpaper(wallpaper_bg, identicon, "/data/wallpaper0.png", 0)
            wallpaper = generate_wallpaper(wallpaper_bg, identicon, "/data/wallpaper1.png", 1)
            wallpaper = generate_wallpaper(wallpaper_bg, identicon, "/data/wallpaper2.png", 2)
        else:
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
    elif key == "identicon":
        core.log(name, "  Generating an Identicon with the data '{}'.".format(param))
        if param:
            generate_identicon(param)
        else:
            generate_identicon()
    else:
        core.log(name, "  Error: illegal command: {}.".format(key))<commit_msg>reduced the visibility of the shadow a but further<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re, pydenticon, core, time, global_variables, requests, cloudinary
from PIL import Image, ImageChops, ImageEnhance, ImageFilter, ImageOps

is_sam_plugin = 1
name = "Images"
keywords = ["identicon", "schedule_h", "wallpaper"]
has_toggle = 0
has_set = 0

DEBUG = 0

def initialize():
    core.log(name, "Startup")
    core.log(name, "Hello World!")

def stop():
    core.log(name, "I'm not even running anymore!")

def get_wallpaper():
    core.log(name, "    Downloading the baseimage.")
    path = "/data/wallpaper_background.png"
    response = requests.get("https://earthview.withgoogle.com/")
    html = response.text
    regex = r"background-image: url\((?P<link>.+)\);"
    m = re.search(regex, html, re.IGNORECASE)
    if m:
        if m.group("link"):
            image_url = m.group("link")
        else:
            image_url = None
    else:
        image_url = None
    core.log(name, "    The URL is {}. Downloading to {}.".format(image_url, global_variables.folder_base + path))

    if image_url:
        f = open(global_variables.folder_base + path, "wb")
        f.write(requests.get(image_url).content)
        f.close()
    else:
        core.log(name, "    Error: Download not completed.")
    core.log(name, "    Download completed to {}.".format(global_variables.folder_base + path))
    return path

def resize(im, size, offset=(0,0)):
    bg = Image.new(im.mode, size)
    if im.size[0] < size[0]:
        factor_width = -1
    else:
        factor_width = 1
    if im.size[1] < size[1]:
        factor_height = -1
    else:
        factor_height = 1
    shift_left = factor_width * (((size[0] - im.size[0]) / 2) - offset[0])
    shift_top = factor_height * (((size[1] - im.size[1]) / 2) - offset[1])
    shift_right = size[0] - (factor_width * shift_left)
    shift_bottom = size[1] - (factor_height * shift_top)
    return im.crop((shift_left, shift_top, shift_right, shift_bottom))

def generate_wallpaper(background_path, mask_path, destination_path="/data/wallpaper.png", offset=2):

    core.log(name, "    Creating the final image")

    #generate the background

    core.log(name, "      Creating the background")
    bg_layer = Image.open(global_variables.folder_base + background_path)    #the background in color
    bg_layer = bg_layer.convert("RGBA")
    converter_color = ImageEnhance.Color(bg_layer)
    bg_layer = converter_color.enhance(0.05)
    converter_brightness = ImageEnhance.Brightness(bg_layer)
    bg_layer = converter_brightness.enhance(0.8)
    if DEBUG:
        bg_layer.save(global_variables.folder_base + "/data/bg_layer.png")

    #generate the mask

    core.log(name, "      Creating the mask")
    size = bg_layer.size
    mask_BoW = Image.open(global_variables.folder_base + mask_path)     #"BoW" means black icon on white background.
    mask_WoB = ImageOps.invert(mask_BoW)                                #"WoB" is a white icon on black bg.
    if not mask_BoW.size == size:
        mask_WoB = resize(mask_WoB, size)
        mask_BoW = ImageOps.invert(mask_WoB)
    mask_WoB = mask_WoB.convert("1")
    mask_BoW = mask_BoW.convert("1")
    if DEBUG:
        mask_WoB.save(global_variables.folder_base + "/data/mask_WoB.png")
        mask_BoW.save(global_variables.folder_base + "/data/mask_BoW.png")

    #generate the colored overlay

    core.log(name, "      Creating the colored overlay")
    overlay_layer = Image.open(global_variables.folder_base + background_path)
    overlay_layer.putalpha(mask_WoB)
    if DEBUG:
        overlay_layer.save(global_variables.folder_base + "/data/overlay_layer.png")

    #generate the shadow

    core.log(name, "      Creating the dropshadow")
    shadow_layer =  mask_BoW.convert("RGBA")
    shadow_layer.putalpha(mask_WoB)
    """
    A copy if the mask is pasted ontop of itself with a 2px offset into the 4 diagonal directions each to increase it's size into each direction. (The shadow would otherwise disappear behind the colored overlay.)
    The whole layer is then blurred multiple times to increase the blur's effect.
    Finally, the shadow_layer is blended with a completely transparent layer to add some transparency to it.
    """

    offset_layers = []
    offsets = [(offset,offset),(-offset,offset),(offset,-offset),(-offset,-offset)]     #offset is defined in the definition of the function. Default is 2.
    for (x, y) in offsets:
        offset_layers.append(ImageChops.offset(shadow_layer, x, y))
    for offset_layer in offset_layers:
        shadow_layer.paste(offset_layer, None, offset_layer)

    core.log(name, "      Blurring the shadow")
    n = 0
    while n < repetitions:      #as noted above, the Blur is applied multiple times for a stronger effect.
        n += 1
        core.log(name, "        Step {}/{}".format(n, repetitions))
        shadow_layer = shadow_layer.filter(ImageFilter.BLUR)
    core.log(name, "      Adding transparency")
    shadow_layer = Image.blend(Image.new("RGBA", size), shadow_layer, 0.6)
    if DEBUG:
        shadow_layer.save(global_variables.folder_base + "/data/shadow_layer.png")

    #merge the shadow with the background

    """
    This will add the shadow manually to the background image. Usign PIL's merge() or composite_alpha() was causing the image underneath the shadow to turn greyish which looked pretty ugly on dark parts of the image.
    To prevent this behaviour, this tool iterates over the pixels of the whole image and changes the colorvalues according to the alphavalue of the corresponding image in the mask.
    If the pixel in the mask is completely transparent (a_mask == 0) the calculation will be skipped and the original pixel won't be changed. If the pixel in the mask is completely opaque (a_mask == 255) the pixel in the original image will be turned black. For every other alpha value the new color is calculated via new = old * (1 - (a_mask / 255)).
    """
    core.log(name, "      Merging the shadow with the background")
    try:
        pixels_changed_count = 0
        pixels_bg = bg_layer.load()
        pixels_mask = shadow_layer.load()
        for x in range(bg_layer.size[0]):
            for y in range(bg_layer.size[1]):
                r_mask, g_mask, b_mask, a_mask = pixels_mask[x, y]
                if 0 < a_mask < 255:
                    r_bg, g_bg, b_bg, a_bg = pixels_bg[x, y]
                    factor = (1.0 - (float(a_mask) / 255.0))
                    r_bg = int(float(r_bg) * factor)
                    g_bg = int(float(g_bg) * factor)
                    b_bg = int(float(b_bg) * factor)
                    pixels_bg[x, y] = (r_bg, g_bg, b_bg, a_bg)
                    pixels_changed_count += 1
                elif a_mask == 255:
                    a_bg = pixels_bg[x, y][-1]
                    pixels_bg[x, y] = (0, 0, 0, a_bg)
                    pixels_changed_count += 1
    except Exception as e:
        core.log(name, "Error: {}".format(e))
    finally:
        core.log(name, "      {} out of {} pixels processed.".format(pixels_changed_count, bg_layer.size[0] * bg_layer.size[1]))
    if DEBUG:
        bg_layer.save(global_variables.folder_base + "/data/bg_layer_shadow.png")
    '''
    #generate the light frame
    frame_layer = overlay_layer.filter(ImageFilter.FIND_EDGES)
    frame_layer.save("./frame_layer.png")
    '''
    #merge the layers

    core.log(name, "      Merging the layers")
    final = Image.new("RGBA", size)
    final.paste(bg_layer)
    #final.paste(shadow_layer, None, shadow_layer)
    final.paste(overlay_layer, None, overlay_layer)
    final.save(global_variables.folder_base + destination_path)
    core.log(name, "    Created the wallpaper at {}.".format(global_variables.folder_base + destination_path))
    return destination_path

def generate_identicon(data="I'm Samantha", path="/data/identicon.png"):
    """
    generates an identicon and sends it to the G2
    possibly via an AutoRemote Plugin?
    """
    core.log(name, "    Generating the Identicon.")
    generator = pydenticon.Generator(5, 5)
    identicon = generator.generate(data, 300, 300)
    core.log(name, "    Generated the Identicon. Saving at {}.".format(global_variables.folder_base + path))
    f = open(global_variables.folder_base + path, "wb")
    f.write(identicon)
    f.close()
    core.log(name, "    Generated the Identicon at {}.".format(global_variables.folder_base + path))
    return path

def process(key, param, comm):
    if key == "schedule_h":
        if param == "0":
            core.log(name, "  Generating the daily wallpaper.")
            wallpaper_bg = get_wallpaper()
            identicon = generate_identicon(str(time.time()))
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
        else:
            core.log(name, "  Error: illegal parameter: {}.".format(param))
    elif key == "wallpaper":
        core.log(name, "  Generating a new wallpaper.")
        wallpaper_bg = get_wallpaper()
        identicon = generate_identicon(str(time.time()))
        if param == "test":
            wallpaper = generate_wallpaper(wallpaper_bg, identicon, "/data/wallpaper0.png", 0)
            wallpaper = generate_wallpaper(wallpaper_bg, identicon, "/data/wallpaper1.png", 1)
            wallpaper = generate_wallpaper(wallpaper_bg, identicon, "/data/wallpaper2.png", 2)
        else:
            wallpaper = generate_wallpaper(wallpaper_bg, identicon)
    elif key == "identicon":
        core.log(name, "  Generating an Identicon with the data '{}'.".format(param))
        if param:
            generate_identicon(param)
        else:
            generate_identicon()
    else:
        core.log(name, "  Error: illegal command: {}.".format(key))<|endoftext|>


<commit_before># Copyright (c) 2005 Iowa State University
# http://mesonet.agron.iastate.edu/ -- mailto:akrherz@iastate.edu
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
""" Hacky meso AFD processor """

from twisted.python import log
import os
log.startLogging(open('/mesonet/data/logs/%s/meso_afd.log' % (os.getenv("USER"),), 'a'))
log.FileLogObserver.timeFormat = "%Y/%m/%d %H:%M:%S %Z"

import sys, logging
import re
import math
import smtplib
import mx.DateTime
import sys
import StringIO
import traceback
from email.MIMEText import MIMEText

# Non standard Stuff
from support import TextProduct, ldmbridge
import common

# Database Connections
import pg, secret
POSTGIS = pg.connect(secret.dbname, secret.dbhost, user=secret.dbuser)

errors = StringIO.StringIO()

from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.words.xish import domish
from twisted.internet import reactor

errors = StringIO.StringIO()

class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        """ I lost my connection, should I do anything else? """
        log.msg("LDM Closed PIPE")

    def process_data(self, raw):
        try:
            real_process(raw)
        except:
            io = StringIO.StringIO()
            traceback.print_exc(file=io)
            msg = MIMEText("%s\n\n>RAW DATA\n\n%s"%(io.getvalue(),
                           raw.replace("\015\015\012", "\n")))
            msg['subject'] = 'meso_afd.py Traceback'
            msg['From'] = "ldm@mesonet.agron.iastate.edu"
            msg['To'] = "akrherz@iastate.edu"

            s = smtplib.SMTP()
            s.connect()
            s.sendmail(msg["From"], msg["To"], msg.as_string())
            s.close()

def real_process(raw):
    tp = TextProduct.TextProduct(raw)
    pil = tp.afos[:3]
    toroom = tp.get_iembot_source()
    sqlraw = raw.replace("'", "\\'").replace("\015\015\012", "\n")
    prod = TextProduct.TextProduct(raw)
    product_id = prod.get_product_id()
    print product_id

    #tokens = re.findall("\.UPDATE\.\.\.MESOSCALE UPDATE", raw)
    tokens = re.findall("MESOSCALE UPDATE", raw)
    if (len(tokens) == 0):
        return

    sql = "INSERT into text_products(product, product_id) \
      values ('%s','%s')" % (sqlraw, product_id)
    POSTGIS.query(sql)

    mess = "%s: %s issues Mesoscale %s http://mesonet.agron.iastate.edu/p.php?pid=%s" % \
        (toroom, toroom, pil, product_id)
    messHTML = "%s issues Mesoscale Forecast Discussion" % \
        (toroom, product_id)
    jabber.sendMessage(mess, messHTML)

myJid = jid.JID('iembot_ingest@%s/meso_afd_%s' % \
      (secret.chatserver, mx.DateTime.gmt().strftime("%Y%m%d%H%M%S") ) )
factory = client.basicClientFactory(myJid, secret.iembot_ingest_password)

jabber = common.JabberClient(myJid)

factory.addBootstrap('//event/stream/authd', jabber.authd)
factory.addBootstrap("//event/client/basicauth/invaliduser", jabber.debug)
factory.addBootstrap("//event/client/basicauth/authfailed", jabber.debug)
factory.addBootstrap("//event/stream/error", jabber.debug)
factory.addBootstrap(xmlstream.STREAM_END_EVENT, jabber._disconnect )

reactor.connectTCP(secret.connect_chatserver, 5222, factory)

ldm = ldmbridge.LDMProductFactory( MyProductIngestor() )
reactor.run()

<commit_msg>here we go<commit_after># Copyright (c) 2005 Iowa State University
# http://mesonet.agron.iastate.edu/ -- mailto:akrherz@iastate.edu
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
""" Hacky meso AFD processor """

from twisted.python import log
import os
log.startLogging(open('/mesonet/data/logs/%s/meso_afd.log' % (os.getenv("USER"),), 'a'))
log.FileLogObserver.timeFormat = "%Y/%m/%d %H:%M:%S %Z"

import sys, logging
import re
import math
import smtplib
import mx.DateTime
import sys
import StringIO
import traceback
from email.MIMEText import MIMEText

# Non standard Stuff
from support import TextProduct, ldmbridge
import common

# Database Connections
import pg, secret
POSTGIS = pg.connect(secret.dbname, secret.dbhost, user=secret.dbuser)

errors = StringIO.StringIO()

from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.words.xish import domish
from twisted.internet import reactor

errors = StringIO.StringIO()

class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        """ I lost my connection, should I do anything else? """
        log.msg("LDM Closed PIPE")

    def process_data(self, raw):
        try:
            real_process(raw)
        except:
            io = StringIO.StringIO()
            traceback.print_exc(file=io)
            msg = MIMEText("%s\n\n>RAW DATA\n\n%s"%(io.getvalue(),
                           raw.replace("\015\015\012", "\n")))
            msg['subject'] = 'meso_afd.py Traceback'
            msg['From'] = secret.parser_user
            msg['To'] = "akrherz@iastate.edu"

            s = smtplib.SMTP()
            s.connect()
            s.sendmail(msg["From"], msg["To"], msg.as_string())
            s.close()

def real_process(raw):
    tp = TextProduct.TextProduct(raw)
    pil = tp.afos[:3]
    toroom = tp.get_iembot_source()
    sqlraw = raw.replace("'", "\\'").replace("\015\015\012", "\n")
    prod = TextProduct.TextProduct(raw)
    product_id = prod.get_product_id()
    print product_id

    #tokens = re.findall("\.UPDATE\.\.\.MESOSCALE UPDATE", raw)
    tokens = re.findall("MESOSCALE UPDATE", raw)
    if (len(tokens) == 0):
        return

    sql = "INSERT into text_products(product, product_id) \
      values ('%s','%s')" % (sqlraw, product_id)
    POSTGIS.query(sql)

    mess = "%s: %s issues Mesoscale %s %s?pid=%s" % \
        (toroom, toroom, pil, secret.parser_user, product_id)
    messHTML = "%s issues Mesoscale Forecast Discussion" % \
        (toroom, secret.parser_user, product_id)
    jabber.sendMessage(mess, messHTML)

myJid = jid.JID('iembot_ingest@%s/meso_afd_%s' % \
      (secret.chatserver, mx.DateTime.gmt().strftime("%Y%m%d%H%M%S") ) )
factory = client.basicClientFactory(myJid, secret.iembot_ingest_password)

jabber = common.JabberClient(myJid)

factory.addBootstrap('//event/stream/authd', jabber.authd)
factory.addBootstrap("//event/client/basicauth/invaliduser", jabber.debug)
factory.addBootstrap("//event/client/basicauth/authfailed", jabber.debug)
factory.addBootstrap("//event/stream/error", jabber.debug)
factory.addBootstrap(xmlstream.STREAM_END_EVENT, jabber._disconnect )

reactor.connectTCP(secret.connect_chatserver, 5222, factory)

ldm = ldmbridge.LDMProductFactory( MyProductIngestor() )
reactor.run()

<|endoftext|>


<commit_before>"""Module with an API for MediaWikis.

Copyright 2017 Stephan Kulla
"""

from abc import ABCMeta, abstractmethod
import urllib.request
from urllib.parse import quote
import re
import os

from mfnf.utils import stablehash, merge, query_path, select_singleton

class MediaWikiAPI(metaclass=ABCMeta):
    """Interface for accessing content of a MediaWiki project."""

    @abstractmethod
    def get_content(self, title):
        """Returns the content of the article with title `title`."""
        raise NotImplementedError()

    @abstractmethod
    def convert_text_to_html(self, title, text):
        """Converts MediaWiki code `text` into HTML representing it."""
        raise NotImplementedError()

    @abstractmethod
    def download_image(self, image_name, image_path):
        """Downloads image `image_name` and stores it to `image_path`."""
        raise NotImplementedError()

    @abstractmethod
    def get_revisions(self, title):
        """Returns the revisions of the article `title`."""
        raise NotImplementedError()

    @abstractmethod
    def normalize_formula(self, formula, mode):
        """Normalizes the formula `formula`. In case the given formula is
        ilformatted a `ValueError` is raised. `mode` can be either `"tex"`
        or `inline-tex`."""
        raise NotImplementedError()

class HTTPMediaWikiAPI(MediaWikiAPI):
    """Implements an API for content stored on a MediaWiki."""

    def __init__(self, req, domain="de.wikibooks.org"):
        """Initializes the object.

        Arguments:
        domain -- domain of the MediaWiki, e.g. `"de.wikibooks.org"`
        req    -- an session object of the `request` framework
        """
        self.domain = domain
        self.req = req

    def _stablehash(self):
        return stablehash((self.__class__.__name__, self.domain))

    @property
    def _index_url(self):
        """Returns the URL to the server's `index.php` file."""
        return "http://" + self.domain + "/w/index.php"

    @property
    def _api_url(self):
        """Returns the URL to the server's `api.php` file."""
        return "https://de.wikibooks.org/w/api.php"

    @property
    def _rest_api_url(self):
        """Returns the URL to the server's REST API endpoints."""
        return "https://de.wikibooks.org/api/rest_v1"

    def _index_call(self, params):
        """Make an HTTP request to the server's `index.php` file."""
        return self.req.get(self._index_url, params=params).text

    def _api_call(self, endpoint, data):
        """Call an REST API endpoint."""
        endpoint_url = "/".join([self._rest_api_url] + endpoint)

        result = self.req.post(endpoint_url, data=data)
        result.raise_for_status()

        return result

    def query(self, params, path_to_result):
        params["format"] = "json"
        params["action"] = "query"
        path_to_result = ["query"] + path_to_result
        result = None

        while True:
            api_result = self.req.get(self._api_url, params=params).json()

            if "error" in api_result:
                message = "Error while making API call."

                raise ConnectionError(api_result.get("info", message))

            result = merge(result, query_path(api_result, path_to_result))

            if "continue" in api_result:
                params.update(api_result["continue"])
            else:
                return result

    def get_image_revisions(self, filename):
        """Returns the history of the image `filename`."""
        params = {"titles": filename, "prop": "imageinfo", "iiprop": "url"}

        return self.query(params, ["pages", select_singleton, "imageinfo"])

    def get_image_url(self, filename):
        """Returns the URL to the current version of the image `filename`."""
        return self.get_image_revisions(filename)[-1]["url"]

    def get_content(self, title):
        return self._index_call({"action": "raw", "title": title})

    def convert_text_to_html(self, title, text):
        endpoint = ["transform", "wikitext", "to", "html", quote(title)]
        data = {"title": title, "wikitext": text, "body_only": True}

        return self._api_call(endpoint, data).text

    def get_revisions(self, title):
        params = {"prop": "revisions", "rvprop": "size|user", "titles": title,
                  "rvlimit": "max"}

        return self.query(params, ["pages", select_singleton, "revisions"])

    def download_image(self, image_name, image_path):
        if not os.path.exists(image_path):
            urllib.request.urlretrieve(self.get_image_url(image_name),
                                       image_path)

    def normalize_formula(self, formula, mode):
        assert mode in ["tex", "inline-tex"]

        endpoint = ["media", "math", "check"] + [mode]
        params = {"type": mode, "q": re.sub(r"\s", " ", formula)}

        try:
            result = self._api_call(endpoint, params).json()

            return result["checked"]
        except IOError as error:
            if error.response.status_code == 400:
                # Wrongly formatted math formula
                raise ValueError()
            else:
                raise error
<commit_msg>api: Raise exception on errors for api calls (#40)<commit_after>"""Module with an API for MediaWikis.

Copyright 2017 Stephan Kulla
"""

from abc import ABCMeta, abstractmethod
import urllib.request
from urllib.parse import quote
import re
import os

from mfnf.utils import stablehash, merge, query_path, select_singleton

class MediaWikiAPI(metaclass=ABCMeta):
    """Interface for accessing content of a MediaWiki project."""

    @abstractmethod
    def get_content(self, title):
        """Returns the content of the article with title `title`."""
        raise NotImplementedError()

    @abstractmethod
    def convert_text_to_html(self, title, text):
        """Converts MediaWiki code `text` into HTML representing it."""
        raise NotImplementedError()

    @abstractmethod
    def download_image(self, image_name, image_path):
        """Downloads image `image_name` and stores it to `image_path`."""
        raise NotImplementedError()

    @abstractmethod
    def get_revisions(self, title):
        """Returns the revisions of the article `title`."""
        raise NotImplementedError()

    @abstractmethod
    def normalize_formula(self, formula, mode):
        """Normalizes the formula `formula`. In case the given formula is
        ilformatted a `ValueError` is raised. `mode` can be either `"tex"`
        or `inline-tex`."""
        raise NotImplementedError()

class HTTPMediaWikiAPI(MediaWikiAPI):
    """Implements an API for content stored on a MediaWiki."""

    def __init__(self, req, domain="de.wikibooks.org"):
        """Initializes the object.

        Arguments:
        domain -- domain of the MediaWiki, e.g. `"de.wikibooks.org"`
        req    -- an session object of the `request` framework
        """
        self.domain = domain
        self.req = req

    def _stablehash(self):
        return stablehash((self.__class__.__name__, self.domain))

    @property
    def _index_url(self):
        """Returns the URL to the server's `index.php` file."""
        return "http://" + self.domain + "/w/index.php"

    @property
    def _api_url(self):
        """Returns the URL to the server's `api.php` file."""
        return "https://de.wikibooks.org/w/api.php"

    @property
    def _rest_api_url(self):
        """Returns the URL to the server's REST API endpoints."""
        return "https://de.wikibooks.org/api/rest_v1"

    def _index_call(self, params):
        """Make an HTTP request to the server's `index.php` file."""
        req = self.req.get(self._index_url, params=params)

        req.raise_for_status()

        return req.text

    def _api_call(self, endpoint, data):
        """Call an REST API endpoint."""
        endpoint_url = "/".join([self._rest_api_url] + endpoint)

        result = self.req.post(endpoint_url, data=data)
        result.raise_for_status()

        return result

    def query(self, params, path_to_result):
        params["format"] = "json"
        params["action"] = "query"
        path_to_result = ["query"] + path_to_result
        result = None

        while True:
            api_result = self.req.get(self._api_url, params=params).json()

            if "error" in api_result:
                message = "Error while making API call."

                raise ConnectionError(api_result.get("info", message))

            result = merge(result, query_path(api_result, path_to_result))

            if "continue" in api_result:
                params.update(api_result["continue"])
            else:
                return result

    def get_image_revisions(self, filename):
        """Returns the history of the image `filename`."""
        params = {"titles": filename, "prop": "imageinfo", "iiprop": "url"}

        return self.query(params, ["pages", select_singleton, "imageinfo"])

    def get_image_url(self, filename):
        """Returns the URL to the current version of the image `filename`."""
        return self.get_image_revisions(filename)[-1]["url"]

    def get_content(self, title):
        return self._index_call({"action": "raw", "title": title})

    def convert_text_to_html(self, title, text):
        endpoint = ["transform", "wikitext", "to", "html", quote(title)]
        data = {"title": title, "wikitext": text, "body_only": True}

        return self._api_call(endpoint, data).text

    def get_revisions(self, title):
        params = {"prop": "revisions", "rvprop": "size|user", "titles": title,
                  "rvlimit": "max"}

        return self.query(params, ["pages", select_singleton, "revisions"])

    def download_image(self, image_name, image_path):
        if not os.path.exists(image_path):
            urllib.request.urlretrieve(self.get_image_url(image_name),
                                       image_path)

    def normalize_formula(self, formula, mode):
        assert mode in ["tex", "inline-tex"]

        endpoint = ["media", "math", "check"] + [mode]
        params = {"type": mode, "q": re.sub(r"\s", " ", formula)}

        try:
            result = self._api_call(endpoint, params).json()

            return result["checked"]
        except IOError as error:
            if error.response.status_code == 400:
                # Wrongly formatted math formula
                raise ValueError()
            else:
                raise error
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Core functions and classes.

Basic Usage: ::

    from mfr.core import detect

    with open('myfile.jpg', 'r') as fp:
        handler = detect(fp)
        if handler:
            html = handler.render(fp)
"""
import os
import shutil
import inspect
import logging
from collections import defaultdict

from mfr._config import Config
from mfr.exceptions import ConfigurationError, MFRError

logger = logging.getLogger(__name__)

_defaults = {
    'INCLUDE_STATIC': False,
    'EXCLUDE_LIBS': []
}
#: Global mfr configuration object
config = Config(defaults=_defaults)

config['HANDLERS'] = []

def register_filehandler(file_handler):
    """Register a new file handler.
    Usage: ::

        register_filehandler(mfr_movie.Handler)

    :param FileHandler file_handler: A FileHandler class.
    """
    reg = get_registry()
    if file_handler not in reg:
        get_registry().append(file_handler)
        return True
    return False

def register_filehandlers(handlers):
    """Register multiple file handlers.
    Usage: ::

        register_file_handlers([
            mfr_image.Handler,
            mfr_movie.Handler,
            mfr_tabular.Handler])

    :param list handlers: A list of FileHandler classes.
    """
    for each in handlers:
        register_filehandler(each)

def get_registry():
    """Get the current list of registered filehandlers.

    :rtype: list
    """
    return config["HANDLERS"]

def clear_registry():
    """Reset the list of registered handlers."""
    config["HANDLERS"] = []

def reset_config():
    """Reset config defaults and empty the registry of FileHandlers."""
    global config
    config.clear()
    config.update(_defaults)
    config["HANDLERS"] = []

def detect(fp, handlers=None, instance=True, many=False, *args, **kwargs):
    """Return a :class:`FileHandler <mfr.core.FileHandler>` for a given file,
    or ``False`` if no handler could be found for the file.

    :param list handlers: A list of filehandler names to try. If ``None``,
        try all registered handlers.
    :return: A FileHandler that can handle the file, or False if no handler was
        found.
    """
    handlers = handlers or get_registry()
    valid_handlers = []
    for HandlerClass in handlers:
        handler = HandlerClass()
        if handler.detect(fp, *args, **kwargs):
            handler_obj = handler if instance else HandlerClass
            if many:
                valid_handlers.append(handler_obj)
            else:
                return handler_obj
    if many:
        return valid_handlers
    else:
        return None

def render(fp, handler=None, renderer=None, *args, **kwargs):
    """Core rendering function. Return the rendered HTML for a given file.

    :param File fp: A file pointer object to render.
    :param FileHandler: The file handler class to use.
    :param str renderer: The name of the renderer function to use (must be a key in
        in the handler class's `renderers` dictionary)
    """
    # Get the specified handler, detect it if not given
    handler = handler or detect(fp, many=False, instance=True)
    if not handler:
        #raise MFRError('No available handler that can handle {name}.'
        #               .format(name=os.path.basename(os.path.normpath(fp.name))))
        raise MFRError('No renderer currently available for this file type.')
    return handler.render(fp, renderer=renderer, *args, **kwargs)

def export(fp, handler=None, exporter=None, *args, **kwargs):
    """Core exporting function. Return the requested file.

    :param File fp: A file pointer object to export.
    :param FileHandler: The file handler class to use.
    :param str exporter: The name of the exporter function to use (must be a key in
        in the handler class's `exporters` dictionary)
    """
    # Get the specified handler, detect it if not given
    HandlerClass = handler or detect(fp)
    if not HandlerClass:
        raise MFRError('No available handler with name {handler}.'
                           .format(handler=handler))

    handler = HandlerClass
    return handler.export(fp, exporter=exporter, *args, **kwargs)

def get_file_extension(path, lower=True):
    """Get the file extension for a given file path."""
    ext = os.path.splitext(path)[1]
    return ext.lower() if lower else ext

def get_file_exporters(handler):
    """Get list of exporters for a given file handler."""
    try:
        return handler.exporters.keys()
    except AttributeError:
        return None

def assets_by_extension(assets):
    """Given a list of assets, return a dictionary keyed by extension

    :param list assets: List of asset paths (strings)
    :rtype: dict
    """
    ret = defaultdict(list)
    for asset in assets:
        key = get_file_extension(asset).lstrip('.')
        ret[key].append(asset)
    return ret

class RenderResult(object):
    """ An object that contains the html representation of content and any
     assets that should be included.
    """

    def __init__(self, content, assets=None):
        """
        Initialize a Render result.

        :param content: html representation of content
        :param assets: css, javascript, or other assets to be included
        """
        self.content = content
        if isinstance(assets, (list, tuple)):
            self.assets = assets_by_extension(assets)
        elif isinstance(assets, dict):  # assets is a dict
            self.assets = defaultdict(list, assets)
        else:  # assets is None
            self.assets = defaultdict(list)

    def __str__(self):
        return str(self.content)

    def __repr__(self):
        return '<RenderResult({0!r})>'.format(self.content)

    def __contains__(self, obj):
        """Implements the ``in`` keyword."""
        return obj in str(self.content)

class FileHandler(object):
    """Abstract base class from which all file handlers must inherit.
    """
    #: Maps renderer names to renderer callables, e.g. {'html': render_img_html}
    renderers = {}
    #: Maps exporter names to exporter callables, e.g. {'png': export_png}
    exporters = {}

    default_renderer = 'html'
    default_exporter = None

    def __init__(self):
        self.__assets = defaultdict(list)

    def detect(self, fp):
        """Return whether a given file can be handled by this file handler.
        MUST be implemented by descendant classes.
        """
        raise NotImplementedError('Must define detect method.')

    def render(self, fp, renderer=None, *args, **kwargs):
        """Return the rendered HTML for a file.

        :param fp: A file-like object.
        :param str renderer: The name of the renderer to use. If `None`,
            the default_renderer is used.
        """
        render_func = self.renderers.get(renderer or self.default_renderer)
        # TODO(sloria): check if render_func is callable
        if render_func:
            return render_func(fp, *args, **kwargs)
        else:
            raise MFRError('`render` method called with no renderer specified and '
                               'no default.')

    def export(self, fp, exporter=None, *args, **kwargs):
        """Export a file to a different format.

        :param str exporter: The name of the exporter to use. If `None`,
            the default_exporter is used.
        """
        export_func = self.exporters.get(exporter or self.default_exporter)
        if export_func:
            return export_func(fp, *args, **kwargs)
        else:
            raise MFRError('`export` method called with no exporter specified and '
                               'no default.')

    def iterstatic(self, url=True):
        """Iterates through the static asset files for the filehandler,
        yielding absolute paths to the files.

        :param bool url: If ``True``, return the static url for each asset.
            If ``False``, return the static folder for each asset.
        """
        static_folder = get_static_path_for_handler(self.__class__)
        for root, dirs, files in os.walk(static_folder):
            for filename in files:
                absolute_path = os.path.join(root, filename)
                if url:
                    namespace = get_namespace(self)
                    static_path = os.path.relpath(absolute_path, static_folder)
                    try:
                        yield os.path.join(config['ASSETS_URL'], namespace, static_path)
                    except KeyError:
                        raise KeyError('ASSETS_URL is not configured.')
                else:
                    yield absolute_path

    def get_assets(self, extension=None):
        """Get the urls for this handler's static assets. Return either a dict
        keyed by extension or a list of assets if ``extension`` is provided.

        Usage: ::

            >>> handler.get_assets()
            {'css': '/static/myformat/style.css', 'js': '/static/myformat/script.js'}

        """

        if not self.__assets:
            for asset in self.iterstatic(url=True):
                ext = get_file_extension(asset).lstrip('.')
                if ext:
                    self.__assets[ext].append(asset)
                else:
                    self.__assets['_'].append(asset)
        if extension:
            return self.__assets[extension]
        return self.__assets

def get_assets_from_list(assets_uri_base, ext, asset_list=None):
    """
    Generate an list of full uri paths to each assets, excluding any files
    listed in the Config's EXCLUDE_LIBS
    """

    return [
        os.path.join(assets_uri_base, ext, filepath)
        for filepath in asset_list
        if filepath not in config.get('EXCLUDE_LIBS')
    ]

def _get_dir_for_class(cls):
    """Return the absolute directory where a class resides."""
    fpath = inspect.getfile(cls)
    return os.path.abspath(os.path.dirname(fpath))

def get_static_path_for_handler(handler_cls):
    """Return the absolute path for a given FileHandler class.
    Defaults to a ``static`` folder within the same directory of the handler's
    module.
    """
    # If STATIC_PATH is defined, use that
    if hasattr(handler_cls, 'STATIC_PATH'):
        static_path = handler_cls.STATIC_PATH
    # Otherwise assume 'static' dir is in the same directory as
    # the handler class's module
    else:
        static_path = os.path.join(_get_dir_for_class(handler_cls), 'static')
    return static_path

def get_static_url_for_handler(handler_cls):
    if hasattr(handler_cls, 'ASSETS_URL'):
        url = os.path.join(config['ASSETS_URL'], handler_cls.ASSETS_URL)
    else:
        url = os.path.join(config['ASSETS_URL'], get_namespace(handler_cls))
    return url

def copy_dir(src, dest):
    """Recursively copies a directory's contents."""
    try:
        shutil.copytree(src, dest)
    except shutil.Error as err:
        logger.warn(err)
    except OSError as err:
        logger.debug('Skipping {src} (already exists)'.format(src=src))

def get_namespace(handler_cls):
    """Given a FileHandler class, return the namespace used by collect_static.
    The namespace defines the name of the folder that a file module's static
    assets will be copied to.
    """
    # If 'namespace' is defined on the class, use that
    if hasattr(handler_cls, 'namespace'):
        return handler_cls.namespace
    # Otherwise use the base name of the module
    else:
        # mypackage.mymodule.MyHandler => 'mymodule'
        return handler_cls.__module__.split('.')[-1]

def collect_static(dest=None, dry_run=False):
    """Collect all static assets for registered handlers to a single directory.
    Files will be copied to ``dest``, if specified, or the STATIC_PATH config
    variable.
    """
    dest_ = dest or config.get('ASSETS_FOLDER')
    if not dest_:
        raise ConfigurationError('ASSETS_FOLDER has not been configured.')
    for handler_cls in get_registry():
        static_path = get_static_path_for_handler(handler_cls)
        namespaced_destination = os.path.join(dest_, get_namespace(handler_cls))
        if dry_run:
            print('Pretending to copy {static_path} to {namespaced_destination}.'
                .format(**locals()))
        else:
            copy_dir(static_path, namespaced_destination)
<commit_msg>Raise excepted OSError on collect_static Unless src exists<commit_after># -*- coding: utf-8 -*-
"""Core functions and classes.

Basic Usage: ::

    from mfr.core import detect

    with open('myfile.jpg', 'r') as fp:
        handler = detect(fp)
        if handler:
            html = handler.render(fp)
"""
import os
import shutil
import inspect
import logging
from collections import defaultdict

from mfr._config import Config
from mfr.exceptions import ConfigurationError, MFRError

logger = logging.getLogger(__name__)

_defaults = {
    'INCLUDE_STATIC': False,
    'EXCLUDE_LIBS': []
}
#: Global mfr configuration object
config = Config(defaults=_defaults)

config['HANDLERS'] = []

def register_filehandler(file_handler):
    """Register a new file handler.
    Usage: ::

        register_filehandler(mfr_movie.Handler)

    :param FileHandler file_handler: A FileHandler class.
    """
    reg = get_registry()
    if file_handler not in reg:
        get_registry().append(file_handler)
        return True
    return False

def register_filehandlers(handlers):
    """Register multiple file handlers.
    Usage: ::

        register_file_handlers([
            mfr_image.Handler,
            mfr_movie.Handler,
            mfr_tabular.Handler])

    :param list handlers: A list of FileHandler classes.
    """
    for each in handlers:
        register_filehandler(each)

def get_registry():
    """Get the current list of registered filehandlers.

    :rtype: list
    """
    return config["HANDLERS"]

def clear_registry():
    """Reset the list of registered handlers."""
    config["HANDLERS"] = []

def reset_config():
    """Reset config defaults and empty the registry of FileHandlers."""
    global config
    config.clear()
    config.update(_defaults)
    config["HANDLERS"] = []

def detect(fp, handlers=None, instance=True, many=False, *args, **kwargs):
    """Return a :class:`FileHandler <mfr.core.FileHandler>` for a given file,
    or ``False`` if no handler could be found for the file.

    :param list handlers: A list of filehandler names to try. If ``None``,
        try all registered handlers.
    :return: A FileHandler that can handle the file, or False if no handler was
        found.
    """
    handlers = handlers or get_registry()
    valid_handlers = []
    for HandlerClass in handlers:
        handler = HandlerClass()
        if handler.detect(fp, *args, **kwargs):
            handler_obj = handler if instance else HandlerClass
            if many:
                valid_handlers.append(handler_obj)
            else:
                return handler_obj
    if many:
        return valid_handlers
    else:
        return None

def render(fp, handler=None, renderer=None, *args, **kwargs):
    """Core rendering function. Return the rendered HTML for a given file.

    :param File fp: A file pointer object to render.
    :param FileHandler: The file handler class to use.
    :param str renderer: The name of the renderer function to use (must be a key in
        in the handler class's `renderers` dictionary)
    """
    # Get the specified handler, detect it if not given
    handler = handler or detect(fp, many=False, instance=True)
    if not handler:
        #raise MFRError('No available handler that can handle {name}.'
        #               .format(name=os.path.basename(os.path.normpath(fp.name))))
        raise MFRError('No renderer currently available for this file type.')
    return handler.render(fp, renderer=renderer, *args, **kwargs)

def export(fp, handler=None, exporter=None, *args, **kwargs):
    """Core exporting function. Return the requested file.

    :param File fp: A file pointer object to export.
    :param FileHandler: The file handler class to use.
    :param str exporter: The name of the exporter function to use (must be a key in
        in the handler class's `exporters` dictionary)
    """
    # Get the specified handler, detect it if not given
    HandlerClass = handler or detect(fp)
    if not HandlerClass:
        raise MFRError('No available handler with name {handler}.'
                           .format(handler=handler))

    handler = HandlerClass
    return handler.export(fp, exporter=exporter, *args, **kwargs)

def get_file_extension(path, lower=True):
    """Get the file extension for a given file path."""
    ext = os.path.splitext(path)[1]
    return ext.lower() if lower else ext

def get_file_exporters(handler):
    """Get list of exporters for a given file handler."""
    try:
        return handler.exporters.keys()
    except AttributeError:
        return None

def assets_by_extension(assets):
    """Given a list of assets, return a dictionary keyed by extension

    :param list assets: List of asset paths (strings)
    :rtype: dict
    """
    ret = defaultdict(list)
    for asset in assets:
        key = get_file_extension(asset).lstrip('.')
        ret[key].append(asset)
    return ret

class RenderResult(object):
    """ An object that contains the html representation of content and any
     assets that should be included.
    """

    def __init__(self, content, assets=None):
        """
        Initialize a Render result.

        :param content: html representation of content
        :param assets: css, javascript, or other assets to be included
        """
        self.content = content
        if isinstance(assets, (list, tuple)):
            self.assets = assets_by_extension(assets)
        elif isinstance(assets, dict):  # assets is a dict
            self.assets = defaultdict(list, assets)
        else:  # assets is None
            self.assets = defaultdict(list)

    def __str__(self):
        return str(self.content)

    def __repr__(self):
        return '<RenderResult({0!r})>'.format(self.content)

    def __contains__(self, obj):
        """Implements the ``in`` keyword."""
        return obj in str(self.content)

class FileHandler(object):
    """Abstract base class from which all file handlers must inherit.
    """
    #: Maps renderer names to renderer callables, e.g. {'html': render_img_html}
    renderers = {}
    #: Maps exporter names to exporter callables, e.g. {'png': export_png}
    exporters = {}

    default_renderer = 'html'
    default_exporter = None

    def __init__(self):
        self.__assets = defaultdict(list)

    def detect(self, fp):
        """Return whether a given file can be handled by this file handler.
        MUST be implemented by descendant classes.
        """
        raise NotImplementedError('Must define detect method.')

    def render(self, fp, renderer=None, *args, **kwargs):
        """Return the rendered HTML for a file.

        :param fp: A file-like object.
        :param str renderer: The name of the renderer to use. If `None`,
            the default_renderer is used.
        """
        render_func = self.renderers.get(renderer or self.default_renderer)
        # TODO(sloria): check if render_func is callable
        if render_func:
            return render_func(fp, *args, **kwargs)
        else:
            raise MFRError('`render` method called with no renderer specified and '
                               'no default.')

    def export(self, fp, exporter=None, *args, **kwargs):
        """Export a file to a different format.

        :param str exporter: The name of the exporter to use. If `None`,
            the default_exporter is used.
        """
        export_func = self.exporters.get(exporter or self.default_exporter)
        if export_func:
            return export_func(fp, *args, **kwargs)
        else:
            raise MFRError('`export` method called with no exporter specified and '
                               'no default.')

    def iterstatic(self, url=True):
        """Iterates through the static asset files for the filehandler,
        yielding absolute paths to the files.

        :param bool url: If ``True``, return the static url for each asset.
            If ``False``, return the static folder for each asset.
        """
        static_folder = get_static_path_for_handler(self.__class__)
        for root, dirs, files in os.walk(static_folder):
            for filename in files:
                absolute_path = os.path.join(root, filename)
                if url:
                    namespace = get_namespace(self)
                    static_path = os.path.relpath(absolute_path, static_folder)
                    try:
                        yield os.path.join(config['ASSETS_URL'], namespace, static_path)
                    except KeyError:
                        raise KeyError('ASSETS_URL is not configured.')
                else:
                    yield absolute_path

    def get_assets(self, extension=None):
        """Get the urls for this handler's static assets. Return either a dict
        keyed by extension or a list of assets if ``extension`` is provided.

        Usage: ::

            >>> handler.get_assets()
            {'css': '/static/myformat/style.css', 'js': '/static/myformat/script.js'}

        """

        if not self.__assets:
            for asset in self.iterstatic(url=True):
                ext = get_file_extension(asset).lstrip('.')
                if ext:
                    self.__assets[ext].append(asset)
                else:
                    self.__assets['_'].append(asset)
        if extension:
            return self.__assets[extension]
        return self.__assets

def get_assets_from_list(assets_uri_base, ext, asset_list=None):
    """
    Generate an list of full uri paths to each assets, excluding any files
    listed in the Config's EXCLUDE_LIBS
    """

    return [
        os.path.join(assets_uri_base, ext, filepath)
        for filepath in asset_list
        if filepath not in config.get('EXCLUDE_LIBS')
    ]

def _get_dir_for_class(cls):
    """Return the absolute directory where a class resides."""
    fpath = inspect.getfile(cls)
    return os.path.abspath(os.path.dirname(fpath))

def get_static_path_for_handler(handler_cls):
    """Return the absolute path for a given FileHandler class.
    Defaults to a ``static`` folder within the same directory of the handler's
    module.
    """
    # If STATIC_PATH is defined, use that
    if hasattr(handler_cls, 'STATIC_PATH'):
        static_path = handler_cls.STATIC_PATH
    # Otherwise assume 'static' dir is in the same directory as
    # the handler class's module
    else:
        static_path = os.path.join(_get_dir_for_class(handler_cls), 'static')
    return static_path

def get_static_url_for_handler(handler_cls):
    if hasattr(handler_cls, 'ASSETS_URL'):
        url = os.path.join(config['ASSETS_URL'], handler_cls.ASSETS_URL)
    else:
        url = os.path.join(config['ASSETS_URL'], get_namespace(handler_cls))
    return url

def copy_dir(src, dest):
    """Recursively copies a directory's contents."""
    try:
        shutil.copytree(src, dest)
    except shutil.Error as err:
        logger.warn(err)
    except OSError as err:
        if os.path.exists(src):
            logger.debug('Skipping {src} (already exists)'.format(src=src))
        else:
            raise err

def get_namespace(handler_cls):
    """Given a FileHandler class, return the namespace used by collect_static.
    The namespace defines the name of the folder that a file module's static
    assets will be copied to.
    """
    # If 'namespace' is defined on the class, use that
    if hasattr(handler_cls, 'namespace'):
        return handler_cls.namespace
    # Otherwise use the base name of the module
    else:
        # mypackage.mymodule.MyHandler => 'mymodule'
        return handler_cls.__module__.split('.')[-1]

def collect_static(dest=None, dry_run=False):
    """Collect all static assets for registered handlers to a single directory.
    Files will be copied to ``dest``, if specified, or the STATIC_PATH config
    variable.
    """
    dest_ = dest or config.get('ASSETS_FOLDER')
    if not dest_:
        raise ConfigurationError('ASSETS_FOLDER has not been configured.')
    for handler_cls in get_registry():
        static_path = get_static_path_for_handler(handler_cls)
        namespaced_destination = os.path.join(dest_, get_namespace(handler_cls))
        if dry_run:
            print('Pretending to copy {static_path} to {namespaced_destination}.'
                .format(**locals()))
        else:
            copy_dir(static_path, namespaced_destination)
<|endoftext|>


<commit_before>from .http import HttpServer, HttpHeaders
from diesel.util.queue import Queue
from diesel import fork, until, receive, first, ConnectionClosed, send
from simplejson import dumps, loads, JSONDecodeError
import cgi, hashlib
from struct import pack, unpack
from base64 import b64encode
from array import array

class WebSocketDisconnect(object): pass
class WebSocketData(dict): pass

server_handshake_hybi = \
'''HTTP/1.1 101 Switching Protocols\r
Upgrade: websocket\r
Connection: Upgrade\r
Sec-WebSocket-Accept: %s\r
'''

class WebSocketServer(HttpServer):
    '''Very simple Web Socket server.
    '''

    GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"

    def __init__(self, web_handler, web_socket_handler, ws_location):
        self.web_handler = web_handler
        self.web_socket_handler = web_socket_handler
        self.ws_location = ws_location
        HttpServer.__init__(self, self.do_upgrade)

    def do_upgrade(self, req):
        if req.headers.get_one('Upgrade') != 'websocket' and req.headers.get_one('Upgrade') != 'WebSocket':
            return self.web_handler(req)

        hybi = False

        # do upgrade response
        org = req.headers.get_one('Origin')
        if 'Sec-WebSocket-Key' in req.headers:
            assert req.headers.get_one('Sec-WebSocket-Version') in ['8', '13'], \
                   "We currently only support Websockets version 8 and 13 (ver=%s)" % \
                   req.headers.get_one('Sec-WebSocket-Version')

            protocol = (req.headers.get_one('Sec-WebSocket-Protocol')
                        if 'Sec-WebSocket-Protocol' in req.headers else None)
            key = req.headers.get_one('Sec-WebSocket-Key')
            accept = b64encode(hashlib.sha1(key + self.GUID).digest())
            send(server_handshake_hybi % accept)
            if protocol:
                print "orot"
                send("Sec-WebSocket-Protocol: %s\r\n" % protocol)
            send("\r\n")
            hybi = True

        elif 'Sec-WebSocket-Key1' in req.headers:
            protocol = (req.headers.get_one('Sec-WebSocket-Protocol')
                        if 'Sec-WebSocket-Protocol' in req.headers else None)
            key1 = req.headers.get_one('Sec-WebSocket-Key1')
            key2 = req.headers.get_one('Sec-WebSocket-Key2')
            key3 = receive(8)
            num1 = int(''.join(c for c in key1 if c in '0123456789'))
            num2 = int(''.join(c for c in key2 if c in '0123456789'))
            assert num1 % key1.count(' ') == 0
            assert num2 % key2.count(' ') == 0
            final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3)
            secure_response = hashlib.md5(final).digest()
            send(
'''HTTP/1.1 101 Web Socket Protocol Handshake\r
Upgrade: WebSocket\r
Connection: Upgrade\r
Sec-WebSocket-Origin: %s\r
Sec-WebSocket-Location: %s\r
'''% (org, self.ws_location))
            if protocol:
                send("Sec-WebSocket-Protocol: %s\r\n" % (protocol,))
            send("\r\n")
            send(secure_response)

        else:
            send(
'''HTTP/1.1 101 Web Socket Protocol Handshake\r
Upgrade: WebSocket\r
Connection: Upgrade\r
WebSocket-Origin: %s\r
WebSocket-Location: %s\r
WebSocket-Protocol: diesel-generic\r
\r
''' % (org, self.ws_location))

        inq = Queue()
        outq = Queue()

        def wrap(req, inq, outq):
            self.web_socket_handler(req, inq, outq)
            outq.put(WebSocketDisconnect())

        fork(wrap, req, inq, outq)

        while True:
            try:
                if hybi:
                    typ, val = first(receive=2, waits=[outq])
                    if typ == 'receive':
                        b1, b2 = unpack(">BB", val)

                        opcode = b1 & 0x0f
                        fin = (b1 & 0x80) >> 7
                        has_mask = (b2 & 0x80) >> 7

                        assert has_mask == 1, "Frames must be masked"

                        if opcode == 8:
                            inq.put(WebSocketDisconnect())
                        else:
                            assert opcode == 1, "Currently only opcode 1 is supported (opcode=%s)" % opcode
                            length = b2 & 0x7f
                            if length == 126:
                                length = unpack('>H', receive(2))[0]
                            elif length == 127:
                                length = unpack('>L', receive(8))[0]

                            mask = unpack('>BBBB', receive(4))
                            payload = array('B', receive(length))
                            for i in xrange(len(payload)):
                                payload[i] ^= mask[i % 4]

                            try:
                                data = loads(payload.tostring())
                                inq.put(data)
                            except JSONDecodeError:
                                pass

                    elif typ == outq:
                        if type(val) is WebSocketDisconnect:
                            b1 = 0x80 | (8 & 0x0f) # FIN + opcode
                            send(pack('>BB', b1, 0))
                            break
                        else:
                            payload = dumps(val)

                            b1 = 0x80 | (1 & 0x0f) # FIN + opcode

                            payload_len = len(payload)
                            if payload_len <= 125:
                                header = pack('>BB', b1, payload_len)
                            elif payload_len > 125 and payload_len < 65536:
                                header = pack('>BBH', b1, 126, payload_len)
                            elif payload_len >= 65536:
                                header = pack('>BBQ', b1, 127, payload_len)

                        send(header + payload)
                else:
                    typ, val = first(receive=1, waits=[outq])
                    if typ == 'receive':
                        assert val == '\x00'
                        val = until('\xff')[:-1]
                        if val == '':
                            inq.put(WebSocketDisconnect())
                        else:
                            try:
                                data = loads(val)
                                inq.put(data)
                            except JSONDecodeError:
                                pass
                    elif typ == outq:
                        if type(val) is WebSocketDisconnect:
                            send('\x00\xff')
                            break
                        else:
                            data = dumps(dict(val))
                            send('\x00%s\xff' % data)

            except ConnectionClosed:
                inq.put(WebSocketDisconnect())
                raise ConnectionClosed("remote disconnected")
<commit_msg>Remove a stupid print<commit_after>from .http import HttpServer, HttpHeaders
from diesel.util.queue import Queue
from diesel import fork, until, receive, first, ConnectionClosed, send
from simplejson import dumps, loads, JSONDecodeError
import cgi, hashlib
from struct import pack, unpack
from base64 import b64encode
from array import array

class WebSocketDisconnect(object): pass
class WebSocketData(dict): pass

server_handshake_hybi = \
'''HTTP/1.1 101 Switching Protocols\r
Upgrade: websocket\r
Connection: Upgrade\r
Sec-WebSocket-Accept: %s\r
'''

class WebSocketServer(HttpServer):
    '''Very simple Web Socket server.
    '''

    GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"

    def __init__(self, web_handler, web_socket_handler, ws_location):
        self.web_handler = web_handler
        self.web_socket_handler = web_socket_handler
        self.ws_location = ws_location
        HttpServer.__init__(self, self.do_upgrade)

    def do_upgrade(self, req):
        if req.headers.get_one('Upgrade') != 'websocket' and req.headers.get_one('Upgrade') != 'WebSocket':
            return self.web_handler(req)

        hybi = False

        # do upgrade response
        org = req.headers.get_one('Origin')
        if 'Sec-WebSocket-Key' in req.headers:
            assert req.headers.get_one('Sec-WebSocket-Version') in ['8', '13'], \
                   "We currently only support Websockets version 8 and 13 (ver=%s)" % \
                   req.headers.get_one('Sec-WebSocket-Version')

            protocol = (req.headers.get_one('Sec-WebSocket-Protocol')
                        if 'Sec-WebSocket-Protocol' in req.headers else None)
            key = req.headers.get_one('Sec-WebSocket-Key')
            accept = b64encode(hashlib.sha1(key + self.GUID).digest())
            send(server_handshake_hybi % accept)
            if protocol:
                send("Sec-WebSocket-Protocol: %s\r\n" % protocol)
            send("\r\n")
            hybi = True

        elif 'Sec-WebSocket-Key1' in req.headers:
            protocol = (req.headers.get_one('Sec-WebSocket-Protocol')
                        if 'Sec-WebSocket-Protocol' in req.headers else None)
            key1 = req.headers.get_one('Sec-WebSocket-Key1')
            key2 = req.headers.get_one('Sec-WebSocket-Key2')
            key3 = receive(8)
            num1 = int(''.join(c for c in key1 if c in '0123456789'))
            num2 = int(''.join(c for c in key2 if c in '0123456789'))
            assert num1 % key1.count(' ') == 0
            assert num2 % key2.count(' ') == 0
            final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3)
            secure_response = hashlib.md5(final).digest()
            send(
'''HTTP/1.1 101 Web Socket Protocol Handshake\r
Upgrade: WebSocket\r
Connection: Upgrade\r
Sec-WebSocket-Origin: %s\r
Sec-WebSocket-Location: %s\r
'''% (org, self.ws_location))
            if protocol:
                send("Sec-WebSocket-Protocol: %s\r\n" % (protocol,))
            send("\r\n")
            send(secure_response)

        else:
            send(
'''HTTP/1.1 101 Web Socket Protocol Handshake\r
Upgrade: WebSocket\r
Connection: Upgrade\r
WebSocket-Origin: %s\r
WebSocket-Location: %s\r
WebSocket-Protocol: diesel-generic\r
\r
''' % (org, self.ws_location))

        inq = Queue()
        outq = Queue()

        def wrap(req, inq, outq):
            self.web_socket_handler(req, inq, outq)
            outq.put(WebSocketDisconnect())

        fork(wrap, req, inq, outq)

        while True:
            try:
                if hybi:
                    typ, val = first(receive=2, waits=[outq])
                    if typ == 'receive':
                        b1, b2 = unpack(">BB", val)

                        opcode = b1 & 0x0f
                        fin = (b1 & 0x80) >> 7
                        has_mask = (b2 & 0x80) >> 7

                        assert has_mask == 1, "Frames must be masked"

                        if opcode == 8:
                            inq.put(WebSocketDisconnect())
                        else:
                            assert opcode == 1, "Currently only opcode 1 is supported (opcode=%s)" % opcode
                            length = b2 & 0x7f
                            if length == 126:
                                length = unpack('>H', receive(2))[0]
                            elif length == 127:
                                length = unpack('>L', receive(8))[0]

                            mask = unpack('>BBBB', receive(4))
                            payload = array('B', receive(length))
                            for i in xrange(len(payload)):
                                payload[i] ^= mask[i % 4]

                            try:
                                data = loads(payload.tostring())
                                inq.put(data)
                            except JSONDecodeError:
                                pass

                    elif typ == outq:
                        if type(val) is WebSocketDisconnect:
                            b1 = 0x80 | (8 & 0x0f) # FIN + opcode
                            send(pack('>BB', b1, 0))
                            break
                        else:
                            payload = dumps(val)

                            b1 = 0x80 | (1 & 0x0f) # FIN + opcode

                            payload_len = len(payload)
                            if payload_len <= 125:
                                header = pack('>BB', b1, payload_len)
                            elif payload_len > 125 and payload_len < 65536:
                                header = pack('>BBH', b1, 126, payload_len)
                            elif payload_len >= 65536:
                                header = pack('>BBQ', b1, 127, payload_len)

                        send(header + payload)
                else:
                    typ, val = first(receive=1, waits=[outq])
                    if typ == 'receive':
                        assert val == '\x00'
                        val = until('\xff')[:-1]
                        if val == '':
                            inq.put(WebSocketDisconnect())
                        else:
                            try:
                                data = loads(val)
                                inq.put(data)
                            except JSONDecodeError:
                                pass
                    elif typ == outq:
                        if type(val) is WebSocketDisconnect:
                            send('\x00\xff')
                            break
                        else:
                            data = dumps(dict(val))
                            send('\x00%s\xff' % data)

            except ConnectionClosed:
                inq.put(WebSocketDisconnect())
                raise ConnectionClosed("remote disconnected")
<|endoftext|>