<commit_before>#!/usr/bin/env python3
# bash-help-gui-qt5.py
#
# Provide a GUI to display bash help.
#
# At a console you can type "& <tab> <tab>" and it will display all bash
# commands. This is due to the bash utility "compgen". This program collects
# all the bash commands by having subprocess execute the "compgen -c" command.
#
# Upon clicking on a command the following will attempt to be displayed:
# 1. Return from "whatis", which is the man page description line.
# 2. -h Short form help
# 3. --help Long form help
# 4. man pages
#
# Tested on:
# Manjaro 21.3.7 / Python 3.10.5 / Qt 5.15.5
#
# Ian Stewart - ©CC0
#
import sys
from subprocess import check_output, Popen, PIPE, STDOUT, run
from PyQt5.QtCore import Qt, qVersion
from PyQt5.QtWidgets import (QApplication, QMainWindow, QSplitter,
        QTextEdit, QTreeWidgetItem,  QTreeWidget)

# version
VERSION = "2022-08-21"
# Qt version
QT_VERSION = "{}".format(qVersion())
# Provide the column header with a heading.
HEADING = 'Bash Categories'
# Welcome message
WELCOME = """
Welcome to Bash Help Gui
Version:{}.
Qt Version:{}.
App:{}

Select a Help Category and then an item within the category.
""".format(VERSION, QT_VERSION, sys.argv[0])

class MainWindow(QMainWindow):
    def __init__(self, dictionary) -> None:
        super().__init__()
        self.setWindowTitle("Help for Bash")
        self.resize(1400,600)
        # Where does QGuiApplication come from?)
        #self.resize(QGuiApplication.primaryScreen().availableGeometry().size() * 0.7)

        splitter = QSplitter()
        self.setCentralWidget(splitter)

        self.textedit = QTextEdit()
        self.textedit.setText("")
        self.textedit.setStyleSheet("font: 14pt Monospace")
        splitter.addWidget(self.textedit)

        self.tree = QTreeWidget()
        splitter.addWidget(self.tree)
        self.tree.clear()
        self.tree.setHeaderLabel(HEADING)
        self.tree.setColumnCount(1)
        self.tree.clicked.connect(self.treewidget_clicked)
        self.fill_tree_widget_item(self.tree.invisibleRootItem(), dictionary)

        splitter.setStretchFactor(0,4)
        splitter.setStretchFactor(1,1)

        self.textedit.setText(WELCOME + MESSAGE)

    def fill_tree_widget_item(self, invisible_root_item, dictionary):
        for keyword, element_list in dictionary.items():
            parent = QTreeWidgetItem([str(keyword)])
            parent.setFlags(parent.flags() & ~ Qt.ItemFlag.ItemIsSelectable)
            invisible_root_item.addChild(parent)
            #parent.setExpanded(True) # Default is False
            for element in element_list:
                child = QTreeWidgetItem([str(element)])
                parent.addChild(child)

    def treewidget_clicked(self, model_index):
        #print(model_index) # PyQt6.QtCore.QModelIndex object
        #print(model_index.flags().value) # 60 parent or 61 child
        # Check is ItemIsSelectable based on model_index.flags()
        if not model_index.flags() & Qt.ItemFlag.ItemIsSelectable:
            #print("Item is not selectable")
            return
        self.item_selected = model_index.data()
        #print("Selected Item: {}".format(self.item_selected))

        self.setWindowTitle("Help for Bash - Selection: {}"
                    .format(self.item_selected))

        # Get the whatis, -h, --help and man pages info.
        one_line_description = self.get_whatis(self.item_selected)
        manpage_info = self.get_manpage(self.item_selected)
        help_short = self.get_help_short(self.item_selected)
        help_long = self.get_help_long(self.item_selected)

        # Build text and display
        string = self.item_selected + ": "
        string += "\n" + one_line_description
        string += "\n" + "== -h ====" + "=" * 80
        string += "\n" + help_short
        string += "\n" + "== --help " + "=" * 80
        string += "\n" + help_long
        string += "\n" + "== man ===" + "=" * 80
        string += "\n" + manpage_info
        string += "\n" + "=" * 90

        self.textedit.setText(string)

    def get_help_short(self, item):
        """
        Get the -h of an item, if there is one.
        """
        cmd = "{} -h".format(item)
        output = run(cmd, capture_output=True, shell=True, executable='/bin/bash')
        output_byte = output.stdout
        output_str = output_byte.decode("utf-8")
        return output_str

    def get_help_long(self, item):
        """
        Get the --help of an item, if there is one.
        """
        cmd = "{} --help".format(item)
        output = run(cmd, capture_output=True, shell=True, executable='/bin/bash')
        output_byte = output.stdout
        output_str = output_byte.decode("utf-8")
        return output_str

    def get_manpage(self, item):
        """
        Get the manpages if they exist
        """
        cmd = "man {}".format(item)
        output = Popen([cmd], shell = True, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        output_byte = output.communicate()[0]
        output_str = output_byte.decode("utf-8")
        return output_str

    def get_whatis(self, item):
        """
        From the man pages get the one-line description
        """
        cmd = "whatis -l {}".format(item)
        output = Popen([cmd], shell = True, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        output_byte = output.communicate()[0]
        output_str = output_byte.decode("utf-8")
        # Clear the "nothing appropriate" entries. Could change to
        if "nothing appropriate." in output_str:
            output_str = " - "
        # Example: cp (1)               - copy files and directories
        # All descriptions prefixed with ' - '
        output_str = output_str.split(" - ")[1]
        #print(item, output_str)
        return output_str

# Functions for Initial Data collection on launch of bash commands
def get_compgen_list():
    """
    Get the commands from the bash utility compgen -c.
    Output is in bytes.
    Convert to a string list
    Remove duplicates.
    Sort
    """
    output = check_output('compgen -c', shell=True, executable='/bin/bash')
    command_list_byte = output.splitlines()
    # Convert from bytes list to string list
    command_list_str = []
    for item in command_list_byte:
        command_list_str.append(item.decode("utf-8"))
    # Remove duplicates in the list
    command_list = list(dict.fromkeys(command_list_str))
    # Sort the list
    command_list.sort()
    return command_list

def build_dict(command_list):
    """
    Build a dictionary. A to Z as keywords, # as numbers, misc as miscellaneous
    >>> chr(65) 'A' >>> chr(90) 'Z'
    """
    command_dict = {}
    # For numbers
    command_dict["#"] = []
    # Miscellaneous
    command_dict["Misc"] = []
    # Create Dictionary keywords from A to Z

    for value in range(65, 91):
        command_dict[chr(value)] = []

    for item in command_list:
        start_char = item[0:1].upper()
        if ord(start_char) >= 65 and ord(start_char) <= 90:
            command_dict[start_char].append(item)
        elif start_char.isdecimal():
            command_dict["#"].append(item)
        else:
            command_dict["Misc"].append(item)
    return command_dict

if __name__ == '__main__':

    print("""
    Collecting information and setting up Bash Help Gui application.
    """)

    # Get the data and build the dictionary to pass data to TreeStore/TreeView
    # Get list of all bash commands from compgen with duplicates removed.
    bash_command_list = get_compgen_list()
    #print(bash_command_list)

    MESSAGE = ("\nA total of {} bash commands have been identified."
                .format(len(bash_command_list)))

    # Convert into a dictionary
    command_dict = build_dict(bash_command_list)
    #print(command_dict)

    #for key, item in command_dict.items():
    #    print(key, len(item))

    app = QApplication([])
    w = MainWindow(command_dict)
    w.show()
    app.exec()

"""
Reference
https://stackoverflow.com/questions/21805047/qtreewidget-to-mirror-python-dictionary

On launching this message is observed on the console:
Warning: Ignoring XDG_SESSION_TYPE=wayland on Gnome.
Use QT_QPA_PLATFORM=wayland to run on Wayland anyway.

"""
<commit_msg>Delete bash-help-gui-qt5.py<commit_after><|endoftext|>


<commit_before>"""
    Generates billing sample data
"""

import datetime
import os
import operator
import pytz
import json
import random
import sys
import time

# The granularity at which the charge collection system collects charges. Once
# per hour makes for a very slow UI, so I upped it to once per 8 hours.
CHARGE_HOURS = 8

MINUTE_SECONDS = 60
HOUR_SECONDS = MINUTE_SECONDS * 60
DAY_SECONDS = HOUR_SECONDS * 24
MONTH_SECONDS = DAY_SECONDS * 30

# XXX hardcoded path
sys.path.append("/opt/xos")

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xos.settings")
#from openstack.manager import OpenStackManager
from core.models import Slice, Sliver, ServiceClass, Reservation, Tag, Network, User, Node, Image, Deployment, Site, NetworkTemplate, NetworkSlice
from core.models import Invoice, Charge, Account, UsableObject, Payment

def delete_all(model):
   for item in model.objects.all():
       item.delete()

def get_usable_object(name):
   objs = UsableObject.objects.filter(name=name)
   if objs:
       return objs[0]
   obj = UsableObject(name=name)
   obj.save()
   return obj

def generate_invoice(account, batch):
   invoice = Invoice(date=batch[-1].date, account=account)
   invoice.save()
   for charge in batch:
       charge.invoice = invoice
       charge.state = "invoiced"
       charge.save()

def generate_invoices(account):
   invoices = sorted(Invoice.objects.filter(account=account), key=operator.attrgetter('date'))
   charges = sorted(Charge.objects.filter(account=account, state="pending"), key=operator.attrgetter('date'))

   if invoices:
       latest_invoice_date = invoices[-1].date()
   else:
       latest_invoice_date = None

   batch = []
   last_week = 0
   for charge in charges:
       # check to see if we crossed a week boundary. If we did, then generate
       # an invoice for the last week's batch of charges
       week = charge.date.isocalendar()[1]
       if (week != last_week) and (batch):
           generate_invoice(account, batch)
           batch = []
           last_week = week
       batch.append(charge)

   # we might still have last week's data batched up, and no data for this week
   # if so, invoice the batch
   this_week = datetime.datetime.now().isocalendar()[1]
   if (this_week != last_week) and (batch):
       generate_invoice(account, batch)

def generate_payments(account):
    invoices = Invoice.objects.filter(account=account)
    for invoice in invoices:
        # let's be optomistic and assume everyone pays exactly two weeks after
        # receiving an invoice
        payment_time = int(invoice.date.strftime("%s")) + 14 * DAY_SECONDS
        if payment_time < time.time():
             payment_time = datetime.datetime.utcfromtimestamp(payment_time).replace(tzinfo=pytz.utc)
             payment = Payment(account=account, amount=invoice.amount, date=payment_time)
             payment.save()

print "deleting old stuff"

delete_all(Invoice)
delete_all(Charge)
delete_all(Payment)
delete_all(Account)
delete_all(UsableObject)

print "creating accounts"

for site in Site.objects.all():
    # only create accounts for sites where some slices exist
    if len(site.slices.all()) > 0:
        account = Account(site=site)
        account.save()

print "generating charges"

for slice in Slice.objects.all():
    site = slice.site
    account = site.accounts.all()[0]
    serviceClass =slice.serviceClass

    if not (slice.name in ["DnsRedir", "DnsDemux", "HyperCache", "Hadoop", "Owl", "Stork", "Syndicate", "test-slice-1", "test-slice-2", "test", "test2"]):
        continue

    print "   generating charges for", slice.name

    now = int(time.time())/HOUR_SECONDS*HOUR_SECONDS

    charge_kind=None
    for resource in slice.serviceClass.serviceresources.all():
        if resource.name == "numberCores":
            charge_kind = "reservation"
            cost = resource.cost
        elif (charge_kind==None) and (resource.name == "cycles") or (resource.name == "Cycles"):
            charge_kind = "besteffort"
            cost = resource.cost

    if not charge_kind:
        print "failed to find resource for", slice.serviceClass
        continue

    for sliver in slice.slivers.all()[:4]:    # only do up to 4 slivers; it's way too much data otherwise
        hostname = sliver.node.name
        for i in range(now-MONTH_SECONDS, now, CHARGE_HOURS*HOUR_SECONDS):
            if charge_kind == "besteffort":
                core_hours = random.randint(20,60)/100.0
            else:
                core_hours = 1

            core_hours = core_hours * CHARGE_HOURS

            amount = float(core_hours * cost) / 100.0

            object = get_usable_object(hostname)

            date = datetime.datetime.utcfromtimestamp(i).replace(tzinfo=pytz.utc)

            charge = Charge(account=account, slice=slice, kind=charge_kind, state="pending", date=date, object=object, coreHours=core_hours, amount=amount)
            charge.save()

print "doing invoices and payments"

for account in Account.objects.all():
    generate_invoices(account)
    generate_payments(account)

<commit_msg>remove obsolete billing sample generator<commit_after><|endoftext|>


<commit_before># Help visualize histograms for hyperspectral images

import os
import cv2
import numpy as np
import pandas as pd
from plantcv.plantcv._debug import _debug
from plantcv.plantcv import fatal_error, params, outputs, color_palette
from plotnine import ggplot, aes, geom_line, geom_errorbar, scale_x_continuous, scale_color_manual
from plantcv.plantcv.visualize import histogram
from plantcv.plantcv.hyperspectral import _find_closest

'''
    == A few notes about color ==

    Color   Wavelength(nm) Frequency(THz)
    Red     620-750        484-400
    Orange  590-620        508-484
    Yellow  570-590        526-508
    Green   495-570        606-526
    Blue    450-495        668-606
    Violet  380-450        789-668

    f is frequency (cycles per second)
    l (lambda) is wavelength (meters per cycle)
    e is energy (Joules)
    h (Plank's constant) = 6.6260695729 x 10^-34 Joule*seconds
                         = 6.6260695729 x 10^-34 m^2*kg/seconds
    c = 299792458 meters per second
    f = c/l
    l = c/f
    e = h*f
    e = c*h/l

    List of peak frequency responses for each type of
    photoreceptor cell in the human eye:
        S cone: 437 nm
        M cone: 533 nm
        L cone: 564 nm
        rod:    550 nm in bright daylight, 498 nm when dark adapted.
                Rods adapt to low light conditions by becoming more sensitive.
                Peak frequency response shifts to 498 nm.
'''

def _wavelength_to_rgb(wavelength, gamma=0.8):
    """
    # reference: "Wavelength to RGB in Python." Noah.org, . 20 Sep 2014, 06:38 UTC. 1 Apr 2021, 19:06 <https://www.noah.org/mediawiki-1.34.2/index.php?title=Wavelength_to_RGB_in_Python&oldid=7665>.
    # http://www.noah.org/wiki/Wavelength_to_RGB_in_Python

    This converts a given wavelength of light to an approximate RGB color value. The wavelength must be given in nanometers in the range from 380 nm through 750 nm
    (789 THz through 400 THz).

    Based on code by Dan Bruton
    http://www.physics.sfasu.edu/astro/color/spectra.html

    Inputs:
    wavelength: wavelength of a visible wavelength in nanometers range: (380, 750)
    gamma: (optional) gamma correction. default value = 0.8

    Returns
    :param wavelength: float
    :param gamma: (optional) float
    :return: tuple
    """

    wavelength = float(wavelength)
    if 380 <= wavelength <= 440:
        attenuation = 0.3 + 0.7 * (wavelength - 380) / (440 - 380)
        R, G, B = ((-(wavelength - 440) / (440 - 380)) * attenuation) ** gamma, 0.0, (1.0 * attenuation) ** gamma
    elif 440 <= wavelength <= 490:
        R, G, B = 0.0, ((wavelength - 440) / (490 - 440)) ** gamma, 1.0
    elif 490 <= wavelength <= 510:
        R, G, B = 0.0, 1.0, (-(wavelength - 510) / (510 - 490)) ** gamma
    elif 510 <= wavelength <= 580:
        R, G, B = ((wavelength - 510) / (580 - 510)) ** gamma, 1.0, 0.0
    elif 580 <= wavelength <= 645:
        R, G, B = 1.0, (-(wavelength - 645) / (645 - 580)) ** gamma, 0.0
    elif 645 <= wavelength <= 750:
        attenuation = 0.3 + 0.7 * (750 - wavelength) / (750 - 645)
        R, G, B = (1.0 * attenuation) ** gamma, 0.0, 0.0
    # else:
    #     R, G, B = 0.0, 0.0, 0.0
    R *= 255
    G *= 255
    B *= 255
    return (int(R), int(G), int(B))

def _RGB_to_webcode(RGB_values):
    """
    RGB_value: a tuple of RGB values (0~255, uint8)
    """
    webcode = "#"
    for value in RGB_values:
        code_ = hex(value).replace('0x', '')
        code = code_.upper() if len(code_) > 1 else '0{}'.format(code_.upper())
        webcode += code
    return webcode

def hyper_histogram(array, mask, wvlengths=[480, 550, 670]):
    """This function calculates the histogram of selected wavelengths hyperspectral images
    The color of the histogram is based on the wavelength: if the wavelength is in the range of visible spectrum
    Inputs:
    array        = Hyperspectral data instance
    mask         = Binary mask made from selected contours
    wvlengths    = (optional) list of wavelengths to show histograms. default = [480,550,670]

    Returns:
    fig_hist = histogram figure

    :param array: plantcv.plantcv.classes.Spectral_data
    :param mask: numpy.array
    :param wvlengths: list
    :return: fig_hist: plotnine.ggplot.ggplot
    """

    # always sort desired wavelengths
    wvlengths.sort()

    wl_keys = array.wavelength_dict.keys()
    wls = np.array([float(i) for i in wl_keys])

    # spectral resolution of the spectral data
    diffs = [wls[i] - wls[i - 1] for i in range(1, len(wls))]
    spc_res = sum(diffs) / len(diffs)

    # check if the distance is greater than 2x the spectral resolution
    checks = [array.min_wavelength-wvlengths[i]>2*spc_res or wvlengths[i]-array.max_wavelength>2*spc_res for i in range(0,len(wvlengths))]

    if sum(checks) > 0:
        fatal_error(f"At least one band is too far from the available wavelength range: ({array.min_wavelength},{array.max_wavelength})!")

    # find indices of bands whose wavelengths are closest to desired ones
    match_ids = [_find_closest(wls, wv) for wv in wvlengths]

    # check if in the visible wavelengths range
    ids_vis = [idx for (idx,wv) in enumerate(wvlengths) if 380 <= wv <= 750]
    # invisible wavelengths
    ids_inv = [idx for (idx,wv) in enumerate(wvlengths) if wv < 380 or wv > 750]

    colors = [tuple(x) for x in color_palette(len(wvlengths))]
    if len(ids_vis) < len(wvlengths):
        print("Warning: at least one of the desired wavelengths is not in the visible spectrum range!")
        # if len(ids_inv) == len(wvlengths): # All wavelengths in invisible range
        #     colors = [tuple(x) for x in color_palette(len(wvlengths))]
    if len(ids_inv) < len(wvlengths):
        colors_vis = []
        colors_inv = colors
        for i in ids_vis:
            # colors_vis.append(_wavelength_to_rgb(wvlengths[i], gamma=0.8))
            color_vis = _wavelength_to_rgb(wvlengths[i], gamma=0.8)
            colors_vis.append(color_vis)
            if color_vis in colors:
                del colors_inv[colors.index(color_vis)]
        colors_inv = colors_inv[0:len(ids_inv)]
        colors = []
        j_vis, j_inv = 0,0
        for i in range(0,len(wvlengths)):
            if i in ids_vis:
                colors.append(colors_vis[j_vis])
                j_vis += 1
            else:
                colors.append(colors_inv[j_inv])
                j_inv += 1

    array_data = array.array_data

    # List of wavelengths recorded created from parsing the header file will be string, make list of floats
    histograms = dict()
    hist_dataset = pd.DataFrame(columns=['reflectance'])
    debug = params.debug
    params.debug = None
    color_codes = []
    for i_wv, (wv, color) in enumerate(zip(wvlengths,colors)):
        idx = match_ids[i_wv]
        code_c = _RGB_to_webcode(color)
        color_codes.append(code_c)
        _, hist_data = histogram(array_data[:, :, idx], mask=mask, bins=100, hist_data=True)
        histograms[wv] = {"label": wv, "graph_color": code_c, "reflectance": hist_data['pixel intensity'].tolist(),
                          "hist": hist_data['proportion of pixels (%)'].tolist()}
        if i_wv == 0:
            hist_dataset['reflectance'] = hist_data['pixel intensity'].tolist()
        hist_dataset[wv] = hist_data['proportion of pixels (%)'].tolist()

    # Make the histogram figure using plotnine
    df_hist = pd.melt(hist_dataset, id_vars=['reflectance'], value_vars=wvlengths,
                      var_name='Wavelength (' + array.wavelength_units + ')', value_name='proportion of pixels (%)')

    fig_hist = (ggplot(df_hist, aes(x='reflectance', y='proportion of pixels (%)',
                                    color='Wavelength (' + array.wavelength_units + ')'))
                + geom_line()
                + scale_color_manual(color_codes)
                )
    params.debug = debug
    _debug(fig_hist, filename=os.path.join(params.debug_outdir, str(params.device) + '_histogram.png'))

    return fig_hist

<commit_msg>Update hyper_histogram.py<commit_after># Help visualize histograms for hyperspectral images

import os
import cv2
import numpy as np
import pandas as pd
from plantcv.plantcv._debug import _debug
from plantcv.plantcv import fatal_error, params, outputs, color_palette
from plotnine import ggplot, aes, geom_line, geom_errorbar, scale_x_continuous, scale_color_manual
from plantcv.plantcv.visualize import histogram
from plantcv.plantcv.hyperspectral import _find_closest

'''
    == A few notes about color ==

    Color   Wavelength(nm) Frequency(THz)
    Red     620-750        484-400
    Orange  590-620        508-484
    Yellow  570-590        526-508
    Green   495-570        606-526
    Blue    450-495        668-606
    Violet  380-450        789-668

    f is frequency (cycles per second)
    l (lambda) is wavelength (meters per cycle)
    e is energy (Joules)
    h (Plank's constant) = 6.6260695729 x 10^-34 Joule*seconds
                         = 6.6260695729 x 10^-34 m^2*kg/seconds
    c = 299792458 meters per second
    f = c/l
    l = c/f
    e = h*f
    e = c*h/l

    List of peak frequency responses for each type of
    photoreceptor cell in the human eye:
        S cone: 437 nm
        M cone: 533 nm
        L cone: 564 nm
        rod:    550 nm in bright daylight, 498 nm when dark adapted.
                Rods adapt to low light conditions by becoming more sensitive.
                Peak frequency response shifts to 498 nm.
'''

def _wavelength_to_rgb(wavelength, gamma=0.8):
    """
    # reference: "Wavelength to RGB in Python." Noah.org, . 20 Sep 2014, 06:38 UTC. 1 Apr 2021, 19:06 <https://www.noah.org/mediawiki-1.34.2/index.php?title=Wavelength_to_RGB_in_Python&oldid=7665>.
    # http://www.noah.org/wiki/Wavelength_to_RGB_in_Python

    This converts a given wavelength of light to an approximate RGB color value. The wavelength must be given in nanometers in the range from 380 nm through 750 nm
    (789 THz through 400 THz).

    Based on code by Dan Bruton
    http://www.physics.sfasu.edu/astro/color/spectra.html

    Inputs:
    wavelength: wavelength of a visible wavelength in nanometers range: (380, 750)
    gamma: (optional) gamma correction. default value = 0.8

    Returns
    :param wavelength: float
    :param gamma: (optional) float
    :return: tuple
    """

    wavelength = float(wavelength)
    if 380 <= wavelength <= 440:
        attenuation = 0.3 + 0.7 * (wavelength - 380) / (440 - 380)
        R, G, B = ((-(wavelength - 440) / (440 - 380)) * attenuation) ** gamma, 0.0, (1.0 * attenuation) ** gamma
    elif 440 <= wavelength <= 490:
        R, G, B = 0.0, ((wavelength - 440) / (490 - 440)) ** gamma, 1.0
    elif 490 <= wavelength <= 510:
        R, G, B = 0.0, 1.0, (-(wavelength - 510) / (510 - 490)) ** gamma
    elif 510 <= wavelength <= 580:
        R, G, B = ((wavelength - 510) / (580 - 510)) ** gamma, 1.0, 0.0
    elif 580 <= wavelength <= 645:
        R, G, B = 1.0, (-(wavelength - 645) / (645 - 580)) ** gamma, 0.0
    elif 645 <= wavelength <= 750:
        attenuation = 0.3 + 0.7 * (750 - wavelength) / (750 - 645)
        R, G, B = (1.0 * attenuation) ** gamma, 0.0, 0.0
    # else:
    #     R, G, B = 0.0, 0.0, 0.0
    R *= 255
    G *= 255
    B *= 255
    return (int(R), int(G), int(B))

def _RGB_to_webcode(RGB_values):
    """
    RGB_value: a tuple of RGB values (0~255, uint8)
    """
    webcode = "#"
    for value in RGB_values:
        code_ = hex(value).replace('0x', '')
        code = code_.upper() if len(code_) > 1 else '0{}'.format(code_.upper())
        webcode += code
    return webcode

def hyper_histogram(array, mask, wvlengths=[480, 550, 670]):
    """This function calculates the histogram of selected wavelengths hyperspectral images
    The color of the histogram is based on the wavelength: if the wavelength is in the range of visible spectrum
    Inputs:
    array        = Hyperspectral data instance
    mask         = Binary mask made from selected contours
    wvlengths    = (optional) list of wavelengths to show histograms. default = [480,550,670]

    Returns:
    fig_hist = histogram figure

    :param array: plantcv.plantcv.classes.Spectral_data
    :param mask: numpy.array
    :param wvlengths: list
    :return: fig_hist: plotnine.ggplot.ggplot
    """

    # always sort desired wavelengths
    wvlengths.sort()

    wl_keys = array.wavelength_dict.keys()
    wls = np.array([float(i) for i in wl_keys])

    # spectral resolution of the spectral data
    diffs = [wls[i] - wls[i - 1] for i in range(1, len(wls))]
    spc_res = sum(diffs) / len(diffs)

    # check if the distance is greater than 2x the spectral resolution
    checks = [array.min_wavelength-wvlengths[i]>2*spc_res or wvlengths[i]-array.max_wavelength>2*spc_res for i in range(0,len(wvlengths))]

    if sum(checks) > 0:
        fatal_error(f"At least one band is too far from the available wavelength range: ({array.min_wavelength},{array.max_wavelength})!")

    # find indices of bands whose wavelengths are closest to desired ones
    match_ids = [_find_closest(wls, wv) for wv in wvlengths]

    # check if in the visible wavelengths range
    ids_vis = [idx for (idx,wv) in enumerate(wvlengths) if 380 <= wv <= 750]
    # invisible wavelengths
    ids_inv = [idx for (idx,wv) in enumerate(wvlengths) if wv < 380 or wv > 750]

    colors = [tuple(x) for x in color_palette(len(wvlengths))]
    if len(ids_vis) < len(wvlengths):
        print("Warning: at least one of the desired wavelengths is not in the visible spectrum range!")
        # if len(ids_inv) == len(wvlengths): # All wavelengths in invisible range
        #     colors = [tuple(x) for x in color_palette(len(wvlengths))]
    if len(ids_inv) < len(wvlengths):
        colors_vis = []
        colors_inv = colors
        for i in ids_vis:
            # colors_vis.append(_wavelength_to_rgb(wvlengths[i], gamma=0.8))
            color_vis = _wavelength_to_rgb(wvlengths[i], gamma=0.8)
            colors_vis.append(color_vis)
        # calculate the distances between every pair of (R,G,B) colors
        dists   = distance.cdist(colors_vis, colors_inv, 'euclidean')
        # exclude those colors "too close" to visible colors
        exclude = np.argmin(dists, axis=1)
        colors_inv = [c for (i,c) in enumerate(colors_inv) if i not in exclude]
        colors = []
        j_vis, j_inv = 0,0
        for i in range(0,len(wvlengths)):
            if i in ids_vis:
                colors.append(colors_vis[j_vis])
                j_vis += 1
            else:
                colors.append(colors_inv[j_inv])
                j_inv += 1

    array_data = array.array_data

    # List of wavelengths recorded created from parsing the header file will be string, make list of floats
    histograms = dict()
    hist_dataset = pd.DataFrame(columns=['reflectance'])
    debug = params.debug
    params.debug = None
    color_codes = []
    for i_wv, (wv, color) in enumerate(zip(wvlengths,colors)):
        idx = match_ids[i_wv]
        code_c = _RGB_to_webcode(color)
        color_codes.append(code_c)
        _, hist_data = histogram(array_data[:, :, idx], mask=mask, bins=100, hist_data=True)
        histograms[wv] = {"label": wv, "graph_color": code_c, "reflectance": hist_data['pixel intensity'].tolist(),
                          "hist": hist_data['proportion of pixels (%)'].tolist()}
        if i_wv == 0:
            hist_dataset['reflectance'] = hist_data['pixel intensity'].tolist()
        hist_dataset[wv] = hist_data['proportion of pixels (%)'].tolist()

    # Make the histogram figure using plotnine
    df_hist = pd.melt(hist_dataset, id_vars=['reflectance'], value_vars=wvlengths,
                      var_name='Wavelength (' + array.wavelength_units + ')', value_name='proportion of pixels (%)')

    fig_hist = (ggplot(df_hist, aes(x='reflectance', y='proportion of pixels (%)',
                                    color='Wavelength (' + array.wavelength_units + ')'))
                + geom_line()
                + scale_color_manual(color_codes)
                )
    params.debug = debug
    _debug(fig_hist, filename=os.path.join(params.debug_outdir, str(params.device) + '_histogram.png'))

    return fig_hist

<|endoftext|>


<commit_before>from datetime import datetime

from sqlalchemy import MetaData, Column, ForeignKey, Table
from sqlalchemy import DateTime, Integer, Unicode, UnicodeText

metadata = MetaData()

old_revision_table = Table('revision', metadata,
    Column('id', Integer, primary_key=True),
    Column('create_time', DateTime, default=datetime.utcnow),
    Column('text', UnicodeText(), nullable=False),
    Column('sentiment', Integer, default=0),
    Column('user_id', Integer, ForeignKey('user.id'), nullable=False),
    Column('comment_id', Integer, ForeignKey('comment.id'), nullable=False),
    Column('title', Unicode(255), nullable=True)
    )

def upgrade(migrate_engine):
    metadata.bind = migrate_engine
    revisions_table = Table('revision', metadata, autoload=True)

    q = migrate_engine.execute(revisions_table.select())
    for (id, _, text, _, _, _, title) in q:
        title = title and title.strip() or ''
        if len(title) < 5:
            continue
        if title.startswith('Re: '):
            continue
        new_text = ('**%(title)s**\n'
                    '\n'
                    '%(text)s') % {'title': title,
                                   'text': text}
        update_statement = revisions_table.update(
            revisions_table.c.id == id, {'text': new_text})
        migrate_engine.execute(update_statement)

    revisions_table.c.title.drop()
    raise Exception('ksjdfkl')

def downgrade(migrate_engine):
    raise NotImplementedError()
<commit_msg>Remove silly exception inserted for testing<commit_after>from datetime import datetime

from sqlalchemy import MetaData, Column, ForeignKey, Table
from sqlalchemy import DateTime, Integer, Unicode, UnicodeText

metadata = MetaData()

old_revision_table = Table('revision', metadata,
    Column('id', Integer, primary_key=True),
    Column('create_time', DateTime, default=datetime.utcnow),
    Column('text', UnicodeText(), nullable=False),
    Column('sentiment', Integer, default=0),
    Column('user_id', Integer, ForeignKey('user.id'), nullable=False),
    Column('comment_id', Integer, ForeignKey('comment.id'), nullable=False),
    Column('title', Unicode(255), nullable=True)
    )

def upgrade(migrate_engine):
    metadata.bind = migrate_engine
    revisions_table = Table('revision', metadata, autoload=True)

    q = migrate_engine.execute(revisions_table.select())
    for (id, _, text, _, _, _, title) in q:
        title = title and title.strip() or ''
        if len(title) < 5:
            continue
        if title.startswith('Re: '):
            continue
        new_text = ('**%(title)s**\n'
                    '\n'
                    '%(text)s') % {'title': title,
                                   'text': text}
        update_statement = revisions_table.update(
            revisions_table.c.id == id, {'text': new_text})
        migrate_engine.execute(update_statement)

    revisions_table.c.title.drop()

def downgrade(migrate_engine):
    raise NotImplementedError()
<|endoftext|>


<commit_before>import urwid
from wikicurses import formats
from wikicurses import settings
from wikicurses.wiki import Wiki
from wikicurses.htmlparse import parseDisambig

def tabComplete(text, matches):
    if not matches:
        return text
    matches = sorted(matches, key=len)
    if matches[0] == text and len(matches)>1:
        match = matches[1]
    else:
        match = matches[0]
    return match

class SearchBox(urwid.Edit):
    def keypress(self, size, key):
        if key == 'enter':
            loop.widget = mainwidget
            setContent(settings.wiki.search(self.edit_text or 'Main page'))
        elif key == 'tab':
            matches = settings.wiki.search_sugestions(self.edit_text)
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        else:
            return super().keypress(size, key)

class SelectorBox(urwid.ListBox):
    def __init__(self):
        def selectButton(radio_button, new_state, parameter):
            if new_state:
                loop.widget = mainwidget
                self._select(parameter)

        super().__init__(urwid.SimpleFocusListWalker([]))
        for i, (name, selected, parameter) in enumerate(self._items()):
            urwid.RadioButton(self.body, name, selected, selectButton, parameter)
            if selected:
                self.set_focus(i)

class Results(SelectorBox):
    def __init__(self, results):
        self.results = results
        super().__init__()

    def _items(self):
        return ((i, False, i) for i in self.results)

    def _select(self, title):
        setContent(settings.wiki.search(title))

class Toc(SelectorBox):
    def _items(self):
        return ((name, widgets.focus>=ind, ind) for name, ind in widgetnames)

    def _select(self, index):
       widgets.set_focus(index)

class Bmarks(SelectorBox):
    def _items(self):
        self.deleted = []
        return ((i, False, i) for i in settings.bmarks)

    def _select(self, name):
        setContent(settings.wiki.search(name))

    def keypress(self, size, key):
        #Undo Delete
        if key == 'u' and self.deleted:
            index, item = self.deleted.pop()
            settings.bmarks.add(item.label)
            self.body.insert(index, item)
            self.set_focus(index)
        elif key in ('meta [', 'x') and self.focus:
            settings.bmarks.discard(self.focus.label)
            self.deleted.append((self.focus_position, self.focus))
            self.body.remove(self.focus)
        else:
            return super().keypress(size, key)

class Wikis(SelectorBox):
    def _items(self):
        wikis = (i for i in settings.conf if i not in ('general', 'DEFAULT'))
        for name, url in settings.wikis().items():
            yield name, settings.wiki.siteurl == url, name

    def _select(self, name):
        settings.openWiki(name)
        setContent(settings.wiki.search('Main page'))

class Feeds(SelectorBox):
    def _items(self):
        return ((i, False, i) for i in settings.wiki.list_featured_feeds())

    def _select(self, feed):
        setContent(settings.wiki.get_featured_feed(feed))

class Disambig(urwid.ListBox):
    def __init__(self, html):
        def selectButton(radio_button, new_state, title):
            if new_state:
                loop.widget = mainwidget
                setContent(settings.wiki.search(title))

        sections = parseDisambig(html)
        super().__init__(urwid.SimpleFocusListWalker([]))
        buttons = []

        for title, items in sections.items():
            if title:
                self.body.append(urwid.Text(['\n', ('h', title)], align='center'))
            for name, text in items:
                if name:
                    button = urwid.RadioButton(buttons, text, False,
                            selectButton, name)
                    self.body.append(button)
                else:
                    self.body.append(urwid.Text(text))

class Ex(urwid.Edit):
    def keypress(self, size, key):
        if key == 'esc' or (key == 'backspace' and not self.edit_text):
            self.exitexmode()
        elif key == 'tab':
            matches = [i for i in cmds if i.startswith(self.edit_text)]
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        elif key == 'enter':
            words = self.edit_text.split(' ')
            self.exitexmode()
            processCmd(*words)
        else:
            return super().keypress(size, key)

    def exitexmode(self):
        self.set_caption('')
        self.set_edit_text('')
        mainwidget.set_focus('body')

    def enterexmode(self):
        mainwidget.set_focus('footer')
        self.set_caption(':')

cmds = ('quit', 'bmark', 'bmarks', 'wikis', 'feeds', 'open', 'contents')
def processCmd(cmd, *args):
    if cmd in ('q', 'quit'):
        raise urwid.ExitMainLoop
    if cmd == 'bmarks':
        openOverlay(Bmarks(), "Bookmarks")
    elif cmd == 'bmark':
        settings.bmarks.add(header.text)
        notify("Bookmark Added")
    elif cmd == 'wikis':
        openOverlay(Wikis(), "Wikis")
    elif cmd == 'feeds':
        openOverlay(Feeds(), "Feeds")
    elif cmd == 'contents':
        openOverlay(Toc(), "Table of Contents")
    elif cmd == 'open':
        if args:
            setContent(settings.wiki.search(' '.join(args)))
        else:
            openOverlay(urwid.ListBox([SearchBox()]), "Search", height=3)
    elif cmd:
        notify(cmd + ': Unknown Command')

def notify(text):
    mainwidget.footer = urwid.Text(text)

def openOverlay(widget, title, height=('relative', 50), width=('relative', 50)):
    box = urwid.LineBox(widget, title)
    overlay = urwid.Overlay(box, mainwidget, 'center', width, 'middle', height)
    loop.widget = overlay

def keymapper(input):
    #TODO: Implement gg and G

    cmdmap = settings.conf['keymap']
    if input == ':':
        mainwidget.footer.enterexmode()
    elif input in cmdmap and cmdmap[input]:
        processCmd(cmdmap[input])
    elif input == 'esc':
        loop.widget = mainwidget
    else:
       return False
    return True

def input_filter(keys, raw):
    if not isinstance(mainwidget.footer, urwid.Edit):
        mainwidget.footer = Ex()
    return keys

def setContent(page):
    if not page.exists:
        results = settings.wiki.search_sugestions(page.title)
        if results:
            openOverlay(Results(results), 'Results')
            return
    elif 'disambiguation' in page.properties:
        openOverlay(Disambig(page.result['text']['*']), 'Disambiguation')
        return

    widgets.clear()
    widgetnames.clear()
    header.set_text(page.title)
    for title, content in page.content.items():
        if title:
            h2 = urwid.Text([('h2', title), '\n'], align="center")
            widgets.append(h2)
            widgetnames.append((title, widgets.index(h2)))
        else:
            widgetnames.append((page.title, 0))
        widgets.append(urwid.Text(content))

palette = [('h1', 'bold', 'dark blue'),
           ('h2', 'underline', ''),
           ('h', 'underline', '')]

#(ITALIC, 'italic') does not work. No italics option?
outputfmt = (('b', 'bold'), ('blockquote', 'dark gray'))
for x in range(1, sum(formats) + 1):
    fmt = ','.join(j for i, j in outputfmt if x&formats[i])
    palette.append((x, fmt, ''))

widgets = urwid.SimpleFocusListWalker([])
widgetnames = []
pager = urwid.ListBox(widgets)

header = urwid.Text('Wikicurses', align='center')
mainwidget = urwid.Frame(pager, urwid.AttrMap(header, 'h1'), Ex())

urwid.command_map['k'] = 'cursor up'
urwid.command_map['j'] = 'cursor down'
urwid.command_map['ctrl b'] = 'cursor page up'
urwid.command_map['ctrl f'] = 'cursor page down'

loop = urwid.MainLoop(mainwidget, palette=palette, handle_mouse=False,
                     unhandled_input=keymapper, input_filter=input_filter)
<commit_msg>Make overlay name a parameter of the class used<commit_after>import urwid
from wikicurses import formats
from wikicurses import settings
from wikicurses.wiki import Wiki
from wikicurses.htmlparse import parseDisambig

def tabComplete(text, matches):
    if not matches:
        return text
    matches = sorted(matches, key=len)
    if matches[0] == text and len(matches)>1:
        match = matches[1]
    else:
        match = matches[0]
    return match

class SearchBox(urwid.Edit):
    title = "Search"
    def keypress(self, size, key):
        if key == 'enter':
            loop.widget = mainwidget
            setContent(settings.wiki.search(self.edit_text or 'Main page'))
        elif key == 'tab':
            matches = settings.wiki.search_sugestions(self.edit_text)
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        else:
            return super().keypress(size, key)

class SelectorBox(urwid.ListBox):
    def __init__(self):
        def selectButton(radio_button, new_state, parameter):
            if new_state:
                loop.widget = mainwidget
                self._select(parameter)

        super().__init__(urwid.SimpleFocusListWalker([]))
        for i, (name, selected, parameter) in enumerate(self._items()):
            urwid.RadioButton(self.body, name, selected, selectButton, parameter)
            if selected:
                self.set_focus(i)

class Results(SelectorBox):
    title = "Results"
    def __init__(self, results):
        self.results = results
        super().__init__()

    def _items(self):
        return ((i, False, i) for i in self.results)

    def _select(self, title):
        setContent(settings.wiki.search(title))

class Toc(SelectorBox):
    title = "Table of Contents"
    def _items(self):
        return ((name, widgets.focus>=ind, ind) for name, ind in widgetnames)

    def _select(self, index):
       widgets.set_focus(index)

class Bmarks(SelectorBox):
    title = "Bookmarks"
    def _items(self):
        self.deleted = []
        return ((i, False, i) for i in settings.bmarks)

    def _select(self, name):
        setContent(settings.wiki.search(name))

    def keypress(self, size, key):
        #Undo Delete
        if key == 'u' and self.deleted:
            index, item = self.deleted.pop()
            settings.bmarks.add(item.label)
            self.body.insert(index, item)
            self.set_focus(index)
        elif key in ('meta [', 'x') and self.focus:
            settings.bmarks.discard(self.focus.label)
            self.deleted.append((self.focus_position, self.focus))
            self.body.remove(self.focus)
        else:
            return super().keypress(size, key)

class Wikis(SelectorBox):
    title = "Wikis"
    def _items(self):
        wikis = (i for i in settings.conf if i not in ('general', 'DEFAULT'))
        for name, url in settings.wikis().items():
            yield name, settings.wiki.siteurl == url, name

    def _select(self, name):
        settings.openWiki(name)
        setContent(settings.wiki.search('Main page'))

class Feeds(SelectorBox):
    title = "Feeds"
    def _items(self):
        return ((i, False, i) for i in settings.wiki.list_featured_feeds())

    def _select(self, feed):
        setContent(settings.wiki.get_featured_feed(feed))

class Disambig(urwid.ListBox):
    title = "Disambiguation"
    def __init__(self, html):
        def selectButton(radio_button, new_state, title):
            if new_state:
                loop.widget = mainwidget
                setContent(settings.wiki.search(title))

        sections = parseDisambig(html)
        super().__init__(urwid.SimpleFocusListWalker([]))
        buttons = []

        for title, items in sections.items():
            if title:
                self.body.append(urwid.Text(['\n', ('h', title)], align='center'))
            for name, text in items:
                if name:
                    button = urwid.RadioButton(buttons, text, False,
                            selectButton, name)
                    self.body.append(button)
                else:
                    self.body.append(urwid.Text(text))

class Ex(urwid.Edit):
    def keypress(self, size, key):
        if key == 'esc' or (key == 'backspace' and not self.edit_text):
            self.exitexmode()
        elif key == 'tab':
            matches = [i for i in cmds if i.startswith(self.edit_text)]
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        elif key == 'enter':
            words = self.edit_text.split(' ')
            self.exitexmode()
            processCmd(*words)
        else:
            return super().keypress(size, key)

    def exitexmode(self):
        self.set_caption('')
        self.set_edit_text('')
        mainwidget.set_focus('body')

    def enterexmode(self):
        mainwidget.set_focus('footer')
        self.set_caption(':')

cmds = ('quit', 'bmark', 'bmarks', 'wikis', 'feeds', 'open', 'contents')
def processCmd(cmd, *args):
    if cmd in ('q', 'quit'):
        raise urwid.ExitMainLoop
    if cmd == 'bmarks':
        openOverlay(Bmarks())
    elif cmd == 'bmark':
        settings.bmarks.add(header.text)
        notify("Bookmark Added")
    elif cmd == 'wikis':
        openOverlay(Wikis())
    elif cmd == 'feeds':
        openOverlay(Feeds())
    elif cmd == 'contents':
        openOverlay(Toc())
    elif cmd == 'open':
        if args:
            setContent(settings.wiki.search(' '.join(args)))
        else:
            openOverlay(urwid.ListBox([SearchBox()]), "Search", height=3)
    elif cmd:
        notify(cmd + ': Unknown Command')

def notify(text):
    mainwidget.footer = urwid.Text(text)

def openOverlay(widget, title=None, height=('relative', 50), width=('relative', 50)):
    box = urwid.LineBox(widget, title or widget.title)
    overlay = urwid.Overlay(box, mainwidget, 'center', width, 'middle', height)
    loop.widget = overlay

def keymapper(input):
    #TODO: Implement gg and G

    cmdmap = settings.conf['keymap']
    if input == ':':
        mainwidget.footer.enterexmode()
    elif input in cmdmap and cmdmap[input]:
        processCmd(cmdmap[input])
    elif input == 'esc':
        loop.widget = mainwidget
    else:
       return False
    return True

def input_filter(keys, raw):
    if not isinstance(mainwidget.footer, urwid.Edit):
        mainwidget.footer = Ex()
    return keys

def setContent(page):
    if not page.exists:
        results = settings.wiki.search_sugestions(page.title)
        if results:
            openOverlay(Results(results))
            return
    elif 'disambiguation' in page.properties:
        openOverlay(Disambig(page.result['text']['*']))
        return

    widgets.clear()
    widgetnames.clear()
    header.set_text(page.title)
    for title, content in page.content.items():
        if title:
            h2 = urwid.Text([('h2', title), '\n'], align="center")
            widgets.append(h2)
            widgetnames.append((title, widgets.index(h2)))
        else:
            widgetnames.append((page.title, 0))
        widgets.append(urwid.Text(content))

palette = [('h1', 'bold', 'dark blue'),
           ('h2', 'underline', ''),
           ('h', 'underline', '')]

#(ITALIC, 'italic') does not work. No italics option?
outputfmt = (('b', 'bold'), ('blockquote', 'dark gray'))
for x in range(1, sum(formats) + 1):
    fmt = ','.join(j for i, j in outputfmt if x&formats[i])
    palette.append((x, fmt, ''))

widgets = urwid.SimpleFocusListWalker([])
widgetnames = []
pager = urwid.ListBox(widgets)

header = urwid.Text('Wikicurses', align='center')
mainwidget = urwid.Frame(pager, urwid.AttrMap(header, 'h1'), Ex())

urwid.command_map['k'] = 'cursor up'
urwid.command_map['j'] = 'cursor down'
urwid.command_map['ctrl b'] = 'cursor page up'
urwid.command_map['ctrl f'] = 'cursor page down'

loop = urwid.MainLoop(mainwidget, palette=palette, handle_mouse=False,
                     unhandled_input=keymapper, input_filter=input_filter)
<|endoftext|>


<commit_before>import urwid
from wikicurses import formats
from wikicurses import settings
from wikicurses.wiki import Wiki
from wikicurses.htmlparse import parseDisambig

def tabComplete(text, matches):
    if not matches:
        return text
    matches = sorted(matches, key=len)
    if matches[0] == text and len(matches)>1:
        match = matches[1]
    else:
        match = matches[0]
    return match

class SearchBox(urwid.Edit):
    title = "Search"
    def keypress(self, size, key):
        if key == 'enter':
            loop.widget = mainwidget
            setContent(settings.wiki.search(self.edit_text or 'Main page'))
        elif key == 'tab':
            matches = settings.wiki.search_sugestions(self.edit_text)
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        else:
            return super().keypress(size, key)

class SelectorBox(urwid.ListBox):
    def __init__(self):
        def selectButton(radio_button, new_state, parameter):
            if new_state:
                loop.widget = mainwidget
                self._select(parameter)

        super().__init__(urwid.SimpleFocusListWalker([]))
        for i, (name, selected, parameter) in enumerate(self._items()):
            urwid.RadioButton(self.body, name, selected, selectButton, parameter)
            if selected:
                self.set_focus(i)

class Results(SelectorBox):
    title = "Results"
    def __init__(self, results):
        self.results = results
        super().__init__()

    def _items(self):
        return ((i, False, i) for i in self.results)

    def _select(self, title):
        setContent(settings.wiki.search(title))

class Toc(SelectorBox):
    title = "Table of Contents"
    def _items(self):
        return ((name, widgets.focus>=ind, ind) for name, ind in widgetnames)

    def _select(self, index):
       widgets.set_focus(index)

class Bmarks(SelectorBox):
    title = "Bookmarks"
    def _items(self):
        self.deleted = []
        return ((i, False, i) for i in settings.bmarks)

    def _select(self, name):
        setContent(settings.wiki.search(name))

    def keypress(self, size, key):
        #Undo Delete
        if key == 'u' and self.deleted:
            index, item = self.deleted.pop()
            settings.bmarks.add(item.label)
            self.body.insert(index, item)
            self.set_focus(index)
        elif key in ('meta [', 'x') and self.focus:
            settings.bmarks.discard(self.focus.label)
            self.deleted.append((self.focus_position, self.focus))
            self.body.remove(self.focus)
        else:
            return super().keypress(size, key)

class Wikis(SelectorBox):
    title = "Wikis"
    def _items(self):
        for name, url in settings.wikis().items():
            yield name, settings.wiki.siteurl == url, name

    def _select(self, name):
        settings.openWiki(name)
        setContent(settings.wiki.search('Main page'))

class Feeds(SelectorBox):
    title = "Feeds"
    def _items(self):
        return ((i, False, i) for i in settings.wiki.list_featured_feeds())

    def _select(self, feed):
        setContent(settings.wiki.get_featured_feed(feed))

class Disambig(urwid.ListBox):
    title = "Disambiguation"
    def __init__(self, html):
        def selectButton(radio_button, new_state, title):
            if new_state:
                loop.widget = mainwidget
                setContent(settings.wiki.search(title))

        sections = parseDisambig(html)
        super().__init__(urwid.SimpleFocusListWalker([]))
        buttons = []

        for title, items in sections.items():
            if title:
                self.body.append(urwid.Text(['\n', ('h', title)], align='center'))
            for name, text in items:
                if name:
                    button = urwid.RadioButton(buttons, text, False,
                            selectButton, name)
                    self.body.append(button)
                else:
                    self.body.append(urwid.Text(text))

class Ex(urwid.Edit):
    def keypress(self, size, key):
        if key == 'esc' or (key == 'backspace' and not self.edit_text):
            self.exitexmode()
        elif key == 'tab':
            matches = [i for i in cmds if i.startswith(self.edit_text)]
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        elif key == 'enter':
            words = self.edit_text.split(' ')
            self.exitexmode()
            processCmd(*words)
        else:
            return super().keypress(size, key)

    def exitexmode(self):
        self.set_caption('')
        self.set_edit_text('')
        mainwidget.set_focus('body')

    def enterexmode(self):
        mainwidget.set_focus('footer')
        self.set_caption(':')

cmds = ('quit', 'bmark', 'bmarks', 'wikis', 'feeds', 'open', 'contents')
def processCmd(cmd, *args):
    if cmd in ('q', 'quit'):
        raise urwid.ExitMainLoop
    elif cmd == 'bmark':
        settings.bmarks.add(header.text)
        notify("Bookmark Added")
    elif cmd in ('bmarks' ,'wikis', 'feeds', 'contents'):
        openOverlay({'bmarks':Bmarks,
                     'wikis':Wikis,
                     'feeds':Feeds,
                     'contents':Toc}[cmd]())
    elif cmd == 'open':
        if args:
            setContent(settings.wiki.search(' '.join(args)))
        else:
            openOverlay(urwid.ListBox([SearchBox()]), "Search", height=3)
    elif cmd:
        notify(cmd + ': Unknown Command')

def notify(text):
    mainwidget.footer = urwid.Text(text)

def openOverlay(widget, title=None, height=('relative', 50), width=('relative', 50)):
    box = urwid.LineBox(widget, title or widget.title)
    overlay = urwid.Overlay(box, mainwidget, 'center', width, 'middle', height)
    loop.widget = overlay

def keymapper(input):
    #TODO: Implement gg and G

    cmdmap = settings.conf['keymap']
    if input == ':':
        mainwidget.footer.enterexmode()
    elif input in cmdmap and cmdmap[input]:
        processCmd(cmdmap[input])
    elif input == 'esc':
        loop.widget = mainwidget
    else:
       return False
    return True

def input_filter(keys, raw):
    if not isinstance(mainwidget.footer, urwid.Edit):
        mainwidget.footer = Ex()
    return keys

def setContent(page):
    if not page.exists:
        results = settings.wiki.search_sugestions(page.title)
        if results:
            openOverlay(Results(results))
            return
    elif 'disambiguation' in page.properties:
        openOverlay(Disambig(page.result['text']['*']))
        return

    widgets.clear()
    widgetnames.clear()
    header.set_text(page.title)
    for title, content in page.content.items():
        if title:
            h2 = urwid.Text([('h2', title), '\n'], align="center")
            widgets.append(h2)
            widgetnames.append((title, widgets.index(h2)))
        else:
            widgetnames.append((page.title, 0))
        widgets.append(urwid.Text(content))

palette = [('h1', 'bold', 'dark blue'),
           ('h2', 'underline', ''),
           ('h', 'underline', '')]

#(ITALIC, 'italic') does not work. No italics option?
outputfmt = (('b', 'bold'), ('blockquote', 'dark gray'))
for x in range(1, sum(formats) + 1):
    fmt = ','.join(j for i, j in outputfmt if x&formats[i])
    palette.append((x, fmt, ''))

widgets = urwid.SimpleFocusListWalker([])
widgetnames = []
pager = urwid.ListBox(widgets)

header = urwid.Text('Wikicurses', align='center')
mainwidget = urwid.Frame(pager, urwid.AttrMap(header, 'h1'), Ex())

urwid.command_map['k'] = 'cursor up'
urwid.command_map['j'] = 'cursor down'
urwid.command_map['ctrl b'] = 'cursor page up'
urwid.command_map['ctrl f'] = 'cursor page down'

loop = urwid.MainLoop(mainwidget, palette=palette, handle_mouse=False,
                     unhandled_input=keymapper, input_filter=input_filter)
<commit_msg>Add logic to SelectorBox to simplify subclasses<commit_after>import urwid
from wikicurses import formats
from wikicurses import settings
from wikicurses.wiki import Wiki
from wikicurses.htmlparse import parseDisambig

def tabComplete(text, matches):
    if not matches:
        return text
    matches = sorted(matches, key=len)
    if matches[0] == text and len(matches)>1:
        match = matches[1]
    else:
        match = matches[0]
    return match

class SearchBox(urwid.Edit):
    title = "Search"
    def keypress(self, size, key):
        if key == 'enter':
            loop.widget = mainwidget
            setContent(settings.wiki.search(self.edit_text or 'Main page'))
        elif key == 'tab':
            matches = settings.wiki.search_sugestions(self.edit_text)
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        else:
            return super().keypress(size, key)

class SelectorBox(urwid.ListBox):
    def __init__(self):
        def selectButton(radio_button, new_state, parameter):
            if new_state:
                loop.widget = mainwidget
                self._select(parameter)

        super().__init__(urwid.SimpleFocusListWalker([]))
        for i, item in enumerate(self._items()):
            if isinstance(item, tuple):
                name, selected, parameter = item
            else:
                parameter = name = item
                selected = False
            urwid.RadioButton(self.body, name, selected, selectButton, parameter)
            if selected:
                self.set_focus(i)

class Results(SelectorBox):
    title = "Results"
    def __init__(self, results):
        self.results = results
        super().__init__()

    def _items(self):
        return self.results

    def _select(self, title):
        setContent(settings.wiki.search(title))

class Toc(SelectorBox):
    title = "Table of Contents"
    def _items(self):
        return ((name, widgets.focus>=ind, ind) for name, ind in widgetnames)

    def _select(self, index):
       widgets.set_focus(index)

class Bmarks(SelectorBox):
    title = "Bookmarks"
    def _items(self):
        self.deleted = []
        return settings.bmarks

    def _select(self, name):
        setContent(settings.wiki.search(name))

    def keypress(self, size, key):
        #Undo Delete
        if key == 'u' and self.deleted:
            index, item = self.deleted.pop()
            settings.bmarks.add(item.label)
            self.body.insert(index, item)
            self.set_focus(index)
        elif key in ('meta [', 'x') and self.focus:
            settings.bmarks.discard(self.focus.label)
            self.deleted.append((self.focus_position, self.focus))
            self.body.remove(self.focus)
        else:
            return super().keypress(size, key)

class Wikis(SelectorBox):
    title = "Wikis"
    def _items(self):
        for name, url in settings.wikis().items():
            yield name, settings.wiki.siteurl == url, name

    def _select(self, name):
        settings.openWiki(name)
        setContent(settings.wiki.search('Main page'))

class Feeds(SelectorBox):
    title = "Feeds"
    def _items(self):
        return settings.wiki.list_featured_feeds()

    def _select(self, feed):
        setContent(settings.wiki.get_featured_feed(feed))

class Disambig(urwid.ListBox):
    title = "Disambiguation"
    def __init__(self, html):
        def selectButton(radio_button, new_state, title):
            if new_state:
                loop.widget = mainwidget
                setContent(settings.wiki.search(title))

        sections = parseDisambig(html)
        super().__init__(urwid.SimpleFocusListWalker([]))
        buttons = []

        for title, items in sections.items():
            if title:
                self.body.append(urwid.Text(['\n', ('h', title)], align='center'))
            for name, text in items:
                if name:
                    button = urwid.RadioButton(buttons, text, False,
                            selectButton, name)
                    self.body.append(button)
                else:
                    self.body.append(urwid.Text(text))

class Ex(urwid.Edit):
    def keypress(self, size, key):
        if key == 'esc' or (key == 'backspace' and not self.edit_text):
            self.exitexmode()
        elif key == 'tab':
            matches = [i for i in cmds if i.startswith(self.edit_text)]
            match = tabComplete(self.edit_text, matches)
            self.set_edit_text(match)
            self.edit_pos = len(match)
        elif key == 'enter':
            words = self.edit_text.split(' ')
            self.exitexmode()
            processCmd(*words)
        else:
            return super().keypress(size, key)

    def exitexmode(self):
        self.set_caption('')
        self.set_edit_text('')
        mainwidget.set_focus('body')

    def enterexmode(self):
        mainwidget.set_focus('footer')
        self.set_caption(':')

cmds = ('quit', 'bmark', 'bmarks', 'wikis', 'feeds', 'open', 'contents')
def processCmd(cmd, *args):
    if cmd in ('q', 'quit'):
        raise urwid.ExitMainLoop
    elif cmd == 'bmark':
        settings.bmarks.add(header.text)
        notify("Bookmark Added")
    elif cmd in ('bmarks' ,'wikis', 'feeds', 'contents'):
        openOverlay({'bmarks':Bmarks,
                     'wikis':Wikis,
                     'feeds':Feeds,
                     'contents':Toc}[cmd]())
    elif cmd == 'open':
        if args:
            setContent(settings.wiki.search(' '.join(args)))
        else:
            openOverlay(urwid.ListBox([SearchBox()]), "Search", height=3)
    elif cmd:
        notify(cmd + ': Unknown Command')

def notify(text):
    mainwidget.footer = urwid.Text(text)

def openOverlay(widget, title=None, height=('relative', 50), width=('relative', 50)):
    box = urwid.LineBox(widget, title or widget.title)
    overlay = urwid.Overlay(box, mainwidget, 'center', width, 'middle', height)
    loop.widget = overlay

def keymapper(input):
    #TODO: Implement gg and G

    cmdmap = settings.conf['keymap']
    if input == ':':
        mainwidget.footer.enterexmode()
    elif input in cmdmap and cmdmap[input]:
        processCmd(cmdmap[input])
    elif input == 'esc':
        loop.widget = mainwidget
    else:
       return False
    return True

def input_filter(keys, raw):
    if not isinstance(mainwidget.footer, urwid.Edit):
        mainwidget.footer = Ex()
    return keys

def setContent(page):
    if not page.exists:
        results = settings.wiki.search_sugestions(page.title)
        if results:
            openOverlay(Results(results))
            return
    elif 'disambiguation' in page.properties:
        openOverlay(Disambig(page.result['text']['*']))
        return

    widgets.clear()
    widgetnames.clear()
    header.set_text(page.title)
    for title, content in page.content.items():
        if title:
            h2 = urwid.Text([('h2', title), '\n'], align="center")
            widgets.append(h2)
            widgetnames.append((title, widgets.index(h2)))
        else:
            widgetnames.append((page.title, 0))
        widgets.append(urwid.Text(content))

palette = [('h1', 'bold', 'dark blue'),
           ('h2', 'underline', ''),
           ('h', 'underline', '')]

#(ITALIC, 'italic') does not work. No italics option?
outputfmt = (('b', 'bold'), ('blockquote', 'dark gray'))
for x in range(1, sum(formats) + 1):
    fmt = ','.join(j for i, j in outputfmt if x&formats[i])
    palette.append((x, fmt, ''))

widgets = urwid.SimpleFocusListWalker([])
widgetnames = []
pager = urwid.ListBox(widgets)

header = urwid.Text('Wikicurses', align='center')
mainwidget = urwid.Frame(pager, urwid.AttrMap(header, 'h1'), Ex())

urwid.command_map['k'] = 'cursor up'
urwid.command_map['j'] = 'cursor down'
urwid.command_map['ctrl b'] = 'cursor page up'
urwid.command_map['ctrl f'] = 'cursor page down'

loop = urwid.MainLoop(mainwidget, palette=palette, handle_mouse=False,
                     unhandled_input=keymapper, input_filter=input_filter)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_hook.py

    Manage hooks
"""
import os
import re
import sys
import tempfile
import mimetypes
from glob import iglob
from importlib import import_module

from moulinette import m18n, msettings
from yunohost.utils.error import YunohostError
from moulinette.utils import log
from moulinette.utils.filesystem import read_json

HOOK_FOLDER = '/usr/share/yunohost/hooks/'
CUSTOM_HOOK_FOLDER = '/etc/yunohost/hooks.d/'

logger = log.getActionLogger('yunohost.hook')

def hook_add(app, file):
    """
    Store hook script to filsystem

    Keyword argument:
        app -- App to link with
        file -- Script to add (/path/priority-file)

    """
    path, filename = os.path.split(file)
    priority, action = _extract_filename_parts(filename)

    try:
        os.listdir(CUSTOM_HOOK_FOLDER + action)
    except OSError:
        os.makedirs(CUSTOM_HOOK_FOLDER + action)

    finalpath = CUSTOM_HOOK_FOLDER + action + '/' + priority + '-' + app
    os.system('cp %s %s' % (file, finalpath))
    os.system('chown -hR admin: %s' % HOOK_FOLDER)

    return {'hook': finalpath}

def hook_remove(app):
    """
    Remove hooks linked to a specific app

    Keyword argument:
        app -- Scripts related to app will be removed

    """
    try:
        for action in os.listdir(CUSTOM_HOOK_FOLDER):
            for script in os.listdir(CUSTOM_HOOK_FOLDER + action):
                if script.endswith(app):
                    os.remove(CUSTOM_HOOK_FOLDER + action + '/' + script)
    except OSError:
        pass

def hook_info(action, name):
    """
    Get information about a given hook

    Keyword argument:
        action -- Action name
        name -- Hook name

    """
    hooks = []
    priorities = set()

    # Search in custom folder first
    for h in iglob('{:s}{:s}/*-{:s}'.format(
            CUSTOM_HOOK_FOLDER, action, name)):
        priority, _ = _extract_filename_parts(os.path.basename(h))
        priorities.add(priority)
        hooks.append({
            'priority': priority,
            'path': h,
        })
    # Append non-overwritten system hooks
    for h in iglob('{:s}{:s}/*-{:s}'.format(
            HOOK_FOLDER, action, name)):
        priority, _ = _extract_filename_parts(os.path.basename(h))
        if priority not in priorities:
            hooks.append({
                'priority': priority,
                'path': h,
            })

    if not hooks:
        raise YunohostError('hook_name_unknown', name=name)
    return {
        'action': action,
        'name': name,
        'hooks': hooks,
    }

def hook_list(action, list_by='name', show_info=False):
    """
    List available hooks for an action

    Keyword argument:
        action -- Action name
        list_by -- Property to list hook by
        show_info -- Show hook information

    """
    result = {}

    # Process the property to list hook by
    if list_by == 'priority':
        if show_info:
            def _append_hook(d, priority, name, path):
                # Use the priority as key and a dict of hooks names
                # with their info as value
                value = {'path': path}
                try:
                    d[priority][name] = value
                except KeyError:
                    d[priority] = {name: value}
        else:
            def _append_hook(d, priority, name, path):
                # Use the priority as key and the name as value
                try:
                    d[priority].add(name)
                except KeyError:
                    d[priority] = set([name])
    elif list_by == 'name' or list_by == 'folder':
        if show_info:
            def _append_hook(d, priority, name, path):
                # Use the name as key and a list of hooks info - the
                # executed ones with this name - as value
                l = d.get(name, list())
                for h in l:
                    # Only one priority for the hook is accepted
                    if h['priority'] == priority:
                        # Custom hooks overwrite system ones and they
                        # are appended at the end - so overwite it
                        if h['path'] != path:
                            h['path'] = path
                        return
                l.append({'priority': priority, 'path': path})
                d[name] = l
        else:
            if list_by == 'name':
                result = set()

            def _append_hook(d, priority, name, path):
                # Add only the name
                d.add(name)
    else:
        raise YunohostError('hook_list_by_invalid')

    def _append_folder(d, folder):
        # Iterate over and add hook from a folder
        for f in os.listdir(folder + action):
            if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc"):
                continue
            path = '%s%s/%s' % (folder, action, f)
            priority, name = _extract_filename_parts(f)
            _append_hook(d, priority, name, path)

    try:
        # Append system hooks first
        if list_by == 'folder':
            result['system'] = dict() if show_info else set()
            _append_folder(result['system'], HOOK_FOLDER)
        else:
            _append_folder(result, HOOK_FOLDER)
    except OSError:
        pass

    try:
        # Append custom hooks
        if list_by == 'folder':
            result['custom'] = dict() if show_info else set()
            _append_folder(result['custom'], CUSTOM_HOOK_FOLDER)
        else:
            _append_folder(result, CUSTOM_HOOK_FOLDER)
    except OSError:
        pass

    return {'hooks': result}

def hook_callback(action, hooks=[], args=None, chdir=None,
                  env=None, pre_callback=None, post_callback=None):
    """
    Execute all scripts binded to an action

    Keyword argument:
        action -- Action name
        hooks -- List of hooks names to execute
        args -- Ordered list of arguments to pass to the scripts
        chdir -- The directory from where the scripts will be executed
        env -- Dictionnary of environment variables to export
        pre_callback -- An object to call before each script execution with
            (name, priority, path, args) as arguments and which must return
            the arguments to pass to the script
        post_callback -- An object to call after each script execution with
            (name, priority, path, succeed) as arguments

    """
    result = {}
    hooks_dict = {}

    # Retrieve hooks
    if not hooks:
        hooks_dict = hook_list(action, list_by='priority',
                               show_info=True)['hooks']
    else:
        hooks_names = hook_list(action, list_by='name',
                                show_info=True)['hooks']

        # Add similar hooks to the list
        # For example: Having a 16-postfix hook in the list will execute a
        # xx-postfix_dkim as well
        all_hooks = []
        for n in hooks:
            for key in hooks_names.keys():
                if key == n or key.startswith("%s_" % n) \
                        and key not in all_hooks:
                    all_hooks.append(key)

        # Iterate over given hooks names list
        for n in all_hooks:
            try:
                hl = hooks_names[n]
            except KeyError:
                raise YunohostError('hook_name_unknown', n)
            # Iterate over hooks with this name
            for h in hl:
                # Update hooks dict
                d = hooks_dict.get(h['priority'], dict())
                d.update({n: {'path': h['path']}})
                hooks_dict[h['priority']] = d
    if not hooks_dict:
        return result

    # Validate callbacks
    if not callable(pre_callback):
        def pre_callback(name, priority, path, args): return args
    if not callable(post_callback):
        def post_callback(name, priority, path, succeed): return None

    # Iterate over hooks and execute them
    for priority in sorted(hooks_dict):
        for name, info in iter(hooks_dict[priority].items()):
            state = 'succeed'
            path = info['path']
            try:
                hook_args = pre_callback(name=name, priority=priority,
                                         path=path, args=args)
                hook_return = hook_exec(path, args=hook_args, chdir=chdir, env=env,
                                        raise_on_error=True)[1]
            except YunohostError as e:
                state = 'failed'
                hook_return = {}
                logger.error(e.strerror, exc_info=1)
                post_callback(name=name, priority=priority, path=path,
                              succeed=False)
            else:
                post_callback(name=name, priority=priority, path=path,
                              succeed=True)
            if name not in result:
                result[name] = {}
            result[name][path] = {'state': state, 'stdreturn': hook_return}
    return result

def hook_exec(path, args=None, raise_on_error=False,
              chdir=None, env=None, return_format="json"):
    """
    Execute hook from a file with arguments

    Keyword argument:
        path -- Path of the script to execute
        args -- Ordered list of arguments to pass to the script
        raise_on_error -- Raise if the script returns a non-zero exit code
        chdir -- The directory from where the script will be executed
        env -- Dictionnary of environment variables to export
    """

    # Validate hook path
    if path[0] != '/':
        path = os.path.realpath(path)
    if not os.path.isfile(path):
        raise YunohostError('file_does_not_exist', path=path)

    def is_relevant_warning(msg):

        # Ignore empty warning messages...
        if not msg:
            return False

        # Some of these are shit sent from apt and we don't give a shit about
        # them because they ain't actual warnings >_>
        irrelevant_warnings = [
            r"invalid value for trace file descriptor",
            r"Creating config file .* with new version",
            r"Created symlink /etc/systemd",
            r"dpkg: warning: while removing .* not empty so not removed"
        ]
        return all(not re.search(w, msg) for w in irrelevant_warnings)

    # Define output loggers and call command
    loggers = (
        lambda l: logger.debug(l.rstrip() + "\r"),
        lambda l: logger.warning(l.rstrip()) if is_relevant_warning(l.rstrip()) else logger.debug(l.rstrip()),
        lambda l: logger.info(l.rstrip())
    )

    # Check the type of the hook (bash by default)
    # For now we support only python and bash hooks.
    hook_type = mimetypes.MimeTypes().guess_type(path)[0]
    if hook_type == 'text/x-python':
        returncode, returndata = _hook_exec_python(path, args, env, loggers)
    else:
        returncode, returndata = _hook_exec_bash(path, args, chdir, env, return_format, loggers)

    # Check and return process' return code
    if returncode is None:
        if raise_on_error:
            raise YunohostError('hook_exec_not_terminated', path=path)
        else:
            logger.error(m18n.n('hook_exec_not_terminated', path=path))
            return 1, {}
    elif raise_on_error and returncode != 0:
        raise YunohostError('hook_exec_failed', path=path)

    return returncode, returndata

def _hook_exec_bash(path, args, chdir, env, return_format, loggers):

    from moulinette.utils.process import call_async_output

    # Construct command variables
    cmd_args = ''
    if args and isinstance(args, list):
        # Concatenate escaped arguments
        cmd_args = ' '.join(shell_quote(s) for s in args)
    if not chdir:
        # use the script directory as current one
        chdir, cmd_script = os.path.split(path)
        cmd_script = './{0}'.format(cmd_script)
    else:
        cmd_script = path

    # Add Execution dir to environment var
    if env is None:
        env = {}
    env['YNH_CWD'] = chdir

    env['YNH_INTERFACE'] = msettings.get('interface')

    stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo")
    env['YNH_STDINFO'] = stdinfo

    stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn")
    with open(stdreturn, 'w') as f:
        f.write('')
    env['YNH_STDRETURN'] = stdreturn

    # use xtrace on fd 7 which is redirected to stdout
    env['BASH_XTRACEFD'] = 7
    cmd = '/bin/bash -x "{script}" {args} 7>&1'
    cmd = cmd.format(script=cmd_script, args=cmd_args)

    logger.debug("Executing command '%s'" % cmd)

    returncode = call_async_output(
        cmd, loggers, shell=True, cwd=chdir,
        stdinfo=stdinfo, env=env
    )

    raw_content = None
    try:
        with open(stdreturn, 'r') as f:
            raw_content = f.read()
        returncontent = {}

        if return_format == "json":
            if raw_content != '':
                try:
                    returncontent = read_json(stdreturn)
                except Exception as e:
                    raise YunohostError('hook_json_return_error',
                                        path=path, msg=str(e),
                                        raw_content=raw_content)

        elif return_format == "plain_dict":
            for line in raw_content.split("\n"):
                if "=" in line:
                    key, value = line.strip().split("=", 1)
                    returncontent[key] = value

        else:
            raise YunohostError("Expected value for return_format is either 'json' or 'plain_dict', got '%s'" % return_format)
    finally:
        stdreturndir = os.path.split(stdreturn)[0]
        os.remove(stdreturn)
        os.rmdir(stdreturndir)

    return returncode, returncontent

def _hook_exec_python(path, args, env, loggers):

    dir_ = os.path.dirname(path)
    name = os.path.splitext(os.path.basename(path))[0]

    if dir_ not in sys.path:
        sys.path = [dir_] + sys.path
    module = import_module(name)

    ret = module.main(args, env, loggers)
    # # Assert that the return is a (int, dict) tuple
    assert isinstance(ret, tuple) \
        and len(ret) == 2 \
        and isinstance(ret[0], int) \
        and isinstance(ret[1], dict), \
        "Module %s did not return a (int, dict) tuple !" % module
    return ret

def _extract_filename_parts(filename):
    """Extract hook parts from filename"""
    if '-' in filename:
        priority, action = filename.split('-', 1)
    else:
        priority = '50'
        action = filename

    # Remove extension if there's one
    action = os.path.splitext(action)[0]
    return priority, action

# Taken from Python 3 shlex module --------------------------------------------

_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.UNICODE).search

def shell_quote(s):
    """Return a shell-escaped version of the string *s*."""
    s = str(s)
    if not s:
        return "''"
    if _find_unsafe(s) is None:
        return s

    # use single quotes, and put single quotes into double quotes
    # the string $'b is then quoted as '$'"'"'b'
    return "'" + s.replace("'", "'\"'\"'") + "'"
<commit_msg>Also copy the script env<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_hook.py

    Manage hooks
"""
import os
import re
import sys
import tempfile
import mimetypes
from glob import iglob
from importlib import import_module

from moulinette import m18n, msettings
from yunohost.utils.error import YunohostError
from moulinette.utils import log
from moulinette.utils.filesystem import read_json

HOOK_FOLDER = '/usr/share/yunohost/hooks/'
CUSTOM_HOOK_FOLDER = '/etc/yunohost/hooks.d/'

logger = log.getActionLogger('yunohost.hook')

def hook_add(app, file):
    """
    Store hook script to filsystem

    Keyword argument:
        app -- App to link with
        file -- Script to add (/path/priority-file)

    """
    path, filename = os.path.split(file)
    priority, action = _extract_filename_parts(filename)

    try:
        os.listdir(CUSTOM_HOOK_FOLDER + action)
    except OSError:
        os.makedirs(CUSTOM_HOOK_FOLDER + action)

    finalpath = CUSTOM_HOOK_FOLDER + action + '/' + priority + '-' + app
    os.system('cp %s %s' % (file, finalpath))
    os.system('chown -hR admin: %s' % HOOK_FOLDER)

    return {'hook': finalpath}

def hook_remove(app):
    """
    Remove hooks linked to a specific app

    Keyword argument:
        app -- Scripts related to app will be removed

    """
    try:
        for action in os.listdir(CUSTOM_HOOK_FOLDER):
            for script in os.listdir(CUSTOM_HOOK_FOLDER + action):
                if script.endswith(app):
                    os.remove(CUSTOM_HOOK_FOLDER + action + '/' + script)
    except OSError:
        pass

def hook_info(action, name):
    """
    Get information about a given hook

    Keyword argument:
        action -- Action name
        name -- Hook name

    """
    hooks = []
    priorities = set()

    # Search in custom folder first
    for h in iglob('{:s}{:s}/*-{:s}'.format(
            CUSTOM_HOOK_FOLDER, action, name)):
        priority, _ = _extract_filename_parts(os.path.basename(h))
        priorities.add(priority)
        hooks.append({
            'priority': priority,
            'path': h,
        })
    # Append non-overwritten system hooks
    for h in iglob('{:s}{:s}/*-{:s}'.format(
            HOOK_FOLDER, action, name)):
        priority, _ = _extract_filename_parts(os.path.basename(h))
        if priority not in priorities:
            hooks.append({
                'priority': priority,
                'path': h,
            })

    if not hooks:
        raise YunohostError('hook_name_unknown', name=name)
    return {
        'action': action,
        'name': name,
        'hooks': hooks,
    }

def hook_list(action, list_by='name', show_info=False):
    """
    List available hooks for an action

    Keyword argument:
        action -- Action name
        list_by -- Property to list hook by
        show_info -- Show hook information

    """
    result = {}

    # Process the property to list hook by
    if list_by == 'priority':
        if show_info:
            def _append_hook(d, priority, name, path):
                # Use the priority as key and a dict of hooks names
                # with their info as value
                value = {'path': path}
                try:
                    d[priority][name] = value
                except KeyError:
                    d[priority] = {name: value}
        else:
            def _append_hook(d, priority, name, path):
                # Use the priority as key and the name as value
                try:
                    d[priority].add(name)
                except KeyError:
                    d[priority] = set([name])
    elif list_by == 'name' or list_by == 'folder':
        if show_info:
            def _append_hook(d, priority, name, path):
                # Use the name as key and a list of hooks info - the
                # executed ones with this name - as value
                l = d.get(name, list())
                for h in l:
                    # Only one priority for the hook is accepted
                    if h['priority'] == priority:
                        # Custom hooks overwrite system ones and they
                        # are appended at the end - so overwite it
                        if h['path'] != path:
                            h['path'] = path
                        return
                l.append({'priority': priority, 'path': path})
                d[name] = l
        else:
            if list_by == 'name':
                result = set()

            def _append_hook(d, priority, name, path):
                # Add only the name
                d.add(name)
    else:
        raise YunohostError('hook_list_by_invalid')

    def _append_folder(d, folder):
        # Iterate over and add hook from a folder
        for f in os.listdir(folder + action):
            if f[0] == '.' or f[-1] == '~' or f.endswith(".pyc"):
                continue
            path = '%s%s/%s' % (folder, action, f)
            priority, name = _extract_filename_parts(f)
            _append_hook(d, priority, name, path)

    try:
        # Append system hooks first
        if list_by == 'folder':
            result['system'] = dict() if show_info else set()
            _append_folder(result['system'], HOOK_FOLDER)
        else:
            _append_folder(result, HOOK_FOLDER)
    except OSError:
        pass

    try:
        # Append custom hooks
        if list_by == 'folder':
            result['custom'] = dict() if show_info else set()
            _append_folder(result['custom'], CUSTOM_HOOK_FOLDER)
        else:
            _append_folder(result, CUSTOM_HOOK_FOLDER)
    except OSError:
        pass

    return {'hooks': result}

def hook_callback(action, hooks=[], args=None, chdir=None,
                  env=None, pre_callback=None, post_callback=None):
    """
    Execute all scripts binded to an action

    Keyword argument:
        action -- Action name
        hooks -- List of hooks names to execute
        args -- Ordered list of arguments to pass to the scripts
        chdir -- The directory from where the scripts will be executed
        env -- Dictionnary of environment variables to export
        pre_callback -- An object to call before each script execution with
            (name, priority, path, args) as arguments and which must return
            the arguments to pass to the script
        post_callback -- An object to call after each script execution with
            (name, priority, path, succeed) as arguments

    """
    result = {}
    hooks_dict = {}

    # Retrieve hooks
    if not hooks:
        hooks_dict = hook_list(action, list_by='priority',
                               show_info=True)['hooks']
    else:
        hooks_names = hook_list(action, list_by='name',
                                show_info=True)['hooks']

        # Add similar hooks to the list
        # For example: Having a 16-postfix hook in the list will execute a
        # xx-postfix_dkim as well
        all_hooks = []
        for n in hooks:
            for key in hooks_names.keys():
                if key == n or key.startswith("%s_" % n) \
                        and key not in all_hooks:
                    all_hooks.append(key)

        # Iterate over given hooks names list
        for n in all_hooks:
            try:
                hl = hooks_names[n]
            except KeyError:
                raise YunohostError('hook_name_unknown', n)
            # Iterate over hooks with this name
            for h in hl:
                # Update hooks dict
                d = hooks_dict.get(h['priority'], dict())
                d.update({n: {'path': h['path']}})
                hooks_dict[h['priority']] = d
    if not hooks_dict:
        return result

    # Validate callbacks
    if not callable(pre_callback):
        def pre_callback(name, priority, path, args): return args
    if not callable(post_callback):
        def post_callback(name, priority, path, succeed): return None

    # Iterate over hooks and execute them
    for priority in sorted(hooks_dict):
        for name, info in iter(hooks_dict[priority].items()):
            state = 'succeed'
            path = info['path']
            try:
                hook_args = pre_callback(name=name, priority=priority,
                                         path=path, args=args)
                hook_return = hook_exec(path, args=hook_args, chdir=chdir, env=env,
                                        raise_on_error=True)[1]
            except YunohostError as e:
                state = 'failed'
                hook_return = {}
                logger.error(e.strerror, exc_info=1)
                post_callback(name=name, priority=priority, path=path,
                              succeed=False)
            else:
                post_callback(name=name, priority=priority, path=path,
                              succeed=True)
            if name not in result:
                result[name] = {}
            result[name][path] = {'state': state, 'stdreturn': hook_return}
    return result

def hook_exec(path, args=None, raise_on_error=False,
              chdir=None, env=None, return_format="json"):
    """
    Execute hook from a file with arguments

    Keyword argument:
        path -- Path of the script to execute
        args -- Ordered list of arguments to pass to the script
        raise_on_error -- Raise if the script returns a non-zero exit code
        chdir -- The directory from where the script will be executed
        env -- Dictionnary of environment variables to export
    """

    # Validate hook path
    if path[0] != '/':
        path = os.path.realpath(path)
    if not os.path.isfile(path):
        raise YunohostError('file_does_not_exist', path=path)

    def is_relevant_warning(msg):

        # Ignore empty warning messages...
        if not msg:
            return False

        # Some of these are shit sent from apt and we don't give a shit about
        # them because they ain't actual warnings >_>
        irrelevant_warnings = [
            r"invalid value for trace file descriptor",
            r"Creating config file .* with new version",
            r"Created symlink /etc/systemd",
            r"dpkg: warning: while removing .* not empty so not removed"
        ]
        return all(not re.search(w, msg) for w in irrelevant_warnings)

    # Define output loggers and call command
    loggers = (
        lambda l: logger.debug(l.rstrip() + "\r"),
        lambda l: logger.warning(l.rstrip()) if is_relevant_warning(l.rstrip()) else logger.debug(l.rstrip()),
        lambda l: logger.info(l.rstrip())
    )

    # Check the type of the hook (bash by default)
    # For now we support only python and bash hooks.
    hook_type = mimetypes.MimeTypes().guess_type(path)[0]
    if hook_type == 'text/x-python':
        returncode, returndata = _hook_exec_python(path, args, env, loggers)
    else:
        returncode, returndata = _hook_exec_bash(path, args, chdir, env, return_format, loggers)

    # Check and return process' return code
    if returncode is None:
        if raise_on_error:
            raise YunohostError('hook_exec_not_terminated', path=path)
        else:
            logger.error(m18n.n('hook_exec_not_terminated', path=path))
            return 1, {}
    elif raise_on_error and returncode != 0:
        raise YunohostError('hook_exec_failed', path=path)

    return returncode, returndata

def _hook_exec_bash(path, args, chdir, env, return_format, loggers):

    from moulinette.utils.process import call_async_output

    # Construct command variables
    cmd_args = ''
    if args and isinstance(args, list):
        # Concatenate escaped arguments
        cmd_args = ' '.join(shell_quote(s) for s in args)
    if not chdir:
        # use the script directory as current one
        chdir, cmd_script = os.path.split(path)
        cmd_script = './{0}'.format(cmd_script)
    else:
        cmd_script = path

    # Add Execution dir to environment var
    if env is None:
        env = {}
    env['YNH_CWD'] = chdir

    env['YNH_INTERFACE'] = msettings.get('interface')

    stdinfo = os.path.join(tempfile.mkdtemp(), "stdinfo")
    env['YNH_STDINFO'] = stdinfo

    stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn")
    with open(stdreturn, 'w') as f:
        f.write('')
    env['YNH_STDRETURN'] = stdreturn

    # use xtrace on fd 7 which is redirected to stdout
    env['BASH_XTRACEFD'] = 7
    cmd = '/bin/bash -x "{script}" {args} 7>&1'
    cmd = cmd.format(script=cmd_script, args=cmd_args)

    logger.debug("Executing command '%s'" % cmd)

    _env = os.environ.copy()
    _env.update(env)

    returncode = call_async_output(
        cmd, loggers, shell=True, cwd=chdir,
        stdinfo=stdinfo, env=_env
    )

    raw_content = None
    try:
        with open(stdreturn, 'r') as f:
            raw_content = f.read()
        returncontent = {}

        if return_format == "json":
            if raw_content != '':
                try:
                    returncontent = read_json(stdreturn)
                except Exception as e:
                    raise YunohostError('hook_json_return_error',
                                        path=path, msg=str(e),
                                        raw_content=raw_content)

        elif return_format == "plain_dict":
            for line in raw_content.split("\n"):
                if "=" in line:
                    key, value = line.strip().split("=", 1)
                    returncontent[key] = value

        else:
            raise YunohostError("Expected value for return_format is either 'json' or 'plain_dict', got '%s'" % return_format)
    finally:
        stdreturndir = os.path.split(stdreturn)[0]
        os.remove(stdreturn)
        os.rmdir(stdreturndir)

    return returncode, returncontent

def _hook_exec_python(path, args, env, loggers):

    dir_ = os.path.dirname(path)
    name = os.path.splitext(os.path.basename(path))[0]

    if dir_ not in sys.path:
        sys.path = [dir_] + sys.path
    module = import_module(name)

    ret = module.main(args, env, loggers)
    # # Assert that the return is a (int, dict) tuple
    assert isinstance(ret, tuple) \
        and len(ret) == 2 \
        and isinstance(ret[0], int) \
        and isinstance(ret[1], dict), \
        "Module %s did not return a (int, dict) tuple !" % module
    return ret

def _extract_filename_parts(filename):
    """Extract hook parts from filename"""
    if '-' in filename:
        priority, action = filename.split('-', 1)
    else:
        priority = '50'
        action = filename

    # Remove extension if there's one
    action = os.path.splitext(action)[0]
    return priority, action

# Taken from Python 3 shlex module --------------------------------------------

_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.UNICODE).search

def shell_quote(s):
    """Return a shell-escaped version of the string *s*."""
    s = str(s)
    if not s:
        return "''"
    if _find_unsafe(s) is None:
        return s

    # use single quotes, and put single quotes into double quotes
    # the string $'b is then quoted as '$'"'"'b'
    return "'" + s.replace("'", "'\"'\"'") + "'"
<|endoftext|>