<commit_before>"""
The ``spams_sandbox`` module provides mechanisms for sandboxing SPAMS.

===============================================================================
Overview
===============================================================================
SPAMS sometimes seems to step on the interpreter. As a result, we provide a
number of strategies to address this, by launching it in a separate process so
that it hopefully does not mess up the main interpreter. We also try to keep
the module space clean. This seems to help.

===============================================================================
API
===============================================================================
"""

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Jun 20, 2014 12:07:48 EDT$"

class SPAMSException(Exception):
    pass

def run_multiprocessing_queue_spams_trainDL(out_queue, *args, **kwargs):
    """
        Designed to run spams.trainDL in a separate process.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            out_queue(multiprocessing.Queue):       what will take the returned dictionary from spams.trainDL.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    out_queue.put(result)

def call_multiprocessing_queue_spams_trainDL(*args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Returns:
            result(numpy.matrix): the dictionary found
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    out_queue = multiprocessing.Queue()

    queue_args = (out_queue,) + args
    p = multiprocessing.Process(
        target=run_multiprocessing_queue_spams_trainDL,
        args=queue_args,
        kwargs=kwargs
    )
    p.start()
    result = out_queue.get()
    result = result.copy()
    p.join()

    if p.exitcode != 0: raise SPAMSException(
        "SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\"."
    )

    return(result)

def run_multiprocessing_array_spams_trainDL(result_array_type, result_array, X_array_type, X_array, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            result_array_type(numpy.ctypeslib.ndpointer):   a pointer type with properties needed by result_array.
            result_array(multiprocessing.RawArray):         shared memory array to store results in.
            X_array_type(numpy.ctypeslib.ndpointer):        a pointer type with properties needed by X_array.
            X_array(numpy.ndarray):                         currently uses numpy ndarray as input.
            *args(list):                                    a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                                 a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.
    """

    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    # Also, it is not needed outside of calling this function.
    import spams

    as_ordered_array_dict = {
        "F_CONTIGUOUS" : numpy.asfortranarray,
        "C_CONTIGUOUS" : numpy.ascontiguousarray
    }

    # Construct X from shared array.
    X_dtype = X_array_type._dtype_
    X_shape = X_array_type._shape_
    X_flags = numpy.core.multiarray.flagsobj(X_array_type._flags_)

    X = numpy.frombuffer(X_array, dtype=X_dtype).reshape(X_shape)
    X.setflags(X_flags)

    for order_name, as_ordered_array in as_ordered_array_dict.items():
        if order_name in X_array_type.__name__:
            X = as_ordered_array(X)

    # Construct the result to use the shared buffer.
    result_dtype = result_array_type._dtype_
    result_shape = result_array_type._shape_
    result_flags = numpy.core.multiarray.flagsobj(result_array_type._flags_)

    result = numpy.frombuffer(
        result_array, dtype=result_dtype
    ).reshape(result_shape)
    result.setflags(result_flags)

    for order_name, as_ordered_array in as_ordered_array_dict.items():
        if order_name in result_array_type.__name__:
            result = as_ordered_array(result)

    result[:] = spams.trainDL(X, *args, **kwargs)

def call_multiprocessing_array_spams_trainDL(X, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            X(numpy.matrix):                        a Fortran order NumPy Matrix with the same name as used by spams.trainDL (so if someone tries to use it as a keyword argument...).
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy

    # Types for X_array
    X_array_type = numpy.ctypeslib.ndpointer(
        dtype=X.dtype, ndim=X.ndim, shape=X.shape, flags=X.flags
    )
    X_array_ctype = type(
        numpy.ctypeslib.as_ctypes(X_array_type._dtype_.type(0)[()])
    )

    # Create a shared array to contain X
    X_array = multiprocessing.Array(X_array_ctype,
                                    X.size,
                                    lock=False)

    # Copy over the contents of X.
    X_array_numpy = numpy.frombuffer(
        X_array, dtype=X_array_type._dtype_
    ).reshape(X_array_type._shape_)
    X_array_numpy[:] = X
    X_array_numpy = None

    # Types for result_array
    result_array_type = numpy.ctypeslib.ndpointer(
        dtype=X.dtype, ndim=X.ndim, shape=(X.shape[0], kwargs["K"])
    )
    result_array_ctype = type(
        numpy.ctypeslib.as_ctypes(result_array_type._dtype_.type(0)[()])
    )

    # Create a shared array to contain the result
    result_array = multiprocessing.Array(
        result_array_ctype,
        numpy.product(result_array_type._shape_),
        lock=False
    )

    p = multiprocessing.Process(
        target=run_multiprocessing_array_spams_trainDL,
        args=(result_array_type, result_array, X_array_type, X_array,) + args,
        kwargs=kwargs
    )
    p.start()
    p.join()

    if p.exitcode != 0: raise SPAMSException(
        "SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\"."
    )

    # Reconstruct the result from the output array
    result = numpy.frombuffer(
        result_array, dtype=result_array_type._dtype_
    ).reshape(result_array_type._shape_)
    result = result.copy()

    return(result)

def call_spams_trainDL(*args, **kwargs):
    """
        Encapsulates call to spams.trainDL. Ensures copy of results occur just in case.
        Designed to be like the multiprocessing calls.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            For legacy.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    result = result.copy()

    return(result)
<commit_msg>nanshe/box/spams_sandbox.py: Wrap long function definition.<commit_after>"""
The ``spams_sandbox`` module provides mechanisms for sandboxing SPAMS.

===============================================================================
Overview
===============================================================================
SPAMS sometimes seems to step on the interpreter. As a result, we provide a
number of strategies to address this, by launching it in a separate process so
that it hopefully does not mess up the main interpreter. We also try to keep
the module space clean. This seems to help.

===============================================================================
API
===============================================================================
"""

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Jun 20, 2014 12:07:48 EDT$"

class SPAMSException(Exception):
    pass

def run_multiprocessing_queue_spams_trainDL(out_queue, *args, **kwargs):
    """
        Designed to run spams.trainDL in a separate process.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            out_queue(multiprocessing.Queue):       what will take the returned dictionary from spams.trainDL.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    out_queue.put(result)

def call_multiprocessing_queue_spams_trainDL(*args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Returns:
            result(numpy.matrix): the dictionary found
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    out_queue = multiprocessing.Queue()

    queue_args = (out_queue,) + args
    p = multiprocessing.Process(
        target=run_multiprocessing_queue_spams_trainDL,
        args=queue_args,
        kwargs=kwargs
    )
    p.start()
    result = out_queue.get()
    result = result.copy()
    p.join()

    if p.exitcode != 0: raise SPAMSException(
        "SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\"."
    )

    return(result)

def run_multiprocessing_array_spams_trainDL(result_array_type,
                                            result_array,
                                            X_array_type,
                                            X_array,
                                            *args,
                                            **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            result_array_type(numpy.ctypeslib.ndpointer):   a pointer type with properties needed by result_array.
            result_array(multiprocessing.RawArray):         shared memory array to store results in.
            X_array_type(numpy.ctypeslib.ndpointer):        a pointer type with properties needed by X_array.
            X_array(numpy.ndarray):                         currently uses numpy ndarray as input.
            *args(list):                                    a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                                 a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.
    """

    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    # Also, it is not needed outside of calling this function.
    import spams

    as_ordered_array_dict = {
        "F_CONTIGUOUS" : numpy.asfortranarray,
        "C_CONTIGUOUS" : numpy.ascontiguousarray
    }

    # Construct X from shared array.
    X_dtype = X_array_type._dtype_
    X_shape = X_array_type._shape_
    X_flags = numpy.core.multiarray.flagsobj(X_array_type._flags_)

    X = numpy.frombuffer(X_array, dtype=X_dtype).reshape(X_shape)
    X.setflags(X_flags)

    for order_name, as_ordered_array in as_ordered_array_dict.items():
        if order_name in X_array_type.__name__:
            X = as_ordered_array(X)

    # Construct the result to use the shared buffer.
    result_dtype = result_array_type._dtype_
    result_shape = result_array_type._shape_
    result_flags = numpy.core.multiarray.flagsobj(result_array_type._flags_)

    result = numpy.frombuffer(
        result_array, dtype=result_dtype
    ).reshape(result_shape)
    result.setflags(result_flags)

    for order_name, as_ordered_array in as_ordered_array_dict.items():
        if order_name in result_array_type.__name__:
            result = as_ordered_array(result)

    result[:] = spams.trainDL(X, *args, **kwargs)

def call_multiprocessing_array_spams_trainDL(X, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.

        Args:
            X(numpy.matrix):                        a Fortran order NumPy Matrix with the same name as used by spams.trainDL (so if someone tries to use it as a keyword argument...).
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.
    """

    # Only necessary for dealing with SPAMS
    import multiprocessing

    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy

    # Types for X_array
    X_array_type = numpy.ctypeslib.ndpointer(
        dtype=X.dtype, ndim=X.ndim, shape=X.shape, flags=X.flags
    )
    X_array_ctype = type(
        numpy.ctypeslib.as_ctypes(X_array_type._dtype_.type(0)[()])
    )

    # Create a shared array to contain X
    X_array = multiprocessing.Array(X_array_ctype,
                                    X.size,
                                    lock=False)

    # Copy over the contents of X.
    X_array_numpy = numpy.frombuffer(
        X_array, dtype=X_array_type._dtype_
    ).reshape(X_array_type._shape_)
    X_array_numpy[:] = X
    X_array_numpy = None

    # Types for result_array
    result_array_type = numpy.ctypeslib.ndpointer(
        dtype=X.dtype, ndim=X.ndim, shape=(X.shape[0], kwargs["K"])
    )
    result_array_ctype = type(
        numpy.ctypeslib.as_ctypes(result_array_type._dtype_.type(0)[()])
    )

    # Create a shared array to contain the result
    result_array = multiprocessing.Array(
        result_array_ctype,
        numpy.product(result_array_type._shape_),
        lock=False
    )

    p = multiprocessing.Process(
        target=run_multiprocessing_array_spams_trainDL,
        args=(result_array_type, result_array, X_array_type, X_array,) + args,
        kwargs=kwargs
    )
    p.start()
    p.join()

    if p.exitcode != 0: raise SPAMSException(
        "SPAMS has terminated with exitcode \"" + repr(p.exitcode) + "\"."
    )

    # Reconstruct the result from the output array
    result = numpy.frombuffer(
        result_array, dtype=result_array_type._dtype_
    ).reshape(result_array_type._shape_)
    result = result.copy()

    return(result)

def call_spams_trainDL(*args, **kwargs):
    """
        Encapsulates call to spams.trainDL. Ensures copy of results occur just in case.
        Designed to be like the multiprocessing calls.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            For legacy.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    result = result.copy()

    return(result)
<|endoftext|>


<commit_before># Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE

import functools
import re

import frappe
from frappe import _

def load_address_and_contact(doc, key=None):
	"""Loads address list and contact list in `__onload`"""
	from frappe.contacts.doctype.address.address import get_address_display, get_condensed_address

	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Address"],
	]
	address_list = frappe.get_list("Address", filters=filters, fields=["*"], order_by="creation asc")

	address_list = [a.update({"display": get_address_display(a)}) for a in address_list]

	address_list = sorted(
		address_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_address - b.is_primary_address))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("addr_list", address_list)

	contact_list = []
	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Contact"],
	]
	contact_list = frappe.get_list("Contact", filters=filters, fields=["*"])

	for contact in contact_list:
		contact["email_ids"] = frappe.get_all(
			"Contact Email",
			filters={"parenttype": "Contact", "parent": contact.name, "is_primary": 0},
			fields=["email_id"],
		)

		contact["phone_nos"] = frappe.get_all(
			"Contact Phone",
			filters={
				"parenttype": "Contact",
				"parent": contact.name,
				"is_primary_phone": 0,
				"is_primary_mobile_no": 0,
			},
			fields=["phone"],
		)

		if contact.address:
			address = frappe.get_doc("Address", contact.address)
			contact["address"] = get_condensed_address(address)

	contact_list = sorted(
		contact_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_contact - b.is_primary_contact))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("contact_list", contact_list)

def has_permission(doc, ptype, user):
	links = get_permitted_and_not_permitted_links(doc.doctype)
	if not links.get("not_permitted_links"):
		# optimization: don't determine permissions based on link fields
		return True

	# True if any one is True or all are empty
	names = []
	for df in links.get("permitted_links") + links.get("not_permitted_links"):
		doctype = df.options
		name = doc.get(df.fieldname)
		names.append(name)

		if name and frappe.has_permission(doctype, ptype, doc=name):
			return True

	if not any(names):
		return True
	return False

def get_permission_query_conditions_for_contact(user):
	return get_permission_query_conditions("Contact")

def get_permission_query_conditions_for_address(user):
	return get_permission_query_conditions("Address")

def get_permission_query_conditions(doctype):
	links = get_permitted_and_not_permitted_links(doctype)

	if not links.get("not_permitted_links"):
		# when everything is permitted, don't add additional condition
		return ""

	elif not links.get("permitted_links"):
		conditions = []

		# when everything is not permitted
		for df in links.get("not_permitted_links"):
			# like ifnull(customer, '')='' and ifnull(supplier, '')=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " and ".join(conditions) + " )"

	else:
		conditions = []

		for df in links.get("permitted_links"):
			# like ifnull(customer, '')!='' or ifnull(supplier, '')!=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " or ".join(conditions) + " )"

def get_permitted_and_not_permitted_links(doctype):
	permitted_links = []
	not_permitted_links = []

	meta = frappe.get_meta(doctype)
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	for df in meta.get_link_fields():
		if df.options not in ("Customer", "Supplier", "Company", "Sales Partner"):
			continue

		if df.options in allowed_doctypes:
			permitted_links.append(df)
		else:
			not_permitted_links.append(df)

	return {"permitted_links": permitted_links, "not_permitted_links": not_permitted_links}

def delete_contact_and_address(doctype, docname):
	for parenttype in ("Contact", "Address"):
		items = frappe.db.sql_list(
			"""select parent from `tabDynamic Link`
			where parenttype=%s and link_doctype=%s and link_name=%s""",
			(parenttype, doctype, docname),
		)

		for name in items:
			doc = frappe.get_doc(parenttype, name)
			if len(doc.links) == 1:
				doc.delete()

@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters):
	if not txt:
		txt = ""

	doctypes = frappe.db.get_all(
		"DocField", filters=filters, fields=["parent"], distinct=True, as_list=True
	)

	doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE))

	filters.update({"dt": ("not in", [d[0] for d in doctypes])})

	_doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True)

	_doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)])

	all_doctypes = [d[0] for d in doctypes + _doctypes]
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes)))
	valid_doctypes = [[doctype] for doctype in valid_doctypes]

	return valid_doctypes

def set_link_title(doc):
	if not doc.links:
		return
	for link in doc.links:
		if not link.link_title:
			linked_doc = frappe.get_doc(link.link_doctype, link.link_name)
			link.link_title = linked_doc.get_title() or link.link_name
<commit_msg>refactor: filter_dynamic_link_doctypes API<commit_after># Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE

import functools
import re
from typing import Dict, List

import frappe
from frappe import _

def load_address_and_contact(doc, key=None):
	"""Loads address list and contact list in `__onload`"""
	from frappe.contacts.doctype.address.address import get_address_display, get_condensed_address

	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Address"],
	]
	address_list = frappe.get_list("Address", filters=filters, fields=["*"], order_by="creation asc")

	address_list = [a.update({"display": get_address_display(a)}) for a in address_list]

	address_list = sorted(
		address_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_address - b.is_primary_address))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("addr_list", address_list)

	contact_list = []
	filters = [
		["Dynamic Link", "link_doctype", "=", doc.doctype],
		["Dynamic Link", "link_name", "=", doc.name],
		["Dynamic Link", "parenttype", "=", "Contact"],
	]
	contact_list = frappe.get_list("Contact", filters=filters, fields=["*"])

	for contact in contact_list:
		contact["email_ids"] = frappe.get_all(
			"Contact Email",
			filters={"parenttype": "Contact", "parent": contact.name, "is_primary": 0},
			fields=["email_id"],
		)

		contact["phone_nos"] = frappe.get_all(
			"Contact Phone",
			filters={
				"parenttype": "Contact",
				"parent": contact.name,
				"is_primary_phone": 0,
				"is_primary_mobile_no": 0,
			},
			fields=["phone"],
		)

		if contact.address:
			address = frappe.get_doc("Address", contact.address)
			contact["address"] = get_condensed_address(address)

	contact_list = sorted(
		contact_list,
		key=functools.cmp_to_key(
			lambda a, b: (int(a.is_primary_contact - b.is_primary_contact))
			or (1 if a.modified - b.modified else 0)
		),
		reverse=True,
	)

	doc.set_onload("contact_list", contact_list)

def has_permission(doc, ptype, user):
	links = get_permitted_and_not_permitted_links(doc.doctype)
	if not links.get("not_permitted_links"):
		# optimization: don't determine permissions based on link fields
		return True

	# True if any one is True or all are empty
	names = []
	for df in links.get("permitted_links") + links.get("not_permitted_links"):
		doctype = df.options
		name = doc.get(df.fieldname)
		names.append(name)

		if name and frappe.has_permission(doctype, ptype, doc=name):
			return True

	if not any(names):
		return True
	return False

def get_permission_query_conditions_for_contact(user):
	return get_permission_query_conditions("Contact")

def get_permission_query_conditions_for_address(user):
	return get_permission_query_conditions("Address")

def get_permission_query_conditions(doctype):
	links = get_permitted_and_not_permitted_links(doctype)

	if not links.get("not_permitted_links"):
		# when everything is permitted, don't add additional condition
		return ""

	elif not links.get("permitted_links"):
		conditions = []

		# when everything is not permitted
		for df in links.get("not_permitted_links"):
			# like ifnull(customer, '')='' and ifnull(supplier, '')=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " and ".join(conditions) + " )"

	else:
		conditions = []

		for df in links.get("permitted_links"):
			# like ifnull(customer, '')!='' or ifnull(supplier, '')!=''
			conditions.append(
				"ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname)
			)

		return "( " + " or ".join(conditions) + " )"

def get_permitted_and_not_permitted_links(doctype):
	permitted_links = []
	not_permitted_links = []

	meta = frappe.get_meta(doctype)
	allowed_doctypes = frappe.permissions.get_doctypes_with_read()

	for df in meta.get_link_fields():
		if df.options not in ("Customer", "Supplier", "Company", "Sales Partner"):
			continue

		if df.options in allowed_doctypes:
			permitted_links.append(df)
		else:
			not_permitted_links.append(df)

	return {"permitted_links": permitted_links, "not_permitted_links": not_permitted_links}

def delete_contact_and_address(doctype, docname):
	for parenttype in ("Contact", "Address"):
		items = frappe.db.sql_list(
			"""select parent from `tabDynamic Link`
			where parenttype=%s and link_doctype=%s and link_name=%s""",
			(parenttype, doctype, docname),
		)

		for name in items:
			doc = frappe.get_doc(parenttype, name)
			if len(doc.links) == 1:
				doc.delete()

@frappe.whitelist()
@frappe.validate_and_sanitize_search_inputs
def filter_dynamic_link_doctypes(txt: str, filters: Dict) -> List[List[str]]:
	from frappe.permissions import get_doctypes_with_read

	txt = txt or ""
	filters = filters or {}
	TXT_PATTERN = re.compile(f"{txt}.*")

	_doctypes_from_df = frappe.get_all(
		"DocField",
		filters=filters,
		pluck="parent",
		distinct=True,
		order_by=None,
	)
	doctypes_from_df = {d for d in _doctypes_from_df if TXT_PATTERN.search(_(d), re.IGNORECASE)}

	filters.update({"dt": ("not in", doctypes_from_df)})
	_doctypes_from_cdf = frappe.get_all(
		"Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None
	)
	doctypes_from_cdf = {d for d in _doctypes_from_cdf if TXT_PATTERN.search(_(d), re.IGNORECASE)}

	all_doctypes = doctypes_from_df.union(doctypes_from_cdf)
	allowed_doctypes = set(get_doctypes_with_read())

	valid_doctypes = sorted(all_doctypes.intersection(allowed_doctypes))

	return [[doctype] for doctype in valid_doctypes]

def set_link_title(doc):
	if not doc.links:
		return
	for link in doc.links:
		if not link.link_title:
			linked_doc = frappe.get_doc(link.link_doctype, link.link_name)
			link.link_title = linked_doc.get_title() or link.link_name
<|endoftext|>


<commit_before># To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the editor.

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$May 1, 2014 2:24:55PM$"

import numpy
import scipy
import scipy.misc

import vigra

import HDF5_recorder

# Need in order to have logging information no matter what.
import debugging_tools

# Get the logger
logger = debugging_tools.logging.getLogger(__name__)

@debugging_tools.log_call(logger)
def binomial_coefficients(n):
    """
        Generates a row in Pascal's triangle (binomial coefficients).

        Args:
            n(int):      which row of Pascal's triangle to return.

        Returns:
            cs(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> binomial_coefficients(0)
            array([1])

            >>> binomial_coefficients(1)
            array([1, 1])

            >>> binomial_coefficients(2)
            array([1, 2, 1])

            >>> binomial_coefficients(4)
            array([1, 4, 6, 4, 1])

            >>> binomial_coefficients(4.0)
            array([1, 4, 6, 4, 1])
    """

    # Must be integer
    n = int(n)

    # Below 0 is irrelevant.
    if n < 0:
        n = 0

    # Get enough repeats of n to get each coefficent
    ns = numpy.repeat(n, n + 1)
    # Get all relevant k's
    ks = numpy.arange(n + 1)

    # Get all the coefficents in order
    cs = scipy.misc.comb(ns, ks)
    cs = numpy.around(cs)
    cs = cs.astype(int)

    return(cs)

@debugging_tools.log_call(logger)
def binomial_1D_array_kernel(i, n = 4):
    """
        Generates a 1D numpy array used to make the kernel for the wavelet transform.

        Args:
            i(int):      which scaling to use.
            n(int):      which row of Pascal's triangle to return.

        Returns:
            r(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> binomial_1D_array_kernel(0)
            array([ 0.0625,  0.25  ,  0.375 ,  0.25  ,  0.0625])

            >>> binomial_1D_array_kernel(0, 4)
            array([ 0.0625,  0.25  ,  0.375 ,  0.25  ,  0.0625])

            >>> binomial_1D_array_kernel(1, 4)
            array([ 0.0625,  0.25  ,  0.375 ,  0.25  ,  0.0625])

            >>> binomial_1D_array_kernel(2, 4)
            array([ 0.0625,  0.    ,  0.25  ,  0.    ,  0.375 ,  0.    ,  0.25  ,
                    0.    ,  0.0625])

            >>> binomial_1D_array_kernel(3, 4)
            array([ 0.0625,  0.    ,  0.    ,  0.    ,  0.25  ,  0.    ,  0.    ,
                    0.    ,  0.375 ,  0.    ,  0.    ,  0.    ,  0.25  ,  0.    ,
                    0.    ,  0.    ,  0.0625])

            >>> binomial_1D_array_kernel(4, 4)
            array([ 0.0625,  0.    ,  0.    ,  0.    ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.25  ,  0.    ,  0.    ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.    ,  0.375 ,  0.    ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.    ,  0.    ,  0.25  ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.    ,  0.    ,  0.    ,  0.0625])

            >>> binomial_1D_array_kernel(2, 1)
            array([ 0.5,  0. ,  0.5])
    """

    # Below 1 is irrelevant.
    if i < 1:
        i = 1

    # Below 0 is irrelevant.
    if n < 0:
        n = 0

    # Get the binomial coefficients.
    cs = list(binomial_coefficients(n))

    # Get the right number of zeros to fill in
    zs = list(numpy.zeros(2 ** (i - 1) - 1, dtype = int))

    # Create the contents of the 1D kernel before normalization
    r = []
    if len(cs) > 1:
        for _ in cs[:-1]:
            r.append(_)
            r.extend(zs)

        r.append(cs[-1])
    else:
        r.append(cs)

    r = numpy.array(r)
    r = r.astype(float)

    # Normalization on the L_1 norm.
    r /= 2 ** n

    return(r)

@debugging_tools.log_call(logger)
def binomial_1D_vigra_kernel(i, n = 4, border_treatment = vigra.filters.BorderTreatmentMode.BORDER_TREATMENT_REFLECT):
    """
        Generates a vigra.filters.Kernel1D using binomial_1D_array_kernel(i).

        Args:
            i(int):                                                 which scaling to use.
            n(int):                                                 which row of Pascal's triangle to return.
            border_treatment(vigra.filters.BorderTreatmentMode):    determines how to deal with the borders.

        Returns:
            k(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> binomial_1D_vigra_kernel(1) # doctest: +ELLIPSIS
            <vigra.filters.Kernel1D object at 0x...>
    """

    # Generate the vector for the kernel
    h_kern = binomial_1D_array_kernel(i, n)

    # Determine the kernel center
    h_kern_half_width = (h_kern.size - 1) / 2

    # Default kernel
    k = vigra.filters.Kernel1D()
    # Center the kernel
    k.initExplicitly(-h_kern_half_width, h_kern_half_width, h_kern)
    # Set the border treatment mode.
    k.setBorderTreatment(border_treatment)

    return(k)

@debugging_tools.log_call(logger)
@HDF5_recorder.static_array_debug_recorder
def wavelet_transform(im0, scale = 5, include_intermediates = False, include_lower_scales = False):
    """
        performs integral steps of the wavelet transform on im0 up to the given scale. If scale is an iterable, then

        Args:
            im0(numpy.ndarray):                                  the original image.
            scale(int):                                          the scale of wavelet transform to apply.
            include_intermediates(bool):                         whether to return intermediates or not (default False).
            include_lower_scales(bool):                          whether to include lower scales or not (default False)
                                                                 (ignored if include_intermediates is True).

        Returns:
            k(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> wavelet_transform(numpy.eye(3).astype(numpy.float32),
            ...     scale = 1,
            ...     include_intermediates = True,
            ...     include_lower_scales = True) # doctest: +NORMALIZE_WHITESPACE
            (array([[[ 0.59375, -0.375  , -0.34375],
                     [-0.375  ,  0.625  , -0.375  ],
                     [-0.34375, -0.375  ,  0.59375]]], dtype=float32),
             array([[[ 1.    ,  0.     ,  0.     ],
                    [ 0.     ,  1.     ,  0.     ],
                    [ 0.     ,  0.     ,  1.     ]],
                   [[ 0.40625,  0.375  ,  0.34375],
                    [ 0.375  ,  0.375  ,  0.375  ],
                    [ 0.34375,  0.375  ,  0.40625]]], dtype=float32))
    """

    im0 = im0.astype(numpy.float32)

    # Make sure that we have scale as a list.
    # If it is not a list, then make a singleton list.
    try:
        scale = numpy.array(list(scale))

        if scale.ndim > 1:
            raise Exception(
                "Scale should only have 1 dimension. Instead, got scale.ndim = \"" + str(scale.ndim) + "\".")

        if scale.shape[0] != im0.ndim:
            raise Exception(
                "Scale should have a value of each dimension of im0. Instead, got scale.shape[0] = \"" + str(
                    scale.shape[0]) + "\" and im0.ndim = \"" + str(im0.ndim) + "\".")

    except TypeError:
        scale = numpy.repeat([scale], im0.ndim)

    if include_intermediates:
        W = numpy.zeros((scale.max(),) + im0.shape, dtype = im0.dtype)
        imOut = numpy.zeros((scale.max() + 1,) + im0.shape, dtype = im0.dtype)
        imOut[0] = im0
    elif include_lower_scales:
        W = numpy.zeros((scale.max(),) + im0.shape, dtype = im0.dtype)

    imPrev = im0.copy()
    imCur = im0.copy()
    imDiff = imPrev - imCur
    for i in xrange(1, scale.max() + 1):

        h_ker = binomial_1D_vigra_kernel(i)

        for d in xrange(len(scale)):
            if i <= scale[d]:
                vigra.filters.convolveOneDimension(imCur, d, h_ker, out=imCur)

        imDiff[:] = imPrev - imCur

        if include_intermediates:
            W[i - 1] = imDiff
            imOut[i] = imCur
        elif include_lower_scales:
            W[i - 1] = imDiff

        imPrev[:] = imCur

    if include_intermediates:
        return((W, imOut))
    elif include_lower_scales:
        return(W)
    else:
        return(imDiff)
<commit_msg>nanshe/wavelet_transform.py: In wavelet_transform, cleaned up doctest spacing.<commit_after># To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the editor.

__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$May 1, 2014 2:24:55PM$"

import numpy
import scipy
import scipy.misc

import vigra

import HDF5_recorder

# Need in order to have logging information no matter what.
import debugging_tools

# Get the logger
logger = debugging_tools.logging.getLogger(__name__)

@debugging_tools.log_call(logger)
def binomial_coefficients(n):
    """
        Generates a row in Pascal's triangle (binomial coefficients).

        Args:
            n(int):      which row of Pascal's triangle to return.

        Returns:
            cs(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> binomial_coefficients(0)
            array([1])

            >>> binomial_coefficients(1)
            array([1, 1])

            >>> binomial_coefficients(2)
            array([1, 2, 1])

            >>> binomial_coefficients(4)
            array([1, 4, 6, 4, 1])

            >>> binomial_coefficients(4.0)
            array([1, 4, 6, 4, 1])
    """

    # Must be integer
    n = int(n)

    # Below 0 is irrelevant.
    if n < 0:
        n = 0

    # Get enough repeats of n to get each coefficent
    ns = numpy.repeat(n, n + 1)
    # Get all relevant k's
    ks = numpy.arange(n + 1)

    # Get all the coefficents in order
    cs = scipy.misc.comb(ns, ks)
    cs = numpy.around(cs)
    cs = cs.astype(int)

    return(cs)

@debugging_tools.log_call(logger)
def binomial_1D_array_kernel(i, n = 4):
    """
        Generates a 1D numpy array used to make the kernel for the wavelet transform.

        Args:
            i(int):      which scaling to use.
            n(int):      which row of Pascal's triangle to return.

        Returns:
            r(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> binomial_1D_array_kernel(0)
            array([ 0.0625,  0.25  ,  0.375 ,  0.25  ,  0.0625])

            >>> binomial_1D_array_kernel(0, 4)
            array([ 0.0625,  0.25  ,  0.375 ,  0.25  ,  0.0625])

            >>> binomial_1D_array_kernel(1, 4)
            array([ 0.0625,  0.25  ,  0.375 ,  0.25  ,  0.0625])

            >>> binomial_1D_array_kernel(2, 4)
            array([ 0.0625,  0.    ,  0.25  ,  0.    ,  0.375 ,  0.    ,  0.25  ,
                    0.    ,  0.0625])

            >>> binomial_1D_array_kernel(3, 4)
            array([ 0.0625,  0.    ,  0.    ,  0.    ,  0.25  ,  0.    ,  0.    ,
                    0.    ,  0.375 ,  0.    ,  0.    ,  0.    ,  0.25  ,  0.    ,
                    0.    ,  0.    ,  0.0625])

            >>> binomial_1D_array_kernel(4, 4)
            array([ 0.0625,  0.    ,  0.    ,  0.    ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.25  ,  0.    ,  0.    ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.    ,  0.375 ,  0.    ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.    ,  0.    ,  0.25  ,  0.    ,  0.    ,  0.    ,
                    0.    ,  0.    ,  0.    ,  0.    ,  0.0625])

            >>> binomial_1D_array_kernel(2, 1)
            array([ 0.5,  0. ,  0.5])
    """

    # Below 1 is irrelevant.
    if i < 1:
        i = 1

    # Below 0 is irrelevant.
    if n < 0:
        n = 0

    # Get the binomial coefficients.
    cs = list(binomial_coefficients(n))

    # Get the right number of zeros to fill in
    zs = list(numpy.zeros(2 ** (i - 1) - 1, dtype = int))

    # Create the contents of the 1D kernel before normalization
    r = []
    if len(cs) > 1:
        for _ in cs[:-1]:
            r.append(_)
            r.extend(zs)

        r.append(cs[-1])
    else:
        r.append(cs)

    r = numpy.array(r)
    r = r.astype(float)

    # Normalization on the L_1 norm.
    r /= 2 ** n

    return(r)

@debugging_tools.log_call(logger)
def binomial_1D_vigra_kernel(i, n = 4, border_treatment = vigra.filters.BorderTreatmentMode.BORDER_TREATMENT_REFLECT):
    """
        Generates a vigra.filters.Kernel1D using binomial_1D_array_kernel(i).

        Args:
            i(int):                                                 which scaling to use.
            n(int):                                                 which row of Pascal's triangle to return.
            border_treatment(vigra.filters.BorderTreatmentMode):    determines how to deal with the borders.

        Returns:
            k(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> binomial_1D_vigra_kernel(1) # doctest: +ELLIPSIS
            <vigra.filters.Kernel1D object at 0x...>
    """

    # Generate the vector for the kernel
    h_kern = binomial_1D_array_kernel(i, n)

    # Determine the kernel center
    h_kern_half_width = (h_kern.size - 1) / 2

    # Default kernel
    k = vigra.filters.Kernel1D()
    # Center the kernel
    k.initExplicitly(-h_kern_half_width, h_kern_half_width, h_kern)
    # Set the border treatment mode.
    k.setBorderTreatment(border_treatment)

    return(k)

@debugging_tools.log_call(logger)
@HDF5_recorder.static_array_debug_recorder
def wavelet_transform(im0, scale = 5, include_intermediates = False, include_lower_scales = False):
    """
        performs integral steps of the wavelet transform on im0 up to the given scale. If scale is an iterable, then

        Args:
            im0(numpy.ndarray):                                  the original image.
            scale(int):                                          the scale of wavelet transform to apply.
            include_intermediates(bool):                         whether to return intermediates or not (default False).
            include_lower_scales(bool):                          whether to include lower scales or not (default False)
                                                                 (ignored if include_intermediates is True).

        Returns:
            k(numpy.ndarray): a numpy array containing the row of Pascal's triangle.

        Examples:
            >>> wavelet_transform(numpy.eye(3).astype(numpy.float32),
            ...     scale = 1,
            ...     include_intermediates = True,
            ...     include_lower_scales = True) # doctest: +NORMALIZE_WHITESPACE
            (array([[[ 0.59375, -0.375  , -0.34375],
                     [-0.375  ,  0.625  , -0.375  ],
                     [-0.34375, -0.375  ,  0.59375]]], dtype=float32),
             array([[[ 1.     ,  0.     ,  0.     ],
                     [ 0.     ,  1.     ,  0.     ],
                     [ 0.     ,  0.     ,  1.     ]],
                    [[ 0.40625,  0.375  ,  0.34375],
                     [ 0.375  ,  0.375  ,  0.375  ],
                     [ 0.34375,  0.375  ,  0.40625]]], dtype=float32))
    """

    im0 = im0.astype(numpy.float32)

    # Make sure that we have scale as a list.
    # If it is not a list, then make a singleton list.
    try:
        scale = numpy.array(list(scale))

        if scale.ndim > 1:
            raise Exception(
                "Scale should only have 1 dimension. Instead, got scale.ndim = \"" + str(scale.ndim) + "\".")

        if scale.shape[0] != im0.ndim:
            raise Exception(
                "Scale should have a value of each dimension of im0. Instead, got scale.shape[0] = \"" + str(
                    scale.shape[0]) + "\" and im0.ndim = \"" + str(im0.ndim) + "\".")

    except TypeError:
        scale = numpy.repeat([scale], im0.ndim)

    if include_intermediates:
        W = numpy.zeros((scale.max(),) + im0.shape, dtype = im0.dtype)
        imOut = numpy.zeros((scale.max() + 1,) + im0.shape, dtype = im0.dtype)
        imOut[0] = im0
    elif include_lower_scales:
        W = numpy.zeros((scale.max(),) + im0.shape, dtype = im0.dtype)

    imPrev = im0.copy()
    imCur = im0.copy()
    imDiff = imPrev - imCur
    for i in xrange(1, scale.max() + 1):

        h_ker = binomial_1D_vigra_kernel(i)

        for d in xrange(len(scale)):
            if i <= scale[d]:
                vigra.filters.convolveOneDimension(imCur, d, h_ker, out=imCur)

        imDiff[:] = imPrev - imCur

        if include_intermediates:
            W[i - 1] = imDiff
            imOut[i] = imCur
        elif include_lower_scales:
            W[i - 1] = imDiff

        imPrev[:] = imCur

    if include_intermediates:
        return((W, imOut))
    elif include_lower_scales:
        return(W)
    else:
        return(imDiff)
<|endoftext|>


<commit_before>from django.utils import timezone
from django.db.models import (
    Count, Subquery, Exists, OuterRef, Value, NullBooleanField,
    Case, When
)

from rest_framework import serializers, viewsets, mixins, status
from rest_framework.response import Response
from rest_framework.reverse import reverse

from oauth2_provider.contrib.rest_framework import TokenHasScope

from froide.foirequest.models.request import (
    FoiRequest, get_absolute_domain_short_url
)
from froide.foirequest.auth import can_read_foirequest_authenticated
from froide.foirequest.api_views import filter_foirequests
from froide.helper.api_utils import CustomLimitOffsetPagination

from .models import FoiRequestFollower

class CreateOnlyWithScopePermission(TokenHasScope):
    def has_permission(self, request, view):
        if view.action not in ('create', 'update'):
            return True
        if not request.user.is_authenticated:
            return False
        if not request.auth:
            return True
        return super(CreateOnlyWithScopePermission, self).has_permission(
            request, view
        )

class CreateFoiRequestFollowSerializer(serializers.ModelSerializer):
    request = serializers.PrimaryKeyRelatedField(queryset=FoiRequest.objects.none())

    class Meta:
        model = FoiRequestFollower
        fields = ('request',)

    def __init__(self, *args, **kwargs):
        super(CreateFoiRequestFollowSerializer, self).__init__(
            *args, **kwargs
        )
        qs = FoiRequestFollower.objects.get_followable_requests(
            self.context['view'].request.user
        )
        self.fields['request'].queryset = qs

    def validate_request(self, value):
        """
        Check that the blog post is about Django.
        """
        if not can_read_foirequest_authenticated(value, self.context['view'].request):
            raise serializers.ValidationError('Cannot follow your own requests')
        return value

    def create(self, validated_data):
        follower, create = FoiRequestFollower.objects.get_or_create(
            request=validated_data['request'],
            user=validated_data['user'],
            defaults={
                'timestamp': timezone.now()
            }
        )
        return follower

class FoiRequestFollowSerializer(serializers.HyperlinkedModelSerializer):
    request = serializers.HyperlinkedRelatedField(
        view_name='api:request-detail',
        lookup_field='pk',
        read_only=True
    )
    resource_uri = serializers.HyperlinkedIdentityField(
        view_name='api:following-detail',
        lookup_field='pk',
        required=False
    )
    request_url = serializers.SerializerMethodField(
        source='get_absolute_domain_url',
        read_only=True
    )
    follow_count = serializers.IntegerField(read_only=True)
    follows = serializers.BooleanField(read_only=True)
    can_follow = serializers.BooleanField(read_only=True)

    class Meta:
        model = FoiRequestFollower
        fields = ('resource_uri', 'request', 'request_url',
                  'timestamp', 'follow_count', 'follows', 'can_follow')

    def get_request_url(self, obj):
        return get_absolute_domain_short_url(obj.request_id)

class FoiRequestFollowRequestSerializer(serializers.HyperlinkedModelSerializer):
    request = serializers.HyperlinkedIdentityField(
        view_name='api:request-detail',
        lookup_field='pk',
        read_only=True
    )
    resource_uri = serializers.HyperlinkedRelatedField(
        view_name='api:following-detail',
        lookup_field='pk',
        read_only=True
    )
    request_url = serializers.SerializerMethodField(
        source='get_absolute_domain_url',
        read_only=True
    )
    follow_count = serializers.IntegerField(read_only=True)
    follows = serializers.BooleanField(read_only=True)
    can_follow = serializers.BooleanField(read_only=True)

    class Meta:
        model = FoiRequest
        fields = ('request', 'request_url', 'resource_uri',
                  'follow_count', 'follows', 'can_follow')

    def get_request_url(self, obj):
        return get_absolute_domain_short_url(obj.id)

class LargeResultsSetPagination(CustomLimitOffsetPagination):
    default_limit = 150
    max_limit = 200

class FoiRequestFollowerViewSet(mixins.CreateModelMixin,
                        mixins.ListModelMixin,
                        mixins.DestroyModelMixin,
                        viewsets.GenericViewSet):
    serializer_class = FoiRequestFollowSerializer
    permission_classes = (CreateOnlyWithScopePermission,)
    read_scopes = ['read:request']
    required_scopes = ['follow:request']
    pagination_class = LargeResultsSetPagination

    def get_serializer_class(self):
        if self.action == 'create':
            return CreateFoiRequestFollowSerializer
        if self.get_request_filter():
            return FoiRequestFollowRequestSerializer
        return self.serializer_class

    def get_foirequest_queryset(self, requests=None):
        user = self.request.user
        token = self.request.auth
        qs = filter_foirequests(user, token)
        if requests is not None:
            qs = qs.filter(id__in=requests)
        if user.is_authenticated and (
                not token or token.is_valid(self.read_scopes)):
            follows = (
                FoiRequestFollower.objects
                .filter(request_id=OuterRef('pk'), user=user)
            )
            qs = qs.annotate(
                follows=Exists(follows),
                resource_uri=Subquery(follows.values('pk')),
                can_follow=Case(
                    When(user_id=user.id, then=Value(False)),
                    default=Value(True),
                    output_field=NullBooleanField()
                )
            )
        else:
            qs = qs.annotate(
                follows=Value(None, output_field=NullBooleanField()),
                can_follow=Value(None, output_field=NullBooleanField())
            )
        qs = qs.annotate(
            follow_count=Count('followers'),
        )
        return qs

    def get_request_filter(self):
        if not hasattr(self, '_requests_filter'):
            requests = self.request.query_params.get('request', '').split(',')
            try:
                requests = [int(r) for r in requests]
            except ValueError:
                requests = []
            self._requests_filter = requests
        return self._requests_filter

    def get_queryset(self):
        user = self.request.user
        token = self.request.auth
        requests = self.get_request_filter()
        qs = None
        if requests:
            return self.get_foirequest_queryset(requests=requests)
        elif user.is_authenticated:
            if not token or token.is_valid(self.read_scopes):
                qs = (
                    FoiRequestFollower.objects
                    .filter(user=user)
                    .order_by('-timestamp')
                )
        if qs is None:
            qs = FoiRequestFollower.objects.none()

        follower_count = (
            FoiRequest.objects
            .filter(id=OuterRef('request'))
            .values('pk')
            .annotate(count=Count('followers'))
        )
        qs = qs.annotate(
            follow_count=Subquery(follower_count.values('count')),
            # follows by query definition
            follows=Value(True, output_field=NullBooleanField()),
            # can_follow by query definition
            can_follow=Value(True, output_field=NullBooleanField())
        )
        return qs

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        instance = self.perform_create(serializer)
        data = {
            'status': 'success',
            'url': reverse('api:following-detail', kwargs={'pk': instance.pk},
                           request=request)
        }
        headers = {'Location': data['url']}
        return Response(data, status=status.HTTP_201_CREATED, headers=headers)

    def perform_create(self, serializer):
        return serializer.save(user=self.request.user)
<commit_msg>Fix foirequest follower api<commit_after>from django.utils import timezone
from django.db.models import (
    Count, Subquery, Exists, OuterRef, Value, NullBooleanField,
    Case, When
)

from rest_framework import serializers, viewsets, mixins, status
from rest_framework.response import Response
from rest_framework.reverse import reverse

from oauth2_provider.contrib.rest_framework import TokenHasScope

from froide.foirequest.models.request import (
    FoiRequest, get_absolute_domain_short_url
)
from froide.foirequest.auth import can_read_foirequest_authenticated
from froide.foirequest.api_views import filter_foirequests
from froide.helper.api_utils import CustomLimitOffsetPagination

from .models import FoiRequestFollower

class CreateOnlyWithScopePermission(TokenHasScope):
    def has_permission(self, request, view):
        if view.action not in ('create', 'update'):
            return True
        if not request.user.is_authenticated:
            return False
        if not request.auth:
            return True
        return super(CreateOnlyWithScopePermission, self).has_permission(
            request, view
        )

class CreateFoiRequestFollowSerializer(serializers.ModelSerializer):
    request = serializers.PrimaryKeyRelatedField(queryset=FoiRequest.objects.none())

    class Meta:
        model = FoiRequestFollower
        fields = ('request',)

    def __init__(self, *args, **kwargs):
        super(CreateFoiRequestFollowSerializer, self).__init__(
            *args, **kwargs
        )
        qs = FoiRequestFollower.objects.get_followable_requests(
            self.context['view'].request.user
        )
        self.fields['request'].queryset = qs

    def validate_request(self, value):
        """
        Check that the blog post is about Django.
        """
        if value.user == self.context['view'].request.user:
            raise serializers.ValidationError('Cannot follow your own requests')
        return value

    def create(self, validated_data):
        follower, create = FoiRequestFollower.objects.get_or_create(
            request=validated_data['request'],
            user=validated_data['user'],
            defaults={
                'timestamp': timezone.now()
            }
        )
        return follower

class FoiRequestFollowSerializer(serializers.HyperlinkedModelSerializer):
    request = serializers.HyperlinkedRelatedField(
        view_name='api:request-detail',
        lookup_field='pk',
        read_only=True
    )
    resource_uri = serializers.HyperlinkedIdentityField(
        view_name='api:following-detail',
        lookup_field='pk',
        required=False
    )
    request_url = serializers.SerializerMethodField(
        source='get_absolute_domain_url',
        read_only=True
    )
    follow_count = serializers.IntegerField(read_only=True)
    follows = serializers.BooleanField(read_only=True)
    can_follow = serializers.BooleanField(read_only=True)

    class Meta:
        model = FoiRequestFollower
        fields = ('resource_uri', 'request', 'request_url',
                  'timestamp', 'follow_count', 'follows', 'can_follow')

    def get_request_url(self, obj):
        return get_absolute_domain_short_url(obj.request_id)

class FoiRequestFollowRequestSerializer(serializers.HyperlinkedModelSerializer):
    request = serializers.HyperlinkedIdentityField(
        view_name='api:request-detail',
        lookup_field='pk',
        read_only=True
    )
    resource_uri = serializers.HyperlinkedRelatedField(
        view_name='api:following-detail',
        lookup_field='pk',
        read_only=True
    )
    request_url = serializers.SerializerMethodField(
        source='get_absolute_domain_url',
        read_only=True
    )
    follow_count = serializers.IntegerField(read_only=True)
    follows = serializers.BooleanField(read_only=True)
    can_follow = serializers.BooleanField(read_only=True)

    class Meta:
        model = FoiRequest
        fields = ('request', 'request_url', 'resource_uri',
                  'follow_count', 'follows', 'can_follow')

    def get_request_url(self, obj):
        return get_absolute_domain_short_url(obj.id)

class LargeResultsSetPagination(CustomLimitOffsetPagination):
    default_limit = 150
    max_limit = 200

class FoiRequestFollowerViewSet(mixins.CreateModelMixin,
                        mixins.ListModelMixin,
                        mixins.DestroyModelMixin,
                        viewsets.GenericViewSet):
    serializer_class = FoiRequestFollowSerializer
    permission_classes = (CreateOnlyWithScopePermission,)
    read_scopes = ['read:request']
    required_scopes = ['follow:request']
    pagination_class = LargeResultsSetPagination

    def get_serializer_class(self):
        if self.action == 'create':
            return CreateFoiRequestFollowSerializer
        if self.get_request_filter():
            return FoiRequestFollowRequestSerializer
        return self.serializer_class

    def get_foirequest_queryset(self, requests=None):
        user = self.request.user
        token = self.request.auth
        qs = filter_foirequests(user, token)
        if requests is not None:
            qs = qs.filter(id__in=requests)
        if user.is_authenticated and (
                not token or token.is_valid(self.read_scopes)):
            follows = (
                FoiRequestFollower.objects
                .filter(request_id=OuterRef('pk'), user=user)
            )
            qs = qs.annotate(
                follows=Exists(follows),
                resource_uri=Subquery(follows.values('pk')),
                can_follow=Case(
                    When(user_id=user.id, then=Value(False)),
                    default=Value(True),
                    output_field=NullBooleanField()
                )
            )
        else:
            qs = qs.annotate(
                follows=Value(None, output_field=NullBooleanField()),
                can_follow=Value(None, output_field=NullBooleanField())
            )
        qs = qs.annotate(
            follow_count=Count('followers'),
        )
        return qs

    def get_request_filter(self):
        if not hasattr(self, '_requests_filter'):
            requests = self.request.query_params.get('request', '').split(',')
            try:
                requests = [int(r) for r in requests]
            except ValueError:
                requests = []
            self._requests_filter = requests
        return self._requests_filter

    def get_queryset(self):
        user = self.request.user
        token = self.request.auth
        requests = self.get_request_filter()
        qs = None
        if requests:
            return self.get_foirequest_queryset(requests=requests)
        elif user.is_authenticated:
            if not token or token.is_valid(self.read_scopes):
                qs = (
                    FoiRequestFollower.objects
                    .filter(user=user)
                    .order_by('-timestamp')
                )
        if qs is None:
            qs = FoiRequestFollower.objects.none()

        follower_count = (
            FoiRequest.objects
            .filter(id=OuterRef('request'))
            .values('pk')
            .annotate(count=Count('followers'))
        )
        qs = qs.annotate(
            follow_count=Subquery(follower_count.values('count')),
            # follows by query definition
            follows=Value(True, output_field=NullBooleanField()),
            # can_follow by query definition
            can_follow=Value(True, output_field=NullBooleanField())
        )
        return qs

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        instance = self.perform_create(serializer)
        data = {
            'status': 'success',
            'url': reverse('api:following-detail', kwargs={'pk': instance.pk},
                           request=request)
        }
        headers = {'Location': data['url']}
        return Response(data, status=status.HTTP_201_CREATED, headers=headers)

    def perform_create(self, serializer):
        return serializer.save(user=self.request.user)
<|endoftext|>


<commit_before>import ast
import copy

from napalm_yang import helpers

def _flatten_dictionary(obj, path, key_name):
    """
    This method tries to use the path `?my_field` to convert:

    a:
        aa: 1
        ab: 2
    b:
        ba: 3
        ba: 4

    into:

    - my_field: a
      aa: 1
      ab: 2
    - my_field: b
      ba: 3
      ba: 4
    """
    result = []
    for k, v in obj.items():
        if path:
            if k == path[0]:
                path.pop(0)
        if k.startswith("#"):
            continue
        r = _resolve_path(v, list(path))

        if isinstance(r, dict):
            # You can either have a dict here, which means your path is like ?a.b
            # or a list, which means you have a path like ?a.?b
            r = [r]

        for e in r:
            e[key_name] = k
            result.append(e)
    return result

def _flatten_list(obj, path, key_name):
    new_key, key = key_name.split(":")
    result = []

    for o in obj:
        r = _resolve_path(o, list(path))

        if isinstance(r, dict):
            # XML gets confused when there is a list with only one element
            # and is treated as a dict when converting to a dict
            r = [r]
        old_value = o.pop(key)
        for e in r:
            e[new_key] = old_value
            result.append(e)

    return result

def _resolve_path(obj, path):
    if not path:
        return obj
    current = path.pop(0)
    if current[0] != "?":
        if isinstance(obj, dict):
            obj = obj[current]
        elif isinstance(obj, list):
            obj = obj[int(current)]
        return _resolve_path(obj, path)
    else:
        if isinstance(obj, dict) and ":" in current:
            # We assume that this is supposed to be a list but xmldict
            # turned a one element list into a dict
            obj = [obj]

        if isinstance(obj, dict):
            return _flatten_dictionary(obj, path, current[1:])
        elif isinstance(obj, list):
            return _flatten_list(obj, path, current[1:])

class BaseParser(object):

    def __init__(self, keys, extra_vars):
        self.keys = keys
        self.extra_vars = extra_vars

    def resolve_path(self, obj, path, default=None, check_presence=False):
        if path == "":
            return obj
        path = path.split(".")
        try:
            r = _resolve_path(obj, path)
        except (KeyError, IndexError):
            return default

        if check_presence:
            return not bool(path)
        return r

    def init_native(self, native):
        return native

    def parse_list(self, attribute, mapping, bookmarks):
        mapping = copy.deepcopy(mapping)
        mapping = helpers.resolve_rule(mapping, attribute, self.keys, self.extra_vars,
                                       None, process_all=False)
        for m in mapping:
            # parent will change as the tree is processed so we save it
            # so we can restore it
            parent = bookmarks["parent"]

            data = self.resolve_path(bookmarks, m.get("from", "parent"))

            for key, block, extra_vars in self._parse_list_default(attribute, m, data):
                yield key, block, extra_vars

            # we restore the parent
            bookmarks["parent"] = parent

    def parse_leaf(self, attribute, mapping, bookmarks):
        mapping = helpers.resolve_rule(mapping, attribute, self.keys,
                                       self.extra_vars, None, process_all=False)
        for m in mapping:
            data = self.resolve_path(bookmarks, m.get("from", "parent"))
            result = self._parse_leaf_default(attribute, m, data)

            if result is not None:
                try:
                    result = ast.literal_eval(result)
                except (ValueError, SyntaxError):
                    pass
                return result

    def parse_container(self, attribute, mapping, bookmarks):
        mapping = helpers.resolve_rule(mapping, attribute, self.keys, self.extra_vars, None,
                                       process_all=False)
        for m in mapping:
            # parent will change as the tree is processed so we save it
            # so we can restore it
            parent = bookmarks["parent"]
            data = self.resolve_path(bookmarks, m.get("from", "parent"))
            result, extra_vars = self._parse_container_default(attribute, m, data)
            if result or extra_vars:
                break

            # we restore the parent
            bookmarks["parent"] = parent
        return result, extra_vars

    def _parse_post_process_filter(self, post_process_filter, **kwargs):
        kwargs.update(self.keys)
        return helpers.template(post_process_filter, extra_vars=self.extra_vars, **kwargs)
<commit_msg>a rule can be debug by setting `pdb: true`<commit_after>import ast
import copy

from napalm_yang import helpers

def _flatten_dictionary(obj, path, key_name):
    """
    This method tries to use the path `?my_field` to convert:

    a:
        aa: 1
        ab: 2
    b:
        ba: 3
        ba: 4

    into:

    - my_field: a
      aa: 1
      ab: 2
    - my_field: b
      ba: 3
      ba: 4
    """
    result = []
    for k, v in obj.items():
        if path:
            if k == path[0]:
                path.pop(0)
        if k.startswith("#"):
            continue
        r = _resolve_path(v, list(path))

        if isinstance(r, dict):
            # You can either have a dict here, which means your path is like ?a.b
            # or a list, which means you have a path like ?a.?b
            r = [r]

        for e in r:
            e[key_name] = k
            result.append(e)
    return result

def _flatten_list(obj, path, key_name):
    new_key, key = key_name.split(":")
    result = []

    for o in obj:
        r = _resolve_path(o, list(path))

        if isinstance(r, dict):
            # XML gets confused when there is a list with only one element
            # and is treated as a dict when converting to a dict
            r = [r]
        old_value = o.pop(key)
        for e in r:
            e[new_key] = old_value
            result.append(e)

    return result

def _resolve_path(obj, path):
    if not path:
        return obj
    current = path.pop(0)
    if current[0] != "?":
        if isinstance(obj, dict):
            obj = obj[current]
        elif isinstance(obj, list):
            obj = obj[int(current)]
        return _resolve_path(obj, path)
    else:
        if isinstance(obj, dict) and ":" in current:
            # We assume that this is supposed to be a list but xmldict
            # turned a one element list into a dict
            obj = [obj]

        if isinstance(obj, dict):
            return _flatten_dictionary(obj, path, current[1:])
        elif isinstance(obj, list):
            return _flatten_list(obj, path, current[1:])

class BaseParser(object):

    def __init__(self, keys, extra_vars):
        self.keys = keys
        self.extra_vars = extra_vars

    def resolve_path(self, obj, path, default=None, check_presence=False):
        if path == "":
            return obj
        path = path.split(".")
        try:
            r = _resolve_path(obj, path)
        except (KeyError, IndexError):
            return default

        if check_presence:
            return not bool(path)
        return r

    def init_native(self, native):
        return native

    def parse_list(self, attribute, mapping, bookmarks):
        mapping = copy.deepcopy(mapping)
        mapping = helpers.resolve_rule(mapping, attribute, self.keys, self.extra_vars,
                                       None, process_all=False)
        for m in mapping:
            if m.get("pdb"):
                import pdb
                pdb.set_trace()
            # parent will change as the tree is processed so we save it
            # so we can restore it
            parent = bookmarks["parent"]

            data = self.resolve_path(bookmarks, m.get("from", "parent"))

            for key, block, extra_vars in self._parse_list_default(attribute, m, data):
                yield key, block, extra_vars

            # we restore the parent
            bookmarks["parent"] = parent

    def parse_leaf(self, attribute, mapping, bookmarks):
        mapping = helpers.resolve_rule(mapping, attribute, self.keys,
                                       self.extra_vars, None, process_all=False)
        for m in mapping:
            if m.get("pdb"):
                import pdb
                pdb.set_trace()
            data = self.resolve_path(bookmarks, m.get("from", "parent"))
            result = self._parse_leaf_default(attribute, m, data)

            if result is not None:
                try:
                    result = ast.literal_eval(result)
                except (ValueError, SyntaxError):
                    pass
                return result

    def parse_container(self, attribute, mapping, bookmarks):
        mapping = helpers.resolve_rule(mapping, attribute, self.keys, self.extra_vars, None,
                                       process_all=False)
        for m in mapping:
            if m.get("pdb"):
                import pdb
                pdb.set_trace()
            # parent will change as the tree is processed so we save it
            # so we can restore it
            parent = bookmarks["parent"]
            data = self.resolve_path(bookmarks, m.get("from", "parent"))
            result, extra_vars = self._parse_container_default(attribute, m, data)
            if result or extra_vars:
                break

            # we restore the parent
            bookmarks["parent"] = parent
        return result, extra_vars

    def _parse_post_process_filter(self, post_process_filter, **kwargs):
        kwargs.update(self.keys)
        return helpers.template(post_process_filter, extra_vars=self.extra_vars, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json
import subprocess
import sys
import time

from os.path import exists, join, split, dirname, abspath

from jupyter_client.kernelspec import (
    KernelSpecManager,
    KernelSpec,
    NATIVE_KERNEL_NAME,
)

CACHE_TIMEOUT = 60

class CondaKernelSpecManager(KernelSpecManager):
    """ A custom KernelSpecManager able to search for conda environments and
        create kernelspecs for them.
    """
    def __init__(self, **kwargs):
        super(CondaKernelSpecManager, self).__init__(**kwargs)

        self._conda_info_cache = None
        self._conda_info_cache_expiry = None

        self._conda_kernels_cache = None
        self._conda_kernels_cache_expiry = None

        self.log.info("[nb_conda_kernels] enabled, %s kernels found",
                      len(self._conda_kspecs))

    @property
    def _conda_info(self):
        """ Get and parse the whole conda information output

            Caches the information for CACHE_TIMEOUT seconds, as this is
            relatively expensive.
        """

        expiry = self._conda_info_cache_expiry

        if expiry is None or expiry < time.time():
            self.log.debug("[nb_conda_kernels] refreshing conda info")
            try:
                p = subprocess.check_output(["conda", "info", "--json"]
                                            ).decode("utf-8")
                conda_info = json.loads(p)
            except Exception as err:
                conda_info = None
                self.log.error("[nb_conda_kernels] couldn't call conda:\n%s",
                               err)
            self._conda_info_cache = conda_info
            self._conda_info_cache_expiry = time.time() + CACHE_TIMEOUT

        return self._conda_info_cache

    def _all_envs(self):
        """ Find the all the executables for each env where jupyter is
            installed.

            Returns a dict with the env names as keys and info about the kernel
            specs, including the paths to the lang executable in each env as
            value if jupyter is installed in that env.

            Caches the information for CACHE_TIMEOUT seconds, as this is
            relatively expensive.
        """
        # play safe with windows
        if sys.platform.startswith('win'):
            python = join("python.exe")
            r = join("Scripts", "R.exe")
            jupyter = join("Scripts", "jupyter.exe")
        else:
            python = join("bin", "python")
            r = join("bin", "R")
            jupyter = join("bin", "jupyter")

        def get_paths_by_env(display_prefix, language_key, language_exe, envs):
            """ Get a dict with name_env:info for kernel executables
            """
            language_envs = {}
            for base in envs:
                exe_path = join(base, language_exe)
                if exists(join(base, jupyter)) and exists(exe_path):
                    env_name = split(base)[1]
                    name = 'conda-env-{}-{}'.format(env_name, language_key)
                    language_envs[name] = {
                        'display_name': '{} [conda env:{}]'.format(
                            display_prefix, env_name),
                        'executable': exe_path,
                        'language_key': language_key,
                    }
            return language_envs

        # Collect all the envs in one dict
        all_envs = {}

        # Get the python envs
        python_envs = get_paths_by_env("Python", "py", python,
                                       self._conda_info["envs"])
        all_envs.update(python_envs)

        # Get the R envs
        r_envs = get_paths_by_env("R", "r", r, self._conda_info["envs"])
        all_envs.update(r_envs)

        # We also add the root prefix into the soup
        root_prefix = join(self._conda_info["root_prefix"], jupyter)
        if exists(root_prefix):
            all_envs.update({
                'conda-root-py': {
                    'display_name': 'Python [conda root]',
                    'executable': join(self._conda_info["root_prefix"],
                                       python),
                    'language_key': 'py',
                }
            })
        # Use Jupyter's default kernel name ('python2' or 'python3') for
        # current env
        if exists(join(sys.prefix, jupyter)) and exists(join(sys.prefix,
                                                             python)):
            all_envs.update({
                NATIVE_KERNEL_NAME: {
                    'display_name': 'Python [default]',
                    'executable': join(sys.prefix, python),
                    'language_key': 'py',
                }
            })

        return all_envs

    @property
    def _conda_kspecs(self):
        """ Get (or refresh) the cache of conda kernels
        """
        if self._conda_info is None:
            return {}

        if (
            self._conda_kernels_cache_expiry is None or
            self._conda_kernels_cache_expiry < time.time()
           ):
            self.log.debug("[nb_conda_kernels] refreshing conda kernelspecs")
            self._conda_kernels_cache = self._load_conda_kspecs()
            self._conda_kernels_cache_expiry = time.time() + CACHE_TIMEOUT

        return self._conda_kernels_cache

    def _load_conda_kspecs(self):
        """ Create a kernelspec for each of the envs where jupyter is installed
        """
        kspecs = {}
        for name, info in self._all_envs().items():
            executable = info['executable']
            display_name = info['display_name']

            if info['language_key'] == 'py':
                kspec = {
                    "argv": [executable, "-m", "ipykernel", "-f",
                             "{connection_file}"],
                    "display_name": display_name,
                    "language": "python",
                    "env": {},
                    "resource_dir": join(dirname(abspath(__file__)), "logos",
                                         "python")
                 }
            elif info['language_key'] == 'r':
                kspec = {
                    "argv": [executable, "--slave", "-e", "IRkernel::main()",
                             "--args", "{connection_file}"],
                    "display_name": display_name,
                    "language": "R",
                    "env": {},
                    "resource_dir": join(dirname(abspath(__file__)), "logos",
                                         "r")
                }

            kspecs.update({
                name: KernelSpec(**kspec)
            })

        return kspecs

    def find_kernel_specs(self):
        """ Returns a dict mapping kernel names to resource directories.

            The update process also adds the resource dir for the conda
            environments.
        """
        kspecs = super(CondaKernelSpecManager, self).find_kernel_specs()

        # add conda envs kernelspecs
        kspecs.update({name: spec.resource_dir
                       for name, spec
                       in self._conda_kspecs.items()})
        return kspecs

    def get_kernel_spec(self, kernel_name):
        """ Returns a :class:`KernelSpec` instance for the given kernel_name.

            Additionally, conda kernelspecs are generated on the fly
            accordingly with the detected envitonments.
        """

        return (
            self._conda_kspecs.get(kernel_name) or
            super(CondaKernelSpecManager, self).get_kernel_spec(kernel_name)
        )
<commit_msg>Support CondaKernelSpecManager.env_filter regex<commit_after># -*- coding: utf-8 -*-
import json
import re
import subprocess
import sys
import time

from os.path import exists, join, split, dirname, abspath
from traitlets import Unicode

from jupyter_client.kernelspec import (
    KernelSpecManager,
    KernelSpec,
    NATIVE_KERNEL_NAME,
)

CACHE_TIMEOUT = 60

class CondaKernelSpecManager(KernelSpecManager):
    """ A custom KernelSpecManager able to search for conda environments and
        create kernelspecs for them.
    """
    env_filter = Unicode(None, config=True, allow_none=True,
                         help="Do not list environment names that match this regex")

    def __init__(self, **kwargs):
        super(CondaKernelSpecManager, self).__init__(**kwargs)

        self._conda_info_cache = None
        self._conda_info_cache_expiry = None

        self._conda_kernels_cache = None
        self._conda_kernels_cache_expiry = None

        if self.env_filter is not None:
            self._env_filter_regex = re.compile(self.env_filter)

        self.log.info("[nb_conda_kernels] enabled, %s kernels found",
                      len(self._conda_kspecs))

    @property
    def _conda_info(self):
        """ Get and parse the whole conda information output

            Caches the information for CACHE_TIMEOUT seconds, as this is
            relatively expensive.
        """

        expiry = self._conda_info_cache_expiry

        if expiry is None or expiry < time.time():
            self.log.debug("[nb_conda_kernels] refreshing conda info")
            try:
                p = subprocess.check_output(["conda", "info", "--json"]
                                            ).decode("utf-8")
                conda_info = json.loads(p)
            except Exception as err:
                conda_info = None
                self.log.error("[nb_conda_kernels] couldn't call conda:\n%s",
                               err)
            self._conda_info_cache = conda_info
            self._conda_info_cache_expiry = time.time() + CACHE_TIMEOUT

        return self._conda_info_cache

    def _skip_env(self, path):
        """Get whether the environment should be included in the kernel specs or
        not based on whether its path matches env_filter.

        If the filter regex is None, always returns False (i.e., never skips).

        Parameters
        ----------
        path: str
            Full path of the conda environment

        Returns
        -------
        bool
            True if the filter matches and the env should not be included in the
            kernel specs.
        """
        if self.env_filter is None:
            return False
        return self._env_filter_regex.search(path) is not None

    def _all_envs(self):
        """ Find the all the executables for each env where jupyter is
            installed.

            Returns a dict with the env names as keys and info about the kernel
            specs, including the paths to the lang executable in each env as
            value if jupyter is installed in that env.

            Caches the information for CACHE_TIMEOUT seconds, as this is
            relatively expensive.
        """
        # play safe with windows
        if sys.platform.startswith('win'):
            python = join("python.exe")
            r = join("Scripts", "R.exe")
            jupyter = join("Scripts", "jupyter.exe")
        else:
            python = join("bin", "python")
            r = join("bin", "R")
            jupyter = join("bin", "jupyter")

        def get_paths_by_env(display_prefix, language_key, language_exe, envs):
            """ Get a dict with name_env:info for kernel executables
            """
            language_envs = {}
            for base in envs:
                exe_path = join(base, language_exe)
                if exists(join(base, jupyter)) and exists(exe_path) and not self._skip_env(base):
                    env_name = split(base)[1]
                    name = 'conda-env-{}-{}'.format(env_name, language_key)
                    language_envs[name] = {
                        'display_name': '{} [conda env:{}]'.format(
                            display_prefix, env_name),
                        'executable': exe_path,
                        'language_key': language_key,
                    }
            return language_envs

        # Collect all the envs in one dict
        all_envs = {}

        # Get the python envs
        python_envs = get_paths_by_env("Python", "py", python,
                                       self._conda_info["envs"])
        all_envs.update(python_envs)

        # Get the R envs
        r_envs = get_paths_by_env("R", "r", r, self._conda_info["envs"])
        all_envs.update(r_envs)

        # We also add the root prefix into the soup
        root_prefix = join(self._conda_info["root_prefix"], jupyter)
        if exists(root_prefix):
            all_envs.update({
                'conda-root-py': {
                    'display_name': 'Python [conda root]',
                    'executable': join(self._conda_info["root_prefix"],
                                       python),
                    'language_key': 'py',
                }
            })
        # Use Jupyter's default kernel name ('python2' or 'python3') for
        # current env
        if exists(join(sys.prefix, jupyter)) and exists(join(sys.prefix,
                                                             python)):
            all_envs.update({
                NATIVE_KERNEL_NAME: {
                    'display_name': 'Python [default]',
                    'executable': join(sys.prefix, python),
                    'language_key': 'py',
                }
            })

        return all_envs

    @property
    def _conda_kspecs(self):
        """ Get (or refresh) the cache of conda kernels
        """
        if self._conda_info is None:
            return {}

        if (
            self._conda_kernels_cache_expiry is None or
            self._conda_kernels_cache_expiry < time.time()
           ):
            self.log.debug("[nb_conda_kernels] refreshing conda kernelspecs")
            self._conda_kernels_cache = self._load_conda_kspecs()
            self._conda_kernels_cache_expiry = time.time() + CACHE_TIMEOUT

        return self._conda_kernels_cache

    def _load_conda_kspecs(self):
        """ Create a kernelspec for each of the envs where jupyter is installed
        """
        kspecs = {}
        for name, info in self._all_envs().items():
            executable = info['executable']
            display_name = info['display_name']

            if info['language_key'] == 'py':
                kspec = {
                    "argv": [executable, "-m", "ipykernel", "-f",
                             "{connection_file}"],
                    "display_name": display_name,
                    "language": "python",
                    "env": {},
                    "resource_dir": join(dirname(abspath(__file__)), "logos",
                                         "python")
                 }
            elif info['language_key'] == 'r':
                kspec = {
                    "argv": [executable, "--slave", "-e", "IRkernel::main()",
                             "--args", "{connection_file}"],
                    "display_name": display_name,
                    "language": "R",
                    "env": {},
                    "resource_dir": join(dirname(abspath(__file__)), "logos",
                                         "r")
                }

            kspecs.update({
                name: KernelSpec(**kspec)
            })

        return kspecs

    def find_kernel_specs(self):
        """ Returns a dict mapping kernel names to resource directories.

            The update process also adds the resource dir for the conda
            environments.
        """
        kspecs = super(CondaKernelSpecManager, self).find_kernel_specs()

        # add conda envs kernelspecs
        kspecs.update({name: spec.resource_dir
                       for name, spec
                       in self._conda_kspecs.items()})
        return kspecs

    def get_kernel_spec(self, kernel_name):
        """ Returns a :class:`KernelSpec` instance for the given kernel_name.

            Additionally, conda kernelspecs are generated on the fly
            accordingly with the detected envitonments.
        """

        return (
            self._conda_kspecs.get(kernel_name) or
            super(CondaKernelSpecManager, self).get_kernel_spec(kernel_name)
        )
<|endoftext|>