<commit_before>from __future__ import division
import numpy as np

def iterateToRoot(x0, f, df=None, steptol=1e-12, roottol=1e-12, maxIter=20, attemptBest=False):
	# iterate to a root using initial point x0
	if df is not None:
		try:
			# uses Newton-Raphson method if f and df are given.

			# SciPy implementation
			# import scipy.optimize
			# root = scipy.optimize.newton(f, x0, df, tol=steptol, maxiter=maxIter)
			# err = abs(f(root))

			root, err = newton(x0, f, df, steptol, 0, maxIter)

		except (RuntimeError, OverflowError):
			return None
	else:
		### For profiling number of function calls:
		# fcalls = [0,0]
		# def secant_f(z):
		# 	fcalls[0] += 1
		# 	return f(z)
		# def muller_f(z):
		# 	fcalls[1] += 1
		# 	return f(z)

		# # Secant method:
		# x1, x2 = x0, x0 + (x0+1)*1e-8
		# root, err = secant(x1, x2, secant_f, steptol, 0, maxIter)
		# print('-----')
		# print('Secant:', fcalls[0], root, err)

		# # Muller's method:
		# x1, x2, x3 = x0, x0*(1 + 1e-8) + 1e-8, x0*(1 - 1e-8) - 1e-8
		# root, err = muller(x1, x2, x3, muller_f, steptol, 0, maxIter)
		# print('Muller:', fcalls[1], root, err)
		# print('-----')
		############################################

		# Muller's method:
		f_muller = lambda z: complex(f(z))
		x1, x2, x3 = x0, x0*(1 + 1e-8) + 1e-8, x0*(1 - 1e-8) - 1e-8
		root, err = muller(x1, x2, x3, f_muller, steptol, 0, maxIter, attemptBest)

	if err < roottol:
		return root

def muller(x1, x2, x3, f, steptol=1e-12, roottol=1e-12, maxIter=20, attemptBest=False, verbose=False, callback=None):
	"""
	Wrapper for mpmath's implementation of Muller's method.

	Parameters
	----------
	x1 : float or complex
		An initial point for iteration, should be close to a
		root of f.
	x2 : float or complex
		An initial point for iteration, should be close to a
		root of f.  Should not equal x1.
	x3 : float or complex
		An initial point for iteration, should be close to a
		root of f.  Should not equal x1 or x2.
	f : function
		Function of a single variable f(x)
	steptol: float, optional
		Routine will end if the step size, dx, between sucessive
		iterations of x satisfies abs(dx) < steptol and attemptBest is False
	roottol: float, optional
		The routine will end if abs(f(x)) < roottol and attemptBest is False
	maxIter : int, optional
		Routine ends after maxIter iterations.
	attemptBest : bool, optional
		If True then routine will exit if error of the previous iteration, x0, was
		at least as good as the current iteration, x, in the sense that
		abs(f(x)) >= abs(f(x0)) and the previous iteration satisfied either
		abs(dx0) < steptol or abs(f(x0)) < roottol.  In this case the preivous
		iteration is returned as the approximation of the root.
	verbose : bool, optional
		Passed to mpmath's Muller function.
	callback : function, optional
		After each iteration the supplied function
		callback(x, dx, f(x), iteration) will be called where 'x' is the current iteration
		of the estimated root, 'dx' is the step size between the previous
		and current 'x' and 'iteration' the number of iterations that have been taken.
		If the callback function evaluates to True then the routine will end

	Returns
	-------
	xf : float
		The approximation to a root of f
	rooterr : float
		The error of the original function at xf, abs(f(xf))
	"""
	from mpmath import mp, mpmathify
	from mpmath.calculus.optimization import Muller

	# mpmath insists on functions accepting mpc
	f_mpmath = lambda z: mpmathify(f(complex(z)))

	mull = Muller(mp, f_mpmath, (x1, x2, x3), verbose=verbose)
	iteration = 0
	x0 = x3

	err0, dx0 = np.inf, np.inf
	try:
		for x, dx in mull:
			err = abs(f_mpmath(x))

			if callback is not None and callback(x, dx, err, iteration+1):
				break

			if not attemptBest and (abs(dx) < steptol or err < roottol) or iteration > maxIter:
				break

			if attemptBest and (abs(dx0) < steptol or err0 < roottol) and err >= err0:
				# The previous iteration was a better appproximation the current one so
				# assume that that was as close to the root as we are going to get.
				x, err = x0, err0
				break

			iteration += 1
			x0 = x

			if attemptBest:
				# record previous error for comparison
				dx0, err0 = dx0, err

	except ZeroDivisionError:
		# ZeroDivisionError comes up if the error is evaluated to be zero
		pass

	# cast mpc and mpf back to regular complex and float
	return complex(x), float(err)

def newton(x0, f, df, steptol=1e-12, roottol=1e-12, maxIter=20, attemptBest=False, callback=None):
	"""
	Find an approximation to a point xf such that f(xf)=0 for a
	scalar function f using Newton-Raphson iteration starting at
	the point x0.

	Parameters
	----------
	x0 : float or complex
		Initial point for Newton iteration, should be as close as
		possible to a root of f
	f : function
		Function of a single variable f(x)
	df : function
		Function of a single variable, df(x), providing the
		derivative of the function f(x) at the point x
	steptol: float, optional
		Routine will end if the step size, dx, between sucessive
		iterations of x satisfies abs(dx) < steptol
	roottol: float, optional
		The routine will end if abs(f(x)) < roottol
	maxIter : int, optional
		Routine ends after maxIter iterations
	attemptBest : bool, optional
		If True then routine will exit if error of the previous iteration, x0, was
		at least as good as the current iteration, x, in the sense that
		abs(f(x)) >= abs(f(x0)) and the previous iteration satisfied either
		abs(dx0) < steptol or abs(f(x0)) < roottol.  In this case the preivous
		iteration is returned as the approximation of the root.
	callback : function, optional
		After each iteration the supplied function
		callback(x, dx, f(x), iteration) will be called where 'x' is the current iteration
		of the estimated root, 'dx' is the step size between the previous
		and current 'x' and 'iteration' the number of iterations that have been taken.
		If the callback function evaluates to True then the routine will end

	Returns
	-------
	xf : float
		The approximation to a root of f
	rooterr : float
		The error of the original function at xf, abs(f(xf))
	"""

	# XXX: Could use deflated polynomials to ensure that known roots are not found again?

	x, y = x0, f(x0)
	dx0, y0 = np.inf, y
	for iteration in range(maxIter):
		dx = -y/df(x)
		x += dx
		y  = f(x)

		if callback is not None and callback(x, dx, y, iteration+1):
			break

		if not attemptBest and (abs(dx) < steptol or abs(y) < roottol):
			break

		if attemptBest and (abs(dx0) < steptol or abs(y0) < roottol) and abs(y) > abs(y0):
			break

		if attemptBest:
			# store previous dx and y
			dx0, y0 = dx, y

	return x, abs(y)

def secant(x1, x2, f, steptol=1e-12, roottol=1e-12, maxIter=30, callback=None):
	"""
	Find an approximation to a point xf such that f(xf)=0 for a
	scalar function f using the secant method.  The method requires
	two initial points x1 and x2, ideally close to a root
	and proceeds iteratively.

	Parameters
	----------
	x1 : float or complex
		An initial point for iteration, should be close to a
		root of f.
	x2 : float or complex
		An initial point for iteration, should be close to a
		root of f.  Should not equal x1.
	f : function
		Function of a single variable f(x)
	steptol: float, optional
		Routine will end if the step size, dx, between sucessive
		iterations of x satisfies abs(dx) < steptol
	roottol: float, optional
		The routine will end if abs(f(x)) < roottol
	maxIter : int, optional
		Routine ends after maxIter iterations
	callback : function, optional
		After each iteration the supplied function
		callback(x, dx, f(x), iteration) will be called where 'x' is the current iteration
		of the estimated root, 'dx' is the step size between the previous
		and current 'x' and 'iteration' the number of iterations that have been taken.
		If the callback function evaluates to True then the routine will end

	Returns
	-------
	xf : float
		The approximation to a root of f
	rooterr : float
		The error of the original function at xf, abs(f(xf))
	"""

	# As in "Numerical Recipies 3rd Edition" pick the bound with the
	# smallest function value as the most recent guess
	y1, y2 = f(x1), f(x2)
	if abs(y1) < abs(y2):
		x1, x2 = x2, x1
		y1, y2 = y2, y1

	# print('--secant--')

	for iteration in range(maxIter):
		dx =  -(x2-x1)*y2/(y2-y1)
		x1, x2 = x2, x2 + dx
		y1, y2 = y2, f(x2)

		if callback is not None and callback(x2, dx, y2, iteration+1):
			break

		# print(iteration, x2, abs(dx), abs(y2))

		if abs(dx) < steptol or abs(y2) < roottol:
			break

	return x2, abs(y2)
<commit_msg>Initialise Muller x and err<commit_after>from __future__ import division
import numpy as np

def iterateToRoot(x0, f, df=None, steptol=1e-12, roottol=1e-12, maxIter=20, attemptBest=False):
	# iterate to a root using initial point x0
	if df is not None:
		try:
			# uses Newton-Raphson method if f and df are given.

			# SciPy implementation
			# import scipy.optimize
			# root = scipy.optimize.newton(f, x0, df, tol=steptol, maxiter=maxIter)
			# err = abs(f(root))

			root, err = newton(x0, f, df, steptol, 0, maxIter)

		except (RuntimeError, OverflowError):
			return None
	else:
		### For profiling number of function calls:
		# fcalls = [0,0]
		# def secant_f(z):
		# 	fcalls[0] += 1
		# 	return f(z)
		# def muller_f(z):
		# 	fcalls[1] += 1
		# 	return f(z)

		# # Secant method:
		# x1, x2 = x0, x0 + (x0+1)*1e-8
		# root, err = secant(x1, x2, secant_f, steptol, 0, maxIter)
		# print('-----')
		# print('Secant:', fcalls[0], root, err)

		# # Muller's method:
		# x1, x2, x3 = x0, x0*(1 + 1e-8) + 1e-8, x0*(1 - 1e-8) - 1e-8
		# root, err = muller(x1, x2, x3, muller_f, steptol, 0, maxIter)
		# print('Muller:', fcalls[1], root, err)
		# print('-----')
		############################################

		# Muller's method:
		f_muller = lambda z: complex(f(z))
		x1, x2, x3 = x0, x0*(1 + 1e-8) + 1e-8, x0*(1 - 1e-8) - 1e-8
		root, err = muller(x1, x2, x3, f_muller, steptol, 0, maxIter, attemptBest)

	if err < roottol:
		return root

def muller(x1, x2, x3, f, steptol=1e-12, roottol=1e-12, maxIter=20, attemptBest=False, verbose=False, callback=None):
	"""
	Wrapper for mpmath's implementation of Muller's method.

	Parameters
	----------
	x1 : float or complex
		An initial point for iteration, should be close to a
		root of f.
	x2 : float or complex
		An initial point for iteration, should be close to a
		root of f.  Should not equal x1.
	x3 : float or complex
		An initial point for iteration, should be close to a
		root of f.  Should not equal x1 or x2.
	f : function
		Function of a single variable f(x)
	steptol: float, optional
		Routine will end if the step size, dx, between sucessive
		iterations of x satisfies abs(dx) < steptol and attemptBest is False
	roottol: float, optional
		The routine will end if abs(f(x)) < roottol and attemptBest is False
	maxIter : int, optional
		Routine ends after maxIter iterations.
	attemptBest : bool, optional
		If True then routine will exit if error of the previous iteration, x0, was
		at least as good as the current iteration, x, in the sense that
		abs(f(x)) >= abs(f(x0)) and the previous iteration satisfied either
		abs(dx0) < steptol or abs(f(x0)) < roottol.  In this case the preivous
		iteration is returned as the approximation of the root.
	verbose : bool, optional
		Passed to mpmath's Muller function.
	callback : function, optional
		After each iteration the supplied function
		callback(x, dx, f(x), iteration) will be called where 'x' is the current iteration
		of the estimated root, 'dx' is the step size between the previous
		and current 'x' and 'iteration' the number of iterations that have been taken.
		If the callback function evaluates to True then the routine will end

	Returns
	-------
	xf : float
		The approximation to a root of f
	rooterr : float
		The error of the original function at xf, abs(f(xf))
	"""
	from mpmath import mp, mpmathify
	from mpmath.calculus.optimization import Muller

	# mpmath insists on functions accepting mpc
	f_mpmath = lambda z: mpmathify(f(complex(z)))

	mull = Muller(mp, f_mpmath, (x1, x2, x3), verbose=verbose)
	iteration = 0
	x0 = x3

	x, err = x0, abs(f(x0))
	err0, dx0 = np.inf, np.inf
	try:
		for x, dx in mull:
			err = abs(f_mpmath(x))

			if callback is not None and callback(x, dx, err, iteration+1):
				break

			if not attemptBest and (abs(dx) < steptol or err < roottol) or iteration > maxIter:
				break

			if attemptBest and (abs(dx0) < steptol or err0 < roottol) and err >= err0:
				# The previous iteration was a better appproximation the current one so
				# assume that that was as close to the root as we are going to get.
				x, err = x0, err0
				break

			iteration += 1
			x0 = x

			if attemptBest:
				# record previous error for comparison
				dx0, err0 = dx0, err

	except ZeroDivisionError:
		# ZeroDivisionError comes up if the error is evaluated to be zero
		pass

	# cast mpc and mpf back to regular complex and float
	return complex(x), float(err)

def newton(x0, f, df, steptol=1e-12, roottol=1e-12, maxIter=20, attemptBest=False, callback=None):
	"""
	Find an approximation to a point xf such that f(xf)=0 for a
	scalar function f using Newton-Raphson iteration starting at
	the point x0.

	Parameters
	----------
	x0 : float or complex
		Initial point for Newton iteration, should be as close as
		possible to a root of f
	f : function
		Function of a single variable f(x)
	df : function
		Function of a single variable, df(x), providing the
		derivative of the function f(x) at the point x
	steptol: float, optional
		Routine will end if the step size, dx, between sucessive
		iterations of x satisfies abs(dx) < steptol
	roottol: float, optional
		The routine will end if abs(f(x)) < roottol
	maxIter : int, optional
		Routine ends after maxIter iterations
	attemptBest : bool, optional
		If True then routine will exit if error of the previous iteration, x0, was
		at least as good as the current iteration, x, in the sense that
		abs(f(x)) >= abs(f(x0)) and the previous iteration satisfied either
		abs(dx0) < steptol or abs(f(x0)) < roottol.  In this case the preivous
		iteration is returned as the approximation of the root.
	callback : function, optional
		After each iteration the supplied function
		callback(x, dx, f(x), iteration) will be called where 'x' is the current iteration
		of the estimated root, 'dx' is the step size between the previous
		and current 'x' and 'iteration' the number of iterations that have been taken.
		If the callback function evaluates to True then the routine will end

	Returns
	-------
	xf : float
		The approximation to a root of f
	rooterr : float
		The error of the original function at xf, abs(f(xf))
	"""

	# XXX: Could use deflated polynomials to ensure that known roots are not found again?

	x, y = x0, f(x0)
	dx0, y0 = np.inf, y
	for iteration in range(maxIter):
		dx = -y/df(x)
		x += dx
		y  = f(x)

		if callback is not None and callback(x, dx, y, iteration+1):
			break

		if not attemptBest and (abs(dx) < steptol or abs(y) < roottol):
			break

		if attemptBest and (abs(dx0) < steptol or abs(y0) < roottol) and abs(y) > abs(y0):
			break

		if attemptBest:
			# store previous dx and y
			dx0, y0 = dx, y

	return x, abs(y)

def secant(x1, x2, f, steptol=1e-12, roottol=1e-12, maxIter=30, callback=None):
	"""
	Find an approximation to a point xf such that f(xf)=0 for a
	scalar function f using the secant method.  The method requires
	two initial points x1 and x2, ideally close to a root
	and proceeds iteratively.

	Parameters
	----------
	x1 : float or complex
		An initial point for iteration, should be close to a
		root of f.
	x2 : float or complex
		An initial point for iteration, should be close to a
		root of f.  Should not equal x1.
	f : function
		Function of a single variable f(x)
	steptol: float, optional
		Routine will end if the step size, dx, between sucessive
		iterations of x satisfies abs(dx) < steptol
	roottol: float, optional
		The routine will end if abs(f(x)) < roottol
	maxIter : int, optional
		Routine ends after maxIter iterations
	callback : function, optional
		After each iteration the supplied function
		callback(x, dx, f(x), iteration) will be called where 'x' is the current iteration
		of the estimated root, 'dx' is the step size between the previous
		and current 'x' and 'iteration' the number of iterations that have been taken.
		If the callback function evaluates to True then the routine will end

	Returns
	-------
	xf : float
		The approximation to a root of f
	rooterr : float
		The error of the original function at xf, abs(f(xf))
	"""

	# As in "Numerical Recipies 3rd Edition" pick the bound with the
	# smallest function value as the most recent guess
	y1, y2 = f(x1), f(x2)
	if abs(y1) < abs(y2):
		x1, x2 = x2, x1
		y1, y2 = y2, y1

	# print('--secant--')

	for iteration in range(maxIter):
		dx =  -(x2-x1)*y2/(y2-y1)
		x1, x2 = x2, x2 + dx
		y1, y2 = y2, f(x2)

		if callback is not None and callback(x2, dx, y2, iteration+1):
			break

		# print(iteration, x2, abs(dx), abs(y2))

		if abs(dx) < steptol or abs(y2) < roottol:
			break

	return x2, abs(y2)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from future import standard_library

standard_library.install_aliases()
from contextlib import contextmanager
import difflib
import fnmatch
import os
import re
import io
import shutil
import tempfile
import textwrap
import zipfile

import requests

import xml.etree.ElementTree as ET

CUMULUSCI_PATH = os.path.realpath(
    os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
)
META_XML_CLEAN_DIRS = ("classes/", "triggers/", "pages/", "aura/", "components/")

def findReplace(find, replace, directory, filePattern, logger=None, max=None):
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in fnmatch.filter(files, filePattern):
            filepath = os.path.join(path, filename)
            with open(filepath) as f:
                s = f.read()
            if max:
                s_updated = s.replace(find, replace, max)
            else:
                s_updated = s.replace(find, replace)
            if s != s_updated:
                if logger:
                    logger.info("Updating {}".format(filepath))
                with open(filepath, "w") as f:
                    f.write(s_updated)

def findReplaceRegex(find, replace, directory, filePattern, logger=None):
    pattern = re.compile(find)
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in fnmatch.filter(files, filePattern):
            filepath = os.path.join(path, filename)
            with open(filepath) as f:
                s = f.read()
            s_updated = pattern.sub(replace, s)
            if s != s_updated:
                if logger:
                    logger.info("Updating {}".format(filepath))
                with open(filepath, "w") as f:
                    f.write(s_updated)

def findRename(find, replace, directory, logger=None):
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in files:
            filepath = os.path.join(path, filename)
            if logger:
                logger.info("Renaming {}".format(filepath))
            os.rename(filepath, os.path.join(path, filename.replace(find, replace)))

def elementtree_parse_file(path):
    try:
        tree = ET.parse(path)
    except ET.ParseError as err:
        err.filename = path
        raise err
    return tree

def removeXmlElement(name, directory, file_pattern, logger=None):
    """ Recursively walk a directory and remove XML elements """
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in fnmatch.filter(files, file_pattern):
            filepath = os.path.join(path, filename)
            remove_xml_element_file(name, filepath)

def remove_xml_element_file(name, path):
    """ Remove XML elements from a single file """
    ET.register_namespace("", "http://soap.sforce.com/2006/04/metadata")
    tree = elementtree_parse_file(path)
    tree = remove_xml_element(name, tree)
    return tree.write(path, encoding="UTF-8")

def remove_xml_element_string(name, content):
    """ Remove XML elements from a string """
    ET.register_namespace("", "http://soap.sforce.com/2006/04/metadata")
    tree = ET.fromstring(content)
    tree = remove_xml_element(name, tree)
    clean_content = ET.tostring(tree, encoding="UTF-8")
    return clean_content

def remove_xml_element(name, tree):
    """ Removes XML elements from an ElementTree content tree """
    # root = tree.getroot()
    remove = tree.findall(
        ".//{{http://soap.sforce.com/2006/04/metadata}}{}".format(name)
    )
    if not remove:
        return tree

    parent_map = {c: p for p in tree.iter() for c in p}

    for elem in remove:
        parent = parent_map[elem]
        parent.remove(elem)

    return tree

def download_extract_zip(url, target=None, subfolder=None, headers=None):
    if not headers:
        headers = {}
    resp = requests.get(url, headers=headers)
    zip_content = io.BytesIO(resp.content)
    zip_file = zipfile.ZipFile(zip_content)
    if subfolder:
        zip_file = zip_subfolder(zip_file, subfolder)
    if target:
        zip_file.extractall(target)
        return
    return zip_file

def zip_subfolder(zip_src, path):
    if not path.endswith("/"):
        path = path + "/"

    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    for name in zip_src.namelist():
        if not name.startswith(path):
            continue

        content = zip_src.read(name)
        rel_name = name.replace(path, "", 1)

        if rel_name:
            zip_dest.writestr(rel_name, content)

    return zip_dest

def zip_inject_namespace(
    zip_src,
    namespace=None,
    managed=None,
    filename_token=None,
    namespace_token=None,
    namespaced_org=None,
    logger=None,
):
    """ Replaces %%%NAMESPACE%%% for all files and ___NAMESPACE___ in all
        filenames in the zip with the either '' if no namespace is provided
        or 'namespace__' if provided.
    """

    # Handle namespace and filename tokens
    if not filename_token:
        filename_token = "___NAMESPACE___"
    if not namespace_token:
        namespace_token = "%%%NAMESPACE%%%"
    if managed is True and namespace:
        namespace_prefix = namespace + "__"
    else:
        namespace_prefix = ""

    # Handle tokens %%%NAMESPACED_ORG%%% and ___NAMESPACED_ORG___
    namespaced_org_token = "%%%NAMESPACED_ORG%%%"
    namespaced_org_file_token = "___NAMESPACED_ORG___"
    namespaced_org = namespace_prefix if namespaced_org else ""

    # Handle token %%%NAMESPACE_OR_C%%% for lightning components
    namespace_or_c_token = "%%%NAMESPACE_OR_C%%%"
    namespace_or_c = namespace if managed and namespace else "c"

    # Handle token %%%NAMESPACED_ORG_OR_C%%%
    namespaced_org_or_c_token = "%%%NAMESPACED_ORG_OR_C%%%"
    namespaced_org_or_c = namespace if namespaced_org else "c"

    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)

    differ = difflib.Differ()

    for name in zip_src.namelist():
        orig_name = unicode(name)
        try:
            content = zip_src.read(name)
            orig_content = unicode(content)
            content = content.replace(namespace_token, namespace_prefix)
            if logger and content != orig_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACE%%% with "{}"'.format(name, namespace)
                )

            prev_content = unicode(content)
            content = content.replace(namespace_or_c_token, namespace_or_c)
            if logger and content != prev_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACE_OR_C%%% with "{}"'.format(
                        name, namespace_or_c
                    )
                )

            prev_content = unicode(content)
            content = content.replace(namespaced_org_token, namespaced_org)
            if logger and content != prev_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACED_ORG%%% with "{}"'.format(
                        name, namespaced_org
                    )
                )

            prev_content = unicode(content)
            content = content.replace(namespaced_org_or_c_token, namespaced_org_or_c)
            if logger and content != prev_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACED_ORG_OR_C%%% with "{}"'.format(
                        name, namespaced_org_or_c
                    )
                )

        except UnicodeDecodeError:
            # if we cannot decode the content, don't try and replace it.
            pass

        # Replace namespace token in file name
        name = name.replace(filename_token, namespace_prefix)
        name = name.replace(namespaced_org_file_token, namespaced_org)
        if logger and name != orig_name:
            logger.info("  {}: renamed to {}".format(orig_name, name))
        zip_dest.writestr(name, content)

    return zip_dest

def zip_strip_namespace(zip_src, namespace, logger=None):
    """ Given a namespace, strips 'namespace__' from all files and filenames
        in the zip
    """
    namespace_prefix = "{}__".format(namespace)
    lightning_namespace = "{}:".format(namespace)
    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    for name in zip_src.namelist():
        try:
            content = zip_src.read(name)
            content = unicode(content)
            content = content.replace(namespace_prefix, "")
            content = content.replace(lightning_namespace, "c:")
            name = name.replace(namespace_prefix, "")
        except UnicodeDecodeError:
            # if we cannot decode the content, don't try and replace it.
            pass
        zip_dest.writestr(name, content)
    return zip_dest

def zip_tokenize_namespace(zip_src, namespace, logger=None):
    """ Given a namespace, replaces 'namespace__' with %%%NAMESPACE%%% for all
        files and ___NAMESPACE___ in all filenames in the zip
    """
    if not namespace:
        return zip_src

    namespace_prefix = "{}__".format(namespace)
    lightning_namespace = "{}:".format(namespace)
    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    for name in zip_src.namelist():
        try:
            content = zip_src.read(name)
            content = unicode(content)
            content = content.replace(namespace_prefix, "%%%NAMESPACE%%%")
            content = content.replace(lightning_namespace, "%%%NAMESPACE_OR_C%%%")
            name = name.replace(namespace_prefix, "___NAMESPACE___")
        except UnicodeDecodeError:
            # if we cannot decode the content, don't try and replace it.
            pass
        zip_dest.writestr(name, content)
    return zip_dest

def zip_clean_metaxml(zip_src, logger=None):
    """ Given a zipfile, cleans all *-meta.xml files in the zip for
        deployment by stripping all <packageVersions/> elements
    """
    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    changed = []
    for name in zip_src.namelist():
        content = zip_src.read(name)
        if name.startswith(META_XML_CLEAN_DIRS) and name.endswith("-meta.xml"):
            try:
                content = content.decode("ascii")
            except UnicodeDecodeError:
                # if we cannot decode the content, don't try and replace it.
                pass
            else:
                clean_content = remove_xml_element_string("packageVersions", content)
                if clean_content != content:
                    changed.append(name)
                    content = clean_content
        zip_dest.writestr(name, content)
    if changed and logger:
        logger.info(
            "Cleaned package versions from {} meta.xml files".format(len(changed))
        )
    return zip_dest

def doc_task(task_name, task_config, project_config=None, org_config=None):
    """ Document a (project specific) task configuration in RST format. """
    from cumulusci.core.utils import import_class

    doc = []
    doc.append("{}\n==========================================\n".format(task_name))
    doc.append("**Description:** {}\n".format(task_config.description))
    doc.append("**Class::** {}\n".format(task_config.class_path))

    task_class = import_class(task_config.class_path)
    task_docs = textwrap.dedent(task_class.task_docs.strip("\n"))
    if task_docs:
        doc.append(task_docs)
    if task_class.task_options:
        doc.append("Options:\n------------------------------------------\n")
        defaults = task_config.options or {}
        for name, option in list(task_class.task_options.items()):
            default = defaults.get(name)
            if default:
                default = " **Default: {}**".format(default)
            else:
                default = ""
            if option.get("required"):
                doc.append(
                    "* **{}** *(required)*: {}{}".format(
                        name, option.get("description"), default
                    )
                )
            else:
                doc.append(
                    "* **{}**: {}{}".format(name, option.get("description"), default)
                )

    return "\n".join(doc)

def package_xml_from_dict(items, api_version, package_name=None):
    lines = []

    # Print header
    lines.append('<?xml version="1.0" encoding="UTF-8"?>')
    lines.append('<Package xmlns="http://soap.sforce.com/2006/04/metadata">')

    # Include package name if specified
    if package_name:
        lines.append("    <fullName>{}</fullName>".format(package_name))

    # Print types sections
    for md_type, members in sorted(items.items()):
        members.sort()
        lines.append("    <types>")
        for member in members:
            lines.append("        <members>{}</members>".format(member))
        lines.append("        <name>{}</name>".format(md_type))
        lines.append("    </types>")

    # Print footer
    lines.append("    <version>{0}</version>".format(api_version))
    lines.append("</Package>")

    return "\n".join(lines)

@contextmanager
def cd(path):
    """Context manager that changes to another directory
    """
    if not path:
        yield
        return
    cwd = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(cwd)

@contextmanager
def temporary_dir():
    """Context manager that creates a temporary directory and chdirs to it.

    When the context manager exits it returns to the previous cwd
    and deletes the temporary directory.
    """
    d = tempfile.mkdtemp()
    try:
        with cd(d):
            yield d
    finally:
        if os.path.exists(d):
            shutil.rmtree(d)

def in_directory(filepath, dirpath):
    """Returns a boolean for whether filepath is contained in dirpath.

    Normalizes the paths (e.g. resolving symlinks and ..)
    so this is the safe way to make sure a user-configured path
    is located inside the user's project repo.
    """
    filepath = os.path.realpath(filepath)
    dirpath = os.path.realpath(dirpath)
    return filepath.startswith(os.path.join(dirpath, ""))
<commit_msg>add a log message when stripping namespace<commit_after>from __future__ import unicode_literals
from future import standard_library

standard_library.install_aliases()
from contextlib import contextmanager
import difflib
import fnmatch
import os
import re
import io
import shutil
import tempfile
import textwrap
import zipfile

import requests

import xml.etree.ElementTree as ET

CUMULUSCI_PATH = os.path.realpath(
    os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
)
META_XML_CLEAN_DIRS = ("classes/", "triggers/", "pages/", "aura/", "components/")

def findReplace(find, replace, directory, filePattern, logger=None, max=None):
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in fnmatch.filter(files, filePattern):
            filepath = os.path.join(path, filename)
            with open(filepath) as f:
                s = f.read()
            if max:
                s_updated = s.replace(find, replace, max)
            else:
                s_updated = s.replace(find, replace)
            if s != s_updated:
                if logger:
                    logger.info("Updating {}".format(filepath))
                with open(filepath, "w") as f:
                    f.write(s_updated)

def findReplaceRegex(find, replace, directory, filePattern, logger=None):
    pattern = re.compile(find)
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in fnmatch.filter(files, filePattern):
            filepath = os.path.join(path, filename)
            with open(filepath) as f:
                s = f.read()
            s_updated = pattern.sub(replace, s)
            if s != s_updated:
                if logger:
                    logger.info("Updating {}".format(filepath))
                with open(filepath, "w") as f:
                    f.write(s_updated)

def findRename(find, replace, directory, logger=None):
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in files:
            filepath = os.path.join(path, filename)
            if logger:
                logger.info("Renaming {}".format(filepath))
            os.rename(filepath, os.path.join(path, filename.replace(find, replace)))

def elementtree_parse_file(path):
    try:
        tree = ET.parse(path)
    except ET.ParseError as err:
        err.filename = path
        raise err
    return tree

def removeXmlElement(name, directory, file_pattern, logger=None):
    """ Recursively walk a directory and remove XML elements """
    for path, dirs, files in os.walk(os.path.abspath(directory)):
        for filename in fnmatch.filter(files, file_pattern):
            filepath = os.path.join(path, filename)
            remove_xml_element_file(name, filepath)

def remove_xml_element_file(name, path):
    """ Remove XML elements from a single file """
    ET.register_namespace("", "http://soap.sforce.com/2006/04/metadata")
    tree = elementtree_parse_file(path)
    tree = remove_xml_element(name, tree)
    return tree.write(path, encoding="UTF-8")

def remove_xml_element_string(name, content):
    """ Remove XML elements from a string """
    ET.register_namespace("", "http://soap.sforce.com/2006/04/metadata")
    tree = ET.fromstring(content)
    tree = remove_xml_element(name, tree)
    clean_content = ET.tostring(tree, encoding="UTF-8")
    return clean_content

def remove_xml_element(name, tree):
    """ Removes XML elements from an ElementTree content tree """
    # root = tree.getroot()
    remove = tree.findall(
        ".//{{http://soap.sforce.com/2006/04/metadata}}{}".format(name)
    )
    if not remove:
        return tree

    parent_map = {c: p for p in tree.iter() for c in p}

    for elem in remove:
        parent = parent_map[elem]
        parent.remove(elem)

    return tree

def download_extract_zip(url, target=None, subfolder=None, headers=None):
    if not headers:
        headers = {}
    resp = requests.get(url, headers=headers)
    zip_content = io.BytesIO(resp.content)
    zip_file = zipfile.ZipFile(zip_content)
    if subfolder:
        zip_file = zip_subfolder(zip_file, subfolder)
    if target:
        zip_file.extractall(target)
        return
    return zip_file

def zip_subfolder(zip_src, path):
    if not path.endswith("/"):
        path = path + "/"

    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    for name in zip_src.namelist():
        if not name.startswith(path):
            continue

        content = zip_src.read(name)
        rel_name = name.replace(path, "", 1)

        if rel_name:
            zip_dest.writestr(rel_name, content)

    return zip_dest

def zip_inject_namespace(
    zip_src,
    namespace=None,
    managed=None,
    filename_token=None,
    namespace_token=None,
    namespaced_org=None,
    logger=None,
):
    """ Replaces %%%NAMESPACE%%% for all files and ___NAMESPACE___ in all
        filenames in the zip with the either '' if no namespace is provided
        or 'namespace__' if provided.
    """

    # Handle namespace and filename tokens
    if not filename_token:
        filename_token = "___NAMESPACE___"
    if not namespace_token:
        namespace_token = "%%%NAMESPACE%%%"
    if managed is True and namespace:
        namespace_prefix = namespace + "__"
    else:
        namespace_prefix = ""

    # Handle tokens %%%NAMESPACED_ORG%%% and ___NAMESPACED_ORG___
    namespaced_org_token = "%%%NAMESPACED_ORG%%%"
    namespaced_org_file_token = "___NAMESPACED_ORG___"
    namespaced_org = namespace_prefix if namespaced_org else ""

    # Handle token %%%NAMESPACE_OR_C%%% for lightning components
    namespace_or_c_token = "%%%NAMESPACE_OR_C%%%"
    namespace_or_c = namespace if managed and namespace else "c"

    # Handle token %%%NAMESPACED_ORG_OR_C%%%
    namespaced_org_or_c_token = "%%%NAMESPACED_ORG_OR_C%%%"
    namespaced_org_or_c = namespace if namespaced_org else "c"

    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)

    differ = difflib.Differ()

    for name in zip_src.namelist():
        orig_name = unicode(name)
        try:
            content = zip_src.read(name)
            orig_content = unicode(content)
            content = content.replace(namespace_token, namespace_prefix)
            if logger and content != orig_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACE%%% with "{}"'.format(name, namespace)
                )

            prev_content = unicode(content)
            content = content.replace(namespace_or_c_token, namespace_or_c)
            if logger and content != prev_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACE_OR_C%%% with "{}"'.format(
                        name, namespace_or_c
                    )
                )

            prev_content = unicode(content)
            content = content.replace(namespaced_org_token, namespaced_org)
            if logger and content != prev_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACED_ORG%%% with "{}"'.format(
                        name, namespaced_org
                    )
                )

            prev_content = unicode(content)
            content = content.replace(namespaced_org_or_c_token, namespaced_org_or_c)
            if logger and content != prev_content:
                logger.info(
                    '  {}: Replaced %%%NAMESPACED_ORG_OR_C%%% with "{}"'.format(
                        name, namespaced_org_or_c
                    )
                )

        except UnicodeDecodeError:
            # if we cannot decode the content, don't try and replace it.
            pass

        # Replace namespace token in file name
        name = name.replace(filename_token, namespace_prefix)
        name = name.replace(namespaced_org_file_token, namespaced_org)
        if logger and name != orig_name:
            logger.info("  {}: renamed to {}".format(orig_name, name))
        zip_dest.writestr(name, content)

    return zip_dest

def zip_strip_namespace(zip_src, namespace, logger=None):
    """ Given a namespace, strips 'namespace__' from all files and filenames
        in the zip
    """
    namespace_prefix = "{}__".format(namespace)
    lightning_namespace = "{}:".format(namespace)
    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    for name in zip_src.namelist():
        try:
            orig_content = zip_src.read(name)
            orig_content = unicode(orig_content)
            new_content = orig_content.replace(namespace_prefix, "")
            new_content = new_content.replace(lightning_namespace, "c:")
            name = name.replace(namespace_prefix, "")  # not...sure...this..gets...used
            if orig_content != new_content and logger:
                logger.info("  {file_name}: removed {namespace}")
        except UnicodeDecodeError:
            # if we cannot decode the content, don't try and replace it.
            new_content = zip_src.read(name)

        zip_dest.writestr(name, new_content)
    return zip_dest

def zip_tokenize_namespace(zip_src, namespace, logger=None):
    """ Given a namespace, replaces 'namespace__' with %%%NAMESPACE%%% for all
        files and ___NAMESPACE___ in all filenames in the zip
    """
    if not namespace:
        return zip_src

    namespace_prefix = "{}__".format(namespace)
    lightning_namespace = "{}:".format(namespace)
    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    for name in zip_src.namelist():
        try:
            content = zip_src.read(name)
            content = unicode(content)
            content = content.replace(namespace_prefix, "%%%NAMESPACE%%%")
            content = content.replace(lightning_namespace, "%%%NAMESPACE_OR_C%%%")
            name = name.replace(namespace_prefix, "___NAMESPACE___")
        except UnicodeDecodeError:
            # if we cannot decode the content, don't try and replace it.
            pass
        zip_dest.writestr(name, content)
    return zip_dest

def zip_clean_metaxml(zip_src, logger=None):
    """ Given a zipfile, cleans all *-meta.xml files in the zip for
        deployment by stripping all <packageVersions/> elements
    """
    zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
    changed = []
    for name in zip_src.namelist():
        content = zip_src.read(name)
        if name.startswith(META_XML_CLEAN_DIRS) and name.endswith("-meta.xml"):
            try:
                content = content.decode("ascii")
            except UnicodeDecodeError:
                # if we cannot decode the content, don't try and replace it.
                pass
            else:
                clean_content = remove_xml_element_string("packageVersions", content)
                if clean_content != content:
                    changed.append(name)
                    content = clean_content
        zip_dest.writestr(name, content)
    if changed and logger:
        logger.info(
            "Cleaned package versions from {} meta.xml files".format(len(changed))
        )
    return zip_dest

def doc_task(task_name, task_config, project_config=None, org_config=None):
    """ Document a (project specific) task configuration in RST format. """
    from cumulusci.core.utils import import_class

    doc = []
    doc.append("{}\n==========================================\n".format(task_name))
    doc.append("**Description:** {}\n".format(task_config.description))
    doc.append("**Class::** {}\n".format(task_config.class_path))

    task_class = import_class(task_config.class_path)
    task_docs = textwrap.dedent(task_class.task_docs.strip("\n"))
    if task_docs:
        doc.append(task_docs)
    if task_class.task_options:
        doc.append("Options:\n------------------------------------------\n")
        defaults = task_config.options or {}
        for name, option in list(task_class.task_options.items()):
            default = defaults.get(name)
            if default:
                default = " **Default: {}**".format(default)
            else:
                default = ""
            if option.get("required"):
                doc.append(
                    "* **{}** *(required)*: {}{}".format(
                        name, option.get("description"), default
                    )
                )
            else:
                doc.append(
                    "* **{}**: {}{}".format(name, option.get("description"), default)
                )

    return "\n".join(doc)

def package_xml_from_dict(items, api_version, package_name=None):
    lines = []

    # Print header
    lines.append('<?xml version="1.0" encoding="UTF-8"?>')
    lines.append('<Package xmlns="http://soap.sforce.com/2006/04/metadata">')

    # Include package name if specified
    if package_name:
        lines.append("    <fullName>{}</fullName>".format(package_name))

    # Print types sections
    for md_type, members in sorted(items.items()):
        members.sort()
        lines.append("    <types>")
        for member in members:
            lines.append("        <members>{}</members>".format(member))
        lines.append("        <name>{}</name>".format(md_type))
        lines.append("    </types>")

    # Print footer
    lines.append("    <version>{0}</version>".format(api_version))
    lines.append("</Package>")

    return "\n".join(lines)

@contextmanager
def cd(path):
    """Context manager that changes to another directory
    """
    if not path:
        yield
        return
    cwd = os.getcwd()
    os.chdir(path)
    try:
        yield
    finally:
        os.chdir(cwd)

@contextmanager
def temporary_dir():
    """Context manager that creates a temporary directory and chdirs to it.

    When the context manager exits it returns to the previous cwd
    and deletes the temporary directory.
    """
    d = tempfile.mkdtemp()
    try:
        with cd(d):
            yield d
    finally:
        if os.path.exists(d):
            shutil.rmtree(d)

def in_directory(filepath, dirpath):
    """Returns a boolean for whether filepath is contained in dirpath.

    Normalizes the paths (e.g. resolving symlinks and ..)
    so this is the safe way to make sure a user-configured path
    is located inside the user's project repo.
    """
    filepath = os.path.realpath(filepath)
    dirpath = os.path.realpath(dirpath)
    return filepath.startswith(os.path.join(dirpath, ""))
<|endoftext|>


<commit_before>"""
registry for plugins
holds the class and the object that contains all code to maintain plugin states
"""
import importlib
import pathlib
import logging
from typing import OrderedDict
from importlib import reload

from django.apps import apps
from django.conf import settings
from django.db.utils import OperationalError, ProgrammingError
from django.conf.urls import url
from django.urls import clear_url_caches
from django.contrib import admin
from django.utils.text import slugify

try:
    from importlib import metadata
except:
    import importlib_metadata as metadata
    # TODO remove when python minimum is 3.8

from maintenance_mode.core import maintenance_mode_on
from maintenance_mode.core import get_maintenance_mode, set_maintenance_mode

from plugin import plugins as inventree_plugins
from .integration import IntegrationPluginBase
from .helpers import get_plugin_error, IntegrationPluginError

logger = logging.getLogger('inventree')

class Plugins:
    def __init__(self) -> None:
        # plugin registry
        self.plugins = {}
        self.plugins_inactive = {}

        self.plugin_modules = []         # Holds all discovered plugins

        self.errors = {}                 # Holds discovering errors

        # flags
        self.is_loading = False
        self.apps_loading = True        # Marks if apps were reloaded yet

        # integration specific
        self.installed_apps = []         # Holds all added plugin_paths
        # mixins
        self.mixins_globalsettings = {}

    # region public plugin functions
    def load_plugins(self):
        """load and activate all IntegrationPlugins"""
        from plugin.helpers import log_plugin_error

        logger.info('Start loading plugins')
        # set maintanace mode
        _maintenance = bool(get_maintenance_mode())
        if not _maintenance:
            set_maintenance_mode(True)

        registered_sucessfull = False
        blocked_plugin = None
        retry_counter = settings.PLUGIN_RETRY
        while not registered_sucessfull:
            try:
                # we are using the db so for migrations etc we need to try this block
                self._init_plugins(blocked_plugin)
                self._activate_plugins()
                registered_sucessfull = True
            except (OperationalError, ProgrammingError):
                # Exception if the database has not been migrated yet
                logger.info('Database not accessible while loading plugins')
                break
            except IntegrationPluginError as error:
                logger.error(f'Encountered an error with {error.path}:\n{error.message}')
                log_plugin_error({error.path: error.message}, 'load')
                blocked_plugin = error.path  # we will not try to load this app again

                # init apps without any integration plugins
                self._clean_registry()
                self._clean_installed_apps()
                self._activate_plugins(force_reload=True)

                # we do not want to end in an endless loop
                retry_counter -= 1
                if retry_counter <= 0:
                    if settings.PLUGIN_TESTING:
                        print('Max retries, breaking loading')
                    break

                # now the loading will re-start up with init

        # remove maintenance
        if not _maintenance:
            set_maintenance_mode(False)
        logger.info('Finished loading plugins')

    def unload_plugins(self):
        """unload and deactivate all IntegrationPlugins"""
        logger.info('Start unloading plugins')
        # set maintanace mode
        _maintenance = bool(get_maintenance_mode())
        if not _maintenance:
            set_maintenance_mode(True)

        # remove all plugins from registry
        self._clean_registry()

        # deactivate all integrations
        self._deactivate_plugins()

        # remove maintenance
        if not _maintenance:
            set_maintenance_mode(False)
        logger.info('Finished unloading plugins')

    def reload_plugins(self):
        """safely reload IntegrationPlugins"""
        logger.info('Start reloading plugins')
        with maintenance_mode_on():
            self.unload_plugins()
            self.load_plugins()
        logger.info('Finished reloading plugins')
    # endregion

    # region general plugin managment mechanisms
    def collect_plugins(self):
        """collect integration plugins from all possible ways of loading"""
        self.plugin_modules = []  # clear

        # Collect plugins from paths
        for plugin in settings.PLUGIN_DIRS:
            modules = inventree_plugins.get_plugins(importlib.import_module(plugin), IntegrationPluginBase, True)
            if modules:
                [self.plugin_modules.append(item) for item in modules]

        # check if not running in testing mode and apps should be loaded from hooks
        if (not settings.PLUGIN_TESTING) or (settings.PLUGIN_TESTING and settings.PLUGIN_TESTING_SETUP):
            # Collect plugins from setup entry points
            for entry in metadata.entry_points().get('inventree_plugins', []):
                plugin = entry.load()
                plugin.is_package = True
                self.plugin_modules.append(plugin)

        # Log collected plugins
        logger.info(f'Collected {len(self.plugin_modules)} plugins!')
        logger.info(", ".join([a.__module__ for a in self.plugin_modules]))

    def _init_plugins(self, disabled=None):
        """initialise all found plugins

        :param disabled: loading path of disabled app, defaults to None
        :type disabled: str, optional
        :raises error: IntegrationPluginError
        """
        from plugin.helpers import log_plugin_error
        from plugin.models import PluginConfig

        logger.info('Starting plugin initialisation')
        # Initialize integration plugins
        for plugin in self.plugin_modules:
            # check if package
            was_packaged = getattr(plugin, 'is_package', False)

            # check if activated
            # these checks only use attributes - never use plugin supplied functions -> that would lead to arbitrary code execution!!
            plug_name = plugin.PLUGIN_NAME
            plug_key = plugin.PLUGIN_SLUG if getattr(plugin, 'PLUGIN_SLUG', None) else plug_name
            plug_key = slugify(plug_key)  # keys are slugs!
            try:
                plugin_db_setting, _ = PluginConfig.objects.get_or_create(key=plug_key, name=plug_name)
            except (OperationalError, ProgrammingError) as error:
                # Exception if the database has not been migrated yet - check if test are running - raise if not
                if not settings.PLUGIN_TESTING:
                    raise error
                plugin_db_setting = None

            # always activate if testing
            if settings.PLUGIN_TESTING or (plugin_db_setting and plugin_db_setting.active):
                # check if the plugin was blocked -> threw an error
                if disabled:
                    if plugin.__name__ == disabled:
                        # errors are bad so disable the plugin in the database
                        # but only if not in testing mode as that breaks in the GH pipeline
                        log_plugin_error({plug_key: 'Disabled'}, 'init')
                        if not settings.PLUGIN_TESTING:
                            plugin_db_setting.active = False
                            # TODO save the error to the plugin
                            plugin_db_setting.save()

                        # add to inactive plugins so it shows up in the ui
                        self.plugins_inactive[plug_key] = plugin_db_setting
                        continue  # continue -> the plugin is not loaded

                # init package
                # now we can be sure that an admin has activated the plugin -> as of Nov 2021 there are not many checks in place
                # but we could enhance those to check signatures, run the plugin against a whitelist etc.
                logger.info(f'Loading integration plugin {plugin.PLUGIN_NAME}')
                try:
                    plugin = plugin()
                except Exception as error:
                    # log error
                    get_plugin_error(error, do_log=True, log_name='init')

                    plugin_db_setting.active = False
                    # TODO save the error to the plugin
                    plugin_db_setting.save()

                    # add to incative plugins
                    self.plugins_inactive[plug_key] = plugin_db_setting
                    continue  # continue -> the plugin is not loaded

                logger.info(f'Loaded integration plugin {plugin.slug}')
                plugin.is_package = was_packaged
                if plugin_db_setting:
                    plugin.pk = plugin_db_setting.pk

                # safe reference
                self.plugins[plugin.slug] = plugin
            else:
                # save for later reference
                self.plugins_inactive[plug_key] = plugin_db_setting

    def _activate_plugins(self, force_reload=False):
        """run integration functions for all plugins

        :param force_reload: force reload base apps, defaults to False
        :type force_reload: bool, optional
        """
        # activate integrations
        plugins = self.plugins.items()
        logger.info(f'Found {len(plugins)} active plugins')

        self.activate_integration_globalsettings(plugins)
        self.activate_integration_app(plugins, force_reload=force_reload)

    def _deactivate_plugins(self):
        """run integration deactivation functions for all plugins"""
        self.deactivate_integration_app()
        self.deactivate_integration_globalsettings()
    # endregion

    # region specific integrations
    # region integration_globalsettings
    def activate_integration_globalsettings(self, plugins):
        from common.models import InvenTreeSetting

        if settings.PLUGIN_TESTING or InvenTreeSetting.get_setting('ENABLE_PLUGINS_GLOBALSETTING'):
            logger.info('Registering IntegrationPlugin global settings')
            for slug, plugin in plugins:
                if plugin.mixin_enabled('globalsettings'):
                    plugin_setting = plugin.globalsettingspatterns
                    self.mixins_globalsettings[slug] = plugin_setting

                    # Add to settings dir
                    InvenTreeSetting.GLOBAL_SETTINGS.update(plugin_setting)

    def deactivate_integration_globalsettings(self):
        from common.models import InvenTreeSetting

        # collect all settings
        plugin_settings = {}
        for _, plugin_setting in self.mixins_globalsettings.items():
            plugin_settings.update(plugin_setting)

        # remove settings
        for setting in plugin_settings:
            InvenTreeSetting.GLOBAL_SETTINGS.pop(setting)

        # clear cache
        self.mixins_globalsettings = {}
    # endregion

    # region integration_app
    def activate_integration_app(self, plugins, force_reload=False):
        """activate AppMixin plugins - add custom apps and reload

        :param plugins: list of IntegrationPlugins that should be installed
        :type plugins: dict
        :param force_reload: only reload base apps, defaults to False
        :type force_reload: bool, optional
        """
        from common.models import InvenTreeSetting

        if settings.PLUGIN_TESTING or InvenTreeSetting.get_setting('ENABLE_PLUGINS_APP'):
            logger.info('Registering IntegrationPlugin apps')
            apps_changed = False

            # add them to the INSTALLED_APPS
            for slug, plugin in plugins:
                if plugin.mixin_enabled('app'):
                    plugin_path = self._get_plugin_path(plugin)
                    if plugin_path not in settings.INSTALLED_APPS:
                        settings.INSTALLED_APPS += [plugin_path]
                        self.installed_apps += [plugin_path]
                        apps_changed = True

            # if apps were changed or force loading base apps -> reload
            if apps_changed or force_reload:
                # first startup or force loading of base apps -> registry is prob false
                if self.apps_loading or force_reload:
                    self.apps_loading = False
                    self._reload_apps(force_reload=True)

                self._reload_apps()

                # rediscover models/ admin sites
                self._reregister_contrib_apps()

                # update urls - must be last as models must be registered for creating admin routes
                self._update_urls()

    def _reregister_contrib_apps(self):
        """fix reloading of contrib apps - models and admin
        this is needed if plugins were loaded earlier and then reloaded as models and admins rely on imports
        those register models and admin in their respective objects (e.g. admin.site for admin)
        """
        for plugin_path in self.installed_apps:
            try:
                app_name = plugin_path.split('.')[-1]
                app_config = apps.get_app_config(app_name)
            except LookupError:
                # the plugin was never loaded correctly
                logger.debug(f'{app_name} App was not found during deregistering')
                break

            # reload models if they were set
            # models_module gets set if models were defined - even after multiple loads
            # on a reload the models registery is empty but models_module is not
            if app_config.models_module and len(app_config.models) == 0:
                reload(app_config.models_module)

            # check for all models if they are registered with the site admin
            model_not_reg = False
            for model in app_config.get_models():
                if not admin.site.is_registered(model):
                    model_not_reg = True

            # reload admin if at least one model is not registered
            # models are registered with admin in the 'admin.py' file - so we check
            # if the app_config has an admin module before trying to laod it
            if model_not_reg and hasattr(app_config.module, 'admin'):
                reload(app_config.module.admin)

    def _get_plugin_path(self, plugin):
        """parse plugin path
        the input can be eiter:
        - a local file / dir
        - a package
        """
        try:
            # for local path plugins
            plugin_path = '.'.join(pathlib.Path(plugin.path).relative_to(settings.BASE_DIR).parts)
        except ValueError:
            # plugin is shipped as package
            plugin_path = plugin.PLUGIN_NAME
        return plugin_path

    def deactivate_integration_app(self):
        """deactivate integration app - some magic required"""
        # unregister models from admin
        for plugin_path in self.installed_apps:
            models = []  # the modelrefs need to be collected as poping an item in a iter is not welcomed
            app_name = plugin_path.split('.')[-1]
            try:
                app_config = apps.get_app_config(app_name)

                # check all models
                for model in app_config.get_models():
                    # remove model from admin site
                    admin.site.unregister(model)
                    models += [model._meta.model_name]
            except LookupError:
                # if an error occurs the app was never loaded right -> so nothing to do anymore
                logger.debug(f'{app_name} App was not found during deregistering')
                break

            # unregister the models (yes, models are just kept in multilevel dicts)
            for model in models:
                # remove model from general registry
                apps.all_models[plugin_path].pop(model)

            # clear the registry for that app
            # so that the import trick will work on reloading the same plugin
            # -> the registry is kept for the whole lifecycle
            if models and app_name in apps.all_models:
                apps.all_models.pop(app_name)

        # remove plugin from installed_apps
        self._clean_installed_apps()

        # reset load flag and reload apps
        settings.INTEGRATION_APPS_LOADED = False
        self._reload_apps()

        # update urls to remove the apps from the site admin
        self._update_urls()

    def _clean_installed_apps(self):
        for plugin in self.installed_apps:
            if plugin in settings.INSTALLED_APPS:
                settings.INSTALLED_APPS.remove(plugin)

        self.installed_apps = []

    def _clean_registry(self):
        # remove all plugins from registry
        self.plugins = {}
        self.plugins_inactive = {}

    def _update_urls(self):
        from InvenTree.urls import urlpatterns
        from plugin.urls import get_plugin_urls

        for index, a in enumerate(urlpatterns):
            if hasattr(a, 'app_name'):
                if a.app_name == 'admin':
                    urlpatterns[index] = url(r'^admin/', admin.site.urls, name='inventree-admin')
                elif a.app_name == 'plugin':
                    urlpatterns[index] = get_plugin_urls()
        clear_url_caches()

    def _reload_apps(self, force_reload: bool = False):
        if force_reload:
            # we can not use the built in functions as we need to brute force the registry
            apps.app_configs = OrderedDict()
            apps.apps_ready = apps.models_ready = apps.loading = apps.ready = False
            apps.clear_cache()
            self._try_reload(apps.populate, settings.INSTALLED_APPS)

        self.is_loading = True
        self._try_reload(apps.set_installed_apps, settings.INSTALLED_APPS)
        self.is_loading = False

    def _try_reload(self, cmd, *args, **kwargs):
        """
        wrapper to try reloading the apps
        throws an custom error that gets handled by the loading function
        """
        try:
            cmd(*args, **kwargs)
            return True, []
        except Exception as error:
            get_plugin_error(error, do_raise=True)
    # endregion
    # endregion

plugins = Plugins()
<commit_msg>just use the default failing mechanism<commit_after>"""
registry for plugins
holds the class and the object that contains all code to maintain plugin states
"""
import importlib
import pathlib
import logging
from typing import OrderedDict
from importlib import reload

from django.apps import apps
from django.conf import settings
from django.db.utils import OperationalError, ProgrammingError
from django.conf.urls import url
from django.urls import clear_url_caches
from django.contrib import admin
from django.utils.text import slugify

try:
    from importlib import metadata
except:
    import importlib_metadata as metadata
    # TODO remove when python minimum is 3.8

from maintenance_mode.core import maintenance_mode_on
from maintenance_mode.core import get_maintenance_mode, set_maintenance_mode

from plugin import plugins as inventree_plugins
from .integration import IntegrationPluginBase
from .helpers import get_plugin_error, IntegrationPluginError

logger = logging.getLogger('inventree')

class Plugins:
    def __init__(self) -> None:
        # plugin registry
        self.plugins = {}
        self.plugins_inactive = {}

        self.plugin_modules = []         # Holds all discovered plugins

        self.errors = {}                 # Holds discovering errors

        # flags
        self.is_loading = False
        self.apps_loading = True        # Marks if apps were reloaded yet

        # integration specific
        self.installed_apps = []         # Holds all added plugin_paths
        # mixins
        self.mixins_globalsettings = {}

    # region public plugin functions
    def load_plugins(self):
        """load and activate all IntegrationPlugins"""
        from plugin.helpers import log_plugin_error

        logger.info('Start loading plugins')
        # set maintanace mode
        _maintenance = bool(get_maintenance_mode())
        if not _maintenance:
            set_maintenance_mode(True)

        registered_sucessfull = False
        blocked_plugin = None
        retry_counter = settings.PLUGIN_RETRY
        while not registered_sucessfull:
            try:
                # we are using the db so for migrations etc we need to try this block
                self._init_plugins(blocked_plugin)
                self._activate_plugins()
                registered_sucessfull = True
            except (OperationalError, ProgrammingError):
                # Exception if the database has not been migrated yet
                logger.info('Database not accessible while loading plugins')
                break
            except IntegrationPluginError as error:
                logger.error(f'Encountered an error with {error.path}:\n{error.message}')
                log_plugin_error({error.path: error.message}, 'load')
                blocked_plugin = error.path  # we will not try to load this app again

                # init apps without any integration plugins
                self._clean_registry()
                self._clean_installed_apps()
                self._activate_plugins(force_reload=True)

                # we do not want to end in an endless loop
                retry_counter -= 1
                if retry_counter <= 0:
                    if settings.PLUGIN_TESTING:
                        print('Max retries, breaking loading')
                    break

                # now the loading will re-start up with init

        # remove maintenance
        if not _maintenance:
            set_maintenance_mode(False)
        logger.info('Finished loading plugins')

    def unload_plugins(self):
        """unload and deactivate all IntegrationPlugins"""
        logger.info('Start unloading plugins')
        # set maintanace mode
        _maintenance = bool(get_maintenance_mode())
        if not _maintenance:
            set_maintenance_mode(True)

        # remove all plugins from registry
        self._clean_registry()

        # deactivate all integrations
        self._deactivate_plugins()

        # remove maintenance
        if not _maintenance:
            set_maintenance_mode(False)
        logger.info('Finished unloading plugins')

    def reload_plugins(self):
        """safely reload IntegrationPlugins"""
        logger.info('Start reloading plugins')
        with maintenance_mode_on():
            self.unload_plugins()
            self.load_plugins()
        logger.info('Finished reloading plugins')
    # endregion

    # region general plugin managment mechanisms
    def collect_plugins(self):
        """collect integration plugins from all possible ways of loading"""
        self.plugin_modules = []  # clear

        # Collect plugins from paths
        for plugin in settings.PLUGIN_DIRS:
            modules = inventree_plugins.get_plugins(importlib.import_module(plugin), IntegrationPluginBase, True)
            if modules:
                [self.plugin_modules.append(item) for item in modules]

        # check if not running in testing mode and apps should be loaded from hooks
        if (not settings.PLUGIN_TESTING) or (settings.PLUGIN_TESTING and settings.PLUGIN_TESTING_SETUP):
            # Collect plugins from setup entry points
            for entry in metadata.entry_points().get('inventree_plugins', []):
                plugin = entry.load()
                plugin.is_package = True
                self.plugin_modules.append(plugin)

        # Log collected plugins
        logger.info(f'Collected {len(self.plugin_modules)} plugins!')
        logger.info(", ".join([a.__module__ for a in self.plugin_modules]))

    def _init_plugins(self, disabled=None):
        """initialise all found plugins

        :param disabled: loading path of disabled app, defaults to None
        :type disabled: str, optional
        :raises error: IntegrationPluginError
        """
        from plugin.helpers import log_plugin_error
        from plugin.models import PluginConfig

        logger.info('Starting plugin initialisation')
        # Initialize integration plugins
        for plugin in self.plugin_modules:
            # check if package
            was_packaged = getattr(plugin, 'is_package', False)

            # check if activated
            # these checks only use attributes - never use plugin supplied functions -> that would lead to arbitrary code execution!!
            plug_name = plugin.PLUGIN_NAME
            plug_key = plugin.PLUGIN_SLUG if getattr(plugin, 'PLUGIN_SLUG', None) else plug_name
            plug_key = slugify(plug_key)  # keys are slugs!
            try:
                plugin_db_setting, _ = PluginConfig.objects.get_or_create(key=plug_key, name=plug_name)
            except (OperationalError, ProgrammingError) as error:
                # Exception if the database has not been migrated yet - check if test are running - raise if not
                if not settings.PLUGIN_TESTING:
                    raise error
                plugin_db_setting = None

            # always activate if testing
            if settings.PLUGIN_TESTING or (plugin_db_setting and plugin_db_setting.active):
                # check if the plugin was blocked -> threw an error
                if disabled:
                    if plugin.__name__ == disabled:
                        # errors are bad so disable the plugin in the database
                        # but only if not in testing mode as that breaks in the GH pipeline
                        log_plugin_error({plug_key: 'Disabled'}, 'init')
                        if not settings.PLUGIN_TESTING:
                            plugin_db_setting.active = False
                            # TODO save the error to the plugin
                            plugin_db_setting.save()

                        # add to inactive plugins so it shows up in the ui
                        self.plugins_inactive[plug_key] = plugin_db_setting
                        continue  # continue -> the plugin is not loaded

                # init package
                # now we can be sure that an admin has activated the plugin -> as of Nov 2021 there are not many checks in place
                # but we could enhance those to check signatures, run the plugin against a whitelist etc.
                logger.info(f'Loading integration plugin {plugin.PLUGIN_NAME}')
                try:
                    plugin = plugin()
                except Exception as error:
                    # log error and raise it -> disable plugin
                    get_plugin_error(error, do_raise=True, do_log=True, log_name='init')

                logger.info(f'Loaded integration plugin {plugin.slug}')
                plugin.is_package = was_packaged
                if plugin_db_setting:
                    plugin.pk = plugin_db_setting.pk

                # safe reference
                self.plugins[plugin.slug] = plugin
            else:
                # save for later reference
                self.plugins_inactive[plug_key] = plugin_db_setting

    def _activate_plugins(self, force_reload=False):
        """run integration functions for all plugins

        :param force_reload: force reload base apps, defaults to False
        :type force_reload: bool, optional
        """
        # activate integrations
        plugins = self.plugins.items()
        logger.info(f'Found {len(plugins)} active plugins')

        self.activate_integration_globalsettings(plugins)
        self.activate_integration_app(plugins, force_reload=force_reload)

    def _deactivate_plugins(self):
        """run integration deactivation functions for all plugins"""
        self.deactivate_integration_app()
        self.deactivate_integration_globalsettings()
    # endregion

    # region specific integrations
    # region integration_globalsettings
    def activate_integration_globalsettings(self, plugins):
        from common.models import InvenTreeSetting

        if settings.PLUGIN_TESTING or InvenTreeSetting.get_setting('ENABLE_PLUGINS_GLOBALSETTING'):
            logger.info('Registering IntegrationPlugin global settings')
            for slug, plugin in plugins:
                if plugin.mixin_enabled('globalsettings'):
                    plugin_setting = plugin.globalsettingspatterns
                    self.mixins_globalsettings[slug] = plugin_setting

                    # Add to settings dir
                    InvenTreeSetting.GLOBAL_SETTINGS.update(plugin_setting)

    def deactivate_integration_globalsettings(self):
        from common.models import InvenTreeSetting

        # collect all settings
        plugin_settings = {}
        for _, plugin_setting in self.mixins_globalsettings.items():
            plugin_settings.update(plugin_setting)

        # remove settings
        for setting in plugin_settings:
            InvenTreeSetting.GLOBAL_SETTINGS.pop(setting)

        # clear cache
        self.mixins_globalsettings = {}
    # endregion

    # region integration_app
    def activate_integration_app(self, plugins, force_reload=False):
        """activate AppMixin plugins - add custom apps and reload

        :param plugins: list of IntegrationPlugins that should be installed
        :type plugins: dict
        :param force_reload: only reload base apps, defaults to False
        :type force_reload: bool, optional
        """
        from common.models import InvenTreeSetting

        if settings.PLUGIN_TESTING or InvenTreeSetting.get_setting('ENABLE_PLUGINS_APP'):
            logger.info('Registering IntegrationPlugin apps')
            apps_changed = False

            # add them to the INSTALLED_APPS
            for slug, plugin in plugins:
                if plugin.mixin_enabled('app'):
                    plugin_path = self._get_plugin_path(plugin)
                    if plugin_path not in settings.INSTALLED_APPS:
                        settings.INSTALLED_APPS += [plugin_path]
                        self.installed_apps += [plugin_path]
                        apps_changed = True

            # if apps were changed or force loading base apps -> reload
            if apps_changed or force_reload:
                # first startup or force loading of base apps -> registry is prob false
                if self.apps_loading or force_reload:
                    self.apps_loading = False
                    self._reload_apps(force_reload=True)

                self._reload_apps()

                # rediscover models/ admin sites
                self._reregister_contrib_apps()

                # update urls - must be last as models must be registered for creating admin routes
                self._update_urls()

    def _reregister_contrib_apps(self):
        """fix reloading of contrib apps - models and admin
        this is needed if plugins were loaded earlier and then reloaded as models and admins rely on imports
        those register models and admin in their respective objects (e.g. admin.site for admin)
        """
        for plugin_path in self.installed_apps:
            try:
                app_name = plugin_path.split('.')[-1]
                app_config = apps.get_app_config(app_name)
            except LookupError:
                # the plugin was never loaded correctly
                logger.debug(f'{app_name} App was not found during deregistering')
                break

            # reload models if they were set
            # models_module gets set if models were defined - even after multiple loads
            # on a reload the models registery is empty but models_module is not
            if app_config.models_module and len(app_config.models) == 0:
                reload(app_config.models_module)

            # check for all models if they are registered with the site admin
            model_not_reg = False
            for model in app_config.get_models():
                if not admin.site.is_registered(model):
                    model_not_reg = True

            # reload admin if at least one model is not registered
            # models are registered with admin in the 'admin.py' file - so we check
            # if the app_config has an admin module before trying to laod it
            if model_not_reg and hasattr(app_config.module, 'admin'):
                reload(app_config.module.admin)

    def _get_plugin_path(self, plugin):
        """parse plugin path
        the input can be eiter:
        - a local file / dir
        - a package
        """
        try:
            # for local path plugins
            plugin_path = '.'.join(pathlib.Path(plugin.path).relative_to(settings.BASE_DIR).parts)
        except ValueError:
            # plugin is shipped as package
            plugin_path = plugin.PLUGIN_NAME
        return plugin_path

    def deactivate_integration_app(self):
        """deactivate integration app - some magic required"""
        # unregister models from admin
        for plugin_path in self.installed_apps:
            models = []  # the modelrefs need to be collected as poping an item in a iter is not welcomed
            app_name = plugin_path.split('.')[-1]
            try:
                app_config = apps.get_app_config(app_name)

                # check all models
                for model in app_config.get_models():
                    # remove model from admin site
                    admin.site.unregister(model)
                    models += [model._meta.model_name]
            except LookupError:
                # if an error occurs the app was never loaded right -> so nothing to do anymore
                logger.debug(f'{app_name} App was not found during deregistering')
                break

            # unregister the models (yes, models are just kept in multilevel dicts)
            for model in models:
                # remove model from general registry
                apps.all_models[plugin_path].pop(model)

            # clear the registry for that app
            # so that the import trick will work on reloading the same plugin
            # -> the registry is kept for the whole lifecycle
            if models and app_name in apps.all_models:
                apps.all_models.pop(app_name)

        # remove plugin from installed_apps
        self._clean_installed_apps()

        # reset load flag and reload apps
        settings.INTEGRATION_APPS_LOADED = False
        self._reload_apps()

        # update urls to remove the apps from the site admin
        self._update_urls()

    def _clean_installed_apps(self):
        for plugin in self.installed_apps:
            if plugin in settings.INSTALLED_APPS:
                settings.INSTALLED_APPS.remove(plugin)

        self.installed_apps = []

    def _clean_registry(self):
        # remove all plugins from registry
        self.plugins = {}
        self.plugins_inactive = {}

    def _update_urls(self):
        from InvenTree.urls import urlpatterns
        from plugin.urls import get_plugin_urls

        for index, a in enumerate(urlpatterns):
            if hasattr(a, 'app_name'):
                if a.app_name == 'admin':
                    urlpatterns[index] = url(r'^admin/', admin.site.urls, name='inventree-admin')
                elif a.app_name == 'plugin':
                    urlpatterns[index] = get_plugin_urls()
        clear_url_caches()

    def _reload_apps(self, force_reload: bool = False):
        if force_reload:
            # we can not use the built in functions as we need to brute force the registry
            apps.app_configs = OrderedDict()
            apps.apps_ready = apps.models_ready = apps.loading = apps.ready = False
            apps.clear_cache()
            self._try_reload(apps.populate, settings.INSTALLED_APPS)

        self.is_loading = True
        self._try_reload(apps.set_installed_apps, settings.INSTALLED_APPS)
        self.is_loading = False

    def _try_reload(self, cmd, *args, **kwargs):
        """
        wrapper to try reloading the apps
        throws an custom error that gets handled by the loading function
        """
        try:
            cmd(*args, **kwargs)
            return True, []
        except Exception as error:
            get_plugin_error(error, do_raise=True)
    # endregion
    # endregion

plugins = Plugins()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.urls import reverse

from InvenTree.api_tester import InvenTreeAPITestCase

class PluginDetailAPITest(InvenTreeAPITestCase):
    """
    Tests the plugin AP I endpoints
    """

    roles = [
        'admin.add',
        'admin.view',
        'admin.change',
        'admin.delete',
    ]

    def setUp(self):
        self.MSG_NO_PKG = 'Either packagenmae of url must be provided'

        self.PKG_NAME = 'minimal'
        super().setUp()

    def test_plugin_install(self):
        """
        Test the plugin install command
        """
        url = reverse('api-plugin-install')

        # valid - Pypi
        data = self.post(url, {
            'confirm': True,
            'packagename': self.PKG_NAME
        }, expected_code=201).data

        self.assertEqual(data['success'], True)

        # invalid tries
        # no input
        self.post(url, {}, expected_code=400)

        # no package info
        data = self.post(url, {
            'confirm': True,
        }, expected_code=400).data
        self.assertEqual(data['url'][0].title().upper(), self.MSG_NO_PKG.upper())
        self.assertEqual(data['packagename'][0].title().upper(), self.MSG_NO_PKG.upper())

        # not confirmed
        data = self.post(url, {
            'packagename': self.PKG_NAME
        }, expected_code=400).data
<commit_msg>check confirm is True<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.urls import reverse

from InvenTree.api_tester import InvenTreeAPITestCase

class PluginDetailAPITest(InvenTreeAPITestCase):
    """
    Tests the plugin AP I endpoints
    """

    roles = [
        'admin.add',
        'admin.view',
        'admin.change',
        'admin.delete',
    ]

    def setUp(self):
        self.MSG_NO_PKG = 'Either packagenmae of url must be provided'

        self.PKG_NAME = 'minimal'
        super().setUp()

    def test_plugin_install(self):
        """
        Test the plugin install command
        """
        url = reverse('api-plugin-install')

        # valid - Pypi
        data = self.post(url, {
            'confirm': True,
            'packagename': self.PKG_NAME
        }, expected_code=201).data

        self.assertEqual(data['success'], True)

        # invalid tries
        # no input
        self.post(url, {}, expected_code=400)

        # no package info
        data = self.post(url, {
            'confirm': True,
        }, expected_code=400).data
        self.assertEqual(data['url'][0].title().upper(), self.MSG_NO_PKG.upper())
        self.assertEqual(data['packagename'][0].title().upper(), self.MSG_NO_PKG.upper())

        # not confirmed
        self.post(url, {
            'packagename': self.PKG_NAME
        }, expected_code=400).data
        data = self.post(url, {
            'packagename': self.PKG_NAME,
            'confirm': False,
        }, expected_code=400).data
        self.assertEqual(data['confirm'][0].title().upper(), 'Installation not confirmed'.upper())
<|endoftext|>


<commit_before>from __future__ import absolute_import

from som.vmobjects.array import Array

class Method(Array):

    # Static field indices and number of method fields
    SIGNATURE_INDEX                 = Array.NUMBER_OF_OBJECT_FIELDS
    HOLDER_INDEX                    = 1 + SIGNATURE_INDEX
    NUMBER_OF_METHOD_FIELDS         = 1 + HOLDER_INDEX

    _immutable_fields_ = ["_bytecodes[*]",
                          "_inline_cache_class",
                          "_receiver_class_table",
                          "_number_of_locals",
                          "_maximum_number_of_stack_elements"]

    def __init__(self, nilObject, num_literals, num_locals, max_stack_elements,
                 num_bytecodes, signature):
        Array.__init__(self, nilObject, num_literals)

        self._receiver_class_table   = []
        self._invoked_methods        = []
        self._receiver_class_index   = 0

        self._invocation_count       = 0

        # Set the number of bytecodes in this method
        self._bytecodes              = ["\x00"] * num_bytecodes
        self._inline_cache_class     = [None]   * num_bytecodes
        self._inline_cache_invokable = [None]   * num_bytecodes

        self._number_of_locals       = num_locals
        self._maximum_number_of_stack_elements = max_stack_elements
        self._set_signature(signature)

    def is_primitive(self):
        return False

    def is_invokable(self):
        """In the RPython version, we use this method to identify methods
           and primitives
        """
        return True

    def get_number_of_locals(self):
        # Get the number of locals (converted to a Java integer)
        return self._number_of_locals

    def get_maximum_number_of_stack_elements(self):
        # Get the maximum number of stack elements
        return self._maximum_number_of_stack_elements

    def get_signature(self):
        # Get the signature of this method by reading the field with signature
        # index
        return self.get_field(self.SIGNATURE_INDEX)

    def _set_signature(self, value):
        # Set the signature of this method by writing to the field with
        # signature index
        self.set_field(self.SIGNATURE_INDEX, value)

    def get_holder(self):
        # Get the holder of this method by reading the field with holder index
        return self.get_field(self.HOLDER_INDEX)

    def set_holder(self, value):
        # Set the holder of this method by writing to the field with holder index
        self.set_field(self.HOLDER_INDEX, value)

        # Make sure all nested invokables have the same holder
        for i in range(0, self.get_number_of_indexable_fields()):
            if self.get_indexable_field(i).is_invokable():
                self.get_indexable_field(i).set_holder(value)

    def get_constant(self, bytecode_index):
        # Get the constant associated to a given bytecode index
        return self.get_indexable_field(self.get_bytecode(bytecode_index + 1))

    def get_number_of_arguments(self):
        # Get the number of arguments of this method
        return self.get_signature().get_number_of_signature_arguments()

    def _get_default_number_of_fields(self):
        # Return the default number of fields in a method
        return self.NUMBER_OF_METHOD_FIELDS

    def get_number_of_bytecodes(self):
        # Get the number of bytecodes in this method
        return len(self._bytecodes)

    def get_bytecode(self, index):
        # Get the bytecode at the given index
        return ord(self._bytecodes[index])

    def set_bytecode(self, index, value):
        # Set the bytecode at the given index to the given value
        assert 0 <= value and value <= 255
        self._bytecodes[index] = chr(value)

    # TODO: remove these things
    def increase_invocation_counter(self):
        self._invocation_count += 1

    # TODO: remove these things
    def get_invocation_count(self):
        return self._invocation_count

    def invoke(self, frame, interpreter):
        # Increase the invocation counter
        self._invocation_count += 1

        # Allocate and push a new frame on the interpreter stack
        new_frame = interpreter.push_new_frame(self,
                                    interpreter.get_universe().nilObject)
        new_frame.copy_arguments_from(frame)

    # TODO: remove these things
    def get_receiver_class(self, index):
        return self._receiver_class_table[index]

    # TODO: remove these things
    def get_invoked_method(self, index):
        # return the last invoked method for a particular send
        return self._invoked_methods[index]

    # TODO: remove these things
    def add_receiver_class_and_method(self, rcvr_class, invokable):
        self._receiver_class_table.append(self._receiver_class_index, rcvr_class)
        self._invoked_methods.append(self._receiver_class_index, invokable)
        self._receiver_class_index += 1

        return self._receiver_class_index - 1

    # TODO: remove these things
    def is_receiver_class_table_full(self):
        return self._receiver_class_index == 255

    def __str__(self):
        return "Method(" + self.get_holder().get_name().get_string() + ">>" + str(self.get_signature()) + ")"

    def get_inline_cache_class(self, bytecode_index):
        return self._inline_cache_class[bytecode_index]

    def get_inline_cache_invokable(self, bytecode_index):
        return self._inline_cache_invokable[bytecode_index]

    def set_inline_cache(self, bytecode_index, receiver_class, invokable):
        self._inline_cache_class[bytecode_index]    = receiver_class
        self._inline_cache_invokable[bytecode_index] = invokable

    def merge_point_string(self):
        """ debug info for the jit """
        return "%s>>%s" % (self.get_holder().get_name().get_string(),
                           self.get_signature().get_string())
<commit_msg>Removed incomplete implementation of invocation tracking<commit_after>from __future__ import absolute_import

from som.vmobjects.array import Array

class Method(Array):

    # Static field indices and number of method fields
    SIGNATURE_INDEX                 = Array.NUMBER_OF_OBJECT_FIELDS
    HOLDER_INDEX                    = 1 + SIGNATURE_INDEX
    NUMBER_OF_METHOD_FIELDS         = 1 + HOLDER_INDEX

    _immutable_fields_ = ["_bytecodes[*]",
                          "_inline_cache_class",
                          "_receiver_class_table",
                          "_number_of_locals",
                          "_maximum_number_of_stack_elements"]

    def __init__(self, nilObject, num_literals, num_locals, max_stack_elements,
                 num_bytecodes, signature):
        Array.__init__(self, nilObject, num_literals)

        # Set the number of bytecodes in this method
        self._bytecodes              = ["\x00"] * num_bytecodes
        self._inline_cache_class     = [None]   * num_bytecodes
        self._inline_cache_invokable = [None]   * num_bytecodes

        self._number_of_locals       = num_locals
        self._maximum_number_of_stack_elements = max_stack_elements
        self._set_signature(signature)

    def is_primitive(self):
        return False

    def is_invokable(self):
        """In the RPython version, we use this method to identify methods
           and primitives
        """
        return True

    def get_number_of_locals(self):
        # Get the number of locals (converted to a Java integer)
        return self._number_of_locals

    def get_maximum_number_of_stack_elements(self):
        # Get the maximum number of stack elements
        return self._maximum_number_of_stack_elements

    def get_signature(self):
        # Get the signature of this method by reading the field with signature
        # index
        return self.get_field(self.SIGNATURE_INDEX)

    def _set_signature(self, value):
        # Set the signature of this method by writing to the field with
        # signature index
        self.set_field(self.SIGNATURE_INDEX, value)

    def get_holder(self):
        # Get the holder of this method by reading the field with holder index
        return self.get_field(self.HOLDER_INDEX)

    def set_holder(self, value):
        # Set the holder of this method by writing to the field with holder index
        self.set_field(self.HOLDER_INDEX, value)

        # Make sure all nested invokables have the same holder
        for i in range(0, self.get_number_of_indexable_fields()):
            if self.get_indexable_field(i).is_invokable():
                self.get_indexable_field(i).set_holder(value)

    def get_constant(self, bytecode_index):
        # Get the constant associated to a given bytecode index
        return self.get_indexable_field(self.get_bytecode(bytecode_index + 1))

    def get_number_of_arguments(self):
        # Get the number of arguments of this method
        return self.get_signature().get_number_of_signature_arguments()

    def _get_default_number_of_fields(self):
        # Return the default number of fields in a method
        return self.NUMBER_OF_METHOD_FIELDS

    def get_number_of_bytecodes(self):
        # Get the number of bytecodes in this method
        return len(self._bytecodes)

    def get_bytecode(self, index):
        # Get the bytecode at the given index
        return ord(self._bytecodes[index])

    def set_bytecode(self, index, value):
        # Set the bytecode at the given index to the given value
        assert 0 <= value and value <= 255
        self._bytecodes[index] = chr(value)

    def invoke(self, frame, interpreter):
        # Allocate and push a new frame on the interpreter stack
        new_frame = interpreter.push_new_frame(self,
                                    interpreter.get_universe().nilObject)
        new_frame.copy_arguments_from(frame)

    def __str__(self):
        return "Method(" + self.get_holder().get_name().get_string() + ">>" + str(self.get_signature()) + ")"

    def get_inline_cache_class(self, bytecode_index):
        return self._inline_cache_class[bytecode_index]

    def get_inline_cache_invokable(self, bytecode_index):
        return self._inline_cache_invokable[bytecode_index]

    def set_inline_cache(self, bytecode_index, receiver_class, invokable):
        self._inline_cache_class[bytecode_index]    = receiver_class
        self._inline_cache_invokable[bytecode_index] = invokable

    def merge_point_string(self):
        """ debug info for the jit """
        return "%s>>%s" % (self.get_holder().get_name().get_string(),
                           self.get_signature().get_string())
<|endoftext|>