<commit_before>import time

from virtualbox import library

"""
Add helper code to the default IGuestSession class.
"""

# Add context management to IGuestSession
class IGuestSession(library.IGuestSession):
    __doc__ = library.IGuestSession.__doc__
    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_val, trace):
        self.close()

    def execute(self, command, arguments=[], stdin="", environment=[],
                 flags=[library.ProcessCreateFlag.wait_for_std_err,
                        library.ProcessCreateFlag.wait_for_std_out,
                        library.ProcessCreateFlag.ignore_orphaned_processes],
                 priority=library.ProcessPriority.default,
                 affinity=[], timeout_ms=0):
        """Execute a command in the Guest

            Arguments:
                command - Command to execute.
                arguments - List of arguments for the command
                stdin - A buffer to write to the stdin of the command.
                environment - See IGuestSession.create_process?
                flags - List of ProcessCreateFlag objects.
                    Default value set to [wait_for_std_err,
                                          wait_for_stdout,
                                          ignore_orphaned_processes]
                timeout_ms - ms to wait for the process to complete.
                    If 0, wait for ever...
                priority - Set the ProcessPriority priority to be used for
                    execution.
                affinity - Process affinity to use for execution.

            Return IProcess, stdout, stderr
        """
        def read_out(process, flags, stdout, stderr):
            if library.ProcessCreateFlag.wait_for_std_err in flags:
                e = process.read(2, 65000, 0)
                stderr.append(e)
            if library.ProcessCreateFlag.wait_for_std_out in flags:
                o = process.read(1, 65000, 0)
                stdout.append(o)

        process = self.process_create_ex(command, arguments, environment,
                            flags, timeout_ms, priority, affinity)
        process.wait_for(int(library.ProcessWaitResult.start), 0)

        # write stdin to the process
        if stdin:
            index = 0
            while index < len(stdin):
                index += process.write(0, [library.ProcessInputFlag.none],
                                        stdin[index:], 0)
            process.write(0, [library.ProcessInputFlag.end_of_file], 0)

        # read the process output and wait for
        stdout = []
        stderr = []
        while process.status == library.ProcessStatus.started:
            read_out(process, flags, stdout, stderr)
            time.sleep(0.2)
        # make sure we have read the remainder of the out
        read_out(process, flags, stdout, stderr)
        return process, "".join(stdout), "".join(stderr)
<commit_msg>added makedirs and fixed windows compatability issue with execute...  ready from stream was a buffer not a str<commit_after>import time

from virtualbox import library

"""
Add helper code to the default IGuestSession class.
"""

# Add context management to IGuestSession
class IGuestSession(library.IGuestSession):
    __doc__ = library.IGuestSession.__doc__
    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_val, trace):
        self.close()

    def execute(self, command, arguments=[], stdin="", environment=[],
                 flags=[library.ProcessCreateFlag.wait_for_std_err,
                        library.ProcessCreateFlag.wait_for_std_out,
                        library.ProcessCreateFlag.ignore_orphaned_processes],
                 priority=library.ProcessPriority.default,
                 affinity=[], timeout_ms=0):
        """Execute a command in the Guest

            Arguments:
                command - Command to execute.
                arguments - List of arguments for the command
                stdin - A buffer to write to the stdin of the command.
                environment - See IGuestSession.create_process?
                flags - List of ProcessCreateFlag objects.
                    Default value set to [wait_for_std_err,
                                          wait_for_stdout,
                                          ignore_orphaned_processes]
                timeout_ms - ms to wait for the process to complete.
                    If 0, wait for ever...
                priority - Set the ProcessPriority priority to be used for
                    execution.
                affinity - Process affinity to use for execution.

            Return IProcess, stdout, stderr
        """
        def read_out(process, flags, stdout, stderr):
            if library.ProcessCreateFlag.wait_for_std_err in flags:
                e = str(process.read(2, 65000, 0))
                stderr.append(e)
            if library.ProcessCreateFlag.wait_for_std_out in flags:
                o = str(process.read(1, 65000, 0))
                stdout.append(o)

        process = self.process_create_ex(command, arguments, environment,
                            flags, timeout_ms, priority, affinity)
        process.wait_for(int(library.ProcessWaitResult.start), 0)

        # write stdin to the process
        if stdin:
            index = 0
            while index < len(stdin):
                index += process.write(0, [library.ProcessInputFlag.none],
                                        stdin[index:], 0)
            process.write(0, [library.ProcessInputFlag.end_of_file], 0)

        # read the process output and wait for
        stdout = []
        stderr = []
        while process.status == library.ProcessStatus.started:
            read_out(process, flags, stdout, stderr)
            time.sleep(0.2)
        # make sure we have read the remainder of the out
        read_out(process, flags, stdout, stderr)
        return process, "".join(stdout), "".join(stderr)

    def makedirs(self, path, mode=0x777):
        """Super-mkdir: create a leaf directory and all intermediate ones."""
        self.directory_create(path, mode, [library.DirectoryCreateFlag.parents])

<|endoftext|>


<commit_before>"""This runs Step 2 of The Cannon:
    uses the model to solve for the labels of the test set."""

from scipy import optimize as opt
import numpy as np

def get_x(labels):
    """
    Constructs a label vector for an arbitrary number of labels
    Assumes that our model is quadratic in the labels
    """
    nlabels = len(labels)
    # should there be a [1] at the beginning of this vector?
    x = labels # linear terms
    # Quadratic terms:
    for i in range(nlabels):
        for j in range(i, nlabels):
            element = labels[i]*labels[j]
            x.append(element)
    x = np.array(x)
    return x

# Extremely annoying...looks like scipy_optimize can't handle an array of parameters. Have to spell all the parameters out.
# So for now we have to hard-code the number of labels into here.
def func(coeffs, a, b, c, d):
    labels = [a,b,c,d]
    x = get_x(labels)
    return np.dot(coeffs, x)

def infer_labels(num_labels, model, test_set):
    """
    Uses the model to solve for labels of the test set.

    Input:
    -----
    num_labels: number of labels being solved for
    model: coeffs_all, covs, scatters, chis, chisqs, pivots
    test_set: Stars object (see stars.py) corresponding to the test set.

    Returns
    -------
    labels_all:
    MCM_rotate_all:
    covs_all: covariance matrix of the fit
    """
    nlabels = num_labels
    spectra = test_set.get_spectra() #(nstars, npixels, 3)
    nstars = spectra.shape[0]
    npixels = spectra.shape[1]
    coeffs_all, covs, scatters, chis, chisqs, pivots = model
    labels_all = np.zeros((nstars, nlabels))
    # Don't understand what this MCM_rotate_all matrix is
    MCM_rotate_all = np.zeros((nstars, coeffs_all.shape[1]-1,
        coeffs_all.shape[1]-1.))
    covs_all = np.zeros((nstars, nlabels, nlabels))

    for jj in range(nstars):
        pixels = spectra[jj,:,0]
        fluxes = spectra[jj,:,1]
        fluxerrs = spectra[jj,:,2]
        fluxes_norm = fluxes - coeffs_all[:,0] #pivot around small value
        Cinv = 1. / (fluxerrs* 2 + scatters**2)
        # something I don't understand --
        coeffs = np.delete(coeffs_all, 0, axis=1) #since x0 = 1...
        weights = 1 / Cinv**0.5
        labels, covs = opt.curve_fit(
                func, coeffs, fluxes_norm, sigma=weights, absolute_sigma = True)
        labels = labels + pivots
        MCM_rotate = np.dot(coeffs.T, Cinv[:,None] * coeffs)
        labels_all[jj,:] = labels
        MCM_rotate_all[jj,:,:] = MCM_rotate
        covs_all[jj,:,:] = covs
    return labels_all, MCM_rotate_all, covs_all
<commit_msg>curve_fit problem fixed, now generalized to an arbitrary number of labels<commit_after>"""This runs Step 2 of The Cannon:
    uses the model to solve for the labels of the test set."""

from scipy import optimize as opt
import numpy as np

def get_x(labels):
    """
    Constructs a label vector for an arbitrary number of labels
    Assumes that our model is quadratic in the labels
    """
    nlabels = len(labels)
    # should there be a [1] at the beginning of this vector?
    x = labels # linear terms
    # Quadratic terms:
    for i in range(nlabels):
        for j in range(i, nlabels):
            element = labels[i]*labels[j]
            x.append(element)
    x = np.array(x)
    return x

def func(coeffs, *labels):
    x = get_x(list(labels))
    return np.dot(coeffs, x)

def infer_labels(num_labels, model, test_set):
    """
    Uses the model to solve for labels of the test set.

    Input:
    -----
    num_labels: number of labels being solved for
    model: coeffs_all, covs, scatters, chis, chisqs, pivots
    test_set: Stars object (see stars.py) corresponding to the test set.

    Returns
    -------
    labels_all:
    MCM_rotate_all:
    covs_all: covariance matrix of the fit
    """
    nlabels = num_labels
    spectra = test_set.get_spectra() #(nstars, npixels, 3)
    nstars = spectra.shape[0]
    npixels = spectra.shape[1]
    coeffs_all, covs, scatters, chis, chisqs, pivots = model
    labels_all = np.zeros((nstars, nlabels))
    # Don't understand what this MCM_rotate_all matrix is
    MCM_rotate_all = np.zeros((nstars, coeffs_all.shape[1]-1,
        coeffs_all.shape[1]-1.))
    covs_all = np.zeros((nstars, nlabels, nlabels))

    for jj in range(nstars):
        pixels = spectra[jj,:,0]
        fluxes = spectra[jj,:,1]
        fluxerrs = spectra[jj,:,2]
        fluxes_norm = fluxes - coeffs_all[:,0] #pivot around small value
        Cinv = 1. / (fluxerrs* 2 + scatters**2)
        # something I don't understand --
        coeffs = np.delete(coeffs_all, 0, axis=1) #since x0 = 1...
        weights = 1 / Cinv**0.5
        labels, covs = opt.curve_fit(func, coeffs, fluxes_norm,
                p0=np.repeat(1,nlabels), sigma=weights, absolute_sigma = True)
        labels = labels + pivots
        MCM_rotate = np.dot(coeffs.T, Cinv[:,None] * coeffs)
        labels_all[jj,:] = labels
        MCM_rotate_all[jj,:,:] = MCM_rotate
        covs_all[jj,:,:] = covs
    return labels_all, MCM_rotate_all, covs_all
<|endoftext|>


<commit_before># coding: utf-8

'''Provides function related to PHP Composer packages.
'''

import json
import os.path
import re
import webbrowser
import requests
import sublime
import sublime_plugin
import mdpopups

URL_JSON = 'https://packagist.org/packages/{package}.json'
URL_PAGE = 'https://packagist.org/packages/{package}'
TEMPLATE = '''
# {name}

{description}

- Stats: DL {downloads_total} / Fav {favers}
- Page: [Packagist]({url_packagist}) / [Repository]({url_repo})
- Command: [require]({command_require}) / [remove]({command_remove})

    {close_btn}

'''
CSS = '''
body {
    margin: 0;
    padding: 0;
}
.mdpopups {
}
.composer-package-info {
    font-size: 12px;
    line-height: 20px;
    padding: 0 18px 10px 10px;
}
.composer-package-info h1 {
    background-color: var(--mdpopups-admon-success-bg);
    color: var(--mdpopups-admon-success-title-fg);
    font-family: var(--mdpopups-font-mono);
    font-size: 12px;
    line-height: 12px;
    margin: 0 -18px 10px -10px;
    padding: 10px;
}
.composer-package-info p {
    margin: 6px 0;
}
.composer-package-info ul {
    margin: 6px 0;
}
.close-btn-wrapper {
    margin-top: 12px;
}
.close-btn-wrapper a {
    text-decoration: none;
}
'''
WRAPPER_CLASS = 'composer-package-info'
CMD_REQUIRE = 'composer require {}'
CMD_REMOVE = 'composer remove {}'
PREFIX_COPY = 'copy: '
MESSAGE_KEY = 'composer_info'
MESSAGE_TTL = 2000
LENGTH_DESC = 100

class ComposerInfoPackageInfo(sublime_plugin.ViewEventListener):
    '''A view event listener for showing composer package data.
    '''

    def on_hover(self, point, hover_zone):
        if not self._is_composer():
            return

        if not self._is_on_text(hover_zone):
            return

        if not self._is_in_scope(point):
            return

        package_name = self._get_selected_pacakge_name(point)
        if not self._is_valid_package_name(package_name):
            return

        self.view.set_status(MESSAGE_KEY, 'Fetching data...')
        try:
            data_raw = self._fetch_package_info(package_name)
            data = self._extract_pakcage_info(data_raw, package_name)
        except BaseException as e:
            sublime.set_timeout(lambda: self.view.erase_status(MESSAGE_KEY),
                                MESSAGE_TTL)
            raise e
        else:
            self._show_popup(data, point)
            self.view.set_status(MESSAGE_KEY, 'Data fetched successfully.')
            sublime.set_timeout(lambda: self.view.erase_status(MESSAGE_KEY),
                                MESSAGE_TTL)

    def _is_composer(self):
        return self._get_basename() == 'composer.json'

    def _get_basename(self):
        return os.path.basename(self.view.file_name())

    def _is_on_text(self, hover_zone):
        return hover_zone == sublime.HOVER_TEXT

    def _is_in_scope(self, point):
        scope_name = self.view.scope_name(point)
        names = [
            'meta.structure.dictionary.key.json',
            'string.quoted.double.json'
        ]
        for n in names:
            if n not in scope_name:
                return False
        return True

    def _get_selected_pacakge_name(self, point):
        quoted_text = self.view.substr(self.view.extract_scope(point))
        package_name = quoted_text.strip('"')
        return package_name

    def _is_valid_package_name(self, name):
        if re.match(r'[\w-]+/[\w-]+', name):
            return True
        return False

    def _fetch_package_info(self, name):
        response = requests.get(URL_JSON.format(package=name))
        if not response.status_code != '200':
            raise BaseException('Package data fetch failed.')
        return json.loads(response.text)

    def _extract_pakcage_info(self, data, name):
        try:
            package = data['package']
            name = package['name']
            url = URL_PAGE.format(package=package['name'])
            description = self._truncate(package['description'], LENGTH_DESC)
            print(data)
            return {
                'name': name,
                'description': description,
                'downloads_total': package['downloads']['total'],
                'favers': package['favers'],
                'url_packagist': url,
                'url_repo': package['repository'],
                'command_require': PREFIX_COPY + CMD_REQUIRE.format(name),
                'command_remove': PREFIX_COPY + CMD_REMOVE.format(name),
                'close_btn': '[' + chr(0x00D7) + ']',
            }
        except Exception as e:
            raise BaseException('Package data extraction failed.')

    def _show_popup(self, data, location):
        if mdpopups.is_popup_visible(self.view):
            mdpopups.hide_popup(self.view)

        mdpopups.show_popup(self.view,
                            TEMPLATE.format(**data),
                            css=CSS,
                            wrapper_class=WRAPPER_CLASS,
                            max_width=400,
                            location=location,
                            flags=sublime.HIDE_ON_MOUSE_MOVE_AWAY,
                            on_navigate=self.on_phantom_click)

    def on_phantom_click(self, href):
        if href.startswith('https://'):
            webbrowser.open_new_tab(href)
        elif href.startswith(PREFIX_COPY):
            sublime.set_clipboard(href[len(PREFIX_COPY):])

        mdpopups.hide_popup(self.view)

    def _truncate(self, string, count):
        return string[:count] + ('...' if string[count:] else '')

class BaseException(Exception):
    pass
<commit_msg>Shorten a loop with all().<commit_after># coding: utf-8

'''Provides function related to PHP Composer packages.
'''

import json
import os.path
import re
import webbrowser
import requests
import sublime
import sublime_plugin
import mdpopups

URL_JSON = 'https://packagist.org/packages/{package}.json'
URL_PAGE = 'https://packagist.org/packages/{package}'
TEMPLATE = '''
# {name}

{description}

- Stats: DL {downloads_total} / Fav {favers}
- Page: [Packagist]({url_packagist}) / [Repository]({url_repo})
- Command: [require]({command_require}) / [remove]({command_remove})

    {close_btn}

'''
CSS = '''
body {
    margin: 0;
    padding: 0;
}
.mdpopups {
}
.composer-package-info {
    font-size: 12px;
    line-height: 20px;
    padding: 0 18px 10px 10px;
}
.composer-package-info h1 {
    background-color: var(--mdpopups-admon-success-bg);
    color: var(--mdpopups-admon-success-title-fg);
    font-family: var(--mdpopups-font-mono);
    font-size: 12px;
    line-height: 12px;
    margin: 0 -18px 10px -10px;
    padding: 10px;
}
.composer-package-info p {
    margin: 6px 0;
}
.composer-package-info ul {
    margin: 6px 0;
}
.close-btn-wrapper {
    margin-top: 12px;
}
.close-btn-wrapper a {
    text-decoration: none;
}
'''
WRAPPER_CLASS = 'composer-package-info'
CMD_REQUIRE = 'composer require {}'
CMD_REMOVE = 'composer remove {}'
PREFIX_COPY = 'copy: '
MESSAGE_KEY = 'composer_info'
MESSAGE_TTL = 2000
LENGTH_DESC = 100

class ComposerInfoPackageInfo(sublime_plugin.ViewEventListener):
    '''A view event listener for showing composer package data.
    '''

    def on_hover(self, point, hover_zone):
        if not self._is_composer():
            return

        if not self._is_on_text(hover_zone):
            return

        if not self._is_in_scope(point):
            return

        package_name = self._get_selected_pacakge_name(point)
        if not self._is_valid_package_name(package_name):
            return

        self.view.set_status(MESSAGE_KEY, 'Fetching data...')
        try:
            data_raw = self._fetch_package_info(package_name)
            data = self._extract_pakcage_info(data_raw, package_name)
        except BaseException as e:
            sublime.set_timeout(lambda: self.view.erase_status(MESSAGE_KEY),
                                MESSAGE_TTL)
            raise e
        else:
            self._show_popup(data, point)
            self.view.set_status(MESSAGE_KEY, 'Data fetched successfully.')
            sublime.set_timeout(lambda: self.view.erase_status(MESSAGE_KEY),
                                MESSAGE_TTL)

    def _is_composer(self):
        return self._get_basename() == 'composer.json'

    def _get_basename(self):
        return os.path.basename(self.view.file_name())

    def _is_on_text(self, hover_zone):
        return hover_zone == sublime.HOVER_TEXT

    def _is_in_scope(self, point):
        scope_name = self.view.scope_name(point)
        names = [
            'meta.structure.dictionary.key.json',
            'string.quoted.double.json'
        ]
        return all(n in scope_name for n in names)

    def _get_selected_pacakge_name(self, point):
        quoted_text = self.view.substr(self.view.extract_scope(point))
        package_name = quoted_text.strip('"')
        return package_name

    def _is_valid_package_name(self, name):
        if re.match(r'[\w-]+/[\w-]+', name):
            return True
        return False

    def _fetch_package_info(self, name):
        response = requests.get(URL_JSON.format(package=name))
        if not response.status_code != '200':
            raise BaseException('Package data fetch failed.')
        return json.loads(response.text)

    def _extract_pakcage_info(self, data, name):
        try:
            package = data['package']
            name = package['name']
            url = URL_PAGE.format(package=package['name'])
            description = self._truncate(package['description'], LENGTH_DESC)
            print(data)
            return {
                'name': name,
                'description': description,
                'downloads_total': package['downloads']['total'],
                'favers': package['favers'],
                'url_packagist': url,
                'url_repo': package['repository'],
                'command_require': PREFIX_COPY + CMD_REQUIRE.format(name),
                'command_remove': PREFIX_COPY + CMD_REMOVE.format(name),
                'close_btn': '[' + chr(0x00D7) + ']',
            }
        except Exception as e:
            raise BaseException('Package data extraction failed.')

    def _show_popup(self, data, location):
        if mdpopups.is_popup_visible(self.view):
            mdpopups.hide_popup(self.view)

        mdpopups.show_popup(self.view,
                            TEMPLATE.format(**data),
                            css=CSS,
                            wrapper_class=WRAPPER_CLASS,
                            max_width=400,
                            location=location,
                            flags=sublime.HIDE_ON_MOUSE_MOVE_AWAY,
                            on_navigate=self.on_phantom_click)

    def on_phantom_click(self, href):
        if href.startswith('https://'):
            webbrowser.open_new_tab(href)
        elif href.startswith(PREFIX_COPY):
            sublime.set_clipboard(href[len(PREFIX_COPY):])

        mdpopups.hide_popup(self.view)

    def _truncate(self, string, count):
        return string[:count] + ('...' if string[count:] else '')

class BaseException(Exception):
    pass
<|endoftext|>


<commit_before>import math

################################################################################
# Point

class Point(object):
    """Point is a point in 2D. It behaves as a vector, too, because that's
convenient."""

    def __init__(self, x=0, y=0):
        """ Point(x: float, y: float) -> Point
        Initialize and return a Point

        """
        self.x = float(x)
        self.y = float(y)

    def reset(self,x,y):
        """ reset(x: float, y: float) -> None
        Resets the point to given coordinates and return nothing

        """
        self.x = float(x)
        self.y = float(y)

    def __neg__(self):
        """ __neg__() -> Point
        Compute a point p such that: self + p == Point(0,0), and return a Point

        """
        return Point(-self.x,-self.y)

    def __add__(self, other):
        """ __add__(other: Point) -> Point
        Returns a point p such that: self + other == p, and return a Point

        """
        return Point(self.x + other.x, self.y + other.y)

    def __sub__(self, other):
        """ __sub__(other: Point) -> Point
        Returns a point p such that: self + other == p, and return a Point

        """
        return Point(self.x - other.x, self.y - other.y)

    def __mul__(self, other):
        """ __mul__(other: float) -> Point
        Interprets self as a vector to perform a scalar multiplication and
        return a Point

        """
        return Point(self.x * other, self.y * other)

    def __rmul__(self, other):
        """ __rmul__(other: float) -> Point
        Interprets self as a vector to perform a scalar multiplication and
        return a Point

        """
        return Point(self.x * other, self.y * other)

    def isInside(self, rect):
        """ isInside(rect: Rect) -> boolean
        Check if the point is falling inside rect and return a boolean

        """
        return (self.x >= rect.lowerLeft.x and
                self.x <= rect.upperRight.x and
                self.y >= rect.lowerLeft.y and
                self.y <= rect.upperRight.y)

    def length(self):
        """ length() -> float
        Interprets self as a vector to compute the L_2 norm and return a float

        """
        return math.sqrt(self.x * self.x + self.y * self.y)

################################################################################
# Unit tests

import unittest
import random

class TestPoint(unittest.TestCase):

    @staticmethod
    def assertDoubleEquals(a, b, eps = 0.00001):
        assert abs(a-b) < eps

    def testAddLength(self):
        """Uses triangle inequality to exercise add and length"""
        for i in range(100):
            x = Point(random.uniform(-1.0, 1.0), random.uniform(-1.0, 1.0))
            y = Point(random.uniform(-1.0, 1.0), random.uniform(-1.0, 1.0))
            assert (x+y).length() <= x.length() + y.length()

    def testMulLength(self):
        """Uses vector space properties to exercise mul, rmul and length"""
        for i in range(100):
            x = Point(random.uniform(-1.0, 1.0), random.uniform(-1.0, 1.0))
            s = random.uniform(0.0, 10.0)
            self.assertDoubleEquals(s * x.length(), (s * x).length())
            self.assertDoubleEquals(s * x.length(), (x * s).length())

if __name__ == '__main__':
    unittest.main()

<commit_msg>Added Point.__eq__(self, other) and Point.__ne__(self, other) comparison methods.<commit_after>import math

################################################################################
# Point

class Point(object):
    """Point is a point in 2D. It behaves as a vector, too, because that's
convenient."""

    def __init__(self, x=0, y=0):
        """ Point(x: float, y: float) -> Point
        Initialize and return a Point

        """
        self.x = float(x)
        self.y = float(y)

    def reset(self,x,y):
        """ reset(x: float, y: float) -> None
        Resets the point to given coordinates and return nothing

        """
        self.x = float(x)
        self.y = float(y)

    def __neg__(self):
        """ __neg__() -> Point
        Compute a point p such that: self + p == Point(0,0), and return a Point

        """
        return Point(-self.x,-self.y)

    def __add__(self, other):
        """ __add__(other: Point) -> Point
        Returns a point p such that: self + other == p, and return a Point

        """
        return Point(self.x + other.x, self.y + other.y)

    def __sub__(self, other):
        """ __sub__(other: Point) -> Point
        Returns a point p such that: self - other == p, and return a Point

        """
        return Point(self.x - other.x, self.y - other.y)

    def __mul__(self, other):
        """ __mul__(other: float) -> Point
        Interprets self as a vector to perform a scalar multiplication and
        return a Point

        """
        return Point(self.x * other, self.y * other)

    def __eq__(self, other):
        """__eq__(other: Point) -> boolean
        Two points are equal if they have the same components

        """
        if other:
            if self.x == other.x and self.y == other.y:
                return True
            else:
                return False
        else:
            return False

    def __ne__(self, other):
        """__ne__(other: Point) -> boolean
        Two points are differenr if they don't have the same components

        """
        return not self.__eq__(other)

    def __rmul__(self, other):
        """ __rmul__(other: float) -> Point
        Interprets self as a vector to perform a scalar multiplication and
        return a Point

        """
        return Point(self.x * other, self.y * other)

    def isInside(self, rect):
        """ isInside(rect: Rect) -> boolean
        Check if the point is falling inside rect and return a boolean

        """
        return (self.x >= rect.lowerLeft.x and
                self.x <= rect.upperRight.x and
                self.y >= rect.lowerLeft.y and
                self.y <= rect.upperRight.y)

    def length(self):
        """ length() -> float
        Interprets self as a vector to compute the L_2 norm and return a float

        """
        return math.sqrt(self.x * self.x + self.y * self.y)

################################################################################
# Unit tests

import unittest
import random

class TestPoint(unittest.TestCase):

    @staticmethod
    def assertDoubleEquals(a, b, eps = 0.00001):
        assert abs(a-b) < eps

    def testAddLength(self):
        """Uses triangle inequality to exercise add and length"""
        for i in range(100):
            x = Point(random.uniform(-1.0, 1.0), random.uniform(-1.0, 1.0))
            y = Point(random.uniform(-1.0, 1.0), random.uniform(-1.0, 1.0))
            assert (x+y).length() <= x.length() + y.length()

    def testMulLength(self):
        """Uses vector space properties to exercise mul, rmul and length"""
        for i in range(100):
            x = Point(random.uniform(-1.0, 1.0), random.uniform(-1.0, 1.0))
            s = random.uniform(0.0, 10.0)
            self.assertDoubleEquals(s * x.length(), (s * x).length())
            self.assertDoubleEquals(s * x.length(), (x * s).length())

    def testComparisonOperators(self):
        """ Test comparison operators """
        a = Point(0, 1)
        b = Point(0, 1)
        assert a == b
        assert a != None
        b = Point(0, 0.1)
        assert a != b

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>############################################################################
##
## Copyright (C) 2006-2007 University of Utah. All rights reserved.
##
## This file is part of VisTrails.
##
## This file may be used under the terms of the GNU General Public
## License version 2.0 as published by the Free Software Foundation
## and appearing in the file LICENSE.GPL included in the packaging of
## this file.  Please review the following to ensure GNU General Public
## Licensing requirements will be met:
## http://www.opensource.org/licenses/gpl-license.php
##
## If you are unsure which license is appropriate for your use (for
## instance, you are interested in developing a commercial derivative
## of VisTrails), please contact us at vistrails@sci.utah.edu.
##
## This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
## WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
##
############################################################################

from gui.open_db_window import QOpenDBWindow
from core.db.locator import DBLocator, XMLFileLocator
from PyQt4 import QtGui
import core.system
import os

##############################################################################
# DB dialogs

def get_load_db_locator_from_gui(parent):
    config, vistrail_id = QOpenDBWindow.getOpenVistrail()
    if config == {} or vistrail_id == -1:
        return None
    return DBLocator(config['host'],
                     config['port'],
                     config['db'],
                     config['user'],
                     config['passwd'],
                     None,
                     vistrail_id,
                     config.get('id', None))

def get_save_db_locator_from_gui(parent, locator=None):
    config, name = QOpenDBWindow.getSaveVistrail()
    if config == {} or name == '':
        return None
    return DBLocator(config['host'],
                     config['port'],
                     config['db'],
                     config['user'],
                     config['passwd'],
                     name,
                     None,
                     config.get('id', None))

##############################################################################
# XML File dialogs

def get_load_xml_file_locator_from_gui(parent):
    fileName = QtGui.QFileDialog.getOpenFileName(
        parent,
        "Open Vistrail...",
        core.system.vistrails_file_directory(),
        "Vistrail files (*.xml)\nOther files (*)")
    if fileName.isEmpty():
        return None
    return XMLFileLocator(str(fileName))

def get_save_xml_file_locator_from_gui(parent, locator=None):
    # Ignore current locator for now
    # In the future, use locator to guide GUI for better starting directory
    print parent
    fileName = QtGui.QFileDialog.getSaveFileName(
        parent,
        "Save Vistrail...",
        core.system.vistrails_file_directory(),
        "VisTrails files (*.xml)",
        None,
        QtGui.QFileDialog.DontConfirmOverwrite)
    if fileName.isEmpty():
        return None
    f = str(fileName)
    if not f.endswith('.xml'):
        f += '.xml'
    if os.path.isfile(f):
        msg = QtGui.QMessageBox(QtGui.QMessageBox.Question,
                                "VisTrails",
                                "File exists. Overwrite?",
                                (QtGui.QMessageBox.Yes |
                                 QtGui.QMessageBox.No),
                                parent)
        if msg.exec_() == QtGui.QMessageBox.No:
            return None
    return XMLFileLocator(f)

<commit_msg>Save file open and save directory to configuration.  This is replacing code that was accidentally removed with a previous reorganization.<commit_after>############################################################################
##
## Copyright (C) 2006-2007 University of Utah. All rights reserved.
##
## This file is part of VisTrails.
##
## This file may be used under the terms of the GNU General Public
## License version 2.0 as published by the Free Software Foundation
## and appearing in the file LICENSE.GPL included in the packaging of
## this file.  Please review the following to ensure GNU General Public
## Licensing requirements will be met:
## http://www.opensource.org/licenses/gpl-license.php
##
## If you are unsure which license is appropriate for your use (for
## instance, you are interested in developing a commercial derivative
## of VisTrails), please contact us at vistrails@sci.utah.edu.
##
## This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
## WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
##
############################################################################

from gui.application import VistrailsApplication
from gui.open_db_window import QOpenDBWindow
from core.db.locator import DBLocator, XMLFileLocator
from PyQt4 import QtGui
import core.system
import os

##############################################################################
# DB dialogs

def get_load_db_locator_from_gui(parent):
    config, vistrail_id = QOpenDBWindow.getOpenVistrail()
    if config == {} or vistrail_id == -1:
        return None
    return DBLocator(config['host'],
                     config['port'],
                     config['db'],
                     config['user'],
                     config['passwd'],
                     None,
                     vistrail_id,
                     config.get('id', None))

def get_save_db_locator_from_gui(parent, locator=None):
    config, name = QOpenDBWindow.getSaveVistrail()
    if config == {} or name == '':
        return None
    return DBLocator(config['host'],
                     config['port'],
                     config['db'],
                     config['user'],
                     config['passwd'],
                     name,
                     None,
                     config.get('id', None))

##############################################################################
# XML File dialogs

def get_load_xml_file_locator_from_gui(parent):
    fileName = QtGui.QFileDialog.getOpenFileName(
        parent,
        "Open Vistrail...",
        core.system.vistrails_file_directory(),
        "Vistrail files (*.xml)\nOther files (*)")
    if fileName.isEmpty():
        return None
    dirName = os.path.dirname(str(fileName))
    setattr(VistrailsApplication.configuration, 'fileDirectory', dirName)
    core.system.set_vistrails_file_directory(dirName)
    return XMLFileLocator(str(fileName))

def get_save_xml_file_locator_from_gui(parent, locator=None):
    # Ignore current locator for now
    # In the future, use locator to guide GUI for better starting directory
    print parent
    fileName = QtGui.QFileDialog.getSaveFileName(
        parent,
        "Save Vistrail...",
        core.system.vistrails_file_directory(),
        "VisTrails files (*.xml)",
        None,
        QtGui.QFileDialog.DontConfirmOverwrite)
    if fileName.isEmpty():
        return None
    f = str(fileName)
    if not f.endswith('.xml'):
        f += '.xml'
    if os.path.isfile(f):
        msg = QtGui.QMessageBox(QtGui.QMessageBox.Question,
                                "VisTrails",
                                "File exists. Overwrite?",
                                (QtGui.QMessageBox.Yes |
                                 QtGui.QMessageBox.No),
                                parent)
        if msg.exec_() == QtGui.QMessageBox.No:
            return None
    dirName = os.path.dirname(str(f))
    setattr(VistrailsApplication.configuration, 'fileDirectory', dirName)
    core.system.set_vistrails_file_directory(dirName)
    return XMLFileLocator(f)

<|endoftext|>


<commit_before># -*- test-case-name: vumi.transports.smpp.tests.test_mica -*-

from vumi.config import ConfigInt
from vumi.components.session import SessionManager
from vumi.message import TransportUserMessage
from vumi.transports.smpp.processors import default

from twisted.internet.defer import inlineCallbacks, returnValue

def make_vumi_session_identifier(msisdn, mica_session_identifier):
    return '%s+%s' % (msisdn, mica_session_identifier)

class DeliverShortMessageProcessorConfig(
        default.DeliverShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class DeliverShortMessageProcessor(default.DeliverShortMessageProcessor):

    CONFIG_CLASS = DeliverShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(DeliverShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_deliver_sm_ussd(self, pdu, pdu_params, pdu_opts):
        service_op = pdu_opts['ussd_service_op']
        mica_session_identifier = pdu_opts['user_message_reference']
        vumi_session_identifier = make_vumi_session_identifier(
            pdu_params['source_addr'], mica_session_identifier)

        session_event = 'close'
        if service_op == '01':
            # PSSR request. Let's assume it means a new session.
            session_event = 'new'
            ussd_code = pdu_params['short_message']
            content = None

            yield self.session_manager.create_session(
                vumi_session_identifier, ussd_code=ussd_code)

        elif service_op == '17':
            # PSSR response. This means session end.
            session_event = 'close'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = None

            yield self.session_manager.clear_session(vumi_session_identifier)

        else:
            session_event = 'continue'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = pdu_params['short_message']

        # This is stashed on the message and available when replying
        # with a `submit_sm`
        session_info = {
            'session_identifier': mica_session_identifier,
        }

        decoded_msg = self.decode_message(content,
                                          pdu_params['data_coding'])

        result = yield self.handle_short_message_content(
            source_addr=pdu_params['source_addr'],
            destination_addr=ussd_code,
            short_message=decoded_msg,
            message_type='ussd',
            session_event=session_event,
            session_info=session_info)
        returnValue(result)

class SubmitShortMessageProcessorConfig(
        default.SubmitShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class SubmitShortMessageProcessor(default.SubmitShortMessageProcessor):

    CONFIG_CLASS = SubmitShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(SubmitShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_outbound_message(self, message, protocol):
        to_addr = message['to_addr']
        from_addr = message['from_addr']
        text = message['content']

        session_event = message['session_event']
        transport_type = message['transport_type']
        optional_parameters = {}

        if transport_type == 'ussd':
            continue_session = (
                session_event != TransportUserMessage.SESSION_CLOSE)
            session_info = message['transport_metadata'].get(
                'session_info', {})
            mica_session_identifier = session_info.get(
                'session_identifier', '')
            vumi_session_identifier = make_vumi_session_identifier(
                to_addr, mica_session_identifier)

            optional_parameters.update({
                'ussd_service_op': ('02' if continue_session else '17'),
                'user_message_reference': mica_session_identifier,
            })

            if not continue_session:
                yield self.session_manager.clear_session(
                    vumi_session_identifier)

        if self.config.send_long_messages:
            resp = yield protocol.submit_sm_long(
                to_addr.encode('ascii'),
                long_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_sar:
            resp = yield protocol.submit_csm_sar(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_udh:
            resp = yield protocol.submit_csm_udh(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )
        else:
            resp = yield protocol.submit_sm(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        returnValue(resp)
<commit_msg>zero pad TLVs (thanks @Jerith)<commit_after># -*- test-case-name: vumi.transports.smpp.tests.test_mica -*-

from vumi.config import ConfigInt
from vumi.components.session import SessionManager
from vumi.message import TransportUserMessage
from vumi.transports.smpp.processors import default

from twisted.internet.defer import inlineCallbacks, returnValue

def make_vumi_session_identifier(msisdn, mica_session_identifier):
    return '%s+%s' % (msisdn, mica_session_identifier)

class DeliverShortMessageProcessorConfig(
        default.DeliverShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class DeliverShortMessageProcessor(default.DeliverShortMessageProcessor):

    CONFIG_CLASS = DeliverShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(DeliverShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_deliver_sm_ussd(self, pdu, pdu_params, pdu_opts):
        service_op = pdu_opts['ussd_service_op']
        mica_session_identifier = pdu_opts['user_message_reference']
        vumi_session_identifier = make_vumi_session_identifier(
            pdu_params['source_addr'], mica_session_identifier)

        session_event = 'close'
        if service_op == '01':
            # PSSR request. Let's assume it means a new session.
            session_event = 'new'
            ussd_code = pdu_params['short_message']
            content = None

            yield self.session_manager.create_session(
                vumi_session_identifier, ussd_code=ussd_code)

        elif service_op == '17':
            # PSSR response. This means session end.
            session_event = 'close'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = None

            yield self.session_manager.clear_session(vumi_session_identifier)

        else:
            session_event = 'continue'

            session = yield self.session_manager.load_session(
                vumi_session_identifier)
            ussd_code = session['ussd_code']
            content = pdu_params['short_message']

        # This is stashed on the message and available when replying
        # with a `submit_sm`
        session_info = {
            'session_identifier': mica_session_identifier,
        }

        decoded_msg = self.decode_message(content,
                                          pdu_params['data_coding'])

        result = yield self.handle_short_message_content(
            source_addr=pdu_params['source_addr'],
            destination_addr=ussd_code,
            short_message=decoded_msg,
            message_type='ussd',
            session_event=session_event,
            session_info=session_info)
        returnValue(result)

class SubmitShortMessageProcessorConfig(
        default.SubmitShortMessageProcessorConfig):

    max_session_length = ConfigInt(
        'Maximum length a USSD sessions data is to be kept for in seconds.',
        default=60 * 3, static=True)

class SubmitShortMessageProcessor(default.SubmitShortMessageProcessor):

    CONFIG_CLASS = SubmitShortMessageProcessorConfig

    def __init__(self, transport, config):
        super(SubmitShortMessageProcessor, self).__init__(transport, config)
        self.transport = transport
        self.redis = transport.redis
        self.config = self.CONFIG_CLASS(config, static=True)
        self.session_manager = SessionManager(
            self.redis, max_session_length=self.config.max_session_length)

    @inlineCallbacks
    def handle_outbound_message(self, message, protocol):
        to_addr = message['to_addr']
        from_addr = message['from_addr']
        text = message['content']

        session_event = message['session_event']
        transport_type = message['transport_type']
        optional_parameters = {}

        if transport_type == 'ussd':
            continue_session = (
                session_event != TransportUserMessage.SESSION_CLOSE)
            session_info = message['transport_metadata'].get(
                'session_info', {})
            mica_session_identifier = session_info.get(
                'session_identifier', '')
            vumi_session_identifier = make_vumi_session_identifier(
                to_addr, mica_session_identifier)

            optional_parameters.update({
                'ussd_service_op': ('02' if continue_session else '17'),
                'user_message_reference': (
                    str(mica_session_identifier).zfill(2)),
            })

            if not continue_session:
                yield self.session_manager.clear_session(
                    vumi_session_identifier)

        if self.config.send_long_messages:
            resp = yield protocol.submit_sm_long(
                to_addr.encode('ascii'),
                long_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_sar:
            resp = yield protocol.submit_csm_sar(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        elif self.config.send_multipart_udh:
            resp = yield protocol.submit_csm_udh(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )
        else:
            resp = yield protocol.submit_sm(
                to_addr.encode('ascii'),
                short_message=text.encode(self.config.submit_sm_encoding),
                data_coding=self.config.submit_sm_data_coding,
                source_addr=from_addr.encode('ascii'),
                optional_parameters=optional_parameters,
            )

        returnValue(resp)
<|endoftext|>


<commit_before>'''
Copyright 2017, Fujitsu Network Communications, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

"""This is the library to hold all api's related to string operations """

from Framework.ClassUtils.configuration_element_class import ConfigurationElement
from Framework.Utils.print_Utils import print_info, print_error, print_exception

import re
import difflib
from xml.etree import ElementTree

def strip_white_spaces(input_list):
    """Takes a list of string as input
    Removes the leading and trailing white spaces from each string in the list
    Returns an output list having the string elements with leading and trailing white spaces removed """

    output_list = []
    for element in input_list:
        if type(element) is str:
            element = element.strip()
        output_list.append(element)

    return output_list

def replace_from_varconfig(varconfigfile, details_dict, var_sub=None):
    """ """
    if varconfigfile is not None or var_sub is not None:
        details_dict["command_list"] = sub_from_varconfig(varconfigfile, details_dict["command_list"], var_sub)
        details_dict["startprompt_list"] = sub_from_varconfig(varconfigfile, details_dict["startprompt_list"], var_sub)
        details_dict["endprompt_list"] = sub_from_varconfig(varconfigfile, details_dict["endprompt_list"], var_sub)

    return details_dict

def return_quote(a, start_pat="${", end_pat="}"):
    """
        return the list of element that needs to be substituted in a string
    :param a:
    :return:
    """
    mode = 0
    start = 0
    count = 0
    result = []
    for index in range(len(a)):
        if mode == 0 and a[index:index+len(start_pat)] == start_pat:
            mode = 1
            start = index+len(start_pat)
            count += 1
        elif mode == 1 and a[index:index+len(start_pat)] == start_pat:
            count += 1
        elif mode == 1 and a[index:index+len(end_pat)] == end_pat:
            count -= 1
            if count == 0:
                result.append(a[start:index])
                mode = 0
    return result

def sub_from_varconfig(varconfigfile, string_list, var_sub=None, start_pat="${", end_pat="}"):
    """
    Replaces the string variables with their values taken from varconfig_file.
    :Arguments:
        1. varconfigfile - xml file or list of xml files from which the values will be taken for subtitution
        2. string_list - List of command strings, where the variables will be replaced by the config values
        3. var_sub(string) = the pattern [var_sub] in the testdata commands,
                                 start_prompt, end_prompt, verification search
                                 will substituted with this value.
    :Returns:
        1. List of command strings with replaced values or\
           False(boolean) - if any one of the variables does not exist in varconfig_file
    """
    newstring_list = []
    for i, string in enumerate(string_list):
        vc_file = varconfigfile[i] if isinstance(varconfigfile, list) else varconfigfile
        if string and var_sub is not None:
            string = sub_from_varsub(string, var_sub)
            if string and vc_file is not None:
                newstring = sub_from_varconfigfile(string, vc_file, start_pat, end_pat)
                newstring_list.append(newstring)
            else:
                newstring_list.append(string)
        else:
            if string and vc_file is not None:
                newstring = sub_from_varconfigfile(string, vc_file, start_pat, end_pat)
                newstring_list.append(newstring)
            else:
                newstring_list.append(string)

    return newstring_list

def sub_from_wdf(datafile, string_list, td_sys_list=None, kw_system_name=None):
    """
    substitute the patterns $wdf{} in the command/verify parameters
    with the values form the datafile
    """
    from Framework.Utils.data_Utils import getSystemData
    newstring_list = []
    for i in range(0, len(string_list)):
        string = string_list[i]
        td_sys = None if td_sys_list is None else td_sys_list[i]
        found = True
        while found:
            if string:
                match = re.search(r"(\$wdf\{)([^\}]*)(\})", string, re.IGNORECASE)
                if match is not None:
                    found = True
                    try:
                        wdf_match = match.group(0).strip("$wdf{")
                        wdf_match = wdf_match.strip("}")
                        system_or_subsystem = wdf_match.split(".")[0].strip()
                        if system_or_subsystem == "kw_system" or system_or_subsystem == "current_system":
                            system_or_subsystem = kw_system_name
                        elif system_or_subsystem == "target_system":
                            system_or_subsystem = td_sys if td_sys is not None else kw_system_name
                        tag_or_attr = wdf_match.split(".")[1].strip()
                        value = getSystemData(datafile, system_or_subsystem, tag_or_attr)
                        if not value:
                            print_error("Value for '{0}' not provided in the datafile={1}"\
                                        "under the system_or_subsystem = {2}".format(tag_or_attr,
                                                                                datafile,
                                                                                system_or_subsystem))
                    except Exception as err:
                        print_error("Incorrect format provided for substituting value"\
                                    "from warrior datafile in the test data file")
                        print_error("Correct format is $wdf{system_or_subsystemname.tag_or_attribute_name}")

                    else:
                        string = string.replace(match.group(0), value)
                else:
                    found = False
            else:
                found = False
        newstring_list.append(string)

    return newstring_list

def sub_from_varsub(string, var_sub):
    """Replace the pattern [var_sub] in the string
    with the value of var_sub

    :Arguments:
        1. string(string) = input string which needs substitution
        3. var_sub(string) = value to replace the pattern [var_sub]
                             in the input string
    :Returns:
        1. updated string
    """
    match = re.search(r".*(\[(var_sub)\]).*", string, re.IGNORECASE)
    if match is not None:
        string = string.replace(match.group(1),var_sub)

    return string

def sub_from_varconfigfile(string, varconfigfile, start_pat="${", end_pat="}"):
    """ """
    try:
        # when varconfigfile is an XMl object(root element) - this happens
        # only when varconfigfile is taken from database server
        if isinstance(varconfigfile, ElementTree.Element) is True:
            cfg_elem_obj = ConfigurationElement("Varconfig_from_Database",
                                                start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile, elem_type="xml_object")
        else:
            cfg_elem_obj = ConfigurationElement(varconfigfile, start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile)
        newstring = cfg_elem_obj.expand_variables(string)
    except TypeError as exception:
        print_info("At least one of the variables in command string is not found in  " + varconfigfile)
        #print_exception(exception)
        return False
    return newstring

def get_list_from_varconfigfile(string, varconfigfile, start_pat="${", end_pat="}"):
    """ """
    try:
        # when varconfigfile is an XMl object(root element) - this happens
        # only when varconfigfile is taken from database server
        if isinstance(varconfigfile, ElementTree.Element) is True:
            cfg_elem_obj = ConfigurationElement("Varconfig_from_Database",
                                                start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile, elem_type="xml_object")
        else:
            cfg_elem_obj = ConfigurationElement(varconfigfile, start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile)
        newstring = cfg_elem_obj.get_list(string)
    except TypeError as exception:
        print_info("At least one of the variables in command string is not found in  " + varconfigfile)
        #print_exception(exception)
        return False
    return newstring

def text_compare(text1, text2, output_file):
    """
        Compares two strings and if they match returns True
        else writes the difference to the output_file.
    """
    if text1 == text2:
        return True
    diff = list(difflib.Differ().compare(text1.split(), text2.split()))
    te = open(output_file, 'w')
    for line in diff:
        te.write(line+"\n")
    te.close()
    return False

def compare_string_using_regex(response, list_of_regex_patterns):
    """
        Checks whether each regular expression is present in response
        and if all are present returns True else False
    """
    status = True
    for regex_pattern in list_of_regex_patterns:
        pattern = regex_pattern.strip("regex=")
        match = re.search(pattern, response)
        if not match:
            status = False
            print_error("The given {0} is not present in api response".format(regex_pattern))
    return status

def seek_next(pattern, data):
    """find the next pattern in the data and return its position
    """
    prev = 0
    search_object = re.search(pattern, data)
    while search_object:
        pos = search_object.start()
        yield prev+pos
        search_object = re.search(pattern, data[pos+1:])
        prev = pos
    else:
        raise StopIteration

def conv_str_to_bool(text, mapping_dict=None):
    """
    Map a text to a boolean value.
    The default text to bool mapping is provided in the
    default_map dict below, however text
    and mapping_dict as inputs to map as per needs.
    default_map = {'yes': True, 'y': True, 'no': False, 'n': False }
    """

    default_map = {
                   'yes': True,
                   'y': True,
                   'no': False,
                   'n': False
                   }
    mapping = mapping_dict if mapping_dict else default_map
    result = mapping.get(text.upper(), False)

    return result

<commit_msg>Update string_Utils.py<commit_after>'''
Copyright 2017, Fujitsu Network Communications, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

"""This is the library to hold all api's related to string operations """

from Framework.ClassUtils.configuration_element_class import ConfigurationElement
from Framework.Utils.print_Utils import print_info, print_error, print_exception

import re
import difflib
from xml.etree import ElementTree

def strip_white_spaces(input_list):
    """Takes a list of string as input
    Removes the leading and trailing white spaces from each string in the list
    Returns an output list having the string elements with leading and trailing white spaces removed """

    output_list = []
    for element in input_list:
        if type(element) is str:
            element = element.strip()
        output_list.append(element)

    return output_list

def replace_from_varconfig(varconfigfile, details_dict, var_sub=None):
    """ """
    if varconfigfile is not None or var_sub is not None:
        details_dict["command_list"] = sub_from_varconfig(varconfigfile, details_dict["command_list"], var_sub)
        details_dict["startprompt_list"] = sub_from_varconfig(varconfigfile, details_dict["startprompt_list"], var_sub)
        details_dict["endprompt_list"] = sub_from_varconfig(varconfigfile, details_dict["endprompt_list"], var_sub)

    return details_dict

def return_quote(a, start_pat="${", end_pat="}"):
    """
        return the list of element that needs to be substituted in a string
    :param a:
    :return:
    """
    mode = 0
    start = 0
    count = 0
    result = []
    for index in range(len(a)):
        if mode == 0 and a[index:index+len(start_pat)] == start_pat:
            mode = 1
            start = index+len(start_pat)
            count += 1
        elif mode == 1 and a[index:index+len(start_pat)] == start_pat:
            count += 1
        elif mode == 1 and a[index:index+len(end_pat)] == end_pat:
            count -= 1
            if count == 0:
                result.append(a[start:index])
                mode = 0
    return result

def sub_from_varconfig(varconfigfile, string_list, var_sub=None, start_pat="${", end_pat="}"):
    """
    Replaces the string variables with their values taken from varconfig_file.
    :Arguments:
        1. varconfigfile - xml file or list of xml files from which the values will be taken for subtitution
        2. string_list - List of command strings, where the variables will be replaced by the config values
        3. var_sub(string) = the pattern [var_sub] in the testdata commands,
                                 start_prompt, end_prompt, verification search
                                 will substituted with this value.
    :Returns:
        1. List of command strings with replaced values or\
           False(boolean) - if any one of the variables does not exist in varconfig_file
    """
    newstring_list = []
    for i, string in enumerate(string_list):
        vc_file = varconfigfile[i] if isinstance(varconfigfile, list) else varconfigfile
        if string and var_sub is not None:
            string = sub_from_varsub(string, var_sub)
            if string and vc_file is not None:
                newstring = sub_from_varconfigfile(string, vc_file, start_pat, end_pat)
                newstring_list.append(newstring)
            else:
                newstring_list.append(string)
        else:
            if string and vc_file is not None:
                newstring = sub_from_varconfigfile(string, vc_file, start_pat, end_pat)
                newstring_list.append(newstring)
            else:
                newstring_list.append(string)

    return newstring_list

def sub_from_wdf(datafile, string_list, td_sys_list=None, kw_system_name=None):
    """
    substitute the patterns $wdf{} in the command/verify parameters
    with the values form the datafile
    """
    from Framework.Utils.data_Utils import getSystemData
    newstring_list = []
    for i in range(0, len(string_list)):
        string = string_list[i]
        td_sys = None if td_sys_list is None else td_sys_list[i]
        found = True
        while found:
            if string:
                match = re.search(r"(\$wdf\{)([^\}]*)(\})", string, re.IGNORECASE)
                if match is not None:
                    found = True
                    try:
                        wdf_match = match.group(0).strip("$wdf{")
                        wdf_match = wdf_match.strip("}")
                        system_or_subsystem = wdf_match.split(".")[0].strip()
                        if system_or_subsystem == "kw_system" or system_or_subsystem == "current_system":
                            system_or_subsystem = kw_system_name
                        elif system_or_subsystem == "target_system":
                            system_or_subsystem = td_sys if td_sys is not None else kw_system_name
                        tag_or_attr = wdf_match.split(".")[1].strip()
                        value = getSystemData(datafile, system_or_subsystem, tag_or_attr)
                        if not value:
                            print_error("Value for '{0}' not provided in the datafile={1}"\
                                        "under the system_or_subsystem = {2}".format(tag_or_attr,
                                                                                datafile,
                                                                                system_or_subsystem))
                    except Exception as err:
                        print_error("Incorrect format provided for substituting value"\
                                    "from warrior datafile in the test data file")
                        print_error("Correct format is $wdf{system_or_subsystemname.tag_or_attribute_name}")

                    else:
                        string = string.replace(match.group(0), value)
                else:
                    found = False
            else:
                found = False
        newstring_list.append(string)

    return newstring_list

def sub_from_varsub(string, var_sub):
    """Replace the pattern [var_sub] in the string
    with the value of var_sub

    :Arguments:
        1. string(string) = input string which needs substitution
        3. var_sub(string) = value to replace the pattern [var_sub]
                             in the input string
    :Returns:
        1. updated string
    """
    match = re.search(r".*(\[(var_sub)\]).*", string, re.IGNORECASE)
    if match is not None:
        string = string.replace(match.group(1),var_sub)

    return string

def sub_from_varconfigfile(string, varconfigfile, start_pat="${", end_pat="}"):
    """ """
    try:
        # when varconfigfile is an XMl object(root element) - this happens
        # only when varconfigfile is taken from database server
        if isinstance(varconfigfile, ElementTree.Element) is True:
            cfg_elem_obj = ConfigurationElement("Varconfig_from_Database",
                                                start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile, elem_type="xml_object")
        else:
            cfg_elem_obj = ConfigurationElement(varconfigfile, start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile)
        newstring = cfg_elem_obj.expand_variables(string)
    except TypeError as exception:
        print_info("At least one of the variables in command string is not found in  " + varconfigfile)
        #print_exception(exception)
        return False
    return newstring

def get_list_from_varconfigfile(string, varconfigfile, start_pat="${", end_pat="}"):
    """ """
    try:
        # when varconfigfile is an XMl object(root element) - this happens
        # only when varconfigfile is taken from database server
        if isinstance(varconfigfile, ElementTree.Element) is True:
            cfg_elem_obj = ConfigurationElement("Varconfig_from_Database",
                                                start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile, elem_type="xml_object")
        else:
            cfg_elem_obj = ConfigurationElement(varconfigfile, start_pat, end_pat)
            cfg_elem_obj.parse_data(varconfigfile)
        newstring = cfg_elem_obj.get_list(string)
    except TypeError as exception:
        print_info("At least one of the variables in command string is not found in  " + varconfigfile)
        #print_exception(exception)
        return False
    return newstring

def text_compare(text1, text2, output_file):
    """
        Compares two strings and if they match returns True
        else writes the difference to the output_file.
    """
    if text1 == text2:
        return True
    diff = list(difflib.Differ().compare(text1.split(), text2.split()))
    te = open(output_file, 'w')
    for line in diff:
        te.write(line+"\n")
    te.close()
    return False

def compare_string_using_regex(response, list_of_regex_patterns):
    """
        Checks whether each regular expression is present in response
        and if all are present returns True else False
    """
    status = True
    for regex_pattern in list_of_regex_patterns:
        pattern = regex_pattern.strip("regex=")
        match = re.search(pattern, response)
        if not match:
            status = False
            print_error("The given {0} is not present in api response".format(regex_pattern))
    return status

def seek_next(pattern, data):
    """find the next pattern in the data and return its position
    """
    prev = 0
    search_object = re.search(pattern, data)
    while search_object:
        pos = search_object.start()
        yield prev+pos
        search_object = re.search(pattern, data[pos+1:])
        prev = pos
    else:
        raise StopIteration

def conv_str_to_bool(text, mapping_dict=None):
    """
    Map a text to a boolean value.
    The default text to bool mapping is provided in the
    default_map dict below, however text
    and mapping_dict as inputs to map as per needs.
    default_map = {'yes': True, 'y': True, 'no': False, 'n': False }
    """

    default_map = {
                   'yes': True,
                   'y': True,
                   'no': False,
                   'n': False
                   }
    mapping = mapping_dict if mapping_dict else default_map
    result = mapping.get(text.lower(), False)

    return result

<|endoftext|>


<commit_before>from __future__ import absolute_import

import sys, os, re, inspect
import imp

try:
    import hashlib
except ImportError:
    import md5 as hashlib

from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext

import Cython
from ..Compiler.Main import Context, CompilationOptions, default_options

from ..Compiler.Visitor import CythonTransform, EnvTransform
from ..Compiler.ParseTreeTransforms import SkipDeclarations
from ..Compiler.TreeFragment import parse_from_strings
from ..Compiler.StringEncoding import _unicode
from .Dependencies import strip_string_literals, cythonize, cached_function
from ..Compiler import Pipeline, Nodes
from ..Utils import get_cython_cache_dir
import cython as cython_module

IS_PY3 = sys.version_info >= (3, 0)

# A utility function to convert user-supplied ASCII strings to unicode.
if sys.version_info[0] < 3:
    def to_unicode(s):
        if isinstance(s, bytes):
            return s.decode('ascii')
        else:
            return s
else:
    to_unicode = lambda x: x

if sys.version_info < (3, 5):
    import imp
    def load_dynamic(name, module_path):
        return imp.load_dynamic(name, module_path)
else:
    import importlib.util as _importlib_util
    def load_dynamic(name, module_path):
        spec = _importlib_util.spec_from_file_location(name, module_path)
        module = _importlib_util.module_from_spec(spec)
        # sys.modules[name] = module
        spec.loader.exec_module(module)
        return module

class UnboundSymbols(EnvTransform, SkipDeclarations):
    def __init__(self):
        CythonTransform.__init__(self, None)
        self.unbound = set()
    def visit_NameNode(self, node):
        if not self.current_env().lookup(node.name):
            self.unbound.add(node.name)
        return node
    def __call__(self, node):
        super(UnboundSymbols, self).__call__(node)
        return self.unbound

@cached_function
def unbound_symbols(code, context=None):
    code = to_unicode(code)
    if context is None:
        context = Context([], default_options)
    from ..Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform
    tree = parse_from_strings('(tree fragment)', code)
    for phase in Pipeline.create_pipeline(context, 'pyx'):
        if phase is None:
            continue
        tree = phase(tree)
        if isinstance(phase, AnalyseDeclarationsTransform):
            break
    try:
        import builtins
    except ImportError:
        import __builtin__ as builtins
    return tuple(UnboundSymbols()(tree) - set(dir(builtins)))

def unsafe_type(arg, context=None):
    py_type = type(arg)
    if py_type is int:
        return 'long'
    else:
        return safe_type(arg, context)

def safe_type(arg, context=None):
    py_type = type(arg)
    if py_type in (list, tuple, dict, str):
        return py_type.__name__
    elif py_type is complex:
        return 'double complex'
    elif py_type is float:
        return 'double'
    elif py_type is bool:
        return 'bint'
    elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray):
        return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim)
    else:
        for base_type in py_type.__mro__:
            if base_type.__module__ in ('__builtin__', 'builtins'):
                return 'object'
            module = context.find_module(base_type.__module__, need_pxd=False)
            if module:
                entry = module.lookup(base_type.__name__)
                if entry.is_type:
                    return '%s.%s' % (base_type.__module__, base_type.__name__)
        return 'object'

def _get_build_extension():
    dist = Distribution()
    # Ensure the build respects distutils configuration by parsing
    # the configuration files
    config_files = dist.find_config_files()
    dist.parse_config_files(config_files)
    build_extension = build_ext(dist)
    build_extension.finalize_options()
    return build_extension

@cached_function
def _create_context(cython_include_dirs):
    return Context(list(cython_include_dirs), default_options)

_cython_inline_cache = {}
_cython_inline_default_context = _create_context(('.',))

def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None):
    for symbol in unbound_symbols:
        if symbol not in kwds:
            if locals is None or globals is None:
                calling_frame = inspect.currentframe().f_back.f_back.f_back
                if locals is None:
                    locals = calling_frame.f_locals
                if globals is None:
                    globals = calling_frame.f_globals
            if symbol in locals:
                kwds[symbol] = locals[symbol]
            elif symbol in globals:
                kwds[symbol] = globals[symbol]
            else:
                print("Couldn't find %r" % symbol)

def _inline_key(orig_code, arg_sigs, language_level):
    key = orig_code, arg_sigs, sys.version_info, sys.executable, language_level, Cython.__version__
    return hashlib.sha1(_unicode(key).encode('utf-8')).hexdigest()

def cython_inline(code, get_type=unsafe_type,
                  lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
                  cython_include_dirs=None, cython_compiler_directives=None,
                  force=False, quiet=False, locals=None, globals=None, language_level=None, **kwds):

    if get_type is None:
        get_type = lambda x: 'object'
    ctx = _create_context(tuple(cython_include_dirs)) if cython_include_dirs else _cython_inline_default_context

    cython_compiler_directives = dict(cython_compiler_directives or {})
    if language_level is None and 'language_level' not in cython_compiler_directives:
        language_level = '3str'
    if language_level is not None:
        cython_compiler_directives['language_level'] = language_level

    # Fast path if this has been called in this session.
    _unbound_symbols = _cython_inline_cache.get(code)
    if _unbound_symbols is not None:
        _populate_unbound(kwds, _unbound_symbols, locals, globals)
        args = sorted(kwds.items())
        arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args])
        key_hash = _inline_key(code, arg_sigs, language_level)
        invoke = _cython_inline_cache.get((code, arg_sigs, key_hash))
        if invoke is not None:
            arg_list = [arg[1] for arg in args]
            return invoke(*arg_list)

    orig_code = code
    code = to_unicode(code)
    code, literals = strip_string_literals(code)
    code = strip_common_indent(code)
    if locals is None:
        locals = inspect.currentframe().f_back.f_back.f_locals
    if globals is None:
        globals = inspect.currentframe().f_back.f_back.f_globals
    try:
        _cython_inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code)
        _populate_unbound(kwds, _unbound_symbols, locals, globals)
    except AssertionError:
        if not quiet:
            # Parsing from strings not fully supported (e.g. cimports).
            print("Could not parse code as a string (to extract unbound symbols).")

    cimports = []
    for name, arg in list(kwds.items()):
        if arg is cython_module:
            cimports.append('\ncimport cython as %s' % name)
            del kwds[name]
    arg_names = sorted(kwds)
    arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
    key_hash = _inline_key(orig_code, arg_sigs, language_level)
    module_name = "_cython_inline_" + key_hash

    if module_name in sys.modules:
        module = sys.modules[module_name]

    else:
        build_extension = None
        if cython_inline.so_ext is None:
            # Figure out and cache current extension suffix
            build_extension = _get_build_extension()
            cython_inline.so_ext = build_extension.get_ext_filename('')

        module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)

        if not os.path.exists(lib_dir):
            os.makedirs(lib_dir)
        if force or not os.path.isfile(module_path):
            cflags = []
            c_include_dirs = []
            qualified = re.compile(r'([.\w]+)[.]')
            for type, _ in arg_sigs:
                m = qualified.match(type)
                if m:
                    cimports.append('\ncimport %s' % m.groups()[0])
                    # one special case
                    if m.groups()[0] == 'numpy':
                        import numpy
                        c_include_dirs.append(numpy.get_include())
                        # cflags.append('-Wno-unused')
            module_body, func_body = extract_func_code(code)
            params = ', '.join(['%s %s' % a for a in arg_sigs])
            module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
    return locals()
            """ % {'cimports': '\n'.join(cimports),
                   'module_body': module_body,
                   'params': params,
                   'func_body': func_body }
            for key, value in literals.items():
                module_code = module_code.replace(key, value)
            pyx_file = os.path.join(lib_dir, module_name + '.pyx')
            fh = open(pyx_file, 'w')
            try:
                fh.write(module_code)
            finally:
                fh.close()
            extension = Extension(
                name = module_name,
                sources = [pyx_file],
                include_dirs = c_include_dirs,
                extra_compile_args = cflags)
            if build_extension is None:
                build_extension = _get_build_extension()
            build_extension.extensions = cythonize(
                [extension],
                include_path=cython_include_dirs or ['.'],
                compiler_directives=cython_compiler_directives,
                quiet=quiet)
            build_extension.build_temp = os.path.dirname(pyx_file)
            build_extension.build_lib  = lib_dir
            build_extension.run()

        module = load_dynamic(module_name, module_path)

    _cython_inline_cache[orig_code, arg_sigs, key_hash] = module.__invoke
    arg_list = [kwds[arg] for arg in arg_names]
    return module.__invoke(*arg_list)

# Cached suffix used by cython_inline above.  None should get
# overridden with actual value upon the first cython_inline invocation
cython_inline.so_ext = None

_find_non_space = re.compile('[^ ]').search

def strip_common_indent(code):
    min_indent = None
    lines = code.splitlines()
    for line in lines:
        match = _find_non_space(line)
        if not match:
            continue  # blank
        indent = match.start()
        if line[indent] == '#':
            continue  # comment
        if min_indent is None or min_indent > indent:
            min_indent = indent
    for ix, line in enumerate(lines):
        match = _find_non_space(line)
        if not match or not line or line[indent:indent+1] == '#':
            continue
        lines[ix] = line[min_indent:]
    return '\n'.join(lines)

module_statement = re.compile(r'^((cdef +(extern|class))|cimport|(from .+ cimport)|(from .+ import +[*]))')
def extract_func_code(code):
    module = []
    function = []
    current = function
    code = code.replace('\t', ' ')
    lines = code.split('\n')
    for line in lines:
        if not line.startswith(' '):
            if module_statement.match(line):
                current = module
            else:
                current = function
        current.append(line)
    return '\n'.join(module), '    ' + '\n    '.join(function)

try:
    from inspect import getcallargs
except ImportError:
    def getcallargs(func, *arg_values, **kwd_values):
        all = {}
        args, varargs, kwds, defaults = inspect.getargspec(func)
        if varargs is not None:
            all[varargs] = arg_values[len(args):]
        for name, value in zip(args, arg_values):
            all[name] = value
        for name, value in list(kwd_values.items()):
            if name in args:
                if name in all:
                    raise TypeError("Duplicate argument %s" % name)
                all[name] = kwd_values.pop(name)
        if kwds is not None:
            all[kwds] = kwd_values
        elif kwd_values:
            raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values))
        if defaults is None:
            defaults = ()
        first_default = len(args) - len(defaults)
        for ix, name in enumerate(args):
            if name not in all:
                if ix >= first_default:
                    all[name] = defaults[ix - first_default]
                else:
                    raise TypeError("Missing argument: %s" % name)
        return all

def get_body(source):
    ix = source.index(':')
    if source[:5] == 'lambda':
        return "return %s" % source[ix+1:]
    else:
        return source[ix+1:]

# Lots to be done here... It would be especially cool if compiled functions
# could invoke each other quickly.
class RuntimeCompiledFunction(object):

    def __init__(self, f):
        self._f = f
        self._body = get_body(inspect.getsource(f))

    def __call__(self, *args, **kwds):
        all = getcallargs(self._f, *args, **kwds)
        if IS_PY3:
            return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all)
        else:
            return cython_inline(self._body, locals=self._f.func_globals, globals=self._f.func_globals, **all)
<commit_msg>Clean up some code in cython.inline() implementation.<commit_after>from __future__ import absolute_import

import hashlib
import inspect
import os
import re
import sys

from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext

import Cython
from ..Compiler.Main import Context, default_options

from ..Compiler.Visitor import CythonTransform, EnvTransform
from ..Compiler.ParseTreeTransforms import SkipDeclarations
from ..Compiler.TreeFragment import parse_from_strings
from ..Compiler.StringEncoding import _unicode
from .Dependencies import strip_string_literals, cythonize, cached_function
from ..Compiler import Pipeline
from ..Utils import get_cython_cache_dir
import cython as cython_module

IS_PY3 = sys.version_info >= (3,)

# A utility function to convert user-supplied ASCII strings to unicode.
if not IS_PY3:
    def to_unicode(s):
        if isinstance(s, bytes):
            return s.decode('ascii')
        else:
            return s
else:
    to_unicode = lambda x: x

if sys.version_info < (3, 5):
    import imp
    def load_dynamic(name, module_path):
        return imp.load_dynamic(name, module_path)
else:
    import importlib.util as _importlib_util
    def load_dynamic(name, module_path):
        spec = _importlib_util.spec_from_file_location(name, module_path)
        module = _importlib_util.module_from_spec(spec)
        # sys.modules[name] = module
        spec.loader.exec_module(module)
        return module

class UnboundSymbols(EnvTransform, SkipDeclarations):
    def __init__(self):
        CythonTransform.__init__(self, None)
        self.unbound = set()
    def visit_NameNode(self, node):
        if not self.current_env().lookup(node.name):
            self.unbound.add(node.name)
        return node
    def __call__(self, node):
        super(UnboundSymbols, self).__call__(node)
        return self.unbound

@cached_function
def unbound_symbols(code, context=None):
    code = to_unicode(code)
    if context is None:
        context = Context([], default_options)
    from ..Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform
    tree = parse_from_strings('(tree fragment)', code)
    for phase in Pipeline.create_pipeline(context, 'pyx'):
        if phase is None:
            continue
        tree = phase(tree)
        if isinstance(phase, AnalyseDeclarationsTransform):
            break
    try:
        import builtins
    except ImportError:
        import __builtin__ as builtins
    return tuple(UnboundSymbols()(tree) - set(dir(builtins)))

def unsafe_type(arg, context=None):
    py_type = type(arg)
    if py_type is int:
        return 'long'
    else:
        return safe_type(arg, context)

def safe_type(arg, context=None):
    py_type = type(arg)
    if py_type in (list, tuple, dict, str):
        return py_type.__name__
    elif py_type is complex:
        return 'double complex'
    elif py_type is float:
        return 'double'
    elif py_type is bool:
        return 'bint'
    elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray):
        return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim)
    else:
        for base_type in py_type.__mro__:
            if base_type.__module__ in ('__builtin__', 'builtins'):
                return 'object'
            module = context.find_module(base_type.__module__, need_pxd=False)
            if module:
                entry = module.lookup(base_type.__name__)
                if entry.is_type:
                    return '%s.%s' % (base_type.__module__, base_type.__name__)
        return 'object'

def _get_build_extension():
    dist = Distribution()
    # Ensure the build respects distutils configuration by parsing
    # the configuration files
    config_files = dist.find_config_files()
    dist.parse_config_files(config_files)
    build_extension = build_ext(dist)
    build_extension.finalize_options()
    return build_extension

@cached_function
def _create_context(cython_include_dirs):
    return Context(list(cython_include_dirs), default_options)

_cython_inline_cache = {}
_cython_inline_default_context = _create_context(('.',))

def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None):
    for symbol in unbound_symbols:
        if symbol not in kwds:
            if locals is None or globals is None:
                calling_frame = inspect.currentframe().f_back.f_back.f_back
                if locals is None:
                    locals = calling_frame.f_locals
                if globals is None:
                    globals = calling_frame.f_globals
            if symbol in locals:
                kwds[symbol] = locals[symbol]
            elif symbol in globals:
                kwds[symbol] = globals[symbol]
            else:
                print("Couldn't find %r" % symbol)

def _inline_key(orig_code, arg_sigs, language_level):
    key = orig_code, arg_sigs, sys.version_info, sys.executable, language_level, Cython.__version__
    return hashlib.sha1(_unicode(key).encode('utf-8')).hexdigest()

def cython_inline(code, get_type=unsafe_type,
                  lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
                  cython_include_dirs=None, cython_compiler_directives=None,
                  force=False, quiet=False, locals=None, globals=None, language_level=None, **kwds):

    if get_type is None:
        get_type = lambda x: 'object'
    ctx = _create_context(tuple(cython_include_dirs)) if cython_include_dirs else _cython_inline_default_context

    cython_compiler_directives = dict(cython_compiler_directives) if cython_compiler_directives else {}
    if language_level is None and 'language_level' not in cython_compiler_directives:
        language_level = '3str'
    if language_level is not None:
        cython_compiler_directives['language_level'] = language_level

    # Fast path if this has been called in this session.
    _unbound_symbols = _cython_inline_cache.get(code)
    if _unbound_symbols is not None:
        _populate_unbound(kwds, _unbound_symbols, locals, globals)
        args = sorted(kwds.items())
        arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args])
        key_hash = _inline_key(code, arg_sigs, language_level)
        invoke = _cython_inline_cache.get((code, arg_sigs, key_hash))
        if invoke is not None:
            arg_list = [arg[1] for arg in args]
            return invoke(*arg_list)

    orig_code = code
    code = to_unicode(code)
    code, literals = strip_string_literals(code)
    code = strip_common_indent(code)
    if locals is None:
        locals = inspect.currentframe().f_back.f_back.f_locals
    if globals is None:
        globals = inspect.currentframe().f_back.f_back.f_globals
    try:
        _cython_inline_cache[orig_code] = _unbound_symbols = unbound_symbols(code)
        _populate_unbound(kwds, _unbound_symbols, locals, globals)
    except AssertionError:
        if not quiet:
            # Parsing from strings not fully supported (e.g. cimports).
            print("Could not parse code as a string (to extract unbound symbols).")

    cimports = []
    for name, arg in list(kwds.items()):
        if arg is cython_module:
            cimports.append('\ncimport cython as %s' % name)
            del kwds[name]
    arg_names = sorted(kwds)
    arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
    key_hash = _inline_key(orig_code, arg_sigs, language_level)
    module_name = "_cython_inline_" + key_hash

    if module_name in sys.modules:
        module = sys.modules[module_name]

    else:
        build_extension = None
        if cython_inline.so_ext is None:
            # Figure out and cache current extension suffix
            build_extension = _get_build_extension()
            cython_inline.so_ext = build_extension.get_ext_filename('')

        module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)

        if not os.path.exists(lib_dir):
            os.makedirs(lib_dir)
        if force or not os.path.isfile(module_path):
            cflags = []
            c_include_dirs = []
            qualified = re.compile(r'([.\w]+)[.]')
            for type, _ in arg_sigs:
                m = qualified.match(type)
                if m:
                    cimports.append('\ncimport %s' % m.groups()[0])
                    # one special case
                    if m.groups()[0] == 'numpy':
                        import numpy
                        c_include_dirs.append(numpy.get_include())
                        # cflags.append('-Wno-unused')
            module_body, func_body = extract_func_code(code)
            params = ', '.join(['%s %s' % a for a in arg_sigs])
            module_code = """
%(module_body)s
%(cimports)s
def __invoke(%(params)s):
%(func_body)s
    return locals()
            """ % {'cimports': '\n'.join(cimports),
                   'module_body': module_body,
                   'params': params,
                   'func_body': func_body }
            for key, value in literals.items():
                module_code = module_code.replace(key, value)
            pyx_file = os.path.join(lib_dir, module_name + '.pyx')
            fh = open(pyx_file, 'w')
            try:
                fh.write(module_code)
            finally:
                fh.close()
            extension = Extension(
                name = module_name,
                sources = [pyx_file],
                include_dirs = c_include_dirs,
                extra_compile_args = cflags)
            if build_extension is None:
                build_extension = _get_build_extension()
            build_extension.extensions = cythonize(
                [extension],
                include_path=cython_include_dirs or ['.'],
                compiler_directives=cython_compiler_directives,
                quiet=quiet)
            build_extension.build_temp = os.path.dirname(pyx_file)
            build_extension.build_lib  = lib_dir
            build_extension.run()

        module = load_dynamic(module_name, module_path)

    _cython_inline_cache[orig_code, arg_sigs, key_hash] = module.__invoke
    arg_list = [kwds[arg] for arg in arg_names]
    return module.__invoke(*arg_list)

# Cached suffix used by cython_inline above.  None should get
# overridden with actual value upon the first cython_inline invocation
cython_inline.so_ext = None

_find_non_space = re.compile('[^ ]').search

def strip_common_indent(code):
    min_indent = None
    lines = code.splitlines()
    for line in lines:
        match = _find_non_space(line)
        if not match:
            continue  # blank
        indent = match.start()
        if line[indent] == '#':
            continue  # comment
        if min_indent is None or min_indent > indent:
            min_indent = indent
    for ix, line in enumerate(lines):
        match = _find_non_space(line)
        if not match or not line or line[indent:indent+1] == '#':
            continue
        lines[ix] = line[min_indent:]
    return '\n'.join(lines)

module_statement = re.compile(r'^((cdef +(extern|class))|cimport|(from .+ cimport)|(from .+ import +[*]))')
def extract_func_code(code):
    module = []
    function = []
    current = function
    code = code.replace('\t', ' ')
    lines = code.split('\n')
    for line in lines:
        if not line.startswith(' '):
            if module_statement.match(line):
                current = module
            else:
                current = function
        current.append(line)
    return '\n'.join(module), '    ' + '\n    '.join(function)

try:
    from inspect import getcallargs
except ImportError:
    def getcallargs(func, *arg_values, **kwd_values):
        all = {}
        args, varargs, kwds, defaults = inspect.getargspec(func)
        if varargs is not None:
            all[varargs] = arg_values[len(args):]
        for name, value in zip(args, arg_values):
            all[name] = value
        for name, value in list(kwd_values.items()):
            if name in args:
                if name in all:
                    raise TypeError("Duplicate argument %s" % name)
                all[name] = kwd_values.pop(name)
        if kwds is not None:
            all[kwds] = kwd_values
        elif kwd_values:
            raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values))
        if defaults is None:
            defaults = ()
        first_default = len(args) - len(defaults)
        for ix, name in enumerate(args):
            if name not in all:
                if ix >= first_default:
                    all[name] = defaults[ix - first_default]
                else:
                    raise TypeError("Missing argument: %s" % name)
        return all

def get_body(source):
    ix = source.index(':')
    if source[:5] == 'lambda':
        return "return %s" % source[ix+1:]
    else:
        return source[ix+1:]

# Lots to be done here... It would be especially cool if compiled functions
# could invoke each other quickly.
class RuntimeCompiledFunction(object):

    def __init__(self, f):
        self._f = f
        self._body = get_body(inspect.getsource(f))

    def __call__(self, *args, **kwds):
        all = getcallargs(self._f, *args, **kwds)
        if IS_PY3:
            return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all)
        else:
            return cython_inline(self._body, locals=self._f.func_globals, globals=self._f.func_globals, **all)
<|endoftext|>