<commit_before>"""
#;+
#; NAME:
#; qso_pca
#;    Version 1.0
#;
#; PURPOSE:
#;    Module for generate QSO PCA templates
#;   24-Nov-2014 by JXP
#;-
#;------------------------------------------------------------------------------
"""
from __future__ import print_function, absolute_import, division, unicode_literals

import numpy as np
import os

from astropy.io import fits

flg_xdb = True
try:
    from xastropy.xutils import xdebug as xdb
except ImportError:
    flg_xdb = False

"""
## Tinkering about
#   JXP on 01 Dec 2014

hdu = fits.open('spEigenQSO-55732.fits')
eigen = hdu[0].data  # There are 4 eigenvectors
wav = 10.**(2.6534 + np.arange(13637)*0.0001)  # Rest-frame, Ang
xdb.xplot(wav,eigen[0,:])
  # Looks sensible enough

# QSO sample used to generate the PCA eigenvectors
qsos = hdu[1].data
xdb.xhist(qsos['REDSHIFT']) # Skewed to high z
"""

"""
Let's test in IDL on a BOSS spectrum

cd /Users/xavier/BOSS/DR10/BOSSLyaDR10_spectra_v2.1/5870

IDL> boss_objinf, [5870,1000], /plot
    Name             RA(2000)   DEC(2000) Mag(R) zobj
5870-56065-100     14:07:37.28 +21:48:38.5 21.06 2.180

Looks like a fine quasar

Read the eigenspectra

eigen = xmrdfits(getenv('IDLSPEC2D_DIR')+'/templates/spEigenQSO-55732.fits',0,head)
eigen_wave = 10.^(sxpar(head,'COEFF0') + dindgen(sxpar(head,'NAXIS1'))*sxpar(head,'COEFF1'))

BOSS spectrum

boss_readspec, 'speclya-5870-56065-1000.fits', flux, wave, ivar=ivar
zqso = 2.180
wrest  = wave / (1+zqso)
npix = n_elements(wrest)

Are they 'registered'?
mn = min(abs(wrest[0]-eigen_wave),imn)
x_splot, findgen(npix), flux, ytwo=eigen[imn:*,0]/5292*4.4
  Yes, they are..

Build the inputs to computechi2

objflux = flux
sqivar = ivar
imx = npix+imn-1
starflux = eigen[imn:imx,*]

chi2 = computechi2(objflux, sqivar, starflux, acoeff=acoeff, yfit=yfit)
  ;; The fit looks ok

IDL> print, acoeff
   0.00045629701  -0.00034376233   0.00047730298   0.00045008259
"""

# Read Eigenspectrum

eigen_fil = os.environ.get('IDLSPEC2D_DIR')+'/templates/spEigenQSO-55732.fits'
print('Using these eigen spectra: {:s}'.format(eigen_fil))
hdu = fits.open(eigen_fil)
eigen = hdu[0].data
head = hdu[0].header
eigen_wave = 10.**(head['COEFF0'] + np.arange(head['NAXIS1'])*head['COEFF1'])

#xdb.xplot(eigen_wave, eigen[0,:])
#xdb.set_trace()

#BOSS spectrum

boss_fil = '/Users/xavier/BOSS/DR10/BOSSLyaDR10_spectra_v2.1/5870/speclya-5870-56065-1000.fits.gz'
print('Fitting this spectrum: {:s}'.format(boss_fil))
boss_hdu = fits.open(boss_fil)
t = boss_hdu[1].data
flux = t['flux']
wave = 10.**t['loglam']
ivar = t['ivar']

zqso = 2.180
wrest  = wave / (1+zqso)
npix = len(wrest)

imn = np.argmin(np.abs(wrest[0]-eigen_wave))
imx = npix+imn
eigen_flux = eigen[:,imn:imx]

# Generate the matrices
"""
Now Python with simple approach following Dan F-M http://dan.iel.fm/emcee/current/user/line/
"""

C = np.diag(1./ivar)
Cinv = np.diag(ivar)
A = eigen_flux.T
y = flux

#alpha = np.dot(A.T, np.linalg.solve(C, A))  # Numerical Recipe notation
alpha = np.dot(A.T, np.dot(Cinv,A))
cov = np.linalg.inv(alpha)
#beta = np.dot(A.T, np.linalg.solve(C, y))
beta = np.dot(A.T, np.dot(Cinv, y))
acoeff = np.dot(cov, beta)

print('acoeff = {:g}, {:g}, {:g}, {:g}'.format(*acoeff))

#In [48]: acoeff
#Out[48]: array([ 0.00045461, -0.00012764,  0.00010843,  0.0003281 ], dtype=float32)

model = np.dot(A,acoeff)
if flg_xdb is True:
    xdb.xplot(wrest, flux, model)
  # Looks good to me
<commit_msg>commented out<commit_after>"""
#;+
#; NAME:
#; qso_pca
#;    Version 1.0
#;
#; PURPOSE:
#;    Module for generate QSO PCA templates
#;   24-Nov-2014 by JXP
#;-
#;------------------------------------------------------------------------------
"""
from __future__ import print_function, absolute_import, division, unicode_literals

import numpy as np
import os

from astropy.io import fits

flg_xdb = True
try:
    from xastropy.xutils import xdebug as xdb
except ImportError:
    flg_xdb = False

"""
## Tinkering about
#   JXP on 01 Dec 2014

hdu = fits.open('spEigenQSO-55732.fits')
eigen = hdu[0].data  # There are 4 eigenvectors
wav = 10.**(2.6534 + np.arange(13637)*0.0001)  # Rest-frame, Ang
xdb.xplot(wav,eigen[0,:])
  # Looks sensible enough

# QSO sample used to generate the PCA eigenvectors
qsos = hdu[1].data
xdb.xhist(qsos['REDSHIFT']) # Skewed to high z
"""

"""
Let's test in IDL on a BOSS spectrum

cd /Users/xavier/BOSS/DR10/BOSSLyaDR10_spectra_v2.1/5870

IDL> boss_objinf, [5870,1000], /plot
    Name             RA(2000)   DEC(2000) Mag(R) zobj
5870-56065-100     14:07:37.28 +21:48:38.5 21.06 2.180

Looks like a fine quasar

Read the eigenspectra

eigen = xmrdfits(getenv('IDLSPEC2D_DIR')+'/templates/spEigenQSO-55732.fits',0,head)
eigen_wave = 10.^(sxpar(head,'COEFF0') + dindgen(sxpar(head,'NAXIS1'))*sxpar(head,'COEFF1'))

BOSS spectrum

boss_readspec, 'speclya-5870-56065-1000.fits', flux, wave, ivar=ivar
zqso = 2.180
wrest  = wave / (1+zqso)
npix = n_elements(wrest)

Are they 'registered'?
mn = min(abs(wrest[0]-eigen_wave),imn)
x_splot, findgen(npix), flux, ytwo=eigen[imn:*,0]/5292*4.4
  Yes, they are..

Build the inputs to computechi2

objflux = flux
sqivar = ivar
imx = npix+imn-1
starflux = eigen[imn:imx,*]

chi2 = computechi2(objflux, sqivar, starflux, acoeff=acoeff, yfit=yfit)
  ;; The fit looks ok

IDL> print, acoeff
   0.00045629701  -0.00034376233   0.00047730298   0.00045008259
"""

"""
# Read Eigenspectrum

eigen_fil = os.environ.get('IDLSPEC2D_DIR')+'/templates/spEigenQSO-55732.fits'
print('Using these eigen spectra: {:s}'.format(eigen_fil))
hdu = fits.open(eigen_fil)
eigen = hdu[0].data
head = hdu[0].header
eigen_wave = 10.**(head['COEFF0'] + np.arange(head['NAXIS1'])*head['COEFF1'])

#xdb.xplot(eigen_wave, eigen[0,:])
#xdb.set_trace()

#BOSS spectrum

boss_fil = '/Users/xavier/BOSS/DR10/BOSSLyaDR10_spectra_v2.1/5870/speclya-5870-56065-1000.fits.gz'
print('Fitting this spectrum: {:s}'.format(boss_fil))
boss_hdu = fits.open(boss_fil)
t = boss_hdu[1].data
flux = t['flux']
wave = 10.**t['loglam']
ivar = t['ivar']

zqso = 2.180
wrest  = wave / (1+zqso)
npix = len(wrest)

imn = np.argmin(np.abs(wrest[0]-eigen_wave))
imx = npix+imn
eigen_flux = eigen[:,imn:imx]

# Generate the matrices
'''
Now Python with simple approach following Dan F-M http://dan.iel.fm/emcee/current/user/line/
'''

C = np.diag(1./ivar)
Cinv = np.diag(ivar)
A = eigen_flux.T
y = flux

#alpha = np.dot(A.T, np.linalg.solve(C, A))  # Numerical Recipe notation
alpha = np.dot(A.T, np.dot(Cinv,A))
cov = np.linalg.inv(alpha)
#beta = np.dot(A.T, np.linalg.solve(C, y))
beta = np.dot(A.T, np.dot(Cinv, y))
acoeff = np.dot(cov, beta)

print('acoeff = {:g}, {:g}, {:g}, {:g}'.format(*acoeff))

#In [48]: acoeff
#Out[48]: array([ 0.00045461, -0.00012764,  0.00010843,  0.0003281 ], dtype=float32)

model = np.dot(A,acoeff)
if flg_xdb is True:
    xdb.xplot(wrest, flux, model)
  # Looks good to me
"""
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.safestring import mark_safe
from table.utils import Accessor
from .base import Column

class LinkColumn(Column):
    def __init__(self, links, delimiter=u' ', *args, **kwargs):
        super(LinkColumn, self).__init__(*args, **kwargs)
        self.links = links
        self.delimiter = delimiter
        self.searchable = False
        self.safe = False

    def render(self, obj):
        return self.delimiter.join([link.render(obj) for link in self.links])

class Link(object):
    """ Represents a link element in html.
    """
    def __init__(self, text, viewname, args=None, kwargs=None, urlconf=None,
                 current_app=None, onclick=None):
        self.text = text
        self.viewname = viewname
        self.args = args or []
        self.kwargs = kwargs or {}
        self.urlconf = urlconf
        self.current_app = current_app
        self.onclick = onclick

    def resolve(self, obj):
        """ Resolving URL paths to the corresponding object. See:
            https://docs.djangoproject.com/en/dev/ref/urlresolvers/#reverse
        """
        viewname = self.viewname
        params = {}

        if self.args:
            params['args'] = [getattr(obj, attr) for attr in self.args]
        if self.kwargs:
            params['kwargs'] = {}
            for key, value in self.kwargs.itmes:
                params['kwargs'][key] = getattr(obj, value)
        if self.urlconf:
            params['urlconf'] = self.urlconf
        if self.current_app:
            params['current_app'] = self.current_app

        try:
            url = reverse(viewname, **params)
        except NoReverseMatch:
            url = ''
        return url

    def render(self, obj):
        """ Render link as HTML output tag .
        """
        url = self.resolve(obj)
        text = self.text.resolve(obj) if isinstance(self.text, Accessor) else self.text
        if self.onclick:
            onclick = self.onclick + "('" + url + "')"
            return mark_safe(u'%s' % (onclick, text))
        else:
            return mark_safe(u'%s' % (url, text))

<commit_msg>remove unused option: onclick<commit_after>#!/usr/bin/env python
# coding: utf-8

from django.core.urlresolvers import reverse, NoReverseMatch
from django.utils.safestring import mark_safe
from table.utils import Accessor
from .base import Column

class LinkColumn(Column):
    def __init__(self, links, delimiter=u' ', *args, **kwargs):
        super(LinkColumn, self).__init__(*args, **kwargs)
        self.links = links
        self.delimiter = delimiter
        self.searchable = False
        self.safe = False

    def render(self, obj):
        return self.delimiter.join([link.render(obj) for link in self.links])

class Link(object):
    """ Represents a link element in html.
    """
    def __init__(self, text, viewname, args=None, kwargs=None, urlconf=None,
                 current_app=None):
        self.text = text
        self.viewname = viewname
        self.args = args or []
        self.kwargs = kwargs or {}
        self.urlconf = urlconf
        self.current_app = current_app

    def resolve(self, obj):
        """ Resolving URL paths to the corresponding object. See:
            https://docs.djangoproject.com/en/dev/ref/urlresolvers/#reverse
        """
        viewname = self.viewname
        params = {}

        if self.args:
            params['args'] = [getattr(obj, attr) for attr in self.args]
        if self.kwargs:
            params['kwargs'] = {}
            for key, value in self.kwargs.itmes:
                params['kwargs'][key] = getattr(obj, value)
        if self.urlconf:
            params['urlconf'] = self.urlconf
        if self.current_app:
            params['current_app'] = self.current_app

        try:
            url = reverse(viewname, **params)
        except NoReverseMatch:
            url = ''
        return url

    def render(self, obj):
        """ Render link as HTML output tag .
        """
        url = self.resolve(obj)
        text = self.text.resolve(obj) if isinstance(self.text, Accessor) else self.text
        return mark_safe(u'%s' % (url, text))

<|endoftext|>


<commit_before>from django.conf import settings
from django.core.mail import get_connection
from django.core.mail.message import EmailMultiAlternatives

NO_HTML_EMAIL_MESSAGE = """
Your email client is trying to display the plaintext version of an email that
is only supported in HTML. Please set your email client to display this message
in HTML, or use an email client that supports HTML emails.
"""

def send_HTML_email(subject, recipient, html_content, text_content=None, cc=None):
    if not text_content:
        text_content = getattr(settings, 'NO_HTML_EMAIL_MESSAGE',
                               NO_HTML_EMAIL_MESSAGE)

    # If you get the return_path header wrong, this may impede mail delivery. It appears that the SMTP server
    # has to recognize the return_path as being valid for the sending host. If we set it to, say, our SMTP
    # server, this will always be the case (as the server is explicitly serving the host).
    email_return_path = getattr(settings, 'HQ_NOTIFICATIONS_EMAIL', None)
    if email_return_path is None:
        # Get last two parts of the SMTP server as a proxy for the domain name from which this mail is sent.
        # This works for gmail, anyway.
        email_return_path = settings.EMAIL_HOST_USER

    email_from = getattr(settings, 'HQ_NOTIFICATIONS_EMAIL', None)
    if email_from is None:
        email_from = email_return_path
    from_header = {'From': email_from}  # From-header
    connection = get_connection()

    msg = EmailMultiAlternatives(subject, text_content, email_return_path, [recipient], headers=from_header, connection=connection, cc=cc)
    msg.attach_alternative(html_content, "text/html")
    msg.send()
<commit_msg>cleanup and adding email_from kwarg to allow override of from field in cases where the in built django assumptions are not valid email addresses (amazon SES)<commit_after>from django.conf import settings
from django.core.mail import get_connection
from django.core.mail.message import EmailMultiAlternatives

NO_HTML_EMAIL_MESSAGE = """
Your email client is trying to display the plaintext version of an email that
is only supported in HTML. Please set your email client to display this message
in HTML, or use an email client that supports HTML emails.
"""

def send_HTML_email(subject, recipient, html_content, text_content=None, cc=None, email_from=None):
    if not text_content:
        text_content = getattr(settings, 'NO_HTML_EMAIL_MESSAGE',
                               NO_HTML_EMAIL_MESSAGE)

    # If you get the return_path header wrong, this may impede mail delivery. It appears that the SMTP server
    # has to recognize the return_path as being valid for the sending host. If we set it to, say, our SMTP
    # server, this will always be the case (as the server is explicitly serving the host).
    if email_from is None:
        #todo: verify that this is even necessary here since it seems like email_return_path == email_from
        email_return_path = getattr(settings, 'EMAIL_RETURN_PATH', None)
        if email_return_path is None:
            email_return_path = settings.EMAIL_LOGIN

        email_from = getattr(settings, 'EMAIL_FROM', None)
        if email_from is None:
            email_from = email_return_path
    else:
        email_return_path = email_from

    from_header = {'From': email_from}  # From-header
    connection = get_connection()

    msg = EmailMultiAlternatives(subject, text_content, email_return_path, [recipient], headers=from_header, connection=connection, cc=cc)
    msg.attach_alternative(html_content, "text/html")
    msg.send()
<|endoftext|>


<commit_before>"""Classes that communicate the results."""

import curses
import threading
import formatter
import executor
import functools
from i18n import translate as _

# TODO: thread mutex
# TODO: cmdline_args, input, tempfile in begin_test
# TODO: help page
# TODO: summary page
# TODO: test info page
# TODO: i18n

_CALLBACKS = {}

def _register_key(*keys):
    """Decorator registering key->callback pairs.

    Keys can be strings or character codes."""
    def decorator(f):
        for k in keys:
            if isinstance(k, str):
                k = ord(k)
            _CALLBACKS[k] = f
        return f
    return decorator

def _synchronized(f):
    @functools.wraps(f)
    def decorated(self, *args, **kwargs):
        # with self._mutex:
        return f(self, *args, **kwargs)
    return decorated

class InteractiveFormatter(formatter.Formatter):
    """Formatter that uses curses to report the result."""

    FOOTER_H = 2  # height of the footer
    MAX_W = 512   # max line length

    COLOR_OK = 1
    COLOR_WARN = 2
    COLOR_ERR = 3

    _RESULT_TABLE = {
        executor.ER_OK: None,
        executor.ER_TIMEOUT: _("TIMEOUT EXPIRED: PROCESS TERMINATED"),
        executor.ER_OUTPUT_LIMIT: _("TOO MANY OUTPUT LINES"),
        executor.ER_SEGFAULT: _("PROCESS ENDED WITH A FAILURE (SEGMENTATION FAULT)"),
        executor.ER_ERROR: ("PROCESS ENDED WITH A FAILURE (ERROR CODE {status})"),
        executor.ER_NOTFILE: _("FAILED TO RUN THE FILE '{progname}' the file does not exist)")
    }

    def __init__(self):
        """Create the interactive formatter."""
        self._reports = []
        self._report_index = 0
        self._screen = None
        self._mutex = threading.Lock()

    @_register_key("q", "Q", 27)  # 27 -> ESC
    def _quit(self):
        self._stop = True

    @_register_key("p", "P", curses.KEY_UP)
    def scroll_line_up(self):
        self._reports[self._report_index].scroll(self._text_height(), -1)
        self._update()

    @_register_key("n", "N", curses.KEY_DOWN, 10)  # 10 -> ENTER
    def scroll_line_down(self):
        self._reports[self._report_index].scroll(self._text_height(), 1)
        self._update()

    @_register_key(curses.KEY_PPAGE)
    def scroll_page_up(self):
        self._reports[self._report_index].scroll(self._text_height(), pages=-1)
        self._update()

    @_register_key(curses.KEY_NPAGE)
    def scroll_page_down(self):
        self._reports[self._report_index].scroll(self._text_height(), pages=1)
        self._update()

    @_register_key(curses.KEY_HOME)
    def scroll_begin(self):
        self._reports[self._report_index].scroll(self._text_height(), documents=-1)
        self._update()

    @_register_key(curses.KEY_END)
    def scroll_end(self):
        self._reports[self._report_index].scroll(self._text_height(), documents=1)
        self._update()

    @_register_key("r", "R", curses.KEY_RESIZE)
    def resize(self):
        self._update()

    @_register_key(curses.KEY_LEFT)
    def previous_document(self):
        self._screen.clear()
        if len(self._reports) > 1:
            self._report_index = max(self._report_index - 1, 1)
        self._update()

    @_register_key(curses.KEY_RIGHT)
    def next_document(self):
        self._screen.clear()
        self._report_index = min(self._report_index + 1, len(self._reports) - 1)
        self._update()

    def _thread_body(self):
        """UI thread."""
        self._stop = False
        curses.wrapper(self._main_loop)

    def _main_loop(self, screen):
        """Main loop managing the interaction with the user."""
        with self._mutex:
            self._screen = screen
            curses.use_default_colors()
            curses.init_pair(self.COLOR_OK, curses.COLOR_GREEN, -1)
            curses.init_pair(self.COLOR_WARN, curses.COLOR_YELLOW, -1)
            curses.init_pair(self.COLOR_ERR, curses.COLOR_RED, -1)
            self._footer = curses.newwin(self.FOOTER_H, self._text_width(), self._text_height(), 0)
            self._footer.bkgd(" ", curses.A_REVERSE)
            self._reports.append(Report("PVCHECK", self.MAX_W))
            self._reports[-1].add_line("Waiting for test results...")
            self._report_index = 0
            self._update()
        while not self._stop:
            ch = screen.getch()
            with self._mutex:
                _CALLBACKS.get(ch, lambda self: None)(self)
                with open("post.p", "a") as f:
                    print("!!!!!!!!!!!!!!!!!!!", ch, chr(ch), file=f, flush=True)

    def _text_height(self):
        """Number of text lines displayed."""
        return self._screen.getmaxyx()[0] - self.FOOTER_H

    def _text_width(self):
        """Width of the displayed text."""
        return min(self._screen.getmaxyx()[1], self.MAX_W)

    def _add_footer(self, line, align, text, *extra):
        k = self._text_width() - 1 - len(text)
        pos = max(0, (0 if align == "left" else (k if align == "right" else k //2 )))
        self._footer.addnstr(line, pos, text, self._text_width() - 1 - pos, *extra)

    def _add_short_report(self):
        texts = [
            "%3d " % self._ok_count, _(" passes, "),
            "%3d " % self._warn_count, _(" warnings, "),
            "%3d " % self._err_count, _(" errors")
        ]
        styles = [
            [curses.color_pair(self.COLOR_OK)], [],
            [curses.color_pair(self.COLOR_WARN)], [],
            [curses.color_pair(self.COLOR_ERR)], []
        ]
        n = self._text_width() - 1
        pos = 0
        for t, s in zip (texts, styles):
            self._footer.addnstr(1, pos, t, n, *s)
            pos += len(t)
            n -= len(t)

    def _update(self):
        """Redraw everything."""
        if self._screen is None:
            return
        self._screen.refresh()
        height = self._text_height()
        width = self._text_width()
        doc = self._reports[self._report_index]
        self._footer.mvwin(height, 0)
        doc.refresh(self._text_height(), self._text_width())
        self._footer.clear()
        self._add_footer(0, "center", _("[Press 'h' for help]"), curses.A_DIM)
        text = _("Test case %d of %d (%s) ") % (self._report_index, len(self._reports) - 1, doc.title)
        self._add_footer(0, "left", text)
        text = _("Lines {%d}-{%d}/{%d}") % (doc.top(), doc.bottom(height), doc.length())
        self._add_footer(0, "right", text)
        self._add_short_report()
        if self._running:
            tot = self._err_count + self._warn_count + self._ok_count
            text = "TEST RUNNING..." + "|/-\\"[tot % 4]
        else:
            text = "TEST COMPLETED"
        self._add_footer(1, "right", text, curses.A_BOLD)
        self._footer.refresh()

    @_synchronized
    def begin_session(self):
        self._err_count = self._warn_count = self._ok_count = 0
        self._running = True
        # Start the UI thread
        self._thread = threading.Thread(target=self._thread_body)
        self._thread.start()

    @_synchronized
    def end_session(self):
        # Wait the termination of the UI thread
        self._running = False
        self._update()
        self._thread.join()

    @_synchronized
    def begin_test(self, description, cmdline_args, input, tempfile):
        description = description or ""
        self._reports.append(Report(description, self.MAX_W))
        if self._report_index == 0:
            self._report_index = 1
        self._update()

    @_synchronized
    def end_test(self):
        pass

    @_synchronized
    def execution_result(self, cmdline_args, execution_result):
        info = {
            'progname': cmdline_args[0],
            'status': execution_result.status
        }
        message = self._RESULT_TABLE[execution_result.result]
        if execution_result.result != executor.ER_OK:
            self._err_count += 1
            message = message.format(**info)
            for line in message.splitlines():
                self._reports[-1].add_line(line, curses.color_pair(self.COLOR_ERR))
        self._update()

    @_synchronized
    def comparison_result(self, expected, got, diffs, matches):
        add = self._reports[-1].add_line
        all_ok = (max(diffs, default=0) <= 0)
        if all_ok:
            self._ok_count += 1
        else:
            self._err_count += 1
        color = curses.color_pair(self.COLOR_OK if all_ok else self.COLOR_ERR)
        add("[%s]" % expected.tag, color | curses.A_BOLD)

        err_diff = _("%s\t\t(expected '%s')")
        err_un = _("%s\t\t(this line was not expected)")
        err_mis = _("\t\t(missing line '%s')")

        for (i, d) in enumerate(diffs):
            if d <= 0:
                # Correct
                add(got.content[i], curses.color_pair(self.COLOR_OK))
            elif matches[i] is None:
                # Extra line
                add(err_un % got.content[i], curses.color_pair(self.COLOR_ERR))
            elif i >= len(got.content):
                # Missing line
                add(err_mis % matches[i], curses.color_pair(self.COLOR_ERR))
            else:
                # Mismatching lines
                add(err_diff % (got.content[i], matches[i]), curses.color_pair(self.COLOR_ERR))
        add("")
        self._update()

    @_synchronized
    def missing_section(self, expected):
        self._warn_count += 1
        message = _("\t\t(section [%s] is missing)") % expected.tag
        self._reports[-1].add_line(message, curses.color_pair(self.COLOR_WARN))
        self._reports[-1].add_line("")
        self._update()

class Report:
    """Test result displayed on the screen."""

    def __init__(self, title, max_width):
        """Create a new report with the given title.

        Lines are truncated to max_width.
        """
        self.title = title
        self._length = 0
        self._max_width = max_width
        self._pad = curses.newpad(1, max_width)
        self._position = 0

    def add_line(self, line, *extra):
        """Add a line at the bottom of the document."""
        self._pad.resize(self._length + 1, self._max_width)
        self._pad.addnstr(self._length, 0, line, self._max_width, *extra)
        self._length += 1

    def scroll(self, page_height, lines=0, pages=0, documents=0):
        """Scroll up or down."""
        amount = lines + page_height * pages + documents * self._length
        self._position = max(0, min(self._position + amount, self._length - page_height))

    def refresh(self, page_height, page_width):
        """Redraw the document on the screen."""
        self._pad.refresh(self._position, 0, 0, 0, page_height - 1, page_width - 1)

    def top(self):
        """Index of the first line on the screen (starting from 1)."""
        return self._position + 1

    def bottom(self, page_height):
        """Index of the last line on the screen."""
        return min(self._position + page_height, self._length)

    def length(self):
        """Number of lines in the document."""
        return self._length
<commit_msg>Proper (?) thread synchronization<commit_after>"""Classes that communicate the results."""

import curses
import threading
import formatter
import executor
import functools
from i18n import translate as _

# TODO: thread mutex
# TODO: cmdline_args, input, tempfile in begin_test
# TODO: help page
# TODO: summary page
# TODO: test info page
# TODO: i18n

_CALLBACKS = {}

def _register_key(*keys):
    """Decorator registering key->callback pairs.

    Keys can be strings or character codes."""
    def decorator(f):
        for k in keys:
            if isinstance(k, str):
                k = ord(k)
            _CALLBACKS[k] = f
        return f
    return decorator

def _synchronized(f):
    @functools.wraps(f)
    def decorated(self, *args, **kwargs):
        with self._mutex:
            return f(self, *args, **kwargs)
    return decorated

class InteractiveFormatter(formatter.Formatter):
    """Formatter that uses curses to report the result."""

    FOOTER_H = 2  # height of the footer
    MAX_W = 512   # max line length

    COLOR_OK = 1
    COLOR_WARN = 2
    COLOR_ERR = 3

    _RESULT_TABLE = {
        executor.ER_OK: None,
        executor.ER_TIMEOUT: _("TIMEOUT EXPIRED: PROCESS TERMINATED"),
        executor.ER_OUTPUT_LIMIT: _("TOO MANY OUTPUT LINES"),
        executor.ER_SEGFAULT: _("PROCESS ENDED WITH A FAILURE (SEGMENTATION FAULT)"),
        executor.ER_ERROR: ("PROCESS ENDED WITH A FAILURE (ERROR CODE {status})"),
        executor.ER_NOTFILE: _("FAILED TO RUN THE FILE '{progname}' the file does not exist)")
    }

    def __init__(self):
        """Create the interactive formatter."""
        self._reports = []
        self._report_index = 0
        self._screen = None
        self._mutex = threading.Lock()

    @_register_key("q", "Q", 27)  # 27 -> ESC
    def _quit(self):
        self._stop = True

    @_register_key("p", "P", curses.KEY_UP)
    def scroll_line_up(self):
        self._reports[self._report_index].scroll(self._text_height(), -1)
        self._update()

    @_register_key("n", "N", curses.KEY_DOWN, 10)  # 10 -> ENTER
    def scroll_line_down(self):
        self._reports[self._report_index].scroll(self._text_height(), 1)
        self._update()

    @_register_key(curses.KEY_PPAGE)
    def scroll_page_up(self):
        self._reports[self._report_index].scroll(self._text_height(), pages=-1)
        self._update()

    @_register_key(curses.KEY_NPAGE)
    def scroll_page_down(self):
        self._reports[self._report_index].scroll(self._text_height(), pages=1)
        self._update()

    @_register_key(curses.KEY_HOME)
    def scroll_begin(self):
        self._reports[self._report_index].scroll(self._text_height(), documents=-1)
        self._update()

    @_register_key(curses.KEY_END)
    def scroll_end(self):
        self._reports[self._report_index].scroll(self._text_height(), documents=1)
        self._update()

    @_register_key("r", "R", curses.KEY_RESIZE)
    def resize(self):
        self._update()

    @_register_key(curses.KEY_LEFT)
    def previous_document(self):
        self._screen.clear()
        if len(self._reports) > 1:
            self._report_index = max(self._report_index - 1, 1)
        self._update()

    @_register_key(curses.KEY_RIGHT)
    def next_document(self):
        self._screen.clear()
        self._report_index = min(self._report_index + 1, len(self._reports) - 1)
        self._update()

    def _thread_body(self):
        """UI thread."""
        self._stop = False
        curses.wrapper(self._main_loop)

    def _main_loop(self, screen):
        """Main loop managing the interaction with the user."""
        with self._mutex:
            self._screen = screen
            curses.use_default_colors()
            curses.init_pair(self.COLOR_OK, curses.COLOR_GREEN, -1)
            curses.init_pair(self.COLOR_WARN, curses.COLOR_YELLOW, -1)
            curses.init_pair(self.COLOR_ERR, curses.COLOR_RED, -1)
            self._footer = curses.newwin(self.FOOTER_H, self._text_width(), self._text_height(), 0)
            self._footer.bkgd(" ", curses.A_REVERSE)
            self._reports.append(Report("PVCHECK", self.MAX_W))
            self._reports[-1].add_line("Waiting for test results...")
            self._report_index = 0
            self._update()
        while not self._stop:
            ch = screen.getch()
            with self._mutex:
                _CALLBACKS.get(ch, lambda self: None)(self)
                with open("post.p", "a") as f:
                    print("!!!!!!!!!!!!!!!!!!!", ch, chr(ch), file=f, flush=True)

    def _text_height(self):
        """Number of text lines displayed."""
        return self._screen.getmaxyx()[0] - self.FOOTER_H

    def _text_width(self):
        """Width of the displayed text."""
        return min(self._screen.getmaxyx()[1], self.MAX_W)

    def _add_footer(self, line, align, text, *extra):
        k = self._text_width() - 1 - len(text)
        pos = max(0, (0 if align == "left" else (k if align == "right" else k //2 )))
        self._footer.addnstr(line, pos, text, self._text_width() - 1 - pos, *extra)

    def _add_short_report(self):
        texts = [
            "%3d " % self._ok_count, _(" passes, "),
            "%3d " % self._warn_count, _(" warnings, "),
            "%3d " % self._err_count, _(" errors")
        ]
        styles = [
            [curses.color_pair(self.COLOR_OK)], [],
            [curses.color_pair(self.COLOR_WARN)], [],
            [curses.color_pair(self.COLOR_ERR)], []
        ]
        n = self._text_width() - 1
        pos = 0
        for t, s in zip (texts, styles):
            self._footer.addnstr(1, pos, t, n, *s)
            pos += len(t)
            n -= len(t)

    def _update(self):
        """Redraw everything."""
        if self._screen is None:
            return
        self._screen.refresh()
        height = self._text_height()
        width = self._text_width()
        doc = self._reports[self._report_index]
        self._footer.mvwin(height, 0)
        doc.refresh(self._text_height(), self._text_width())
        self._footer.clear()
        self._add_footer(0, "center", _("[Press 'h' for help]"), curses.A_DIM)
        text = _("Test case %d of %d (%s) ") % (self._report_index, len(self._reports) - 1, doc.title)
        self._add_footer(0, "left", text)
        text = _("Lines {%d}-{%d}/{%d}") % (doc.top(), doc.bottom(height), doc.length())
        self._add_footer(0, "right", text)
        self._add_short_report()
        if self._running:
            tot = self._err_count + self._warn_count + self._ok_count
            text = "TEST RUNNING..." + "|/-\\"[tot % 4]
        else:
            text = "TEST COMPLETED"
        self._add_footer(1, "right", text, curses.A_BOLD)
        self._footer.refresh()

    def begin_session(self):
        self._err_count = self._warn_count = self._ok_count = 0
        self._running = True
        # Start the UI thread
        self._thread = threading.Thread(target=self._thread_body)
        self._thread.start()

    def end_session(self):
        # Wait the termination of the UI thread
        with self._mutex:
            self._running = False
            self._update()
        self._thread.join()

    @_synchronized
    def begin_test(self, description, cmdline_args, input, tempfile):
        description = description or ""
        self._reports.append(Report(description, self.MAX_W))
        if self._report_index == 0:
            self._report_index = 1
        self._update()

    @_synchronized
    def end_test(self):
        pass

    @_synchronized
    def execution_result(self, cmdline_args, execution_result):
        info = {
            'progname': cmdline_args[0],
            'status': execution_result.status
        }
        message = self._RESULT_TABLE[execution_result.result]
        if execution_result.result != executor.ER_OK:
            self._err_count += 1
            message = message.format(**info)
            for line in message.splitlines():
                self._reports[-1].add_line(line, curses.color_pair(self.COLOR_ERR))
        self._update()

    @_synchronized
    def comparison_result(self, expected, got, diffs, matches):
        add = self._reports[-1].add_line
        all_ok = (max(diffs, default=0) <= 0)
        if all_ok:
            self._ok_count += 1
        else:
            self._err_count += 1
        color = curses.color_pair(self.COLOR_OK if all_ok else self.COLOR_ERR)
        add("[%s]" % expected.tag, color | curses.A_BOLD)

        err_diff = _("%s\t\t(expected '%s')")
        err_un = _("%s\t\t(this line was not expected)")
        err_mis = _("\t\t(missing line '%s')")

        for (i, d) in enumerate(diffs):
            if d <= 0:
                # Correct
                add(got.content[i], curses.color_pair(self.COLOR_OK))
            elif matches[i] is None:
                # Extra line
                add(err_un % got.content[i], curses.color_pair(self.COLOR_ERR))
            elif i >= len(got.content):
                # Missing line
                add(err_mis % matches[i], curses.color_pair(self.COLOR_ERR))
            else:
                # Mismatching lines
                add(err_diff % (got.content[i], matches[i]), curses.color_pair(self.COLOR_ERR))
        add("")
        self._update()

    @_synchronized
    def missing_section(self, expected):
        self._warn_count += 1
        message = _("\t\t(section [%s] is missing)") % expected.tag
        self._reports[-1].add_line(message, curses.color_pair(self.COLOR_WARN))
        self._reports[-1].add_line("")
        self._update()

class Report:
    """Test result displayed on the screen."""

    def __init__(self, title, max_width):
        """Create a new report with the given title.

        Lines are truncated to max_width.
        """
        self.title = title
        self._length = 0
        self._max_width = max_width
        self._pad = curses.newpad(1, max_width)
        self._position = 0

    def add_line(self, line, *extra):
        """Add a line at the bottom of the document."""
        self._pad.resize(self._length + 1, self._max_width)
        self._pad.addnstr(self._length, 0, line, self._max_width, *extra)
        self._length += 1

    def scroll(self, page_height, lines=0, pages=0, documents=0):
        """Scroll up or down."""
        amount = lines + page_height * pages + documents * self._length
        self._position = max(0, min(self._position + amount, self._length - page_height))

    def refresh(self, page_height, page_width):
        """Redraw the document on the screen."""
        self._pad.refresh(self._position, 0, 0, 0, page_height - 1, page_width - 1)

    def top(self):
        """Index of the first line on the screen (starting from 1)."""
        return self._position + 1

    def bottom(self, page_height):
        """Index of the last line on the screen."""
        return min(self._position + page_height, self._length)

    def length(self):
        """Number of lines in the document."""
        return self._length
<|endoftext|>


<commit_before># Tai Sakuma <tai.sakuma@cern.ch>
import itertools

##__________________________________________________________________||
class BackrefMultipleArrayReader(object):
    def __init__(self, arrays, idxs_conf, backref_idxs):
        self.backref_idxs = backref_idxs
        self._use_backref = any([e is not None for e in backref_idxs])
        self._zipped = zip(range(len(arrays)), arrays, idxs_conf, backref_idxs)

    def read(self):
        return self._read_zipped(self._zipped, self.backref_idxs)

    def _read_zipped(self, zipped, backref_idxs):

        varis = self._unzip_and_read_event_attributes(zipped)

        if not self._use_backref:
            return self._fast_path_without_backref(varis)

        # e.g.,
        # keys = [
        #     [1001],
        #     [15.0, 12.0, None, 10.0],
        #     [None, 1.2, 2.2, 0.5],
        #     [20.0, 11.0, 13.0],
        #     [2.5, None, 1.0],
        #     [0.1, 0.6, None, 0.3]
        # ]
        # vals = [
        #     [21.0, 13.0, 15.0, 11.0],
        #     [22.0, 15.0, 16.0]
        # ]

        # e.g.,
        # backref_idxs = [None, None, 1, None, 3, 1, 1, 3]

        uniq_idxs, ref_key_idxs = self._build_uniq_ref_idxs(varis, backref_idxs)
        # e.g.,
        # uniq_idxs = [
        #     [0],
        #     [0, 1, 2, 3],
        #     [0, 1, 2]
        # ]
        #
        # ref_key_idxs = [
        #     [0],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        #     [0, 1, 2],
        #     [0, 1, 2],
        #     [0, 1, 2, 3],
        # ]
        #
        # ref_val_idxs = [
        #     [0, 1, 2, 3],
        #     [0, 1, 2]]
        # ]
        #
        # these 5 lists are the same object:
        #   binIdxsList_uniq[1]
        #   ref_key_idxs[1]
        #   ref_key_idxs[2]
        #   ref_key_idxs[5]
        #   ref_val_idxs[0]
        #
        # so are these 3 lists:
        #   binIdxsList_uniq[2]
        #   ref_key_idxs[3]
        #   ref_key_idxs[4]
        #   ref_val_idxs[1]

        # e.g.,
        # uniq_idxs = [
        #     [0],
        #     [1, 3],
        #     [0, 2]
        # ]
        #
        # ref_key_idxs = [
        #     [0],
        #     [1, 3],
        #     [1, 3],
        #     [0, 2],
        #     [0, 2],
        # ]
        #
        # ref_val_idxs = [
        #     [1, 3],
        #     [0, 2]
        # ]

        self._expand_idxs_with_all_combinations(uniq_idxs)
        # e.g.,
        # uniq_idxs = [
        #     [0, 0, 0, 0],
        #     [1, 1, 3, 3],
        #     [0, 2, 0, 2]
        # ]
        #
        # ref_key_idxs = [
        #     [0, 0, 0, 0],
        #     [1, 1, 3, 3],
        #     [1, 1, 3, 3],
        #     [0, 2, 0, 2],
        #     [0, 2, 0, 2],
        #     [1, 1, 3, 3],
        # ]
        #
        # ref_val_idxs = [
        #     [1, 1, 3, 3],
        #     [0, 2, 0, 2]
        # ]

        varis = self._build_ret(varis, ref_key_idxs)
        return varis

    def _unzip_and_read_event_attributes(self, zipped):
        backref_map = { }
        varis = [ ]
        for var_idx, attr, conf_attr_idx, backref_idx in zipped:
            attr_idxs = self._determine_attr_indices_to_read(attr, conf_attr_idx, var_idx, backref_idx, backref_map)
            attr_vals = [(attr[i] if i < len(attr) else None) for i in attr_idxs]
            varis.append(attr_vals)
        return varis

    def _determine_attr_indices_to_read(self, attr, conf_attr_idx, var_idx, backref_idx, backref_map):
        if backref_idx is None:
            if conf_attr_idx == '*': ret = range(len(attr))
            elif conf_attr_idx < len(attr): ret = [conf_attr_idx]
            else: ret = [ ] # conf_attr_idx is out of the range
        else:
            ret = backref_map[backref_idx]
        backref_map[var_idx] = ret
        return ret

    def _fast_path_without_backref(self, varis):
        varis = tuple(itertools.product(*varis))
        return varis

    def _build_uniq_ref_idxs(self, keys, backref_idxs):
        uniq_idxs = [ ]
        ref_idxs = [ ]
        for keys, backrefIdx in zip(keys, backref_idxs):
            if backrefIdx is None:
                idxs = range(len(keys))
                uniq_idxs.append(idxs)
                ref_idxs.append(idxs)
            else:
                ref_idxs.append(ref_idxs[backrefIdx])
        return uniq_idxs, ref_idxs

    def _expand_idxs_with_all_combinations(self, idxs):
        prod = tuple(itertools.product(*idxs))
        for i in range(len(idxs)):
            idxs[i][:] = [p[i] for p in prod]

    def _build_ret(self, varis, idxs):
        if not idxs: return tuple()
        ret = [ ]
        for i in range(len(idxs[0])):
            ret.append(tuple([b[subidxs[i]] for b, subidxs in zip(varis, idxs)]))
        val = [ ]
        return tuple(ret)
##__________________________________________________________________||
<commit_msg>clean up the code in BackrefMultipleArrayReader<commit_after># Tai Sakuma <tai.sakuma@cern.ch>
import itertools

##__________________________________________________________________||
class BackrefMultipleArrayReader(object):
    def __init__(self, arrays, idxs_conf, backref_idxs):
        self.backref_idxs = backref_idxs
        self._use_backref = any([e is not None for e in backref_idxs])
        self._zipped = zip(range(len(arrays)), arrays, idxs_conf, backref_idxs)

        self.arrays = arrays

    def read(self):
        return self._read_zipped(self._zipped, self.backref_idxs)

    def _read_zipped(self, zipped, backref_idxs):

        varis = self._unzip_and_read_arrays(zipped)
        # e.g.,
        # varis = [
        #     [1001],
        #     [15.3, 12.9, 9.2, 10.5],
        #     [-1.2, 5.2, 2.2, 0.5],
        #     [20.2, 11.9, 13.3, 5.2],
        #     [2.2, 1.2, -1.5, -0.5],
        #     [0.1, 0.6, 1.2, None],
        #     [16.2, 13.1, 10.1, 11.8],
        #     [22.1, 15.2, 16.3, None]
        # ]

        if not self._use_backref:
            # fast path without backref
            return tuple(itertools.product(*varis))

        # e.g.,
        # backref_idxs = [None, None, 1, None, 3, 1, 1, 3]

        uniq_idxs, ref_key_idxs = self._build_uniq_ref_idxs(varis, backref_idxs)
        # e.g.,
        # uniq_idxs = [
        #     [0],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3]
        # ]
        #
        # ref_key_idxs = [
        #     [0],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        #     [0, 1, 2, 3],
        # ]
        #
        # these 5 lists are the same object:
        #   uniq_idxs[1]
        #   ref_key_idxs[1]
        #   ref_key_idxs[2]
        #   ref_key_idxs[5]
        #   ref_key_idxs[]
        #
        # so are these 3 lists:
        #   uniq_idxs[2]
        #   ref_key_idxs[3]
        #   ref_key_idxs[4]
        #   ref_key_idxs[7]

        self._expand_idxs_with_all_combinations(uniq_idxs)
        # e.g.,
        # uniq_idxs = [
        #     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        #     [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3],
        #     [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]
        # ]
        #
        # e.g.,
        # ref_key_idxs = [
        #     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        #     [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3],
        #     [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3],
        #     [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3],
        #     [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3],
        #     [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3],
        #     [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3],
        #     [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]
        # ]

        varis = self._build_ret(varis, ref_key_idxs)
        # e.g.,
        # varis = (
        #     (1001, 15.3, -1.2, 20.2, 2.2, 0.1, 16.2, 22.1),
        #     (1001, 15.3, -1.2, 11.9, 1.2, 0.1, 16.2, 15.2),
        #     (1001, 15.3, -1.2, 13.3, -1.5, 0.1, 16.2, 16.3),
        #     (1001, 15.3, -1.2, 5.2, -0.5, 0.1, 16.2, None),
        #     (1001, 12.9, 5.2, 20.2, 2.2, 0.6, 13.1, 22.1),
        #     (1001, 12.9, 5.2, 11.9, 1.2, 0.6, 13.1, 15.2),
        #     (1001, 12.9, 5.2, 13.3, -1.5, 0.6, 13.1, 16.3),
        #     (1001, 12.9, 5.2, 5.2, -0.5, 0.6, 13.1, None),
        #     (1001, 9.2, 2.2, 20.2, 2.2, 1.2, 10.1, 22.1),
        #     (1001, 9.2, 2.2, 11.9, 1.2, 1.2, 10.1, 15.2),
        #     (1001, 9.2, 2.2, 13.3, -1.5, 1.2, 10.1, 16.3),
        #     (1001, 9.2, 2.2, 5.2, -0.5, 1.2, 10.1, None),
        #     (1001, 10.5, 0.5, 20.2, 2.2, None, 11.8, 22.1),
        #     (1001, 10.5, 0.5, 11.9, 1.2, None, 11.8, 15.2),
        #     (1001, 10.5, 0.5, 13.3, -1.5, None, 11.8, 16.3),
        #     (1001, 10.5, 0.5, 5.2, -0.5, None, 11.8, None)
        # )

        return varis

    def _unzip_and_read_arrays(self, zipped):
        backref_map = { }
        varis = [ ]
        for var_idx, attr, conf_attr_idx, backref_idx in zipped:
            attr_idxs = self._determine_array_indices_to_read(attr, conf_attr_idx, var_idx, backref_idx, backref_map)
            attr_vals = [(attr[i] if i < len(attr) else None) for i in attr_idxs]
            varis.append(attr_vals)
        return varis

    def _determine_array_indices_to_read(self, attr, conf_attr_idx, var_idx, backref_idx, backref_map):
        if backref_idx is None:
            if conf_attr_idx == '*': ret = range(len(attr))
            elif conf_attr_idx < len(attr): ret = [conf_attr_idx]
            else: ret = [ ] # conf_attr_idx is out of the range
        else:
            ret = backref_map[backref_idx]
        backref_map[var_idx] = ret
        return ret

    def _build_uniq_ref_idxs(self, keys, backref_idxs):
        uniq_idxs = [ ]
        ref_idxs = [ ]
        for keys, backrefIdx in zip(keys, backref_idxs):
            if backrefIdx is None:
                idxs = range(len(keys))
                uniq_idxs.append(idxs)
                ref_idxs.append(idxs)
            else:
                ref_idxs.append(ref_idxs[backrefIdx])
        return uniq_idxs, ref_idxs

    def _expand_idxs_with_all_combinations(self, idxs):
        prod = tuple(itertools.product(*idxs))
        for i in range(len(idxs)):
            idxs[i][:] = [p[i] for p in prod]

    def _build_ret(self, varis, idxs):
        if not idxs: return tuple()
        ret = [ ]
        for i in range(len(idxs[0])):
            ret.append(tuple([b[subidxs[i]] for b, subidxs in zip(varis, idxs)]))
        val = [ ]
        return tuple(ret)
##__________________________________________________________________||
<|endoftext|>


<commit_before>from i3pystatus import IntervalModule
# requires python-novaclient
from novaclient.v2 import client
import webbrowser

class Openstack_vms(IntervalModule):
    """
    Displays the number of VMs in an openstack cluster in ACTIVE and
    non-ACTIVE states.
    Requires: python-novaclient
    """

    settings = (
        ("auth_url", "OpenStack cluster authentication URL (OS_AUTH_URL)"),
        ("username", "Username for OpenStack authentication (OS_USERNAME)"),
        ("password", "Password for Openstack authentication (OS_PASSWORD)"),
        ("tenant_name", "Tenant/Project name to view (OS_TENANT_NAME)"),
        ("color", "Display color when non-active VMs are =< `threshold`"),
        ("crit_color", "Display color when non-active VMs are => `threshold`"),
        ("threshold", "Set critical indicators when non-active VM pass this "
            "number"),
        ("horizon_url", "When clicked, open this URL in a browser")
    )
    required = ("auth_url", "password", "tenant_name", "username")
    color = "#00FF00"
    crit_color = "#FF0000"
    threshold = 0
    horizon_url = None
    format = "{tenant_name}: {active_servers} up, "\
        "{nonactive_servers} down"

    on_leftclick = "openurl"

    def run(self):
        nclient = client.Client(
            self.username,
            self.password,
            self.tenant_name,
            self.auth_url
        )

        active_servers = 0
        nonactive_servers = 0
        server_list = nclient.servers.list()
        for server in server_list:
            if server.status == 'ACTIVE':
                active_servers = active_servers + 1
            else:
                nonactive_servers = nonactive_servers + 1

        if nonactive_servers > self.threshold:
            display_color = self.crit_color
        else:
            display_color = self.color
        cdict = {
            "tenant_name": self.tenant_name,
            "active_servers": active_servers,
            "nonactive_servers": nonactive_servers,
        }

        self.output = {
            "full_text": self.format.format(**cdict),
            "color": display_color
        }

    def openurl(self):
        webbrowser.open_new_tab(self.horizon_url)
<commit_msg>openstack_vms: add format option<commit_after>from i3pystatus import IntervalModule
# requires python-novaclient
from novaclient.v2 import client
import webbrowser

class Openstack_vms(IntervalModule):
    """
    Displays the number of VMs in an openstack cluster in ACTIVE and
    non-ACTIVE states.
    Requires: python-novaclient
    """

    settings = (
        ("auth_url", "OpenStack cluster authentication URL (OS_AUTH_URL)"),
        ("username", "Username for OpenStack authentication (OS_USERNAME)"),
        ("password", "Password for Openstack authentication (OS_PASSWORD)"),
        ("tenant_name", "Tenant/Project name to view (OS_TENANT_NAME)"),
        ("color", "Display color when non-active VMs are =< `threshold`"),
        ("crit_color", "Display color when non-active VMs are => `threshold`"),
        ("threshold", "Set critical indicators when non-active VM pass this "
            "number"),
        ("horizon_url", "When clicked, open this URL in a browser"),
        "format"
    )
    required = ("auth_url", "password", "tenant_name", "username")
    color = "#00FF00"
    crit_color = "#FF0000"
    threshold = 0
    horizon_url = None
    format = "{tenant_name}: {active_servers} up, "\
        "{nonactive_servers} down"

    on_leftclick = "openurl"

    def run(self):
        nclient = client.Client(
            self.username,
            self.password,
            self.tenant_name,
            self.auth_url
        )

        active_servers = 0
        nonactive_servers = 0
        server_list = nclient.servers.list()
        for server in server_list:
            if server.status == 'ACTIVE':
                active_servers = active_servers + 1
            else:
                nonactive_servers = nonactive_servers + 1

        if nonactive_servers > self.threshold:
            display_color = self.crit_color
        else:
            display_color = self.color
        cdict = {
            "tenant_name": self.tenant_name,
            "active_servers": active_servers,
            "nonactive_servers": nonactive_servers,
        }

        self.output = {
            "full_text": self.format.format(**cdict),
            "color": display_color
        }

    def openurl(self):
        webbrowser.open_new_tab(self.horizon_url)
<|endoftext|>


<commit_before># Copyright 2005 Divmod, Inc.  See LICENSE file for details

import itertools

from OpenSSL import SSL
from OpenSSL.crypto import PKey, X509, X509Req
from OpenSSL.crypto import TYPE_RSA

from twisted.trial import unittest, util
from twisted.internet import protocol, defer, reactor
from twisted.python import log

from vertex import sslverify

counter = itertools.count().next
def makeCertificate(**kw):
    keypair = PKey()
    keypair.generate_key(TYPE_RSA, 1024)

    certificate = X509()
    certificate.gmtime_adj_notBefore(0)
    certificate.gmtime_adj_notAfter(60 * 60 * 24 * 365) # One year
    for xname in certificate.get_issuer(), certificate.get_subject():
        for (k, v) in kw.items():
            setattr(xname, k, v)

    certificate.set_serial_number(counter())
    certificate.set_pubkey(keypair)
    certificate.sign(keypair, "md5")

    return keypair, certificate

def otherMakeCertificate(**kw):
    keypair = PKey()
    keypair.generate_key(TYPE_RSA, 1024)

    req = X509Req()
    subj = req.get_subject()
    for (k, v) in kw.items():
        setattr(subj, k, v)

    req.set_pubkey(keypair)
    req.sign(keypair, "md5")

    cert = X509()
    cert.set_serial_number(counter())
    cert.gmtime_adj_notBefore(0)
    cert.gmtime_adj_notAfter(60 * 60 * 24 * 365) # One year

    cert.set_issuer(req.get_subject())
    cert.set_subject(req.get_subject())
    cert.set_pubkey(req.get_pubkey())
    cert.sign(keypair, "md5")

    return keypair, cert

class DataCallbackProtocol(protocol.Protocol):
    def dataReceived(self, data):
        d, self.factory.onData = self.factory.onData, None
        if d is not None:
            d.callback(data)

    def connectionLost(self, reason):
        d, self.factory.onLost = self.factory.onLost, None
        if d is not None:
            d.errback(reason)

class WritingProtocol(protocol.Protocol):
    byte = 'x'
    def connectionMade(self):
        self.transport.write(self.byte)

    def connectionLost(self, reason):
        self.factory.onLost.errback(reason)

class OpenSSLOptions(unittest.TestCase):
    serverPort = clientConn = None
    onServerLost = onClientLost = None

    def setUpClass(self):
        self.sKey, self.sCert = makeCertificate(
            O="Server Test Certificate",
            CN="server")
        self.cKey, self.cCert = makeCertificate(
            O="Client Test Certificate",
            CN="client")

    def tearDown(self):
        if self.serverPort is not None:
            self.serverPort.stopListening()
        if self.clientConn is not None:
            self.clientConn.disconnect()

        L = []
        if self.onServerLost is not None:
            L.append(self.onServerLost)
        if self.onClientLost is not None:
            L.append(self.onClientLost)

        return defer.DeferredList(L, consumeErrors=True)

    def loopback(self, serverCertOpts, clientCertOpts,
                 onServerLost=None, onClientLost=None, onData=None):
        if onServerLost is None:
            self.onServerLost = onServerLost = defer.Deferred()
        if onClientLost is None:
            self.onClientLost = onClientLost = defer.Deferred()
        if onData is None:
            onData = defer.Deferred()

        serverFactory = protocol.ServerFactory()
        serverFactory.protocol = DataCallbackProtocol
        serverFactory.onLost = onServerLost
        serverFactory.onData = onData

        clientFactory = protocol.ClientFactory()
        clientFactory.protocol = WritingProtocol
        clientFactory.onLost = onClientLost

        self.serverPort = reactor.listenSSL(0, serverFactory, serverCertOpts)
        self.clientConn = reactor.connectSSL('127.0.0.1', self.serverPort.getHost().port,
                                             clientFactory, clientCertOpts)

    def testAbbreviatingDistinguishedNames(self):
        self.assertEquals(sslverify.DN(CN='a', OU='hello'),
                          sslverify.DistinguishedName(commonName='a', organizationalUnitName='hello'))
        self.assertNotEquals(sslverify.DN(CN='a', OU='hello'),
                             sslverify.DN(CN='a', OU='hello', emailAddress='xxx'))
        dn = sslverify.DN(CN='abcdefg')
        self.assertRaises(AttributeError, setattr, dn, 'Cn', 'x')
        self.assertEquals(dn.CN, dn.commonName)
        dn.CN = 'bcdefga'
        self.assertEquals(dn.CN, dn.commonName)

    def testAllowedAnonymousClientConnection(self):
        onData = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(requireCertificate=False),
                      onData=onData)

        return onData.addCallback(
            lambda result: self.assertEquals(result, WritingProtocol.byte))

    def testRefusedAnonymousClientConnection(self):
        onServerLost = defer.Deferred()
        onClientLost = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=True, caCerts=[self.sCert], requireCertificate=True),
                      sslverify.OpenSSLCertificateOptions(requireCertificate=False),
                      onServerLost=onServerLost,
                      onClientLost=onClientLost)

        d = defer.DeferredList([onClientLost, onServerLost], consumeErrors=True)

        def afterLost(((cSuccess, cResult), (sSuccess, sResult))):

            self.failIf(cSuccess)
            self.failIf(sSuccess)

            # XXX Twisted doesn't report SSL errors as SSL errors, but in the
            # future it will.

            # cResult.trap(SSL.Error)
            # sResult.trap(SSL.Error)

        return d.addCallback(afterLost)

    def testFailedCertificateVerification(self):
        onServerLost = defer.Deferred()
        onClientLost = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=False, requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(verify=True, requireCertificate=False, caCerts=[self.cCert]),
                      onServerLost=onServerLost,
                      onClientLost=onClientLost)

        d = defer.DeferredList([onClientLost, onServerLost], consumeErrors=True)
        def afterLost(((cSuccess, cResult), (sSuccess, sResult))):

            self.failIf(cSuccess)
            self.failIf(sSuccess)

        return d.addCallback(afterLost)

    def testSuccessfulCertificateVerification(self):
        onData = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=False, requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(verify=True, requireCertificate=True, caCerts=[self.sCert]),
                      onData=onData)

        return onData.addCallback(lambda result: self.assertEquals(result, WritingProtocol.byte))

    def testSuccessfulSymmetricSelfSignedCertificateVerification(self):
        onData = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=True, requireCertificate=True, caCerts=[self.cCert]),
                      sslverify.OpenSSLCertificateOptions(privateKey=self.cKey, certificate=self.cCert, verify=True, requireCertificate=True, caCerts=[self.sCert]),
                      onData=onData)

        return onData.addCallback(lambda result: self.assertEquals(result, WritingProtocol.byte))

    def testVerification(self):
        clientDN = sslverify.DistinguishedName(commonName='client')
        clientKey = sslverify.KeyPair.generate()
        clientCertReq = clientKey.certificateRequest(clientDN)

        serverDN = sslverify.DistinguishedName(commonName='server')
        serverKey = sslverify.KeyPair.generate()
        serverCertReq = serverKey.certificateRequest(serverDN)

        ##
        clientSelfCertReq = clientKey.certificateRequest(clientDN)
        clientSelfCertData = clientKey.signCertificateRequest(clientDN, clientSelfCertReq,
                                                              lambda dn: True,
                                                              132)
        clientSelfCert = clientKey.newCertificate(clientSelfCertData)
        ##

        ##
        serverSelfCertReq = serverKey.certificateRequest(serverDN)
        serverSelfCertData = serverKey.signCertificateRequest(serverDN, serverSelfCertReq,
                                                              lambda dn: True,
                                                              516)
        serverSelfCert = serverKey.newCertificate(serverSelfCertData)
        ##

        ##
        clientCertData = serverKey.signCertificateRequest(serverDN, clientCertReq,
                                                          lambda dn: True,
                                                          7)
        clientCert = clientKey.newCertificate(clientCertData)
        ##

        ##
        serverCertData = clientKey.signCertificateRequest(clientDN, serverCertReq,
                                                          lambda dn: True,
                                                          42)
        serverCert = serverKey.newCertificate(serverCertData)
        ##

        onData = defer.Deferred()

        serverOpts = serverCert.options(serverSelfCert)
        clientOpts = clientCert.options(clientSelfCert)

        self.loopback(serverOpts,
                      clientOpts,
                      onData=onData)

        return onData.addCallback(lambda result: self.assertEquals(result, WritingProtocol.byte))
<commit_msg>Put flushErrors calls back in until buildbot is fixed.<commit_after># Copyright 2005 Divmod, Inc.  See LICENSE file for details

import itertools

from OpenSSL import SSL
from OpenSSL.crypto import PKey, X509, X509Req
from OpenSSL.crypto import TYPE_RSA

from twisted.trial import unittest, util
from twisted.internet import protocol, defer, reactor
from twisted.python import log

from vertex import sslverify

counter = itertools.count().next
def makeCertificate(**kw):
    keypair = PKey()
    keypair.generate_key(TYPE_RSA, 1024)

    certificate = X509()
    certificate.gmtime_adj_notBefore(0)
    certificate.gmtime_adj_notAfter(60 * 60 * 24 * 365) # One year
    for xname in certificate.get_issuer(), certificate.get_subject():
        for (k, v) in kw.items():
            setattr(xname, k, v)

    certificate.set_serial_number(counter())
    certificate.set_pubkey(keypair)
    certificate.sign(keypair, "md5")

    return keypair, certificate

def otherMakeCertificate(**kw):
    keypair = PKey()
    keypair.generate_key(TYPE_RSA, 1024)

    req = X509Req()
    subj = req.get_subject()
    for (k, v) in kw.items():
        setattr(subj, k, v)

    req.set_pubkey(keypair)
    req.sign(keypair, "md5")

    cert = X509()
    cert.set_serial_number(counter())
    cert.gmtime_adj_notBefore(0)
    cert.gmtime_adj_notAfter(60 * 60 * 24 * 365) # One year

    cert.set_issuer(req.get_subject())
    cert.set_subject(req.get_subject())
    cert.set_pubkey(req.get_pubkey())
    cert.sign(keypair, "md5")

    return keypair, cert

class DataCallbackProtocol(protocol.Protocol):
    def dataReceived(self, data):
        d, self.factory.onData = self.factory.onData, None
        if d is not None:
            d.callback(data)

    def connectionLost(self, reason):
        d, self.factory.onLost = self.factory.onLost, None
        if d is not None:
            d.errback(reason)

class WritingProtocol(protocol.Protocol):
    byte = 'x'
    def connectionMade(self):
        self.transport.write(self.byte)

    def connectionLost(self, reason):
        self.factory.onLost.errback(reason)

class OpenSSLOptions(unittest.TestCase):
    serverPort = clientConn = None
    onServerLost = onClientLost = None

    def setUpClass(self):
        self.sKey, self.sCert = makeCertificate(
            O="Server Test Certificate",
            CN="server")
        self.cKey, self.cCert = makeCertificate(
            O="Client Test Certificate",
            CN="client")

    def tearDown(self):
        if self.serverPort is not None:
            self.serverPort.stopListening()
        if self.clientConn is not None:
            self.clientConn.disconnect()

        L = []
        if self.onServerLost is not None:
            L.append(self.onServerLost)
        if self.onClientLost is not None:
            L.append(self.onClientLost)

        return defer.DeferredList(L, consumeErrors=True)

    def loopback(self, serverCertOpts, clientCertOpts,
                 onServerLost=None, onClientLost=None, onData=None):
        if onServerLost is None:
            self.onServerLost = onServerLost = defer.Deferred()
        if onClientLost is None:
            self.onClientLost = onClientLost = defer.Deferred()
        if onData is None:
            onData = defer.Deferred()

        serverFactory = protocol.ServerFactory()
        serverFactory.protocol = DataCallbackProtocol
        serverFactory.onLost = onServerLost
        serverFactory.onData = onData

        clientFactory = protocol.ClientFactory()
        clientFactory.protocol = WritingProtocol
        clientFactory.onLost = onClientLost

        self.serverPort = reactor.listenSSL(0, serverFactory, serverCertOpts)
        self.clientConn = reactor.connectSSL('127.0.0.1', self.serverPort.getHost().port,
                                             clientFactory, clientCertOpts)

    def testAbbreviatingDistinguishedNames(self):
        self.assertEquals(sslverify.DN(CN='a', OU='hello'),
                          sslverify.DistinguishedName(commonName='a', organizationalUnitName='hello'))
        self.assertNotEquals(sslverify.DN(CN='a', OU='hello'),
                             sslverify.DN(CN='a', OU='hello', emailAddress='xxx'))
        dn = sslverify.DN(CN='abcdefg')
        self.assertRaises(AttributeError, setattr, dn, 'Cn', 'x')
        self.assertEquals(dn.CN, dn.commonName)
        dn.CN = 'bcdefga'
        self.assertEquals(dn.CN, dn.commonName)

    def testAllowedAnonymousClientConnection(self):
        onData = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(requireCertificate=False),
                      onData=onData)

        return onData.addCallback(
            lambda result: self.assertEquals(result, WritingProtocol.byte))

    def testRefusedAnonymousClientConnection(self):
        onServerLost = defer.Deferred()
        onClientLost = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=True, caCerts=[self.sCert], requireCertificate=True),
                      sslverify.OpenSSLCertificateOptions(requireCertificate=False),
                      onServerLost=onServerLost,
                      onClientLost=onClientLost)

        d = defer.DeferredList([onClientLost, onServerLost], consumeErrors=True)

        def afterLost(((cSuccess, cResult), (sSuccess, sResult))):

            self.failIf(cSuccess)
            self.failIf(sSuccess)

            # XXX Twisted doesn't report SSL errors as SSL errors, but in the
            # future it will.

            # cResult.trap(SSL.Error)
            # sResult.trap(SSL.Error)

            # Twisted trunk will do the correct thing here, and not log any
            # errors.  Twisted 2.1 will do the wrong thing.  We're flushing
            # errors until the buildbot is updated to a reasonable facsimilie
            # of 2.2.
            log.flushErrors(SSL.Error)

        return d.addCallback(afterLost)

    def testFailedCertificateVerification(self):
        onServerLost = defer.Deferred()
        onClientLost = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=False, requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(verify=True, requireCertificate=False, caCerts=[self.cCert]),
                      onServerLost=onServerLost,
                      onClientLost=onClientLost)

        d = defer.DeferredList([onClientLost, onServerLost], consumeErrors=True)
        def afterLost(((cSuccess, cResult), (sSuccess, sResult))):

            self.failIf(cSuccess)
            self.failIf(sSuccess)

            # Twisted trunk will do the correct thing here, and not log any
            # errors.  Twisted 2.1 will do the wrong thing.  We're flushing
            # errors until the buildbot is updated to a reasonable facsimilie
            # of 2.2.
            log.flushErrors(SSL.Error)

        return d.addCallback(afterLost)

    def testSuccessfulCertificateVerification(self):
        onData = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=False, requireCertificate=False),
                      sslverify.OpenSSLCertificateOptions(verify=True, requireCertificate=True, caCerts=[self.sCert]),
                      onData=onData)

        return onData.addCallback(lambda result: self.assertEquals(result, WritingProtocol.byte))

    def testSuccessfulSymmetricSelfSignedCertificateVerification(self):
        onData = defer.Deferred()
        self.loopback(sslverify.OpenSSLCertificateOptions(privateKey=self.sKey, certificate=self.sCert, verify=True, requireCertificate=True, caCerts=[self.cCert]),
                      sslverify.OpenSSLCertificateOptions(privateKey=self.cKey, certificate=self.cCert, verify=True, requireCertificate=True, caCerts=[self.sCert]),
                      onData=onData)

        return onData.addCallback(lambda result: self.assertEquals(result, WritingProtocol.byte))

    def testVerification(self):
        clientDN = sslverify.DistinguishedName(commonName='client')
        clientKey = sslverify.KeyPair.generate()
        clientCertReq = clientKey.certificateRequest(clientDN)

        serverDN = sslverify.DistinguishedName(commonName='server')
        serverKey = sslverify.KeyPair.generate()
        serverCertReq = serverKey.certificateRequest(serverDN)

        ##
        clientSelfCertReq = clientKey.certificateRequest(clientDN)
        clientSelfCertData = clientKey.signCertificateRequest(clientDN, clientSelfCertReq,
                                                              lambda dn: True,
                                                              132)
        clientSelfCert = clientKey.newCertificate(clientSelfCertData)
        ##

        ##
        serverSelfCertReq = serverKey.certificateRequest(serverDN)
        serverSelfCertData = serverKey.signCertificateRequest(serverDN, serverSelfCertReq,
                                                              lambda dn: True,
                                                              516)
        serverSelfCert = serverKey.newCertificate(serverSelfCertData)
        ##

        ##
        clientCertData = serverKey.signCertificateRequest(serverDN, clientCertReq,
                                                          lambda dn: True,
                                                          7)
        clientCert = clientKey.newCertificate(clientCertData)
        ##

        ##
        serverCertData = clientKey.signCertificateRequest(clientDN, serverCertReq,
                                                          lambda dn: True,
                                                          42)
        serverCert = serverKey.newCertificate(serverCertData)
        ##

        onData = defer.Deferred()

        serverOpts = serverCert.options(serverSelfCert)
        clientOpts = clientCert.options(clientSelfCert)

        self.loopback(serverOpts,
                      clientOpts,
                      onData=onData)

        return onData.addCallback(lambda result: self.assertEquals(result, WritingProtocol.byte))
<|endoftext|>


<commit_before>import abc
import json

import requests

import woodpecker.misc.utils as utils

from woodpecker.elements.transactions.simpletransaction import SimpleTransaction

__author__ = 'Stefano'

class HttpTransaction(SimpleTransaction):
    __metaclass__ = abc.ABCMeta

    def __init__(self, **kwargs):
        super(HttpTransaction, self).__init__(**kwargs)

        # Check for existence of '_session' key in thread variables dictionary.
        # If it does not exist a new session is created
        if '_session' not in self.thread_variables:
            self.thread_variables['_session'] = requests.Session()

    def http_request(self, str_request_name, str_url, **kwargs):
        str_method = kwargs.get('method', 'GET')
        obj_data = kwargs.get('data', {})
        obj_headers = kwargs.get('headers', {})
        obj_cookies = kwargs.get('cookies', {})
        bool_redirects = kwargs.get('allow_redirects', True)
        bool_verify_ssl = kwargs.get('verify_ssl', False)

        # Sets the proxy settings
        if 'proxy' in self.settings:
            obj_proxy = self.settings['proxy']
        else:
            obj_proxy = {}

        # If the Verify SSL option is set to false, disables the urllib InsecureRequestWarning message
        if not bool_verify_ssl:
            requests.packages.urllib3.disable_warnings()

        # Build kwargs for request according to method
        dic_request_kwargs = {
            'headers': obj_headers,
            'cookies': obj_cookies,
            'allow_redirects': bool_redirects,
            'proxies': obj_proxy,
            'verify': bool_verify_ssl
        }

        if str_method == 'GET' or str_method == 'DELETE':
            dic_request_kwargs['params'] = obj_data
        elif str_method == 'POST' or str_method == 'PUT' or str_method == 'PATCH':
            if self.is_json(obj_data):
                dic_request_kwargs['json'] = obj_data
            else:
                dic_request_kwargs['data'] = obj_data

        # Send unique request with kwargs defined in dict
        str_timestamp = utils.get_timestamp()
        self.thread_variables['_last_response'] = \
            self.thread_variables['_session'].request(str_method, str_url, **dic_request_kwargs)

        # Send request data using sender object
        self.sender.send('request',
                         {
                             'hostName': utils.get_ip_address(),
                             'spawnID': self.spawn_id,
                             'testName': self.navigation_name,
                             'iteration': self.iteration,
                             'timestamp': str_timestamp,
                             'requestName': str_request_name,
                             'requestType': '_'.join(('HTTP', str_method)),
                             'requestSkeleton': None,
                             'requestSpecs': json.dumps(obj_data),
                             'duration': self.thread_variables['_last_response'].elapsed.total_seconds() * 1000,
                             'status': self.thread_variables['_last_response'].status_code,
                             'responseSize': self.thread_variables['_last_response'].headers.get('content-length',
                                                                                None),
                             'assertionResult': 1
                         })

    @staticmethod
    def is_json(str_json):
        try:
            json.loads(str_json)
        except ValueError:
            return False
        return True
<commit_msg>Fixed missing URL retrieval for requests<commit_after>import abc
import json

import requests

import woodpecker.misc.utils as utils

from woodpecker.elements.transactions.simpletransaction import SimpleTransaction

__author__ = 'Stefano'

class HttpTransaction(SimpleTransaction):
    __metaclass__ = abc.ABCMeta

    def __init__(self, **kwargs):
        super(HttpTransaction, self).__init__(**kwargs)

        # Check for existence of '_session' key in thread variables dictionary.
        # If it does not exist a new session is created
        if '_session' not in self.thread_variables:
            self.thread_variables['_session'] = requests.Session()

    def http_request(self, str_request_name, str_url, **kwargs):
        str_method = kwargs.get('method', 'GET')
        obj_data = kwargs.get('data', {})
        obj_headers = kwargs.get('headers', {})
        obj_cookies = kwargs.get('cookies', {})
        bool_redirects = kwargs.get('allow_redirects', True)
        bool_verify_ssl = kwargs.get('verify_ssl', False)

        # Sets the proxy settings
        if 'proxy' in self.settings:
            obj_proxy = self.settings['proxy']
        else:
            obj_proxy = {}

        # If the Verify SSL option is set to false, disables the urllib InsecureRequestWarning message
        if not bool_verify_ssl:
            requests.packages.urllib3.disable_warnings()

        # Build kwargs for request according to method
        dic_request_kwargs = {
            'headers': obj_headers,
            'cookies': obj_cookies,
            'allow_redirects': bool_redirects,
            'proxies': obj_proxy,
            'verify': bool_verify_ssl
        }

        if str_method == 'GET' or str_method == 'DELETE':
            dic_request_kwargs['params'] = obj_data
        elif str_method == 'POST' or str_method == 'PUT' or str_method == 'PATCH':
            if self.is_json(obj_data):
                dic_request_kwargs['json'] = obj_data
            else:
                dic_request_kwargs['data'] = obj_data

        # Send unique request with kwargs defined in dict
        str_timestamp = utils.get_timestamp()
        self.thread_variables['_last_response'] = \
            self.thread_variables['_session'].request(str_method, str_url, **dic_request_kwargs)

        # Send request data using sender object
        self.sender.send('request',
                         {
                             'hostName': utils.get_ip_address(),
                             'spawnID': self.spawn_id,
                             'testName': self.navigation_name,
                             'iteration': self.iteration,
                             'timestamp': str_timestamp,
                             'requestName': str_request_name,
                             'requestType': '_'.join(('HTTP', str_method)),
                             'requestSkeleton': str_url,
                             'requestSpecs': json.dumps(obj_data),
                             'duration': self.thread_variables['_last_response'].elapsed.total_seconds() * 1000,
                             'status': self.thread_variables['_last_response'].status_code,
                             'responseSize': self.thread_variables['_last_response'].headers.get('content-length',
                                                                                None),
                             'assertionResult': 1
                         })

    @staticmethod
    def is_json(str_json):
        try:
            json.loads(str_json)
        except ValueError:
            return False
        return True
<|endoftext|>


<commit_before># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaCdbCdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaCdbCdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.1)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.1)
        basal.apoptosisTimeInDays = 8000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.00002

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.09)
        intermediate.apoptosisTimeInDays = 20000000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.00002

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.08)
        umbrella.apoptosisTimeInDays = 1000000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.00004

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500, #SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<commit_msg>no message<commit_after># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaCdbCdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaCdbCdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.1)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.1)
        basal.apoptosisTimeInDays = 8000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.000008

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.09)
        intermediate.apoptosisTimeInDays = 20000000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.000006

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.08)
        umbrella.apoptosisTimeInDays = 1000000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.00001

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500, #SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<|endoftext|>


<commit_before>#!/usr/bin/python

# Program to provide numeric string output of HDD temperature using netcat,
# so non-root user can query HDD temperature

# Depends on //netcat//, //hddtemp// (daemon on port)

# Copyright 2015 Vihar Malviya

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# st__ : String variable
# nm__ : Numeric (integer or float) variable
# ar__ : Array variable (associative arrays are dictionaries, non-associative are lists or tuples )
# ak__ : Array key
# ss__ : Status variable
# bl__ : Boolean variable
# ag__ : Arguments
# ob__ : Object

import sys , getopt
import subprocess

# Initialise exit status
ss__ExitStatus = 1

# Define the main function
def main ( ag__ArgList ) :

  # Initialise the working variables, tweak as per need
  ar__Netcat = {
    "ak__Bin" : "/bin/netcat" ,
    "ak__Host" : "localhost" ,
    "ak__Port" : "7634" }
# st__NetcatBin = "/bin/netcat"
# st__NetcatHost = "localhost"
# st__NetcatPort = "7634"

  # Initialise internal variables
  bl__DiskFound = False
  st__DiskRequest = ""

  # Scan for script arguments
  try :
    # Parse the supplied arguments
    ar__ArgOption , ar__ArgRemainder = getopt.getopt ( ag__ArgList , "hd:" , [ "help" , "disk=" ] )
    # Call help if no aruments supplied
    if len ( ar__ArgOption ) == 0 :
      fn__Help ( )
  # Call help if confused
  except getopt.GetoptError :
    fn__Help ( )

  # Check supplied arguments/parameters
  for st__ArgParam , st__ArgValue in ar__ArgOption :
    # Call help if requested so
    if st__ArgParam in ( "-h" , "--help" ) :
      fn__Help ( )
    # Read disk request
    elif st__ArgParam in ( "-d" , "--disk" ) :
      st__DiskRequest = st__ArgValue

  # Run //netcat// using the supplied working variables
  st__HddtempProc = subprocess.Popen (
    [
      ar__Netcat [ "ak__Bin" ] ,
      ar__Netcat [ "ak__Host" ] ,
      ar__Netcat [ "ak__Port" ] ] ,
    stdout = subprocess.PIPE ,
    stderr = subprocess.STDOUT ,
    universal_newlines = True )
  # Store the output of //netcat//
  st__HddtempRaw = st__HddtempProc.stdout.read ( )
  # Notify if the requested disk is not found in the //netcat// output
  if st__HddtempRaw.find ( st__DiskRequest ) == -1 :
    fn__NotFound ( )
  # Finally, start parsing the raw //netcat// output
  else :
    # //netcat// splits each disk info with a double pipe symbol
    # use it split the output into list of disks
    ar__DisksTemp = st__HddtempRaw.split ( "||" )
    # Go through each disk in the list
    for st__DiskInfo in ar__DisksTemp :
      # Search for the requested disk in the list item
      if st__DiskInfo.find ( st__DiskRequest ) != -1 :
        # If requested disk found, then split further using //netcat//'s default delimited (|)
        ar__DiskInfo = st__DiskInfo.split ( "|" )
        # Go through each fragment of output for the current disk from the list of disks found
        for st__DiskInfoFragment in ar__DiskInfo :
          # Check if the requested disk matches at least one of the fragments; it should.
          if st__DiskInfoFragment == st__DiskRequest :
            # If any one fragment matches, then set the flag for disk found
            bl__DiskFound = True
        # If the disk is found if one of the fragments is "C" (Celsius)
        if bl__DiskFound and "C" in ar__DiskInfo :
          # Locate "C"
          nm__CelsiusLocation = ar__DiskInfo.index ( "C" )
          # The temperature is the item just before "C"
          print ( ar__DiskInfo [ nm__CelsiusLocation - 1 ] )
          # Set exit status success flag
          ss__ExitStatus = 0
        else :
          fn__NotFound ( )
  return ss__ExitStatus

def fn__Help ( ) :
  print ( "Usage:" , sys.argv [ 0 ] , "-d <disk> (--disk=<disk>)" )
  sys.exit ( 2 )

def fn__NotFound ( ) :
  print ( "Disk not found" )
  sys.exit ( 3 )

if __name__ == "__main__" :
# main ( sys.argv [ 1: ] )
  sys.exit ( main ( sys.argv [ 1: ] ) )

<commit_msg>+ Added script name, version and date<commit_after>#!/usr/bin/python

# HDDtempNC
# Version 01.00.00
# Date: 2015-02-05

# Program to provide numeric string output of HDD temperature using netcat,
# so non-root user can query HDD temperature

# Depends on //netcat//, //hddtemp// (daemon on port)

# Copyright 2015 Vihar Malviya

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# st__ : String variable
# nm__ : Numeric (integer or float) variable
# ar__ : Array variable (associative arrays are dictionaries, non-associative are lists or tuples )
# ak__ : Array key
# ss__ : Status variable
# bl__ : Boolean variable
# ag__ : Arguments
# ob__ : Object

import sys , getopt
import subprocess

# Initialise exit status
ss__ExitStatus = 1

# Define the main function
def main ( ag__ArgList ) :

  # Initialise the working variables, tweak as per need
  ar__Netcat = {
    "ak__Bin" : "/bin/netcat" ,
    "ak__Host" : "localhost" ,
    "ak__Port" : "7634" }
# st__NetcatBin = "/bin/netcat"
# st__NetcatHost = "localhost"
# st__NetcatPort = "7634"

  # Initialise internal variables
  bl__DiskFound = False
  st__DiskRequest = ""

  # Scan for script arguments
  try :
    # Parse the supplied arguments
    ar__ArgOption , ar__ArgRemainder = getopt.getopt ( ag__ArgList , "hd:" , [ "help" , "disk=" ] )
    # Call help if no aruments supplied
    if len ( ar__ArgOption ) == 0 :
      fn__Help ( )
  # Call help if confused
  except getopt.GetoptError :
    fn__Help ( )

  # Check supplied arguments/parameters
  for st__ArgParam , st__ArgValue in ar__ArgOption :
    # Call help if requested so
    if st__ArgParam in ( "-h" , "--help" ) :
      fn__Help ( )
    # Read disk request
    elif st__ArgParam in ( "-d" , "--disk" ) :
      st__DiskRequest = st__ArgValue

  # Run //netcat// using the supplied working variables
  st__HddtempProc = subprocess.Popen (
    [
      ar__Netcat [ "ak__Bin" ] ,
      ar__Netcat [ "ak__Host" ] ,
      ar__Netcat [ "ak__Port" ] ] ,
    stdout = subprocess.PIPE ,
    stderr = subprocess.STDOUT ,
    universal_newlines = True )
  # Store the output of //netcat//
  st__HddtempRaw = st__HddtempProc.stdout.read ( )
  # Notify if the requested disk is not found in the //netcat// output
  if st__HddtempRaw.find ( st__DiskRequest ) == -1 :
    fn__NotFound ( )
  # Finally, start parsing the raw //netcat// output
  else :
    # //netcat// splits each disk info with a double pipe symbol
    # use it split the output into list of disks
    ar__DisksTemp = st__HddtempRaw.split ( "||" )
    # Go through each disk in the list
    for st__DiskInfo in ar__DisksTemp :
      # Search for the requested disk in the list item
      if st__DiskInfo.find ( st__DiskRequest ) != -1 :
        # If requested disk found, then split further using //netcat//'s default delimited (|)
        ar__DiskInfo = st__DiskInfo.split ( "|" )
        # Go through each fragment of output for the current disk from the list of disks found
        for st__DiskInfoFragment in ar__DiskInfo :
          # Check if the requested disk matches at least one of the fragments; it should.
          if st__DiskInfoFragment == st__DiskRequest :
            # If any one fragment matches, then set the flag for disk found
            bl__DiskFound = True
        # If the disk is found if one of the fragments is "C" (Celsius)
        if bl__DiskFound and "C" in ar__DiskInfo :
          # Locate "C"
          nm__CelsiusLocation = ar__DiskInfo.index ( "C" )
          # The temperature is the item just before "C"
          print ( ar__DiskInfo [ nm__CelsiusLocation - 1 ] )
          # Set exit status success flag
          ss__ExitStatus = 0
        else :
          fn__NotFound ( )
  return ss__ExitStatus

def fn__Help ( ) :
  print ( "Usage:" , sys.argv [ 0 ] , "-d <disk> (--disk=<disk>)" )
  sys.exit ( 2 )

def fn__NotFound ( ) :
  print ( "Disk not found" )
  sys.exit ( 3 )

if __name__ == "__main__" :
# main ( sys.argv [ 1: ] )
  sys.exit ( main ( sys.argv [ 1: ] ) )

<|endoftext|>