<commit_before>#  -*- coding: utf-8 -*-

import unittest

from faker import Faker

class TestPyint(unittest.TestCase):
    def setUp(self):
        self.factory = Faker()

    def test_pyint(self):
        self.assertIsInstance(self.factory.pyint(), int)

    def test_pyint_bounds(self):
        self.assertTrue(0 <= self.factory.pyint() <= 9999)

    def test_pyint_step(self):
        random_int = self.factory.pyint(step=2)
        self.assertEqual(0, random_int % 2)

    def test_pyint_bound_0(self):
        self.assertEqual(0, self.factory.pyint(min_value=0, max_value=0))

    def test_pyint_bound_positive(self):
        self.assertEqual(5, self.factory.pyint(min_value=5, max_value=5))

    def test_pyint_bound_negative(self):
        self.assertEqual(-5, self.factory.pyint(min_value=-5, max_value=-5))

    def test_pyint_range(self):
        self.assertTrue(0 <= self.factory.pyint(min_value=0, max_value=2) <= 2)

class TestPyfloat(unittest.TestCase):
    def setUp(self):
        self.factory = Faker()

    def test_pyfloat(self):
        result = self.factory.pyfloat()

        self.assertIsInstance(result, float)

    def test_left_digits(self):
        expected_left_digits = 10

        result = self.factory.pyfloat(left_digits=expected_left_digits)

        left_digits = len(str(abs(int(result))))
        self.assertGreaterEqual(expected_left_digits, left_digits)

    def test_right_digits(self):
        expected_right_digits = 10

        result = self.factory.pyfloat(right_digits=expected_right_digits)

        right_digits = len(str(result).split('.')[1])
        self.assertGreaterEqual(expected_right_digits, right_digits)

    def test_positive(self):
        result = self.factory.pyfloat(positive=True)

        self.assertGreaterEqual(result, 0)
        self.assertEqual(result, abs(result))

    def test_min_value(self):
        min_values = (0, 10, -1000, 1000, 999999)

        for min_value in min_values:
            result = self.factory.pyfloat(min_value=min_value)
            self.assertGreaterEqual(result, min_value)

    def test_max_value(self):
        max_values = (0, 10, -1000, 1000, 999999)

        for max_value in max_values:
            result = self.factory.pyfloat(max_value=max_value)
            self.assertLessEqual(result, max_value)

    def test_max_value_should_be_greater_than_min_value(self):
        """
        An exception should be raised if min_value is greater than max_value
        """
        expected_message = 'Min value cannot be greater than max value'
        with self.assertRaises(ValueError) as raises:
            self.factory.pyfloat(min_value=100, max_value=0)

        message = str(raises.exception)
        self.assertEqual(message, expected_message)
<commit_msg>fix float test (#1004)<commit_after>#  -*- coding: utf-8 -*-

import unittest

from faker import Faker

class TestPyint(unittest.TestCase):
    def setUp(self):
        self.factory = Faker()

    def test_pyint(self):
        self.assertIsInstance(self.factory.pyint(), int)

    def test_pyint_bounds(self):
        self.assertTrue(0 <= self.factory.pyint() <= 9999)

    def test_pyint_step(self):
        random_int = self.factory.pyint(step=2)
        self.assertEqual(0, random_int % 2)

    def test_pyint_bound_0(self):
        self.assertEqual(0, self.factory.pyint(min_value=0, max_value=0))

    def test_pyint_bound_positive(self):
        self.assertEqual(5, self.factory.pyint(min_value=5, max_value=5))

    def test_pyint_bound_negative(self):
        self.assertEqual(-5, self.factory.pyint(min_value=-5, max_value=-5))

    def test_pyint_range(self):
        self.assertTrue(0 <= self.factory.pyint(min_value=0, max_value=2) <= 2)

class TestPyfloat(unittest.TestCase):
    def setUp(self):
        self.factory = Faker()

    def test_pyfloat(self):
        result = self.factory.pyfloat()

        self.assertIsInstance(result, float)

    def test_left_digits(self):
        expected_left_digits = 10

        result = self.factory.pyfloat(left_digits=expected_left_digits)

        left_digits = len(str(abs(int(result))))
        self.assertGreaterEqual(expected_left_digits, left_digits)

    def test_right_digits(self):
        expected_right_digits = 10

        result = self.factory.pyfloat(right_digits=expected_right_digits)

        right_digits = len(('%r' % result).split('.')[1])
        self.assertGreaterEqual(expected_right_digits, right_digits)

    def test_positive(self):
        result = self.factory.pyfloat(positive=True)

        self.assertGreaterEqual(result, 0)
        self.assertEqual(result, abs(result))

    def test_min_value(self):
        min_values = (0, 10, -1000, 1000, 999999)

        for min_value in min_values:
            result = self.factory.pyfloat(min_value=min_value)
            self.assertGreaterEqual(result, min_value)

    def test_max_value(self):
        max_values = (0, 10, -1000, 1000, 999999)

        for max_value in max_values:
            result = self.factory.pyfloat(max_value=max_value)
            self.assertLessEqual(result, max_value)

    def test_max_value_should_be_greater_than_min_value(self):
        """
        An exception should be raised if min_value is greater than max_value
        """
        expected_message = 'Min value cannot be greater than max value'
        with self.assertRaises(ValueError) as raises:
            self.factory.pyfloat(min_value=100, max_value=0)

        message = str(raises.exception)
        self.assertEqual(message, expected_message)
<|endoftext|>


<commit_before>from django.contrib import admin

from models import *

class VendorAdmin(admin.ModelAdmin):
    list_display = ('code', 'name', 'summary')
    prepopulated_fields = {'name': ('code',)}
admin.site.register(Vendor,VendorAdmin)

class DataFieldAdmin(admin.ModelAdmin):
    list_display = ('code', 'description', 'format')
admin.site.register(DataField,DataFieldAdmin)
class VendorDataFieldAdmin(admin.ModelAdmin):
    list_display = ('vendor', 'field', 'code')
admin.site.register(VendorDataField,VendorDataFieldAdmin)

class VendorIdInline(admin.TabularInline):
    model = VendorId

class VendorIdAdmin(admin.ModelAdmin):
    list_display  = ('ticker', 'vendor', 'dataid',)
admin.site.register(VendorId,VendorIdAdmin)

class DataIdAdmin(admin.ModelAdmin):
    list_display  = ('code', 'name', 'live', 'get_country', 'tags')
    inlines = [VendorIdInline]
    search_fields = ('code', 'name', 'description', 'tags')
    #search_fields = ('code','name', 'vendor__name')
admin.site.register(DataId,DataIdAdmin)

class ExchangeAdmin(admin.ModelAdmin):
    list_display = ('code','name',)
admin.site.register(Exchange,ExchangeAdmin)
class BondMaturityTypeAdmin(admin.ModelAdmin):
    list_display = ('code','description',)
admin.site.register(BondMaturityType,BondMaturityTypeAdmin)
class CouponTypeAdmin(admin.ModelAdmin):
    list_display = ('code','month_frequency','day_count','description')
    ordering = ('code','month_frequency','day_count')
admin.site.register(CouponType,CouponTypeAdmin)
class FutureContractAdmin(admin.ModelAdmin):
    list_display = ('code','description','type','curncy','country','index','exchange')
admin.site.register(FutureContract,FutureContractAdmin)
class BondClassAdmin(admin.ModelAdmin):
    list_display = ('code','bondcode','description','coupon_type','curncy','country','settlement','issuer','sovereign','callable','putable','convertible')
    search_fields = ('bondcode','description', 'curncy', 'country')
    list_filter = ('sovereign','callable','putable','convertible','curncy', 'country')
admin.site.register(BondClass,BondClassAdmin)
class BondIssuerAdmin(admin.ModelAdmin):
    list_display = ('bond_class','issuer','dt')
admin.site.register(BondIssuer,BondIssuerAdmin)
class CollateralTypeAdmin(admin.ModelAdmin):
    list_display = ('name','order')
admin.site.register(CollateralType,CollateralTypeAdmin)

class FundTypeAdmin(admin.ModelAdmin):
    list_display = ('code','name','openended','description')
admin.site.register(FundType,FundTypeAdmin)
class FundManagerAdmin(admin.ModelAdmin):
    list_display = ('code','name','description','website')
admin.site.register(FundManager,FundManagerAdmin)

class IcAdmin(admin.ModelAdmin):
    list_display = ('id','code','firm_code','instype','data_id')
    search_fields = ('code','firm_code')
admin.site.register(InstrumentCode,IcAdmin)

admin.site.register(Cash3, list_display = ('id','code','curncy','type','extended'))
admin.site.register(FwdCash, list_display = ('id','code','curncy','value_date'))

class BondAdmin(admin.ModelAdmin):
    list_display = ('code','bond_class','ISIN','coupon','maturity_date')
    search_fields = ('ISIN',)
admin.site.register(Bond,BondAdmin)

#class EquityAdmin(admin.ModelAdmin):
#    pass
#admin.safe_register(Equity,EquityAdmin)
#class EtfAdmin(admin.ModelAdmin):
#    exclude = ['benchmark','underlying']
#admin.safe_register(Etf,EtfAdmin)
#class FundAdmin(admin.ModelAdmin):
#    exclude = ['benchmark','underlying']
#admin.safe_register(Fund,FundAdmin)

class InstDecompAdmin(admin.ModelAdmin):
    list_display = ('code','instrument','name','composition')
admin.site.register(InstDecomp,InstDecompAdmin)
class InstDecompHistoryAdmin(admin.ModelAdmin):
    list_display = ('inst_decomp','dt','composition')
admin.site.register(InstDecompHistory,InstDecompHistoryAdmin)

class MktDataAdmin(admin.ModelAdmin):
    list_display = ('vendor_id','field','dt','mkt_value')
    search_fields = ('vendor_id__ticker',)
    ordering      = ('-dt',)
admin.site.register(MktData,MktDataAdmin)

<commit_msg>refactoring<commit_after>from django.contrib import admin

from models import *

class VendorAdmin(admin.ModelAdmin):
    list_display = ('code', 'name', 'summary')
    prepopulated_fields = {'name': ('code',)}
admin.site.register(Vendor,VendorAdmin)

class DataFieldAdmin(admin.ModelAdmin):
    list_display = ('code', 'description', 'format')
admin.site.register(DataField,DataFieldAdmin)
class VendorDataFieldAdmin(admin.ModelAdmin):
    list_display = ('vendor', 'field', 'code')
admin.site.register(VendorDataField,VendorDataFieldAdmin)

class VendorIdInline(admin.TabularInline):
    model = VendorId

class VendorIdAdmin(admin.ModelAdmin):
    list_display  = ('ticker', 'vendor', 'dataid',)
admin.site.register(VendorId,VendorIdAdmin)

class DataIdAdmin(admin.ModelAdmin):
    list_display  = ('code', 'name', 'live', 'get_country', 'tags')
    inlines = [VendorIdInline]
    search_fields = ('code', 'name', 'description', 'tags')
    #search_fields = ('code','name', 'vendor__name')
admin.site.register(DataId,DataIdAdmin)

class ExchangeAdmin(admin.ModelAdmin):
    list_display = ('code','name',)
admin.site.register(Exchange,ExchangeAdmin)
class BondMaturityTypeAdmin(admin.ModelAdmin):
    list_display = ('code','description',)
admin.site.register(BondMaturityType,BondMaturityTypeAdmin)
class CouponTypeAdmin(admin.ModelAdmin):
    list_display = ('code','month_frequency','day_count','description')
    ordering = ('code','month_frequency','day_count')
admin.site.register(CouponType,CouponTypeAdmin)
class FutureContractAdmin(admin.ModelAdmin):
    list_display = ('code','description','type','curncy','country','index','exchange')
admin.site.register(FutureContract,FutureContractAdmin)
class BondClassAdmin(admin.ModelAdmin):
    list_display = ('code','bondcode','description','coupon_type','curncy','country','settlement','issuer','sovereign','callable','putable','convertible')
    search_fields = ('bondcode',)
    list_filter = ('sovereign','callable','putable','convertible','curncy', 'country')
admin.site.register(BondClass,BondClassAdmin)
class BondIssuerAdmin(admin.ModelAdmin):
    search_fields = ('bond_class__code',)
    list_display = ('bond_class','issuer','dt')
admin.site.register(BondIssuer,BondIssuerAdmin)
class CollateralTypeAdmin(admin.ModelAdmin):
    list_display = ('name','order')
admin.site.register(CollateralType,CollateralTypeAdmin)

class FundTypeAdmin(admin.ModelAdmin):
    list_display = ('code','name','openended','description')
admin.site.register(FundType,FundTypeAdmin)
class FundManagerAdmin(admin.ModelAdmin):
    list_display = ('code','name','description','website')
admin.site.register(FundManager,FundManagerAdmin)

class IcAdmin(admin.ModelAdmin):
    list_display = ('id','code','firm_code','instype','data_id')
    search_fields = ('code','firm_code')
admin.site.register(InstrumentCode,IcAdmin)

admin.site.register(Cash3, list_display = ('id','code','curncy','type','extended'))
admin.site.register(FwdCash, list_display = ('id','code','curncy','value_date'))

class BondAdmin(admin.ModelAdmin):
    list_display = ('code','bond_class','ISIN','coupon','maturity_date')
    search_fields = ('ISIN',)
admin.site.register(Bond,BondAdmin)

#class EquityAdmin(admin.ModelAdmin):
#    pass
#admin.safe_register(Equity,EquityAdmin)
#class EtfAdmin(admin.ModelAdmin):
#    exclude = ['benchmark','underlying']
#admin.safe_register(Etf,EtfAdmin)
#class FundAdmin(admin.ModelAdmin):
#    exclude = ['benchmark','underlying']
#admin.safe_register(Fund,FundAdmin)

class InstDecompAdmin(admin.ModelAdmin):
    list_display = ('code','instrument','name','composition')
admin.site.register(InstDecomp,InstDecompAdmin)
class InstDecompHistoryAdmin(admin.ModelAdmin):
    list_display = ('inst_decomp','dt','composition')
admin.site.register(InstDecompHistory,InstDecompHistoryAdmin)

class MktDataAdmin(admin.ModelAdmin):
    list_display = ('vendor_id','field','dt','mkt_value')
    search_fields = ('vendor_id__ticker',)
    ordering      = ('-dt',)
admin.site.register(MktData,MktDataAdmin)

<|endoftext|>


<commit_before>r"""
Definition
----------
This model calculates the structure factor of a polyelectrolyte solution with
the RPA expression derived by Borue and Erukhimovich\ [#Borue]_.  Note however
that the fitting procedure here does not follow the notation in that reference
as 's' and 't' are **not** decoupled. Instead the scattering intensity $I(q)$
is calculated as

.. math::

    I(q) = K\frac{q^2+k^2}{4\pi L_b\alpha ^2}
    \frac{1}{1+r_{0}^4(q^2+k^2)(q^2-12hC_a/b^2)} + background

    k^2 = 4\pi L_b(2C_s + \alpha C_a)

    r_{0}^2 = \frac{b}{\alpha \sqrt{C_a 48\pi L_b}}

where

$K$ is the contrast factor for the polymer which is defined differently than in
other models and is given in barns where $1 barn = 10^{-24} cm^2$.  $K$ is
defined as:

.. math::

    K = a^2

    a = b_p - (v_p/v_s) b_s

where $b_p$ and $b_s$ are sum of the scattering lengths of the atoms
constituting the monomer of the polymer and the sum of the scattering lengths
of the atoms constituting the solvent molecules respectively, and $v_p$ and
$v_s$ are the partial molar volume of the polymer and the solvent respectively

$L_b$ is the Bjerrum length(|Ang|) - **Note:** This parameter needs to be
kept constant for a given solvent and temperature!

$h$ is the virial parameter (|Ang^3|) - **Note:** See [#Borue]_ for the
correct interpretation of this parameter.  It incorporates second and third
virial coefficients and can be Negative.

$b$ is the monomer length(|Ang|), $C_s$ is the concentration of monovalent
salt(1/|Ang|- converted from mol/L), $\alpha$ is the ionization degree
(ionization degree : ratio of charged monomers  to total number of monomers),
$C_a$ is the polymer molar concentration(1/|Ang|- converted from mol/L),
and $background$ is the incoherent background.

For 2D data the scattering intensity is calculated in the same way as 1D,
where the $\vec q$ vector is defined as

.. math::

    q = \sqrt{q_x^2 + q_y^2}

Validation
----------

As of the last revision, this code is believed to be correct.  However it
needs further validation and should be used with caution at this time.  The
history of this code goes back to a 1998 implementation. It was recently noted
that in that implementation, while both the polymer concentration and salt
concentration were converted from experimental units of mol/L to more
dimensionally useful units of 1/|Ang|^3, only the converted version of the
polymer concentration was actually being used in the calculation while the
unconverted salt concentration (still in apparent units of mol/L) was being used.  This
was carried through to sasmodels as used for sasview 4.1(though the line of code converting
the salt concentration to the new units was removed somewhere along the line).
Simple dimensional analysis of the calculation shows that the
converted salt concentration should be used, which the original code suggests
was the intention, so this has now been corrected (for sasview 4.2). Once better
validation has been performed this note will be removed.

References
----------

.. [#Borue] V Y Borue, I Y Erukhimovich, *Macromolecules*, 21 (1988) 3240
.. [#] J F Joanny, L Leibler, *Journal de Physique*, 51 (1990) 545
.. [#] A Moussaid, F Schosseler, J P Munch, S Candau, *J. Journal de Physique
   II France*, 3 (1993) 573
.. [#] E Raphael, J F Joanny, *Europhysics Letters*, 11 (1990) 179

Authorship and Verification
----------------------------

* **Author:** NIST IGOR/DANSE **Date:** pre 2010
* **Last Modified by:** Paul Butler **Date:** September 25, 2018
* **Last Reviewed by:** Paul Butler **Date:** September 25, 2018
"""

import numpy as np
from numpy import inf, pi, sqrt

name = "be_polyelectrolyte"
title = "Polyelectrolyte with the RPA expression derived by Borue and Erukhimovich"
description = """
            Evaluate
            F(x) = K 1/(4 pi Lb (alpha)^(2)) (q^(2)+k2)/(1+(r02)^(2))
                 (q^(2)+k2) (q^(2)-(12 h C/b^(2)))

            has 3 internal parameters :
                   The inverse Debye Length: K2 = 4 pi Lb (2 Cs+alpha C)
                   r02 =1/alpha/Ca^(0.5) (B/(48 pi Lb)^(0.5))
                   Ca = 6.022136e-4 C
            """
category = "shape-independent"

# pylint: disable=bad-whitespace, line-too-long
#   ["name", "units", default, [lower, upper], "type", "description"],
parameters = [
    ["contrast_factor",       "barns",   10.0,  [-inf, inf], "", "Contrast factor of the polymer"],
    ["bjerrum_length",        "Ang",      7.1,  [0, inf],    "", "Bjerrum length"],
    ["virial_param",          "Ang^3", 12.0,  [-inf, inf], "", "Virial parameter"],
    ["monomer_length",        "Ang",     10.0,  [0, inf],    "", "Monomer length"],
    ["salt_concentration",    "mol/L",    0.0,  [-inf, inf], "", "Concentration of monovalent salt"],
    ["ionization_degree",     "",         0.05, [0, inf],    "", "Degree of ionization"],
    ["polymer_concentration", "mol/L",    0.7,  [0, inf],    "", "Polymer molar concentration"],
    ]
# pylint: enable=bad-whitespace, line-too-long

def Iq(q,
       contrast_factor,
       bjerrum_length,
       virial_param,
       monomer_length,
       salt_concentration,
       ionization_degree,
       polymer_concentration):
    """
    :params: see parameter table
    :return: 1-D form factor for polyelectrolytes in low salt

    parameter names, units, default values, and behavior (volume, sld etc) are
    defined in the parameter table.  The concentrations are converted from
    experimental mol/L to dimensionaly useful 1/A3 in first two lines
    """

    concentration_pol = polymer_concentration * 6.022136e-4
    concentration_salt = salt_concentration * 6.022136e-4

    k_square = 4.0 * pi * bjerrum_length * (2*concentration_salt +
                                            ionization_degree * concentration_pol)

    r0_square = 1.0/ionization_degree/sqrt(concentration_pol) * \
                (monomer_length/sqrt((48.0*pi*bjerrum_length)))

    term1 = contrast_factor/(4.0 * pi * bjerrum_length *
                             ionization_degree**2) * (q**2 + k_square)

    term2 = 1.0 + r0_square**2 * (q**2 + k_square) * \
        (q**2 - (12.0 * virial_param * concentration_pol/(monomer_length**2)))

    return term1/term2

Iq.vectorized = True  # Iq accepts an array of q values

def random():
    # TODO: review random be_polyelectrolyte model generation
    pars = dict(
        scale=10000, #background=0,
        #polymer_concentration=0.7,
        polymer_concentration=np.random.beta(5, 3), # around 70%
        #salt_concentration=0.0,
        # keep salt concentration extremely low
        # and use explicit molar to match polymer concentration
        salt_concentration=np.random.beta(1, 100)*6.022136e-4,
        #contrast_factor=10.0,
        contrast_fact=np.random.uniform(1, 100),
        #bjerrum_length=7.1,
        bjerrum_length=np.random.uniform(1, 10),
        #virial_param=12.0,
        virial_param=np.random.uniform(-1000, 30),
        #monomer_length=10.0,
        monomer_length=10.0**(4*np.random.beta(1.5, 3)),
        #ionization_degree=0.05,
        ionization_degree=np.random.beta(1.5, 4),
    )
    return pars

demo = dict(scale=1, background=0.1,
            contrast_factor=10.0,
            bjerrum_length=7.1,
            virial_param=12.0,
            monomer_length=10.0,
            salt_concentration=0.0,
            ionization_degree=0.05,
            polymer_concentration=0.7)

tests = [

    # Accuracy tests based on content in test/utest_other_models.py
    # Note that these should some day be validated beyond this self validation
    # (circular reasoning). -- i.e. the "good value," at least for those with
    # non zero salt concentrations, were obtained by running the current
    # model in SasView and copying the appropriate result here.
    #    PDB -- Sep 26, 2018
    [{'contrast_factor':       10.0,
      'bjerrum_length':         7.1,
      'virial_param':          12.0,
      'monomer_length':        10.0,
      'salt_concentration':     0.0,
      'ionization_degree':      0.05,
      'polymer_concentration':  0.7,
      'background':             0.001,
     }, 0.001, 0.0948379],

    [{'contrast_factor':       10.0,
      'bjerrum_length':       100.0,
      'virial_param':           3.0,
      'monomer_length':         5.0,
      'salt_concentration':     1.0,
      'ionization_degree':      0.1,
      'polymer_concentration':  1.0,
      'background':             0.0,
     }, 0.1, 0.253469484],

    [{'contrast_factor':       10.0,
      'bjerrum_length':       100.0,
      'virial_param':           3.0,
      'monomer_length':         5.0,
      'salt_concentration':     1.0,
      'ionization_degree':      0.1,
      'polymer_concentration':  1.0,
      'background':             1.0,
     }, 0.05, 1.738358122],

    [{'contrast_factor':     100.0,
      'bjerrum_length':       10.0,
      'virial_param':         12.0,
      'monomer_length':       10.0,
      'salt_concentration':    0.1,
      'ionization_degree':     0.5,
      'polymer_concentration': 0.1,
      'background':           0.01,
     }, 0.5, 0.012881893],
    ]
<commit_msg>Fixed inline substitution errors & improved formatting and presentation<commit_after>r"""
.. note:: Please read the Validation section below.

Definition
----------
This model calculates the structure factor of a polyelectrolyte solution with
the RPA expression derived by Borue and Erukhimovich\ [#Borue]_.  Note however
that the fitting procedure here does not follow the notation in that reference
as 's' and 't' are **not** decoupled. Instead the scattering intensity $I(q)$
is calculated as

.. math::

    I(q) = K\frac{q^2+k^2}{4\pi L_b\alpha ^2}
    \frac{1}{1+r_{0}^4(q^2+k^2)(q^2-12hC_a/b^2)} + background

    k^2 = 4\pi L_b(2C_s + \alpha C_a)

    r_{0}^2 = \frac{b}{\alpha \sqrt{C_a 48\pi L_b}}

where

$K$ is the contrast factor for the polymer which is defined differently than in
other models and is given in barns where 1 $barn = 10^{-24}$ $cm^2$.  $K$ is
defined as:

.. math::

    K = a^2

    a = b_p - (v_p/v_s) b_s

where:

- $b_p$ and $b_s$ are **sum of the scattering lengths of the atoms**
  constituting the polymer monomer and the solvent molecules, respectively.

- $v_p$ and $v_s$ are the partial molar volume of the polymer and the
  solvent, respectively.

- $L_b$ is the Bjerrum length (|Ang|) - **Note:** This parameter needs to be
  kept constant for a given solvent and temperature!

- $h$ is the virial parameter (|Ang^3|) - **Note:** See [#Borue]_ for the
  correct interpretation of this parameter.  It incorporates second and third
  virial coefficients and can be *negative*.

- $b$ is the monomer length (|Ang|).

- $C_s$ is the concentration of monovalent salt(1/|Ang| - converted from mol/L).

- $\alpha$ is the degree of ionization (the ratio of charged monomers to the total
  number of monomers)

- $C_a$ is the polymer molar concentration (1/|Ang| - converted from mol/L)

- $background$ is the incoherent background.

For 2D data the scattering intensity is calculated in the same way as 1D,
where the $\vec q$ vector is defined as

.. math::

    q = \sqrt{q_x^2 + q_y^2}

Validation
----------

As of the last revision, this code is believed to be correct.  However it
needs further validation and should be used with caution at this time.  The
history of this code goes back to a 1998 implementation. It was recently noted
that in that implementation, while both the polymer concentration and salt
concentration were converted from experimental units of mol/L to more
dimensionally useful units of 1/|Ang^3|, only the converted version of the
polymer concentration was actually being used in the calculation while the
unconverted salt concentration (still in apparent units of mol/L) was being
used.  This was carried through to Sasmodels as used for SasView 4.1 (though
the line of code converting the salt concentration to the new units was removed
somewhere along the line). Simple dimensional analysis of the calculation shows
that the converted salt concentration should be used, which the original code
suggests was the intention, so this has now been corrected (for SasView 4.2).
Once better validation has been performed this note will be removed.

References
----------

.. [#Borue] V Y Borue, I Y Erukhimovich, *Macromolecules*, 21 (1988) 3240
.. [#] J F Joanny, L Leibler, *Journal de Physique*, 51 (1990) 545
.. [#] A Moussaid, F Schosseler, J P Munch, S Candau, *J. Journal de Physique
   II France*, 3 (1993) 573
.. [#] E Raphael, J F Joanny, *Europhysics Letters*, 11 (1990) 179

Authorship and Verification
----------------------------

* **Author:** NIST IGOR/DANSE **Date:** pre 2010
* **Last Modified by:** Paul Butler **Date:** September 25, 2018
* **Last Reviewed by:** Paul Butler **Date:** September 25, 2018
"""

import numpy as np
from numpy import inf, pi, sqrt

name = "be_polyelectrolyte"
title = "Polyelectrolyte with the RPA expression derived by Borue and Erukhimovich"
description = """
            Evaluate
            F(x) = K 1/(4 pi Lb (alpha)^(2)) (q^(2)+k2)/(1+(r02)^(2))
                 (q^(2)+k2) (q^(2)-(12 h C/b^(2)))

            has 3 internal parameters :
                   The inverse Debye Length: K2 = 4 pi Lb (2 Cs+alpha C)
                   r02 =1/alpha/Ca^(0.5) (B/(48 pi Lb)^(0.5))
                   Ca = 6.022136e-4 C
            """
category = "shape-independent"

# pylint: disable=bad-whitespace, line-too-long
#   ["name", "units", default, [lower, upper], "type", "description"],
parameters = [
    ["contrast_factor",       "barns",   10.0,  [-inf, inf], "", "Contrast factor of the polymer"],
    ["bjerrum_length",        "Ang",      7.1,  [0, inf],    "", "Bjerrum length"],
    ["virial_param",          "Ang^3", 12.0,  [-inf, inf], "", "Virial parameter"],
    ["monomer_length",        "Ang",     10.0,  [0, inf],    "", "Monomer length"],
    ["salt_concentration",    "mol/L",    0.0,  [-inf, inf], "", "Concentration of monovalent salt"],
    ["ionization_degree",     "",         0.05, [0, inf],    "", "Degree of ionization"],
    ["polymer_concentration", "mol/L",    0.7,  [0, inf],    "", "Polymer molar concentration"],
    ]
# pylint: enable=bad-whitespace, line-too-long

def Iq(q,
       contrast_factor,
       bjerrum_length,
       virial_param,
       monomer_length,
       salt_concentration,
       ionization_degree,
       polymer_concentration):
    """
    :params: see parameter table
    :return: 1-D form factor for polyelectrolytes in low salt

    parameter names, units, default values, and behavior (volume, sld etc) are
    defined in the parameter table.  The concentrations are converted from
    experimental mol/L to dimensionaly useful 1/A3 in first two lines
    """

    concentration_pol = polymer_concentration * 6.022136e-4
    concentration_salt = salt_concentration * 6.022136e-4

    k_square = 4.0 * pi * bjerrum_length * (2*concentration_salt +
                                            ionization_degree * concentration_pol)

    r0_square = 1.0/ionization_degree/sqrt(concentration_pol) * \
                (monomer_length/sqrt((48.0*pi*bjerrum_length)))

    term1 = contrast_factor/(4.0 * pi * bjerrum_length *
                             ionization_degree**2) * (q**2 + k_square)

    term2 = 1.0 + r0_square**2 * (q**2 + k_square) * \
        (q**2 - (12.0 * virial_param * concentration_pol/(monomer_length**2)))

    return term1/term2

Iq.vectorized = True  # Iq accepts an array of q values

def random():
    # TODO: review random be_polyelectrolyte model generation
    pars = dict(
        scale=10000, #background=0,
        #polymer_concentration=0.7,
        polymer_concentration=np.random.beta(5, 3), # around 70%
        #salt_concentration=0.0,
        # keep salt concentration extremely low
        # and use explicit molar to match polymer concentration
        salt_concentration=np.random.beta(1, 100)*6.022136e-4,
        #contrast_factor=10.0,
        contrast_fact=np.random.uniform(1, 100),
        #bjerrum_length=7.1,
        bjerrum_length=np.random.uniform(1, 10),
        #virial_param=12.0,
        virial_param=np.random.uniform(-1000, 30),
        #monomer_length=10.0,
        monomer_length=10.0**(4*np.random.beta(1.5, 3)),
        #ionization_degree=0.05,
        ionization_degree=np.random.beta(1.5, 4),
    )
    return pars

demo = dict(scale=1, background=0.1,
            contrast_factor=10.0,
            bjerrum_length=7.1,
            virial_param=12.0,
            monomer_length=10.0,
            salt_concentration=0.0,
            ionization_degree=0.05,
            polymer_concentration=0.7)

tests = [

    # Accuracy tests based on content in test/utest_other_models.py
    # Note that these should some day be validated beyond this self validation
    # (circular reasoning). -- i.e. the "good value," at least for those with
    # non zero salt concentrations, were obtained by running the current
    # model in SasView and copying the appropriate result here.
    #    PDB -- Sep 26, 2018
    [{'contrast_factor':       10.0,
      'bjerrum_length':         7.1,
      'virial_param':          12.0,
      'monomer_length':        10.0,
      'salt_concentration':     0.0,
      'ionization_degree':      0.05,
      'polymer_concentration':  0.7,
      'background':             0.001,
     }, 0.001, 0.0948379],

    [{'contrast_factor':       10.0,
      'bjerrum_length':       100.0,
      'virial_param':           3.0,
      'monomer_length':         5.0,
      'salt_concentration':     1.0,
      'ionization_degree':      0.1,
      'polymer_concentration':  1.0,
      'background':             0.0,
     }, 0.1, 0.253469484],

    [{'contrast_factor':       10.0,
      'bjerrum_length':       100.0,
      'virial_param':           3.0,
      'monomer_length':         5.0,
      'salt_concentration':     1.0,
      'ionization_degree':      0.1,
      'polymer_concentration':  1.0,
      'background':             1.0,
     }, 0.05, 1.738358122],

    [{'contrast_factor':     100.0,
      'bjerrum_length':       10.0,
      'virial_param':         12.0,
      'monomer_length':       10.0,
      'salt_concentration':    0.1,
      'ionization_degree':     0.5,
      'polymer_concentration': 0.1,
      'background':           0.01,
     }, 0.5, 0.012881893],
    ]
<|endoftext|>


<commit_before>"""
config.py

Holds config information for each system.
"""
DBFILE = "/dragonDB.db"
isMCP = False
# Use 'localhost' for base station.
MCPIP = "192.168.0.10"
STATIONNAME = 'computer'
CHECK_RATE = 60
SENSORS = [{
    'poller': 'Serial',
    'pollRate': 4,
    'sensorName': 'test sensor',
    'report': False,
    'controls': []
}, {
    'poller': 'Crypto',
    'pollRate': 60,
    'sensorName': 'Ethereum',
    'id': 'ethereum',
    'report': False,
    'controls': []
}, {
    'poller': 'Crypto',
    'pollRate': 60,
    'sensorName': 'Ripple',
    'id': 'ripple',
    'report': False,
    'controls': []
}]
<commit_msg>removing config from repo<commit_after><|endoftext|>


<commit_before>"""Definition of the RAB class.

The RAB provides access to capabilities hosted on remote nodes in MP
calculations.  It implements much of the component interface, but it isn't a
component as such, and in some respects it works very differently to proper
components.

The name RAB was originally an acronym for "Remote Access Broker", but mostly
it's just a name.  Also, "RAB" are my mom's initials.  Hi, Mom!

This module should _only_ be imported by the my.py module.  It imports mpi4py,
which will have the side effect of trying to initialize MPI if mp.py hasn't
already done it.
"""

from cassandra.constants import TAG_REQ, TAG_REQID_BASE
from mpi4py import MPI
import concurrent.futures as ft
import threading
import logging

# The choice of the sleep length in RAB listener loop is a tradeoff.  Making it
# shorter improves responsiveness, but creates more waiting in the GIL,
# potentially slowing down threads that are actually running models (especially
# if those models are implemented in python and running in the same process).
RAB_LOOP_SLEEP = 0.05          # 50 ms

class RAB(object):
    def __init__(self, cap_tbl, comm=MPI.COMM_WORLD):
        self.cap_tbl = cap_tbl
        self.comm = comm
        self.rank = comm.Get_rank()
        self.status = 0         # status indicator follows ComponentBase convention
        self.terminate = False   # sentinel indicating when it's time for the RAB to exit
        self.remote_caps = {}   # Table of remote capabilities
        self.requests_outstanding = {} # Table of requests in process

        # members for managing message tags
        self.taglock = threading.Condition() # Lock for working with the list of tags
        self.tags = set()                    # MPI message tags in use

    def run(self):
        """Execute the RAB's listen() method in a separate thread."""
        thread = threading.Thread(target=lambda: self.listen())
        thread.start()
        # returns immediately
        return thread

    def shutdown(self):
        """Order the RAB to exit its listen() thread."""
        # The GIL should take care of any locking that needs to be done here.
        logging.debug(f'{self.comm.Get_rank()}: shutdown.')
        self.terminate = True

    def addremote(self, remote_rank, remote_cap_tbl):
        """Add capabilities from a remote process to our remote capability table.

        :param remote_rank: MPI rank of the remote process
        :param remote_cap_tbl: List of capabilities held by the remote process
        """

        for cap in remote_cap_tbl:
            if cap in self.cap_tbl:
                raise RuntimeError(f'Duplicate definition of capability {cap}.')
            self.cap_tbl[cap] = self
            self.remote_caps[cap] = remote_rank

    def unique_tag(self):
        """Get a message tag not already in use by another thread.
        """
        # Start with the base value of TAG_REQ, and search forward until we find
        # one that isn't already in use.  We have to acquire a lock to do this,
        # in order to avoid a race condition if another thread is trying to get
        # a tag assignment at the same time.
        tag = TAG_REQ
        with self.taglock:
            while tag in self.tags:
                tag += 1
            self.tags.add(tag)
            return tag

    def fetch(self, capability):
        """Fetch a capability from a remote process.

        This method uses blocking sends and receives, so it will automatically
        block the calling thread until the data is available (i.e., without
        having to wait on a condition variable).

        We use MPI message tags to disambiguate multiple requests from different
        threads on the same node.  By issuing a unique tag for each request, we
        ensure that the combination of source and tag uniquely identifies a
        particular request in the system.  Thus, if the responding RAB sends its
        result back to the source of the request, using the same tag as the one
        sent with the request, then we can be certain that the result will be
        delivered to the thread that requested it.

        """

        # This call could have come either from a component asking for a remote
        # capability, or from a spawned thread from the listener.  In the latter
        # case, we need to forward the request to the component that has the
        # data.  We *don't* raise CapabilityNotFound exceptions here because
        # everything that gets here should have been forwarded by a component's
        # fetch method, which will already have checked for the capability's
        # existence.
        provider = self.cap_tbl[capability]
        if self is not provider:
            return provider.fetch(capability)

        provider_rank = self.remote_caps[capability]
        reqtag = self.unique_tag() # get a unique tag for the response

        # send both the capability and the tag we will be expecting for the
        # response to the remote RAB
        data = (capability, reqtag)
        logging.debug(f'requesting {capability} from {provider_rank} on tag {reqtag}')
        self.comm.send(data, dest=provider_rank, tag=TAG_REQ)
        # wait for the response
        logging.debug(f'waiting on {provider_rank} with tag {reqtag}')
        rslt = self.comm.recv(source=provider_rank, tag=reqtag)
        logging.debug(f'got {reqtag} from {provider_rank}')
        return rslt

    def listen(self):
        """Main control loop for RAB listener.

        The control loop proceeds as follows:

        1. Check for incoming requests from other nodes.
        2. If a request is outstanding:
           A. Spawn a thread to fetch the data required (using a concurrent.future)
           B. Record the source of the request alongside the Future object in the
              table of running requests
           C. repeat until no further requests are outstanding
        3. Check running requests to see if any have completed.  If so:
           A. Harvest the result.
           B. Look up the original source of the request
           C. Send (nonblocking) the result to the requestor
           D. Remove the completed Future from the request table
           E. Repeat until there are no further running requests.
        4. Check whether we have been asked to shut down.  If so, terminate.
        5. Sleep for delay = RAB_LOOP_SLEEP
        6. Repeat until we terminate in step 4.
        """

        from time import sleep
        with ft.ThreadPoolExecutor() as self.executor:
            logging.debug(f'begin listen')
            while True:
                ### Check for incoming requests.
                self.process_incoming()

                ### Check outstanding requests (the ones we have running)
                self.process_outstanding()

                ### Check for termination request.  The barrier in mp.finalize()
                ### ensures that this can't happen until all other components on
                ### this and other nodes have finished; therefore, all the
                ### outstanding requests will have been processed.
                if self.terminate:
                    logging.debug('listen loop got request to terminate')
                    assert(len(self.requests_outstanding) == 0)
                    break

                sleep(RAB_LOOP_SLEEP)

        # Record our status as successful
        self.status = 1
        logging.debug(f'self.comm.Get_rank(): listen exiting')
        return 0

    def process_incoming(self):
        """Dispatch incoming requests from other nodes, if any.
        """

        stat = MPI.Status()
        while self.comm.iprobe(tag=TAG_REQ, status=stat):
            source = stat.Get_source()
            capability, rtag = self.comm.recv(source=source)
            logging.debug(f'{self.rank}: processing {capability} from {source} on tag {rtag}')

            # Create a thread to fetch the capability.  This thread might block.
            future = self.executor.submit(self.fetch, capability)

            # Add the source and the remote tag to the table, indexed by thread.
            # We don't need the capability anymore, so we don't store it.
            self.requests_outstanding[future] = (source, rtag)

    # End of process_incoming()

    def process_outstanding(self):
        """Process any threads that have finished servicing their requests.

        For each thread in the list of outstanding requests, check to see if it
        has completed.  If so, send the response back to the requestor and
        remove the thread object from the outstanding requests list.

        """

        threads = list(self.requests_outstanding.keys())
        for thread in threads:
            if not thread.done():
                continue

            rslt = thread.result() # no need to specify a timeout, since we
                                   # already verified that the thread completed.

            source, rtag = self.requests_outstanding.pop(thread)
            logging.debug(f'sending result to {source} on tag {rtag}')
            # theoretically this could block, but the fetch method on the remote
            # node will have posted a receive as soon as the request was sent.
            self.comm.send(rslt, dest=source, tag=rtag)
            logging.debug(f'sent {rtag} to {source}')

    # End of process_outstanding

<commit_msg>Set RAB status to successful on init.<commit_after>"""Definition of the RAB class.

The RAB provides access to capabilities hosted on remote nodes in MP
calculations.  It implements much of the component interface, but it isn't a
component as such, and in some respects it works very differently to proper
components.

The name RAB was originally an acronym for "Remote Access Broker", but mostly
it's just a name.  Also, "RAB" are my mom's initials.  Hi, Mom!

This module should _only_ be imported by the my.py module.  It imports mpi4py,
which will have the side effect of trying to initialize MPI if mp.py hasn't
already done it.
"""

from cassandra.constants import TAG_REQ, TAG_REQID_BASE
from mpi4py import MPI
import concurrent.futures as ft
import threading
import logging

# The choice of the sleep length in RAB listener loop is a tradeoff.  Making it
# shorter improves responsiveness, but creates more waiting in the GIL,
# potentially slowing down threads that are actually running models (especially
# if those models are implemented in python and running in the same process).
RAB_LOOP_SLEEP = 0.05          # 50 ms

class RAB(object):
    def __init__(self, cap_tbl, comm=MPI.COMM_WORLD):
        self.cap_tbl = cap_tbl
        self.comm = comm
        self.rank = comm.Get_rank()
        self.status = 1         # Always show RAB status as successful
        self.terminate = False   # sentinel indicating when it's time for the RAB to exit
        self.remote_caps = {}   # Table of remote capabilities
        self.requests_outstanding = {} # Table of requests in process

        # members for managing message tags
        self.taglock = threading.Condition() # Lock for working with the list of tags
        self.tags = set()                    # MPI message tags in use

    def run(self):
        """Execute the RAB's listen() method in a separate thread."""
        thread = threading.Thread(target=lambda: self.listen())
        thread.start()
        # returns immediately
        return thread

    def shutdown(self):
        """Order the RAB to exit its listen() thread."""
        # The GIL should take care of any locking that needs to be done here.
        logging.debug(f'{self.comm.Get_rank()}: shutdown.')
        self.terminate = True

    def addremote(self, remote_rank, remote_cap_tbl):
        """Add capabilities from a remote process to our remote capability table.

        :param remote_rank: MPI rank of the remote process
        :param remote_cap_tbl: List of capabilities held by the remote process
        """

        for cap in remote_cap_tbl:
            if cap in self.cap_tbl:
                raise RuntimeError(f'Duplicate definition of capability {cap}.')
            self.cap_tbl[cap] = self
            self.remote_caps[cap] = remote_rank

    def unique_tag(self):
        """Get a message tag not already in use by another thread.
        """
        # Start with the base value of TAG_REQ, and search forward until we find
        # one that isn't already in use.  We have to acquire a lock to do this,
        # in order to avoid a race condition if another thread is trying to get
        # a tag assignment at the same time.
        tag = TAG_REQ
        with self.taglock:
            while tag in self.tags:
                tag += 1
            self.tags.add(tag)
            return tag

    def fetch(self, capability):
        """Fetch a capability from a remote process.

        This method uses blocking sends and receives, so it will automatically
        block the calling thread until the data is available (i.e., without
        having to wait on a condition variable).

        We use MPI message tags to disambiguate multiple requests from different
        threads on the same node.  By issuing a unique tag for each request, we
        ensure that the combination of source and tag uniquely identifies a
        particular request in the system.  Thus, if the responding RAB sends its
        result back to the source of the request, using the same tag as the one
        sent with the request, then we can be certain that the result will be
        delivered to the thread that requested it.

        """

        # This call could have come either from a component asking for a remote
        # capability, or from a spawned thread from the listener.  In the latter
        # case, we need to forward the request to the component that has the
        # data.  We *don't* raise CapabilityNotFound exceptions here because
        # everything that gets here should have been forwarded by a component's
        # fetch method, which will already have checked for the capability's
        # existence.
        provider = self.cap_tbl[capability]
        if self is not provider:
            return provider.fetch(capability)

        provider_rank = self.remote_caps[capability]
        reqtag = self.unique_tag() # get a unique tag for the response

        # send both the capability and the tag we will be expecting for the
        # response to the remote RAB
        data = (capability, reqtag)
        logging.debug(f'requesting {capability} from {provider_rank} on tag {reqtag}')
        self.comm.send(data, dest=provider_rank, tag=TAG_REQ)
        # wait for the response
        logging.debug(f'waiting on {provider_rank} with tag {reqtag}')
        rslt = self.comm.recv(source=provider_rank, tag=reqtag)
        logging.debug(f'got {reqtag} from {provider_rank}')
        return rslt

    def listen(self):
        """Main control loop for RAB listener.

        The control loop proceeds as follows:

        1. Check for incoming requests from other nodes.
        2. If a request is outstanding:
           A. Spawn a thread to fetch the data required (using a concurrent.future)
           B. Record the source of the request alongside the Future object in the
              table of running requests
           C. repeat until no further requests are outstanding
        3. Check running requests to see if any have completed.  If so:
           A. Harvest the result.
           B. Look up the original source of the request
           C. Send (nonblocking) the result to the requestor
           D. Remove the completed Future from the request table
           E. Repeat until there are no further running requests.
        4. Check whether we have been asked to shut down.  If so, terminate.
        5. Sleep for delay = RAB_LOOP_SLEEP
        6. Repeat until we terminate in step 4.
        """

        from time import sleep
        with ft.ThreadPoolExecutor() as self.executor:
            logging.debug(f'begin listen')
            while True:
                ### Check for incoming requests.
                self.process_incoming()

                ### Check outstanding requests (the ones we have running)
                self.process_outstanding()

                ### Check for termination request.  The barrier in mp.finalize()
                ### ensures that this can't happen until all other components on
                ### this and other nodes have finished; therefore, all the
                ### outstanding requests will have been processed.
                if self.terminate:
                    logging.debug('listen loop got request to terminate')
                    assert(len(self.requests_outstanding) == 0)
                    break

                sleep(RAB_LOOP_SLEEP)

        logging.debug(f'self.comm.Get_rank(): listen exiting')
        return 0

    def process_incoming(self):
        """Dispatch incoming requests from other nodes, if any.
        """

        stat = MPI.Status()
        while self.comm.iprobe(tag=TAG_REQ, status=stat):
            source = stat.Get_source()
            capability, rtag = self.comm.recv(source=source)
            logging.debug(f'{self.rank}: processing {capability} from {source} on tag {rtag}')

            # Create a thread to fetch the capability.  This thread might block.
            future = self.executor.submit(self.fetch, capability)

            # Add the source and the remote tag to the table, indexed by thread.
            # We don't need the capability anymore, so we don't store it.
            self.requests_outstanding[future] = (source, rtag)

    # End of process_incoming()

    def process_outstanding(self):
        """Process any threads that have finished servicing their requests.

        For each thread in the list of outstanding requests, check to see if it
        has completed.  If so, send the response back to the requestor and
        remove the thread object from the outstanding requests list.

        """

        threads = list(self.requests_outstanding.keys())
        for thread in threads:
            if not thread.done():
                continue

            rslt = thread.result() # no need to specify a timeout, since we
                                   # already verified that the thread completed.

            source, rtag = self.requests_outstanding.pop(thread)
            logging.debug(f'sending result to {source} on tag {rtag}')
            # theoretically this could block, but the fetch method on the remote
            # node will have posted a receive as soon as the request was sent.
            self.comm.send(rslt, dest=source, tag=rtag)
            logging.debug(f'sent {rtag} to {source}')

    # End of process_outstanding

<|endoftext|>


<commit_before>import codecs
from collections import namedtuple
import logging
import serial
import struct
import time

_PROTOCOL_HEADER = b'CLS'
_PROTOCOL_VERSION = b'0'
_DEFAULT_BAUD = 9600
_DEFAULT_TIMEOUT = 10

# Instructions
_READ_433 = b'1'
_WRITE_433 = b'2'

# Responses
_AWAITING_DATA = b'B'
_INCOMING_DATA = b'C'
_RADIO_TIMEOUT = b'D'
_HELLO = b'E'
_GOODBYE = b'F'
_BAD_HEADER = b'G'
_WRONG_VERSION = b'H'
_HEARTBEAT = b'I'
_UNKNOWN_OP_CODE = b'Z'

_RESPONSE_DEFINITIONS = {
    _AWAITING_DATA: "awaiting additional data",
    _INCOMING_DATA: "there is an incoming radio transmission",
    _RADIO_TIMEOUT: "timed out awaiting radio transmission",
    _HELLO: "protocol hello (beginning handshake)",
    _GOODBYE: "protocol goodbye (terminating communication)",
    _BAD_HEADER: "bad protocol header received",
    _WRONG_VERSION: "bad protocol version received",
    _UNKNOWN_OP_CODE: "unknown request received"
}

Signal = namedtuple('Signal', ['protocol', 'pulse_length', 'message'])

LOG = logging.getLogger(__name__)

class BadResponseError(Exception):
    def __init__(self, actual, expected):
        actual_msg = _RESPONSE_DEFINITIONS \
                     .get(actual, "Strange response: %s" % repr(actual))
        expected_msg = _RESPONSE_DEFINITIONS.get(expected, repr(expected))
        message = "[Expected] %s, [Actual] %s" \
                  % (expected_msg, actual_msg)
        super(BadResponseError, self).__init__(message)

class RadioTimeout(Exception):
    pass

class SignalDriver(object):

    def __init__(self, serial_device_file, baud_rate=_DEFAULT_BAUD,
                 timeout=_DEFAULT_TIMEOUT):

        self.serial_device_file = serial_device_file
        self.baud_rate = baud_rate
        self.timeout = timeout
        self.conn = None
        self.reconnect()

    def reconnect(self):
        if self.conn:
            try:
                self.conn.close()
            except Exception as e:
                LOG.warn("Exception when terminating connection: %s", str(e))

        self.conn = serial.Serial(self.serial_device_file,
                                  self.baud_rate,
                                  timeout=self.timeout)

        # Time for the ATMega328P to complete boot in the event of a
        # serial-triggered reset.
        time.sleep(3)

    def _assert_response(self, expected, actual=None):
        if not actual:
            actual = self.conn.read()
        if actual != expected:
            raise BadResponseError(actual, expected)

    def _read_2byte_int(self):
        return struct.unpack('<H', self.conn.read(2))

    def _write_as_2bytes(self, x):
        if isinstance(x, str):
            x = ord(x)
        self.conn.write(struct.pack("<H", x))

    def _perform_handshake(self):
        self.conn.write(_PROTOCOL_HEADER)
        self.conn.write(_PROTOCOL_VERSION)
        self.conn.flush()
        self._assert_response(_HELLO)

    def send_signal(self, message, pulse_length, repetitions=1, protocol=1):
        self._perform_handshake()
        self.conn.write(_WRITE_433)
        self.conn.flush()
        self._assert_response(_AWAITING_DATA)

        # Send the message definition payload.
        self._write_as_2bytes(protocol)
        self._write_as_2bytes(pulse_length)
        self._write_as_2bytes(repetitions)
        self._write_as_2bytes(len(message))
        if isinstance(message, bytes):
            self.conn.write(message)
        else:
            self.conn.write(codecs.decode(message, 'hex'))
        self._assert_response(_GOODBYE)

    def read_signals(self, message_num, radio_timeout=10000):
        self._perform_handshake()
        self.conn.write(_READ_433)
        self.conn.flush()
        self._assert_response(_AWAITING_DATA)

        # Send desired spec.
        self._write_as_2bytes(message_num)
        self._write_as_2bytes(radio_timeout)

        # Begin reading message data.
        for _ in range(message_num):
            while True:
                resp = self.conn.read()
                if resp == _HEARTBEAT:
                    continue
                if resp == _RADIO_TIMEOUT:
                    raise RadioTimeout(radio_timeout)
                self._assert_response(_INCOMING_DATA, actual=resp)
            protocol = self._read_2byte_int()[0]
            delay = self._read_2byte_int()[0]
            size = self._read_2byte_int()[0]
            message = self.conn.read(size)
            yield Signal(protocol, delay, message)
        self._assert_response(_GOODBYE)
<commit_msg>Added missing break statement<commit_after>import codecs
from collections import namedtuple
import logging
import serial
import struct
import time

_PROTOCOL_HEADER = b'CLS'
_PROTOCOL_VERSION = b'0'
_DEFAULT_BAUD = 9600
_DEFAULT_TIMEOUT = 10

# Instructions
_READ_433 = b'1'
_WRITE_433 = b'2'

# Responses
_AWAITING_DATA = b'B'
_INCOMING_DATA = b'C'
_RADIO_TIMEOUT = b'D'
_HELLO = b'E'
_GOODBYE = b'F'
_BAD_HEADER = b'G'
_WRONG_VERSION = b'H'
_HEARTBEAT = b'I'
_UNKNOWN_OP_CODE = b'Z'

_RESPONSE_DEFINITIONS = {
    _AWAITING_DATA: "awaiting additional data",
    _INCOMING_DATA: "there is an incoming radio transmission",
    _RADIO_TIMEOUT: "timed out awaiting radio transmission",
    _HELLO: "protocol hello (beginning handshake)",
    _GOODBYE: "protocol goodbye (terminating communication)",
    _BAD_HEADER: "bad protocol header received",
    _WRONG_VERSION: "bad protocol version received",
    _UNKNOWN_OP_CODE: "unknown request received"
}

Signal = namedtuple('Signal', ['protocol', 'pulse_length', 'message'])

LOG = logging.getLogger(__name__)

class BadResponseError(Exception):
    def __init__(self, actual, expected):
        actual_msg = _RESPONSE_DEFINITIONS \
                     .get(actual, "Strange response: %s" % repr(actual))
        expected_msg = _RESPONSE_DEFINITIONS.get(expected, repr(expected))
        message = "[Expected] %s, [Actual] %s" \
                  % (expected_msg, actual_msg)
        super(BadResponseError, self).__init__(message)

class RadioTimeout(Exception):
    pass

class SignalDriver(object):

    def __init__(self, serial_device_file, baud_rate=_DEFAULT_BAUD,
                 timeout=_DEFAULT_TIMEOUT):

        self.serial_device_file = serial_device_file
        self.baud_rate = baud_rate
        self.timeout = timeout
        self.conn = None
        self.reconnect()

    def reconnect(self):
        if self.conn:
            try:
                self.conn.close()
            except Exception as e:
                LOG.warn("Exception when terminating connection: %s", str(e))

        self.conn = serial.Serial(self.serial_device_file,
                                  self.baud_rate,
                                  timeout=self.timeout)

        # Time for the ATMega328P to complete boot in the event of a
        # serial-triggered reset.
        time.sleep(3)

    def _assert_response(self, expected, actual=None):
        if not actual:
            actual = self.conn.read()
        if actual != expected:
            raise BadResponseError(actual, expected)

    def _read_2byte_int(self):
        return struct.unpack('<H', self.conn.read(2))

    def _write_as_2bytes(self, x):
        if isinstance(x, str):
            x = ord(x)
        self.conn.write(struct.pack("<H", x))

    def _perform_handshake(self):
        self.conn.write(_PROTOCOL_HEADER)
        self.conn.write(_PROTOCOL_VERSION)
        self.conn.flush()
        self._assert_response(_HELLO)

    def send_signal(self, message, pulse_length, repetitions=1, protocol=1):
        self._perform_handshake()
        self.conn.write(_WRITE_433)
        self.conn.flush()
        self._assert_response(_AWAITING_DATA)

        # Send the message definition payload.
        self._write_as_2bytes(protocol)
        self._write_as_2bytes(pulse_length)
        self._write_as_2bytes(repetitions)
        self._write_as_2bytes(len(message))
        if isinstance(message, bytes):
            self.conn.write(message)
        else:
            self.conn.write(codecs.decode(message, 'hex'))
        self._assert_response(_GOODBYE)

    def read_signals(self, message_num, radio_timeout=10000):
        self._perform_handshake()
        self.conn.write(_READ_433)
        self.conn.flush()
        self._assert_response(_AWAITING_DATA)

        # Send desired spec.
        self._write_as_2bytes(message_num)
        self._write_as_2bytes(radio_timeout)

        # Begin reading message data.
        for _ in range(message_num):
            while True:
                resp = self.conn.read()
                if resp == _HEARTBEAT:
                    continue
                if resp == _RADIO_TIMEOUT:
                    raise RadioTimeout(radio_timeout)
                self._assert_response(_INCOMING_DATA, actual=resp)
                break
            protocol = self._read_2byte_int()[0]
            delay = self._read_2byte_int()[0]
            size = self._read_2byte_int()[0]
            message = self.conn.read(size)
            yield Signal(protocol, delay, message)
        self._assert_response(_GOODBYE)
<|endoftext|>


<commit_before>from hwt.hdlObjects.constants import INTF_DIRECTION, DIRECTION
from hwt.synthesizer.exceptions import IntfLvlConfErr

class InterfaceDirectionFns():

    def _setDirectionsLikeIn(self, intfDir):
        # [TODO] array elements
        d = DIRECTION.asIntfDirection(self._masterDir)
        if intfDir == INTF_DIRECTION.MASTER or d == INTF_DIRECTION.TRISTATE:
            self._direction = d
            for i in self._interfaces:
                i._setDirectionsLikeIn(d)
        else:
            opDir = INTF_DIRECTION.opposite(d)
            self._direction = opDir
            for i in self._interfaces:
                i._setDirectionsLikeIn(opDir)

    def __directionProbe(self):
        if not self._interfaces:
            s = self._sig
            d = self._masterDir
            return (not bool(s.drivers), bool(s.drivers))

        allInMasterConf = True
        allInSlaveConf = True
        for i in self._interfaces:
            i._resolveDirections(updateDir=False)
            d = i._direction
            md = DIRECTION.asIntfDirection(i._masterDir)
            if d != INTF_DIRECTION.UNKNOWN:
                isLikeInM = d == md
                isLikeInS = d == INTF_DIRECTION.opposite(md)
                allInMasterConf = allInMasterConf and isLikeInM
                allInSlaveConf = allInSlaveConf and isLikeInS
        return (allInMasterConf, allInSlaveConf)

    def _resolveDirections(self, updateDir=True):
        allM, allS = self.__directionProbe()

        if allM and allS and self._arrayElemCache:  # if direction is nod clear from this intf. and it has elems.
            allM, allS = self._arrayElemCache[0].__directionProbe()

        if allM and allS:
            self._direction = INTF_DIRECTION.UNKNOWN
        elif allM:
            self._direction = INTF_DIRECTION.MASTER
        elif allS:
            self._direction = INTF_DIRECTION.SLAVE
        else:
            raise IntfLvlConfErr("Subinterfaces on %s \nhave not consistent directions\n%s" %
                                 (repr(self),
                                  '\n'.join([str((i._direction, repr(i)))
                                             for i in self._interfaces])))

        if updateDir:
            if self._direction == INTF_DIRECTION.UNKNOWN:
                self._direction = INTF_DIRECTION.MASTER
            self._setDirectionsLikeIn(self._direction)

    def _setAsExtern(self, isExtern):
        """Set interface as extern"""
        self._isExtern = isExtern
        for prop in self._interfaces:
            prop._setAsExtern(isExtern)

    def _setDirLock(self, dirLock):
        self._dirLocked = dirLock
        for intf in self._interfaces:
            intf._setDirLock(dirLock)

    def _reverseDirection(self):
        """Reverse direction of this interface in implementation stage"""
        if self._dirLocked:
            raise IntfLvlConfErr("Can not reverse direction on interface %s because it was locked (%s)" %
                                 (repr(self), self._direction))
        self._direction = INTF_DIRECTION.opposite(self._direction)
        for intf in self._interfaces:
            intf._reverseDirection()
<commit_msg>codestyle: rm unused<commit_after>from hwt.hdlObjects.constants import INTF_DIRECTION, DIRECTION
from hwt.synthesizer.exceptions import IntfLvlConfErr

class InterfaceDirectionFns():

    def _setDirectionsLikeIn(self, intfDir):
        # [TODO] array elements
        d = DIRECTION.asIntfDirection(self._masterDir)
        if intfDir == INTF_DIRECTION.MASTER or d == INTF_DIRECTION.TRISTATE:
            self._direction = d
            for i in self._interfaces:
                i._setDirectionsLikeIn(d)
        else:
            opDir = INTF_DIRECTION.opposite(d)
            self._direction = opDir
            for i in self._interfaces:
                i._setDirectionsLikeIn(opDir)

    def __directionProbe(self):
        if not self._interfaces:
            s = self._sig
            return (not bool(s.drivers), bool(s.drivers))

        allInMasterConf = True
        allInSlaveConf = True
        for i in self._interfaces:
            i._resolveDirections(updateDir=False)
            d = i._direction
            md = DIRECTION.asIntfDirection(i._masterDir)
            if d != INTF_DIRECTION.UNKNOWN:
                isLikeInM = d == md
                isLikeInS = d == INTF_DIRECTION.opposite(md)
                allInMasterConf = allInMasterConf and isLikeInM
                allInSlaveConf = allInSlaveConf and isLikeInS
        return (allInMasterConf, allInSlaveConf)

    def _resolveDirections(self, updateDir=True):
        allM, allS = self.__directionProbe()

        if allM and allS and self._arrayElemCache:  # if direction is nod clear from this intf. and it has elems.
            allM, allS = self._arrayElemCache[0].__directionProbe()

        if allM and allS:
            self._direction = INTF_DIRECTION.UNKNOWN
        elif allM:
            self._direction = INTF_DIRECTION.MASTER
        elif allS:
            self._direction = INTF_DIRECTION.SLAVE
        else:
            raise IntfLvlConfErr("Subinterfaces on %s \nhave not consistent directions\n%s" %
                                 (repr(self),
                                  '\n'.join([str((i._direction, repr(i)))
                                             for i in self._interfaces])))

        if updateDir:
            if self._direction == INTF_DIRECTION.UNKNOWN:
                self._direction = INTF_DIRECTION.MASTER
            self._setDirectionsLikeIn(self._direction)

    def _setAsExtern(self, isExtern):
        """Set interface as extern"""
        self._isExtern = isExtern
        for prop in self._interfaces:
            prop._setAsExtern(isExtern)

    def _setDirLock(self, dirLock):
        self._dirLocked = dirLock
        for intf in self._interfaces:
            intf._setDirLock(dirLock)

    def _reverseDirection(self):
        """Reverse direction of this interface in implementation stage"""
        if self._dirLocked:
            raise IntfLvlConfErr("Can not reverse direction on interface %s because it was locked (%s)" %
                                 (repr(self), self._direction))
        self._direction = INTF_DIRECTION.opposite(self._direction)
        for intf in self._interfaces:
            intf._reverseDirection()
<|endoftext|>


<commit_before># importing modules/ libraries
import pandas as pd
import random

# create sample of order products train data
n = 1384617
s = round(0.1 * n)
skip = sorted(random.sample(range(1,n), n-s))
order_products__train_sample_df = pd.read_csv('Data/order_products__train.csv',
                                       skiprows = skip)
order_products__train_sample_df.to_csv('Data/order_products__train_sample.csv',
                                       index = False)

# create sample of order products prior data
n = 32434489
s = round(0.1 * n)
skip = sorted(random.sample(range(1,n), n-s))
order_products__prior_sample_df = pd.read_csv('Data/order_products__prior.csv',
                                       skiprows = skip)
order_products__prior_sample_df.to_csv('Data/order_products__prior_sample.csv',
                                       index = False)

# create sample of orders data
n = 3421083
s = round(0.1 * n)
skip = sorted(random.sample(range(1,n), n-s))
order_sample_df = pd.read_csv('Data/orders.csv',
                                       skiprows = skip)
order_sample_df.to_csv('Data/orders_sample.csv',
                                       index = False)<commit_msg>fix: Change create sample code to ensure matching order ids data<commit_after># importing modules/ libraries
import pandas as pd
import random
import numpy as np

# create sample of order products train data
n = 1384617
s = round(0.1 * n)
skip = sorted(random.sample(range(1,n), n-s))
order_products__train_sample_df = pd.read_csv('Data/order_products__train.csv',
                                       index = 'order_Id', skiprows = skip)
order_products__train_sample_df.to_csv('Data/order_products__train_sample.csv')

# create sample of order products prior data
n = 32434489
s = round(0.1 * n)
skip = sorted(random.sample(range(1,n), n-s))
order_products__prior_sample_df = pd.read_csv('Data/order_products__prior.csv',
                                       index = 'order_Id', skiprows = skip)
order_products__prior_sample_df.to_csv('Data/order_products__prior_sample.csv')

# create sample of orders data
prior_unique_ids = np.array(order_products__prior_sample_df.index.unique())
train_unique_ids = np.array(order_products__train_sample_df.index.unique())
match_ids = np.concatenate((prior_unique_ids, train_unique_ids), axis = 0)
order_sample_df = pd.read_csv('Data/orders.csv', index_col = 'order_id')
order_sample_df = order_sample_df.loc[match_ids,:]
order_sample_df.to_csv('Data/orders_sample.csv')<|endoftext|>


<commit_before># -------------------------------------------------------------------------------
# Copyright IBM Corp. 2016
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------
import os
import sys
import nbformat
import re
import glob
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert.preprocessors.execute import CellExecutionError
import logging
from ipykernel.kernelspec import KernelSpecManager, write_kernel_spec
from jupyter_client.manager import KernelManager
from jupyter_client.kernelspec import NoSuchKernel
import shutil
from difflib import SequenceMatcher
from optparse import OptionParser

__TEST_KERNEL_NAME__ = "PixiedustTravisTest"

if "debug" in os.environ:
    logging.basicConfig(level=logging.DEBUG)

if "compareratio" in os.environ:
    try:
        compareRatio = float(os.environ["compareratio"])
    except:
        compareRatio = 0.98
else:
    compareRatio = 0.98

def getPyJ(sparkHome):
    pyjs = glob.glob('{0}/python/lib/py4j-*.zip'.format(sparkHome))
    if not pyjs:
        return '{0}/python/lib/py4j-0.9-src.zip'.format(sparkHome)
    else:
        return pyjs[0]

def createKernelSpecIfNeeded(kernelName, useSpark):
    try:
        km = KernelManager(kernel_name=kernelName)
        km.kernel_spec
        return None
    except NoSuchKernel:
        print("Creating new Kernel {} target: {}".format(kernelName, useSpark) )
        if useSpark:
            sparkHome = os.environ["SPARK_HOME"]
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "SCALA_HOME": "{0}".format(os.environ["SCALA_HOME"]),
                    "SPARK_HOME": "{0}".format(sparkHome),
                    "PYTHONPATH": "{0}/python/:{1}".format(sparkHome, getPyJ(sparkHome)),
                    "PYTHONSTARTUP": "{0}/python/pyspark/shell.py".format(sparkHome),
                    "PYSPARK_SUBMIT_ARGS": "--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell".format(os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))),
                    "SPARK_DRIVER_MEMORY":"10G",
                    "SPARK_LOCAL_IP":"127.0.0.1"
                }
            }
        else:
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "PIXIEDUST_HOME": os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))
                }
            }

        path = write_kernel_spec(overrides=overrides)
        dest = KernelSpecManager().install_kernel_spec(path, kernel_name=kernelName, user=True)
        # cleanup afterward
        shutil.rmtree(path)
        return dest

class RestartKernelException(Exception):
    pass

class CompareOutputException(Exception):
    pass

class PixieDustTestExecutePreprocessor( ExecutePreprocessor ):
    def skipCell(self, cell):
        m = re.search("#TARGET=(.*)", cell.source, re.IGNORECASE)
        if m is not None:
            return (m.group(1).lower() == "spark") != self.useSpark or m.group(1) == "NO_RUN"
        return False

    def preprocess_cell(self, cell, resources, cell_index):
        if self.skipCell(cell):
            return cell, resources
        beforeOutputs = cell.outputs
        skipCompareOutput = "#SKIP_COMPARE_OUTPUT" in cell.source
        pixiedustDisplay = "display(" in cell.source
        try:
            logging.debug("Processing cell:\r\n{0}".format(cell.source))
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            for output in cell.outputs:
                if "text" in output and "restart kernel" in output["text"].lower():
                    raise RestartKernelException()
            if not skipCompareOutput:
                if not pixiedustDisplay:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source)
                else:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source, True)
            return cell, resources
        except Exception as e:
            if isinstance(e, CellExecutionError):
                for output in cell.outputs:
                    logging.warn("Output Error is {}".format(output))
            cell.source="%pixiedustLog -l debug"
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            logging.error("An error occurred executing the cell:\r\n{0}".format(cell.source))
            logging.info("Fetching pixiedust log...")
            if len(cell.outputs) > 0 and "text" in cell.outputs[0]:
                logging.warn(cell.outputs[0].text)
            else:
                logging.warn("Pixiedust Log is empty")
            raise

    def compareOutputs(self, beforeOutputs, afterOutputs, cellsource, useRatio=False):
        #return a measure of the sequences similarity as a float in the range [0, 1]
        seqmatcher = SequenceMatcher(None, '', '')

        #filter transient data from output
        def filterOutput(output):
            return "data" in output and "application/javascript" in output["data"]

        afterOutputs = [output for output in afterOutputs if not filterOutput(output)]

        if ( len(beforeOutputs) != len(afterOutputs)):
            logging.debug("Outputs do not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
            raise CompareOutputException("Outputs do not match for cell:\r\n{0}".format(cellsource))

        for beforeOutput, afterOutput in list(zip(beforeOutputs,afterOutputs)):
            if "output_type" in beforeOutput and beforeOutput["output_type"] != "execute_result":
                if len(beforeOutput) != len(afterOutput):
                    logging.debug("output_type does not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
                    raise CompareOutputException("output_type does not match for cell:\r\n{0}".format(cellsource))
                skip = ["execution_count"]
                expected = 0
                actual = 0
                longest = 0
                for key in beforeOutput:
                    if key not in skip:
                        before = str(beforeOutput[key])
                        after = str(afterOutput[key])
                        if useRatio:
                            seqmatcher.set_seqs(before, after)
                            expected = len(before)
                            actual = len(after)
                            ratio = seqmatcher.quick_ratio()
                            logging.info("expected_length: {0}, actual_length: {1}, sequence_ratio: {2}, compare_ratio: {3}".format(expected, actual, ratio, compareRatio))
                            if ratio < compareRatio:
                                logging.debug("output_type ({0}) below ratio ({1}) threshold \r\nExpected:\r\n {2} \r\n\r\nActual:\r\n {3}".format(key, ratio, before, after))
                                raise CompareOutputException("output_type ({0}) below ratio ({1}) threshold for cell:\r\n{2}".format(key, ratio, cellsource))
                        elif beforeOutput[key] != afterOutput[key]:
                            logging.debug("output_type ({0}) does not match \r\nExpected:\r\n {1} \r\n\r\nActual:\r\n {2}".format(key, before, after))
                            raise CompareOutputException("output_type ({0}) does not match for cell:\r\n{1}".format(key, cellsource))

def runNotebook(path, useSpark):
    ep = PixieDustTestExecutePreprocessor(timeout=3600, kernel_name= __TEST_KERNEL_NAME__)
    ep.useSpark = useSpark
    nb=nbformat.read(path, as_version=4)
    #set the kernel name to test
    nb.metadata.kernelspec.name=__TEST_KERNEL_NAME__
    if "pixiedust_test" in nb.metadata:
        testMeta = nb.metadata["pixiedust_test"]
        targetKernel = testMeta.get("target")
        if targetKernel is not None and (targetKernel.lower() == "spark") != useSpark:
            logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
            return
        skipVersions = testMeta.get("skipPython",[])
        for skipVersion in skipVersions:
            skipVersion = int(float(skipVersion))
            if skipVersion == sys.version_info.major:
                logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
                return
    try:
        ep.preprocess(nb, {'metadata':{'path': os.path.dirname(path)}})
    finally:
        dir = os.environ.get("PIXIEDUST_TEST_OUTPUT", os.path.expanduser('~') + "/pixiedust") + "/tests"
        if not os.path.exists(dir):
            os.makedirs( dir )
        nbformat.write(nb, dir + "/" + os.path.basename(path) + ".out")

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option("-t", "--target", dest="target", default="spark", help="target kernel: spark or plain")

    (options, args) = parser.parse_args()
    useSpark = options.target == "spark"
    print("Starting Test Suite with target: {}".format(options.target) )
    if "PIXIEDUST_HOME" in os.environ:
        print("Using PIXIEDUST_HOME: ", os.environ["PIXIEDUST_HOME"])
    kernelPath = createKernelSpecIfNeeded(__TEST_KERNEL_NAME__, useSpark)
    try:
        inputDir = os.environ.get("PIXIEDUST_TEST_INPUT", './tests')
        for path in os.listdir( inputDir ):
            if path.endswith(".ipynb"):
                print("Processing notebook {0}".format(path))
                processed = False
                count = 0
                while (not processed and count < 10 ):
                    try:
                        processed = True
                        count += 1
                        runNotebook(inputDir + "/" + path, useSpark)
                    except RestartKernelException:
                        logging.warn("Restarting kernel...")
                        processed = False
                    except:
                        logging.warn("Fatal Error in Notebook {}.".format(path))
                        raise
                print("Finished processing notebook {0}".format(path))
    finally:
        if kernelPath:
            shutil.rmtree(kernelPath)

        #clean up PIXIEDUST_HOME if provided
        def rmTree(path):
            if os.path.exists(path):
                shutil.rmtree(path)
        def rmFile(path):
            if os.path.exists(path):
                os.remove(path)
        if "PIXIEDUST_HOME" in os.environ:
            rmTree(os.environ["PIXIEDUST_HOME"] + "/data/libs")
            rmTree(os.environ["PIXIEDUST_HOME"] + "/pixiedust")
            rmFile(os.environ["PIXIEDUST_HOME"] + "/pixiedust.db")<commit_msg>list test notebooks<commit_after># -------------------------------------------------------------------------------
# Copyright IBM Corp. 2016
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------
import os
import sys
import nbformat
import re
import glob
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert.preprocessors.execute import CellExecutionError
import logging
from ipykernel.kernelspec import KernelSpecManager, write_kernel_spec
from jupyter_client.manager import KernelManager
from jupyter_client.kernelspec import NoSuchKernel
import shutil
from difflib import SequenceMatcher
from optparse import OptionParser

__TEST_KERNEL_NAME__ = "PixiedustTravisTest"

if "debug" in os.environ:
    logging.basicConfig(level=logging.DEBUG)

if "compareratio" in os.environ:
    try:
        compareRatio = float(os.environ["compareratio"])
    except:
        compareRatio = 0.98
else:
    compareRatio = 0.98

def getPyJ(sparkHome):
    pyjs = glob.glob('{0}/python/lib/py4j-*.zip'.format(sparkHome))
    if not pyjs:
        return '{0}/python/lib/py4j-0.9-src.zip'.format(sparkHome)
    else:
        return pyjs[0]

def createKernelSpecIfNeeded(kernelName, useSpark):
    try:
        km = KernelManager(kernel_name=kernelName)
        km.kernel_spec
        return None
    except NoSuchKernel:
        print("Creating new Kernel {} target: {}".format(kernelName, useSpark) )
        if useSpark:
            sparkHome = os.environ["SPARK_HOME"]
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "SCALA_HOME": "{0}".format(os.environ["SCALA_HOME"]),
                    "SPARK_HOME": "{0}".format(sparkHome),
                    "PYTHONPATH": "{0}/python/:{1}".format(sparkHome, getPyJ(sparkHome)),
                    "PYTHONSTARTUP": "{0}/python/pyspark/shell.py".format(sparkHome),
                    "PYSPARK_SUBMIT_ARGS": "--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell".format(os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))),
                    "SPARK_DRIVER_MEMORY":"10G",
                    "SPARK_LOCAL_IP":"127.0.0.1"
                }
            }
        else:
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "PIXIEDUST_HOME": os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))
                }
            }

        path = write_kernel_spec(overrides=overrides)
        dest = KernelSpecManager().install_kernel_spec(path, kernel_name=kernelName, user=True)
        # cleanup afterward
        shutil.rmtree(path)
        return dest

class RestartKernelException(Exception):
    pass

class CompareOutputException(Exception):
    pass

class PixieDustTestExecutePreprocessor( ExecutePreprocessor ):
    def skipCell(self, cell):
        m = re.search("#TARGET=(.*)", cell.source, re.IGNORECASE)
        if m is not None:
            return (m.group(1).lower() == "spark") != self.useSpark or m.group(1) == "NO_RUN"
        return False

    def preprocess_cell(self, cell, resources, cell_index):
        if self.skipCell(cell):
            return cell, resources
        beforeOutputs = cell.outputs
        skipCompareOutput = "#SKIP_COMPARE_OUTPUT" in cell.source
        pixiedustDisplay = "display(" in cell.source
        try:
            logging.debug("Processing cell:\r\n{0}".format(cell.source))
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            for output in cell.outputs:
                if "text" in output and "restart kernel" in output["text"].lower():
                    raise RestartKernelException()
            if not skipCompareOutput:
                if not pixiedustDisplay:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source)
                else:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source, True)
            return cell, resources
        except Exception as e:
            if isinstance(e, CellExecutionError):
                for output in cell.outputs:
                    logging.warn("Output Error is {}".format(output))
            cell.source="%pixiedustLog -l debug"
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            logging.error("An error occurred executing the cell:\r\n{0}".format(cell.source))
            logging.info("Fetching pixiedust log...")
            if len(cell.outputs) > 0 and "text" in cell.outputs[0]:
                logging.warn(cell.outputs[0].text)
            else:
                logging.warn("Pixiedust Log is empty")
            raise

    def compareOutputs(self, beforeOutputs, afterOutputs, cellsource, useRatio=False):
        #return a measure of the sequences similarity as a float in the range [0, 1]
        seqmatcher = SequenceMatcher(None, '', '')

        #filter transient data from output
        def filterOutput(output):
            return "data" in output and "application/javascript" in output["data"]

        afterOutputs = [output for output in afterOutputs if not filterOutput(output)]

        if ( len(beforeOutputs) != len(afterOutputs)):
            logging.debug("Outputs do not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
            raise CompareOutputException("Outputs do not match for cell:\r\n{0}".format(cellsource))

        for beforeOutput, afterOutput in list(zip(beforeOutputs,afterOutputs)):
            if "output_type" in beforeOutput and beforeOutput["output_type"] != "execute_result":
                if len(beforeOutput) != len(afterOutput):
                    logging.debug("output_type does not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
                    raise CompareOutputException("output_type does not match for cell:\r\n{0}".format(cellsource))
                skip = ["execution_count"]
                expected = 0
                actual = 0
                longest = 0
                for key in beforeOutput:
                    if key not in skip:
                        before = str(beforeOutput[key])
                        after = str(afterOutput[key])
                        if useRatio:
                            seqmatcher.set_seqs(before, after)
                            expected = len(before)
                            actual = len(after)
                            ratio = seqmatcher.quick_ratio()
                            logging.info("expected_length: {0}, actual_length: {1}, sequence_ratio: {2}, compare_ratio: {3}".format(expected, actual, ratio, compareRatio))
                            if ratio < compareRatio:
                                logging.debug("output_type ({0}) below ratio ({1}) threshold \r\nExpected:\r\n {2} \r\n\r\nActual:\r\n {3}".format(key, ratio, before, after))
                                raise CompareOutputException("output_type ({0}) below ratio ({1}) threshold for cell:\r\n{2}".format(key, ratio, cellsource))
                        elif beforeOutput[key] != afterOutput[key]:
                            logging.debug("output_type ({0}) does not match \r\nExpected:\r\n {1} \r\n\r\nActual:\r\n {2}".format(key, before, after))
                            raise CompareOutputException("output_type ({0}) does not match for cell:\r\n{1}".format(key, cellsource))

def runNotebook(path, useSpark):
    ep = PixieDustTestExecutePreprocessor(timeout=3600, kernel_name= __TEST_KERNEL_NAME__)
    ep.useSpark = useSpark
    nb=nbformat.read(path, as_version=4)
    #set the kernel name to test
    nb.metadata.kernelspec.name=__TEST_KERNEL_NAME__
    if "pixiedust_test" in nb.metadata:
        testMeta = nb.metadata["pixiedust_test"]
        targetKernel = testMeta.get("target")
        if targetKernel is not None and (targetKernel.lower() == "spark") != useSpark:
            logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
            return
        skipVersions = testMeta.get("skipPython",[])
        for skipVersion in skipVersions:
            skipVersion = int(float(skipVersion))
            if skipVersion == sys.version_info.major:
                logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
                return
    try:
        ep.preprocess(nb, {'metadata':{'path': os.path.dirname(path)}})
    finally:
        dir = os.environ.get("PIXIEDUST_TEST_OUTPUT", os.path.expanduser('~') + "/pixiedust") + "/tests"
        if not os.path.exists(dir):
            os.makedirs( dir )
        nbformat.write(nb, dir + "/" + os.path.basename(path) + ".out")

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option("-t", "--target", dest="target", default="spark", help="target kernel: spark or plain")

    (options, args) = parser.parse_args()
    useSpark = options.target == "spark"
    print("Starting Test Suite with target: {}".format(options.target) )
    if "PIXIEDUST_HOME" in os.environ:
        print("Using PIXIEDUST_HOME: ", os.environ["PIXIEDUST_HOME"])
    kernelPath = createKernelSpecIfNeeded(__TEST_KERNEL_NAME__, useSpark)
    try:
        inputDir = os.environ.get("PIXIEDUST_TEST_INPUT", './tests')
        print("Processing notebooks from directory {0}:".format(inputDir))
        nb_paths = sorted(os.listdir( inputDir ))
        print(nb_paths)
        for path in nb_paths:
            if path.endswith(".ipynb"):
                print("Processing notebook {0}".format(path))
                processed = False
                count = 0
                while (not processed and count < 10 ):
                    try:
                        processed = True
                        count += 1
                        runNotebook(inputDir + "/" + path, useSpark)
                    except RestartKernelException:
                        logging.warn("Restarting kernel...")
                        processed = False
                    except:
                        logging.warn("Fatal Error in Notebook {}.".format(path))
                        raise
                print("Finished processing notebook {0}".format(path))
    finally:
        if kernelPath:
            shutil.rmtree(kernelPath)

        #clean up PIXIEDUST_HOME if provided
        def rmTree(path):
            if os.path.exists(path):
                shutil.rmtree(path)
        def rmFile(path):
            if os.path.exists(path):
                os.remove(path)
        if "PIXIEDUST_HOME" in os.environ:
            rmTree(os.environ["PIXIEDUST_HOME"] + "/data/libs")
            rmTree(os.environ["PIXIEDUST_HOME"] + "/pixiedust")
            rmFile(os.environ["PIXIEDUST_HOME"] + "/pixiedust.db")<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import unicode_literals

import errno
from contextlib import contextmanager
import sys

from . import errors

from six import reraise

class _ConvertOSErrors(object):
    """Context manager to convert OSErrors in to FS Errors."""

    FILE_ERRORS = {
        64: errors.RemoteConnectionError,  # ENONET
        errno.ENOENT: errors.ResourceNotFound,
        errno.EFAULT: errors.ResourceNotFound,
        errno.ESRCH: errors.ResourceNotFound,
        errno.ENOTEMPTY: errors.DirectoryNotEmpty,
        errno.EEXIST: errors.FileExists,
        183: errors.DirectoryExists,
        #errno.ENOTDIR: errors.DirectoryExpected,
        errno.ENOTDIR: errors.ResourceNotFound,
        errno.EISDIR: errors.FileExpected,
        errno.EINVAL: errors.DirectoryExpected,
        errno.ENOSPC: errors.InsufficientStorage,
        errno.EPERM: errors.PermissionDenied,
        errno.ENETDOWN: errors.RemoteConnectionError,
        errno.ECONNRESET: errors.RemoteConnectionError,
        errno.ENAMETOOLONG: errors.PathError,
        errno.EOPNOTSUPP: errors.Unsupported,
        errno.ENOSYS: errors.Unsupported
    }

    DIR_ERRORS = FILE_ERRORS.copy()
    DIR_ERRORS[errno.ENOTDIR] = errors.DirectoryExpected
    DIR_ERRORS[errno.EEXIST] = errors.DirectoryExists
    DIR_ERRORS[errno.EINVAL] = errors.FileExpected

    def __init__(self, opname, path, directory=False):
        self._opname = opname
        self._path = path
        self._directory = directory

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        os_errors = (
            self.DIR_ERRORS
            if self._directory
            else self.FILE_ERRORS
        )
        if exc_type and isinstance(exc_value, EnvironmentError):
            _errno = exc_value.errno
            fserror = os_errors.get(_errno, errors.OperationFailed)
            if _errno == errno.EACCES and sys.platform == "win32":
                if getattr(exc_value, 'args', None) == 32:  # pragma: no cover
                    fserror = errors.ResourceLocked
            reraise(
                fserror,
                fserror(
                    self._path,
                    exc=exc_value
                ),
                traceback
            )

# Stops linter complaining about invalid class name
convert_os_errors = _ConvertOSErrors

@contextmanager
def unwrap_errors(path_replace):
    """
    A context manager to re-write the paths in resource exceptions to be
    in the same context as the wrapped filesystem.

    The only parameter may be the path from the parent, if only one path
    is to be unwrapped. Or it may be a dictionary that maps wrapped
    paths on to unwrapped paths.

    """
    try:
        yield
    except errors.ResourceError as e:
        if hasattr(e, 'path'):
            if isinstance(path_replace, dict):
                e.path = path_replace.get(e.path, e.path)
            else:
                e.path = path_replace
        reraise(type(e), e)
<commit_msg>error message fix<commit_after>from __future__ import print_function
from __future__ import unicode_literals

import errno
from contextlib import contextmanager
import sys

from . import errors

from six import reraise

class _ConvertOSErrors(object):
    """Context manager to convert OSErrors in to FS Errors."""

    FILE_ERRORS = {
        64: errors.RemoteConnectionError,  # ENONET
        errno.ENOENT: errors.ResourceNotFound,
        errno.EFAULT: errors.ResourceNotFound,
        errno.ESRCH: errors.ResourceNotFound,
        errno.ENOTEMPTY: errors.DirectoryNotEmpty,
        errno.EEXIST: errors.FileExists,
        183: errors.DirectoryExists,
        #errno.ENOTDIR: errors.DirectoryExpected,
        errno.ENOTDIR: errors.ResourceNotFound,
        errno.EISDIR: errors.FileExpected,
        errno.EINVAL: errors.FileExpected,
        errno.ENOSPC: errors.InsufficientStorage,
        errno.EPERM: errors.PermissionDenied,
        errno.ENETDOWN: errors.RemoteConnectionError,
        errno.ECONNRESET: errors.RemoteConnectionError,
        errno.ENAMETOOLONG: errors.PathError,
        errno.EOPNOTSUPP: errors.Unsupported,
        errno.ENOSYS: errors.Unsupported
    }

    DIR_ERRORS = FILE_ERRORS.copy()
    DIR_ERRORS[errno.ENOTDIR] = errors.DirectoryExpected
    DIR_ERRORS[errno.EEXIST] = errors.DirectoryExists
    DIR_ERRORS[errno.EINVAL] = errors.DirectoryExpected

    def __init__(self, opname, path, directory=False):
        self._opname = opname
        self._path = path
        self._directory = directory

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        os_errors = (
            self.DIR_ERRORS
            if self._directory
            else self.FILE_ERRORS
        )
        if exc_type and isinstance(exc_value, EnvironmentError):
            _errno = exc_value.errno
            fserror = os_errors.get(_errno, errors.OperationFailed)
            if _errno == errno.EACCES and sys.platform == "win32":
                if getattr(exc_value, 'args', None) == 32:  # pragma: no cover
                    fserror = errors.ResourceLocked
            reraise(
                fserror,
                fserror(
                    self._path,
                    exc=exc_value
                ),
                traceback
            )

# Stops linter complaining about invalid class name
convert_os_errors = _ConvertOSErrors

@contextmanager
def unwrap_errors(path_replace):
    """
    A context manager to re-write the paths in resource exceptions to be
    in the same context as the wrapped filesystem.

    The only parameter may be the path from the parent, if only one path
    is to be unwrapped. Or it may be a dictionary that maps wrapped
    paths on to unwrapped paths.

    """
    try:
        yield
    except errors.ResourceError as e:
        if hasattr(e, 'path'):
            if isinstance(path_replace, dict):
                e.path = path_replace.get(e.path, e.path)
            else:
                e.path = path_replace
        reraise(type(e), e)
<|endoftext|>