<commit_before>"""
Digest objects for the messages app
"""

from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string

from actstream.models import Action, user_stream
from datetime import datetime, timedelta
import logging

class Digest(EmailMultiAlternatives):
    """
    A digest describes a collection of activity over a duration, which
    is then rendered into an email and delivered at a scheduled interval.
    """
    text_template = None
    html_template = None
    interval = None
    activity = {
        'count': 0,
        'requests': None,
        'following': None
    }

    def __init__(self, user, **kwargs):
        """Initialize the notification"""
        super(Digest, self).__init__(**kwargs)
        if isinstance(user, User):
            self.user = user
            self.to = [user.email]
        else:
            raise TypeError('Digest requires a User to recieve it')
        self.activity = self.get_activity()
        context = self.get_context_data()
        text_email = render_to_string(self.get_text_template(), context)
        html_email = render_to_string(self.get_html_template(), context)
        self.from_email = 'MuckRock <info@muckrock.com>'
        self.bcc = ['diagnostics@muckrock.com']
        self.subject = self.get_subject()
        self.body = text_email
        self.attach_alternative(html_email, 'text/html')

    def get_activity(self):
        """Returns a list of activities to be sent in the email"""
        duration = self.get_duration()
        f_stream = self.get_foia_activity(duration)
        u_stream = user_stream(self.user).filter(timestamp__gte=duration)\
                                         .exclude(verb__icontains='following')
        self.activity['count'] = f_stream.count() + u_stream.count()
        self.activity['requests'] = f_stream
        self.activity['following'] = u_stream
        return self.activity

    def get_duration(self):
        if not self.interval:
            raise NotImplementedError('No interval specified.')
        if not isinstance(self.interval, timedelta):
            raise TypeError('Interval attribute must be a datetime.timedelta object.')
        return datetime.now() - self.interval

    def get_foia_activity(self, period):
        """Returns activity on requests owned by the user."""
        foia_stream = Action.objects.requests_for_user(self.user)
        foia_stream = foia_stream.filter(timestamp__gte=period)
        user_ct = ContentType.objects.get_for_model(self.user)
        # exclude actions where the user is the Actor
        # since they know which actions they've taken themselves
        foia_stream.exclude(actor_content_type=user_ct, actor_object_id=self.user.id)
        return foia_stream

    def get_context_data(self):
        """Adds classified activity to the context"""
        classified_foia_activity = self.classify_foia_activity()
        context = {
            'user': self.user,
            'activity': self.activity,
            'foia_activity': classified_foia_activity,
            'base_url': 'https://www.muckrock.com'
        }
        return context

    def classify_foia_activity(self):
        """Segment and classify the activity"""
        foia_activity = self.activity['requests']
        return {
            'granted': foia_activity.filter(verb__icontains='completed'),
            'denied': foia_activity.filter(verb__icontains='rejected'),
            'unsuccessful': foia_activity.filter(verb__icontains='no responsive documents'),
            'needs_action': foia_activity.filter(verb__icontains='payment', verb__icontains='fix'),
            'unembargo': foia_activity.filter(verb__icontains='unembargo'),
            'response': foia_activity.filter(verb__icontains='responded'),
            'auto_follow_up': foia_activity.filter(verb__icontains='automatically followed up'),
        }

    def get_text_template(self):
        """Returns the text template"""
        if not self.text_template:
            raise NotImplementedError('No text template specified.')
        return self.text_template

    def get_html_template(self):
        """Returns the html template"""
        if not self.html_template:
            raise NotImplementedError('No HTML template specified.')
        return self.html_template

    def get_subject(self):
        """Summarizes the activities in the notification"""
        subject = str(self.activity['count']) + ' Update'
        if self.activity['count'] > 1:
            subject += 's'
        return subject

    def send(self, *args):
        """Don't send the email if there's no activity."""
        if self.activity['count'] < 1:
            return 0
        return super(Digest, self).send(*args)

class HourlyDigest(Digest):
    """An hourly email digest"""
    text_template = 'message/digest/hourly.txt'
    html_template = 'message/digest/hourly.html'
    interval = timedelta(hours=1)

class DailyDigest(Digest):
    """A daily email digest"""
    text_template = 'message/notification/daily.txt'
    html_template = 'message/notification/daily.html'
    interval = timedelta(days=1)

class WeeklyDigest(Digest):
    """A weekly email digest"""
    text_template = 'message/digest/weekly.txt'
    html_template = 'message/digest/weekly.html'
    interval = timedelta(days=7)
<commit_msg>fixes bug in digest code<commit_after>"""
Digest objects for the messages app
"""

from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string

from actstream.models import Action, user_stream
from datetime import datetime, timedelta
import logging

class Digest(EmailMultiAlternatives):
    """
    A digest describes a collection of activity over a duration, which
    is then rendered into an email and delivered at a scheduled interval.
    """
    text_template = None
    html_template = None
    interval = None
    activity = {
        'count': 0,
        'requests': None,
        'following': None
    }

    def __init__(self, user, **kwargs):
        """Initialize the notification"""
        super(Digest, self).__init__(**kwargs)
        if isinstance(user, User):
            self.user = user
            self.to = [user.email]
        else:
            raise TypeError('Digest requires a User to recieve it')
        self.activity = self.get_activity()
        context = self.get_context_data()
        text_email = render_to_string(self.get_text_template(), context)
        html_email = render_to_string(self.get_html_template(), context)
        self.from_email = 'MuckRock <info@muckrock.com>'
        self.bcc = ['diagnostics@muckrock.com']
        self.subject = self.get_subject()
        self.body = text_email
        self.attach_alternative(html_email, 'text/html')

    def get_activity(self):
        """Returns a list of activities to be sent in the email"""
        duration = self.get_duration()
        f_stream = self.get_foia_activity(duration)
        u_stream = user_stream(self.user).filter(timestamp__gte=duration)\
                                         .exclude(verb__icontains='following')
        self.activity['count'] = f_stream.count() + u_stream.count()
        self.activity['requests'] = f_stream
        self.activity['following'] = u_stream
        return self.activity

    def get_duration(self):
        if not self.interval:
            raise NotImplementedError('No interval specified.')
        if not isinstance(self.interval, timedelta):
            raise TypeError('Interval attribute must be a datetime.timedelta object.')
        return datetime.now() - self.interval

    def get_foia_activity(self, period):
        """Returns activity on requests owned by the user."""
        foia_stream = Action.objects.requests_for_user(self.user)
        foia_stream = foia_stream.filter(timestamp__gte=period)
        # exclude actions where the user is the Actor
        # since they know which actions they've taken themselves
        user_ct = ContentType.objects.get_for_model(self.user)
        foia_stream.exclude(actor_content_type=user_ct, actor_object_id=self.user.id)
        return foia_stream

    def get_context_data(self):
        """Adds classified activity to the context"""
        classified_foia_activity = self.classify_foia_activity()
        context = {
            'user': self.user,
            'activity': self.activity,
            'foia_activity': classified_foia_activity,
            'base_url': 'https://www.muckrock.com'
        }
        return context

    def classify_foia_activity(self):
        """Segment and classify the activity"""
        foia_activity = self.activity['requests']
        return {
            'granted': foia_activity.filter(verb__icontains='completed'),
            'denied': foia_activity.filter(verb__icontains='rejected'),
            'unsuccessful': foia_activity.filter(verb__icontains='no responsive documents'),
            'payment': foia_activity.filter(verb__icontains='payment'),
            'fix': foia_activity.filter(verb__icontains='fix'),
            'unembargo': foia_activity.filter(verb__icontains='unembargo'),
            'response': foia_activity.filter(verb__icontains='responded'),
            'auto_follow_up': foia_activity.filter(verb__icontains='automatically followed up'),
        }

    def get_text_template(self):
        """Returns the text template"""
        if not self.text_template:
            raise NotImplementedError('No text template specified.')
        return self.text_template

    def get_html_template(self):
        """Returns the html template"""
        if not self.html_template:
            raise NotImplementedError('No HTML template specified.')
        return self.html_template

    def get_subject(self):
        """Summarizes the activities in the notification"""
        subject = str(self.activity['count']) + ' Update'
        if self.activity['count'] > 1:
            subject += 's'
        return subject

    def send(self, *args):
        """Don't send the email if there's no activity."""
        if self.activity['count'] < 1:
            return 0
        return super(Digest, self).send(*args)

class HourlyDigest(Digest):
    """An hourly email digest"""
    text_template = 'message/digest/hourly.txt'
    html_template = 'message/digest/hourly.html'
    interval = timedelta(hours=1)

class DailyDigest(Digest):
    """A daily email digest"""
    text_template = 'message/notification/daily.txt'
    html_template = 'message/notification/daily.html'
    interval = timedelta(days=1)

class WeeklyDigest(Digest):
    """A weekly email digest"""
    text_template = 'message/digest/weekly.txt'
    html_template = 'message/digest/weekly.html'
    interval = timedelta(days=7)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2017 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from calvin.runtime.south.calvinlib import base_calvinlib_object
import base64

class Base64(base_calvinlib_object.BaseCalvinlibObject):
    """
    Functions for base64 encoding/decoding data.
    """

    encode_schema = {
        "description": "base64 encode binary string",
        "type": "string"

    }

    decode_schema = {
        "description": "decode base64 encoded string",
        "type": "string"
    }

    def init(self):
        pass

    def encode(self, string):
        return base64.b64encode(string)

    def decode(self, b64string):
        return base64.b64decode(b64string)

<commit_msg>calvinlib: add init_schema to base64 implementation<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2017 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from calvin.runtime.south.calvinlib import base_calvinlib_object
import base64

class Base64(base_calvinlib_object.BaseCalvinlibObject):
    """
    Functions for base64 encoding/decoding data.
    """

    init_schema = {
            "description": "base64 encoding/decoding of data"
    }

    encode_schema = {
        "description": "base64 encode binary string",
        "type": "string"

    }

    decode_schema = {
        "description": "decode base64 encoded string",
        "type": "string"
    }

    def init(self):
        pass

    def encode(self, string):
        return base64.b64encode(string)

    def decode(self, b64string):
        return base64.b64decode(b64string)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2017 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from calvin.runtime.south.calvinsys import base_calvinsys_object

class StandardOut(base_calvinsys_object.BaseCalvinsysObject):
    """
    StandardOut
    """

    init_schema = {
        "type": "object",
        "properties": {
            "prefix": {
                "description": "String to prefix all data",
                "type": "string"
            }
        },
        "description": "Send all incoming data to terminal (usually not very useful)"
    }

    can_write_schema = {
        "description": "Always true",
        "type": "boolean"
    }

    write_schema = {
        "description": "Write data to standard out",
        "type": ["boolean", "integer", "number", "string", "array", "object", "null"]
    }

    def init(self, prefix=None, **kwargs):
        self._prefix = prefix

    def can_write(self):
        return True

    def write(self, data=None):
        msg = ""
        if data is not None and self._prefix is not None:
            msg = u"{}: {}".format(self._prefix, data)
        elif data is not None:
            msg = u"{}".format(data)
        elif self._prefix is not None:
            msg = u"{}".format(self._prefix)
        print(msg)

    def close(self):
        pass
<commit_msg>StandardOut: Once more with the utf-8.<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2017 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from calvin.runtime.south.calvinsys import base_calvinsys_object

class StandardOut(base_calvinsys_object.BaseCalvinsysObject):
    """
    StandardOut
    """

    init_schema = {
        "type": "object",
        "properties": {
            "prefix": {
                "description": "String to prefix all data",
                "type": "string"
            }
        },
        "description": "Send all incoming data to terminal (usually not very useful)"
    }

    can_write_schema = {
        "description": "Always true",
        "type": "boolean"
    }

    write_schema = {
        "description": "Write data to standard out",
        "type": ["boolean", "integer", "number", "string", "array", "object", "null"]
    }

    def init(self, prefix=None, **kwargs):
        self._prefix = prefix

    def can_write(self):
        return True

    def write(self, data=None):
        msg = ""
        if data is not None and self._prefix is not None:
            msg = u"{}: {}".format(self._prefix, data)
        elif data is not None:
            msg = u"{}".format(data)
        elif self._prefix is not None:
            msg = u"{}".format(self._prefix)
        print(msg.encode("utf-8"))

    def close(self):
        pass
<|endoftext|>


<commit_before>from SimPEG import Utils, Survey, np
from SimPEG.Survey import BaseSurvey
from SimPEG.EM.Utils import *
from BaseTDEM import FieldsTDEM

class RxTDEM(Survey.BaseTimeRx):

    knownRxTypes = {
                    'ex':['e', 'Ex', 'N'],
                    'ey':['e', 'Ey', 'N'],
                    'ez':['e', 'Ez', 'N'],

                    'bx':['b', 'Fx', 'N'],
                    'by':['b', 'Fy', 'N'],
                    'bz':['b', 'Fz', 'N'],

                    'dbxdt':['b', 'Fx', 'CC'],
                    'dbydt':['b', 'Fy', 'CC'],
                    'dbzdt':['b', 'Fz', 'CC'],
                   }

    def __init__(self, locs, times, rxType):
        Survey.BaseTimeRx.__init__(self, locs, times, rxType)

    @property
    def projField(self):
        """Field Type projection (e.g. e b ...)"""
        return self.knownRxTypes[self.rxType][0]

    @property
    def projGLoc(self):
        """Grid Location projection (e.g. Ex Fy ...)"""
        return self.knownRxTypes[self.rxType][1]

    @property
    def projTLoc(self):
        """Time Location projection (e.g. CC N)"""
        return self.knownRxTypes[self.rxType][2]

    def getTimeP(self, timeMesh):
        """
            Returns the time projection matrix.

            .. note::

                This is not stored in memory, but is created on demand.
        """
        if self.rxType in ['dbxdt','dbydt','dbzdt']:
            return timeMesh.getInterpolationMat(self.times, self.projTLoc)*timeMesh.faceDiv
        else:
            return timeMesh.getInterpolationMat(self.times, self.projTLoc)

    def projectFields(self, src, mesh, timeMesh, u):
        P = self.getP(mesh, timeMesh)
        u_part = Utils.mkvc(u[src, self.projField, :])
        return P*u_part

    def projectFieldsDeriv(self, src, mesh, timeMesh, u, v, adjoint=False):
        P = self.getP(mesh, timeMesh)

        if not adjoint:
            return P * Utils.mkvc(v[src, self.projField, :])
        elif adjoint:
            return P.T * v[src, self]

class SrcTDEM(Survey.BaseSrc):
    rxPair = RxTDEM
    radius = None

    def getInitialFields(self, mesh):
        F0 = getattr(self, '_getInitialFields_' + self.srcType)(mesh)
        return F0

    def getJs(self, mesh, time):
        return None

class SrcTDEM_VMD_MVP(SrcTDEM):

    def __init__(self,rxList,loc):
        self.loc = loc
        SrcTDEM.__init__(self,rxList)

    def getInitialFields(self, mesh):
        """Vertical magnetic dipole, magnetic vector potential"""
        if mesh._meshType is 'CYL':
            if mesh.isSymmetric:
                MVP = MagneticDipoleVectorPotential(self.loc, mesh, 'Ey')
            else:
                raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
        elif mesh._meshType is 'TENSOR':
            MVP = MagneticDipoleVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'])
        else:
            raise Exception('Unknown mesh for VMD')

        return {"b": mesh.edgeCurl*MVP}

class SrcTDEM_CircularLoop_MVP(SrcTDEM):
    def __init__(self,rxList,loc,radius,waveformType):
        self.loc = loc
        self.radius = radius
        self.waveformType = waveformType
        SrcTDEM.__init__(self,rxList)

    def getInitialFields(self, mesh):
        """Circular Loop, magnetic vector potential"""
        if self.waveformType == "STEPOFF":
            print ">> Step waveform: Non-zero initial condition"
            if mesh._meshType is 'CYL':
                if mesh.isSymmetric:
                    MVP = MagneticLoopVectorPotential(self.loc, mesh, 'Ey', self.radius)
                else:
                    raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
            elif mesh._meshType is 'TENSOR':
                MVP = MagneticLoopVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'], self.radius)
            else:
                raise Exception('Unknown mesh for CircularLoop')
            return {"b": mesh.edgeCurl*MVP}
        elif self.waveformType == "GENERAL":
            print ">> General waveform: Zero initial condition"
            return {"b": np.zeros(mesh.nF)}
        else:
            raise NotImplementedError("Only use STEPOFF or GENERAL")

    def getMeS(self, mesh, MfMui):
        if mesh._meshType is 'CYL':
            if mesh.isSymmetric:
                MVP = MagneticLoopVectorPotential(self.loc, mesh, 'Ey', self.radius)
            else:
                raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
        elif mesh._meshType is 'TENSOR':
            MVP = MagneticLoopVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'], self.radius)
        else:
            raise Exception('Unknown mesh for CircularLoop')
        return mesh.edgeCurl.T*MfMui*mesh.edgeCurl*MVP

class SurveyTDEM(Survey.BaseSurvey):
    """
        docstring for SurveyTDEM
    """
    srcPair = SrcTDEM

    def __init__(self, srcList, **kwargs):
        # Sort these by frequency
        self.srcList = srcList
        Survey.BaseSurvey.__init__(self, **kwargs)

    def projectFields(self, u):
        data = Survey.Data(self)
        for src in self.srcList:
            for rx in src.rxList:
                data[src, rx] = rx.projectFields(src, self.mesh, self.prob.timeMesh, u)
        return data

    def projectFieldsDeriv(self, u, v=None, adjoint=False):
        assert v is not None, 'v to multiply must be provided.'

        if not adjoint:
            data = Survey.Data(self)
            for src in self.srcList:
                for rx in src.rxList:
                    data[src, rx] = rx.projectFieldsDeriv(src, self.mesh, self.prob.timeMesh, u, v)
            return data
        else:
            f = FieldsTDEM(self.mesh, self)
            for src in self.srcList:
                for rx in src.rxList:
                    Ptv = rx.projectFieldsDeriv(src, self.mesh, self.prob.timeMesh, u, v, adjoint=True)
                    Ptv = Ptv.reshape((-1, self.prob.timeMesh.nN), order='F')
                    if rx.projField not in f: # first time we are projecting
                        f[src, rx.projField, :] = Ptv
                    else: # there are already fields, so let's add to them!
                        f[src, rx.projField, :] += Ptv
            return f

<commit_msg>Checkout a updated file from Seogi.<commit_after>from SimPEG import Utils, Survey, np
from SimPEG.Survey import BaseSurvey
from SimPEG.EM.Utils import *
from BaseTDEM import FieldsTDEM

class RxTDEM(Survey.BaseTimeRx):

    knownRxTypes = {
                    'ex':['e', 'Ex', 'N'],
                    'ey':['e', 'Ey', 'N'],
                    'ez':['e', 'Ez', 'N'],

                    'bx':['b', 'Fx', 'N'],
                    'by':['b', 'Fy', 'N'],
                    'bz':['b', 'Fz', 'N'],

                    'dbxdt':['b', 'Fx', 'CC'],
                    'dbydt':['b', 'Fy', 'CC'],
                    'dbzdt':['b', 'Fz', 'CC'],
                   }

    def __init__(self, locs, times, rxType):
        Survey.BaseTimeRx.__init__(self, locs, times, rxType)

    @property
    def projField(self):
        """Field Type projection (e.g. e b ...)"""
        return self.knownRxTypes[self.rxType][0]

    @property
    def projGLoc(self):
        """Grid Location projection (e.g. Ex Fy ...)"""
        return self.knownRxTypes[self.rxType][1]

    @property
    def projTLoc(self):
        """Time Location projection (e.g. CC N)"""
        return self.knownRxTypes[self.rxType][2]

    def getTimeP(self, timeMesh):
        """
            Returns the time projection matrix.

            .. note::

                This is not stored in memory, but is created on demand.
        """
        if self.rxType in ['dbxdt','dbydt','dbzdt']:
            return timeMesh.getInterpolationMat(self.times, self.projTLoc)*timeMesh.faceDiv
        else:
            return timeMesh.getInterpolationMat(self.times, self.projTLoc)

    def projectFields(self, src, mesh, timeMesh, u):
        P = self.getP(mesh, timeMesh)
        u_part = Utils.mkvc(u[src, self.projField, :])
        return P*u_part

    def projectFieldsDeriv(self, src, mesh, timeMesh, u, v, adjoint=False):
        P = self.getP(mesh, timeMesh)

        if not adjoint:
            return P * Utils.mkvc(v[src, self.projField, :])
        elif adjoint:
            return P.T * v[src, self]

class SrcTDEM(Survey.BaseSrc):
    rxPair = RxTDEM
    radius = None

    def getInitialFields(self, mesh):
        F0 = getattr(self, '_getInitialFields_' + self.srcType)(mesh)
        return F0

    def getJs(self, mesh, time):
        return None

class SrcTDEM_VMD_MVP(SrcTDEM):

    def __init__(self,rxList,loc,waveformType="STEPOFF"):
        self.loc = loc
        self.waveformType = waveformType
        SrcTDEM.__init__(self,rxList)

    def getInitialFields(self, mesh):
        """Vertical magnetic dipole, magnetic vector potential"""
        if self.waveformType == "STEPOFF":
            print ">> Step waveform: Non-zero initial condition"
            if mesh._meshType is 'CYL':
                if mesh.isSymmetric:
                    MVP = MagneticDipoleVectorPotential(self.loc, mesh, 'Ey')
                else:
                    raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
            elif mesh._meshType is 'TENSOR':
                MVP = MagneticDipoleVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'])
            else:
                raise Exception('Unknown mesh for VMD')
            return {"b": mesh.edgeCurl*MVP}
        elif self.waveformType == "GENERAL":
            print ">> General waveform: Zero initial condition"
            return {"b": np.zeros(mesh.nF)}
        else:
            raise NotImplementedError("Only use STEPOFF or GENERAL")

    def getMeS(self, mesh, MfMui):
        if mesh._meshType is 'CYL':
            if mesh.isSymmetric:
                MVP = MagneticDipoleVectorPotential(self.loc, mesh, 'Ey')
            else:
                raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
        elif mesh._meshType is 'TENSOR':
            MVP = MagneticDipoleVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'])
        else:
            raise Exception('Unknown mesh for VMD')
        return mesh.edgeCurl.T*MfMui*mesh.edgeCurl*MVP

class SrcTDEM_CircularLoop_MVP(SrcTDEM):
    def __init__(self,rxList,loc,radius,waveformType="STEPOFF"):
        self.loc = loc
        self.radius = radius
        self.waveformType = waveformType
        SrcTDEM.__init__(self,rxList)

    def getInitialFields(self, mesh):
        """Circular Loop, magnetic vector potential"""
        if self.waveformType == "STEPOFF":
            print ">> Step waveform: Non-zero initial condition"
            if mesh._meshType is 'CYL':
                if mesh.isSymmetric:
                    MVP = MagneticLoopVectorPotential(self.loc, mesh, 'Ey', self.radius)
                else:
                    raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
            elif mesh._meshType is 'TENSOR':
                MVP = MagneticLoopVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'], self.radius)
            else:
                raise Exception('Unknown mesh for CircularLoop')
            return {"b": mesh.edgeCurl*MVP}
        elif self.waveformType == "GENERAL":
            print ">> General waveform: Zero initial condition"
            return {"b": np.zeros(mesh.nF)}
        else:
            raise NotImplementedError("Only use STEPOFF or GENERAL")

    def getMeS(self, mesh, MfMui):
        if mesh._meshType is 'CYL':
            if mesh.isSymmetric:
                MVP = MagneticLoopVectorPotential(self.loc, mesh, 'Ey', self.radius)
            else:
                raise NotImplementedError('Non-symmetric cyl mesh not implemented yet!')
        elif mesh._meshType is 'TENSOR':
            MVP = MagneticLoopVectorPotential(self.loc, mesh, ['Ex','Ey','Ez'], self.radius)
        else:
            raise Exception('Unknown mesh for CircularLoop')
        return mesh.edgeCurl.T*MfMui*mesh.edgeCurl*MVP

class SurveyTDEM(Survey.BaseSurvey):
    """
        docstring for SurveyTDEM
    """
    srcPair = SrcTDEM

    def __init__(self, srcList, **kwargs):
        # Sort these by frequency
        self.srcList = srcList
        Survey.BaseSurvey.__init__(self, **kwargs)

    def projectFields(self, u):
        data = Survey.Data(self)
        for src in self.srcList:
            for rx in src.rxList:
                data[src, rx] = rx.projectFields(src, self.mesh, self.prob.timeMesh, u)
        return data

    def projectFieldsDeriv(self, u, v=None, adjoint=False):
        assert v is not None, 'v to multiply must be provided.'

        if not adjoint:
            data = Survey.Data(self)
            for src in self.srcList:
                for rx in src.rxList:
                    data[src, rx] = rx.projectFieldsDeriv(src, self.mesh, self.prob.timeMesh, u, v)
            return data
        else:
            f = FieldsTDEM(self.mesh, self)
            for src in self.srcList:
                for rx in src.rxList:
                    Ptv = rx.projectFieldsDeriv(src, self.mesh, self.prob.timeMesh, u, v, adjoint=True)
                    Ptv = Ptv.reshape((-1, self.prob.timeMesh.nN), order='F')
                    if rx.projField not in f: # first time we are projecting
                        f[src, rx.projField, :] = Ptv
                    else: # there are already fields, so let's add to them!
                        f[src, rx.projField, :] += Ptv
            return f

<|endoftext|>


<commit_before>import sys

import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst

def on_message(bus, message, udata):
    pipeline, loop = udata

    if message.type == Gst.MessageType.EOS:
        pipeline.set_state(Gst.State.NULL)
        loop.quit()
    elif message.type == Gst.MessageType.ERROR:
        print message.parse_error()
        pipeline.set_state(Gst.State.NULL)
        loop.quit()

    return True

def on_autoplug_factories(element, pad, caps, udata):
    factories = Gst.ElementFactory.list_get_elements(Gst.ELEMENT_FACTORY_TYPE_DEMUXER | Gst.ELEMENT_FACTORY_TYPE_PARSER, Gst.Rank.MARGINAL)
    factories = Gst.ElementFactory.list_filter(factories, caps, Gst.PadDirection.SINK, caps.is_fixed())

    if len(factories) == 0:
        #TODO check if this is indeed a parsed type and not some unhandled format (missing elements)
        return None

    return factories

def on_autoplug_continue(element, pad, caps, udata):
    s = caps.get_structure(0)
    if s.has_field('parsed'):
        if s.get_value('parsed') == True:
            return False
    if s.has_field('framed'):
        if s.get_value('framed') == True:
            return False

    return True

def caps_is_video(caps):
    s = caps.get_structure(0)
    return s.get_name().startswith('video/')
def caps_is_audio(caps):
    s = caps.get_structure(0)
    return s.get_name().startswith('audio/')

def on_pad_added(element, pad, udata):
    pipeline, splitmuxsink = udata
    other_pad = None

    # Can't use 'get_compatible_pad' because splitmuxsink pad templates
    # are all ANY so it will always match the first
    if caps_is_video(pad.get_current_caps()):
        klass = type(splitmuxsink)
        tmpl = klass.get_pad_template('video')
        other_pad = splitmuxsink.request_pad(tmpl, None, None)
    elif caps_is_audio(pad.get_current_caps()):
        klass = type(splitmuxsink)
        tmpl = klass.get_pad_template('audio_%u')
        other_pad = splitmuxsink.request_pad(tmpl, None, None)
    else:
        caps = pad.get_current_caps()
        print 'leaving pad %s unlinked - "%s"' % (pad.get_name(), caps.to_string() if caps else 'no caps')

    if other_pad:
        queue = Gst.ElementFactory.make('queue')
        pipeline.add(queue)
        queue.sync_state_with_parent()
        pad.link(queue.get_static_pad('sink'))
        queue.get_static_pad('src').link(other_pad)

def split(input_uri, output_dir):
    loop = GObject.MainLoop()

    pipeline = Gst.Pipeline()
    uridecodebin = Gst.ElementFactory.make('uridecodebin')
    splitmuxsink = Gst.ElementFactory.make('splitmuxsink')
    pipeline.add(uridecodebin)
    pipeline.add(splitmuxsink)

    uridecodebin.set_property('uri', input_uri)
    uridecodebin.connect('autoplug-factories', on_autoplug_factories, None)
    uridecodebin.connect('autoplug-continue', on_autoplug_continue, None)
    uridecodebin.connect('pad-added', on_pad_added, (pipeline, splitmuxsink))

    # TODO fix mp4mux to properly segment files
    splitmuxsink.set_property('muxer', Gst.ElementFactory.make('matroska'))
    splitmuxsink.set_property('location', output_dir + '/' + 'segment_%09d.mkv')
    splitmuxsink.set_property('max-size-time', 10000000000) #10s segments

    bus = pipeline.get_bus()
    bus.add_watch(0, on_message, (pipeline, loop))

    pipeline.set_state(Gst.State.PLAYING)
    loop.run()

if __name__ == '__main__':
    Gst.init()

    input_uri = sys.argv[1]
    output_dir = sys.argv[2]
    #TODO validate args

    split(input_uri, output_dir)
<commit_msg>splitter: use correct name for muxer<commit_after>import sys

import gi
gi.require_version('Gst', '1.0')
from gi.repository import GObject, Gst

def on_message(bus, message, udata):
    pipeline, loop = udata

    if message.type == Gst.MessageType.EOS:
        pipeline.set_state(Gst.State.NULL)
        loop.quit()
    elif message.type == Gst.MessageType.ERROR:
        print message.parse_error()
        pipeline.set_state(Gst.State.NULL)
        loop.quit()

    return True

def on_autoplug_factories(element, pad, caps, udata):
    factories = Gst.ElementFactory.list_get_elements(Gst.ELEMENT_FACTORY_TYPE_DEMUXER | Gst.ELEMENT_FACTORY_TYPE_PARSER, Gst.Rank.MARGINAL)
    factories = Gst.ElementFactory.list_filter(factories, caps, Gst.PadDirection.SINK, caps.is_fixed())

    if len(factories) == 0:
        #TODO check if this is indeed a parsed type and not some unhandled format (missing elements)
        return None

    return factories

def on_autoplug_continue(element, pad, caps, udata):
    s = caps.get_structure(0)
    if s.has_field('parsed'):
        if s.get_value('parsed') == True:
            return False
    if s.has_field('framed'):
        if s.get_value('framed') == True:
            return False

    return True

def caps_is_video(caps):
    s = caps.get_structure(0)
    return s.get_name().startswith('video/')
def caps_is_audio(caps):
    s = caps.get_structure(0)
    return s.get_name().startswith('audio/')

def on_pad_added(element, pad, udata):
    pipeline, splitmuxsink = udata
    other_pad = None

    # Can't use 'get_compatible_pad' because splitmuxsink pad templates
    # are all ANY so it will always match the first
    if caps_is_video(pad.get_current_caps()):
        klass = type(splitmuxsink)
        tmpl = klass.get_pad_template('video')
        other_pad = splitmuxsink.request_pad(tmpl, None, None)
    elif caps_is_audio(pad.get_current_caps()):
        klass = type(splitmuxsink)
        tmpl = klass.get_pad_template('audio_%u')
        other_pad = splitmuxsink.request_pad(tmpl, None, None)
    else:
        caps = pad.get_current_caps()
        print 'leaving pad %s unlinked - "%s"' % (pad.get_name(), caps.to_string() if caps else 'no caps')

    if other_pad:
        queue = Gst.ElementFactory.make('queue')
        pipeline.add(queue)
        queue.sync_state_with_parent()
        pad.link(queue.get_static_pad('sink'))
        queue.get_static_pad('src').link(other_pad)

def split(input_uri, output_dir):
    loop = GObject.MainLoop()

    pipeline = Gst.Pipeline()
    uridecodebin = Gst.ElementFactory.make('uridecodebin')
    splitmuxsink = Gst.ElementFactory.make('splitmuxsink')
    pipeline.add(uridecodebin)
    pipeline.add(splitmuxsink)

    uridecodebin.set_property('uri', input_uri)
    uridecodebin.connect('autoplug-factories', on_autoplug_factories, None)
    uridecodebin.connect('autoplug-continue', on_autoplug_continue, None)
    uridecodebin.connect('pad-added', on_pad_added, (pipeline, splitmuxsink))

    # TODO fix mp4mux to properly segment files
    splitmuxsink.set_property('muxer', Gst.ElementFactory.make('matroskamux'))
    splitmuxsink.set_property('location', output_dir + '/' + 'segment_%09d.mkv')
    splitmuxsink.set_property('max-size-time', 10000000000) #10s segments

    bus = pipeline.get_bus()
    bus.add_watch(0, on_message, (pipeline, loop))

    pipeline.set_state(Gst.State.PLAYING)
    loop.run()

if __name__ == '__main__':
    Gst.init()

    input_uri = sys.argv[1]
    output_dir = sys.argv[2]
    #TODO validate args

    split(input_uri, output_dir)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from polymodels.fields import PolymorphicTypeField

from ... import forms

class FieldDefinitionTypeField(PolymorphicTypeField):
    def __init__(self, *args, **kwargs):
        super(FieldDefinitionTypeField, self).__init__(
            'mutant.FieldDefinition', *args, **kwargs
        )

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.FieldDefinitionTypeField}
        defaults.update(kwargs)
        return super(FieldDefinitionTypeField, self).formfield(**kwargs)
<commit_msg>Defaulted on_delete to CASCADE for FieldDefinitionTypeField.<commit_after>from __future__ import unicode_literals

from django.db import models
from polymodels.fields import PolymorphicTypeField

from ... import forms

class FieldDefinitionTypeField(PolymorphicTypeField):
    def __init__(self, on_delete=models.CASCADE, *args, **kwargs):
        super(FieldDefinitionTypeField, self).__init__(
            'mutant.FieldDefinition', on_delete=on_delete, *args, **kwargs
        )

    def formfield(self, **kwargs):
        defaults = {'form_class': forms.FieldDefinitionTypeField}
        defaults.update(kwargs)
        return super(FieldDefinitionTypeField, self).formfield(**kwargs)
<|endoftext|>


<commit_before>import os
import sys

#Base class for instructions revolving aroung TODO list.
class TodoInstruction(object):

    #location parameter = location of todo list file
    def __init__(self, location):
        self.location = location

    #execute instruction
    def doInstruction(self, args):
        print ("Doing instruction! Parameters:")
        print (self.prepareArguments(self, args))

    def prepareArguments(self, args):
        arguments = args.split(",")
        problems = []
        for arg in arguments:
            if arg.strip() != "":
                problems.append(arg.strip())
        return problems

    #read todo list
    def readTodo(self):
        if not os.path.isfile(self.location):
            return []
        f = open(self.location, "r")
        result = []
        a = f.readline().strip()
        while a != "":
            result.append(a)
            a = f.readline().strip()
        f.close()
        return result

    #write todo list
    def writeTodo(self, lst):
        f = open(self.location, "w")
        for prob in lst:
            f.write(prob+"\n")
        f.close()

#Add to TODO list instruction
class Add(TodoInstruction):

    def __init__(self, location):
        super(Add, self).__init__(location)

    def doInstruction(self, args):
        problems = self.prepareArguments(args)
        todo = self.readTodo()
        finalList = sorted(list(set(problems + todo)))
        self.writeTodo(finalList)

#Delete problems from TODO list
class Del(TodoInstruction):

    def __init__(self, location):
        super(Del, self).__init__(location)

    def doInstruction(self, args):
        problems = self.prepareArguments(args)
        todo = self.readTodo()
        finalList = []
        for prob in todo:
            if not prob in problems:
                finalList.append(prob)
        self.writeTodo(finalList)

class List(TodoInstruction):

    def __init__(self, location):
        super(List, self).__init__(location)

    def doInstruction(self, args):
        todo = self.readTodo()
        print "TODO List:"
        print "================"
        for prob in todo:
            print prob
        print "================"

class Clear(TodoInstruction):

    def __init__(self, location):
        super(Clear, self).__init__(location)

    def doInstruction(self, args):
        self.writeTodo([])

#init function, reads init file, initializes instructions
def init_function():
    todo_location = "TODO_LOCATION"
    init_file = "init.txt"
    init_params = {}
    location = os.path.realpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))
    init = open(os.path.join(location, init_file), "r")
    s = init.readline().strip()
    while s != "":
        params = s.split("=")
        init_params[params[0]] = params[1]
        s = init.readline().strip()
    init.close()

    global instructions
    instructions = {}
    instructions["add"] = Add(init_params[todo_location])
    instructions["del"] = Del(init_params[todo_location])
    instructions["ls"] = List(init_params[todo_location])
    instructions["clear"] = Clear(init_params[todo_location])

if __name__ == "__main__":
    instructions = {}
    print(("Initializing..."))
    try:
        init_function()
    except:
        print (("Error initializing: ", sys.exc_info()[0], sys.exc_info()[1]))
        print ("Closing.")
        sys.exit(0)
    print(("Completed"))

    while(True):
        s = raw_input(">")
        command = s.split(" ")
        if not command[0] in instructions:
            print("Unknown instruction.")
            continue
        if len(command) == 1:
            command.append("")
        try:
            instructions[command[0]].doInstruction(command[1])
        except:
            print (("Error: ", sys.exc_info()[0], sys.exc_info()[1]))
<commit_msg>Added getHtml and ls instructions<commit_after>import os
import sys

#Base class for instructions revolving aroung TODO list.
class TodoInstruction(object):

    #location parameter = location of todo list file
    def __init__(self, location):
        self.location = location

    #execute instruction
    def doInstruction(self, args):
        print ("Doing instruction! Parameters:")
        print (self.prepareArguments(self, args))

    def prepareArguments(self, args):
        arguments = args.split(",")
        problems = []
        for arg in arguments:
            if arg.strip() != "":
                problems.append(arg.strip())
        return problems

    #read todo list
    def readTodo(self):
        if not os.path.isfile(self.location):
            return []
        f = open(self.location, "r")
        result = []
        a = f.readline().strip()
        while a != "":
            result.append(a)
            a = f.readline().strip()
        f.close()
        return result

    #write todo list
    def writeTodo(self, lst):
        f = open(self.location, "w")
        for prob in lst:
            if prob.strip() != "":
                f.write(prob.strip()+"\n")
        f.close()

#Add to TODO list instruction
class Add(TodoInstruction):

    def __init__(self, location):
        super(Add, self).__init__(location)

    def doInstruction(self, args):
        problems = self.prepareArguments(args)
        todo = self.readTodo()
        finalList = sorted(list(set(problems + todo)))
        self.writeTodo(finalList)

#Delete problems from TODO list
class Del(TodoInstruction):

    def __init__(self, location):
        super(Del, self).__init__(location)

    def doInstruction(self, args):
        problems = self.prepareArguments(args)
        todo = self.readTodo()
        finalList = []
        for prob in todo:
            if not prob in problems:
                finalList.append(prob)
        self.writeTodo(finalList)

#Prints todo list to console
class List(TodoInstruction):

    def __init__(self, location):
        super(List, self).__init__(location)

    def doInstruction(self, args):
        todo = self.readTodo()
        print "TODO List:"
        print "================"
        for prob in todo:
            print prob
        print "================"

#Clears todo list
class Clear(TodoInstruction):

    def __init__(self, location):
        super(Clear, self).__init__(location)

    def doInstruction(self, args):
        self.writeTodo([])

#Generates Html document for todo list
class getHtml(TodoInstruction):

    def __init__(self, location):
        super(getHtml, self).__init__(location)

    def doInstruction(self, args):
        arguments = self.prepareArguments(args)
        if len(arguments) == 0:
            print "No argument given."
            return
        path = arguments[0]
        todo = self.readTodo()
        f = open(path, "w")
        f.write("\n")
        f.write("
 TODO List:
\n")
        f.write("\n")
        for prob in todo:
            f.write("" + prob + "
")
        f.write("\n")
        f.write("\n")
        f.close()

#init function, reads init file, initializes instructions
def init_function():
    todo_location = "TODO_LOCATION"
    init_file = "init.txt"
    init_params = {}
    location = os.path.realpath(
    os.path.join(os.getcwd(), os.path.dirname(__file__)))
    init = open(os.path.join(location, init_file), "r")
    s = init.readline().strip()
    while s != "":
        params = s.split("=")
        init_params[params[0]] = params[1]
        s = init.readline().strip()
    init.close()

    global instructions
    instructions = {}
    instructions["add"] = Add(init_params[todo_location])
    instructions["del"] = Del(init_params[todo_location])
    instructions["ls"] = List(init_params[todo_location])
    instructions["clear"] = Clear(init_params[todo_location])
    instructions["getHtml"] = getHtml(init_params[todo_location])

if __name__ == "__main__":
    instructions = {}
    print(("Initializing..."))
    try:
        init_function()
    except:
        print (("Error initializing: ", sys.exc_info()[0], sys.exc_info()[1]))
        print ("Closing.")
        sys.exit(0)
    print(("Completed"))

    while(True):
        s = raw_input(">")
        command = s.split(" ")
        if not command[0] in instructions:
            print("Unknown instruction.")
            continue
        if len(command) == 1:
            command.append("")
        try:
            instructions[command[0]].doInstruction(command[1])
        except:
            print (("Error: ", sys.exc_info()[0], sys.exc_info()[1]))
<|endoftext|>


<commit_before>
import os
import re
import math
from string import Template
from datetime import timedelta
from mycluster import get_timedelta
from subprocess import Popen, PIPE, check_output
from mycluster import get_data

"""
sacctmgr show cluster
"""

def scheduler_type():
    return 'slurm'

def name():
    with os.popen('sacctmgr show cluster') as f:
        f.readline()
        f.readline()
        return f.readline().strip().split(' ')[0]

def accounts():
    pass

def queues():
    queue_list = []

    with os.popen('sinfo -s') as f:
        f.readline(); # read header
        for line in f:
            q = line.split(' ')[0].strip().replace("*","")
            queue_list.append(q)

    return queue_list

def available_tasks(queue_id):

    # split queue id into queue and parallel env
    # list free slots
    free_tasks = 0
    max_tasks = 0
    queue_name   = queue_id
    nc = node_config(queue_id)
    with os.popen('sinfo -s -p '+queue_name) as f:
        f.readline(); # read header
        line = f.readline();
        new_line = re.sub(' +',' ',line.strip())
        line = new_line.split(' ')[3]
        free_tasks = int(line.split('/')[1])*nc['max task']
        max_tasks  = int(line.split('/')[3])*nc['max task']

    return {'available' : free_tasks, 'max tasks' : max_tasks}

def tasks_per_node(queue_id):
    queue_name   = queue_id
    tasks=0
    with os.popen('sinfo -Nel -p '+queue_name) as f:
        f.readline(); # read header
        f.readline(); # read header
        line = f.readline();
        new_line = re.sub(' +',' ',line.strip())
        tasks = int(new_line.split(' ')[4])
    return tasks

def node_config(queue_id):
    # Find first node with queue and record node config
    queue_name   = queue_id
    tasks=0
    config = {}
    with os.popen('sinfo -Nel -p '+queue_name) as f:
        f.readline(); # read header
        f.readline(); # read header
        line = f.readline();
        new_line = re.sub(' +',' ',line.strip())
        tasks = int(new_line.split(' ')[4])
        memory = int(new_line.split(' ')[6])
        config['max task']   = tasks
        config['max thread'] = tasks
        config['max memory'] = memory

    return config

def create_submit(queue_id,**kwargs):

    queue_name   = queue_id

    num_tasks = 1
    if 'num_tasks' in kwargs:
        num_tasks = kwargs['num_tasks']

    tpn = tasks_per_node(queue_id)
    queue_tpn = tpn
    if 'tasks_per_node' in kwargs:
        tpn = min(tpn,kwargs['tasks_per_node'])

    nc = node_config(queue_id)
    qc = available_tasks(queue_id)

    num_tasks = min(num_tasks,qc['max tasks'])

    num_threads_per_task = nc['max thread']
    if 'num_threads_per_task' in kwargs:
        num_threads_per_task = kwargs['num_threads_per_task']
    num_threads_per_task = min(num_threads_per_task,int(math.ceil(float(nc['max thread'])/float(tpn))))

    my_name = "myclusterjob"
    if 'my_name' in kwargs:
        my_name = kwargs['my_name']
    my_output = "myclusterjob.out"
    if 'my_output' in kwargs:
        my_output = kwargs['my_output']
    if 'my_script' not in kwargs:
        pass
    my_script = kwargs['my_script']
    if 'mycluster-' in my_script:
        my_script = get_data(my_script)

    if 'user_email' not in kwargs:
        pass
    user_email = kwargs['user_email']

    project_name = 'default'
    if 'project_name' in kwargs:
        project_name = kwargs['project_name']

    wall_clock = '12:00:00'
    if 'wall_clock' in kwargs:
        if ':' not in kwargs['wall_clock']:
            wall_clock = str(kwargs['wall_clock'])+':00:00'

    num_nodes = int(math.ceil(float(num_tasks)/float(tpn)))

    num_queue_slots = num_nodes*queue_tpn

    script=Template(r"""#!/bin/bash
#
# SLURM job submission script generated by MyCluster
#
# Job name
#SBATCH -J $my_name
# Send status information to this email address.
#SBATCH --mail-user=$user_email
# Send me an e-mail when the job has finished.
#SBATCH --mail-type=ALL
# Redirect output stream to this file.
#SBATCH --output $my_output.%j
# Which project should be charged
#SBATCH -A $project_name
# Partition name
#SBATCH -p $queue_name
# Number of nodes
#SBATCH --nodes $num_nodes
# Number of tasks
#SBATCH --ntasks $num_tasks
# Exclusive node use
#SBATCH --exclusive
# How much wallclock time will be required?
#SBATCH --time=$wall_clock

export MYCLUSTER_QUEUE=$queue_name
export MYCLUSTER_JOB_NAME=$my_name
export NUM_TASKS=$num_tasks
export TASKS_PER_NODE=$tpn
export THREADS_PER_TASK=$num_threads_per_task
export NUM_NODES=$num_nodes
export JOBID=$$SLURM_JOB_ID

# OpenMP configuration
export OMP_NUM_THREADS=$$THREADS_PER_TASK
export OMP_PROC_BIND=true
export OMP_PLACES=sockets

# OpenMPI
export OMPI_CMD="mpiexec -n $$NUM_TASKS -npernode $$TASKS_PER_NODE -bysocket -bind-to-socket"

# MVAPICH2
export MV2_CPU_BINDING_LEVEL=SOCKET
export MV2_CPU_BINDING_POLICY=scatter
export MVAPICH_CMD="mpiexec -n $$NUM_TASKS -ppn $$TASKS_PER_NODE -bind-to-socket"

# Intel MPI
# The following variables define a sensible pinning strategy for Intel MPI tasks -
# this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $$OMP_NUM_THREADS cores in size
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
#export I_MPI_FABRICS=shm:ofa
export IMPI_CMD="mpiexec -n $$NUM_TASKS -ppn $$TASKS_PER_NODE"

# Summarise environment
echo -e "JobID: $$JOBID\n======"
echo "Time: `date`"
echo "Running on master node: `hostname`"
echo "Current directory: `pwd`"

if [ "$$SLURM_JOB_NODELIST" ]; then
        #! Create a machine file:
        echo $$SLURM_JOB_NODELIST | uniq > machine.file.$$JOBID
        echo -e "\nNodes allocated:\n================"
        echo `cat machine.file.$$JOBID | sed -e 's/\..*$$//g'`
fi

echo -e "\nnumtasks=$num_tasks, numnodes=$num_nodes, tasks_per_node=$tpn (OMP_NUM_THREADS=$$OMP_NUM_THREADS)"

echo -e "\nExecuting command:\n==================\n$my_script\n"

# Run user script
./$my_script

# Report on completion
echo -e "\nJob Complete:\n==================\n"
echo -e "\nRecording hardware setup\n==================\n"
mycluster --sysscribe $$JOBID
if [ "$$MYCLUSTER_APP_NAME" ]; then
    mycluster --jobid $$JOBID --appname=$$MYCLUSTER_APP_NAME
fi
if [ "$$MYCLUSTER_APP_DATA" ]; then
    mycluster --jobid $$JOBID --appdata=$$MYCLUSTER_APP_DATA
fi
echo -e "Complete========\n"
""")
    script_str = script.substitute({'my_name':my_name,
                                   'my_script':my_script,
                                   'my_output':my_output,
                                   'user_email':user_email,
                                   'queue_name':queue_name,
                                   'num_queue_slots':num_queue_slots,
                                   'num_tasks':num_tasks,
                                   'tpn':tpn,
                                   'num_threads_per_task':num_threads_per_task,
                                   'num_queue_slots':num_queue_slots,
                                   'num_nodes':num_nodes,
                                   'project_name': project_name,
                                   'wall_clock' : wall_clock,
                                   })

    return script_str

def submit(script_name,immediate):
    job_id = None
    if not immediate:
        with os.popen('sbatch '+script_name) as f:
            output = f.readline()
            try:
                job_id = int(output.split(' ')[-1].strip())
            except:
                print('Job submission failed: '+output)
            # Get job id and record in database
    else:
        with os.popen('srun '+script_name) as f:
            output = f.readline()
            try:
                job_id = int(output.split(' ')[-1].strip())
            except:
                print('Job submission failed: '+output)
    return job_id

def delete(job_id):
    with os.popen('scancel '+job_id) as f:
        pass

def status():
    status_dict = {}
    with os.popen('squeue -u `whoami`') as f:
        try:
            f.readline(); # read header
            for line in f:
                new_line = re.sub(' +',' ',line.strip())
                job_id = int(new_line.split(' ')[0])
                state = new_line.split(' ')[4]
                if state == 'R':
                    status_dict[job_id] = 'r'
                else:
                    status_dict[job_id] = state
        except e:
            print e

    return status_dict

def job_stats(job_id):
    stats_dict = {}
    with os.popen('sacct --noheader --format Elapsed,TotalCPU,Partition,NTasks,AveRSS -j '+str(job_id)) as f:
        try:
            line = f.readline();
            new_line = re.sub(' +',' ',line.strip())

            wallclock_str = new_line.split(' ')[0]
            stats_dict['wallclock']  = get_timedelta(wallclock_str)

            cpu_str = new_line.split(' ')[1]
            stats_dict['cpu'] = get_timedelta(cpu_str)

            stats_dict['queue'] = new_line.split(' ')[2]
            stats_dict['mem'] = 0 #float(new_line.split(' ')[4])*int(new_line.split(' ')[3])
        except:
            print('SLURM: Error reading job stats')

    return stats_dict

def running_stats(job_id):
    stats_dict = {}
    with os.popen('sacct --noheader --format Elapsed -j '+str(job_id)) as f:
        try:
            line = f.readline();
            new_line = re.sub(' +',' ',line.strip())
            stats_dict['wallclock']  = get_timedelta(new_line)
        except:
            pass

    with os.popen('sstat --noheader --format AveCPU,AveRSS,NTasks -j '+str(job_id)) as f:
        try:
            line = f.readline();
            new_line = re.sub(' +',' ',line.strip())
            ntasks = int(new_line.split(' ')[2])
            stats_dict['mem']  = float(new_line.split(' ')[1].replace('K',''))*ntasks
            stats_dict['cpu']  = '-' #float(new_line.split(' ')[0])*ntasks
        except:
            pass

    return stats_dict
<commit_msg>bug fix to script execution<commit_after>
import os
import re
import math
from string import Template
from datetime import timedelta
from mycluster import get_timedelta
from subprocess import Popen, PIPE, check_output
from mycluster import get_data

"""
sacctmgr show cluster
"""

def scheduler_type():
    return 'slurm'

def name():
    with os.popen('sacctmgr show cluster') as f:
        f.readline()
        f.readline()
        return f.readline().strip().split(' ')[0]

def accounts():
    pass

def queues():
    queue_list = []

    with os.popen('sinfo -s') as f:
        f.readline(); # read header
        for line in f:
            q = line.split(' ')[0].strip().replace("*","")
            queue_list.append(q)

    return queue_list

def available_tasks(queue_id):

    # split queue id into queue and parallel env
    # list free slots
    free_tasks = 0
    max_tasks = 0
    queue_name   = queue_id
    nc = node_config(queue_id)
    with os.popen('sinfo -s -p '+queue_name) as f:
        f.readline(); # read header
        line = f.readline();
        new_line = re.sub(' +',' ',line.strip())
        line = new_line.split(' ')[3]
        free_tasks = int(line.split('/')[1])*nc['max task']
        max_tasks  = int(line.split('/')[3])*nc['max task']

    return {'available' : free_tasks, 'max tasks' : max_tasks}

def tasks_per_node(queue_id):
    queue_name   = queue_id
    tasks=0
    with os.popen('sinfo -Nel -p '+queue_name) as f:
        f.readline(); # read header
        f.readline(); # read header
        line = f.readline();
        new_line = re.sub(' +',' ',line.strip())
        tasks = int(new_line.split(' ')[4])
    return tasks

def node_config(queue_id):
    # Find first node with queue and record node config
    queue_name   = queue_id
    tasks=0
    config = {}
    with os.popen('sinfo -Nel -p '+queue_name) as f:
        f.readline(); # read header
        f.readline(); # read header
        line = f.readline();
        new_line = re.sub(' +',' ',line.strip())
        tasks = int(new_line.split(' ')[4])
        memory = int(new_line.split(' ')[6])
        config['max task']   = tasks
        config['max thread'] = tasks
        config['max memory'] = memory

    return config

def create_submit(queue_id,**kwargs):

    queue_name   = queue_id

    num_tasks = 1
    if 'num_tasks' in kwargs:
        num_tasks = kwargs['num_tasks']

    tpn = tasks_per_node(queue_id)
    queue_tpn = tpn
    if 'tasks_per_node' in kwargs:
        tpn = min(tpn,kwargs['tasks_per_node'])

    nc = node_config(queue_id)
    qc = available_tasks(queue_id)

    num_tasks = min(num_tasks,qc['max tasks'])

    num_threads_per_task = nc['max thread']
    if 'num_threads_per_task' in kwargs:
        num_threads_per_task = kwargs['num_threads_per_task']
    num_threads_per_task = min(num_threads_per_task,int(math.ceil(float(nc['max thread'])/float(tpn))))

    my_name = "myclusterjob"
    if 'my_name' in kwargs:
        my_name = kwargs['my_name']
    my_output = "myclusterjob.out"
    if 'my_output' in kwargs:
        my_output = kwargs['my_output']
    if 'my_script' not in kwargs:
        pass
    my_script = kwargs['my_script']
    if 'mycluster-' in my_script:
        my_script = get_data(my_script)

    if 'user_email' not in kwargs:
        pass
    user_email = kwargs['user_email']

    project_name = 'default'
    if 'project_name' in kwargs:
        project_name = kwargs['project_name']

    wall_clock = '12:00:00'
    if 'wall_clock' in kwargs:
        if ':' not in kwargs['wall_clock']:
            wall_clock = str(kwargs['wall_clock'])+':00:00'

    num_nodes = int(math.ceil(float(num_tasks)/float(tpn)))

    num_queue_slots = num_nodes*queue_tpn

    script=Template(r"""#!/bin/bash
#
# SLURM job submission script generated by MyCluster
#
# Job name
#SBATCH -J $my_name
# Send status information to this email address.
#SBATCH --mail-user=$user_email
# Send me an e-mail when the job has finished.
#SBATCH --mail-type=ALL
# Redirect output stream to this file.
#SBATCH --output $my_output.%j
# Which project should be charged
#SBATCH -A $project_name
# Partition name
#SBATCH -p $queue_name
# Number of nodes
#SBATCH --nodes $num_nodes
# Number of tasks
#SBATCH --ntasks $num_tasks
# Exclusive node use
#SBATCH --exclusive
# How much wallclock time will be required?
#SBATCH --time=$wall_clock

export MYCLUSTER_QUEUE=$queue_name
export MYCLUSTER_JOB_NAME=$my_name
export NUM_TASKS=$num_tasks
export TASKS_PER_NODE=$tpn
export THREADS_PER_TASK=$num_threads_per_task
export NUM_NODES=$num_nodes
export JOBID=$$SLURM_JOB_ID

# OpenMP configuration
export OMP_NUM_THREADS=$$THREADS_PER_TASK
export OMP_PROC_BIND=true
export OMP_PLACES=sockets

# OpenMPI
export OMPI_CMD="mpiexec -n $$NUM_TASKS -npernode $$TASKS_PER_NODE -bysocket -bind-to-socket"

# MVAPICH2
export MV2_CPU_BINDING_LEVEL=SOCKET
export MV2_CPU_BINDING_POLICY=scatter
export MVAPICH_CMD="mpiexec -n $$NUM_TASKS -ppn $$TASKS_PER_NODE -bind-to-socket"

# Intel MPI
# The following variables define a sensible pinning strategy for Intel MPI tasks -
# this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $$OMP_NUM_THREADS cores in size
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
#export I_MPI_FABRICS=shm:ofa
export IMPI_CMD="mpiexec -n $$NUM_TASKS -ppn $$TASKS_PER_NODE"

# Summarise environment
echo -e "JobID: $$JOBID\n======"
echo "Time: `date`"
echo "Running on master node: `hostname`"
echo "Current directory: `pwd`"

if [ "$$SLURM_JOB_NODELIST" ]; then
        #! Create a machine file:
        echo $$SLURM_JOB_NODELIST | uniq > machine.file.$$JOBID
        echo -e "\nNodes allocated:\n================"
        echo `cat machine.file.$$JOBID | sed -e 's/\..*$$//g'`
fi

echo -e "\nnumtasks=$num_tasks, numnodes=$num_nodes, tasks_per_node=$tpn (OMP_NUM_THREADS=$$OMP_NUM_THREADS)"

echo -e "\nExecuting command:\n==================\n$my_script\n"

# Run user script
. $my_script

# Report on completion
echo -e "\nJob Complete:\n==================\n"
echo -e "\nRecording hardware setup\n==================\n"
mycluster --sysscribe $$JOBID
if [ "$$MYCLUSTER_APP_NAME" ]; then
    mycluster --jobid $$JOBID --appname=$$MYCLUSTER_APP_NAME
fi
if [ "$$MYCLUSTER_APP_DATA" ]; then
    mycluster --jobid $$JOBID --appdata=$$MYCLUSTER_APP_DATA
fi
echo -e "Complete========\n"
""")
    script_str = script.substitute({'my_name':my_name,
                                   'my_script':my_script,
                                   'my_output':my_output,
                                   'user_email':user_email,
                                   'queue_name':queue_name,
                                   'num_queue_slots':num_queue_slots,
                                   'num_tasks':num_tasks,
                                   'tpn':tpn,
                                   'num_threads_per_task':num_threads_per_task,
                                   'num_queue_slots':num_queue_slots,
                                   'num_nodes':num_nodes,
                                   'project_name': project_name,
                                   'wall_clock' : wall_clock,
                                   })

    return script_str

def submit(script_name,immediate):
    job_id = None
    if not immediate:
        with os.popen('sbatch '+script_name) as f:
            output = f.readline()
            try:
                job_id = int(output.split(' ')[-1].strip())
            except:
                print('Job submission failed: '+output)
            # Get job id and record in database
    else:
        with os.popen('srun '+script_name) as f:
            output = f.readline()
            try:
                job_id = int(output.split(' ')[-1].strip())
            except:
                print('Job submission failed: '+output)
    return job_id

def delete(job_id):
    with os.popen('scancel '+job_id) as f:
        pass

def status():
    status_dict = {}
    with os.popen('squeue -u `whoami`') as f:
        try:
            f.readline(); # read header
            for line in f:
                new_line = re.sub(' +',' ',line.strip())
                job_id = int(new_line.split(' ')[0])
                state = new_line.split(' ')[4]
                if state == 'R':
                    status_dict[job_id] = 'r'
                else:
                    status_dict[job_id] = state
        except e:
            print e

    return status_dict

def job_stats(job_id):
    stats_dict = {}
    with os.popen('sacct --noheader --format Elapsed,TotalCPU,Partition,NTasks,AveRSS -j '+str(job_id)) as f:
        try:
            line = f.readline();
            new_line = re.sub(' +',' ',line.strip())

            wallclock_str = new_line.split(' ')[0]
            stats_dict['wallclock']  = get_timedelta(wallclock_str)

            cpu_str = new_line.split(' ')[1]
            stats_dict['cpu'] = get_timedelta(cpu_str)

            stats_dict['queue'] = new_line.split(' ')[2]
            stats_dict['mem'] = 0 #float(new_line.split(' ')[4])*int(new_line.split(' ')[3])
        except:
            print('SLURM: Error reading job stats')

    return stats_dict

def running_stats(job_id):
    stats_dict = {}
    with os.popen('sacct --noheader --format Elapsed -j '+str(job_id)) as f:
        try:
            line = f.readline();
            new_line = re.sub(' +',' ',line.strip())
            stats_dict['wallclock']  = get_timedelta(new_line)
        except:
            pass

    with os.popen('sstat --noheader --format AveCPU,AveRSS,NTasks -j '+str(job_id)) as f:
        try:
            line = f.readline();
            new_line = re.sub(' +',' ',line.strip())
            ntasks = int(new_line.split(' ')[2])
            stats_dict['mem']  = float(new_line.split(' ')[1].replace('K',''))*ntasks
            stats_dict['cpu']  = '-' #float(new_line.split(' ')[0])*ntasks
        except:
            pass

    return stats_dict
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2018-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the wallet balance RPC methods."""
from decimal import Decimal

from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY
from test_framework.test_framework import SyscoinTestFramework

from test_framework.util import (
    assert_equal,
    assert_raises_rpc_error,
)

def create_transactions(node, address, amt, fees):
    # Create and sign raw transactions from node to address for amt.
    # Creates a transaction for each fee and returns an array
    # of the raw transactions.
    utxos = node.listunspent(0)

    # Create transactions
    inputs = []
    ins_total = 0
    for utxo in utxos:
        inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]})
        ins_total += utxo['amount']
        if ins_total > amt:
            break

    txs = []
    for fee in fees:
        outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee}
        raw_tx = node.createrawtransaction(inputs, outputs, 0, True)
        raw_tx = node.signrawtransactionwithwallet(raw_tx)
        txs.append(raw_tx)

    return txs

class WalletTest(SyscoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 2
        self.setup_clean_chain = True

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def run_test(self):
        self.nodes[0].importaddress(ADDRESS_WATCHONLY)
        # Check that nodes don't own any UTXOs
        assert_equal(len(self.nodes[0].listunspent()), 0)
        assert_equal(len(self.nodes[1].listunspent()), 0)

        self.log.info("Mining blocks ...")

        self.nodes[0].generate(1)
        self.sync_all()
        self.nodes[1].generate(1)
        self.nodes[1].generatetoaddress(101, ADDRESS_WATCHONLY)
        self.sync_all()

        assert_equal(self.nodes[0].getbalance(), 50)
        assert_equal(self.nodes[1].getbalance(), 50)

        self.log.info("Test getbalance with different arguments")
        assert_equal(self.nodes[0].getbalance("*"), 50)
        assert_equal(self.nodes[0].getbalance("*", 1), 50)
        assert_equal(self.nodes[0].getbalance("*", 1, True), 100)
        assert_equal(self.nodes[0].getbalance(minconf=1), 50)
        assert_equal(self.nodes[0].getbalance(minconf=0, include_watchonly=True), 100)
        assert_equal(self.nodes[1].getbalance(minconf=0, include_watchonly=True), 50)

        # Send 40 SYS from 0 to 1 and 60 SYS from 1 to 0.
        txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')])
        self.nodes[0].sendrawtransaction(txs[0]['hex'])
        self.nodes[1].sendrawtransaction(txs[0]['hex'])  # sending on both nodes is faster than waiting for propagation

        self.sync_all()
        txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), 60, [Decimal('0.01'), Decimal('0.02')])
        self.nodes[1].sendrawtransaction(txs[0]['hex'])
        self.nodes[0].sendrawtransaction(txs[0]['hex'])  # sending on both nodes is faster than waiting for propagation
        self.sync_all()

        # First argument of getbalance must be set to "*"
        assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "")

        self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs")

        # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions
        assert_equal(self.nodes[0].getbalance(), Decimal('9.99'))  # change from node 0's send
        assert_equal(self.nodes[1].getbalance(), Decimal('29.99'))  # change from node 1's send
        # Same with minconf=0
        assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99'))
        assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('29.99'))
        # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago
        # TODO: fix getbalance tracking of coin spentness depth
        assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0'))
        assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0'))
        # getunconfirmedbalance
        assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60'))  # output of node 1's spend
        assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0'))  # Doesn't include output of node 0's send since it was spent

        # Node 1 bumps the transaction fee and resends
        self.nodes[1].sendrawtransaction(txs[1]['hex'])
        self.sync_all()

        self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs")

        assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60'))  # output of node 1's send
        assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60'))
        assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0'))  # Doesn't include output of node 0's send since it was spent
        assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0'))

        self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY)
        self.sync_all()

        # balances are correct after the transactions are confirmed
        assert_equal(self.nodes[0].getbalance(), Decimal('69.99'))  # node 1's send plus change from node 0's send
        assert_equal(self.nodes[1].getbalance(), Decimal('29.98'))  # change from node 0's send

        # Send total balance away from node 1
        txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')])
        self.nodes[1].sendrawtransaction(txs[0]['hex'])
        self.nodes[1].generatetoaddress(2, ADDRESS_WATCHONLY)
        self.sync_all()

        # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago
        # TODO: fix getbalance tracking of coin spentness depth
        # getbalance with minconf=3 should still show the old balance
        assert_equal(self.nodes[1].getbalance(minconf=3), Decimal('0'))

        # getbalance with minconf=2 will show the new balance.
        assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0'))

        # check mempool transactions count for wallet unconfirmed balance after
        # dynamically loading the wallet.
        before = self.nodes[1].getunconfirmedbalance()
        dst = self.nodes[1].getnewaddress()
        self.nodes[1].unloadwallet('')
        self.nodes[0].sendtoaddress(dst, 0.1)
        self.sync_all()
        self.nodes[1].loadwallet('')
        after = self.nodes[1].getunconfirmedbalance()
        assert_equal(before + Decimal('0.1'), after)

if __name__ == '__main__':
    WalletTest().main()
<commit_msg>test: Add getunconfirmedbalance test with conflicts<commit_after>#!/usr/bin/env python3
# Copyright (c) 2018-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the wallet balance RPC methods."""
from decimal import Decimal

from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY
from test_framework.test_framework import SyscoinTestFramework

from test_framework.util import (
    assert_equal,
    assert_raises_rpc_error,
)

def create_transactions(node, address, amt, fees):
    # Create and sign raw transactions from node to address for amt.
    # Creates a transaction for each fee and returns an array
    # of the raw transactions.
    utxos = node.listunspent(0)

    # Create transactions
    inputs = []
    ins_total = 0
    for utxo in utxos:
        inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]})
        ins_total += utxo['amount']
        if ins_total > amt:
            break

    txs = []
    for fee in fees:
        outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee}
        raw_tx = node.createrawtransaction(inputs, outputs, 0, True)
        raw_tx = node.signrawtransactionwithwallet(raw_tx)
        txs.append(raw_tx)

    return txs

class WalletTest(SyscoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 2
        self.setup_clean_chain = True

    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    def run_test(self):
        self.nodes[0].importaddress(ADDRESS_WATCHONLY)
        # Check that nodes don't own any UTXOs
        assert_equal(len(self.nodes[0].listunspent()), 0)
        assert_equal(len(self.nodes[1].listunspent()), 0)

        self.log.info("Mining blocks ...")

        self.nodes[0].generate(1)
        self.sync_all()
        self.nodes[1].generate(1)
        self.nodes[1].generatetoaddress(101, ADDRESS_WATCHONLY)
        self.sync_all()

        assert_equal(self.nodes[0].getbalance(), 50)
        assert_equal(self.nodes[1].getbalance(), 50)

        self.log.info("Test getbalance with different arguments")
        assert_equal(self.nodes[0].getbalance("*"), 50)
        assert_equal(self.nodes[0].getbalance("*", 1), 50)
        assert_equal(self.nodes[0].getbalance("*", 1, True), 100)
        assert_equal(self.nodes[0].getbalance(minconf=1), 50)
        assert_equal(self.nodes[0].getbalance(minconf=0, include_watchonly=True), 100)
        assert_equal(self.nodes[1].getbalance(minconf=0, include_watchonly=True), 50)

        # Send 40 SYS from 0 to 1 and 60 SYS from 1 to 0.
        txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')])
        self.nodes[0].sendrawtransaction(txs[0]['hex'])
        self.nodes[1].sendrawtransaction(txs[0]['hex'])  # sending on both nodes is faster than waiting for propagation

        self.sync_all()
        txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), 60, [Decimal('0.01'), Decimal('0.02')])
        self.nodes[1].sendrawtransaction(txs[0]['hex'])
        self.nodes[0].sendrawtransaction(txs[0]['hex'])  # sending on both nodes is faster than waiting for propagation
        self.sync_all()

        # First argument of getbalance must be set to "*"
        assert_raises_rpc_error(-32, "dummy first argument must be excluded or set to \"*\"", self.nodes[1].getbalance, "")

        self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs")

        def test_balances(*, fee_node_1=0):
            # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions
            assert_equal(self.nodes[0].getbalance(), Decimal('9.99'))  # change from node 0's send
            assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1)  # change from node 1's send
            # Same with minconf=0
            assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99'))
            assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1)
            # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago
            # TODO: fix getbalance tracking of coin spentness depth
            assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0'))
            assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0'))
            # getunconfirmedbalance
            assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60'))  # output of node 1's spend
            assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60'))
            assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0'))  # Doesn't include output of node 0's send since it was spent
            assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0'))

        test_balances(fee_node_1=Decimal('0.01'))

        # Node 1 bumps the transaction fee and resends
        self.nodes[1].sendrawtransaction(txs[1]['hex'])
        self.nodes[0].sendrawtransaction(txs[1]['hex'])  # sending on both nodes is faster than waiting for propagation
        self.sync_all()

        self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs")
        test_balances(fee_node_1=Decimal('0.02'))

        self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY)
        self.sync_all()

        # balances are correct after the transactions are confirmed
        assert_equal(self.nodes[0].getbalance(), Decimal('69.99'))  # node 1's send plus change from node 0's send
        assert_equal(self.nodes[1].getbalance(), Decimal('29.98'))  # change from node 0's send

        # Send total balance away from node 1
        txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')])
        self.nodes[1].sendrawtransaction(txs[0]['hex'])
        self.nodes[1].generatetoaddress(2, ADDRESS_WATCHONLY)
        self.sync_all()

        # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago
        # TODO: fix getbalance tracking of coin spentness depth
        # getbalance with minconf=3 should still show the old balance
        assert_equal(self.nodes[1].getbalance(minconf=3), Decimal('0'))

        # getbalance with minconf=2 will show the new balance.
        assert_equal(self.nodes[1].getbalance(minconf=2), Decimal('0'))

        # check mempool transactions count for wallet unconfirmed balance after
        # dynamically loading the wallet.
        before = self.nodes[1].getunconfirmedbalance()
        dst = self.nodes[1].getnewaddress()
        self.nodes[1].unloadwallet('')
        self.nodes[0].sendtoaddress(dst, 0.1)
        self.sync_all()
        self.nodes[1].loadwallet('')
        after = self.nodes[1].getunconfirmedbalance()
        assert_equal(before + Decimal('0.1'), after)

if __name__ == '__main__':
    WalletTest().main()
<|endoftext|>


<commit_before>"""Delete user."""
# :license: MIT, see LICENSE for more details.

import click

import SoftLayer
from SoftLayer.CLI import environment
from SoftLayer.CLI import helpers

@click.command()
@click.argument('identifier')
@environment.pass_env
def cli(env, identifier):
    """Delete a User.

    Example: slcli user delete userId
    """

    mgr = SoftLayer.UserManager(env.client)

    user_id = helpers.resolve_id(mgr.resolve_ids, identifier, 'username')

    user_template = {'userStatusId': 1021}

    result = mgr.edit_user(user_id, user_template)
    if result:
        click.secho("%s deleted successfully" % identifier, fg='green')
    else:
        click.secho("Failed to delete %s" % identifier, fg='red')
<commit_msg>Updating help message<commit_after>"""Delete user."""
# :license: MIT, see LICENSE for more details.

import click

import SoftLayer
from SoftLayer.CLI import environment
from SoftLayer.CLI import helpers

@click.command()
@click.argument('identifier')
@environment.pass_env
def cli(env, identifier):
    """Sets a user's status to CANCEL_PENDING, which will immediately disable the account,

    and will eventually be fully removed from the account by an automated internal process.

    Example: slcli user delete userId
    """

    mgr = SoftLayer.UserManager(env.client)

    user_id = helpers.resolve_id(mgr.resolve_ids, identifier, 'username')

    user_template = {'userStatusId': 1021}

    result = mgr.edit_user(user_id, user_template)
    if result:
        click.secho("%s deleted successfully" % identifier, fg='green')
    else:
        click.secho("Failed to delete %s" % identifier, fg='red')
<|endoftext|>