<commit_before>from nodeconductor.logging.loggers import EventLogger, event_logger

class BackupEventLogger(EventLogger):
    resource = 'structure.Resource'

    class Meta:
        event_types = ('resource_backup_creation_scheduled',
                       'resource_backup_creation_succeeded',
                       'resource_backup_creation_failed',
                       'resource_backup_restoration_scheduled',
                       'resource_backup_restoration_succeeded',
                       'resource_backup_restoration_failed',
                       'resource_backup_deletion_scheduled',
                       'resource_backup_deletion_succeeded',
                       'resource_backup_deletion_failed',
                       'resource_backup_schedule_creation_succeeded',
                       'resource_backup_schedule_update_succeeded',
                       'resource_backup_schedule_deletion_succeeded',
                       'resource_backup_schedule_activated',
                       'resource_backup_schedule_deactivated')

class InstanceFlavorChangeEventLogger(EventLogger):
    resource = 'structure.Resource'
    flavor = 'openstack.Flavor'

    class Meta:
        event_types = ('resource_flavor_change_scheduled',
                       'resource_flavor_change_succeeded',
                       'resource_flavor_change_failed')

class InstanceVolumeChangeEventLogger(EventLogger):
    resource = 'structure.Resource'
    volume_size = int

    class Meta:
        nullable_fields = ['volume_size']
        event_types = ('resource_volume_extension_scheduled',
                       'resource_volume_extension_succeeded',
                       'resource_volume_extension_failed')

# TODO: move this handler to itacloud assembly
class TenantQuotaLogger(EventLogger):
    quota = 'quotas.Quota'
    service = 'structure.Service'
    project = 'structure.Project'
    project_group = 'structure.ProjectGroup'
    tenant = 'openstack.Tenant'
    threshold = float

    class Meta:
        nullable_fields = ['project_group']
        event_types = ('quota_threshold_reached',)

event_logger.register('openstack_backup', BackupEventLogger)
event_logger.register('openstack_flavor', InstanceFlavorChangeEventLogger)
event_logger.register('openstack_volume', InstanceVolumeChangeEventLogger)
event_logger.register('openstack_tenant_quota', TenantQuotaLogger)
<commit_msg>Use OpenStack instance instead of general purpose resource in logger.<commit_after>from nodeconductor.logging.loggers import EventLogger, event_logger

class BackupEventLogger(EventLogger):
    resource = 'openstack.Instance'

    class Meta:
        event_types = ('resource_backup_creation_scheduled',
                       'resource_backup_creation_succeeded',
                       'resource_backup_creation_failed',
                       'resource_backup_restoration_scheduled',
                       'resource_backup_restoration_succeeded',
                       'resource_backup_restoration_failed',
                       'resource_backup_deletion_scheduled',
                       'resource_backup_deletion_succeeded',
                       'resource_backup_deletion_failed',
                       'resource_backup_schedule_creation_succeeded',
                       'resource_backup_schedule_update_succeeded',
                       'resource_backup_schedule_deletion_succeeded',
                       'resource_backup_schedule_activated',
                       'resource_backup_schedule_deactivated')

class InstanceFlavorChangeEventLogger(EventLogger):
    resource = 'openstack.Instance'
    flavor = 'openstack.Flavor'

    class Meta:
        event_types = ('resource_flavor_change_scheduled',
                       'resource_flavor_change_succeeded',
                       'resource_flavor_change_failed')

class InstanceVolumeChangeEventLogger(EventLogger):
    resource = 'openstack.Instance'
    volume_size = int

    class Meta:
        nullable_fields = ['volume_size']
        event_types = ('resource_volume_extension_scheduled',
                       'resource_volume_extension_succeeded',
                       'resource_volume_extension_failed')

# TODO: move this handler to itacloud assembly
class TenantQuotaLogger(EventLogger):
    quota = 'quotas.Quota'
    service = 'structure.Service'
    project = 'structure.Project'
    project_group = 'structure.ProjectGroup'
    tenant = 'openstack.Tenant'
    threshold = float

    class Meta:
        nullable_fields = ['project_group']
        event_types = ('quota_threshold_reached',)

event_logger.register('openstack_backup', BackupEventLogger)
event_logger.register('openstack_flavor', InstanceFlavorChangeEventLogger)
event_logger.register('openstack_volume', InstanceVolumeChangeEventLogger)
event_logger.register('openstack_tenant_quota', TenantQuotaLogger)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###
# Copyright (c) 2012, spline
# All rights reserved.
#
#
###

# my libs
import urllib2
import urllib
import string
from collections import defaultdict # we use defaultdict vs. OrderedDict for compatability. Requires some workarounds.
try:
    import xml.etree.cElementTree as ElementTree
except ImportError:
    import xml.etree.ElementTree as ElementTree

# supybot libs
import supybot.utils as utils
from supybot.commands import *
import supybot.plugins as plugins
import supybot.ircutils as ircutils
import supybot.callbacks as callbacks
from supybot.i18n import PluginInternationalization, internationalizeDocstring

_ = PluginInternationalization('WolframAlpha')

@internationalizeDocstring
class WolframAlpha(callbacks.Plugin):
    """Add the help for "@plugin help WolframAlpha" here
    This should describe *how* to use this plugin."""
    threaded = True

    # http://products.wolframalpha.com/api/documentation.html
    def wolframalpha(self, irc, msg, args, optlist, optinput):
        """[--num #|--reinterpret|--usemetric|--shortest|--fulloutput] <input>

        Returns answer from Wolfram Alpha API based on input.

        Use --num number to display a specific amount of lines.
        Use --reinterpret to have WA logic to interpret question if not understood.
        Use --usemetric to not display in imperial units.
        Use --shortest for the shortest output (ignores lines).
        Use --fulloutput to display everything from the API (can flood).
        """

        # check for API key before we can do anything.
        apiKey = self.registryValue('apiKey')
        if not apiKey or apiKey == "Not set":
            irc.reply("Wolfram Alpha API key not set. see 'config help supybot.plugins.WolframAlpha.apiKey'.")
            return

        # first, url arguments, some of which getopts and config variables can manipulate.
        urlArgs = { 'input':optinput, 'appid':apiKey, 'reinterpret':'false', 'format':'plaintext', 'units':'nonmetric' }

        # check for config variables to manipulate URL arguments.
        if not self.registryValue('useImperial'):
            urlArgs['units'] = 'metric'
        if self.registryValue('reinterpretInput'):
            urlArgs['reinterpret'] = 'true'

        # args we use internally to control output.
        args = { 'maxoutput': self.registryValue('maxOutput'), 'shortest':None, 'fulloutput':None }

        # handle getopts.
        if optlist:
            for (key, value) in optlist:
                if key == 'shortest':
                    args['shortest'] = True
                if key == 'fulloutput':
                    args['fulloutput'] = True
                if key == 'num':
                    args['maxoutput'] = value
                if key == 'usemetric':
                    urlArgs['units'] = 'metric'
                if key == 'reinterpret':
                    urlArgs['reinterpret'] = 'true'

        # build url.
        url = 'http://api.wolframalpha.com/v2/query?' + urllib.urlencode(urlArgs)
        self.log.info(url)

        # try and query.
        try:
            request = urllib2.Request(url)
            u = urllib2.urlopen(request)
        except:
            irc.reply("Failed to load url: %s" % url)
            return

        # now try to process XML.
        try:
            tree = ElementTree.parse(u)
            document = tree.getroot()
        except:
            irc.reply("Something broke processing the XML.")
            return

        # check if we have an error. reports to irc but more detailed in the logs.
        if document.attrib['success'] == 'false' and document.attrib['error'] == 'true':
            errormsgs = []
            for error in document.findall('.//error'):
                errorcode = error.find('code').text
                errormsg = error.find('msg').text
                errormsgs.append("{0} - {1}".format(errorcode, errormsg))

            self.log.debug("ERROR processing request for: {0} message: {1}".format(optinput, errormsgs))
            irc.reply("Something went wrong processing request for: {0} ERROR: {1}".format(optinput, errormsgs))
            return
        elif document.attrib['success'] == 'false' and document.attrib['error'] == 'false': # no success but no error.
            errormsgs = []
            for error in document.findall('.//futuretopic'):
                errormsg = error.attrib['msg']
                errormsgs.append("FUTURE TOPIC: {0}".format(errormsg))

            for error in document.findall('.//didyoumeans'):
                errormsg = error.find('didyoumean').text
                errormsgs.append("Did you mean? {0}".format(errormsg))

            for error in document.findall('.//tips'):
                errormsg = error.find('tip').attrib['text'].text
                errormsgs.append("TIPS: {0}".format(errormsg))

            self.log.debug("ERROR with input: {0} API returned: {1}".format(optinput, errormsgs))
            irc.reply("ERROR with input: {0} API returned: {1}".format(optinput, errormsgs))
            return
        else: # this means we have success and no error messages.
            # each pod has a title, position and a number of subtexts. output contains the plaintext.
            # outputlist is used in sorting since defaultdict does not remember order/position.
            output = defaultdict(list)
            outputlist = {}

            for pod in document.findall('.//pod'):
                title = pod.attrib['title'].encode('utf-8')
                position = pod.attrib['position'].encode('utf-8')
                outputlist[position] = title
                for plaintext in pod.findall('.//plaintext'):
                    if plaintext.text:
                        output[title].append(plaintext.text.encode('utf-8').replace('\n',' '))

        # done processing the XML so lets work on the output.
        if len(output) < 1:
            irc.reply("Something went wrong looking up: {0}".format(optinput))
            return
        else: # rarely, output doesn't have things. validated so now output.
            if args['shortest']: # just show the question and answer.
                # outputlist has pod titles, sort, get the first and second title, string, fetch key.
                # other approach would be to use the keys above, which can be unreliable. Not every q/a has a set podtitle.
                question = output.get(str("".join(sorted(outputlist.values())[0])), None)
                answer = output.get(str("".join(sorted(outputlist.values())[1])), None)

                irc.reply("{0} :: {1}".format(string.join([item for item in question]), string.join([item for item in answer])))

            elif args['fulloutput']: # show everything. no limits.
                for k, v in sorted(outputlist.items()): # grab all values, sorted via the position number. output one per line.
                    itemout = "".join(output.get(v, None)) # output contains lists, so we need to join prior to going out.
                    if itemout is not None or len(itemout) > 0:
                        irc.reply("{0} :: {1}".format(v, itemout))

            else: # regular output, dictated by --lines or maxoutput.
                for q, k in enumerate(sorted(outputlist.keys())):
                    if q < args['maxoutput']:
                        itemout = "".join(output.get(outputlist[k], None)) # have the key, get the value, use for output.
                        if itemout is not None or len(itemout) > 0:
                            irc.reply("{0} :: {1}".format(outputlist[k], itemout))

    wolframalpha = wrap(wolframalpha, [getopts({'num':'int',
                                                'reinterpret':'',
                                                'usemetric':'',
                                                'shortest':'',
                                                'fulloutput':''}), 'text'])

Class = WolframAlpha

# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=250:
<commit_msg>Fix if value in output is empty.<commit_after># -*- coding: utf-8 -*-
###
# Copyright (c) 2012, spline
# All rights reserved.
#
#
###

# my libs
import urllib2
import urllib
import string
from collections import defaultdict # we use defaultdict vs. OrderedDict for compatability. Requires some workarounds.
try:
    import xml.etree.cElementTree as ElementTree
except ImportError:
    import xml.etree.ElementTree as ElementTree

# supybot libs
import supybot.utils as utils
from supybot.commands import *
import supybot.plugins as plugins
import supybot.ircutils as ircutils
import supybot.callbacks as callbacks
from supybot.i18n import PluginInternationalization, internationalizeDocstring

_ = PluginInternationalization('WolframAlpha')

@internationalizeDocstring
class WolframAlpha(callbacks.Plugin):
    """Add the help for "@plugin help WolframAlpha" here
    This should describe *how* to use this plugin."""
    threaded = True

    # http://products.wolframalpha.com/api/documentation.html
    def wolframalpha(self, irc, msg, args, optlist, optinput):
        """[--num #|--reinterpret|--usemetric|--shortest|--fulloutput] <input>

        Returns answer from Wolfram Alpha API based on input.

        Use --num number to display a specific amount of lines.
        Use --reinterpret to have WA logic to interpret question if not understood.
        Use --usemetric to not display in imperial units.
        Use --shortest for the shortest output (ignores lines).
        Use --fulloutput to display everything from the API (can flood).
        """

        # check for API key before we can do anything.
        apiKey = self.registryValue('apiKey')
        if not apiKey or apiKey == "Not set":
            irc.reply("Wolfram Alpha API key not set. see 'config help supybot.plugins.WolframAlpha.apiKey'.")
            return

        # first, url arguments, some of which getopts and config variables can manipulate.
        urlArgs = { 'input':optinput, 'appid':apiKey, 'reinterpret':'false', 'format':'plaintext', 'units':'nonmetric' }

        # check for config variables to manipulate URL arguments.
        if not self.registryValue('useImperial'):
            urlArgs['units'] = 'metric'
        if self.registryValue('reinterpretInput'):
            urlArgs['reinterpret'] = 'true'

        # args we use internally to control output.
        args = { 'maxoutput': self.registryValue('maxOutput'), 'shortest':None, 'fulloutput':None }

        # handle getopts.
        if optlist:
            for (key, value) in optlist:
                if key == 'shortest':
                    args['shortest'] = True
                if key == 'fulloutput':
                    args['fulloutput'] = True
                if key == 'num':
                    args['maxoutput'] = value
                if key == 'usemetric':
                    urlArgs['units'] = 'metric'
                if key == 'reinterpret':
                    urlArgs['reinterpret'] = 'true'

        # build url.
        url = 'http://api.wolframalpha.com/v2/query?' + urllib.urlencode(urlArgs)
        self.log.info(url)

        # try and query.
        try:
            request = urllib2.Request(url)
            u = urllib2.urlopen(request)
        except:
            irc.reply("Failed to load url: %s" % url)
            return

        # now try to process XML.
        try:
            tree = ElementTree.parse(u)
            document = tree.getroot()
        except:
            irc.reply("Something broke processing the XML.")
            return

        # check if we have an error. reports to irc but more detailed in the logs.
        if document.attrib['success'] == 'false' and document.attrib['error'] == 'true':
            errormsgs = []
            for error in document.findall('.//error'):
                errorcode = error.find('code').text
                errormsg = error.find('msg').text
                errormsgs.append("{0} - {1}".format(errorcode, errormsg))

            self.log.debug("ERROR processing request for: {0} message: {1}".format(optinput, errormsgs))
            irc.reply("Something went wrong processing request for: {0} ERROR: {1}".format(optinput, errormsgs))
            return
        elif document.attrib['success'] == 'false' and document.attrib['error'] == 'false': # no success but no error.
            errormsgs = []
            for error in document.findall('.//futuretopic'):
                errormsg = error.attrib['msg']
                errormsgs.append("FUTURE TOPIC: {0}".format(errormsg))

            for error in document.findall('.//didyoumeans'):
                errormsg = error.find('didyoumean').text
                errormsgs.append("Did you mean? {0}".format(errormsg))

            for error in document.findall('.//tips'):
                errormsg = error.find('tip').attrib['text'].text
                errormsgs.append("TIPS: {0}".format(errormsg))

            self.log.debug("ERROR with input: {0} API returned: {1}".format(optinput, errormsgs))
            irc.reply("ERROR with input: {0} API returned: {1}".format(optinput, errormsgs))
            return
        else: # this means we have success and no error messages.
            # each pod has a title, position and a number of subtexts. output contains the plaintext.
            # outputlist is used in sorting since defaultdict does not remember order/position.
            output = defaultdict(list)
            outputlist = {}

            for pod in document.findall('.//pod'):
                title = pod.attrib['title'].encode('utf-8')
                position = pod.attrib['position'].encode('utf-8')
                outputlist[position] = title
                for plaintext in pod.findall('.//plaintext'):
                    if plaintext.text:
                        output[title].append(plaintext.text.encode('utf-8').replace('\n',' '))

        # done processing the XML so lets work on the output.
        if len(output) < 1:
            irc.reply("Something went wrong looking up: {0}".format(optinput))
            return
        else: # rarely, output doesn't have things. validated so now output.
            if args['shortest']: # just show the question and answer.
                # outputlist has pod titles, sort, get the first and second title, string, fetch key.
                # other approach would be to use the keys above, which can be unreliable. Not every q/a has a set podtitle.
                question = output.get(str("".join(sorted(outputlist.values())[0])), None)
                answer = output.get(str("".join(sorted(outputlist.values())[1])), None)

                irc.reply("{0} :: {1}".format(string.join([item for item in question]), string.join([item for item in answer])))

            elif args['fulloutput']: # show everything. no limits.
                for k, v in sorted(outputlist.items()): # grab all values, sorted via the position number. output one per line.
                    itemout = output.get(v, None) # output contains lists, so we need to join prior to going out.
                    if itemout and itemout is not None:
                        irc.reply("{0} :: {1}".format(v, "".join(itemout)))

            else: # regular output, dictated by --lines or maxoutput.
                for q, k in enumerate(sorted(outputlist.keys())):
                    if q < args['maxoutput']:
                        itemout = output.get(outputlist[k], None) # have the key, get the value, use for output.
                        if itemout and itemout is not None:
                            irc.reply("{0} :: {1}".format(outputlist[k], "".join(itemout)))

    wolframalpha = wrap(wolframalpha, [getopts({'num':'int',
                                                'reinterpret':'',
                                                'usemetric':'',
                                                'shortest':'',
                                                'fulloutput':''}), 'text'])

Class = WolframAlpha

# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=250:
<|endoftext|>


<commit_before>import numpy as N

from odict import odict
from attributes import readonly

from neuroimaging import reorder, reverse, hasattrs

class CoordinateSystem(odict):
    "A simple class to carry around coordinate information in one bundle."

    class name (readonly):
        "coordinate system name"
        implements=str
    class axes (readonly):
        "list of axes"
        implements=tuple
        get=lambda _,s: s.values()
    class axisnames (readonly):
        "list of axis names"
        implements=tuple
        get=lambda _,s: s.keys()
    class ndim (readonly):
        "number of dimensions"
        get=lambda _,s: len(s.axes)

    def __init__(self, name, axes):
        self.name = name
        odict.__init__(self, [(ax.name, ax) for ax in axes])

    def __getitem__(self, axisname):
        try:
            return odict.__getitem__(self, axisname)
        except KeyError:
            raise KeyError(
              "axis '%s' not found, names are %s"%(axisname,self.keys()))

    def __setitem__(self, name, value):
        raise TypeError, "CoordinateSystem does not support axis assignment"

    def __eq__(self, other):
        if not hasattrs(other, "name", "axes"): return False
        return (self.name, self.axes)==(other.name, other.axes)

    def __str__(self):
        _dict = {'name': self.name, 'axes':self.axes}
        return `_dict`

    def reorder(self, name, order):
        """
        Given a name for the reordered coordinates, and a new order, return a
        reordered coordinate system.
        """
        if name is None: name = self.name
        return CoordinateSystem(name, reorder(self.axes, order))

    def reverse(self, name=None):
        if name is None: name = self.name
        return CoordinateSystem(name, reverse(self.axes))

    def hasaxis(self, name):
        return self.has_key(name)

    def getaxis(self, name):
        return self.get(name)

class VoxelCoordinateSystem(CoordinateSystem):
    """
    Coordinates with a shape -- assumed to be
    voxel coordinates, i.e. if shape = [3,4,5] then valid range
    interpreted as [0,2] X [0,3] X [0,4].
    """
    class shape (readonly): implements=list

    def __init__(self, name, axes, shape=None):
        if shape is None:
            try:
                self.shape = [dim.length for dim in axes]
            except:
                raise ValueError, 'must specify a shape or axes must have lengths'
        else:
            self.shape = list(shape)
        CoordinateSystem.__init__(self, name, axes)

    def isvalid(self, x):
        """
        Verify whether x is a valid (voxel) coordinate.
        """
        if not hasattr(self, '_box'):
            self._getbox()
        test = 1
        for i in range(self.ndim):
            test *= (N.greater_equal(x[i], self._box[i][0]) *
                     N.less_equal(x[i], self._box[i][1]))
        return test

    def _getbox(self):
        self._box = []
        for i in range(self.ndim):
            dim = self.axes[i]
            try:
                v = dim.values()
            except:
                v = range(self.shape[i])
            self._box.append(min(v), max(v))

class DiagonalCoordinateSystem(VoxelCoordinateSystem):

    def __init__(self, name, axes):
        shape = [dim.length for dim in axes]
        VoxelCoordinateSystem.__init__(self, name, axes, shape=shape)

    def transform(self):
        """
        Return an orthogonal homogeneous transformation matrix based on the
        step, start, length attributes of each axis.
        """
        value = N.zeros((self.ndim+1,)*2, N.float64)
        value[self.ndim, self.ndim] = 1.0
        for i in range(self.ndim):
            value[i,i] = self.axes[i].step
            value[i, self.ndim] = self.axes[i].start
        return value

<commit_msg>a DiagonalCoordinate system should derive from CoordinateSystem, not VoxelCoordinateSystem<commit_after>import numpy as N

from odict import odict
from attributes import readonly

from neuroimaging import reorder, reverse, hasattrs

class CoordinateSystem(odict):
    "A simple class to carry around coordinate information in one bundle."

    class name (readonly):
        "coordinate system name"
        implements=str
    class axes (readonly):
        "list of axes"
        implements=tuple
        get=lambda _,s: s.values()
    class axisnames (readonly):
        "list of axis names"
        implements=tuple
        get=lambda _,s: s.keys()
    class ndim (readonly):
        "number of dimensions"
        get=lambda _,s: len(s.axes)

    def __init__(self, name, axes):
        self.name = name
        odict.__init__(self, [(ax.name, ax) for ax in axes])

    def __getitem__(self, axisname):
        try:
            return odict.__getitem__(self, axisname)
        except KeyError:
            raise KeyError(
              "axis '%s' not found, names are %s"%(axisname,self.keys()))

    def __setitem__(self, name, value):
        raise TypeError, "CoordinateSystem does not support axis assignment"

    def __eq__(self, other):
        if not hasattrs(other, "name", "axes"): return False
        return (self.name, self.axes)==(other.name, other.axes)

    def __str__(self):
        _dict = {'name': self.name, 'axes':self.axes}
        return `_dict`

    def reorder(self, name, order):
        """
        Given a name for the reordered coordinates, and a new order, return a
        reordered coordinate system.
        """
        if name is None: name = self.name
        return CoordinateSystem(name, reorder(self.axes, order))

    def reverse(self, name=None):
        if name is None: name = self.name
        return CoordinateSystem(name, reverse(self.axes))

    def hasaxis(self, name):
        return self.has_key(name)

    def getaxis(self, name):
        return self.get(name)

class VoxelCoordinateSystem(CoordinateSystem):
    """
    Coordinates with a shape -- assumed to be
    voxel coordinates, i.e. if shape = [3,4,5] then valid range
    interpreted as [0,2] X [0,3] X [0,4].
    """
    class shape (readonly): implements=list

    def __init__(self, name, axes, shape=None):
        if shape is None:
            try:
                self.shape = [dim.length for dim in axes]
            except:
                raise ValueError, 'must specify a shape or axes must have lengths'
        else:
            self.shape = list(shape)
        CoordinateSystem.__init__(self, name, axes)

    def isvalid(self, x):
        """
        Verify whether x is a valid (voxel) coordinate.
        """
        if not hasattr(self, '_box'):
            self._getbox()
        test = 1
        for i in range(self.ndim):
            test *= (N.greater_equal(x[i], self._box[i][0]) *
                     N.less_equal(x[i], self._box[i][1]))
        return test

    def _getbox(self):
        self._box = []
        for i in range(self.ndim):
            dim = self.axes[i]
            try:
                v = dim.values()
            except:
                v = range(self.shape[i])
            self._box.append(min(v), max(v))

class DiagonalCoordinateSystem(CoordinateSystem):

    def __init__(self, name, axes):
        self.shape = [dim.length for dim in axes]
        CoordinateSystem.__init__(self, name, axes)

    def transform(self):
        """
        Return an orthogonal homogeneous transformation matrix based on the
        step, start, length attributes of each axis.
        """
        value = N.zeros((self.ndim+1,)*2, N.float64)
        value[self.ndim, self.ndim] = 1.0
        for i in range(self.ndim):
            value[i,i] = self.axes[i].step
            value[i, self.ndim] = self.axes[i].start
        return value

<|endoftext|>


<commit_before># Copyright (c) 2015 Uncharted Inc.
# http://www.uncharted.software/
#
# Released under the MIT License.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

__author__ = 'Chris Bethune'

#!/usr/bin/env python
#!/usr/bin/python

import urllib2
import json
import sys
import pytz
from dateutil.parser import parse

components = {'build', 'server', 'client', 'tiling', 'examples'}
types = {'feature', 'bug', 'doc', 'style', 'refactor', 'test', 'maintenance'}

prs_by_component = {}

# takes two args - start + end date for allowed PRs
# example format: 2014-05-01
start_date = pytz.utc.localize(parse(sys.argv[1]))
end_date = pytz.utc.localize(parse(sys.argv[2]))

# fetch the 100 most recent pulls
response = urllib2.urlopen('https://api.github.com/repos/unchartedsoftware/aperture-tiles/pulls?state=all&per_page=100')
pulls = json.loads(response.read())
response.close()

for pull in pulls:
    # Only process merged files
    merged_at_val = pull['merged_at']
    merge_date = None
    if merged_at_val is None:
        continue

    # Only process files merged between our supplied start and end dates
    merge_date = parse(merged_at_val)
    if start_date > merge_date or merge_date > end_date:
        continue

    # fetch the labels associated with the pull
    issue = pull['_links']['issue']['href']
    issue_response = urllib2.urlopen(issue)
    issue_data = json.loads(issue_response.read())
    issue_response.close()

    labels = issue_data['labels']
    # skip this entry when the label is NOLOG
    if 'NOLOG' in labels:
        continue

    # pull out type and component
    component = "other"
    change_type = "other"
    for label in labels:
        label_name = label['name']
        if label_name in components:
            component = label_name
        elif label_name in types:
            change_type = label_name

    # create an entry
    component_list = []
    if component in prs_by_component:
        component_list = prs_by_component[component]
    else:
        prs_by_component[component] = component_list

    component_list.append('#' + str(pull['number']) + ' ' + change_type + ': ' + pull['title'])

print '<Release Description Here>'
print '# Changes'
for component, pulls in prs_by_component.iteritems():
    print '\n'
    print '## ' + component
    for pull in pulls:
        print '* ' + pull<commit_msg>updates labels to match github<commit_after># Copyright (c) 2015 Uncharted Inc.
# http://www.uncharted.software/
#
# Released under the MIT License.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

__author__ = 'Chris Bethune'

#!/usr/bin/env python
#!/usr/bin/python

import urllib2
import json
import sys
import pytz
from dateutil.parser import parse

components = {'a1 - build', 'a2 - maintenance', 'a3 - binning', 'a4 -tiling', 'a5 - server', 'a6 - client','a7 - examples', 'a8 - doc', 'a9 - test'}
types = {'t1 - bug', 't2 - feature', 't3 - refactor', 't4 - style', 't5 - question'}

prs_by_component = {}

# takes two args - start + end date for allowed PRs
# example format: 2014-05-01
start_date = pytz.utc.localize(parse(sys.argv[1]))
end_date = pytz.utc.localize(parse(sys.argv[2]))

# fetch the 100 most recent pulls
response = urllib2.urlopen('https://api.github.com/repos/unchartedsoftware/aperture-tiles/pulls?state=all&per_page=100')
pulls = json.loads(response.read())
response.close()

for pull in pulls:
    # Only process merged files
    merged_at_val = pull['merged_at']
    merge_date = None
    if merged_at_val is None:
        continue

    # Only process files merged between our supplied start and end dates
    merge_date = parse(merged_at_val)
    if start_date > merge_date or merge_date > end_date:
        continue

    # fetch the labels associated with the pull
    issue = pull['_links']['issue']['href']
    issue_response = urllib2.urlopen(issue)
    issue_data = json.loads(issue_response.read())
    issue_response.close()

    labels = issue_data['labels']
    # skip this entry when the label is NOLOG
    if 'NOLOG' in labels:
        continue

    # pull out type and component
    component = "other"
    change_type = "other"
    for label in labels:
        label_name = label['name']
        if label_name in components:
            component = label_name
        elif label_name in types:
            change_type = label_name

    # create an entry
    component_list = []
    if component in prs_by_component:
        component_list = prs_by_component[component]
    else:
        prs_by_component[component] = component_list

    component_list.append('#' + str(pull['number']) + ' ' + change_type + ': ' + pull['title'])

print '<Release Description Here>'
print '# Changes'
for component, pulls in prs_by_component.iteritems():
    print '\n'
    print '## ' + component
    for pull in pulls:
        print '* ' + pull<|endoftext|>


<commit_before>from piston.handler import BaseHandler
from emitters import *
from piston.utils import rc
from util.helpers import parse_polygon_wkt, merge_dict
from django.contrib.gis.geos.geometry import GEOSGeometry
import datetime
from climatedata import models
import inspect
from util.raw_sql import get_dataset, execute
import netCDF4
from slug import *
import climatedata.models
from experimental.in_memory_oo_update import OcgDataset
from util.ncconv.in_memory_oo_multi_core import multipolygon_multicore_operation

class ocg(object):
    """Structure class to hold keyword arguments."""

    def __repr__(self):
        prints = []
        mems = inspect.getmembers(self)
        for mem in mems:
            if not mem[0].startswith('__'):
                prints.append('{0}={1}\n'.format(*mem))
        return(''.join(prints))

class OpenClimateHandler(BaseHandler):
    """Superclass for all OpenClimate handlers."""

    allowed_methods = ('GET',)

    def __init__(self,*args,**kwds):
        ## set some default parameters for the handlers
        self.ocg = ocg()

        super(OpenClimateHandler,self).__init__(*args,**kwds)

    def read(self,request,**kwds):
        """
        Subclasses should not overload this method. Each return will be checked
        for basic validity.
        """
#        import ipdb;ipdb.set_trace()
        ## parse query
#        self._query_string_(request)
        ## parse URL arguments
        self._parse_slugs_(kwds)
        ## add OCG object to request object for use by emitters
        request.ocg = self.ocg
#        import ipdb;ipdb.set_trace()
        ## call the subclass read methods
        return self.check(self._read_(request))

    def check(self,payload):
        """Basic checks on returned data."""

        if len(payload) == 0:
            return rc.NOT_FOUND
        else:
            return payload

    def _read_(self,request,**kwds):
        """Overload in subclasses."""

        raise NotImplementedError

    def _parse_slugs_(self,kwds):

        self.ocg.temporal = TemporalSlug('temporal',possible=kwds).value
        self.ocg.aoi = PolygonSlug('aoi',possible=kwds).value
        self.ocg.aggregate = AggregateSlug('aggregate',possible=kwds).value
        self.ocg.operation = OperationSlug('operation',possible=kwds).value
        self.ocg.html_template = OcgSlug('html_template',possible=kwds).value
        self.ocg.run = IntegerSlug('run',possible=kwds).value

        self.ocg.variable = IExactQuerySlug(models.Variable,'variable',possible=kwds,one=True).value
        self.ocg.scenario = IExactQuerySlug(models.Scenario,'scenario',possible=kwds,one=True).value
        self.ocg.archive = IExactQuerySlug(models.Archive,'archive',possible=kwds,one=True).value
        self.ocg.climate_model = IExactQuerySlug(models.ClimateModel,'model',possible=kwds,one=True).value

        ## return the dataset object if all components passed
        if all([self.ocg.scenario,self.ocg.archive,self.ocg.climate_model,self.ocg.run,self.ocg.variable]):
            fkwds = dict(archive=self.ocg.archive,
                         scenario=self.ocg.scenario,
                         climate_model=self.ocg.climate_model,
                         run=self.ocg.run,
                         variable=self.ocg.variable)
            qs = climatedata.models.SimulationOutput.objects.filter(**fkwds)
            assert(len(qs) == 1)
            self.ocg.simulation_output = qs[0]
        else:
            self.ocg.simulation_output = None

#        import ipdb;ipdb.set_trace()

#    def _parse_kwds_(self,kwds):
#        """Parser and formatter for potential URL keyword arguments."""
#
#        def _format_date_(start,end):
#            return([datetime.datetime.strptime(d,'%Y-%m-%d') for d in [start,end]])
#        def _get_iexact_(model,code):
##            if code == 'ps': import ipdb;ipdb.set_trace()
#            "Return a single record from the database. Otherwise raise exception."
#            ## this is the null case and should be treated as such
#            if code == None:
#                ret = None
#            else:
#                obj = model.objects.filter(code__iexact=code)
#                if len(obj) != 1:
#                    obj = model.objects.filter(name__iexact=code)
#                    if len(obj) != 1:
#                        msg = '{0} records returned for model {1} with code query {2}'.format(len(obj),model,code)
#                        raise ValueError(msg)
#                    else:
#                        ret = obj[0]
#                else:
#                    ret = obj[0]
#            return(ret)
#
#        ## name of the scenario
#        self.ocg.scenario = kwds.get('scenario')
#        ## the temporal arguments
#        t = kwds.get('temporal')
#        if t != None:
#            if '+' in t:
#                start,end = t.split('+')
#            else:
#                start = t
#                end = t
#                self.ocg.temporal = _format_date_(start,end)
#        else:
#            self.ocg.temporal = None
#        ## the polygon overlay
#        aoi = kwds.get('aoi')
#        if aoi:
#            self.ocg.aoi = GEOSGeometry(parse_polygon_wkt(aoi))
#        else:
#            self.ocg.aoi = None
#        ## the model archive
#        self.ocg.archive = kwds.get('archive')
#        ## target variable
#        self.ocg.variable = kwds.get('variable')
#        ## aggregation boolean
#        agg = kwds.get('aggregate')
#        ## the None case is different than 'true' or 'false'
#        if agg:
#            if agg.lower() == 'true':
#                self.ocg.aggregate = True
#            elif agg.lower() == 'false':
#                self.ocg.aggregate = False
#            else:
#                msg = '"{0}" aggregating boolean operation not recognized.'
#                raise(ValueError(msg.format(agg)))
#        else:
#            self.ocg.aggregate = None
#        ## the model designation
#        self.ocg.model = kwds.get('model')
#        ## the overlay operation
#        self.ocg.operation = kwds.get('operation')
#
#        ## these queries return objects from the database classifying the NetCDF.
#        ## the goal is to return the prediction.
#        if self.ocg.archive:
#            self.ocg.archive_obj = _get_iexact_(models.Archive,self.ocg.archive)
#            self.ocg.climatemodel_obj = models.ClimateModel.objects.filter(archive=self.ocg.archive_obj,
#                                                                           code__iexact=self.ocg.model)[0]
##        self.ocg.climatemodel_obj = _get_iexact_(models.ClimateModel,self.ocg.model)
##        self.ocg.scenario_obj = _get_iexact_(models.Scenario,self.ocg.scenario)
##        self.ocg.variable_obj = _get_iexact_(models.Variable,self.ocg.variable)
#        ## if we have data for each component, we can return a prediction
#        if all([self.ocg.archive,self.ocg.model,self.ocg.scenario,self.ocg.variable,self.ocg.temporal]):
#            ## execute the raw sql select statement to return the dataset(s)
#            sql = get_dataset(self.ocg.archive_obj.id,
#                              self.ocg.variable,
#                              self.ocg.scenario,
#                              self.ocg.temporal,
#                              self.ocg.climatemodel_obj.code)
#            rows = execute(sql)
#            ## return the dataset object(s)
##            if len(rows) > 1:
#            pks = [ii[0] for ii in rows]
#            self.ocg.dataset_obj = models.Dataset.objects.filter(pk__in=pks)
#
#
##            else:
##                self.ocg.dataset_obj = [models.Dataset.objects.filter(pk=rows[0][0])[0]]
#
##            ## return potential climate models from the archive selection
##            cms = ClimateModel.objects.filter(archive=self.ocg.archive_obj)
##            ## find potential datasets
##            datasets = Dataset.objects.filter(climatemodel__in=cms,
##                                              scenario=self.ocg.scenario_obj)
##            ## filter by variable
##            var_dataset_ids = Variable.objects.filter()
##            ## narrow down the search by looking through the time data
##            dataset = IndexTime.objects.filter(dataset__in=datasets,
##                                                  value__range=self.ocg.temporal).\
##                                           values('dataset').\
##                                           distinct()
##            import ipdb;ipdb.set_trace()
##            fkwds = dict(archive=self.ocg.archive_obj,
##                         climate_model=self.ocg.model_obj,
##                         experiment=self.ocg.scenario_obj,
##                         variable=self.ocg.variable_obj)
##            self.ocg.prediction_obj = models.Prediction.objects.filter(**fkwds)
##            if len(self.ocg.prediction_obj) != 1:
##                raise ValueError('prediction query should return 1 record.')
##            self.ocg.prediction_obj = self.ocg.prediction_obj[0]
#        else:
#            self.ocg.dataset_obj = None

class NonSpatialHandler(OpenClimateHandler):
    __data_kwds__ = {}

    def _read_(self,request,code=None):
        if code:
            query = self.model.objects.filter(code__iexact=str(code))
        else:
            query = self.model.objects.all()
        return query

#class ArchiveHandler(NonSpatialHandler):
#    model = Archive
#
#
#class ClimateModelHandler(NonSpatialHandler):
#    model = ClimateModel
#
#
#class ExperimentHandler(NonSpatialHandler):
#    model = Scenario
#
#
#class VariableHandler(NonSpatialHandler):
#    model = Variable

class SpatialHandler(OpenClimateHandler):

    def _read_(self,request):

#        import ipdb;ipdb.set_trace()

        dataset = self.ocg.simulation_output.netcdf_variable.netcdf_dataset

        ## arguments for the dataset object
        kwds = dict(rowbnds_name=dataset.rowbnds_name,
                    colbnds_name=dataset.colbnds_name,
                    time_name=dataset.time_name,
                    time_units=dataset.time_units,
                    calendar=dataset.calendar,)
#                    level_name=self.ocg.dataset.level_name,) TODO: add level variable name

        ## construct arguments to clip operation
        if self.ocg.operation == 'clip':
            clip = True
        else:
            clip = False

#        import ipdb;ipdb.set_trace()

        elements = multipolygon_multicore_operation(dataset.uri,
                                      self.ocg.simulation_output.netcdf_variable.code,
                                      [self.ocg.aoi],
                                      time_range=self.ocg.temporal,
                                      clip=clip,
                                      dissolve=self.ocg.aggregate,
                                      levels=None,
                                      ocgOpts=kwds,
                                      subdivide=True,
                                      #subres = 90
                                      )

#        import ipdb;ipdb.set_trace()

#        try:
#            dataset = netCDF4.Dataset(self.ocg.dataset.uri,'r')
#            d = OcgDataset(dataset,**kwds)
#            ## pull the elements
#            elements = d.extract_elements(self.ocg.variable.title(), # TODO: variable formatting
#                                          dissolve=self.ocg.aggregate,
#                                          polygon=self.ocg.aoi,
#                                          time_range=self.ocg.temporal,
#                                          clip=self.ocg.operation)
#        ## close the connection...
#        finally:
#            dataset.close()

        return(elements)

#        self.rowbnds_name = kwds.get('rowbnds_name') or 'bounds_latitude'
#        self.colbnds_name = kwds.get('colbnds_name') or 'bounds_longitude'
#        self.time_name = kwds.get('time_name') or 'time'
#        self.time_units = kwds.get('time_units') or 'days since 1950-01-01 00:00:00'
#        self.calendar = kwds.get('calendar') or 'proleptic_gregorian'
#        self.level_name = kwds.get('level_name') or 'levels'

#        return(climatedata.models.Dataset.objects.all())

#        from tests import get_example_netcdf
#
#        attrs = get_example_netcdf()

#        ## SPATIAL QUERYING ----------------------------------------------------
#
#        ocgeom = OpenClimateGeometry(self.ocg.climatemodel_obj,
#                                     self.ocg.aoi,
#                                     self.ocg.operation,
#                                     self.ocg.aggregate)
#        geom_list = ocgeom.get_geom_list()
#        row,col = ocgeom.get_indices()
#        weights = ocgeom.get_weights()
#
#        ## it is possible that multiple datasets are returned by a query. this
#        ## occurs when a temporal range overlaps multiple datasets. hence, we
#        ## always put the retrieval in a loop.
#
#        for ii,dataset in enumerate(self.ocg.dataset_obj):
#
#            ## TEMPORAL QUERYING -----------------------------------------------
#
#            ti = IndexTime.objects.filter(dataset=dataset)\
#                                  .filter(value__range=self.ocg.temporal)
#            ti = ti.order_by('index').values_list('index',flat=True)
#
#            ## RETRIEVE NETCDF DATA --------------------------------------------
#
#            rootgrp = netCDF4.Dataset(dataset.uri,'r')
#            try:
#                na = NetCdfAccessor(rootgrp,self.ocg.variable)
#                ## extract a dictionary representation of the netcdf
#                dl = na.get_dict(geom_list,
#                                 time_indices=ti,
#                                 row=row,
#                                 col=col,
#                                 aggregate=self.ocg.aggregate,
#                                 weights=weights)
#                ## append the data
#                if ii == 0:
#                    dls = dl
#                else:
#                    dls += dl
#            finally:
#                rootgrp.close()
#
#        return(dls)

#class OpenClimateGeometry(object):
#    """
#    Perform OpenClimateGIS geometry operations. Manages clip v. intersect
#    operations and spatial unioning in the case of an aggregation.
#
#    aoi -- GEOSGeometry Polygon object acting as the geometric selection overlay.
#    op -- 'intersect(s)' or 'clip'
#    aggregate -- set to True to union the geometries.
#    """
#
#    def __init__(self,climatemodel,aoi,op,aggregate):
#        self.aoi = aoi
#        self.op = op
#        self.aggregate = aggregate
#        self.climatemodel = climatemodel
#
#        self.__qs = None ## queryset with correct spatial operations
#        self.__geoms = None ## list of geometries with correct attribute selected
#
#        ## set the geometry attribute depending on the operation
#        if op in ['intersect','intersects']:
#            self._gattr = 'geom'
#        elif op == 'clip':
#            self._gattr = 'intersection'
#        else:
#            msg = 'spatial operation "{0}" not recognized.'.format(op)
#            raise NotImplementedError(msg)
#
#    def get_indices(self):
#        "Returning row and column indices used to index into NetCDF."
#
#        row = [obj.row for obj in self._qs]
#        col = [obj.col for obj in self._qs]
#        return((row,col))
#
#    def get_weights(self):
#        "Returns weights for each polygon in the case of an aggregation."
#
##        if self.aggregate:
#            ## calculate weights for each polygon
#        areas = [obj.area for obj in self._geoms]
#        asum = sum(areas)
#        weights = [a/asum for a in areas]
#        weights = [dict(weight=w,row=obj.row,col=obj.col) for w,obj in zip(weights,self._qs)]
##            weights = dict(zip(weights,
##                               [dict(row=obj.row,col=obj.col) for obj in self._qs]))
##            import ipdb;ipdb.set_trace()
##            ret = weights
##        else:
##            ret = None
#        return(weights)
#
#    def get_geom_list(self):
#        "Return the list of geometries accounting for processing parms."
#
#        if self.aggregate:
#            ret = self._union_geoms_()
#        else:
#            ret = self._geoms
#        return(ret)
#
#    @property
#    def _qs(self):
#        if self.__qs == None:
#            ## always perform the spatial select to limit returned records.
#            self.__qs = IndexSpatial.objects.filter(climatemodel=self.climatemodel,
#                                                    geom__intersects=self.aoi)\
#                                            .order_by('row','col')
#            ## intersection operations require element-by-element intersection
#            ## operations.
#            if self.op == 'clip':
#                self.__qs = self.__qs.intersection(self.aoi)
#        return(self.__qs)
#
#    @property
#    def _geoms(self):
#        if self.__geoms == None:
#            self.__geoms = [getattr(obj,self._gattr) for obj in self._qs]
#        return(self.__geoms)
#
#    def _union_geoms_(self):
#        "Returns the union of geometries in the case of an aggregation."
#
#        first = True
#        for geom in self._geoms:
#            if first:
#                union = geom
#                first = False
#            else:
#                union = union.union(geom)
#        return(union)
<commit_msg>cleaned up handlers.py<commit_after>from piston.handler import BaseHandler
from emitters import *
from piston.utils import rc
from climatedata import models
import inspect
from slug import *
import climatedata.models
from util.ncconv.in_memory_oo_multi_core import multipolygon_multicore_operation

class ocg(object):
    """Structure class to hold keyword arguments."""

    def __repr__(self):
        prints = []
        mems = inspect.getmembers(self)
        for mem in mems:
            if not mem[0].startswith('__'):
                prints.append('{0}={1}\n'.format(*mem))
        return(''.join(prints))

class OpenClimateHandler(BaseHandler):
    """Superclass for all OpenClimate handlers."""

    allowed_methods = ('GET',)

    def __init__(self,*args,**kwds):
        ## set some default parameters for the handlers
        self.ocg = ocg()

        super(OpenClimateHandler,self).__init__(*args,**kwds)

    def read(self,request,**kwds):
        """
        Subclasses should not overload this method. Each return will be checked
        for basic validity.
        """

        ## parse URL arguments
        self._parse_slugs_(kwds)
        ## add OCG object to request object for use by emitters
        request.ocg = self.ocg
        ## call the subclass read methods
        return self.check(self._read_(request))

    def check(self,payload):
        """Basic checks on returned data."""

        if len(payload) == 0:
            return rc.NOT_FOUND
        else:
            return payload

    def _read_(self,request,**kwds):
        """Overload in subclasses."""

        raise NotImplementedError

    def _parse_slugs_(self,kwds):

        self.ocg.temporal = TemporalSlug('temporal',possible=kwds).value
        self.ocg.aoi = PolygonSlug('aoi',possible=kwds).value
        self.ocg.aggregate = AggregateSlug('aggregate',possible=kwds).value
        self.ocg.operation = OperationSlug('operation',possible=kwds).value
        self.ocg.html_template = OcgSlug('html_template',possible=kwds).value
        self.ocg.run = IntegerSlug('run',possible=kwds).value

        self.ocg.variable = IExactQuerySlug(models.Variable,'variable',possible=kwds,one=True).value
        self.ocg.scenario = IExactQuerySlug(models.Scenario,'scenario',possible=kwds,one=True).value
        self.ocg.archive = IExactQuerySlug(models.Archive,'archive',possible=kwds,one=True).value
        self.ocg.climate_model = IExactQuerySlug(models.ClimateModel,'model',possible=kwds,one=True).value

        ## return the dataset object if all components passed
        if all([self.ocg.scenario,self.ocg.archive,self.ocg.climate_model,self.ocg.run,self.ocg.variable]):
            fkwds = dict(archive=self.ocg.archive,
                         scenario=self.ocg.scenario,
                         climate_model=self.ocg.climate_model,
                         run=self.ocg.run,
                         variable=self.ocg.variable)
            qs = climatedata.models.SimulationOutput.objects.filter(**fkwds)
            assert(len(qs) == 1)
            self.ocg.simulation_output = qs[0]
        else:
            self.ocg.simulation_output = None

class NonSpatialHandler(OpenClimateHandler):
    __data_kwds__ = {}

    def _read_(self,request,code=None):
        if code:
            query = self.model.objects.filter(code__iexact=str(code))
        else:
            query = self.model.objects.all()
        return query

#class ArchiveHandler(NonSpatialHandler):
#    model = Archive
#
#
#class ClimateModelHandler(NonSpatialHandler):
#    model = ClimateModel
#
#
#class ExperimentHandler(NonSpatialHandler):
#    model = Scenario
#
#
#class VariableHandler(NonSpatialHandler):
#    model = Variable

class SpatialHandler(OpenClimateHandler):

    def _read_(self,request):
        dataset = self.ocg.simulation_output.netcdf_variable.netcdf_dataset

        ## arguments for the dataset object
        kwds = dict(rowbnds_name=dataset.rowbnds_name,
                    colbnds_name=dataset.colbnds_name,
                    time_name=dataset.time_name,
                    time_units=dataset.time_units,
                    calendar=dataset.calendar,)
#                    level_name=self.ocg.dataset.level_name,) TODO: add level variable name

        ## construct arguments to clip operation
        if self.ocg.operation == 'clip':
            clip = True
        else:
            clip = False

        elements = multipolygon_multicore_operation(dataset.uri,
                                      self.ocg.simulation_output.netcdf_variable.code,
                                      [self.ocg.aoi],
                                      time_range=self.ocg.temporal,
                                      clip=clip,
                                      dissolve=self.ocg.aggregate,
                                      levels=None,
                                      ocgOpts=kwds,
                                      subdivide=True,
                                      #subres = 90
                                      )

#        import ipdb;ipdb.set_trace()

#        try:
#            dataset = netCDF4.Dataset(self.ocg.dataset.uri,'r')
#            d = OcgDataset(dataset,**kwds)
#            ## pull the elements
#            elements = d.extract_elements(self.ocg.variable.title(), # TODO: variable formatting
#                                          dissolve=self.ocg.aggregate,
#                                          polygon=self.ocg.aoi,
#                                          time_range=self.ocg.temporal,
#                                          clip=self.ocg.operation)
#        ## close the connection...
#        finally:
#            dataset.close()

        return(elements)
<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from .models import User

admin.site.register(User, UserAdmin)
<commit_msg>Fix admin forms for our custom User model<commit_after>from django import forms
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.forms import UserCreationForm, UserChangeForm
from .models import User

class MyUserCreationForm(UserCreationForm):
    class Meta(UserCreationForm.Meta):
        model = User

    def clean_username(self):
        username = self.cleaned_data["username"]
        try:
            User.objects.get(username=username)
        except User.DoesNotExist:
            return username
        raise forms.ValidationError(self.error_messages['duplicate_username'])

class MyUserChangeForm(UserChangeForm):
    class Meta(UserChangeForm.Meta):
        model = User

class MyUserAdmin(UserAdmin):
    form = MyUserChangeForm
    add_form = MyUserCreationForm

admin.site.register(User, MyUserAdmin)
<|endoftext|>


<commit_before>from flask import Flask, Blueprint, jsonify, request, abort, current_app, \
        make_response
from sqlalchemy.sql import func
from friendbot.models import Friend
from slackclient import SlackClient
import re

friendbot = Blueprint('fb', __name__)

@friendbot.route('/friendbot', methods=['POST'])
def index():
    TOKEN = current_app.config['SLACK_VERIFY_TOKEN']
    form = request.get_json()

    # current_app.logger.error(form)

    if form['token'] != TOKEN:
        abort(403)

    type = form['type']

    if type == 'url_verification':
        return jsonify({'challenge': form['challenge']})

    elif type == 'event_callback':
        event = form['event']
        event_type = event['type']

        if event.get('subtype', '') == 'bot_message':
            return make_response("Please don't respond to other bots", 200,)

        if event_type == 'message':
            text = event['text']

            if text.startswith('!') and text.endswith('bot'):
                friend = Friend.query.filter_by(name=text[1:-3]).first()

            elif text.startswith('<@'):
                # This is a file upload, ignore it
                if ('uploaded a file:' in text or
                        'commented on' in text or
                        'set the' in text):
                    return make_response('Ignoring', 200,)

                text = re.sub('>.*$', '', text)

                friend = Friend.query.filter_by(slack_id=text[2:]).first()

                if friend is None:
                    return make_response('Friend not found', 200,)
            else:
                return make_response('Not a friendbot request', 200,)

            output = friend.phrases.order_by(func.random()).first().line
            output = output.replace('${newline}', '\n')

            sc = SlackClient(current_app.config['SLACK_BOT_TOKEN'])

            name = friend.name

            args = {
                'username': name + 'bot',
                'channel': event['channel'],
                'text': output,
                'parse': 'full'
            }

            if name != 'friendbot':
                args['icon_url'] = 'https://maxtimkovich.com/img/friends/{}.jpg'.format(name)

            sc.api_call(
                'chat.postMessage',
                **args
            )

            return make_response('Mission accomplished', 200,)

    return make_response('Not for friendbot', 200,)
<commit_msg>making friendbot do stupid things<commit_after>from flask import Flask, Blueprint, jsonify, request, abort, current_app, \
        make_response
from sqlalchemy.sql import func
from friendbot.models import Friend
from slackclient import SlackClient
import re

friendbot = Blueprint('fb', __name__)

def get_phrase(text, event):
    if text.startswith('!') and text.endswith('bot'):
        friend = Friend.query.filter_by(name=text[1:-3]).first()

    elif text.startswith('<@'):
        # This is a file upload, ignore it
        if ('uploaded a file:' in text or
                'commented on' in text or
                'set the' in text):
            return make_response('Ignoring', 200,)

        text = re.sub('>.*$', '', text)

        friend = Friend.query.filter_by(slack_id=text[2:]).first()

        if friend is None:
            return make_response('Friend not found', 200,)
    else:
        return make_response('Not a friendbot request', 200,)

    output = friend.phrases.order_by(func.random()).first().line
    output = output.replace('${newline}', '\n')

    sc = SlackClient(current_app.config['SLACK_BOT_TOKEN'])

    name = friend.name

    args = {
        'username': name + 'bot',
        'channel': event['channel'],
        'text': output,
        'parse': 'full'
    }

    if name != 'friendbot':
        args['icon_url'] = 'https://maxtimkovich.com/img/friends/{}.jpg'.format(name)

    sc.api_call(
        'chat.postMessage',
        **args
    )

    return make_response('Mission accomplished', 200,)

@friendbot.route('/friendbot', methods=['POST'])
def index():
    TOKEN = current_app.config['SLACK_VERIFY_TOKEN']
    form = request.get_json()

    current_app.logger.error(form)

    if form['token'] != TOKEN:
        abort(403)

    type = form['type']

    if type == 'url_verification':
        return jsonify({'challenge': form['challenge']})

    elif type == 'event_callback':
        event = form['event']
        event_type = event['type']

        if event.get('subtype', '') == 'bot_message':
            return make_response("Please don't respond to other bots", 200,)

        if event_type == 'message':
            text = event.get('text', '')

            if ((text.startswith('!') and text.endswith('bot')) or
                    text.startswith('<@')):
                return get_phrase(text, event)
            elif 'max' in text.lower():
                sc = SlackClient(current_app.config['SLACK_BOT_TOKEN'])

                sc.api_call(
                    'reactions.add',
                    channel=event['channel'],
                    timestamp=event['ts'],
                    name='max',
                )

                return make_response("Mission Complete!", 200,)

    return make_response('Not for friendbot', 200,)
<|endoftext|>


<commit_before>__author__ = "Christian Kongsgaard"
__license__ = 'MIT'

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules
import sys
import os
import numpy as np

try:
    import matplotlib.pyplot as plt
except ModuleNotFoundError:
    pass

# RiBuild Modules
from delphin_6_automation.logging.ribuild_logger import ribuild_logger
from delphin_6_automation.sampling import sampling
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions import sampling_interactions
from delphin_6_automation.database_interactions.db_templates import sample_entry

# Logger
logger = ribuild_logger()

# -------------------------------------------------------------------------------------------------------------------- #
# RIBuild

def main():
    """Entrance to the sampling worker"""

    print_header()

    try:
        while True:
            menu()
    except KeyboardInterrupt:
        sys.exit()

def print_header():
    print('---------------------------------------------------')
    print('|                                                  |')
    print('|           RiBuild EU Research Project            |')
    print('|           for Hygrothermal Simulations           |')
    print('|                                                  |')
    print('|                 WORK IN PROGRESS                 |')
    print('|               Sampling Environment               |')
    print('|                                                  |')
    print('---------------------------------------------------')

def menu():
    print('')
    print('------------------- SAMPLING MENU ---------------------')
    print('')
    print("Available Actions:")
    print("[a] Start Sampling")
    print("[b] Create Sampling Strategy")
    print("[c] View Current Progress")
    print("[x] Exit")

    choice = input("> ").strip().lower()

    if choice == 'a':
        logger.debug('starting sampling')
        sampling_strategy_id = input("Define sampling strategy ID > ")
        logger.debug(sampling_strategy_id)

        logger.info('\nStarting sampling\n')
        sampling_worker(sampling_strategy_id)

    elif choice == 'b':
        strategy = sampling.create_sampling_strategy(os.path.dirname(__file__))
        strategy_id = sampling_interactions.upload_sampling_strategy(strategy)
        logger.info(f'Created sampling and uploaded it with ID: {strategy_id}')

    elif choice == 'c':
        sampling_strategy_id = input("Define sampling strategy ID > ")
        sampling_overview(sampling_strategy_id)

    elif choice == 'x':
        logger.info("Goodbye")

def sampling_worker(strategy_id: str) -> None:
    """
    Main sampling worker, which creates new Delphin projects based on a sample strategy and checks for convergence,
    when all Delphin simulations in a given iteration is done.
    If convergence was not reached then it creates new Delphin projects.
    """

    strategy_doc = sampling_interactions.get_sampling_strategy(strategy_id)
    (sample_iteration, convergence,
     new_samples_per_set, used_samples_per_set) = sampling.initialize_sampling(strategy_doc)

    # Run loop
    while not convergence:
        logger.info(f'Running sampling iteration #{sample_iteration}')
        logger.info(f'New Samples per set: {new_samples_per_set}')
        logger.info(f'Used samples per set: {used_samples_per_set}')

        strategy_doc.reload()
        existing_sample = sampling.sample_exists(strategy_doc)

        if not existing_sample:
            delphin_ids, sample_id = create_new_samples(strategy_doc, used_samples_per_set, sample_iteration)

        else:
            logger.info('Found existing sample')
            delphin_ids = sampling_interactions.get_delphin_for_sample(existing_sample)
            sample_id = existing_sample.id

        is_all_simulated = simulation_interactions.wait_until_simulated(delphin_ids, is_sampling_ahead(strategy_doc))

        if is_all_simulated:
            sampling.calculate_sample_output(strategy_doc.strategy, sample_id)
            current_error = sampling.calculate_error(strategy_doc)
            sampling_interactions.upload_standard_error(strategy_doc, current_error)
            convergence = sampling.check_convergence(strategy_doc)

            logger.debug(f'Standard Error at iteration {sample_iteration} is: {current_error}\n')

            # Update parameters for next iteration
            used_samples_per_set = used_samples_per_set + new_samples_per_set
            sample_iteration += 1
            sampling_interactions.upload_sample_iteration_parameters(strategy_doc,
                                                                     sample_iteration,
                                                                     used_samples_per_set)

            if used_samples_per_set >= strategy_doc.strategy['settings']['max samples']:
                logger.info(f'Maximum number of samples reached. Simulated {used_samples_per_set} samples per set\n')
                logger.info('Exits. Bye')
                sys.exit()
        else:
            create_new_samples(strategy_doc, used_samples_per_set + new_samples_per_set, sample_iteration + 1)

    logger.info(f'Convergence reached at iteration #{sample_iteration}\n')
    logger.info('Exits. Bye')
    sys.exit()

def generate_samples(strategy_id):
    strategy_doc = sampling_interactions.get_sampling_strategy(strategy_id)
    (sample_iteration, convergence,
     new_samples_per_set, used_samples_per_set) = sampling.initialize_sampling(strategy_doc)

    # Run loop
    while not sample_iteration < 50:
        logger.info(f'Running sampling iteration #{sample_iteration}')
        logger.info(f'New Samples per set: {new_samples_per_set}')
        logger.info(f'Used samples per set: {used_samples_per_set}')

        strategy_doc.reload()
        existing_sample = sampling.sample_exists(strategy_doc)

        if not existing_sample:
            delphin_ids, sample_id = create_new_samples(strategy_doc, used_samples_per_set, sample_iteration)

        else:
            logger.info('Found existing sample')
            delphin_ids = sampling_interactions.get_delphin_for_sample(existing_sample)
            sample_id = existing_sample.id

        # Update parameters for next iteration
        used_samples_per_set = used_samples_per_set + new_samples_per_set
        sample_iteration += 1
        sampling_interactions.upload_sample_iteration_parameters(strategy_doc,
                                                                 sample_iteration,
                                                                 used_samples_per_set)

    logger.info(f'Convergence reached at iteration #{sample_iteration}\n')
    logger.info('Exits. Bye')
    sys.exit()

def is_sampling_ahead(strategy_doc: sample_entry.Strategy) -> bool:

    num_samples = len(strategy_doc.samples) - 1

    if num_samples > strategy_doc.current_iteration:
        return True
    else:
        return False

def create_new_samples(strategy_doc, used_samples_per_set, sample_iteration):

    logger.info('Creating new samples')

    strategy_id = strategy_doc.id
    new_samples = sampling.create_samples(strategy_doc, used_samples_per_set)
    sample_id = sampling_interactions.upload_samples(new_samples, sample_iteration)
    delphin_ids = sampling.create_delphin_projects(strategy_doc.strategy, new_samples)

    sampling_interactions.add_delphin_to_sampling(sample_id, delphin_ids)
    #sampling_interactions.update_time_estimation_model(strategy_id)
    sampling_interactions.predict_simulation_time(delphin_ids, strategy_id)
    #sampling_interactions.update_queue_priorities(sample_id)
    sampling_interactions.add_sample_to_strategy(strategy_id, sample_id)

    return delphin_ids, sample_id

def sampling_overview(strategy_id):
    """Shows the progress in the sampling strategy"""

    strategy_doc = sampling_interactions.get_sampling_strategy(strategy_id)

    def create_figure(damage_model: str):
        plt.figure()

        for design in strategy_doc.standard_error.keys():
            data = strategy_doc.standard_error[design][damage_model]
            x = np.arange(0, len(data))
            if len(data) == 1:
                plt.scatter(x, data, label=design)
            else:
                plt.plot(x, data, label=design)
            plt.xlim(x[0] - 1, x[-1] + 1)

        plt.legend()
        plt.title(f'Sampling Convergence:\n{damage_model.capitalize()}')
        plt.xlabel('Iterations')
        plt.ylabel('Standard Error')

    create_figure('mould')
    create_figure('heat_loss')

    plt.show()
<commit_msg>fixed bug in generate samples<commit_after>__author__ = "Christian Kongsgaard"
__license__ = 'MIT'

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules
import sys
import os
import numpy as np

try:
    import matplotlib.pyplot as plt
except ModuleNotFoundError:
    pass

# RiBuild Modules
from delphin_6_automation.logging.ribuild_logger import ribuild_logger
from delphin_6_automation.sampling import sampling
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions import sampling_interactions
from delphin_6_automation.database_interactions.db_templates import sample_entry

# Logger
logger = ribuild_logger()

# -------------------------------------------------------------------------------------------------------------------- #
# RIBuild

def main():
    """Entrance to the sampling worker"""

    print_header()

    try:
        while True:
            menu()
    except KeyboardInterrupt:
        sys.exit()

def print_header():
    print('---------------------------------------------------')
    print('|                                                  |')
    print('|           RiBuild EU Research Project            |')
    print('|           for Hygrothermal Simulations           |')
    print('|                                                  |')
    print('|                 WORK IN PROGRESS                 |')
    print('|               Sampling Environment               |')
    print('|                                                  |')
    print('---------------------------------------------------')

def menu():
    print('')
    print('------------------- SAMPLING MENU ---------------------')
    print('')
    print("Available Actions:")
    print("[a] Start Sampling")
    print("[b] Create Sampling Strategy")
    print("[c] View Current Progress")
    print("[x] Exit")

    choice = input("> ").strip().lower()

    if choice == 'a':
        logger.debug('starting sampling')
        sampling_strategy_id = input("Define sampling strategy ID > ")
        logger.debug(sampling_strategy_id)

        logger.info('\nStarting sampling\n')
        sampling_worker(sampling_strategy_id)

    elif choice == 'b':
        strategy = sampling.create_sampling_strategy(os.path.dirname(__file__))
        strategy_id = sampling_interactions.upload_sampling_strategy(strategy)
        logger.info(f'Created sampling and uploaded it with ID: {strategy_id}')

    elif choice == 'c':
        sampling_strategy_id = input("Define sampling strategy ID > ")
        sampling_overview(sampling_strategy_id)

    elif choice == 'x':
        logger.info("Goodbye")

def sampling_worker(strategy_id: str) -> None:
    """
    Main sampling worker, which creates new Delphin projects based on a sample strategy and checks for convergence,
    when all Delphin simulations in a given iteration is done.
    If convergence was not reached then it creates new Delphin projects.
    """

    strategy_doc = sampling_interactions.get_sampling_strategy(strategy_id)
    (sample_iteration, convergence,
     new_samples_per_set, used_samples_per_set) = sampling.initialize_sampling(strategy_doc)

    # Run loop
    while not convergence:
        logger.info(f'Running sampling iteration #{sample_iteration}')
        logger.info(f'New Samples per set: {new_samples_per_set}')
        logger.info(f'Used samples per set: {used_samples_per_set}')

        strategy_doc.reload()
        existing_sample = sampling.sample_exists(strategy_doc)

        if not existing_sample:
            delphin_ids, sample_id = create_new_samples(strategy_doc, used_samples_per_set, sample_iteration)

        else:
            logger.info('Found existing sample')
            delphin_ids = sampling_interactions.get_delphin_for_sample(existing_sample)
            sample_id = existing_sample.id

        is_all_simulated = simulation_interactions.wait_until_simulated(delphin_ids, is_sampling_ahead(strategy_doc))

        if is_all_simulated:
            sampling.calculate_sample_output(strategy_doc.strategy, sample_id)
            current_error = sampling.calculate_error(strategy_doc)
            sampling_interactions.upload_standard_error(strategy_doc, current_error)
            convergence = sampling.check_convergence(strategy_doc)

            logger.debug(f'Standard Error at iteration {sample_iteration} is: {current_error}\n')

            # Update parameters for next iteration
            used_samples_per_set = used_samples_per_set + new_samples_per_set
            sample_iteration += 1
            sampling_interactions.upload_sample_iteration_parameters(strategy_doc,
                                                                     sample_iteration,
                                                                     used_samples_per_set)

            if used_samples_per_set >= strategy_doc.strategy['settings']['max samples']:
                logger.info(f'Maximum number of samples reached. Simulated {used_samples_per_set} samples per set\n')
                logger.info('Exits. Bye')
                sys.exit()
        else:
            create_new_samples(strategy_doc, used_samples_per_set + new_samples_per_set, sample_iteration + 1)

    logger.info(f'Convergence reached at iteration #{sample_iteration}\n')
    logger.info('Exits. Bye')
    sys.exit()

def generate_samples(strategy_id):
    strategy_doc = sampling_interactions.get_sampling_strategy(strategy_id)
    (sample_iteration, convergence,
     new_samples_per_set, used_samples_per_set) = sampling.initialize_sampling(strategy_doc)

    # Run loop
    while sample_iteration < 50:
        logger.info(f'Running sampling iteration #{sample_iteration}')
        logger.info(f'New Samples per set: {new_samples_per_set}')
        logger.info(f'Used samples per set: {used_samples_per_set}')

        strategy_doc.reload()
        existing_sample = sampling.sample_exists(strategy_doc)

        if not existing_sample:
            delphin_ids, sample_id = create_new_samples(strategy_doc, used_samples_per_set, sample_iteration)

        else:
            logger.info('Found existing sample')
            delphin_ids = sampling_interactions.get_delphin_for_sample(existing_sample)
            sample_id = existing_sample.id

        # Update parameters for next iteration
        used_samples_per_set = used_samples_per_set + new_samples_per_set
        sample_iteration += 1
        sampling_interactions.upload_sample_iteration_parameters(strategy_doc,
                                                                 sample_iteration,
                                                                 used_samples_per_set)

    logger.info(f'Convergence reached at iteration #{sample_iteration}\n')
    logger.info('Exits. Bye')
    sys.exit()

def is_sampling_ahead(strategy_doc: sample_entry.Strategy) -> bool:

    num_samples = len(strategy_doc.samples) - 1

    if num_samples > strategy_doc.current_iteration:
        return True
    else:
        return False

def create_new_samples(strategy_doc, used_samples_per_set, sample_iteration):

    logger.info('Creating new samples')

    strategy_id = strategy_doc.id
    new_samples = sampling.create_samples(strategy_doc, used_samples_per_set)
    sample_id = sampling_interactions.upload_samples(new_samples, sample_iteration)
    delphin_ids = sampling.create_delphin_projects(strategy_doc.strategy, new_samples)

    sampling_interactions.add_delphin_to_sampling(sample_id, delphin_ids)
    #sampling_interactions.update_time_estimation_model(strategy_id)
    sampling_interactions.predict_simulation_time(delphin_ids, strategy_id)
    #sampling_interactions.update_queue_priorities(sample_id)
    sampling_interactions.add_sample_to_strategy(strategy_id, sample_id)

    return delphin_ids, sample_id

def sampling_overview(strategy_id):
    """Shows the progress in the sampling strategy"""

    strategy_doc = sampling_interactions.get_sampling_strategy(strategy_id)

    def create_figure(damage_model: str):
        plt.figure()

        for design in strategy_doc.standard_error.keys():
            data = strategy_doc.standard_error[design][damage_model]
            x = np.arange(0, len(data))
            if len(data) == 1:
                plt.scatter(x, data, label=design)
            else:
                plt.plot(x, data, label=design)
            plt.xlim(x[0] - 1, x[-1] + 1)

        plt.legend()
        plt.title(f'Sampling Convergence:\n{damage_model.capitalize()}')
        plt.xlabel('Iterations')
        plt.ylabel('Standard Error')

    create_figure('mould')
    create_figure('heat_loss')

    plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2017-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test message sending before handshake completion.

A node should never send anything other than VERSION/VERACK until it's
received a VERACK.

This test connects to a node and sends it a few messages, trying to entice it
into sending us something it shouldn't."""

import time

from test_framework.messages import (
    msg_getaddr,
    msg_ping,
    msg_verack,
    msg_version,
)
from test_framework.mininode import mininode_lock, P2PInterface
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
    assert_greater_than_or_equal,
    wait_until,
)

banscore = 10

class CLazyNode(P2PInterface):
    def __init__(self):
        super().__init__()
        self.unexpected_msg = False
        self.ever_connected = False

    def bad_message(self, message):
        self.unexpected_msg = True
        self.log.info("should not have received message: %s" % message.msgtype)

    def on_open(self):
        self.ever_connected = True

    def on_version(self, message): self.bad_message(message)
    def on_verack(self, message): self.bad_message(message)
    def on_inv(self, message): self.bad_message(message)
    def on_addr(self, message): self.bad_message(message)
    def on_getdata(self, message): self.bad_message(message)
    def on_getblocks(self, message): self.bad_message(message)
    def on_tx(self, message): self.bad_message(message)
    def on_block(self, message): self.bad_message(message)
    def on_getaddr(self, message): self.bad_message(message)
    def on_headers(self, message): self.bad_message(message)
    def on_getheaders(self, message): self.bad_message(message)
    def on_ping(self, message): self.bad_message(message)
    def on_mempool(self, message): self.bad_message(message)
    def on_pong(self, message): self.bad_message(message)
    def on_feefilter(self, message): self.bad_message(message)
    def on_sendheaders(self, message): self.bad_message(message)
    def on_sendcmpct(self, message): self.bad_message(message)
    def on_cmpctblock(self, message): self.bad_message(message)
    def on_getblocktxn(self, message): self.bad_message(message)
    def on_blocktxn(self, message): self.bad_message(message)

# Node that never sends a version. We'll use this to send a bunch of messages
# anyway, and eventually get disconnected.
class CNodeNoVersionBan(CLazyNode):
    # send a bunch of veracks without sending a message. This should get us disconnected.
    # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes
    def on_open(self):
        super().on_open()
        for i in range(banscore):
            self.send_message(msg_verack())

# Node that never sends a version. This one just sits idle and hopes to receive
# any message (it shouldn't!)
class CNodeNoVersionIdle(CLazyNode):
    def __init__(self):
        super().__init__()

# Node that sends a version but not a verack.
class CNodeNoVerackIdle(CLazyNode):
    def __init__(self):
        self.version_received = False
        super().__init__()

    def on_verack(self, message): pass
    # When version is received, don't reply with a verack. Instead, see if the
    # node will give us a message that it shouldn't. This is not an exhaustive
    # list!
    def on_version(self, message):
        self.version_received = True
        self.send_message(msg_ping())
        self.send_message(msg_getaddr())

class P2PVersionStore(P2PInterface):
    version_received = None

    def on_version(self, msg):
        super().on_version(msg)
        self.version_received = msg

class P2PLeakTest(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 1
        self.extra_args = [['-banscore=' + str(banscore)]]

    def run_test(self):
        no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False, wait_for_verack=False)
        no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False)
        no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False)

        # Wait until we got the verack in response to the version. Though, don't wait for the other node to receive the
        # verack, since we never sent one
        no_verack_idlenode.wait_for_verack()

        wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock)
        wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock)
        wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock)

        # Mine a block and make sure that it's not sent to the connected nodes
        self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)

        #Give the node enough time to possibly leak out a message
        time.sleep(5)

        #This node should have been banned
        assert not no_version_bannode.is_connected

        self.nodes[0].disconnect_p2ps()

        # Wait until all connections are closed
        wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0)

        # Make sure no unexpected messages came in
        assert no_version_bannode.unexpected_msg == False
        assert no_version_idlenode.unexpected_msg == False
        assert no_verack_idlenode.unexpected_msg == False

        self.log.info('Check that the version message does not leak the local address of the node')
        time_begin = int(time.time())
        p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore())
        time_end = time.time()
        ver = p2p_version_store.version_received
        assert_greater_than_or_equal(ver.nTime, time_begin)
        assert_greater_than_or_equal(time_end, ver.nTime)
        assert_equal(ver.addrFrom.port, 0)
        assert_equal(ver.addrFrom.ip, '0.0.0.0')
        assert_equal(ver.nStartingHeight, 201)
        assert_equal(ver.nRelay, 1)

        self.log.info('Check that old nodes are disconnected')
        p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False)
        old_version_msg = msg_version()
        old_version_msg.nVersion = 31799
        wait_until(lambda: p2p_old_node.is_connected)
        with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']):
            p2p_old_node.send_message(old_version_msg)
            p2p_old_node.wait_for_disconnect()

if __name__ == '__main__':
    P2PLeakTest().main()
<commit_msg>test: Fix p2p_leak intermittent issue<commit_after>#!/usr/bin/env python3
# Copyright (c) 2017-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test message sending before handshake completion.

A node should never send anything other than VERSION/VERACK until it's
received a VERACK.

This test connects to a node and sends it a few messages, trying to entice it
into sending us something it shouldn't."""

import time

from test_framework.messages import (
    msg_getaddr,
    msg_ping,
    msg_verack,
    msg_version,
)
from test_framework.mininode import mininode_lock, P2PInterface
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
    assert_greater_than_or_equal,
    wait_until,
)

banscore = 10

class CLazyNode(P2PInterface):
    def __init__(self):
        super().__init__()
        self.unexpected_msg = False
        self.ever_connected = False

    def bad_message(self, message):
        self.unexpected_msg = True
        self.log.info("should not have received message: %s" % message.msgtype)

    def on_open(self):
        self.ever_connected = True

    def on_version(self, message): self.bad_message(message)
    def on_verack(self, message): self.bad_message(message)
    def on_inv(self, message): self.bad_message(message)
    def on_addr(self, message): self.bad_message(message)
    def on_getdata(self, message): self.bad_message(message)
    def on_getblocks(self, message): self.bad_message(message)
    def on_tx(self, message): self.bad_message(message)
    def on_block(self, message): self.bad_message(message)
    def on_getaddr(self, message): self.bad_message(message)
    def on_headers(self, message): self.bad_message(message)
    def on_getheaders(self, message): self.bad_message(message)
    def on_ping(self, message): self.bad_message(message)
    def on_mempool(self, message): self.bad_message(message)
    def on_pong(self, message): self.bad_message(message)
    def on_feefilter(self, message): self.bad_message(message)
    def on_sendheaders(self, message): self.bad_message(message)
    def on_sendcmpct(self, message): self.bad_message(message)
    def on_cmpctblock(self, message): self.bad_message(message)
    def on_getblocktxn(self, message): self.bad_message(message)
    def on_blocktxn(self, message): self.bad_message(message)

# Node that never sends a version. We'll use this to send a bunch of messages
# anyway, and eventually get disconnected.
class CNodeNoVersionBan(CLazyNode):
    # send a bunch of veracks without sending a message. This should get us disconnected.
    # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes
    def on_open(self):
        super().on_open()
        for i in range(banscore):
            self.send_message(msg_verack())

# Node that never sends a version. This one just sits idle and hopes to receive
# any message (it shouldn't!)
class CNodeNoVersionIdle(CLazyNode):
    def __init__(self):
        super().__init__()

# Node that sends a version but not a verack.
class CNodeNoVerackIdle(CLazyNode):
    def __init__(self):
        self.version_received = False
        super().__init__()

    def on_verack(self, message): pass
    # When version is received, don't reply with a verack. Instead, see if the
    # node will give us a message that it shouldn't. This is not an exhaustive
    # list!
    def on_version(self, message):
        self.version_received = True
        self.send_message(msg_ping())
        self.send_message(msg_getaddr())

class P2PVersionStore(P2PInterface):
    version_received = None

    def on_version(self, msg):
        super().on_version(msg)
        self.version_received = msg

class P2PLeakTest(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 1
        self.extra_args = [['-banscore=' + str(banscore)]]

    def run_test(self):
        no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False, wait_for_verack=False)
        no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False)
        no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False)

        # Wait until we got the verack in response to the version. Though, don't wait for the other node to receive the
        # verack, since we never sent one
        no_verack_idlenode.wait_for_verack()

        wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock)
        wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock)
        wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock)

        # Mine a block and make sure that it's not sent to the connected nodes
        self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)

        #Give the node enough time to possibly leak out a message
        time.sleep(5)

        #This node should have been banned
        assert not no_version_bannode.is_connected

        self.nodes[0].disconnect_p2ps()

        # Wait until all connections are closed
        wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0)

        # Make sure no unexpected messages came in
        assert no_version_bannode.unexpected_msg == False
        assert no_version_idlenode.unexpected_msg == False
        assert no_verack_idlenode.unexpected_msg == False

        self.log.info('Check that the version message does not leak the local address of the node')
        p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore())
        ver = p2p_version_store.version_received
        # Check that received time is within one hour of now
        assert_greater_than_or_equal(ver.nTime, time.time() - 3600)
        assert_greater_than_or_equal(time.time() + 3600, ver.nTime)
        assert_equal(ver.addrFrom.port, 0)
        assert_equal(ver.addrFrom.ip, '0.0.0.0')
        assert_equal(ver.nStartingHeight, 201)
        assert_equal(ver.nRelay, 1)

        self.log.info('Check that old nodes are disconnected')
        p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False)
        old_version_msg = msg_version()
        old_version_msg.nVersion = 31799
        wait_until(lambda: p2p_old_node.is_connected)
        with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']):
            p2p_old_node.send_message(old_version_msg)
            p2p_old_node.wait_for_disconnect()

if __name__ == '__main__':
    P2PLeakTest().main()
<|endoftext|>