<commit_before>"""Client/manager side object for querying Agent via SNMPProtocol"""
from twisted.internet import defer, reactor
from pysnmp.proto import v2c, v1, error
from pysnmp.proto.api import generic
from twistedsnmp import datatypes, tableretriever
import traceback

__metaclass__ = type

class AgentProxy:
	"""Proxy object for querying a remote agent"""
	verbose = 0
	def __init__(self, ip, port=161, community='public', snmpVersion = '1', protocol=None):
		"""Initialize the SNMPProtocol object

		ip -- ipAddress for the protocol
		port -- port for the connection
		community -- community to use for SNMP conversations
		snmpVersion -- '1' or '2', indicating the supported version
		protocol -- SNMPProtocol object to use for actual connection
		"""
		self.ip = str(ip)
		self.port = int(port)
		self.community = str(community)
		self.snmpVersion = snmpVersion
		if self.snmpVersion in ("2",'2c','v2','v2c'):
			self.implementation = v2c
		else:
			self.implementation = v1
		self.protocol = protocol
	def __repr__( self ):
		"""Get nice string representation of the proxy"""
		ip,port,community,snmpVersion,protocol = self.ip,self.port,self.community,self.snmpVersion,self.protocol
		className = self.__class__.__name__
		try:
			snmpVersionName = snmpVersion.__name__
		except AttributeError:
			snmpVersionName = snmpVersion
		return """%(className)s(%(ip)s,%(port)s,%(community)s,%(snmpVersionName)s,%(protocol)r)"""%locals()
	def get(self, oids, timeout=2.0, retryCount=4):
		"""Retrieve a single set of OIDs from the remote agent

		oids -- list of dotted-numeric oids to retrieve
		retryCount -- number of retries
		timeout -- initial timeout, is multipled by 1.5 on each
			timeout iteration.

		return value is a defered for an { oid : value } mapping
		for each oid in requested set

		XXX Should be raising an error if the response has an
		error message, will raise error if the connection times
		out.
		"""
		if not self.protocol:
			raise ValueError( """Expected a non-null protocol object! Got %r"""%(protocol,))
		df = defer.Deferred()
		def asDictionary( value ):
			return dict(value)
		df.addCallback( self.getResponseResults )
		df.addCallback( asDictionary )
		request = self.encode(oids, self.community)
		key = self.getRequestKey( request )
		self.send(request.encode())
		oids = [str(oid) for oid in oids ]
		timer = reactor.callLater(timeout, self._timeout, key, df, oids, timeout, retryCount)
		self.protocol.requests[key] = df, timer
		return df
	def set( self, oids, timeout=2.0, retryCount=4):
		"""Set a variable on our connected agent

		oids -- dictionary of oid:value pairs, or a list of
			(oid,value) tuples to be set on the agent

		raises errors if the setting fails
		"""
		if not self.protocol:
			raise ValueError( """Expected a non-null protocol object! Got %r"""%(protocol,))
		df = defer.Deferred()
		if hasattr( oids, "items"):
			oids = oids.items()
		request = self.encode(oids, self.community, set=1)
		key = self.getRequestKey( request )
		def raiseOnError( response ):
			pdu = response.apiGenGetPdu()
			if pdu.apiGenGetErrorStatus():
				raise error.ProtoError( """Set failure""", pdu.apiGenGetErrorStatus() )
			return response
		df.addCallback( raiseOnError )
		self.send(request.encode())
		timer = reactor.callLater(timeout, self._timeout, key, df, oids, timeout, retryCount)
		self.protocol.requests[key] = df, timer
		return df

	def getTable( self, roots, includeStart=0, recordCallback=None, retryCount=4, timeout= 2.0 ):
		"""Convenience method for creating and running a TableRetriever

		roots -- root OIDs to retrieve
		includeStart -- whether to include the starting OID
			in the set of results, by default, return the OID
			*after* the root oids.
			Note:  Only implemented for v1 protocols, and likely
				to be dropped eventually, as it seems somewhat
				superfluous.
		recordCallback -- called for each new record discovered
			recordCallback( root, oid, value )
		retryCount -- number of retries
		timeout -- initial timeout, is multipled by 1.5 on each
			timeout iteration.

		Will use bulk downloading when available (i.e. if
		we have implementation v2c, not v1).

		return value is a defered for a { rootOID: { oid: value } } mapping
		"""
		if not self.protocol:
			raise ValueError( """Expected a non-null protocol object! Got %r"""%(protocol,))
		roots = [str(oid) for oid in roots ]
		retriever = tableretriever.TableRetriever(
			self, roots, includeStart=includeStart,
			retryCount=retryCount, timeout= timeout,
		)
		return retriever( recordCallback = recordCallback )

	def send(self, request):
		"""Send a request (string) to the network"""
		return self.protocol.send(request, (self.ip, self.port))

	## Utility methods...
	def getImplementation( self ):
		"""Get the implementation module for this request

		returns v2c or v1 pysnmp modules depending on our
		snmpVersion property.
		"""
		return self.implementation
	def getRequestKey( self, request ):
		"""Get the request key from a request/response"""
		return self.protocol.getRequestKey( request, (self.ip, self.port) )
	def encode( self, oids, community, next=0, bulk=0, set=0 ):
		"""Encode a datagram message"""
		implementation = self.getImplementation()
		if bulk:
			request = implementation.GetBulkRequest()
		elif set:
			request = implementation.SetRequest()
		elif next:
			request = implementation.GetNextRequest()
		else:
			request = implementation.GetRequest()
		request.apiGenSetCommunity( community )
		pdu = request.apiGenGetPdu()
		def oidFix( value, implementation=implementation ):
			if isinstance( value, tuple ) and len(value) == 2:
				oid,value = value
				value = datatypes.typeCoerce( value, implementation )
				return oid, value
			else:
				return value, None
		variables = [oidFix(oid) for oid in oids]
		pdu.apiGenSetVarBind(variables)
		return request
	def getResponseResults( self, response ):
		"""Get [(oid,value)...] list from response

		This callback is part of the callback chain for get
		response processing.  In essence, if you have a callback
		that wants [(oid,value)...] format instead of response
		objects register this callback before the needy callback.
		"""
		if response and not response.apiGenGetPdu().apiGenGetErrorStatus():
			pdu = response.apiGenGetPdu()
			answer = pdu.apiGenGetVarBind()
			return [
				(a,b.getTerminal().get())
				for a,b in answer
				if not isinstance( b, v2c.EndOfMibView)
			]
		return []
	def _timeout(self, key, df, oids, timeout, retryCount):
		try:
			try:
				del self.protocol.requests[ key ]
			except KeyError:
				pass
			if not df.called:
				if self.verbose:
					print 'timeout', self
				if retryCount:
					timeout *= 1.5
					retryCount -= 1
					if self.verbose:
						print '    trying again', timeout, retryCount
					request = self.encode(oids, self.community)
					key = self.getRequestKey( request )
					self.send(request.encode())
					timer = reactor.callLater(
						timeout,
						self._timeout, key, df, oids, timeout, retryCount
					)
					self.protocol.requests[key] = df, timer
					return
				if self.verbose:
					print '    RAISING ERROR'
				df.errback(defer.TimeoutError('SNMP request timed out'))
		except Exception, err:
			df.errback( err )
		<commit_msg>Added a few more prints for the verbose version<commit_after>"""Client/manager side object for querying Agent via SNMPProtocol"""
from twisted.internet import defer, reactor
from pysnmp.proto import v2c, v1, error
from pysnmp.proto.api import generic
from twistedsnmp import datatypes, tableretriever
import traceback

__metaclass__ = type

class AgentProxy:
	"""Proxy object for querying a remote agent"""
	verbose = 0
	def __init__(self, ip, port=161, community='public', snmpVersion = '1', protocol=None):
		"""Initialize the SNMPProtocol object

		ip -- ipAddress for the protocol
		port -- port for the connection
		community -- community to use for SNMP conversations
		snmpVersion -- '1' or '2', indicating the supported version
		protocol -- SNMPProtocol object to use for actual connection
		"""
		self.ip = str(ip)
		self.port = int(port)
		self.community = str(community)
		self.snmpVersion = snmpVersion
		if self.snmpVersion in ("2",'2c','v2','v2c'):
			self.implementation = v2c
		else:
			self.implementation = v1
		self.protocol = protocol
	def __repr__( self ):
		"""Get nice string representation of the proxy"""
		ip,port,community,snmpVersion,protocol = self.ip,self.port,self.community,self.snmpVersion,self.protocol
		className = self.__class__.__name__
		try:
			snmpVersionName = snmpVersion.__name__
		except AttributeError:
			snmpVersionName = snmpVersion
		return """%(className)s(%(ip)s,%(port)s,%(community)s,%(snmpVersionName)s,%(protocol)r)"""%locals()
	def get(self, oids, timeout=2.0, retryCount=4):
		"""Retrieve a single set of OIDs from the remote agent

		oids -- list of dotted-numeric oids to retrieve
		retryCount -- number of retries
		timeout -- initial timeout, is multipled by 1.5 on each
			timeout iteration.

		return value is a defered for an { oid : value } mapping
		for each oid in requested set

		XXX Should be raising an error if the response has an
		error message, will raise error if the connection times
		out.
		"""
		if not self.protocol:
			raise ValueError( """Expected a non-null protocol object! Got %r"""%(protocol,))
		df = defer.Deferred()
		def asDictionary( value ):
			return dict(value)
		df.addCallback( self.getResponseResults )
		df.addCallback( asDictionary )
		request = self.encode(oids, self.community)
		key = self.getRequestKey( request )
		self.send(request.encode())
		oids = [str(oid) for oid in oids ]
		timer = reactor.callLater(timeout, self._timeout, key, df, oids, timeout, retryCount)
		self.protocol.requests[key] = df, timer
		return df
	def set( self, oids, timeout=2.0, retryCount=4):
		"""Set a variable on our connected agent

		oids -- dictionary of oid:value pairs, or a list of
			(oid,value) tuples to be set on the agent

		raises errors if the setting fails
		"""
		if not self.protocol:
			raise ValueError( """Expected a non-null protocol object! Got %r"""%(protocol,))
		df = defer.Deferred()
		if hasattr( oids, "items"):
			oids = oids.items()
		request = self.encode(oids, self.community, set=1)
		key = self.getRequestKey( request )
		def raiseOnError( response ):
			pdu = response.apiGenGetPdu()
			if pdu.apiGenGetErrorStatus():
				raise error.ProtoError( """Set failure""", pdu.apiGenGetErrorStatus() )
			return response
		df.addCallback( raiseOnError )
		self.send(request.encode())
		timer = reactor.callLater(timeout, self._timeout, key, df, oids, timeout, retryCount)
		self.protocol.requests[key] = df, timer
		return df

	def getTable( self, roots, includeStart=0, recordCallback=None, retryCount=4, timeout= 2.0 ):
		"""Convenience method for creating and running a TableRetriever

		roots -- root OIDs to retrieve
		includeStart -- whether to include the starting OID
			in the set of results, by default, return the OID
			*after* the root oids.
			Note:  Only implemented for v1 protocols, and likely
				to be dropped eventually, as it seems somewhat
				superfluous.
		recordCallback -- called for each new record discovered
			recordCallback( root, oid, value )
		retryCount -- number of retries
		timeout -- initial timeout, is multipled by 1.5 on each
			timeout iteration.

		Will use bulk downloading when available (i.e. if
		we have implementation v2c, not v1).

		return value is a defered for a { rootOID: { oid: value } } mapping
		"""
		if self.verbose:
			print 'getTable( %(roots)r, %(includeStart)r, %(recordCallback)r,%(retryCount)r )'%locals()
		if not self.protocol:
			raise ValueError( """Expected a non-null protocol object! Got %r"""%(protocol,))
		roots = [str(oid) for oid in roots ]
		retriever = tableretriever.TableRetriever(
			self, roots, includeStart=includeStart,
			retryCount=retryCount, timeout= timeout,
		)
		if self.verbose:
			retriever.verbose = 1
		return retriever( recordCallback = recordCallback )

	def send(self, request):
		"""Send a request (string) to the network"""
		return self.protocol.send(request, (self.ip, self.port))

	## Utility methods...
	def getImplementation( self ):
		"""Get the implementation module for this request

		returns v2c or v1 pysnmp modules depending on our
		snmpVersion property.
		"""
		return self.implementation
	def getRequestKey( self, request ):
		"""Get the request key from a request/response"""
		return self.protocol.getRequestKey( request, (self.ip, self.port) )
	def encode( self, oids, community, next=0, bulk=0, set=0 ):
		"""Encode a datagram message"""
		if self.verbose:
			print 'encode( %(oids)r, %(community)r, %(next)r, %(bulk)r, %(set)r)'%locals()
		implementation = self.getImplementation()
		if bulk:
			request = implementation.GetBulkRequest()
		elif set:
			request = implementation.SetRequest()
		elif next:
			request = implementation.GetNextRequest()
		else:
			request = implementation.GetRequest()
		request.apiGenSetCommunity( community )
		pdu = request.apiGenGetPdu()
		def oidFix( value, implementation=implementation ):
			if isinstance( value, tuple ) and len(value) == 2:
				oid,value = value
				value = datatypes.typeCoerce( value, implementation )
				return oid, value
			else:
				return value, None
		variables = [oidFix(oid) for oid in oids]
		pdu.apiGenSetVarBind(variables)
		return request
	def getResponseResults( self, response ):
		"""Get [(oid,value)...] list from response

		This callback is part of the callback chain for get
		response processing.  In essence, if you have a callback
		that wants [(oid,value)...] format instead of response
		objects register this callback before the needy callback.
		"""
		if self.verbose:
			print 'getResponseResults( %(response)r )'%locals()
		if response and not response.apiGenGetPdu().apiGenGetErrorStatus():
			pdu = response.apiGenGetPdu()
			answer = pdu.apiGenGetVarBind()
			return [
				(a,b.getTerminal().get())
				for a,b in answer
				if not isinstance( b, v2c.EndOfMibView)
			]
		return []
	def _timeout(self, key, df, oids, timeout, retryCount):
		try:
			try:
				del self.protocol.requests[ key ]
			except KeyError:
				pass
			if not df.called:
				if self.verbose:
					print 'timeout', self
				if retryCount:
					timeout *= 1.5
					retryCount -= 1
					if self.verbose:
						print '    trying again', timeout, retryCount
					request = self.encode(oids, self.community)
					key = self.getRequestKey( request )
					self.send(request.encode())
					timer = reactor.callLater(
						timeout,
						self._timeout, key, df, oids, timeout, retryCount
					)
					self.protocol.requests[key] = df, timer
					return
				if self.verbose:
					print '    RAISING ERROR'
				df.errback(defer.TimeoutError('SNMP request timed out'))
		except Exception, err:
			df.errback( err )
		<|endoftext|>


<commit_before>import datetime
import os

import pkg_resources
from gi.repository import GtkSource, Gdk, Gtk, GObject, GdkPixbuf, Notify

from ..core.toolwindow import ToolWindow, question_message, error_message, info_message
from ...core.commands.script import Script, Command

# ToDo: closing the script window results in GtkWarnings (not a GTK_WIDGET etc.). Something to do with buffer marks.

language_def_path = pkg_resources.resource_filename(
    'cct', 'resource/language-specs')
langman = GtkSource.LanguageManager.get_default()
langman.set_search_path(langman.get_search_path() + [language_def_path])

class ScriptMeasurement(ToolWindow):
    def _init_gui(self, *args):
        view=self._builder.get_object('sourceview')
        self._sourcebuffer=GtkSource.Buffer()
        view.set_buffer(self._sourcebuffer)
        self._sourcebuffer.set_language(langman.get_language('cct'))
        ssman = GtkSource.StyleSchemeManager.get_default()
        self._sourcebuffer.set_style_scheme(
            ssman.get_scheme(ssman.get_scheme_ids()[0]))
        self._sourcebuffer.set_highlight_syntax(True)
        self._sourcebuffer.set_highlight_matching_brackets(True)
        self._sourcebuffer.set_modified(False)
        self._sourcebuffer.connect('modified-changed', self.on_modified_changed)
        self._window.set_title('Unnamed script')
        ma = GtkSource.MarkAttributes()
        ma.set_icon_name('media-playback-start')
        ma.set_background(Gdk.RGBA(0, 1, 0, 1))
        view.set_mark_attributes('Executing', ma, 0)

    def confirm_save(self):
        if self._sourcebuffer.get_modified():
            if question_message(self._window,'Script has been modified. Do you want to save it first?'):
                self.on_toolbutton_save(self._builder.get_object('save_toolbutton'))
                self._sourcebuffer.set_modified(False)
                return True
        self._sourcebuffer.set_modified(False)
        return False

    def on_modified_changed(self, sourcebuffer):
        self._builder.get_object('save_toolbutton').set_sensitive(sourcebuffer.get_modified())
        self._builder.get_object('saveas_toolbutton').set_sensitive(sourcebuffer.get_modified())
        title=self._window.get_title()
        if title.endswith('*'):
            title=title[:-1].strip()
        if sourcebuffer.get_modified():
            self._window.set_title(title+' *')
        else:
            self._window.set_title(title)

    def on_toolbutton_new(self, toolbutton):
        self.confirm_save()
        self._sourcebuffer.set_text('')
        self._sourcebuffer.set_modified(False)

    def on_toolbutton_open(self, toolbutton):
        self.confirm_save()
        if not hasattr(self, '_filechooser'):
            self._filechooser = Gtk.FileChooserDialog('Open script file', self._window, Gtk.FileChooserAction.OPEN,
                                                      ['OK', Gtk.ResponseType.OK, 'Cancel', Gtk.ResponseType.CANCEL])
            self._filechooser.add_shortcut_folder(os.path.join(os.getcwd(),'scripts'))
        self._filechooser.set_action(Gtk.FileChooserAction.OPEN)
        self._filechooser.set_title('Open script file')
        self._filechooser.set_transient_for(self._window)
        if self._filechooser.run()==Gtk.ResponseType.OK:
            self._filename=self._filechooser.get_filename()
            self._window.set_title(self._filename)
        self._filechooser.hide()
        with open(self._filename, 'rt', encoding='utf-8') as f:
            self._sourcebuffer.set_text(f.read())
        self._sourcebuffer.set_modified(False)

    def on_toolbutton_save(self, toolbutton):
        if not hasattr(self, '_filename'):
            self.on_toolbutton_saveas(toolbutton)
        with open(self._filename, 'wt', encoding='utf-8') as f:
            f.write(self._sourcebuffer.get_text(self._sourcebuffer.get_start_iter(), self._sourcebuffer.get_end_iter(), True))
        self._sourcebuffer.set_modified(False)

    def on_toolbutton_saveas(self, toolbutton):
        if not hasattr(self, '_filechooser'):
            self._filechooser = Gtk.FileChooserDialog('Save script file as...', self._window,
                                                      Gtk.FileChooserAction.SAVE, ['OK', Gtk.ResponseType.OK, 'Cancel', Gtk.ResponseType.CANCEL])
            self._filechooser.add_shortcut_folder(os.path.join(os.getcwd(),'scripts'))
        self._filechooser.set_action(Gtk.FileChooserAction.SAVE)
        self._filechooser.set_do_overwrite_confirmation(True)
        self._filechooser.set_title('Save script file as...')
        self._filechooser.set_transient_for(self._window)
        if self._filechooser.run()==Gtk.ResponseType.OK:
            self._filename=self._filechooser.get_filename()
            if not self._filename.lower().endswith('.cct'):
                self._filename=self._filename+'.cct'
            self._window.set_title(self._filename)
            self.on_toolbutton_save(toolbutton)
        self._filechooser.hide()

    def on_toolbutton_undo(self, toolbutton):
        self._sourcebuffer.undo()

    def on_toolbutton_redo(self, toolbutton):
        self._sourcebuffer.redo()

    def on_toolbutton_cut(self, toolbutton):
        self._sourcebuffer.copy_clipboard(Gtk.Clipboard.get_default(Gdk.Display.get_default()))

    def on_toolbutton_copy(self, toolbutton):
        self._sourcebuffer.cut_clipboard(Gtk.Clipboard.get_default(Gdk.Display.get_default()),
                                         self._inhibit_close_reason is not None)

    def on_toolbutton_paste(self, toolbutton):
        self._sourcebuffer.paste_clipboard(Gtk.Clipboard.get_default(Gdk.Display.get_default()),
                                           self._inhibit_close_reason is not None)

    def on_toolbutton_execute(self, toolbutton):
        self._scriptcommand=Script(self._sourcebuffer.get_text(self._sourcebuffer.get_start_iter(),
                                                               self._sourcebuffer.get_end_iter(),
                                                               True))
        self._make_insensitive('Script is running', ['sourceview', 'new_toolbutton', 'save_toolbutton', 'saveas_toolbutton',
                                                     'open_toolbutton', 'undo_toolbutton', 'redo_toolbutton', 'cut_toolbutton',
                                                     'copy_toolbutton', 'paste_toolbutton', 'help_toolbutton', 'execute_toolbutton'])
        self._interpreter_connections=[
            self._instrument.interpreter.connect('cmd-return', self.on_script_end),
            self._instrument.interpreter.connect('cmd-fail', self.on_script_fail),
            self._instrument.interpreter.connect('pulse', self.on_script_pulse),
            self._instrument.interpreter.connect('progress',self.on_script_progress),
            self._instrument.interpreter.connect('cmd-message', self.on_script_message),
                                       ]
        self._instrument.interpreter.clear_flag(None)  # clear all flags
        flagsbb = self._builder.get_object('flags_buttonbox')
        for b in flagsbb:
            if b.get_active():
                self._instrument.interpreter.set_flag(b.get_label())

        self._builder.get_object('sourceview').set_editable(False)
        self._scriptconnections = [self._scriptcommand.connect('cmd-start', self.on_command_start)]
        buf = self._builder.get_object('messagesbuffer')
        buf.insert(buf.get_end_iter(),
                   '----------------------- %s -----------------------\n' % str(datetime.datetime.now()))
        self._builder.get_object('messagesview').scroll_to_iter(buf.get_end_iter(), 0, False, 0, 0)
        buf.place_cursor(buf.get_end_iter())
        try:
            self._instrument.interpreter.execute_command(self._scriptcommand)
        except Exception:
            self._cleanup()
            raise
        try:
            with open(self._filename[:-len('.cct')] + '.log', 'at', encoding='utf-8') as f:
                f.write(str(datetime.datetime.now()) + ': ----------------------- Started -----------------------\n')
        except AttributeError:
            pass

    def _cleanup(self):
        self._make_sensitive()
        for c in self._interpreter_connections:
            self._instrument.interpreter.disconnect(c)
        del self._interpreter_connections
        self._builder.get_object('sourceview').set_editable(True)
        self._sourcebuffer.remove_source_marks(
            self._sourcebuffer.get_start_iter(), self._sourcebuffer.get_end_iter(), 'Executing')
        self._builder.get_object('progressbar').set_text('')
        self._builder.get_object('progressbar').set_fraction(0)
        for c in self._scriptconnections:
            self._scriptcommand.disconnect(c)
        del self._scriptconnections
        del self._scriptcommand

    def on_script_end(self, interpreter, commandname, returnvalue):
        info_message(self._window,'Script ended','Result: %s'%str(returnvalue))
        self._cleanup()
        n=Notify.Notification(summary='Script ended',body='Script execution ended with result: %s'%str(returnvalue))
        n.set_image_from_pixbuf(GdkPixbuf.Pixbuf.new_from_file_at_size(pkg_resources.resource_filename('cct','resource/icons/scalable/cctlogo.svg'),256,256))
        n.show()

    def on_script_fail(self, interpreter, commandname, exc, tb):
        error_message(self._window, 'Error while executing script', tb)

    def on_script_pulse(self, interpreter, commandname, message):
        pb=self._builder.get_object('progressbar')
        pb.set_visible(True)
        pb.set_text(message)
        pb.pulse()

    def on_script_progress(self, interpreter, commandname, message, fraction):
        pb=self._builder.get_object('progressbar')
        pb.set_visible(True)
        pb.set_text(message)
        pb.set_fraction(fraction)

    def on_script_message(self, interpreter, commandname, message):
        buf=self._builder.get_object('messagesview').get_buffer()
        buf.insert(buf.get_end_iter(), message+'\n')
        self._builder.get_object('messagesview').scroll_to_iter(buf.get_end_iter(), 0,False, 0,0)
        buf.place_cursor(buf.get_end_iter())
        try:
            with open(self._filename[:-len('.cct')]+'.log','at', encoding='utf-8') as f:
                f.write(str(datetime.datetime.now())+': '+message+'\n')
        except AttributeError:
            pass

    def on_command_start(self, scriptcmd, lineno, command ):
        it = self._sourcebuffer.get_iter_at_line(lineno)
        self._sourcebuffer.remove_source_marks(
            self._sourcebuffer.get_start_iter(), self._sourcebuffer.get_end_iter(), 'Executing')
        self._sourcebuffer.create_source_mark(
            'Line #%d' % lineno, 'Executing', it)
        self._builder.get_object('sourceview').scroll_to_iter(it, 0, False, 0, 0)
        self._sourcebuffer.place_cursor(it)
        self._builder.get_object('progressbar').hide()

    def on_toolbutton_pause(self, toolbutton):
        pass

    def on_toolbutton_stop(self, toolbutton):
        self._instrument.interpreter.kill()

    def on_toolbutton_help(self, toolbutton):
        if not hasattr(self, '_helpdialog'):
            self._helpdialog = CommandHelpDialog('help_commandhelpbrowser.glade', 'commandhelpbrowser',
                                                 self._instrument, self._application, 'Help on commands')
            self._helpdialog.connect('insert', self.on_insert)
        self._helpdialog._window.show_all()

    def on_insert(self, helpdialog, text):
        self._sourcebuffer.insert_at_cursor(text)

    def on_close(self, widget, event=None):
        self.confirm_save()
        ToolWindow.on_close(self, widget, event)

    def on_flag_toggled(self, flagtoggle):
        if flagtoggle.get_active():
            self._instrument.interpreter.set_flag(flagtoggle.get_label())
        else:
            self._instrument.interpreter.clear_flag(flagtoggle.get_label())

class CommandHelpDialog(ToolWindow):
    __gsignals__ = {'insert': (GObject.SignalFlags.RUN_FIRST, None, (str,))}

    def _init_gui(self, *args):
        model = self._builder.get_object('commandnames')
        for command in sorted([c.name for c in Command.allcommands()]):
            model.append((command,))
        tc = Gtk.TreeViewColumn('Command', Gtk.CellRendererText(), text=0)
        self._builder.get_object('commandsview').append_column(tc)
        self._builder.get_object('commandsview').get_selection().select_iter(model.get_iter_first())

    def on_command_selected(self, treeviewselection):
        model, it = treeviewselection.get_selected()
        buf = self._builder.get_object('helptextbuffer')
        buf.set_text([c.__doc__ for c in Command.allcommands() if c.name == model[it][0]][0])

    def on_insert(self, button):
        model, it = self._builder.get_object('commandsview').get_selection().get_selected()
        self.emit('insert', model[it][0] + '()')
<commit_msg>Script GUI: Save As button not made insensitive<commit_after>import datetime
import os

import pkg_resources
from gi.repository import GtkSource, Gdk, Gtk, GObject, GdkPixbuf, Notify

from ..core.toolwindow import ToolWindow, question_message, error_message, info_message
from ...core.commands.script import Script, Command

# ToDo: closing the script window results in GtkWarnings (not a GTK_WIDGET etc.). Something to do with buffer marks.

language_def_path = pkg_resources.resource_filename(
    'cct', 'resource/language-specs')
langman = GtkSource.LanguageManager.get_default()
langman.set_search_path(langman.get_search_path() + [language_def_path])

class ScriptMeasurement(ToolWindow):
    def _init_gui(self, *args):
        view=self._builder.get_object('sourceview')
        self._sourcebuffer=GtkSource.Buffer()
        view.set_buffer(self._sourcebuffer)
        self._sourcebuffer.set_language(langman.get_language('cct'))
        ssman = GtkSource.StyleSchemeManager.get_default()
        self._sourcebuffer.set_style_scheme(
            ssman.get_scheme(ssman.get_scheme_ids()[0]))
        self._sourcebuffer.set_highlight_syntax(True)
        self._sourcebuffer.set_highlight_matching_brackets(True)
        self._sourcebuffer.set_modified(False)
        self._sourcebuffer.connect('modified-changed', self.on_modified_changed)
        self._window.set_title('Unnamed script')
        ma = GtkSource.MarkAttributes()
        ma.set_icon_name('media-playback-start')
        ma.set_background(Gdk.RGBA(0, 1, 0, 1))
        view.set_mark_attributes('Executing', ma, 0)

    def confirm_save(self):
        if self._sourcebuffer.get_modified():
            if question_message(self._window,'Script has been modified. Do you want to save it first?'):
                self.on_toolbutton_save(self._builder.get_object('save_toolbutton'))
                self._sourcebuffer.set_modified(False)
                return True
        self._sourcebuffer.set_modified(False)
        return False

    def on_modified_changed(self, sourcebuffer):
        self._builder.get_object('save_toolbutton').set_sensitive(sourcebuffer.get_modified())
        title=self._window.get_title()
        if title.endswith('*'):
            title=title[:-1].strip()
        if sourcebuffer.get_modified():
            self._window.set_title(title+' *')
        else:
            self._window.set_title(title)

    def on_toolbutton_new(self, toolbutton):
        self.confirm_save()
        self._sourcebuffer.set_text('')
        self._sourcebuffer.set_modified(False)

    def on_toolbutton_open(self, toolbutton):
        self.confirm_save()
        if not hasattr(self, '_filechooser'):
            self._filechooser = Gtk.FileChooserDialog('Open script file', self._window, Gtk.FileChooserAction.OPEN,
                                                      ['OK', Gtk.ResponseType.OK, 'Cancel', Gtk.ResponseType.CANCEL])
            self._filechooser.add_shortcut_folder(os.path.join(os.getcwd(),'scripts'))
        self._filechooser.set_action(Gtk.FileChooserAction.OPEN)
        self._filechooser.set_title('Open script file')
        self._filechooser.set_transient_for(self._window)
        if self._filechooser.run()==Gtk.ResponseType.OK:
            self._filename=self._filechooser.get_filename()
            self._window.set_title(self._filename)
        self._filechooser.hide()
        with open(self._filename, 'rt', encoding='utf-8') as f:
            self._sourcebuffer.set_text(f.read())
        self._sourcebuffer.set_modified(False)

    def on_toolbutton_save(self, toolbutton):
        if not hasattr(self, '_filename'):
            self.on_toolbutton_saveas(toolbutton)
        with open(self._filename, 'wt', encoding='utf-8') as f:
            f.write(self._sourcebuffer.get_text(self._sourcebuffer.get_start_iter(), self._sourcebuffer.get_end_iter(), True))
        self._sourcebuffer.set_modified(False)

    def on_toolbutton_saveas(self, toolbutton):
        if not hasattr(self, '_filechooser'):
            self._filechooser = Gtk.FileChooserDialog('Save script file as...', self._window,
                                                      Gtk.FileChooserAction.SAVE, ['OK', Gtk.ResponseType.OK, 'Cancel', Gtk.ResponseType.CANCEL])
            self._filechooser.add_shortcut_folder(os.path.join(os.getcwd(),'scripts'))
        self._filechooser.set_action(Gtk.FileChooserAction.SAVE)
        self._filechooser.set_do_overwrite_confirmation(True)
        self._filechooser.set_title('Save script file as...')
        self._filechooser.set_transient_for(self._window)
        if self._filechooser.run()==Gtk.ResponseType.OK:
            self._filename=self._filechooser.get_filename()
            if not self._filename.lower().endswith('.cct'):
                self._filename=self._filename+'.cct'
            self._window.set_title(self._filename)
            self.on_toolbutton_save(toolbutton)
        self._filechooser.hide()

    def on_toolbutton_undo(self, toolbutton):
        self._sourcebuffer.undo()

    def on_toolbutton_redo(self, toolbutton):
        self._sourcebuffer.redo()

    def on_toolbutton_cut(self, toolbutton):
        self._sourcebuffer.copy_clipboard(Gtk.Clipboard.get_default(Gdk.Display.get_default()))

    def on_toolbutton_copy(self, toolbutton):
        self._sourcebuffer.cut_clipboard(Gtk.Clipboard.get_default(Gdk.Display.get_default()),
                                         self._inhibit_close_reason is not None)

    def on_toolbutton_paste(self, toolbutton):
        self._sourcebuffer.paste_clipboard(Gtk.Clipboard.get_default(Gdk.Display.get_default()),
                                           self._inhibit_close_reason is not None)

    def on_toolbutton_execute(self, toolbutton):
        self._scriptcommand=Script(self._sourcebuffer.get_text(self._sourcebuffer.get_start_iter(),
                                                               self._sourcebuffer.get_end_iter(),
                                                               True))
        self._make_insensitive('Script is running', ['sourceview', 'new_toolbutton', 'save_toolbutton', 'saveas_toolbutton',
                                                     'open_toolbutton', 'undo_toolbutton', 'redo_toolbutton', 'cut_toolbutton',
                                                     'copy_toolbutton', 'paste_toolbutton', 'help_toolbutton', 'execute_toolbutton'])
        self._interpreter_connections=[
            self._instrument.interpreter.connect('cmd-return', self.on_script_end),
            self._instrument.interpreter.connect('cmd-fail', self.on_script_fail),
            self._instrument.interpreter.connect('pulse', self.on_script_pulse),
            self._instrument.interpreter.connect('progress',self.on_script_progress),
            self._instrument.interpreter.connect('cmd-message', self.on_script_message),
                                       ]
        self._instrument.interpreter.clear_flag(None)  # clear all flags
        flagsbb = self._builder.get_object('flags_buttonbox')
        for b in flagsbb:
            if b.get_active():
                self._instrument.interpreter.set_flag(b.get_label())

        self._builder.get_object('sourceview').set_editable(False)
        self._scriptconnections = [self._scriptcommand.connect('cmd-start', self.on_command_start)]
        buf = self._builder.get_object('messagesbuffer')
        buf.insert(buf.get_end_iter(),
                   '----------------------- %s -----------------------\n' % str(datetime.datetime.now()))
        self._builder.get_object('messagesview').scroll_to_iter(buf.get_end_iter(), 0, False, 0, 0)
        buf.place_cursor(buf.get_end_iter())
        try:
            self._instrument.interpreter.execute_command(self._scriptcommand)
        except Exception:
            self._cleanup()
            raise
        try:
            with open(self._filename[:-len('.cct')] + '.log', 'at', encoding='utf-8') as f:
                f.write(str(datetime.datetime.now()) + ': ----------------------- Started -----------------------\n')
        except AttributeError:
            pass

    def _cleanup(self):
        self._make_sensitive()
        for c in self._interpreter_connections:
            self._instrument.interpreter.disconnect(c)
        del self._interpreter_connections
        self._builder.get_object('sourceview').set_editable(True)
        self._sourcebuffer.remove_source_marks(
            self._sourcebuffer.get_start_iter(), self._sourcebuffer.get_end_iter(), 'Executing')
        self._builder.get_object('progressbar').set_text('')
        self._builder.get_object('progressbar').set_fraction(0)
        for c in self._scriptconnections:
            self._scriptcommand.disconnect(c)
        del self._scriptconnections
        del self._scriptcommand

    def on_script_end(self, interpreter, commandname, returnvalue):
        info_message(self._window,'Script ended','Result: %s'%str(returnvalue))
        self._cleanup()
        n=Notify.Notification(summary='Script ended',body='Script execution ended with result: %s'%str(returnvalue))
        n.set_image_from_pixbuf(GdkPixbuf.Pixbuf.new_from_file_at_size(pkg_resources.resource_filename('cct','resource/icons/scalable/cctlogo.svg'),256,256))
        n.show()

    def on_script_fail(self, interpreter, commandname, exc, tb):
        error_message(self._window, 'Error while executing script', tb)

    def on_script_pulse(self, interpreter, commandname, message):
        pb=self._builder.get_object('progressbar')
        pb.set_visible(True)
        pb.set_text(message)
        pb.pulse()

    def on_script_progress(self, interpreter, commandname, message, fraction):
        pb=self._builder.get_object('progressbar')
        pb.set_visible(True)
        pb.set_text(message)
        pb.set_fraction(fraction)

    def on_script_message(self, interpreter, commandname, message):
        buf=self._builder.get_object('messagesview').get_buffer()
        buf.insert(buf.get_end_iter(), message+'\n')
        self._builder.get_object('messagesview').scroll_to_iter(buf.get_end_iter(), 0,False, 0,0)
        buf.place_cursor(buf.get_end_iter())
        try:
            with open(self._filename[:-len('.cct')]+'.log','at', encoding='utf-8') as f:
                f.write(str(datetime.datetime.now())+': '+message+'\n')
        except AttributeError:
            pass

    def on_command_start(self, scriptcmd, lineno, command ):
        it = self._sourcebuffer.get_iter_at_line(lineno)
        self._sourcebuffer.remove_source_marks(
            self._sourcebuffer.get_start_iter(), self._sourcebuffer.get_end_iter(), 'Executing')
        self._sourcebuffer.create_source_mark(
            'Line #%d' % lineno, 'Executing', it)
        self._builder.get_object('sourceview').scroll_to_iter(it, 0, False, 0, 0)
        self._sourcebuffer.place_cursor(it)
        self._builder.get_object('progressbar').hide()

    def on_toolbutton_pause(self, toolbutton):
        pass

    def on_toolbutton_stop(self, toolbutton):
        self._instrument.interpreter.kill()

    def on_toolbutton_help(self, toolbutton):
        if not hasattr(self, '_helpdialog'):
            self._helpdialog = CommandHelpDialog('help_commandhelpbrowser.glade', 'commandhelpbrowser',
                                                 self._instrument, self._application, 'Help on commands')
            self._helpdialog.connect('insert', self.on_insert)
        self._helpdialog._window.show_all()

    def on_insert(self, helpdialog, text):
        self._sourcebuffer.insert_at_cursor(text)

    def on_close(self, widget, event=None):
        self.confirm_save()
        ToolWindow.on_close(self, widget, event)

    def on_flag_toggled(self, flagtoggle):
        if flagtoggle.get_active():
            self._instrument.interpreter.set_flag(flagtoggle.get_label())
        else:
            self._instrument.interpreter.clear_flag(flagtoggle.get_label())

class CommandHelpDialog(ToolWindow):
    __gsignals__ = {'insert': (GObject.SignalFlags.RUN_FIRST, None, (str,))}

    def _init_gui(self, *args):
        model = self._builder.get_object('commandnames')
        for command in sorted([c.name for c in Command.allcommands()]):
            model.append((command,))
        tc = Gtk.TreeViewColumn('Command', Gtk.CellRendererText(), text=0)
        self._builder.get_object('commandsview').append_column(tc)
        self._builder.get_object('commandsview').get_selection().select_iter(model.get_iter_first())

    def on_command_selected(self, treeviewselection):
        model, it = treeviewselection.get_selected()
        buf = self._builder.get_object('helptextbuffer')
        buf.set_text([c.__doc__ for c in Command.allcommands() if c.name == model[it][0]][0])

    def on_insert(self, button):
        model, it = self._builder.get_object('commandsview').get_selection().get_selected()
        self.emit('insert', model[it][0] + '()')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020  The SymbiFlow Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC

import os
import sys
import tqdm
import glob
import json
import pandas
from multiprocessing import Pool, cpu_count
from contextlib import redirect_stdout

from fpgaperf import run
from dataframe import generate_dataframe
import sow

class Runner:
    """Class to create a runner object that, given a list of tasks
    runs all of them parallely.
    """
    def __init__(
        self, task_list, verbose, out_prefix, root_dir, build_type,
        build_numbers, overwrite
    ):
        self.verbose = verbose
        self.out_prefix = out_prefix
        self.root_dir = root_dir
        self.build_format = "{:03d}"
        self.build_type = build_type
        self.results = dict()
        self.task_list = task_list
        self.build_numbers = build_numbers
        self.overwrite = overwrite

    def worker(self, arglist):
        """Single worker function that is run in the Pool of workers.

        This takes, as argument list, the various tasks to perform.
        """
        def eprint(*args, **kwargs):
            print(*args, file=sys.stderr, **kwargs)

        project, toolchain, board, seed, option, build_number = arglist

        build = self.build_format.format(build_number)

        # We don't want output of all subprocesses here
        # Log files for each build will be placed in build directory
        with redirect_stdout(open(os.devnull, 'w')):
            try:
                run(
                    board,
                    toolchain,
                    project,
                    None,  #params_file
                    option,  #params_string
                    None,  #out_dir
                    self.out_prefix,
                    self.overwrite,
                    self.verbose,
                    None,  #strategy
                    seed,
                    None,  #carry
                    build,
                    self.build_type,
                )
            except Exception as e:
                eprint("\n---------------------")
                eprint(
                    "ERROR: {} {} {} test has failed (build type {}, build nr. {})\n"
                    .format(project, toolchain, board, self.build_type, build)
                )
                # Limit output to max 300 characters for exhaust.py to make sure log files are not too large.
                exception_str = str(e)
                eprint(
                    "ERROR MESSAGE: ", (exception_str[:296] + " [...]")
                    if len(str(exception_str)) > 300 else exception_str
                )
                eprint("---------------------\n")

    def run(self):
        os.makedirs(os.path.expanduser(self.out_prefix), exist_ok=True)
        print('Writing to %s' % self.out_prefix)

        with Pool(cpu_count()) as pool:
            for _ in tqdm.tqdm(pool.imap_unordered(self.worker,
                                                   self.task_list),
                               total=len(self.task_list), unit='test'):
                pass

    def get_reports(self):
        reports = []
        for build_number in self.build_numbers:
            if build_number is not None:
                metadata_path = '*{}_{}*/meta.json'.format(
                    self.build_type, self.build_format.format(build_number)
                )
            else:
                metadata_path = '*{}*/meta.json'.format(self.build_type)
            for filename in glob.iglob(os.path.join(self.root_dir,
                                                    self.out_prefix,
                                                    metadata_path)):
                reports.append(filename)

        return reports

    def collect_results(self):
        reports = self.get_reports()

        if len(reports) == 0:
            return

        for report in reports:
            sow.merge(self.results, json.load(open(report, 'r')))

        dataframe = generate_dataframe(self.results)
        dataframe = dataframe.reset_index(drop=True)

        dataframe_path = os.path.join(
            self.root_dir, self.out_prefix, 'dataframe.json'
        )
        if os.path.exists(dataframe_path):
            old_dataframe = pandas.read_json(
                dataframe_path, convert_dates=False
            )
            dataframe = pandas.concat(
                [old_dataframe, dataframe], ignore_index=True
            )

        dataframe.to_json(dataframe_path)
<commit_msg>runner: allow printing last 1000 characters of the error message<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2020  The SymbiFlow Authors.
#
# Use of this source code is governed by a ISC-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/ISC
#
# SPDX-License-Identifier: ISC

import os
import sys
import tqdm
import glob
import json
import pandas
from multiprocessing import Pool, cpu_count
from contextlib import redirect_stdout

from fpgaperf import run
from dataframe import generate_dataframe
import sow

class Runner:
    """Class to create a runner object that, given a list of tasks
    runs all of them parallely.
    """
    def __init__(
        self, task_list, verbose, out_prefix, root_dir, build_type,
        build_numbers, overwrite
    ):
        self.verbose = verbose
        self.out_prefix = out_prefix
        self.root_dir = root_dir
        self.build_format = "{:03d}"
        self.build_type = build_type
        self.results = dict()
        self.task_list = task_list
        self.build_numbers = build_numbers
        self.overwrite = overwrite

    def worker(self, arglist):
        """Single worker function that is run in the Pool of workers.

        This takes, as argument list, the various tasks to perform.
        """
        def eprint(*args, **kwargs):
            print(*args, file=sys.stderr, **kwargs)

        project, toolchain, board, seed, option, build_number = arglist

        build = self.build_format.format(build_number)

        # We don't want output of all subprocesses here
        # Log files for each build will be placed in build directory
        with redirect_stdout(open(os.devnull, 'w')):
            try:
                run(
                    board,
                    toolchain,
                    project,
                    None,  #params_file
                    option,  #params_string
                    None,  #out_dir
                    self.out_prefix,
                    self.overwrite,
                    self.verbose,
                    None,  #strategy
                    seed,
                    None,  #carry
                    build,
                    self.build_type,
                )
            except Exception as e:
                eprint("\n---------------------")
                eprint(
                    "ERROR: {} {} {} test has failed (build type {}, build nr. {})\n"
                    .format(project, toolchain, board, self.build_type, build)
                )
                # Limit output to max 300 characters for exhaust.py to make sure log files are not too large.
                exception_str = str(e)
                eprint(
                    "ERROR MESSAGE: ",
                    ("[...]\n{}".format(exception_str[-1000:]))
                    if len(str(exception_str)) > 1000 else exception_str
                )
                eprint("---------------------\n")

    def run(self):
        os.makedirs(os.path.expanduser(self.out_prefix), exist_ok=True)
        print('Writing to %s' % self.out_prefix)

        with Pool(cpu_count()) as pool:
            for _ in tqdm.tqdm(pool.imap_unordered(self.worker,
                                                   self.task_list),
                               total=len(self.task_list), unit='test'):
                pass

    def get_reports(self):
        reports = []
        for build_number in self.build_numbers:
            if build_number is not None:
                metadata_path = '*{}_{}*/meta.json'.format(
                    self.build_type, self.build_format.format(build_number)
                )
            else:
                metadata_path = '*{}*/meta.json'.format(self.build_type)
            for filename in glob.iglob(os.path.join(self.root_dir,
                                                    self.out_prefix,
                                                    metadata_path)):
                reports.append(filename)

        return reports

    def collect_results(self):
        reports = self.get_reports()

        if len(reports) == 0:
            return

        for report in reports:
            sow.merge(self.results, json.load(open(report, 'r')))

        dataframe = generate_dataframe(self.results)
        dataframe = dataframe.reset_index(drop=True)

        dataframe_path = os.path.join(
            self.root_dir, self.out_prefix, 'dataframe.json'
        )
        if os.path.exists(dataframe_path):
            old_dataframe = pandas.read_json(
                dataframe_path, convert_dates=False
            )
            dataframe = pandas.concat(
                [old_dataframe, dataframe], ignore_index=True
            )

        dataframe.to_json(dataframe_path)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

from tastypie import fields
from tastypie.cache import SimpleCache
from tastypie.constants import ALL, ALL_WITH_RELATIONS

from akvo.api.fields import ConditionalFullToManyField

from akvo.rsr.models import Organisation, Project

from .resources import ConditionalFullResource, get_extra_thumbnails

class OrganisationMapResource(ConditionalFullResource):
    """
    a limited resource for delivering data to be used when creating maps
    """
    locations           = ConditionalFullToManyField('akvo.api.resources.OrganisationLocationResource', 'locations', null=True)
    primary_location    = fields.ToOneField('akvo.api.resources.OrganisationLocationResource', 'primary_location', full=True, null=True)

    class Meta:
        allowed_methods = ['get']
        queryset = Organisation.objects.all()
        resource_name = 'map_for_organisation'
        include_absolute_url = True
        cache = SimpleCache(timeout=900) # 15 minutes

        filtering       = dict(
            # other fields
            iati_org_id         = ALL,
            name                = ALL,
            organisation_type   = ALL,
            # foreign keys
            locations           = ALL_WITH_RELATIONS,
            partnerships        = ALL_WITH_RELATIONS,
            )

    def dehydrate(self, bundle):
        """ add thumbnails inline info for Organisation.logo
        """
        bundle = super(OrganisationMapResource, self).dehydrate(bundle)
        del bundle.data['description']
        bundle.data['logo'] = {
            'original': bundle.data['logo'],
            'thumbnails': get_extra_thumbnails(bundle.obj.logo),
            }
        return bundle

class ProjectMapResource(ConditionalFullResource):
    """
    a limited resource for delivering data to be used when creating maps
    """
    locations           = ConditionalFullToManyField('akvo.api.resources.ProjectLocationResource', 'locations')
    primary_location    = fields.ToOneField('akvo.api.resources.ProjectLocationResource', 'primary_location', full=True, null=True)

    class Meta:
        allowed_methods = ['get']
        queryset = Project.objects.published()
        resource_name = 'map_for_project'
        include_absolute_url = True
        cache = SimpleCache(timeout=900) # 15 minutes

        filtering               = dict(
            # other fields
            status              = ALL,
            title               = ALL,
            budget              = ALL,
            funds               = ALL,
            funds_needed        = ALL,
            # foreign keys
            benchmarks          = ALL_WITH_RELATIONS,
            budget_items        = ALL_WITH_RELATIONS,
            categories          = ALL_WITH_RELATIONS,
            goals               = ALL_WITH_RELATIONS,
            invoices            = ALL_WITH_RELATIONS,
            links               = ALL_WITH_RELATIONS,
            locations           = ALL_WITH_RELATIONS,
            partnerships        = ALL_WITH_RELATIONS,
            project_comments    = ALL_WITH_RELATIONS,
            project_updates     = ALL_WITH_RELATIONS,
        )

    def dehydrate(self, bundle):
        """ add thumbnails inline info for Project.current_image
        """
        bundle = super(ProjectMapResource, self).dehydrate(bundle)
        ignored_fields = ('goals_overview', 'current_status', 'project_plan', 'sustainability', 'background', 'project_rating', 'notes',)
        for field in ignored_fields:
            del bundle.data[field]
        bundle.data['current_image'] = {
            'original': bundle.data['current_image'],
            'thumbnails': get_extra_thumbnails(bundle.obj.current_image),
            }
        return bundle
<commit_msg>[#331] Optimizations of ProjectMapResource<commit_after># -*- coding: utf-8 -*-

# Akvo RSR is covered by the GNU Affero General Public License.
# See more details in the license.txt file located at the root folder of the Akvo RSR module.
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.

from tastypie import fields
from tastypie.cache import SimpleCache
from tastypie.constants import ALL, ALL_WITH_RELATIONS
from tastypie.resources import ModelResource

from akvo.api.fields import ConditionalFullToManyField

from akvo.rsr.models import Organisation, Project

from .resources import ConditionalFullResource, get_extra_thumbnails

class OrganisationMapResource(ConditionalFullResource):
    """
    a limited resource for delivering data to be used when creating maps
    """
    locations           = ConditionalFullToManyField('akvo.api.resources.OrganisationLocationResource', 'locations', null=True)
    primary_location    = fields.ToOneField('akvo.api.resources.OrganisationLocationResource', 'primary_location', full=True, null=True)

    class Meta:
        allowed_methods = ['get']
        queryset = Organisation.objects.all()
        resource_name = 'map_for_organisation'
        include_absolute_url = True
        cache = SimpleCache(timeout=900) # 15 minutes

        filtering       = dict(
            # other fields
            iati_org_id         = ALL,
            name                = ALL,
            organisation_type   = ALL,
            # foreign keys
            locations           = ALL_WITH_RELATIONS,
            partnerships        = ALL_WITH_RELATIONS,
            )

    def dehydrate(self, bundle):
        """ add thumbnails inline info for Organisation.logo
        """
        bundle = super(OrganisationMapResource, self).dehydrate(bundle)
        del bundle.data['description']
        bundle.data['logo'] = {
            'original': bundle.data['logo'],
            'thumbnails': get_extra_thumbnails(bundle.obj.logo),
            }
        return bundle

class ProjectMapResource(ModelResource):
    """
    a limited resource for delivering data to be used when creating maps
    """
    #locations           = ConditionalFullToManyField('akvo.api.resources.ProjectLocationResource', 'locations')
    primary_location    = fields.ToOneField('akvo.api.resources.ProjectLocationResource', 'primary_location', full=True, null=True)

    class Meta:
        allowed_methods = ['get']
        queryset = Project.objects.select_related('primary_location', 'primary_location__country').published()
        resource_name = 'map_for_project'
        include_absolute_url = True
        cache = SimpleCache(timeout=900) # 15 minutes

        filtering               = dict(
            # other fields
            status              = ALL,
            title               = ALL,
            budget              = ALL,
            funds               = ALL,
            funds_needed        = ALL,
            # foreign keys
            benchmarks          = ALL_WITH_RELATIONS,
            budget_items        = ALL_WITH_RELATIONS,
            categories          = ALL_WITH_RELATIONS,
            goals               = ALL_WITH_RELATIONS,
            invoices            = ALL_WITH_RELATIONS,
            links               = ALL_WITH_RELATIONS,
            locations           = ALL_WITH_RELATIONS,
            partnerships        = ALL_WITH_RELATIONS,
            project_comments    = ALL_WITH_RELATIONS,
            project_updates     = ALL_WITH_RELATIONS,
        )

    def dehydrate(self, bundle):
        """ add thumbnails inline info for Project.current_image
        """
        bundle = super(ProjectMapResource, self).dehydrate(bundle)
        ignored_fields = ('goals_overview', 'current_status', 'project_plan', 'sustainability', 'background', 'project_rating', 'notes',)
        for field in ignored_fields:
            del bundle.data[field]
        bundle.data['current_image'] = {
            'original': bundle.data['current_image'],
            'thumbnails': get_extra_thumbnails(bundle.obj.current_image),
            }
        return bundle
<|endoftext|>


<commit_before>import datetime

from django.test import TestCase

from custom.icds_reports.const import GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
from custom.icds_reports.reports.governance_apis import get_home_visit_data, get_state_names
from custom.icds_reports.utils import india_now

class GovernanceApiTest(TestCase):

    def test_data_fetching_total_record_count_for_home_visit_api(self):
        """
        test to check the total count of records that are returned from the home visit api
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 5, order, query_filters)
        expected_count = 55
        self.assertEqual(count, expected_count)

    def test_data_fetching_without_start_for_home_visit_api(self):
        """
        test to check the first record that is returned from the home visit api without start parameter
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 5, order, query_filters)
        expected_first_row = {
            'state': 'st1', 'district': 'd1', 'block': 'b1', 'sector': 's1', 'awc': 'a1',
            'month': datetime.date(2017, 5, 1), 'valid_visits': 0, 'expected_visits': 4,
        }
        self.assertEqual(data[0], expected_first_row)

    def test_data_fetching_with_start_for_home_visit_api(self):
        """
        test to check the first record that is returned from the home visit api with start parameter
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(1, limit,
                                          2017, 5, order, query_filters)
        expected_first_row = {
            'state': 'st1', 'district': 'd1', 'block': 'b1', 'sector': 's1', 'awc': 'a17',
            'month': datetime.date(2017, 5, 1), 'valid_visits': 0, 'expected_visits': 3
        }
        self.assertEqual(data[0], expected_first_row)

    def test_data_fetching_total_record_count_for_no_records_for_home_visit_api(self):
        """
        test to check the no records are returned from the home visit api
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 6, order, query_filters)
        expected_count = 0
        self.assertEqual(count, expected_count)
        self.assertEqual(data, [])

    def test_data_fetching_total_record_count_with_state_id_for_home_visit_api(self):
        """
        test to check the total count of records that are returned from the home visit api
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5, 'state_id': 'st1'}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 5, order, query_filters)
        expected_count = 26
        self.assertEqual(count, expected_count)

    def test_data_fetching_state_names_api(self):
        """
        test to check if state names are getting retreived from the api
        """
        data = get_state_names()
        expected_count = 6
        self.assertEqual(len(data), expected_count)
<commit_msg>removed unused import<commit_after>import datetime

from django.test import TestCase

from custom.icds_reports.const import GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
from custom.icds_reports.reports.governance_apis import get_home_visit_data, get_state_names

class GovernanceApiTest(TestCase):

    def test_data_fetching_total_record_count_for_home_visit_api(self):
        """
        test to check the total count of records that are returned from the home visit api
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 5, order, query_filters)
        expected_count = 55
        self.assertEqual(count, expected_count)

    def test_data_fetching_without_start_for_home_visit_api(self):
        """
        test to check the first record that is returned from the home visit api without start parameter
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 5, order, query_filters)
        expected_first_row = {
            'state': 'st1', 'district': 'd1', 'block': 'b1', 'sector': 's1', 'awc': 'a1',
            'month': datetime.date(2017, 5, 1), 'valid_visits': 0, 'expected_visits': 4,
        }
        self.assertEqual(data[0], expected_first_row)

    def test_data_fetching_with_start_for_home_visit_api(self):
        """
        test to check the first record that is returned from the home visit api with start parameter
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(1, limit,
                                          2017, 5, order, query_filters)
        expected_first_row = {
            'state': 'st1', 'district': 'd1', 'block': 'b1', 'sector': 's1', 'awc': 'a17',
            'month': datetime.date(2017, 5, 1), 'valid_visits': 0, 'expected_visits': 3
        }
        self.assertEqual(data[0], expected_first_row)

    def test_data_fetching_total_record_count_for_no_records_for_home_visit_api(self):
        """
        test to check the no records are returned from the home visit api
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 6, order, query_filters)
        expected_count = 0
        self.assertEqual(count, expected_count)
        self.assertEqual(data, [])

    def test_data_fetching_total_record_count_with_state_id_for_home_visit_api(self):
        """
        test to check the total count of records that are returned from the home visit api
        """
        limit = GOVERNANCE_API_HOME_VISIT_RECORDS_PAGINATION
        query_filters = {'aggregation_level': 5, 'state_id': 'st1'}
        order = ['state_name', 'district_name', 'block_name', 'supervisor_name', 'awc_name']
        data, count = get_home_visit_data(0, limit,
                                          2017, 5, order, query_filters)
        expected_count = 26
        self.assertEqual(count, expected_count)

    def test_data_fetching_state_names_api(self):
        """
        test to check if state names are getting retreived from the api
        """
        data = get_state_names()
        expected_count = 6
        self.assertEqual(len(data), expected_count)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# author:         rex
# blog:           http://iregex.org
# filename        tr.py
# created:        2010-08-01 20:24
# source uri:     http://iregex.org/blog/trie-in-python.html

import re

class Trie():
    """Regexp::Trie in python"""

    def __init__(self):
        self.data = dict()

    def add(self, word):
        ref = self.data
        for char in word:
            ref[char] = char in ref and ref[char] or dict()
            ref = ref[char]
        ref[""] = 1

    def dump(self):
        return self.data

    def quote(self, char):
        return re.escape(char)

    def _regexp(self, pData):
        data = pData
        if "" in data and len(data.keys()) == 1:
            return None

        alt = list()
        cc = list()
        q = 0
        for char in sorted(data.keys()):
            if isinstance(data[char], dict):
                try:
                    recurse = self._regexp(data[char])
                    alt.append(self.quote(char)+recurse)
                except:
                    cc.append(self.quote(char))
            else:
                q = 1
        cconly = not len(alt) > 0

        if len(cc) > 0:
            if len(cc) == 1:
                alt.append(cc[0])
            else:
                alt.append("[%s]" % "".join(cc))

        if len(alt) == 1:
            result = alt[0]
        else:
            result = "(?:%s)" % "|".join(alt)

        if q:
            if cconly:
                result += "?"
            else:
                result = "(?:%s)?" % result
        return result

    def regexp(self):
        return self._regexp(self.dump())

if __name__ == '__main__':
    a = Trie()
    for w in ['foobar', 'foobah', 'fooxar', 'foozap', 'fooza']:
        a.add(w)
    print a.regexp()
<commit_msg>added my normal import header and corrected file name<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# author:         rex
# blog:           http://iregex.org
# filename        trie.py
# created:        2010-08-01 20:24
# source uri:     http://iregex.org/blog/trie-in-python.html

# Standard Library
import re

class Trie():
    """Regexp::Trie in python"""

    def __init__(self):
        self.data = dict()

    def add(self, word):
        ref = self.data
        for char in word:
            ref[char] = char in ref and ref[char] or dict()
            ref = ref[char]
        ref[""] = 1

    def dump(self):
        return self.data

    def quote(self, char):
        return re.escape(char)

    def _regexp(self, pData):
        data = pData
        if "" in data and len(data.keys()) == 1:
            return None

        alt = list()
        cc = list()
        q = 0
        for char in sorted(data.keys()):
            if isinstance(data[char], dict):
                try:
                    recurse = self._regexp(data[char])
                    alt.append(self.quote(char)+recurse)
                except:
                    cc.append(self.quote(char))
            else:
                q = 1
        cconly = not len(alt) > 0

        if len(cc) > 0:
            if len(cc) == 1:
                alt.append(cc[0])
            else:
                alt.append("[%s]" % "".join(cc))

        if len(alt) == 1:
            result = alt[0]
        else:
            result = "(?:%s)" % "|".join(alt)

        if q:
            if cconly:
                result += "?"
            else:
                result = "(?:%s)?" % result
        return result

    def regexp(self):
        return self._regexp(self.dump())

if __name__ == '__main__':
    a = Trie()
    for w in ['foobar', 'foobah', 'fooxar', 'foozap', 'fooza']:
        a.add(w)
    print a.regexp()
<|endoftext|>


<commit_before>import os
from celery import Celery
from importlib import import_module
from lib.config import ALERTS, LOGGING, RABBITMQ
from logging.config import dictConfig

# Alert files to include
alerts_include = []
for alert in ALERTS.keys():
    alerts_include.append(".".join((alert).split(".")[:-1]))
alerts_include = list(set(alerts_include))

# XXX TBD this should get wrapped into an object that provides pyconfig
if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
    BROKER_URL = "sqs://@"
    BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1]}
    CELERY_RESULT_BACKEND = None
    alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
    CELERY_DEFAULT_QUEUE = alert_queue_name
    CELERY_QUEUES = {
        {alert_queue_name}: {"exchange": "alert_queue_name, "binding_key": alert_queue_name}
    }
else:
    BROKER_URL = "amqp://{0}:{1}@{2}:{3}//".format(
        RABBITMQ["mquser"], RABBITMQ["mqpassword"], RABBITMQ["mqserver"], RABBITMQ["mqport"]
    )
    CELERY_QUEUES = {
        "celery-default": {"exchange": "celery-default", "binding_key": "celery-default"}
    }
    CELERY_DEFAULT_QUEUE = 'celery-default'

CELERY_DISABLE_RATE_LIMITS = True
CELERYD_CONCURRENCY = 1
CELERY_IGNORE_RESULT = True
CELERY_ACCEPT_CONTENT = ["json"]
CELERY_TASK_SERIALIZER = "json"
CELERYBEAT_SCHEDULE = {}

# Register frequency of the tasks in the scheduler
for alert in ALERTS.keys():
    CELERYBEAT_SCHEDULE[alert] = {
        "task": alert,
        "schedule": ALERTS[alert]["schedule"],
        "options": {"queue": "celery-default", "exchange": "celery-default"},
    }
    # add optional parameters:
    if "args" in ALERTS[alert]:
        CELERYBEAT_SCHEDULE[alert]["args"] = ALERTS[alert]["args"]
    if "kwargs" in ALERTS[alert]:
        CELERYBEAT_SCHEDULE[alert]["kwargs"] = ALERTS[alert]["kwargs"]

# Load logging config
dictConfig(LOGGING)

# print CELERYBEAT_SCHEDULE

# Optional configuration, see the application user guide.
# app.conf.update(
#     CELERY_TASK_RESULT_EXPIRES=3600,
# )
app = Celery("alerts", include=alerts_include)
app.config_from_object("celeryconfig", force=True)

# As a result of celery 3 to celery 4, we need to dynamically
# register all of the alert tasks specifically
for alert_namespace in CELERYBEAT_SCHEDULE:
    try:
        alert_tokens = alert_namespace.split(".")
        alert_module_name = alert_tokens[0]
        alert_classname = alert_tokens[1]
        alert_module = import_module(alert_module_name)
        alert_class = getattr(alert_module, alert_classname)
        app.register_task(alert_class())
    except ImportError as e:
        print("Error importing {}").format(alert_namespace)
        print(e)
        pass
    except Exception as e:
        print("Error addding alert")
        print(e)

if __name__ == "__main__":
    app.start()
<commit_msg>fix quote typo<commit_after>import os
from celery import Celery
from importlib import import_module
from lib.config import ALERTS, LOGGING, RABBITMQ
from logging.config import dictConfig

# Alert files to include
alerts_include = []
for alert in ALERTS.keys():
    alerts_include.append(".".join((alert).split(".")[:-1]))
alerts_include = list(set(alerts_include))

# XXX TBD this should get wrapped into an object that provides pyconfig
if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
    BROKER_URL = "sqs://@"
    BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1]}
    CELERY_RESULT_BACKEND = None
    alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
    CELERY_DEFAULT_QUEUE = alert_queue_name
    CELERY_QUEUES = {
        {alert_queue_name}: {"exchange": alert_queue_name, "binding_key": alert_queue_name}
    }
else:
    BROKER_URL = "amqp://{0}:{1}@{2}:{3}//".format(
        RABBITMQ["mquser"], RABBITMQ["mqpassword"], RABBITMQ["mqserver"], RABBITMQ["mqport"]
    )
    CELERY_QUEUES = {
        "celery-default": {"exchange": "celery-default", "binding_key": "celery-default"}
    }
    CELERY_DEFAULT_QUEUE = 'celery-default'

CELERY_DISABLE_RATE_LIMITS = True
CELERYD_CONCURRENCY = 1
CELERY_IGNORE_RESULT = True
CELERY_ACCEPT_CONTENT = ["json"]
CELERY_TASK_SERIALIZER = "json"
CELERYBEAT_SCHEDULE = {}

# Register frequency of the tasks in the scheduler
for alert in ALERTS.keys():
    CELERYBEAT_SCHEDULE[alert] = {
        "task": alert,
        "schedule": ALERTS[alert]["schedule"],
        "options": {"queue": "celery-default", "exchange": "celery-default"},
    }
    # add optional parameters:
    if "args" in ALERTS[alert]:
        CELERYBEAT_SCHEDULE[alert]["args"] = ALERTS[alert]["args"]
    if "kwargs" in ALERTS[alert]:
        CELERYBEAT_SCHEDULE[alert]["kwargs"] = ALERTS[alert]["kwargs"]

# Load logging config
dictConfig(LOGGING)

# print CELERYBEAT_SCHEDULE

# Optional configuration, see the application user guide.
# app.conf.update(
#     CELERY_TASK_RESULT_EXPIRES=3600,
# )
app = Celery("alerts", include=alerts_include)
app.config_from_object("celeryconfig", force=True)

# As a result of celery 3 to celery 4, we need to dynamically
# register all of the alert tasks specifically
for alert_namespace in CELERYBEAT_SCHEDULE:
    try:
        alert_tokens = alert_namespace.split(".")
        alert_module_name = alert_tokens[0]
        alert_classname = alert_tokens[1]
        alert_module = import_module(alert_module_name)
        alert_class = getattr(alert_module, alert_classname)
        app.register_task(alert_class())
    except ImportError as e:
        print("Error importing {}").format(alert_namespace)
        print(e)
        pass
    except Exception as e:
        print("Error addding alert")
        print(e)

if __name__ == "__main__":
    app.start()
<|endoftext|>


<commit_before>import datetime
from flask_restful import Resource, request, abort
from flask_restful_swagger import swagger
from hpcpm.api import log
from hpcpm.api.helpers.database import database
from hpcpm.api.helpers.utils import abort_when_not_int, abort_when_node_not_found, abort_when_device_not_found
from hpcpm.api.helpers.constants import COMPUTATION_NODE_PARAM_NAME, COMPUTATION_NODE_NOT_FOUND_RESPONSE, \
    DEVICE_NOT_FOUND_AND_COMPUTATION_NODE_FETCHED_RESPONSES, DEVICE_IDENTIFIER_PARAM, DATETIME_PARAM, \
    STATISTICS_DATA_UPDATED_RESPONSE, POWER_USAGE_PARAM

class StatisticsData(Resource):
    @swagger.operation(
        notes='This endpoint is used for setting statistics data for given device and time.',
        nickname='/nodes/computation_node/<string:name>/<string:device_id>/statistics_data/<string:date_time>',
        parameters=[
            COMPUTATION_NODE_PARAM_NAME,
            DEVICE_IDENTIFIER_PARAM,
            DATETIME_PARAM,
            POWER_USAGE_PARAM
        ],
        responseMessages=[
            COMPUTATION_NODE_NOT_FOUND_RESPONSE,
            STATISTICS_DATA_UPDATED_RESPONSE
        ]
    )
    def put(self, name, device_id, date_time):
        power_usage = request.args.get('power_usage')
        abort_when_not_int(power_usage)
        power_usage = int(power_usage)

        try:
            datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M')
        except ValueError:
            abort(400)

        computation_node = abort_when_node_not_found(name)

        abort_when_device_not_found(device_id, computation_node)

        if database.check_stats_present(name, device_id):
            database.update_stats_data(name, device_id, {date_time: power_usage})
            log.debug('Stats updated for device %s:%s, date: %s, power usage: %s', name, device_id, date_time,
                      power_usage)
        else:
            database.replace_stats_data(name, device_id, {'name': name, 'device_id': device_id, date_time: power_usage})
            log.debug('Stats set for device %s:%s, date: %s, power usage: %s', name, device_id, date_time, power_usage)

        return 'Statistics gathering interval successfully set', 201

    @swagger.operation(
        notes='This endpoint is used for getting statistics data from database for given device and time.',
        nickname='/nodes/computation_node/<string:name>/<string:device_id>/statistics_data/<string:date_time>',
        parameters=[
            COMPUTATION_NODE_PARAM_NAME,
            DEVICE_IDENTIFIER_PARAM,
            DATETIME_PARAM
        ],
        responseMessages=DEVICE_NOT_FOUND_AND_COMPUTATION_NODE_FETCHED_RESPONSES
    )
    def get(self, name, device_id, date_time):
        try:
            datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M')
        except ValueError:
            abort(400)

        computation_node = abort_when_node_not_found(name)

        abort_when_device_not_found(device_id, computation_node)

        result = database.get_stats_data_for_time(name, device_id, date_time)

        if not result:
            log.info('No such statistic for device %s:%s', name, device_id)
            abort(404)
        if not result.get(date_time):
            log.info('No statistics for %s', date_time)
            abort(404)
        log.info('Successfully get device %s:%s statistics data info: %s', name, device_id, result)
        return result, 200

    @swagger.operation(
        notes='This endpoint is used for removing statistics data from database',
        nickname='/nodes/computation_node/<string:name>/<string:device_id>/statistics_data/<string:date_time>',
        parameters=[
            COMPUTATION_NODE_PARAM_NAME,
            DEVICE_IDENTIFIER_PARAM,
            DATETIME_PARAM
        ],
        responseMessages=DEVICE_NOT_FOUND_AND_COMPUTATION_NODE_FETCHED_RESPONSES
    )
    def delete(self, name, device_id, date_time):
        try:
            datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M')
        except ValueError:
            abort(400)

        computation_node = abort_when_node_not_found(name)
        abort_when_device_not_found(device_id, computation_node)

        result = database.delete_stats_for_time(name, device_id, {date_time: ""})
        if not result:
            log.info('No such statistic for device %s:%s', name, device_id)
            abort(404)
        if not result.matched_count:
            log.info('Did not manage to delete statistic for device %s:%s', name, device_id)
            abort(404)
        log.info('Successfully deleted device %s:%s statistics data info', name, device_id)
        return 204
<commit_msg>updating next measurement time on data put<commit_after>import datetime
from flask_restful import Resource, request, abort
from flask_restful_swagger import swagger
from hpcpm.api import log
from hpcpm.api.helpers.database import database
from hpcpm.api.helpers.utils import abort_when_not_int, abort_when_node_not_found, abort_when_device_not_found
from hpcpm.api.helpers.constants import COMPUTATION_NODE_PARAM_NAME, COMPUTATION_NODE_NOT_FOUND_RESPONSE, \
    DEVICE_NOT_FOUND_AND_COMPUTATION_NODE_FETCHED_RESPONSES, DEVICE_IDENTIFIER_PARAM, DATETIME_PARAM, \
    STATISTICS_DATA_UPDATED_RESPONSE, POWER_USAGE_PARAM

class StatisticsData(Resource):
    @swagger.operation(
        notes='This endpoint is used for setting statistics data for given device and time.',
        nickname='/nodes/computation_node/<string:name>/<string:device_id>/statistics_data/<string:date_time>',
        parameters=[
            COMPUTATION_NODE_PARAM_NAME,
            DEVICE_IDENTIFIER_PARAM,
            DATETIME_PARAM,
            POWER_USAGE_PARAM
        ],
        responseMessages=[
            COMPUTATION_NODE_NOT_FOUND_RESPONSE,
            STATISTICS_DATA_UPDATED_RESPONSE
        ]
    )
    def put(self, name, device_id, date_time):
        power_usage = request.args.get('power_usage')
        abort_when_not_int(power_usage)
        power_usage = int(power_usage)

        try:
            datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M')
        except ValueError:
            abort(400)

        computation_node = abort_when_node_not_found(name)

        abort_when_device_not_found(device_id, computation_node)

        if database.check_stats_present(name, device_id):
            database.update_stats_data(name, device_id, {date_time: power_usage})
            log.debug('Stats updated for device %s:%s, date: %s, power usage: %s', name, device_id, date_time,
                      power_usage)
        else:
            database.replace_stats_data(name, device_id, {'name': name, 'device_id': device_id, date_time: power_usage})
            log.debug('Stats set for device %s:%s, date: %s, power usage: %s', name, device_id, date_time, power_usage)

        interval_info = database.get_stats_interval_info(name, device_id)
        interval = datetime.timedelta(minutes=interval_info['interval'])
        new_next_measurement = (datetime.datetime.utcnow() + interval)
        new_next_measurement_str = new_next_measurement.strftime('%Y-%m-%dT%H:%M')  # pylint: disable=no-member
        interval_info['next_measurement'] = new_next_measurement_str

        database.replace_stats_interval_info(name, device_id, interval_info)

        return 'Statistics data successfully set', 201

    @swagger.operation(
        notes='This endpoint is used for getting statistics data from database for given device and time.',
        nickname='/nodes/computation_node/<string:name>/<string:device_id>/statistics_data/<string:date_time>',
        parameters=[
            COMPUTATION_NODE_PARAM_NAME,
            DEVICE_IDENTIFIER_PARAM,
            DATETIME_PARAM
        ],
        responseMessages=DEVICE_NOT_FOUND_AND_COMPUTATION_NODE_FETCHED_RESPONSES
    )
    def get(self, name, device_id, date_time):
        try:
            datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M')
        except ValueError:
            abort(400)

        computation_node = abort_when_node_not_found(name)

        abort_when_device_not_found(device_id, computation_node)

        result = database.get_stats_data_for_time(name, device_id, date_time)

        if not result:
            log.info('No such statistic for device %s:%s', name, device_id)
            abort(404)
        if not result.get(date_time):
            log.info('No statistics for %s', date_time)
            abort(404)
        log.info('Successfully get device %s:%s statistics data info: %s', name, device_id, result)
        return result, 200

    @swagger.operation(
        notes='This endpoint is used for removing statistics data from database',
        nickname='/nodes/computation_node/<string:name>/<string:device_id>/statistics_data/<string:date_time>',
        parameters=[
            COMPUTATION_NODE_PARAM_NAME,
            DEVICE_IDENTIFIER_PARAM,
            DATETIME_PARAM
        ],
        responseMessages=DEVICE_NOT_FOUND_AND_COMPUTATION_NODE_FETCHED_RESPONSES
    )
    def delete(self, name, device_id, date_time):
        try:
            datetime.datetime.strptime(date_time, '%Y-%m-%dT%H:%M')
        except ValueError:
            abort(400)

        computation_node = abort_when_node_not_found(name)
        abort_when_device_not_found(device_id, computation_node)

        result = database.delete_stats_for_time(name, device_id, {date_time: ""})
        if not result:
            log.info('No such statistic for device %s:%s', name, device_id)
            abort(404)
        if not result.matched_count:
            log.info('Did not manage to delete statistic for device %s:%s', name, device_id)
            abort(404)
        log.info('Successfully deleted device %s:%s statistics data info', name, device_id)
        return 204
<|endoftext|>


<commit_before>"""Utilities for reformatting and analyzing spiking data"""
import numpy as np
from copy import deepcopy

def to_spiketimes(times, spikes):
    """Convert spikes to two 1d arrays"""

    n_steps = len(times)
    n = spikes.shape[1]

    ns, ts = [], []
    for i in range(n_steps):
        for j in range(n):
            if spikes[i,j] == 1:
                ns.append(j)  # save neuron and
                ts.append(times[i])  # look up dt time

    return np.array(ns), np.array(ts)

def to_spikedict(ns, ts):
    """Convert from arrays to a neuron-keyed dict"""
    d_sp = {}
    for n, t in zip(ns, ts):
        try:
            d_sp[n].append(t)
        except KeyError:
            d_sp[n] = [t, ]

    for k in d_sp.keys():
        d_sp[k] = np.array(d_sp[k])

    return d_sp

def fano(spikes):
    """Calculate spike-count Fano"""
    return spikes.sum(0).std() ** 2 / spikes.sum(0).mean()

def isi(d_sp):
    """ISIs, in a neuron-keyed dict"""

    d_isi = {}
    for k, v in d_sp.items():
        tlast = 0

        intervals = []
        for t in v:
            intervals.append(t - tlast)
            tlast = deepcopy(t)

        d_isi[k] = np.array(intervals)

    return d_isi

<commit_msg>Added several new fn to util.<commit_after># -*- coding: utf-8 -*-

"""Utilities for reformatting and analyzing spiking data"""
import numpy as np
from copy import deepcopy
from itertools import product

def to_spiketimes(times, spikes):
    """Convert spikes to two 1d arrays"""

    n_steps = len(times)
    n = spikes.shape[1]

    ns, ts = [], []
    for i in range(n_steps):
        for j in range(n):
            if spikes[i, j] == 1:
                ns.append(j)  # save neuron and
                ts.append(times[i])  # look up dt time

    return np.array(ns), np.array(ts)

def to_spikedict(ns, ts):
    """Convert from seperate time and neuron
    arrays to a neuron-keyed dict"""
    d_sp = {}
    for n, t in zip(ns, ts):
        try:
            d_sp[n].append(t)
        except KeyError:
            d_sp[n] = [t, ]

    for k in d_sp.keys():
        d_sp[k] = np.array(d_sp[k])

    return d_sp

def spikedict_to(d_sp):
    """"Undoes `to_spikedict`"""

    ts, ns = [], []
    for n, ts_n in d_sp.iteritems():
        ns.extend([n] * len(ts_n))
        ts.extend(list(ts_n))

    return np.array(ns), np.array(ts)

def ts_sort(ns, ts):
    """Sort by ts"""
    ts = np.array(ts)
    ns = np.array(ns)
    idx = ts.argsort()

    return ns[idx], ts[idx]

def bin_times(ts, t_range, dt):
    """ts into a grid of dt sized bins"""

    if len(t_range) != 2:
        raise ValueError("t_range must contain two elements")
    if t_range[0] > t_range[1]:
        raise ValueError("t_range[0] must be less then t_range[1]")

    n_sample = int((t_range[1] - t_range[0]) * (1.0 / dt))

    bins = np.linspace(t_range[0], t_range[1], n_sample)
    binned, _ = np.histogram(ts[1:], bins=bins)

    return bins[1:], binned

def levenshtein(a, b):
    """Calculates the Levenshtein distance between a and b.

    Note: a and b are two sequences
    """

    a = list(a)
    b = list(b)
    n, m = len(a), len(b)

    # Make sure n <= m, to use O(min(n,m)) space
    if n > m:
        a, b = b, a
        n, m = m, n

    current = range(n + 1)
    for i in range(1, m + 1):
        previous, current = current, [i] + [0] * n
        for j in range(1, n + 1):
            add, delete = previous[j] + 1, current[j - 1] + 1
            change = previous[j - 1]
            if a[j - 1] != b[i - 1]:
                change = change + 1
            current[j] = min(add, delete, change)

    return current[-1]

def kappa(ns1, ts1, ns2, ts2, t_range, dt):
    """Measure Bruno's Kappa correlation[0].

    [0]: Wang, X.-J. & Buzsaki, G., 1996. Gamma Oscillation by Synaptic
    Inhibition in a Hippocampal Interneuronal Network Model. J. Neurosci.,
    16(20), pp.6402–6413.
    """

    d_1 = to_spikedict(ns1, ts1)
    d_2 = to_spikedict(ns2, ts2)

    pairs = product(list(np.unique(ns1)), list(np.unique(ns2)))
    corrs = []
    for n1, n2 in pairs:
        _, b1 = bin_times(d_1[n1], t_range, dt)
        _, b2 = bin_times(d_2[n2], t_range, dt)
        b1[b1 > 1] = 1
        b2[b2 > 1] = 1
        corrs.append(_kappa(b1, b2))

    return np.nanmean(corrs)

def _kappa(bini, binj):
    return np.sum(bini * binj) / np.sqrt(np.sum(bini) * np.sum(binj))

def fano(ns, ts):
    """Calculate isi Fano"""

    d_sp = isi(ns, ts)
    d_fano = {}
    for n, v in d_sp.items():
        d_fano[n] = v.std() ** 2 / v.mean()

    return d_fano

def isi(ns, ts):
    """Return ISIs, in a neuron-keyed dict"""

    d_sp = to_spikedict(ns, ts)

    d_isi = {}
    for k, v in d_sp.items():
        tlast = 0

        intervals = []
        for t in v:
            intervals.append(t - tlast)
            tlast = deepcopy(t)

        d_isi[k] = np.array(intervals)

    return d_isi
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Zip up cleaned raw data files (and error logs) and archive zip files.
"""
import os
import logging
import shutil
from sys import exit
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from django.core.files import File
from calaccess_raw import get_download_directory
from calaccess_raw.management.commands import CalAccessCommand
from calaccess_raw.models.tracking import RawDataVersion
logger = logging.getLogger(__name__)

class Command(CalAccessCommand):
    """
    Zip up cleaned raw data files (and error logs) and archive zip files.
    """
    help = 'Zip up cleaned raw data files (and error logs) and archive zip files'

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)
        self.clean_zip_path = os.path.join(
            get_download_directory(),
            'calaccess_cleaned.zip'
        )

        self.data_dir = get_download_directory()

        if os.path.exists(self.data_dir):
            shutil.rmtree(self.data_dir)

        os.makedirs(self.data_dir)

        versions = RawDataVersion.objects.filter(clean_zip_archive='')

        if versions:
            for version in versions:
                print version.release_datetime
                logger.debug(
                    'Creating zip file for {:%Y-%m-%d_%H-%M-%S} version'.format(
                        version.release_datetime
                    )
                )

                self.download_clean_files(version)
                self.create_zip_file(version)
                self.archive_zip_file(version)

    def download_clean_files(self, version):
        for raw_file in version.files.all():
            if raw_file.clean_file_archive:
                logger.debug(
                    'Downloading {0}'.format(os.path.basename(
                            raw_file.clean_file_archive.name
                        )
                    )
                )
                local_clean_file_path = os.path.join(
                    self.data_dir,
                    os.path.basename(raw_file.clean_file_archive.name),
                )
                local_clean_file = open(local_clean_file_path, 'wb')
                local_clean_file.write(raw_file.clean_file_archive.read())
                local_clean_file.close()
            if raw_file.error_log_archive:
                logger.debug(
                    'Downloading {0}'.format(os.path.basename(
                            raw_file.error_log_archive.name
                        )
                    )
                )
                local_error_file_path = os.path.join(
                    self.data_dir,
                    os.path.basename(raw_file.error_log_archive.name),
                )
                local_error_file = open(local_error_file_path, 'wb')
                local_error_file.write(raw_file.error_log_archive.read())
                local_error_file.close()

    def create_zip_file(self, version):
        logger.debug('Creating zip file')
        compression = ZIP_DEFLATED
        try:
            zf = ZipFile(self.clean_zip_path, 'w', compression, allowZip64=True)
        except RuntimeError:
            logger.error('Zip file cannot be compressed (check zlib module).')
            compression = ZIP_STORED
            zf = ZipFile(self.clean_zip_path, 'w', compression, allowZip64=True)

        csv_files = [f for f in os.listdir(self.data_dir) if '.csv' in f]

        for f in csv_files:
            logger.debug('Writing %s to zip file' % os.path.basename(f))
            file_path = os.path.join(self.data_dir, f)
            zf.write(file_path, f)
            os.remove(file_path)

        zf.close()

    def archive_zip_file(self, version):
        version.clean_zip_archive.delete()
        logger.debug('Archiving zip file')
        with open(self.clean_zip_path) as zf:
            # Save the zip on the raw data version
            version.clean_zip_archive.save(
                os.path.basename(self.clean_zip_path), File(zf)
            )<commit_msg>Remove print statement<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Zip up cleaned raw data files (and error logs) and archive zip files.
"""
import os
import logging
import shutil
from sys import exit
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile
from django.core.files import File
from calaccess_raw import get_download_directory
from calaccess_raw.management.commands import CalAccessCommand
from calaccess_raw.models.tracking import RawDataVersion
logger = logging.getLogger(__name__)

class Command(CalAccessCommand):
    """
    Zip up cleaned raw data files (and error logs) and archive zip files.
    """
    help = 'Zip up cleaned raw data files (and error logs) and archive zip files'

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)
        self.clean_zip_path = os.path.join(
            get_download_directory(),
            'calaccess_cleaned.zip'
        )

        self.data_dir = get_download_directory()

        if os.path.exists(self.data_dir):
            shutil.rmtree(self.data_dir)

        os.makedirs(self.data_dir)

        versions = RawDataVersion.objects.filter(clean_zip_archive='')

        if versions:
            for version in versions:
                logger.debug(
                    'Creating zip file for {:%Y-%m-%d_%H-%M-%S} version'.format(
                        version.release_datetime
                    )
                )

                self.download_clean_files(version)
                self.create_zip_file(version)
                self.archive_zip_file(version)

    def download_clean_files(self, version):
        for raw_file in version.files.all():
            if raw_file.clean_file_archive:
                logger.debug(
                    'Downloading {0}'.format(os.path.basename(
                            raw_file.clean_file_archive.name
                        )
                    )
                )
                local_clean_file_path = os.path.join(
                    self.data_dir,
                    os.path.basename(raw_file.clean_file_archive.name),
                )
                local_clean_file = open(local_clean_file_path, 'wb')
                local_clean_file.write(raw_file.clean_file_archive.read())
                local_clean_file.close()
            if raw_file.error_log_archive:
                logger.debug(
                    'Downloading {0}'.format(os.path.basename(
                            raw_file.error_log_archive.name
                        )
                    )
                )
                local_error_file_path = os.path.join(
                    self.data_dir,
                    os.path.basename(raw_file.error_log_archive.name),
                )
                local_error_file = open(local_error_file_path, 'wb')
                local_error_file.write(raw_file.error_log_archive.read())
                local_error_file.close()

    def create_zip_file(self, version):
        logger.debug('Creating zip file')
        compression = ZIP_DEFLATED
        try:
            zf = ZipFile(self.clean_zip_path, 'w', compression, allowZip64=True)
        except RuntimeError:
            logger.error('Zip file cannot be compressed (check zlib module).')
            compression = ZIP_STORED
            zf = ZipFile(self.clean_zip_path, 'w', compression, allowZip64=True)

        csv_files = [f for f in os.listdir(self.data_dir) if '.csv' in f]

        for f in csv_files:
            logger.debug('Writing %s to zip file' % os.path.basename(f))
            file_path = os.path.join(self.data_dir, f)
            zf.write(file_path, f)
            os.remove(file_path)

        zf.close()

    def archive_zip_file(self, version):
        version.clean_zip_archive.delete()
        logger.debug('Archiving zip file')
        with open(self.clean_zip_path) as zf:
            # Save the zip on the raw data version
            version.clean_zip_archive.save(
                os.path.basename(self.clean_zip_path), File(zf)
            )<|endoftext|>