<commit_before>import logging; logger = logging.getLogger("minimalKB."+__name__);
DEBUG_LEVEL=logging.DEBUG

import shlex
import traceback

from sqlite_store import SQLStore

def api(fn):
    fn._api = True
    return fn

def compat(fn):
    fn._compat = True
    return fn

class KbServerError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class Event:

    NEW_INSTANCE = "NEW_INSTANCE"
    NEW_INSTANCE_ONE_SHOT = "NEW_INSTANCE_ONE_SHOT"

    def __init__(self, kb, type, trigger, var, patterns, models):
        self.kb = kb
        self.type = type
        self.trigger = trigger
        self.var = var
        self.patterns = patterns
        self.models = models

        self.id =  "evt_" + str(hash(self.type + \
                    self.trigger + \
                    self.var + \
                    str(sorted(self.patterns)) + \
                    str(sorted(self.models))))

        self.content = None

        self.valid = True

        self.previous_instances = set()
        if type in [Event.NEW_INSTANCE, Event.NEW_INSTANCE_ONE_SHOT]:
            instances = self.kb.find([var], patterns, None, models)
            logger.debug("Creating a NEW_INSTANCE event with initial instances %s"%instances)
            self.previous_instances = set(instances)

    def __hash__(self):
        return hash(self.id)

    def __cmp__(self, other):
        return hash(self).__cmp__(hash(other))

    def evaluate(self):
        if "ONE_SHOT" in self.trigger:
            self.valid = False

        if self.type in [Event.NEW_INSTANCE, Event.NEW_INSTANCE_ONE_SHOT]:
            instances = set(self.kb.find([self.var], self.patterns, None, self.models))
            newinstances = instances - self.previous_instances

            if not newinstances:
                return False

            self.content = [i for i in newinstances] # for some reason, calling list() does not work
            self.previous_instances = self.previous_instances | instances
            return True

class MinimalKB:

    MEMORYPROFILE_DEFAULT = ""
    MEMORYPROFILE_SHORTTERM = "SHORTTERM"

    def __init__(self):
        _api = [getattr(self, fn) for fn in dir(self) if hasattr(getattr(self, fn), "_api")]
        self._api = {fn.__name__:fn for fn in _api}

        self.store = SQLStore()

        self.models = {"default"}

        apilist = [key + (" (compatibility)" if hasattr(val, "_compat") else "") for key, val in self._api.items()]

        logger.info("Initializing the MinimalKB with the following API: \n\t- " + \
                "\n\t- ".join(apilist))

        self.active_evts = set()
        self.triggered_evts = []

    @api
    def hello(self):
        return "MinimalKB, v.0.1"

    @api
    def load(self, filename):
        logger.info("Loading triples from %s" % filename)
        with open(filename, 'r') as triples:
            self.store.add([s.strip() for s in triples.readlines()])

    @api
    def clear(self):
        logger.warn("Clearing the knowledge base!")
        self.store.clear()

    @compat
    @api
    def listSimpleMethods(self):
        return self.methods()

    @api
    def methods(self):
        return self._api.keys()

    @api
    def about(self, resource, models = None):
        return self.store.about(resource, self.normalize_models(models))

    @api
    def lookup(self, resource, models = None):
        logger.info("Lookup for " + str(resource) + \
                    " in " + (str(models) if models else "any model."))

        about =  self.store.about(resource, self.normalize_models(models))
        if not about:
            return []

        return [(resource, "unknown")]

    @api
    def exist(self, stmts, models = None):
        logger.info("Checking existence of " + str(stmts) + \
                    " in " + (str(models) if models else "any model."))
        return self.store.has(stmts,
                              self.normalize_models(models))

    @api
    def revise(self, stmts, policy):

        if isinstance(stmts, (str, unicode)):
            raise KbServerError("A list of statements is expected")

        if type(policy) != dict:
            raise KbServerError("Expected a dictionary as policy")
        models = self.normalize_models(policy.get('models', []))

        if policy["method"] in ["add", "safe_add"]:
            logger.info("Adding to " + str(list(models)) + ":\n\t- " + "\n\t- ".join(stmts))
            for model in models:
                self.store.add(stmts, model)

        if policy["method"] == "retract":
            logger.info("Deleting from " + str(list(models)) +":\n\t- " + "\n\t- ".join(stmts))
            for model in models:
                self.store.delete(stmts, model)

        self.onupdate()

    @api
    def add(self, stmts, models = None):
        return self.revise(stmts,
                           {"method": "add",
                            "models": models})

    @api
    def retract(self, stmts, models = None):
        return self.revise(stmts,
                           {"method": "retract",
                            "models": models})

    @compat
    @api
    def findForAgent(self, agent, var, stmts):
        return self.find([var], stmts, None, [agent])

    @api
    def find(self, vars, pattern, constraints = None, models = None):

        if not models:
            models = self.models

        logger.info("Searching " + str(vars) + \
                    " in models " + str(models) + \
                    " matching:\n\t- " + "\n\t- ".join(pattern))

        res = self.store.query(vars, pattern, models)

        logger.info("Found: " + str(res))
        return res

    @api
    def findmpe(self, vars, pattern, constraints = None, models = None):
        """ Finds the most probable explanation. Strictly equivalent to
        'find' until we support probabilities.
        """
        return find(self, vars, pattern, constraints = None, models = None)

    @api
    def subscribe(self, type, trigger, var, patterns, models = None):

        models = self.normalize_models(models)

        logger.info("Registering a new event: %s %s for %s on %s" % (type, trigger, var, patterns) + \
                    " in " + (str(models) if models else "any model."))

        event = Event(self, type, trigger, var, patterns, models)

        self.active_evts.add(event)

        return event.id

    def onupdate(self):
        for e in self.active_evts:
            if e.evaluate():
                logger.info("Event %s triggered." % e.id)
                self.triggered_evts.append(e)
                if not e.valid:
                    self.active_evts.discard(e)

    def normalize_models(self, models):
        """ If 'models' is None, [] or contains 'all', then
        returns the set of all models known to the KB.
        Else, add the models to the list of all models, and return
        only the models passed as argument.
        """
        if models:
            if "all" in models:
                return self.models
            else:
                if isinstance(models, (str, unicode)):
                    models = [models]
                #add to the set of all models
                self.models = self.models | set(models)
                return set(models)
        else:
            return self.models

    def execute(self, name, *args):
        f = getattr(self, name)
        if hasattr(f, "_compat"):
                logger.warn("Using non-standard method %s. This may be " + \
                        "removed in the future!" % f.__name__)
        try:
            return f(*args)
        except TypeError:
            traceback.print_exc()
            return f()

<commit_msg>When clearing knowledge base, also clear registered events<commit_after>import logging; logger = logging.getLogger("minimalKB."+__name__);
DEBUG_LEVEL=logging.DEBUG

import shlex
import traceback

from sqlite_store import SQLStore

def api(fn):
    fn._api = True
    return fn

def compat(fn):
    fn._compat = True
    return fn

class KbServerError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class Event:

    NEW_INSTANCE = "NEW_INSTANCE"
    NEW_INSTANCE_ONE_SHOT = "NEW_INSTANCE_ONE_SHOT"

    def __init__(self, kb, type, trigger, var, patterns, models):
        self.kb = kb
        self.type = type
        self.trigger = trigger
        self.var = var
        self.patterns = patterns
        self.models = models

        self.id =  "evt_" + str(hash(self.type + \
                    self.trigger + \
                    self.var + \
                    str(sorted(self.patterns)) + \
                    str(sorted(self.models))))

        self.content = None

        self.valid = True

        self.previous_instances = set()
        if type in [Event.NEW_INSTANCE, Event.NEW_INSTANCE_ONE_SHOT]:
            instances = self.kb.find([var], patterns, None, models)
            logger.debug("Creating a NEW_INSTANCE event with initial instances %s"%instances)
            self.previous_instances = set(instances)

    def __hash__(self):
        return hash(self.id)

    def __cmp__(self, other):
        return hash(self).__cmp__(hash(other))

    def evaluate(self):
        if "ONE_SHOT" in self.trigger:
            self.valid = False

        if self.type in [Event.NEW_INSTANCE, Event.NEW_INSTANCE_ONE_SHOT]:
            instances = set(self.kb.find([self.var], self.patterns, None, self.models))
            newinstances = instances - self.previous_instances

            if not newinstances:
                return False

            self.content = [i for i in newinstances] # for some reason, calling list() does not work
            self.previous_instances = self.previous_instances | instances
            return True

class MinimalKB:

    MEMORYPROFILE_DEFAULT = ""
    MEMORYPROFILE_SHORTTERM = "SHORTTERM"

    def __init__(self):
        _api = [getattr(self, fn) for fn in dir(self) if hasattr(getattr(self, fn), "_api")]
        self._api = {fn.__name__:fn for fn in _api}

        self.store = SQLStore()

        self.models = {"default"}

        apilist = [key + (" (compatibility)" if hasattr(val, "_compat") else "") for key, val in self._api.items()]

        logger.info("Initializing the MinimalKB with the following API: \n\t- " + \
                "\n\t- ".join(apilist))

        self.active_evts = set()
        self.triggered_evts = []

    @api
    def hello(self):
        return "MinimalKB, v.0.1"

    @api
    def load(self, filename):
        logger.info("Loading triples from %s" % filename)
        with open(filename, 'r') as triples:
            self.store.add([s.strip() for s in triples.readlines()])

    @api
    def clear(self):
        logger.warn("Clearing the knowledge base!")
        self.store.clear()
        self.active_evts.clear()

    @compat
    @api
    def listSimpleMethods(self):
        return self.methods()

    @api
    def methods(self):
        return self._api.keys()

    @api
    def about(self, resource, models = None):
        return self.store.about(resource, self.normalize_models(models))

    @api
    def lookup(self, resource, models = None):
        logger.info("Lookup for " + str(resource) + \
                    " in " + (str(models) if models else "any model."))

        about =  self.store.about(resource, self.normalize_models(models))
        if not about:
            return []

        return [(resource, "unknown")]

    @api
    def exist(self, stmts, models = None):
        logger.info("Checking existence of " + str(stmts) + \
                    " in " + (str(models) if models else "any model."))
        return self.store.has(stmts,
                              self.normalize_models(models))

    @api
    def revise(self, stmts, policy):

        if isinstance(stmts, (str, unicode)):
            raise KbServerError("A list of statements is expected")

        if type(policy) != dict:
            raise KbServerError("Expected a dictionary as policy")
        models = self.normalize_models(policy.get('models', []))

        if policy["method"] in ["add", "safe_add"]:
            logger.info("Adding to " + str(list(models)) + ":\n\t- " + "\n\t- ".join(stmts))
            for model in models:
                self.store.add(stmts, model)

        if policy["method"] == "retract":
            logger.info("Deleting from " + str(list(models)) +":\n\t- " + "\n\t- ".join(stmts))
            for model in models:
                self.store.delete(stmts, model)

        self.onupdate()

    @api
    def add(self, stmts, models = None):
        return self.revise(stmts,
                           {"method": "add",
                            "models": models})

    @api
    def retract(self, stmts, models = None):
        return self.revise(stmts,
                           {"method": "retract",
                            "models": models})

    @compat
    @api
    def findForAgent(self, agent, var, stmts):
        return self.find([var], stmts, None, [agent])

    @api
    def find(self, vars, pattern, constraints = None, models = None):

        if not models:
            models = self.models

        logger.info("Searching " + str(vars) + \
                    " in models " + str(models) + \
                    " matching:\n\t- " + "\n\t- ".join(pattern))

        res = self.store.query(vars, pattern, models)

        logger.info("Found: " + str(res))
        return res

    @api
    def findmpe(self, vars, pattern, constraints = None, models = None):
        """ Finds the most probable explanation. Strictly equivalent to
        'find' until we support probabilities.
        """
        return find(self, vars, pattern, constraints = None, models = None)

    @api
    def subscribe(self, type, trigger, var, patterns, models = None):

        models = self.normalize_models(models)

        logger.info("Registering a new event: %s %s for %s on %s" % (type, trigger, var, patterns) + \
                    " in " + (str(models) if models else "any model."))

        event = Event(self, type, trigger, var, patterns, models)

        self.active_evts.add(event)

        return event.id

    def onupdate(self):
        for e in self.active_evts:
            if e.evaluate():
                logger.info("Event %s triggered." % e.id)
                self.triggered_evts.append(e)
                if not e.valid:
                    self.active_evts.discard(e)

    def normalize_models(self, models):
        """ If 'models' is None, [] or contains 'all', then
        returns the set of all models known to the KB.
        Else, add the models to the list of all models, and return
        only the models passed as argument.
        """
        if models:
            if "all" in models:
                return self.models
            else:
                if isinstance(models, (str, unicode)):
                    models = [models]
                #add to the set of all models
                self.models = self.models | set(models)
                return set(models)
        else:
            return self.models

    def execute(self, name, *args):
        f = getattr(self, name)
        if hasattr(f, "_compat"):
                logger.warn("Using non-standard method %s. This may be " + \
                        "removed in the future!" % f.__name__)
        try:
            return f(*args)
        except TypeError:
            traceback.print_exc()
            return f()

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- encoding: utf-8 -*-

# Copyright (c) 2009 Alexander van der Mey <alexvandermey@gmail.com>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import gtk

CURRENT = "CURRENT"
LINE = "\hline\n"

class Importer:

	def __init__(self, editorpane, builder):
		self.editorpane = editorpane

		self.mainwindow = builder.get_object("mainwindow")
		self.import_tabs = builder.get_object("import_tabs")

		self.image_pane = builder.get_object("image_pane")
		self.image_file = builder.get_object("image_file")
		self.image_caption = builder.get_object("image_caption")
		self.image_label = builder.get_object("image_label")
		self.image_scale = builder.get_object("image_scale")
		self.scaler = builder.get_object("image_scaler")

		self.table_pane = builder.get_object("table_pane")
		self.table_comboallign = builder.get_object("table_comboallign")
		self.table_comboborder = builder.get_object("table_comboborder")
		self.table_rows = builder.get_object("table_rows")
		self.table_cols = builder.get_object("table_cols")

		self.matrix_rows = builder.get_object("matrix_rows")
		self.matrix_cols = builder.get_object("matrix_cols")
		self.matrix_combobracket = builder.get_object("matrix_combobracket")

		self.table_cols.set_value(3)
		self.table_rows.set_value(3)
		self.matrix_cols.set_value(3)
		self.matrix_rows.set_value(3)

	def prepare_image(self):
		imagefile = None
		chooser = gtk.FileChooserDialog("Open File...", self.mainwindow,
								gtk.FILE_CHOOSER_ACTION_OPEN,
								(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
								gtk.STOCK_OPEN, gtk.RESPONSE_OK))
		imagefilter = gtk.FileFilter()
		imagefilter.set_name('Image files')
		imagefilter.add_mime_type("image/*")
		chooser.add_filter(imagefilter)

		response = chooser.run()
		if response == gtk.RESPONSE_OK: imagefile = chooser.get_filename()
		chooser.destroy()
		self.image_label.set_sensitive(True)
		self.image_scale.set_sensitive(True)
		self.image_caption.set_sensitive(True)
		self.scaler.set_value(1.00)
		self.image_file.set_text(imagefile)
		return imagefile

	def insert_image(self):
		if os.path.exists(self.image_file.get_text()):
			self.editorpane.insert_package("graphicx")
			f = self.image_file.get_text()
			s = self.scaler.get_value()
			c = self.image_caption.get_text()
			l = self.image_label.get_text()
			code = self.generate_image(f, s, c, l)
			position = self.editorpane.get_iterator(CURRENT)
			self.editorpane.editorbuffer.insert(position, code)
			self.editorpane.set_buffer_changed()
			self.image_file.set_text("")
		self.import_tabs.set_current_page(0)

	def insert_table(self):
		position = self.editorpane.get_iterator(CURRENT)
		code = self.generate_table(self.table_rows.get_value(), self.table_cols.get_value())
		position = self.editorpane.get_iterator(CURRENT)
		self.editorpane.editorbuffer.insert(position, code)
		self.editorpane.set_buffer_changed()
		self.import_tabs.set_current_page(0)

	def insert_matrix(self):
		self.editorpane.insert_package("amsmath")
		position = self.editorpane.get_iterator(CURRENT)
		code = self.generate_matrix(self.matrix_rows.get_value(), self.matrix_cols.get_value())
		position = self.editorpane.get_iterator(CURRENT)
		self.editorpane.editorbuffer.insert(position, code)
		self.editorpane.set_buffer_changed()
		self.import_tabs.set_current_page(0)

	def generate_table(self, rows, columns):
		table = ""
		rows = int(rows) + 1
		columns = int(columns) + 1
		borders = self.table_comboborder.get_active()
		# get column with l/c/r directly?
		allignment = self.table_comboallign.get_active()
		if allignment is 0: alligntype = "l"
		elif allignment is 1: alligntype = "c"
		else: alligntype = "r"
		if borders is 2:
				alligntype = alligntype + "|"
		for f in range(1,columns):
				allign = f * alligntype
		if borders is 1: allign = "|" + allign + "|"
		elif borders is 2: allign = "|" + allign
		begin_tabular = "\\begin{tabular}{" + allign + "}\n"
		end_tabular = "\\end{tabular}\n"
		for k in range(1, rows):
			for i in range(1,columns):
				if i == (columns-1):
					new = str(k) + str(i) + "\\\\ \n"
					if borders is 2: new = new + LINE
				elif i == 1: new = "\t" + str(k) + str(i) + " & "
				else: new = str(k) + str(i) + " & "
				table = table + new
		if borders is 1: table = LINE + table + LINE
		if borders is 2: table = LINE + table
		return begin_tabular + table + end_tabular

	def generate_image(self, imagefile,  scale, caption, label):
		include = "\\includegraphics"
		scale = "[scale=" + str(scale) + "]"
		filename = "{" + imagefile + "}\n"
		caption = "\\captionof{" + caption + "}\n"
		label = "\\label{" + label + "}\n"
		return include + scale + filename + caption + label

	def generate_matrix(self, rows, columns):
		bracket = self.matrix_combobracket.get_active_text()
		if bracket == "Unbracketed": mode = "matrix"
		elif bracket == "Parentheses": mode = "pmatrix"
		elif bracket == "Brackets": mode = "bmatrix"
		elif bracket == "Braces": mode = "Bmatrix"
		elif bracket == "Single line": mode = "vmatrix"
		elif bracket == "Double line": mode = "Vmatrix"
		matrix = ""
		rows = int(rows) + 1
		columns = int(columns) + 1
		begin_matrix = "$\\begin{" + mode + "}\n"
		end_matrix = "\\end{" + mode +"}$\n"
		for k in range(1, rows):
			for i in range(1,columns):
				if i == (columns-1): new = str(k) + str(i) + "\\\\ \n"
				elif i == 1: new = "\t" + str(k) + str(i) + " & "
				else: new = str(k) + str(i) + " & "
				matrix = matrix + new
		return begin_matrix + matrix + end_matrix

<commit_msg>enhanced image code generation and fixed issue 42<commit_after>#!/usr/bin/python
# -*- encoding: utf-8 -*-

# Copyright (c) 2009 Alexander van der Mey <alexvandermey@gmail.com>

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import gtk

CURRENT = "CURRENT"
LINE = "\hline\n"

class Importer:

	def __init__(self, editorpane, builder):
		self.editorpane = editorpane

		self.mainwindow = builder.get_object("mainwindow")
		self.import_tabs = builder.get_object("import_tabs")

		self.image_pane = builder.get_object("image_pane")
		self.image_file = builder.get_object("image_file")
		self.image_caption = builder.get_object("image_caption")
		self.image_label = builder.get_object("image_label")
		self.image_scale = builder.get_object("image_scale")
		self.scaler = builder.get_object("image_scaler")

		self.table_pane = builder.get_object("table_pane")
		self.table_comboallign = builder.get_object("table_comboallign")
		self.table_comboborder = builder.get_object("table_comboborder")
		self.table_rows = builder.get_object("table_rows")
		self.table_cols = builder.get_object("table_cols")

		self.matrix_rows = builder.get_object("matrix_rows")
		self.matrix_cols = builder.get_object("matrix_cols")
		self.matrix_combobracket = builder.get_object("matrix_combobracket")

		self.table_cols.set_value(3)
		self.table_rows.set_value(3)
		self.matrix_cols.set_value(3)
		self.matrix_rows.set_value(3)

	def prepare_image(self):
		imagefile = None
		chooser = gtk.FileChooserDialog("Open File...", self.mainwindow,
								gtk.FILE_CHOOSER_ACTION_OPEN,
								(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
								gtk.STOCK_OPEN, gtk.RESPONSE_OK))
		imagefilter = gtk.FileFilter()
		imagefilter.set_name('Image files')
		imagefilter.add_mime_type("image/*")
		chooser.add_filter(imagefilter)

		response = chooser.run()
		if response == gtk.RESPONSE_OK: imagefile = chooser.get_filename()
		chooser.destroy()
		self.image_label.set_sensitive(True)
		self.image_scale.set_sensitive(True)
		self.image_caption.set_sensitive(True)
		self.scaler.set_value(1.00)
		self.image_file.set_text(imagefile)
		return imagefile

	def insert_image(self):
		if os.path.exists(self.image_file.get_text()):
			self.editorpane.insert_package("graphicx")
			f = self.image_file.get_text()
			s = self.scaler.get_value()
			c = self.image_caption.get_text()
			l = self.image_label.get_text()
			code = self.generate_image(f, s, c, l)
			position = self.editorpane.get_iterator(CURRENT)
			self.editorpane.editorbuffer.insert(position, code)
			self.editorpane.set_buffer_changed()
			self.image_file.set_text("")
			self.image_caption.set_text("")
			self.image_label.set_text("")
		self.import_tabs.set_current_page(0)

	def insert_table(self):
		position = self.editorpane.get_iterator(CURRENT)
		code = self.generate_table(self.table_rows.get_value(), self.table_cols.get_value())
		position = self.editorpane.get_iterator(CURRENT)
		self.editorpane.editorbuffer.insert(position, code)
		self.editorpane.set_buffer_changed()
		self.import_tabs.set_current_page(0)

	def insert_matrix(self):
		self.editorpane.insert_package("amsmath")
		position = self.editorpane.get_iterator(CURRENT)
		code = self.generate_matrix(self.matrix_rows.get_value(), self.matrix_cols.get_value())
		position = self.editorpane.get_iterator(CURRENT)
		self.editorpane.editorbuffer.insert(position, code)
		self.editorpane.set_buffer_changed()
		self.import_tabs.set_current_page(0)

	def generate_table(self, rows, columns):
		table = ""
		rows = int(rows) + 1
		columns = int(columns) + 1
		borders = self.table_comboborder.get_active()
		# get column with l/c/r directly?
		allignment = self.table_comboallign.get_active()
		if allignment is 0: alligntype = "l"
		elif allignment is 1: alligntype = "c"
		else: alligntype = "r"
		if borders is 2:
				alligntype = alligntype + "|"
		for f in range(1,columns):
				allign = f * alligntype
		if borders is 1: allign = "|" + allign + "|"
		elif borders is 2: allign = "|" + allign
		begin_tabular = "\\begin{tabular}{" + allign + "}\n"
		end_tabular = "\\end{tabular}\n"
		for k in range(1, rows):
			for i in range(1,columns):
				if i == (columns-1):
					new = str(k) + str(i) + "\\\\ \n"
					if borders is 2: new = new + LINE
				elif i == 1: new = "\t" + str(k) + str(i) + " & "
				else: new = str(k) + str(i) + " & "
				table = table + new
		if borders is 1: table = LINE + table + LINE
		if borders is 2: table = LINE + table
		return begin_tabular + table + end_tabular

	def generate_image(self, imagefile,  scale, caption, label):
		caption = self.image_caption.get_text()
		label = self.image_label.get_text()
		begin = "\\begin{figure}[htp]\n\\centering\n"
		end = "\\end{figure}"
		include = "\\includegraphics"
		scale = "[scale=" + str(scale) + "]"
		filename = "{" + imagefile + "}\n"
		if self.image_caption.get_text() is not "":
			caption = "\\caption{" + caption + "}\n"
		if self.image_label.get_text() is not "":
			label = "\\label{" + label + "}\n"
		return begin + include + scale + filename + caption + label + end

	def generate_matrix(self, rows, columns):
		bracket = self.matrix_combobracket.get_active_text()
		if bracket == "Unbracketed": mode = "matrix"
		elif bracket == "Parentheses": mode = "pmatrix"
		elif bracket == "Brackets": mode = "bmatrix"
		elif bracket == "Braces": mode = "Bmatrix"
		elif bracket == "Single line": mode = "vmatrix"
		elif bracket == "Double line": mode = "Vmatrix"
		matrix = ""
		rows = int(rows) + 1
		columns = int(columns) + 1
		begin_matrix = "$\\begin{" + mode + "}\n"
		end_matrix = "\\end{" + mode +"}$\n"
		for k in range(1, rows):
			for i in range(1,columns):
				if i == (columns-1): new = str(k) + str(i) + "\\\\ \n"
				elif i == 1: new = "\t" + str(k) + str(i) + " & "
				else: new = str(k) + str(i) + " & "
				matrix = matrix + new
		return begin_matrix + matrix + end_matrix

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Print a json file with the participants information for their badges."""

import json
import logging as log
from   optparse import make_option
from   collections import OrderedDict

from   django.core.management.base import BaseCommand, CommandError

from   assopy import models as assopy_models

### Helpers
def conference_year(conference='ep2016'):
    return conference[-2:]

def get_all_order_tickets(conference='ep2016'):

    year = conference_year(conference)

    orders          = assopy_models.Order.objects.filter(_complete=True)
    conf_orders     = [order for order in orders if order.code.startswith('O/{}.'.format(year))]
    order_tkts      = [ordi.ticket
                       for order in conf_orders
                       for ordi in order.orderitem_set.all()
                       if ordi.ticket is not None]
    conf_order_tkts = [ot for ot in order_tkts if ot.fare.code.startswith('T')]

    return conf_order_tkts

###
class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--status',
                    action='store',
                    dest='status',
                    default='all',
                    choices=['all', 'complete', 'incomplete'],
                    help='Status of the orders related with the tickets.',
                ),
        make_option('--nondups',
                    action='store_true',
                    dest='nondups',
                    default=False,
                    help='If enables will remove the tickets with '
                         'same owner/email.',
                ),
        make_option('--raise',
                    action='store_true',
                    dest='raise',
                    default=False,
                    help='If enabled will raise any error that it may find.',
                ),
        make_option('--ticket-id',
                    action='store',
                    dest='ticket_id',
                    help='Will output the profile of the given ticket only.',
                ),
    )

    def handle(self, *args, **options):
        try:
            conference = args[0]
        except IndexError:
            raise CommandError('conference not specified')

        if options['status'] not in ('all', 'complete', 'incomplete'):
            raise CommandError('--status should be one of '
                               '(all, complete, incomplete)' )

        tkts = get_all_order_tickets(conference)
        if not tkts:
            raise IndexError('Could not find any tickets for '
                             'conference {}.'.format(conference))

        if options['ticket_id']:
            tkt_id = int(options['ticket_id'])
            tkts = [t for t in tkts if t.id == tkt_id]
            if not tkts:
                raise IndexError('Could not find any ticket with '
                                 'ticket_id {}.'.format(options['ticket_id']))

        dflt_profile = {'name':    '',
                        'surname': '',
                        'tagline': '',
                        'company': '',
                        'pypower': '0',
                        'tshirt':  'l',
                        'email':   '',
                        'phone':   '',
                        'compweb': '',
                        'persweb': '',
                        'id'     : '',
                        }

        profiles = OrderedDict()
        for t in tkts:
            p3_tkt = t.p3_conference
            subj   = dflt_profile.copy()

            try:
                profile = p3_tkt.profile()
            except:
                msg = 'Could not find a profile for ticket_id {}.'.format(t.id)
                if options['raise']:
                    #raise AttributeError(msg)
                    log.error(msg)
                    import ipdb
                    ipdb.set_trace()
                else:
                    log.error(msg)

            if profile.job_title and profile.company:
                if profile.company.lower().strip() in profile.job_title.lower().strip():
                    subj['company'] = profile.job_title
                else:
                    subj['company'] = profile.job_title + " @ " + profile.company
            elif profile.job_title:
                subj['company'] = profile.job_title
            elif profile.company:
                subj['company'] = profile.company

            subj['name'   ] = profile.user.first_name
            subj['surname'] = profile.user.last_name
            subj['tagline'] = p3_tkt.tagline
            subj['pypower'] = str(p3_tkt.python_experience)
            subj['tshirt']  = p3_tkt.shirt_size
            subj['email'  ] = profile.user.email
            subj['phone'  ] = profile.phone
            subj['compweb'] = profile.company_homepage
            subj['persweb'] = profile.personal_homepage

            subj = {k: v.encode('utf-8') for k, v in subj.items()}

            profiles[t.id] = subj

        print(json.dumps(profiles, indent=2, separators=(',', ': ')))
<commit_msg>fix ticket_profiles command<commit_after># -*- coding: utf-8 -*-
""" Print a json file with the participants information for their badges."""

import json
import logging as log
from   optparse import make_option
from   collections import OrderedDict

from   django.core.management.base import BaseCommand, CommandError

from   assopy import models as assopy_models

### Helpers
def conference_year(conference='ep2016'):
    return conference[-2:]

def get_all_order_tickets(conference='ep2016'):

    year = conference_year(conference)

    orders          = assopy_models.Order.objects.filter(_complete=True)
    conf_orders     = [order for order in orders if order.code.startswith('O/{}.'.format(year))]
    order_tkts      = [ordi.ticket
                       for order in conf_orders
                       for ordi in order.orderitem_set.all()
                       if ordi.ticket is not None]
    conf_order_tkts = [ot for ot in order_tkts if ot.fare.code.startswith('T')]

    return conf_order_tkts

###
class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--status',
                    action='store',
                    dest='status',
                    default='all',
                    choices=['all', 'complete', 'incomplete'],
                    help='Status of the orders related with the tickets.',
                ),
        make_option('--nondups',
                    action='store_true',
                    dest='nondups',
                    default=False,
                    help='If enables will remove the tickets with '
                         'same owner/email.',
                ),
        make_option('--raise',
                    action='store_true',
                    dest='raise',
                    default=False,
                    help='If enabled will raise any error that it may find.',
                ),
        make_option('--ticket-id',
                    action='store',
                    dest='ticket_id',
                    help='Will output the profile of the given ticket only.',
                ),
    )

    def handle(self, *args, **options):
        try:
            conference = args[0]
        except IndexError:
            raise CommandError('conference not specified')

        if options['status'] not in ('all', 'complete', 'incomplete'):
            raise CommandError('--status should be one of '
                               '(all, complete, incomplete)' )

        tkts = get_all_order_tickets(conference)
        if not tkts:
            raise IndexError('Could not find any tickets for '
                             'conference {}.'.format(conference))

        if options['ticket_id']:
            tkt_id = int(options['ticket_id'])
            tkts = [t for t in tkts if t.id == tkt_id]
            if not tkts:
                raise IndexError('Could not find any ticket with '
                                 'ticket_id {}.'.format(options['ticket_id']))

        dflt_profile = {'name':    '',
                        'surname': '',
                        'tagline': '',
                        'company': '',
                        'pypower': '0',
                        'tshirt':  'l',
                        'email':   '',
                        'phone':   '',
                        'compweb': '',
                        'persweb': '',
                        'id'     : '',
                        }

        profiles = [] #OrderedDict()
        for t in tkts:
            p3_tkt = t.p3_conference
            subj   = dflt_profile.copy()

            try:
                profile = p3_tkt.profile()
            except:
                msg = 'Could not find a profile for ticket_id {}.'.format(t.id)
                if options['raise']:
                    #raise AttributeError(msg)
                    log.error(msg)
                    import ipdb
                    ipdb.set_trace()
                else:
                    log.error(msg)

            if profile.job_title and profile.company:
                if profile.company.lower().strip() in profile.job_title.lower().strip():
                    subj['company'] = profile.job_title
                else:
                    subj['company'] = profile.job_title + " @ " + profile.company
            elif profile.job_title:
                subj['company'] = profile.job_title
            elif profile.company:
                subj['company'] = profile.company

            subj['name'   ] = profile.user.first_name
            subj['surname'] = profile.user.last_name
            subj['tagline'] = p3_tkt.tagline
            subj['pypower'] = str(p3_tkt.python_experience)
            subj['tshirt']  = p3_tkt.shirt_size
            subj['email'  ] = profile.user.email
            subj['phone'  ] = profile.phone
            subj['compweb'] = profile.company_homepage
            subj['persweb'] = profile.personal_homepage
            subj['id']      = str(t.id)

            subj = {k: v.encode('utf-8') for k, v in subj.items()}

            profiles.append(subj)

        print(json.dumps(profiles, indent=2, separators=(',', ': ')))
<|endoftext|>


<commit_before>"""
Check to see if there are webcams offline, generate emails and such
"""

import os
import mx.DateTime
import stat
import access
import tracker
track = tracker.Engine()
import iemdb
IEM = iemdb.connect('iem', bypass=True)
icursor = IEM.cursor()
MESOSITE = iemdb.connect('mesosite', bypass=True)
mcursor = MESOSITE.cursor()
now = mx.DateTime.now()

# Determine sites offline
offline = {}
icursor.execute("""SELECT station, valid from offline
    WHERE network IN ('KCCI','KELO','KCRG')""")
for row in icursor:
    offline[ row[0] ] = row[1]

# Now lets check files
dir = "/home/ldm/data/camera/stills"
files = os.listdir(dir)

mcursor.execute("""
    SELECT id, network, name from webcams where network in ('KELO','KCCI','KCRG')
    and online ORDER by id ASC
""")

emails = 0
for row in mcursor:
    fn = "%s/%s.jpg" % (dir, row[0])
    if not os.path.isfile(fn):
        print 'Missing webcam file: %s' % (fn,)
        continue

    mtime = os.stat(fn)[stat.ST_MTIME]
    ts = mx.DateTime.DateTimeFromTicks(mtime)
    age = float(now) - mtime
    network = row[1]
    portfolio = "%ssnet" % (network.lower(),)

    if age > 3600 and not offline.has_key(row[0]):
        emails += 1
        track.doAlert(row[0], {'ts': ts, 'sname': row[2]},
                      network, portfolio, False)
    elif age < 3600 and offline.has_key(row[0]):
        emails += 1
        track.checkStation(row[0], {'ts': ts, 'sname': row[2]},
                           network, portfolio, False)

if emails < 5:
    track.send()
else:
    print 'Skipping check_webcams email due to too many emails'

# done<commit_msg>add webcam to the location name<commit_after>"""
Check to see if there are webcams offline, generate emails and such
"""

import os
import mx.DateTime
import stat
import access
import tracker
track = tracker.Engine()
import iemdb
IEM = iemdb.connect('iem', bypass=True)
icursor = IEM.cursor()
MESOSITE = iemdb.connect('mesosite', bypass=True)
mcursor = MESOSITE.cursor()
now = mx.DateTime.now()

# Determine sites offline
offline = {}
icursor.execute("""SELECT station, valid from offline
    WHERE network IN ('KCCI','KELO','KCRG')""")
for row in icursor:
    offline[ row[0] ] = row[1]

# Now lets check files
dir = "/home/ldm/data/camera/stills"
files = os.listdir(dir)

mcursor.execute("""
    SELECT id, network, name from webcams where network in ('KELO','KCCI','KCRG')
    and online ORDER by id ASC
""")

emails = 0
for row in mcursor:
    fn = "%s/%s.jpg" % (dir, row[0])
    if not os.path.isfile(fn):
        print 'Missing webcam file: %s' % (fn,)
        continue

    mtime = os.stat(fn)[stat.ST_MTIME]
    ts = mx.DateTime.DateTimeFromTicks(mtime)
    age = float(now) - mtime
    network = row[1]
    portfolio = "%ssnet" % (network.lower(),)

    if age > 3600 and not offline.has_key(row[0]):
        emails += 1
        track.doAlert(row[0], {'ts': ts, 'sname': row[2] +' Webcam'},
                      network, portfolio, False)
    elif age < 3600 and offline.has_key(row[0]):
        emails += 1
        track.checkStation(row[0], {'ts': ts, 'sname': row[2] +' Webcam'},
                           network, portfolio, False)

if emails < 5:
    track.send()
else:
    print 'Skipping check_webcams email due to too many emails'

# done<|endoftext|>


<commit_before>#!/usr/bin/env python3

from plumbum import cli, local, TF, FG, ProcessExecutionError
from plumbum.cmd import tar, mkdir, mv, rm, bash
from benchbuild import settings
from benchbuild.utils import user_interface as ui
from benchbuild.utils import log
from benchbuild.utils.run import uchroot_no_args
from benchbuild.utils.downloader import Copy, update_hash

import logging
import sys
import os

def ask(question, default_answer=False, default_answer_str="no"):
    response = default_answer
    if sys.stdin.isatty():
        response = ui.query_yes_no(question, default_answer_str)
    return response

def clean_directories(builddir, in_dir=True, out_dir=True):
    with local.cwd(builddir):
        if in_dir and os.path.exists("container-in") and ask(
                "Should I delete '{0}'?".format(os.path.abspath(
                    "container-in"))):
            rm("-rf", "container-in")
        if out_dir and os.path.exists("container-out") and ask(
                "Should I delete '{0}'?".format(os.path.abspath(
                    "container-out"))):
            rm("-rf", "container-out")

def setup_directories(builddir):
    with local.cwd(builddir):
        if not os.path.exists("container-in"):
            mkdir("-p", "container-in")
        if not os.path.exists("container-out"):
            mkdir("-p", "container-out")

def setup_container(builddir, container):
    with local.cwd(builddir):
        container_filename = str(container).split(os.path.sep)[-1]
        container_in = os.path.join("container-in", container_filename)
        Copy(container, container_in)
        uchroot = uchroot_no_args()

        with local.cwd("container-in"):
            uchroot = uchroot["-E", "-A", "-u", "0", "-g", "0", "-C", "-r",
                              "/", "-w", os.path.abspath("."), "--"]

        # Check, if we need erlent support for this archive.
        has_erlent = bash[
            "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
                container_in)]
        has_erlent = (has_erlent & TF)

        # Unpack input container to: container-in
        if not has_erlent:
            cmd = local["/bin/tar"]["xf"]
            cmd = uchroot[cmd[container_filename]]
        else:
            cmd = tar["xf"]
            cmd = cmd[os.path.abspath(container_in)]

        with local.cwd("container-in"):
            cmd("--exclude=dev/*")
        rm(container_in)
    return os.path.join(builddir, "container-in")

def run_in_container(command, container_dir, mounts):
    uchroot = uchroot_no_args()["-E", "-A", "-u", "0", "-g", "0", "-C", "-w",
                                "/", "-r", os.path.abspath(container_dir)]
    uchroot_m = uchroot
    uchroot = uchroot["--"]

    for mount in mounts:
        absdir = os.path.abspath(str(mount))
        dirname = os.path.split(absdir)[-1]
        uchroot_m = uchroot_m["-m", "{0}:/mnt/{1}".format(absdir, dirname)]
        mount_path = os.path.join(container_dir, "mnt", dirname)
        if not os.path.exists(mount_path):
            uchroot("mkdir", "-p", "/mnt/{0}".format(dirname))
        print("Mounting: '{0}' inside container at '/mnt/{1}'".format(mount,
                                                                      dirname))

    cmd_path = os.path.join(container_dir, command.lstrip('/'))
    if not os.path.exists(cmd_path):
        logging.error(
            "The command does not exist inside the container! {0}".format(
                cmd_path))
        return

    cmd = uchroot_m[command]
    return cmd & FG

def setup_bash_in_container(builddir, container, outfile, mounts, shell):
    with local.cwd(builddir):
        # Switch to bash inside uchroot
        print("Entering bash inside User-Chroot. Prepare your image and "
              "type 'exit' when you are done. If bash exits with a non-zero"
              "exit code, no new container will be stored.")
        store_new_container = True
        try:
            run_in_container(shell, container, mounts)
        except ProcessExecutionError:
            store_new_container = False

        if store_new_container:  # pylint: disable=W0104
            print("Packing new container image.")

            container_filename = os.path.split(container)[-1]
            container_out = os.path.join("container-out", container_filename)
            container_out = os.path.abspath(container_out)

            # Pack the results to: container-out
            with local.cwd("container-in"):
                tar("cjf", container_out, ".")
            update_hash(container_filename, os.path.dirname(container_out))
            outdir = os.path.dirname(outfile)
            if not os.path.exists(outdir):
                mkdir("-p", outdir)
            mv(container_out, outfile)

class Container(cli.Application):
    """Manage uchroot containers."""
    VERSION = settings.CFG["version"].value()

    def __init__(self, exe):
        super(Container, self).__init__(exe)

    @cli.switch(["-i", "--input-file"], str, help="Input container path")
    def input_file(self, container):
        if os.path.exists(container):
            settings.CFG["container"]["input"] = container
        else:
            raise ValueError("The path '{0}' does not exist.".format(
                container))

    @cli.switch(["-o", "--output-file"], str, help="Output container path")
    def output_file(self, container):
        if os.path.exists(container):
            if not ask("Path '{0}' already exists. Overwrite?"):
                sys.exit(0)
        settings.CFG["container"]["output"] = container

    @cli.switch(["-s", "--shell"],
                str,
                help="The shell command we invoke inside the container.")
    def shell(self, custom_shell):
        settings.CFG["container"]["shell"] = custom_shell

    @cli.switch(["-t", "-tmp-dir"],
                cli.ExistingDirectory,
                help="Temporary directory")
    def builddir(self, tmpdir):
        settings.CFG["build_dir"] = tmpdir

    @cli.switch(
        ["m", "--mount"],
        cli.ExistingDirectory,
        list=True,
        help="Mount the given directory under / inside the uchroot container")
    def mounts(self, user_mount):
        settings.CFG["container"]["mounts"] = user_mount

    verbosity = cli.CountOf('-v', help="Enable verbose output")

    def main(self, *args):

        log.configure()
        LOG = logging.getLogger()
        LOG.setLevel({
            3: logging.DEBUG,
            2: logging.INFO,
            1: logging.WARNING,
            0: logging.ERROR
        }[self.verbosity])

        settings.update_env()

        builddir = os.path.abspath(settings.CFG["build_dir"].value())
        if not os.path.exists(builddir):
            response = ask("The build directory {dirname} does not exist yet. "
                           "Should I create it?".format(dirname=builddir))

            if response:
                mkdir("-p", builddir)
                print("Created directory {0}.".format(builddir))

        setup_directories(builddir)

@Container.subcommand("run")
class ContainerRun(cli.Application):
    def main(self, *args):
        builddir = settings.CFG["build_dir"].value()
        in_container = settings.CFG["container"]["input"].value()
        mounts = settings.CFG["container"]["mounts"].value()
        in_is_file = os.path.isfile(in_container)
        if in_is_file:
            in_container = setup_container(builddir, in_container)
        run_in_container(" ".join(args), in_container, mounts)
        clean_directories(builddir, in_is_file, False)

@Container.subcommand("create")
class ContainerCreate(cli.Application):
    def main(self, *args):
        builddir = settings.CFG["build_dir"].value()
        in_container = settings.CFG["container"]["input"].value()
        out_container = settings.CFG["container"]["output"].value()
        mounts = settings.CFG["container"]["mounts"].value()
        shell = settings.CFG["container"]["shell"].value()
        in_is_file = os.path.isfile(in_container)
        if in_is_file:
            in_container = setup_container(builddir, in_container)
        setup_bash_in_container(builddir, container_dir, out_container, mounts,
                                shell)
        clean_directories(builddir, in_is_file, True)

def find_package(binary):
    try:
        from plumbum import cmd
        cmd.__getattr__(binary)
    except AttributeError:
        print("Checking for {}  - No".format(binary))
        return False
    print("Checking for {} - Yes".format(binary))
    return True

from plumbum.cmd import git

@Container.subcommand("bootstrap")
class ContainerBootstrap(cli.Application):
    def install_uchroot(self):
        builddir = settings.CFG["build_dir"].value()
        with local.cwd(builddir):
            if not os.path.exists("erlent/.git"):
                git("clone", "git@github.com:PolyJIT/erlent")
            else:
                with local.cwd("erlent"):
                    git("pull", "--rebase")
            mkdir("-p", "erlent/build")
            with local.cwd("erlent/build"):
                from plumbum.cmd import cmake, make, cp
                cmake("../")
                make()
        erlent_path = os.path.abspath(os.path.join(builddir, "erlent",
                                                   "build"))
        os.environ["PATH"] = os.path.pathsep.join([erlent_path, os.environ[
            "PATH"]])
        local.env.update(PATH=os.environ["PATH"])
        if not find_package("uchroot"):
            sys.exit(-1)
        settings.CFG["env"]["lookup_path"].value().append(erlent_path)

    def install_cmake_and_exit(self):
        print("You need to  install cmake via your package manager manually."
              " Exiting.")
        sys.exit(-1)

    def main(self, *args):
        print("Checking container binary dependencies...")
        if not find_package("uchroot"):
            if not find_package("cmake"):
                self.install_cmake_and_exit()
            self.install_uchroot()
        print("...OK")
        config_path = ".benchbuild.json"
        settings.CFG.store(config_path)
        print("Storing config in {0}".format(os.path.abspath(config_path)))
        print(
            "Future container commands from this directory will automatically"
            " source the config file.")

def main(*args):
    return Container.run(*args)
<commit_msg>container: container_dir -> in_container<commit_after>#!/usr/bin/env python3

from plumbum import cli, local, TF, FG, ProcessExecutionError
from plumbum.cmd import tar, mkdir, mv, rm, bash
from benchbuild import settings
from benchbuild.utils import user_interface as ui
from benchbuild.utils import log
from benchbuild.utils.run import uchroot_no_args
from benchbuild.utils.downloader import Copy, update_hash

import logging
import sys
import os

def ask(question, default_answer=False, default_answer_str="no"):
    response = default_answer
    if sys.stdin.isatty():
        response = ui.query_yes_no(question, default_answer_str)
    return response

def clean_directories(builddir, in_dir=True, out_dir=True):
    with local.cwd(builddir):
        if in_dir and os.path.exists("container-in") and ask(
                "Should I delete '{0}'?".format(os.path.abspath(
                    "container-in"))):
            rm("-rf", "container-in")
        if out_dir and os.path.exists("container-out") and ask(
                "Should I delete '{0}'?".format(os.path.abspath(
                    "container-out"))):
            rm("-rf", "container-out")

def setup_directories(builddir):
    with local.cwd(builddir):
        if not os.path.exists("container-in"):
            mkdir("-p", "container-in")
        if not os.path.exists("container-out"):
            mkdir("-p", "container-out")

def setup_container(builddir, container):
    with local.cwd(builddir):
        container_filename = str(container).split(os.path.sep)[-1]
        container_in = os.path.join("container-in", container_filename)
        Copy(container, container_in)
        uchroot = uchroot_no_args()

        with local.cwd("container-in"):
            uchroot = uchroot["-E", "-A", "-u", "0", "-g", "0", "-C", "-r",
                              "/", "-w", os.path.abspath("."), "--"]

        # Check, if we need erlent support for this archive.
        has_erlent = bash[
            "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
                container_in)]
        has_erlent = (has_erlent & TF)

        # Unpack input container to: container-in
        if not has_erlent:
            cmd = local["/bin/tar"]["xf"]
            cmd = uchroot[cmd[container_filename]]
        else:
            cmd = tar["xf"]
            cmd = cmd[os.path.abspath(container_in)]

        with local.cwd("container-in"):
            cmd("--exclude=dev/*")
        rm(container_in)
    return os.path.join(builddir, "container-in")

def run_in_container(command, container_dir, mounts):
    uchroot = uchroot_no_args()["-E", "-A", "-u", "0", "-g", "0", "-C", "-w",
                                "/", "-r", os.path.abspath(container_dir)]
    uchroot_m = uchroot
    uchroot = uchroot["--"]

    for mount in mounts:
        absdir = os.path.abspath(str(mount))
        dirname = os.path.split(absdir)[-1]
        uchroot_m = uchroot_m["-m", "{0}:/mnt/{1}".format(absdir, dirname)]
        mount_path = os.path.join(container_dir, "mnt", dirname)
        if not os.path.exists(mount_path):
            uchroot("mkdir", "-p", "/mnt/{0}".format(dirname))
        print("Mounting: '{0}' inside container at '/mnt/{1}'".format(mount,
                                                                      dirname))

    cmd_path = os.path.join(container_dir, command.lstrip('/'))
    if not os.path.exists(cmd_path):
        logging.error(
            "The command does not exist inside the container! {0}".format(
                cmd_path))
        return

    cmd = uchroot_m[command]
    return cmd & FG

def setup_bash_in_container(builddir, container, outfile, mounts, shell):
    with local.cwd(builddir):
        # Switch to bash inside uchroot
        print("Entering bash inside User-Chroot. Prepare your image and "
              "type 'exit' when you are done. If bash exits with a non-zero"
              "exit code, no new container will be stored.")
        store_new_container = True
        try:
            run_in_container(shell, container, mounts)
        except ProcessExecutionError:
            store_new_container = False

        if store_new_container:  # pylint: disable=W0104
            print("Packing new container image.")

            container_filename = os.path.split(container)[-1]
            container_out = os.path.join("container-out", container_filename)
            container_out = os.path.abspath(container_out)

            # Pack the results to: container-out
            with local.cwd("container-in"):
                tar("cjf", container_out, ".")
            update_hash(container_filename, os.path.dirname(container_out))
            outdir = os.path.dirname(outfile)
            if not os.path.exists(outdir):
                mkdir("-p", outdir)
            mv(container_out, outfile)

class Container(cli.Application):
    """Manage uchroot containers."""
    VERSION = settings.CFG["version"].value()

    def __init__(self, exe):
        super(Container, self).__init__(exe)

    @cli.switch(["-i", "--input-file"], str, help="Input container path")
    def input_file(self, container):
        if os.path.exists(container):
            settings.CFG["container"]["input"] = container
        else:
            raise ValueError("The path '{0}' does not exist.".format(
                container))

    @cli.switch(["-o", "--output-file"], str, help="Output container path")
    def output_file(self, container):
        if os.path.exists(container):
            if not ask("Path '{0}' already exists. Overwrite?"):
                sys.exit(0)
        settings.CFG["container"]["output"] = container

    @cli.switch(["-s", "--shell"],
                str,
                help="The shell command we invoke inside the container.")
    def shell(self, custom_shell):
        settings.CFG["container"]["shell"] = custom_shell

    @cli.switch(["-t", "-tmp-dir"],
                cli.ExistingDirectory,
                help="Temporary directory")
    def builddir(self, tmpdir):
        settings.CFG["build_dir"] = tmpdir

    @cli.switch(
        ["m", "--mount"],
        cli.ExistingDirectory,
        list=True,
        help="Mount the given directory under / inside the uchroot container")
    def mounts(self, user_mount):
        settings.CFG["container"]["mounts"] = user_mount

    verbosity = cli.CountOf('-v', help="Enable verbose output")

    def main(self, *args):

        log.configure()
        LOG = logging.getLogger()
        LOG.setLevel({
            3: logging.DEBUG,
            2: logging.INFO,
            1: logging.WARNING,
            0: logging.ERROR
        }[self.verbosity])

        settings.update_env()

        builddir = os.path.abspath(settings.CFG["build_dir"].value())
        if not os.path.exists(builddir):
            response = ask("The build directory {dirname} does not exist yet. "
                           "Should I create it?".format(dirname=builddir))

            if response:
                mkdir("-p", builddir)
                print("Created directory {0}.".format(builddir))

        setup_directories(builddir)

@Container.subcommand("run")
class ContainerRun(cli.Application):
    def main(self, *args):
        builddir = settings.CFG["build_dir"].value()
        in_container = settings.CFG["container"]["input"].value()
        mounts = settings.CFG["container"]["mounts"].value()
        in_is_file = os.path.isfile(in_container)
        if in_is_file:
            in_container = setup_container(builddir, in_container)
        run_in_container(" ".join(args), in_container, mounts)
        clean_directories(builddir, in_is_file, False)

@Container.subcommand("create")
class ContainerCreate(cli.Application):
    def main(self, *args):
        builddir = settings.CFG["build_dir"].value()
        in_container = settings.CFG["container"]["input"].value()
        out_container = settings.CFG["container"]["output"].value()
        mounts = settings.CFG["container"]["mounts"].value()
        shell = settings.CFG["container"]["shell"].value()
        in_is_file = os.path.isfile(in_container)
        if in_is_file:
            in_container = setup_container(builddir, in_container)
        setup_bash_in_container(builddir, in_container, out_container, mounts,
                                shell)
        clean_directories(builddir, in_is_file, True)

def find_package(binary):
    try:
        from plumbum import cmd
        cmd.__getattr__(binary)
    except AttributeError:
        print("Checking for {}  - No".format(binary))
        return False
    print("Checking for {} - Yes".format(binary))
    return True

from plumbum.cmd import git

@Container.subcommand("bootstrap")
class ContainerBootstrap(cli.Application):
    def install_uchroot(self):
        builddir = settings.CFG["build_dir"].value()
        with local.cwd(builddir):
            if not os.path.exists("erlent/.git"):
                git("clone", "git@github.com:PolyJIT/erlent")
            else:
                with local.cwd("erlent"):
                    git("pull", "--rebase")
            mkdir("-p", "erlent/build")
            with local.cwd("erlent/build"):
                from plumbum.cmd import cmake, make, cp
                cmake("../")
                make()
        erlent_path = os.path.abspath(os.path.join(builddir, "erlent",
                                                   "build"))
        os.environ["PATH"] = os.path.pathsep.join([erlent_path, os.environ[
            "PATH"]])
        local.env.update(PATH=os.environ["PATH"])
        if not find_package("uchroot"):
            sys.exit(-1)
        settings.CFG["env"]["lookup_path"].value().append(erlent_path)

    def install_cmake_and_exit(self):
        print("You need to  install cmake via your package manager manually."
              " Exiting.")
        sys.exit(-1)

    def main(self, *args):
        print("Checking container binary dependencies...")
        if not find_package("uchroot"):
            if not find_package("cmake"):
                self.install_cmake_and_exit()
            self.install_uchroot()
        print("...OK")
        config_path = ".benchbuild.json"
        settings.CFG.store(config_path)
        print("Storing config in {0}".format(os.path.abspath(config_path)))
        print(
            "Future container commands from this directory will automatically"
            " source the config file.")

def main(*args):
    return Container.run(*args)
<|endoftext|>


<commit_before>from social_auth.backends import PIPELINE
from social_auth.utils import setting

PIPELINE_ENTRY = 'social_auth.backends.pipeline.misc.save_status_to_session'

def save_status_to_session(request, auth, *args, **kwargs):
    """Saves current social-auth status to session."""
    next_entry = setting('SOCIAL_AUTH_PIPELINE_RESUME_ENTRY')

    try:
        if next_entry:
            idx = PIPELINE.index(next_entry)
        else:
            idx = PIPELINE.index(PIPELINE_ENTRY) + 1
    except ValueError:
        idx = None

    data = auth.to_session_dict(idx, *args, **kwargs)

    name = setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY', 'partial_pipeline')
    request.session[name] = data
    request.session.modified = True
<commit_msg>Add own tuple_index function to stay compatible with python 2.5<commit_after>from social_auth.backends import PIPELINE
from social_auth.utils import setting

PIPELINE_ENTRY = 'social_auth.backends.pipeline.misc.save_status_to_session'

def tuple_index(t, e):
    for (i, te) in enumerate(t):
        if te == e:
            return i
    return None

def save_status_to_session(request, auth, *args, **kwargs):
    """Saves current social-auth status to session."""
    next_entry = setting('SOCIAL_AUTH_PIPELINE_RESUME_ENTRY')

    if next_entry:
        idx = tuple_index(PIPELINE, next_entry)
    else:
        idx = tuple_index(PIPELINE, PIPELINE_ENTRY)
        if idx:
            idx += 1

    data = auth.to_session_dict(idx, *args, **kwargs)

    name = setting('SOCIAL_AUTH_PARTIAL_PIPELINE_KEY', 'partial_pipeline')
    request.session[name] = data
    request.session.modified = True
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from django.db import connections
from django.conf import settings
from datetime import datetime

from dimagi.utils.chunked import chunked

from corehq.warehouse.models.meta import Batch
from corehq.warehouse.const import DJANGO_MAX_BATCH_SIZE
from corehq.sql_db.routers import db_for_read_write
from corehq.util.quickcache import quickcache
from corehq.util.soft_assert import soft_assert
from custom.icds.const import ACCEPTABLE_WAREHOUSE_LAG_IN_MINUTES

def django_batch_records(cls, record_iter, field_mapping, batch_id):
    for batch in chunked(record_iter, DJANGO_MAX_BATCH_SIZE):
        records = []
        for raw_record in batch:
            record = {'batch_id': batch_id}
            for source_key, destination_key in field_mapping:
                value = raw_record
                for key in source_key.split('.'):
                    if isinstance(raw_record, dict):
                        value = value.get(key)
                    else:
                        value = getattr(value, key, None)
                record[destination_key] = value

            records.append(cls(**record))

        cls.objects.bulk_create(records, batch_size=DJANGO_MAX_BATCH_SIZE)

def truncate_records_for_cls(cls, cascade=False):
    if settings.UNIT_TESTING:
        cls.objects.all().delete()
        return
    database = db_for_read_write(cls)
    with connections[database].cursor() as cursor:
        cursor.execute("TRUNCATE {} {}".format(cls._meta.db_table, 'CASCADE' if cascade else ''))

@quickcache([], timeout=60 * 60)
def get_warehouse_latest_modified_date(email_on_delay=False):
    """
    Return in minutes how fresh is the data of app_status warehouse model.
    """
    last_completed_app_status_batch = Batch.objects.filter(
        dag_slug='app_status_batch', completed_on__isnull=False
    ).order_by('completed_on').last()
    # The end_datetime of a batch is used to filter on forms by last_modified (received_on, edited_on, deleted_on)
    if not last_completed_app_status_batch:
        return datetime(2000, 1, 1)
    latest_date = last_completed_app_status_batch.end_datetime
    if email_on_delay:
        SMS_TEAM = ['{}@{}'.format('icds-sms-rule', 'dimagi.com')]
        _soft_assert = soft_assert(to=SMS_TEAM, send_to_ops=False)
        lag = (datetime.utcnow() - latest_date).total_seconds() / 60
        if lag > ACCEPTABLE_WAREHOUSE_LAG_IN_MINUTES:
            _soft_assert(False,
                "The weekly inactive SMS rule is skipped for this week. Warehouse lag is {} minutes"
                .format(str(lag))
            )
        else:
            _soft_assert(False, "The weekly inactive SMS rule is successfully triggered for this week")

    return latest_date
<commit_msg>future<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from django.db import connections
from django.conf import settings
from datetime import datetime

from dimagi.utils.chunked import chunked

from corehq.warehouse.models.meta import Batch
from corehq.warehouse.const import DJANGO_MAX_BATCH_SIZE
from corehq.sql_db.routers import db_for_read_write
from corehq.util.quickcache import quickcache
from corehq.util.soft_assert import soft_assert
from custom.icds.const import ACCEPTABLE_WAREHOUSE_LAG_IN_MINUTES

def django_batch_records(cls, record_iter, field_mapping, batch_id):
    for batch in chunked(record_iter, DJANGO_MAX_BATCH_SIZE):
        records = []
        for raw_record in batch:
            record = {'batch_id': batch_id}
            for source_key, destination_key in field_mapping:
                value = raw_record
                for key in source_key.split('.'):
                    if isinstance(raw_record, dict):
                        value = value.get(key)
                    else:
                        value = getattr(value, key, None)
                record[destination_key] = value

            records.append(cls(**record))

        cls.objects.bulk_create(records, batch_size=DJANGO_MAX_BATCH_SIZE)

def truncate_records_for_cls(cls, cascade=False):
    if settings.UNIT_TESTING:
        cls.objects.all().delete()
        return
    database = db_for_read_write(cls)
    with connections[database].cursor() as cursor:
        cursor.execute("TRUNCATE {} {}".format(cls._meta.db_table, 'CASCADE' if cascade else ''))

@quickcache([], timeout=60 * 60)
def get_warehouse_latest_modified_date(email_on_delay=False):
    """
    Return in minutes how fresh is the data of app_status warehouse model.
    """
    last_completed_app_status_batch = Batch.objects.filter(
        dag_slug='app_status_batch', completed_on__isnull=False
    ).order_by('completed_on').last()
    # The end_datetime of a batch is used to filter on forms by last_modified (received_on, edited_on, deleted_on)
    if not last_completed_app_status_batch:
        return datetime(2000, 1, 1)
    latest_date = last_completed_app_status_batch.end_datetime
    if email_on_delay:
        SMS_TEAM = ['{}@{}'.format('icds-sms-rule', 'dimagi.com')]
        _soft_assert = soft_assert(to=SMS_TEAM, send_to_ops=False)
        lag = (datetime.utcnow() - latest_date).total_seconds() / 60
        if lag > ACCEPTABLE_WAREHOUSE_LAG_IN_MINUTES:
            _soft_assert(False,
                "The weekly inactive SMS rule is skipped for this week. Warehouse lag is {} minutes"
                .format(str(lag))
            )
        else:
            _soft_assert(False, "The weekly inactive SMS rule is successfully triggered for this week")

    return latest_date
<|endoftext|>


<commit_before>"""epoll loop backend

This module is deprecated and will be removed once pyuv_cffi is fully implemented.
"""
import logging
import heapq
import select
import errno
import greenlet

from .. import patcher
from ..const import READ, WRITE
from ..support import compat, get_errno
from . import abc
from .watchers import PollFdListener
from .timer import Timer

compat.patch()

log = logging.getLogger('guv')

epoll = patcher.original('select').epoll
time = patcher.original('time')
queue = patcher.original('queue')

EXC_MASK = select.POLLERR | select.POLLHUP
READ_MASK = select.POLLIN | select.POLLPRI
WRITE_MASK = select.POLLOUT
SYSTEM_EXCEPTIONS = (KeyboardInterrupt, SystemExit)

class Hub(abc.AbstractHub):
    def __init__(self):
        super().__init__()
        self.Listener = PollFdListener
        self.stopping = False
        self.running = False
        self.timers = []  # heap of timers
        self.callbacks = []  # list of callbacks
        self.epoll = epoll()

    def _fire_timers(self, up_to):
        """Fire timers scheduled to be fire before `up_to`

        :param float up_to: timers scheduled to be fired before this specified absolute time
            (seconds) will be fired
        """
        timers = self.timers

        while timers:
            timer = timers[0]

            if timer.called:
                heapq.heappop(timers)
                continue

            if timer > up_to:
                break

            if timer.absolute_time > time.monotonic():
                break

            # call the timer to call the callback
            timer()

            heapq.heappop(timers)

    def _time_until_next_timer(self):
        """Get number of seconds until next timer has to be fired
        """
        if not self.timers:
            return None

        return self.timers[0].absolute_time - time.monotonic()

    def _remove_descriptor(self, fd):
        #: :type: dict[fd, listener]
        readers = self.listeners[READ]
        #: :type: dict[fd, listener]
        writers = self.listeners[WRITE]

        if fd in readers:
            del readers[fd]

        if fd in writers:
            del writers[fd]

    def _fire_callbacks(self):
        """Fire immediate callbacks
        """
        callbacks = self.callbacks
        self.callbacks = []
        for cb, args, kwargs in callbacks:
            cb(*args, **kwargs)

            # TODO: (urgent) add support for idle_h, as used in pyuv_cffi.py

    def run(self):
        assert self is greenlet.getcurrent()

        if self.stopping:
            return

        if self.running:
            raise RuntimeError("The hub's runloop is already running")

        log.debug('Start runloop')
        try:
            self.running = True
            self.stopping = False
            self._run_loop()
        finally:
            self.running = False
            self.stopping = False

    def _run_loop(self):
        try:
            while True:
                self._fire_timers(time.monotonic())

                # call scheduled callbacks (made via spawn(), etc.)
                self._fire_callbacks()

                # poll for I/O and call I/O callbacks
                self._poll()

        except KeyboardInterrupt:
            self.abort()
            self.parent.throw(KeyboardInterrupt)

    def _poll(self):
        # get time until next timer, or 60 seconds if there are no timers
        sleep_time = self._time_until_next_timer()
        if sleep_time is None:
            sleep_time = 60

        readers = self.listeners[READ]
        writers = self.listeners[WRITE]

        # sleep if no I/O listeners are registered
        if not (readers or writers):
            if sleep_time:
                time.sleep(sleep_time)
            return

        # call epoll()
        try:
            poll_result = self.epoll.poll(sleep_time)
        except (IOError, select.error) as e:
            if get_errno(e) == errno.EINTR:
                return
            raise

        # build a set of listeners to process from the results of epoll()
        listeners = set()

        for fd, event in poll_result:
            if event & READ_MASK:
                listeners.add(readers[fd])
            if event & WRITE_MASK:
                listeners.add(writers[fd])
            if event & select.POLLNVAL:
                self._remove_descriptor(fd)
                continue
            if event & EXC_MASK:
                if fd in readers:
                    listeners.add(readers[fd])
                if fd in writers:
                    listeners.add(writers[fd])

        # call listener callbacks
        for listener in listeners:
            try:
                listener.cb()
            except SYSTEM_EXCEPTIONS:
                raise

    def abort(self):
        print()
        log.debug('Abort loop')
        if self.running:
            self.stopping = True

    def schedule_call_now(self, cb, *args, **kwargs):
        self.callbacks.append((cb, args, kwargs))

    def schedule_call_global(self, seconds, cb, *args, **kwargs):
        timer = Timer(seconds, cb, *args, **kwargs)
        heapq.heappush(self.timers, timer)

        return timer

    def add(self, evtype, fd, cb, tb):
        """Signals an intent to or write a particular file descriptor

        Signature of Callable cb: cb(fd: int)

        :param str evtype: either the constant READ or WRITE
        :param int fd: file number of the file of interest
        :param cb: callback which will be called when the file is ready for reading/writing
        :param tb: throwback used to signal (into the greenlet) that the file was closed
        :return: listener
        :rtype: self.Listener
        """
        # create and add listener object
        listener = PollFdListener(evtype, fd, cb)
        self._add_listener(listener)

        # register the fd with the epoll object
        flags = 0
        if evtype == READ:
            flags = select.EPOLLIN | select.EPOLLPRI
        elif evtype == WRITE:
            flags = select.EPOLLOUT

        self.epoll.register(fd, flags)

        return listener

    def remove(self, listener):
        """Remove listener

        :param listener: listener to remove
        :type listener: self.Listener
        """
        super()._remove_listener(listener)
        # log.debug('call w.handle.stop(), fd: {}'.format(listener.handle.fileno()))

        # initiate correct cleanup sequence (these three statements are critical)
        self.epoll.unregister(listener.fd)
        # self.debug()
<commit_msg>Add note that epoll.py is currently broken<commit_after>"""epoll loop backend

This module is deprecated and will be removed once pyuv_cffi is fully implemented.

This module is currently broken until support for non-I/O callbacks is added, similar to how this is
handled in pyuv_cffi.py
"""
# FIXME: (critical) add support for non-I/O callbacks for gyield()
import logging
import heapq
import select
import errno
import greenlet

from .. import patcher
from ..const import READ, WRITE
from ..support import compat, get_errno
from . import abc
from .watchers import PollFdListener
from .timer import Timer

compat.patch()

log = logging.getLogger('guv')

epoll = patcher.original('select').epoll
time = patcher.original('time')
queue = patcher.original('queue')

EXC_MASK = select.POLLERR | select.POLLHUP
READ_MASK = select.POLLIN | select.POLLPRI
WRITE_MASK = select.POLLOUT
SYSTEM_EXCEPTIONS = (KeyboardInterrupt, SystemExit)

class Hub(abc.AbstractHub):
    def __init__(self):
        super().__init__()
        self.Listener = PollFdListener
        self.stopping = False
        self.running = False
        self.timers = []  # heap of timers
        self.callbacks = []  # list of callbacks
        self.epoll = epoll()

    def _fire_timers(self, up_to):
        """Fire timers scheduled to be fire before `up_to`

        :param float up_to: timers scheduled to be fired before this specified absolute time
            (seconds) will be fired
        """
        timers = self.timers

        while timers:
            timer = timers[0]

            if timer.called:
                heapq.heappop(timers)
                continue

            if timer > up_to:
                break

            if timer.absolute_time > time.monotonic():
                break

            # call the timer to call the callback
            timer()

            heapq.heappop(timers)

    def _time_until_next_timer(self):
        """Get number of seconds until next timer has to be fired
        """
        if not self.timers:
            return None

        return self.timers[0].absolute_time - time.monotonic()

    def _remove_descriptor(self, fd):
        #: :type: dict[fd, listener]
        readers = self.listeners[READ]
        #: :type: dict[fd, listener]
        writers = self.listeners[WRITE]

        if fd in readers:
            del readers[fd]

        if fd in writers:
            del writers[fd]

    def _fire_callbacks(self):
        """Fire immediate callbacks
        """
        callbacks = self.callbacks
        self.callbacks = []
        for cb, args, kwargs in callbacks:
            cb(*args, **kwargs)

    def run(self):
        assert self is greenlet.getcurrent()

        if self.stopping:
            return

        if self.running:
            raise RuntimeError("The hub's runloop is already running")

        log.debug('Start runloop')
        try:
            self.running = True
            self.stopping = False
            self._run_loop()
        finally:
            self.running = False
            self.stopping = False

    def _run_loop(self):
        try:
            while True:
                self._fire_timers(time.monotonic())

                # call scheduled callbacks (made via spawn(), etc.)
                self._fire_callbacks()

                # poll for I/O and call I/O callbacks
                self._poll()

        except KeyboardInterrupt:
            self.abort()
            self.parent.throw(KeyboardInterrupt)

    def _poll(self):
        # get time until next timer, or 60 seconds if there are no timers
        sleep_time = self._time_until_next_timer()
        if sleep_time is None:
            sleep_time = 60

        readers = self.listeners[READ]
        writers = self.listeners[WRITE]

        # sleep if no I/O listeners are registered
        if not (readers or writers):
            if sleep_time:
                time.sleep(sleep_time)
            return

        # call epoll()
        try:
            poll_result = self.epoll.poll(sleep_time)
        except (IOError, select.error) as e:
            if get_errno(e) == errno.EINTR:
                return
            raise

        # build a set of listeners to process from the results of epoll()
        listeners = set()

        for fd, event in poll_result:
            if event & READ_MASK:
                listeners.add(readers[fd])
            if event & WRITE_MASK:
                listeners.add(writers[fd])
            if event & select.POLLNVAL:
                self._remove_descriptor(fd)
                continue
            if event & EXC_MASK:
                if fd in readers:
                    listeners.add(readers[fd])
                if fd in writers:
                    listeners.add(writers[fd])

        # call listener callbacks
        for listener in listeners:
            try:
                listener.cb()
            except SYSTEM_EXCEPTIONS:
                raise

    def abort(self):
        print()
        log.debug('Abort loop')
        if self.running:
            self.stopping = True

    def schedule_call_now(self, cb, *args, **kwargs):
        self.callbacks.append((cb, args, kwargs))

    def schedule_call_global(self, seconds, cb, *args, **kwargs):
        timer = Timer(seconds, cb, *args, **kwargs)
        heapq.heappush(self.timers, timer)

        return timer

    def add(self, evtype, fd, cb, tb):
        """Signals an intent to or write a particular file descriptor

        Signature of Callable cb: cb(fd: int)

        :param str evtype: either the constant READ or WRITE
        :param int fd: file number of the file of interest
        :param cb: callback which will be called when the file is ready for reading/writing
        :param tb: throwback used to signal (into the greenlet) that the file was closed
        :return: listener
        :rtype: self.Listener
        """
        # create and add listener object
        listener = PollFdListener(evtype, fd, cb)
        self._add_listener(listener)

        # register the fd with the epoll object
        flags = 0
        if evtype == READ:
            flags = select.EPOLLIN | select.EPOLLPRI
        elif evtype == WRITE:
            flags = select.EPOLLOUT

        self.epoll.register(fd, flags)

        return listener

    def remove(self, listener):
        """Remove listener

        :param listener: listener to remove
        :type listener: self.Listener
        """
        super()._remove_listener(listener)
        # log.debug('call w.handle.stop(), fd: {}'.format(listener.handle.fileno()))

        # initiate correct cleanup sequence (these three statements are critical)
        self.epoll.unregister(listener.fd)
        # self.debug()
<|endoftext|>


<commit_before>import logging.config as lc

__LOG_DICT = {
    "version": 1,
    "formatters": {
        "brief": {
            'class': 'logging.Formatter',
            'format': '%(message)s'
        },
        "plumbum": {
            'class': 'logging.Formatter',
            'format': '--- %(message)s'
        }
    },
    "handlers": {
        "console": {
            'class': 'logging.StreamHandler',
            'formatter': 'brief'
        },
        "plumbum": {
            'class': 'logging.StreamHandler',
            'formatter': 'plumbum'
        }
    },
    "loggers": {
        "benchbuild": {},
        "benchbuild.steps": {},
        "plumbum.local": {'level': 'DEBUG',
                          'handlers': ['plumbum'],
                          'propagate': False}
    },
    "root": {
        "level": 'WARN',
        "handlers": ["console"]
    }
}

def configure():
    lc.dictConfig(__LOG_DICT)
<commit_msg>log: default log level for plumbum should be 'WARN'<commit_after>import logging.config as lc

__LOG_DICT = {
    "version": 1,
    "formatters": {
        "brief": {
            'class': 'logging.Formatter',
            'format': '%(message)s'
        },
        "plumbum": {
            'class': 'logging.Formatter',
            'format': '--- %(message)s'
        }
    },
    "handlers": {
        "console": {
            'class': 'logging.StreamHandler',
            'formatter': 'brief'
        },
        "plumbum": {
            'class': 'logging.StreamHandler',
            'formatter': 'plumbum'
        }
    },
    "loggers": {
        "benchbuild": {},
        "benchbuild.steps": {},
        "plumbum.local": {'level': 'WARN',
                          'handlers': ['plumbum'],
                          'propagate': False}
    },
    "root": {
        "level": 'WARN',
        "handlers": ["console"]
    }
}

def configure():
    lc.dictConfig(__LOG_DICT)
<|endoftext|>


<commit_before>from django import http
from urlparse import urlparse

from corsheaders import defaults as settings

ACCESS_CONTROL_ALLOW_ORIGIN = 'Access-Control-Allow-Origin'
ACCESS_CONTROL_EXPOSE_HEADERS = 'Access-Control-Expose-Headers'
ACCESS_CONTROL_ALLOW_CREDENTIALS = 'Access-Control-Allow-Credentials'
ACCESS_CONTROL_ALLOW_HEADERS = 'Access-Control-Allow-Headers'
ACCESS_CONTROL_ALLOW_METHODS = 'Access-Control-Allow-Methods'
ACCESS_CONTROL_MAX_AGE = 'Access-Control-Max-Age'

class CorsMiddleware(object):

    def process_request(self, request):
        '''
            If CORS preflight header, then create an empty body response (200 OK) and return it

            Django won't bother calling any other request view/exception middleware along with
            the requested view; it will call any response middlewares
        '''
        if (request.method == 'OPTIONS' and
            'HTTP_ACCESS_CONTROL_REQUEST_METHOD' in request.META):
            response = http.HttpResponse()
            return response
        return None

    def process_response(self, request, response):
        '''
            Add the respective CORS headers
        '''
        origin = request.META.get('HTTP_ORIGIN')
        if origin:
            # todo: check hostname from db instead
            url = urlparse(origin)

            if not settings.CORS_ORIGIN_ALLOW_ALL and url.hostname not in settings.CORS_ORIGIN_WHITELIST:
                return response

            response[ACCESS_CONTROL_ALLOW_ORIGIN] = "*" if settings.CORS_ORIGIN_ALLOW_ALL else origin

            if len(settings.CORS_EXPOSE_HEADERS):
                response[ACCESS_CONTROL_EXPOSE_HEADERS] = ', '.join(settings.CORS_EXPOSE_HEADERS)

            if settings.CORS_ALLOW_CREDENTIALS:
                response[ACCESS_CONTROL_ALLOW_CREDENTIALS] = 'true'

            if request.method == 'OPTIONS':
                response[ACCESS_CONTROL_ALLOW_HEADERS] = ', '.join(settings.CORS_ALLOW_HEADERS)
                response[ACCESS_CONTROL_ALLOW_METHODS] = ', '.join(settings.CORS_ALLOW_METHODS)
                if settings.CORS_PREFLIGHT_MAX_AGE:
                    response[ACCESS_CONTROL_MAX_AGE] = settings.CORS_PREFLIGHT_MAX_AGE

        return response
<commit_msg>Use url.netloc instead of url.hostname to support ports for CORS<commit_after>from django import http
from urlparse import urlparse

from corsheaders import defaults as settings

ACCESS_CONTROL_ALLOW_ORIGIN = 'Access-Control-Allow-Origin'
ACCESS_CONTROL_EXPOSE_HEADERS = 'Access-Control-Expose-Headers'
ACCESS_CONTROL_ALLOW_CREDENTIALS = 'Access-Control-Allow-Credentials'
ACCESS_CONTROL_ALLOW_HEADERS = 'Access-Control-Allow-Headers'
ACCESS_CONTROL_ALLOW_METHODS = 'Access-Control-Allow-Methods'
ACCESS_CONTROL_MAX_AGE = 'Access-Control-Max-Age'

class CorsMiddleware(object):

    def process_request(self, request):
        '''
            If CORS preflight header, then create an empty body response (200 OK) and return it

            Django won't bother calling any other request view/exception middleware along with
            the requested view; it will call any response middlewares
        '''
        if (request.method == 'OPTIONS' and
            'HTTP_ACCESS_CONTROL_REQUEST_METHOD' in request.META):
            response = http.HttpResponse()
            return response
        return None

    def process_response(self, request, response):
        '''
            Add the respective CORS headers
        '''
        origin = request.META.get('HTTP_ORIGIN')
        if origin:
            # todo: check hostname from db instead
            url = urlparse(origin)

            if not settings.CORS_ORIGIN_ALLOW_ALL and url.netloc not in settings.CORS_ORIGIN_WHITELIST:
                return response

            response[ACCESS_CONTROL_ALLOW_ORIGIN] = "*" if settings.CORS_ORIGIN_ALLOW_ALL else origin

            if len(settings.CORS_EXPOSE_HEADERS):
                response[ACCESS_CONTROL_EXPOSE_HEADERS] = ', '.join(settings.CORS_EXPOSE_HEADERS)

            if settings.CORS_ALLOW_CREDENTIALS:
                response[ACCESS_CONTROL_ALLOW_CREDENTIALS] = 'true'

            if request.method == 'OPTIONS':
                response[ACCESS_CONTROL_ALLOW_HEADERS] = ', '.join(settings.CORS_ALLOW_HEADERS)
                response[ACCESS_CONTROL_ALLOW_METHODS] = ', '.join(settings.CORS_ALLOW_METHODS)
                if settings.CORS_PREFLIGHT_MAX_AGE:
                    response[ACCESS_CONTROL_MAX_AGE] = settings.CORS_PREFLIGHT_MAX_AGE

        return response
<|endoftext|>