<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# Copyright (C) 2014  Philipp Rosenkranz <philipp.rosenkranz@fu-berlin.de>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

import sys, signal, threading

from twisted.internet import reactor
from twisted.internet.protocol import Protocol, Factory

class PubProtocol(Protocol):
    def __init__(self, factory):
        self.factory = factory

    def connectionMade(self):
        print("new connection made")

    def connectionLost(self, reason):
        self.factory.numProtocols = self.factory.numProtocols - 1

    def connectionLost(self, reason):
        self.factory.clients = {key: value for key, value in self.factory.clients.items()
             if value is not self.transport}

    def dataReceived(self, data):
        if data.startswith("hostname: "):
            remoteHostname = data.split()[1].strip()
            print("dataReceived added: " + remoteHostname)
            self.factory.clients[remoteHostname] = self.transport
        else:
            print("received some useless data...")

class PubFactory(Factory):
    def __init__(self):
        self.clients = dict()

    def buildProtocol(self, addr):
        return PubProtocol(self)

class ExperimentRunner():
    def __init__(self, experiment, testbed):
        self.publisher = PubFactory()
        self.port = int(testbed.serverPort)
        self.experiment = experiment(self.publisher, self)
        self.testbed = testbed

    def run(self):
        signal.signal(signal.SIGINT, self.handle_sigint)
        self.experiment.run()
        reactor.listenTCP(self.port, self.publisher)
        # clean logfiles and start nodes but don't flash nodes
        self.testbed.initClean()
        reactor.run()

    def stop(self):
        self.testbed.stop()
        reactor.callFromThread(self.safeReactorStop)

    def safeReactorStop(self):
        if reactor.running:
            try:
                reactor.stop()
            except:
                print("tried to shutdown reactor twice!")

    def handle_sigint(self, signal, frame):
        self.experiment.stop()
        self.testbed.stop()
        self.stop() # shutdown if experiment didn't already

class Experiment():
    def __init__(self, factory, runner):
        self.factory = factory
        self.runner = runner
        self.hostid = dict()
        self.sumDelay = 0

    def run(self):
        print("Running preHook")
        self.preHook()
        print("Running experiment")
        self.start()

    def start(self):
        raise NotImplementedError("Inherit from Experiment and implement start")

    def stop(self):
        print("Running postHook")
        self.postHook()
        self.runner.stop()

    def preHook(self):
        pass

    def postHook(self):
        pass

    def readHostFile(self, path):
        id = 1
        with open(path) as f:
            for line in f:
                self.hostid[line.strip()] = id
                id += 1

    def sendToHost(self, host=None, cmd=""):
        if host in self.factory.clients:
            self.factory.clients[host].write(cmd + "\n")
        else:
            print("sendToHost: no such host known: " + host + " !")

    def sendToAll(self, cmd=""):
       for host, transport in self.factory.clients.items():
            self.sendToHost(host, cmd)

    def pauseInSeconds(self, seconds=0):
        from time import time, sleep
        start = time()
        while (time() - start < seconds):
            sleep(seconds - (time() - start))

    def callLater(self, absoluteDelay = 0.0, function = None):
        reactor.callLater(absoluteDelay, function)

    def waitAndCall(self, relativeDelay = 0.0, function = None):
        self.sumDelay += relativeDelay
        self.callLater(self.sumDelay, function)

    def clientIterator(self):
        return self.factory.clients.items()

    def connectionByHostname(self, host=None):
        if host in self.factory.clients:
            return self.factory.clients[host]

    @staticmethod
    def sendToConnection(connection, line):
        connection.write(line + "\n")
<commit_msg>dist/tools/pyterm: remove duplicate method in PubProtocol<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# Copyright (C) 2014  Philipp Rosenkranz <philipp.rosenkranz@fu-berlin.de>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

import sys, signal, threading

from twisted.internet import reactor
from twisted.internet.protocol import Protocol, Factory

class PubProtocol(Protocol):
    def __init__(self, factory):
        self.factory = factory

    def connectionMade(self):
        print("new connection made")

    def connectionLost(self, reason):
        self.factory.clients = {
            key: value for key, value in self.factory.clients.items()
            if value is not self.transport}

    def dataReceived(self, data):
        if data.startswith("hostname: "):
            remoteHostname = data.split()[1].strip()
            print("dataReceived added: " + remoteHostname)
            self.factory.clients[remoteHostname] = self.transport
        else:
            print("received some useless data...")

class PubFactory(Factory):
    def __init__(self):
        self.clients = dict()

    def buildProtocol(self, addr):
        return PubProtocol(self)

class ExperimentRunner():
    def __init__(self, experiment, testbed):
        self.publisher = PubFactory()
        self.port = int(testbed.serverPort)
        self.experiment = experiment(self.publisher, self)
        self.testbed = testbed

    def run(self):
        signal.signal(signal.SIGINT, self.handle_sigint)
        self.experiment.run()
        reactor.listenTCP(self.port, self.publisher)
        # clean logfiles and start nodes but don't flash nodes
        self.testbed.initClean()
        reactor.run()

    def stop(self):
        self.testbed.stop()
        reactor.callFromThread(self.safeReactorStop)

    def safeReactorStop(self):
        if reactor.running:
            try:
                reactor.stop()
            except:
                print("tried to shutdown reactor twice!")

    def handle_sigint(self, signal, frame):
        self.experiment.stop()
        self.testbed.stop()
        self.stop() # shutdown if experiment didn't already

class Experiment():
    def __init__(self, factory, runner):
        self.factory = factory
        self.runner = runner
        self.hostid = dict()
        self.sumDelay = 0

    def run(self):
        print("Running preHook")
        self.preHook()
        print("Running experiment")
        self.start()

    def start(self):
        raise NotImplementedError("Inherit from Experiment and implement start")

    def stop(self):
        print("Running postHook")
        self.postHook()
        self.runner.stop()

    def preHook(self):
        pass

    def postHook(self):
        pass

    def readHostFile(self, path):
        id = 1
        with open(path) as f:
            for line in f:
                self.hostid[line.strip()] = id
                id += 1

    def sendToHost(self, host=None, cmd=""):
        if host in self.factory.clients:
            self.factory.clients[host].write(cmd + "\n")
        else:
            print("sendToHost: no such host known: " + host + " !")

    def sendToAll(self, cmd=""):
       for host, transport in self.factory.clients.items():
            self.sendToHost(host, cmd)

    def pauseInSeconds(self, seconds=0):
        from time import time, sleep
        start = time()
        while (time() - start < seconds):
            sleep(seconds - (time() - start))

    def callLater(self, absoluteDelay = 0.0, function = None):
        reactor.callLater(absoluteDelay, function)

    def waitAndCall(self, relativeDelay = 0.0, function = None):
        self.sumDelay += relativeDelay
        self.callLater(self.sumDelay, function)

    def clientIterator(self):
        return self.factory.clients.items()

    def connectionByHostname(self, host=None):
        if host in self.factory.clients:
            return self.factory.clients[host]

    @staticmethod
    def sendToConnection(connection, line):
        connection.write(line + "\n")
<|endoftext|>


<commit_before>from datetime import datetime
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponseRedirect, Http404
from django.template import RequestContext
from django.db import transaction

from django_fixmystreet.fixmystreet.utils import render_to_pdf
from django_fixmystreet.fixmystreet.models import Report, FMSUser, OrganisationEntity, ReportComment, ReportFile
from django_fixmystreet.fixmystreet.forms import ReportCommentForm, ReportFileForm, RefuseForm

@transaction.commit_on_success
def accept( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
<<<<<<< HEAD
    #Update the status and persist to the database
    report.status = Report.MANAGER_ASSIGNED
    report.save()
    #Redirect to the report show page
=======
    report.status = Report.MANAGER_ASSIGNED
    report.save()

>>>>>>> remove pyflakes
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def refuse( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    #Update the status
    report.status = Report.REFUSED
    form = RefuseForm(request)
    #Save the refusal motivation in the database
    report.refusal_motivation = form.data.POST.get('refusal_motivation')
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def fixed( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    #Update the status
    report.status = Report.SOLVED
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def close( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    #Update the status and set the close date
    report.status = Report.PROCESSED
    report.close_date = datetime.now()
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def new( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    if request.method == 'POST':
        if request.POST['form-type'] == u"comment-form":
            comment_form = ReportCommentForm(request.POST)
            if comment_form.is_valid():
                comment_form.save(request.user, report)

        if request.POST['form-type'] == u"file-form":
            if request.POST['title'] == "":
                request.POST['title']= request.FILES.get('file').name
            file_form = ReportFileForm(request.POST,request.FILES)
            if file_form.is_valid:
                file_form.save(request.user, report)

        if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
        else:
            return HttpResponseRedirect(report.get_absolute_url())
    raise Http404()

def switchPrivacy(request,report_id):
    report = get_object_or_404(Report, id=report_id)
    privacy = request.REQUEST.get("privacy")
    report.private = ('true' == privacy)
    report.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def changeManager(request,report_id):
    report = Report.objects.get(pk=report_id)
    report.status = Report.MANAGER_ASSIGNED
    manId = request.REQUEST.get("manId")
    if manId.split("_")[0] == "manager":
        newRespMan = FMSUser.objects.get(pk=int(manId.split("_")[1]))
        report.responsible_manager = newRespMan
        report.save()
    if manId.split("_")[0] == "entity":
        orgId = int(manId.split("_")[1])
        report.responsible_entity = OrganisationEntity.objects.get(id=orgId)
        managers = FMSUser.objects.filter(organisation_id = orgId).filter(manager=True)
        for manager in managers:
            if manager.categories.all().filter(id = report.category.id):
                report.responsible_manager = manager
                report.save()
                break

    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())
def changeContractor(request,report_id):
    report = get_object_or_404(Report, id=report_id)
    contractorId = request.REQUEST.get("contractorId")
    if contractorId=='-1':
        report.subcontractor = None
        #Restore status to IN_PROGRESS
        report.status = Report.IN_PROGRESS
    else:
        organisation = OrganisationEntity.objects.get(pk=int(contractorId))
        report.contractor = organisation
        if organisation.is_subcontractor() == True:
            report.status = Report.CONTRACTOR_ASSIGNED
        else:
            report.status = Report.APPLICANT_RESPONSIBLE

    report.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def reportPdf(request, report_id, pro_version):
    '''reportPdf is called from report details page to generate the pdf with report story. When pro_version == 0 then filter pdf content'''
    report = get_object_or_404(Report, id=report_id)

    #Verify if the connected user is well pro ! (Server side protection)
    # if request.user.fmsuser.is_citizen():
       # pro_Version = 0

    if request.GET.get('output', False):
        return render_to_response("pro/pdf.html", {'report' : report, 'file_list' : report.get_files(), 'comment_list' : report.get_comments(), 'pro_version': pro_version},
                context_instance=RequestContext(request))
    else:
        return render_to_pdf("pro/pdf.html", {'report' : report, 'file_list' : report.get_files(), 'comment_list' : report.get_comments(), 'pro_version' : pro_version},
                context_instance=RequestContext(request))

def acceptAndValidate(request, report_id):
    report = get_object_or_404(Report, id=report_id)
    report.status = Report.MANAGER_ASSIGNED
    report.save()
    comments = ReportComment.objects.filter(report_id=report_id)
    for comment in comments:
        comment.is_validated= True
        comment.save()
    files = ReportFile.objects.filter(report_id=report_id)
    for f in files:
        f.is_validated = True
        f.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def validateAll(request,report_id):
    report = get_object_or_404(Report, id=report_id)
    comments = ReportComment.objects.filter(report_id=report_id)
    files = ReportFile.objects.filter(report_id=report_id)
    for comment in comments:
        comment.is_validated = True
        comment.is_visible = True
        comment.save()
    for f in files:
        f.is_validated = True
        f.is_visible = True
        f.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def updateComment(request,report_id):
    report = get_object_or_404(Report,id=report_id)
    updateType = request.REQUEST.get('updateType')
    comment = ReportComment.objects.get(pk=request.REQUEST.get('commentId'))
    if updateType == "valid":
        comment.is_validated = (request.REQUEST.get('updateValue')=='checked')
        if comment.is_validated:
            comment.is_visible= True
    if updateType == 'confidential':
        comment.is_visible = not (request.REQUEST.get('updateValue')=='checked')
        if comment.is_visible == False:
            comment.is_validated = False #When setting element to confidential then it becomes unvalidated automatically

    comment.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def updateFile(request,report_id):
    report = get_object_or_404(Report,id=report_id)
    updateType = request.REQUEST.get('updateType')
    f = ReportFile.objects.get(pk=request.REQUEST.get('fileId'))
    if updateType == "valid":
        f.is_validated= (request.REQUEST.get('updateValue')=='checked')
        if f.is_validated:
            f.is_visible= True
    if updateType == 'confidential':
        f.is_visible = not (request.REQUEST.get('updateValue')=='checked')
        if (f.is_visible == False):
            f.is_validated = False #When setting element to confidential then it becomes unvalidated automatically
    f.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())
<commit_msg>remove merge<commit_after>from datetime import datetime
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponseRedirect, Http404
from django.template import RequestContext
from django.db import transaction

from django_fixmystreet.fixmystreet.utils import render_to_pdf
from django_fixmystreet.fixmystreet.models import Report, FMSUser, OrganisationEntity, ReportComment, ReportFile
from django_fixmystreet.fixmystreet.forms import ReportCommentForm, ReportFileForm, RefuseForm

@transaction.commit_on_success
def accept( request, report_id ):
    report = get_object_or_404(Report, id=report_id)

    #Update the status and persist to the database
    report.status = Report.MANAGER_ASSIGNED
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def refuse( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    #Update the status
    report.status = Report.REFUSED
    form = RefuseForm(request)
    #Save the refusal motivation in the database
    report.refusal_motivation = form.data.POST.get('refusal_motivation')
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def fixed( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    #Update the status
    report.status = Report.SOLVED
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def close( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    #Update the status and set the close date
    report.status = Report.PROCESSED
    report.close_date = datetime.now()
    report.save()
    #Redirect to the report show page
    if "pro" in request.path:
        return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
        return HttpResponseRedirect(report.get_absolute_url())

def new( request, report_id ):
    report = get_object_or_404(Report, id=report_id)
    if request.method == 'POST':
        if request.POST['form-type'] == u"comment-form":
            comment_form = ReportCommentForm(request.POST)
            if comment_form.is_valid():
                comment_form.save(request.user, report)

        if request.POST['form-type'] == u"file-form":
            if request.POST['title'] == "":
                request.POST['title']= request.FILES.get('file').name
            file_form = ReportFileForm(request.POST,request.FILES)
            if file_form.is_valid:
                file_form.save(request.user, report)

        if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
        else:
            return HttpResponseRedirect(report.get_absolute_url())
    raise Http404()

def switchPrivacy(request,report_id):
    report = get_object_or_404(Report, id=report_id)
    privacy = request.REQUEST.get("privacy")
    report.private = ('true' == privacy)
    report.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def changeManager(request,report_id):
    report = Report.objects.get(pk=report_id)
    report.status = Report.MANAGER_ASSIGNED
    manId = request.REQUEST.get("manId")
    if manId.split("_")[0] == "manager":
        newRespMan = FMSUser.objects.get(pk=int(manId.split("_")[1]))
        report.responsible_manager = newRespMan
        report.save()
    if manId.split("_")[0] == "entity":
        orgId = int(manId.split("_")[1])
        report.responsible_entity = OrganisationEntity.objects.get(id=orgId)
        managers = FMSUser.objects.filter(organisation_id = orgId).filter(manager=True)
        for manager in managers:
            if manager.categories.all().filter(id = report.category.id):
                report.responsible_manager = manager
                report.save()
                break

    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())
def changeContractor(request,report_id):
    report = get_object_or_404(Report, id=report_id)
    contractorId = request.REQUEST.get("contractorId")
    if contractorId=='-1':
        report.subcontractor = None
        #Restore status to IN_PROGRESS
        report.status = Report.IN_PROGRESS
    else:
        organisation = OrganisationEntity.objects.get(pk=int(contractorId))
        report.contractor = organisation
        if organisation.is_subcontractor() == True:
            report.status = Report.CONTRACTOR_ASSIGNED
        else:
            report.status = Report.APPLICANT_RESPONSIBLE

    report.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def reportPdf(request, report_id, pro_version):
    '''reportPdf is called from report details page to generate the pdf with report story. When pro_version == 0 then filter pdf content'''
    report = get_object_or_404(Report, id=report_id)

    #Verify if the connected user is well pro ! (Server side protection)
    # if request.user.fmsuser.is_citizen():
       # pro_Version = 0

    if request.GET.get('output', False):
        return render_to_response("pro/pdf.html", {'report' : report, 'file_list' : report.get_files(), 'comment_list' : report.get_comments(), 'pro_version': pro_version},
                context_instance=RequestContext(request))
    else:
        return render_to_pdf("pro/pdf.html", {'report' : report, 'file_list' : report.get_files(), 'comment_list' : report.get_comments(), 'pro_version' : pro_version},
                context_instance=RequestContext(request))

def acceptAndValidate(request, report_id):
    report = get_object_or_404(Report, id=report_id)
    report.status = Report.MANAGER_ASSIGNED
    report.save()
    comments = ReportComment.objects.filter(report_id=report_id)
    for comment in comments:
        comment.is_validated= True
        comment.save()
    files = ReportFile.objects.filter(report_id=report_id)
    for f in files:
        f.is_validated = True
        f.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def validateAll(request,report_id):
    report = get_object_or_404(Report, id=report_id)
    comments = ReportComment.objects.filter(report_id=report_id)
    files = ReportFile.objects.filter(report_id=report_id)
    for comment in comments:
        comment.is_validated = True
        comment.is_visible = True
        comment.save()
    for f in files:
        f.is_validated = True
        f.is_visible = True
        f.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def updateComment(request,report_id):
    report = get_object_or_404(Report,id=report_id)
    updateType = request.REQUEST.get('updateType')
    comment = ReportComment.objects.get(pk=request.REQUEST.get('commentId'))
    if updateType == "valid":
        comment.is_validated = (request.REQUEST.get('updateValue')=='checked')
        if comment.is_validated:
            comment.is_visible= True
    if updateType == 'confidential':
        comment.is_visible = not (request.REQUEST.get('updateValue')=='checked')
        if comment.is_visible == False:
            comment.is_validated = False #When setting element to confidential then it becomes unvalidated automatically

    comment.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())

def updateFile(request,report_id):
    report = get_object_or_404(Report,id=report_id)
    updateType = request.REQUEST.get('updateType')
    f = ReportFile.objects.get(pk=request.REQUEST.get('fileId'))
    if updateType == "valid":
        f.is_validated= (request.REQUEST.get('updateValue')=='checked')
        if f.is_validated:
            f.is_visible= True
    if updateType == 'confidential':
        f.is_visible = not (request.REQUEST.get('updateValue')=='checked')
        if (f.is_visible == False):
            f.is_validated = False #When setting element to confidential then it becomes unvalidated automatically
    f.save()
    if "pro" in request.path:
            return HttpResponseRedirect(report.get_absolute_url_pro())
    else:
            return HttpResponseRedirect(report.get_absolute_url())
<|endoftext|>


<commit_before>import copy
import inspect
from django.db.models import DateTimeField
from djangoautoconf.import_export_utils import get_import_export_resource

__author__ = 'q19420'
from django.conf import settings
from django.contrib.admin import ModelAdmin
from django.contrib import admin

if "guardian" in settings.INSTALLED_APPS:
    g_is_guardian_included = True
    from guardian.admin import GuardedModelAdmin

def register_admin_without_duplicated_register(class_inst, admin_class, admin_site=admin.site):
    try:
        if (not (class_inst in admin_site._registry)) and (not class_inst._meta.abstract):
            admin_site.register(class_inst, admin_class)
    except Exception, e:
        if True:  # not (' is already registered' in e.message):
            print class_inst, admin_class
            import traceback
            traceback.print_exc()

class AdminRegister(object):
    def __init__(self, parent_admin_list=[]):
        super(AdminRegister, self).__init__()
        self.parent_admin_list = parent_admin_list
        self.base_model_admin = ModelAdmin
        self.admin_class_attributes = {}
        #self.is_import_export_supported = False

    def get_admin_class(self, class_instance):
        if self.admin_list is None:
            pass

    def get_valid_admin_class_with_list(self, class_inst):
        #print admin_list
        try:
            from import_export.admin import ImportExportActionModelAdmin
            self.base_model_admin = ImportExportActionModelAdmin
            resource_class = get_import_export_resource(class_inst)
            self.admin_class_attributes.update({
                "resource_class": resource_class
            })
        except ImportError:
            pass
        copied_admin_list = copy.copy(self.parent_admin_list)
        copied_admin_list.append(self.base_model_admin)
        #self.include_additional_admin_mixins(class_inst, copied_admin_list)
        #print ModelAdmin
        #print final_parents
        admin_class = type(class_inst.__name__ + "Admin", tuple(copied_admin_list), self.admin_class_attributes)
        return admin_class

    def register(self, class_inst, list_display=None, search_fields=None):
        self.admin_class_attributes = {}
        if not (list_display is None):
            self.admin_class_attributes["list_display"] = list_display
        if not (search_fields is None):
            self.admin_class_attributes["search_fields"] = search_fields
        admin_class = self.get_valid_admin_class_with_list(class_inst)
        register_admin_without_duplicated_register(class_inst, admin_class)

    def register_all_with_additional_attributes(self, class_inst, admin_class_attributes={}):
        attr_list = self.get_class_attributes(class_inst)
        self.admin_class_attributes = copy.copy(admin_class_attributes)
        self.admin_class_attributes.update({"list_display": attr_list})
        self.admin_class_attributes.update({"search_fields": attr_list})
        admin_class = self.get_valid_admin_class_with_list(class_inst)
        register_admin_without_duplicated_register(class_inst, admin_class)

    def get_class_attributes(self, class_inst):
        res = []
        for field in class_inst.__dict__['_meta'].fields:
            if type(field) == DateTimeField:
                continue
            res.append(field.name)
        return res

    def register_with_all_in_list_display(self, class_inst):
        self.admin_class_attributes["list_display"] = []
        self.admin_class_attributes["search_fields"] = []
        for field in class_inst.__dict__['_meta'].fields:
            if type(field) == DateTimeField:
                continue
            self.admin_class_attributes["list_display"].append(field.name)
            self.admin_class_attributes["search_fields"].append(field.name)
        admin_class = self.get_valid_admin_class_with_list(class_inst)
        register_admin_without_duplicated_register(class_inst, admin_class)

    def class_enumerator(self, module_instance, exclude_name_list):
        for name, obj in inspect.getmembers(module_instance):
            if inspect.isclass(obj):
                if name in exclude_name_list:
                    continue
                yield obj

    def register_all_model(self, module_instance, exclude_name_list=[]):
        """
        :param module_instance: mostly the models module
        :param exclude_name_list: class does not need to register or is already registered
        :param admin_class_list:
        :return: N/A
        """
        for class_instance in self.class_enumerator(module_instance, exclude_name_list):
            self.register_with_all_in_list_display(class_instance)

    def add_list_filter(self, filter_field):
        if not ("list_filter" in self.admin_class_attributes):
            self.admin_class_attributes["list_display"] = []
        self.admin_class_attributes["list_display"].append(filter_field)

    # def include_additional_admin_mixins(self, class_instance, existing_list):
    #     try:
    #         from djangoautoconf.import_export_utils import get_import_export_admin_mixin
    #         existing_list.append(get_import_export_admin_mixin(class_instance))
    #         #from import_export.admin import ImportExportActionModelAdmin
    #         #existing_list.append(ImportExportActionModelAdmin)
    #     except ImportError:
    #         pass

<commit_msg>Only use import export when it is installed.<commit_after>import copy
import inspect
from django.db.models import DateTimeField
from djangoautoconf.import_export_utils import get_import_export_resource

__author__ = 'q19420'
from django.conf import settings
from django.contrib.admin import ModelAdmin
from django.contrib import admin

if "guardian" in settings.INSTALLED_APPS:
    g_is_guardian_included = True
    from guardian.admin import GuardedModelAdmin

def register_admin_without_duplicated_register(class_inst, admin_class, admin_site=admin.site):
    try:
        if (not (class_inst in admin_site._registry)) and (not class_inst._meta.abstract):
            admin_site.register(class_inst, admin_class)
    except Exception, e:
        if True:  # not (' is already registered' in e.message):
            print class_inst, admin_class
            import traceback
            traceback.print_exc()

class AdminRegister(object):
    def __init__(self, parent_admin_list=[]):
        super(AdminRegister, self).__init__()
        self.parent_admin_list = parent_admin_list
        self.base_model_admin = ModelAdmin
        self.admin_class_attributes = {}
        #self.is_import_export_supported = False

    def get_admin_class(self, class_instance):
        if self.admin_list is None:
            pass

    def get_valid_admin_class_with_list(self, class_inst):
        #print admin_list
        try:
            if "import_export" in settings.INSTALLED_APPS:
                from import_export.admin import ImportExportActionModelAdmin
                self.base_model_admin = ImportExportActionModelAdmin
                resource_class = get_import_export_resource(class_inst)
                self.admin_class_attributes.update({
                    "resource_class": resource_class
                })
        except ImportError:
            pass
        copied_admin_list = copy.copy(self.parent_admin_list)
        copied_admin_list.append(self.base_model_admin)
        #self.include_additional_admin_mixins(class_inst, copied_admin_list)
        #print ModelAdmin
        #print final_parents
        admin_class = type(class_inst.__name__ + "Admin", tuple(copied_admin_list), self.admin_class_attributes)
        return admin_class

    def register(self, class_inst, list_display=None, search_fields=None):
        self.admin_class_attributes = {}
        if not (list_display is None):
            self.admin_class_attributes["list_display"] = list_display
        if not (search_fields is None):
            self.admin_class_attributes["search_fields"] = search_fields
        admin_class = self.get_valid_admin_class_with_list(class_inst)
        register_admin_without_duplicated_register(class_inst, admin_class)

    def register_all_with_additional_attributes(self, class_inst, admin_class_attributes={}):
        attr_list = self.get_class_attributes(class_inst)
        self.admin_class_attributes = copy.copy(admin_class_attributes)
        self.admin_class_attributes.update({"list_display": attr_list})
        self.admin_class_attributes.update({"search_fields": attr_list})
        admin_class = self.get_valid_admin_class_with_list(class_inst)
        register_admin_without_duplicated_register(class_inst, admin_class)

    def get_class_attributes(self, class_inst):
        res = []
        for field in class_inst.__dict__['_meta'].fields:
            if type(field) == DateTimeField:
                continue
            res.append(field.name)
        return res

    def register_with_all_in_list_display(self, class_inst):
        self.admin_class_attributes["list_display"] = []
        self.admin_class_attributes["search_fields"] = []
        for field in class_inst.__dict__['_meta'].fields:
            if type(field) == DateTimeField:
                continue
            self.admin_class_attributes["list_display"].append(field.name)
            self.admin_class_attributes["search_fields"].append(field.name)
        admin_class = self.get_valid_admin_class_with_list(class_inst)
        register_admin_without_duplicated_register(class_inst, admin_class)

    def class_enumerator(self, module_instance, exclude_name_list):
        for name, obj in inspect.getmembers(module_instance):
            if inspect.isclass(obj):
                if name in exclude_name_list:
                    continue
                yield obj

    def register_all_model(self, module_instance, exclude_name_list=[]):
        """
        :param module_instance: mostly the models module
        :param exclude_name_list: class does not need to register or is already registered
        :param admin_class_list:
        :return: N/A
        """
        for class_instance in self.class_enumerator(module_instance, exclude_name_list):
            self.register_with_all_in_list_display(class_instance)

    def add_list_filter(self, filter_field):
        if not ("list_filter" in self.admin_class_attributes):
            self.admin_class_attributes["list_display"] = []
        self.admin_class_attributes["list_display"].append(filter_field)

    # def include_additional_admin_mixins(self, class_instance, existing_list):
    #     try:
    #         from djangoautoconf.import_export_utils import get_import_export_admin_mixin
    #         existing_list.append(get_import_export_admin_mixin(class_instance))
    #         #from import_export.admin import ImportExportActionModelAdmin
    #         #existing_list.append(ImportExportActionModelAdmin)
    #     except ImportError:
    #         pass

<|endoftext|>


<commit_before>"""
Utility functions for  data harvesting

Commandline usage can fire API calls with an ID e.g.

dc.py organization_purge hscic
"""
import ConfigParser
import json
import sys

import ffs
from ffs.contrib import http
import ckanapi

inifile = ffs.Path.here()/'../config.ini'
CONF = ConfigParser.ConfigParser()
CONF.read(inifile)

ckan = ckanapi.RemoteCKAN(CONF.get('ckan', 'url'),  apikey=CONF.get('ckan', 'api_key'))

class Error(Exception):
    def __init__(self, msg):
        Exception.__init__(self, '\n\n\n{0}\n\n\n'.format(msg))

class NHSEnglandNotFoundException(Error): pass

def tags(*tags):
    """
    Given a list of tags as positional arguments TAGS, return
    a list of dictionaries in the format that the CKAN API
    wants!
    """
    return [{'name': t} for t in tags]

def fh_for_url(url):
    """
    Return a file-like-object for URL!
    """
    return http.HTTPPath(url).open()

def _org_existsp(name):
    orglist = ckan.action.organization_list()
    return name in orglist

def ensure_publisher(name):
    """
    Ensure that the publisher NAME exists.
    if not, attempt to create it from our settings file or COMPLAIN LOUDLY!
    """
    if _org_existsp(name):
        return # YAY
    if not CONF.has_section('publisher:'+name):
        what = 'Publisher "{0}" does not exist in the catalogue or inifile'.format(name)
        raise NHSEnglandNotFoundException(what)

    ckan.action.organization_create(
        name=CONF.get('publisher:'+name, 'name'),
        title=CONF.get('publisher:'+name, 'title'),
        description=CONF.get('publisher:'+name, 'description'),
        image_url= CONF.get('publisher:'+name, 'image_url')
    )
    return

class Dataset(object):
    """
    Not really a class.

    Namespaces are one honking...
    """
    @staticmethod
    def create_or_update(**deets):
        resources = deets.pop('resources')
        try:
            pkg =  ckan.action.package_show(id=deets['name'])
            pkg.update(deets)
            ckan.action.package_update(**pkg)
        except ckanapi.errors.NotFound:
            pkg = ckan.action.package_create(**deets)

        for resource in resources:
            resource['package_id'] = pkg['id']
            ckan.action.resource_create(**resource)
        return

if __name__ == '__main__':
    getattr(ckan.action, sys.argv[-2])(id=sys.argv[-1])
<commit_msg>Don't allow 's in tags, check that resources exist and update/create as required<commit_after>"""
Utility functions for  data harvesting

Commandline usage can fire API calls with an ID e.g.

dc.py organization_purge hscic
"""
import ConfigParser
import json
import sys

import ffs
from ffs.contrib import http
import ckanapi

inifile = ffs.Path.here()/'../config.ini'
CONF = ConfigParser.ConfigParser()
CONF.read(inifile)

ckan = ckanapi.RemoteCKAN(CONF.get('ckan', 'url'),  apikey=CONF.get('ckan', 'api_key'))

class Error(Exception):
    def __init__(self, msg):
        Exception.__init__(self, '\n\n\n{0}\n\n\n'.format(msg))

class NHSEnglandNotFoundException(Error): pass

def tags(*tags):
    """
    Given a list of tags as positional arguments TAGS, return
    a list of dictionaries in the format that the CKAN API
    wants!
    """
    return [{'name': t.replace("'", "") } for t in tags]

def fh_for_url(url):
    """
    Return a file-like-object for URL!
    """
    return http.HTTPPath(url).open()

def _org_existsp(name):
    orglist = ckan.action.organization_list()
    return name in orglist

def ensure_publisher(name):
    """
    Ensure that the publisher NAME exists.
    if not, attempt to create it from our settings file or COMPLAIN LOUDLY!
    """
    if _org_existsp(name):
        return # YAY
    if not CONF.has_section('publisher:'+name):
        what = 'Publisher "{0}" does not exist in the catalogue or inifile'.format(name)
        raise NHSEnglandNotFoundException(what)

    ckan.action.organization_create(
        name=CONF.get('publisher:'+name, 'name'),
        title=CONF.get('publisher:'+name, 'title'),
        description=CONF.get('publisher:'+name, 'description'),
        image_url= CONF.get('publisher:'+name, 'image_url')
    )
    return

class Dataset(object):
    """
    Not really a class.

    Namespaces are one honking...
    """
    @staticmethod
    def create_or_update(**deets):
        resources = deets.pop('resources')
        try:
            pkg =  ckan.action.package_show(id=deets['name'])
            pkg.update(deets)
            ckan.action.package_update(**pkg)
        except ckanapi.errors.NotFound:
            pkg = ckan.action.package_create(**deets)

        print json.dumps(pkg, indent=2)
        for resource in resources:
            resource['package_id'] = pkg['id']
            name = resource['name']
            existing = [r for r in pkg['resources'] if r['name'] == name]
            if not existing:
                ckan.action.resource_create(**resource)
            else:
                existing = existing[0]
                existing.update(resource)
                ckan.action.resource_update(**existing)
        return

if __name__ == '__main__':
    getattr(ckan.action, sys.argv[-2])(id=sys.argv[-1])
<|endoftext|>


<commit_before>import collections
import hashlib
import logging
import time

from django.conf import settings
from django.core.cache import DEFAULT_CACHE_ALIAS
from django.utils import six

from cacheback.utils import enqueue_task, get_cache, get_job_class

logger = logging.getLogger('cacheback')

MEMCACHE_MAX_EXPIRATION = 2592000

# Container for call args (which makes things simpler to pass around)
Call = collections.namedtuple("Call", ['args', 'kwargs'])

def to_bytestring(value):
    """
    Encode a string as a UTF8 bytestring.  This function could be passed a
    bytestring or unicode string so must distinguish between the two.
    """
    if isinstance(value, six.text_type):
        return value.encode('utf8')
    if isinstance(value, six.binary_type):
        return value
    if six.PY2:
        return str(value)
    return bytes(str(value), 'utf8')

class Job(object):
    """
    A cached read job.

    This is the core class for the package which is intended to be subclassed
    to allow the caching behaviour to be customised.
    """
    # All items are stored in memcache as a tuple (expiry, data).  We don't use
    # the TTL functionality within memcache but implement on own.  If the
    # expiry value is None, this indicates that there is already a job created
    # for refreshing this item.

    #: Default cache lifetime is 10 minutes.  After this time, the result will
    #: be considered stale and requests will trigger a job to refresh it.
    lifetime = 600

    #: Timeout period during which no new tasks will be created for a
    #: single cache item.  This time should cover the normal time required to
    #: refresh the cache.
    refresh_timeout = 60

    #: Time to store items in the cache.  After this time, we will get a cache
    #: miss which can lead to synchronous refreshes if you have
    #: fetch_on_miss=True.
    cache_ttl = MEMCACHE_MAX_EXPIRATION

    #: Whether to perform a synchronous refresh when a result is missing from
    #: the cache.  Default behaviour is to do a synchronous fetch when the cache is empty.
    #: Stale results are generally ok, but not no results.
    fetch_on_miss = True

    #: Whether to perform a synchronous refresh when a result is in the cache
    #: but stale from. Default behaviour is never to do a synchronous fetch but
    #: there will be times when an item is _too_ stale to be returned.
    fetch_on_stale_threshold = None

    #: Overrides options for `refresh_cache.apply_async` (e.g. `queue`).
    task_options = None

    #: Cache statuses
    MISS, HIT, STALE = range(3)

    def __init__(self):
        self.cache_alias = getattr(settings, 'CACHEBACK_CACHE_ALIAS', DEFAULT_CACHE_ALIAS)
        self.cache = get_cache(self.cache_alias)
        self.task_options = self.task_options or {}

    # --------
    # MAIN API
    # --------

    def get(self, *raw_args, **raw_kwargs):
        """
        Return the data for this function (using the cache if possible).

        This method is not intended to be overidden
        """
        # We pass args and kwargs through a filter to allow them to be
        # converted into values that can be pickled.
        args = self.prepare_args(*raw_args)
        kwargs = self.prepare_kwargs(**raw_kwargs)

        # Build the cache key and attempt to fetch the cached item
        key = self.key(*args, **kwargs)
        item = self.cache.get(key)

        call = Call(args=raw_args, kwargs=raw_kwargs)

        if item is None:
            # Cache MISS - we can either:
            # a) fetch the data immediately, blocking execution until
            #    the fetch has finished, or
            # b) trigger an async refresh and return an empty result
            if self.should_missing_item_be_fetched_synchronously(*args, **kwargs):
                logger.debug(("Job %s with key '%s' - cache MISS - running "
                              "synchronous refresh"),
                             self.class_path, key)
                result = self.refresh(*args, **kwargs)
                return self.process_result(
                    result, call=call, cache_status=self.MISS, sync_fetch=True)

            else:
                logger.debug(("Job %s with key '%s' - cache MISS - triggering "
                              "async refresh and returning empty result"),
                             self.class_path, key)
                # To avoid cache hammering (ie lots of identical tasks
                # to refresh the same cache item), we reset the cache with an
                # empty result which will be returned until the cache is
                # refreshed.
                result = self.empty()
                self.cache_set(key, self.timeout(*args, **kwargs), result)
                self.async_refresh(*args, **kwargs)
                return self.process_result(
                    result, call=call, cache_status=self.MISS,
                    sync_fetch=False)

        expiry, data = item
        delta = time.time() - expiry
        if delta > 0:
            # Cache HIT but STALE expiry - we can either:
            # a) fetch the data immediately, blocking execution until
            #    the fetch has finished, or
            # b) trigger a refresh but allow the stale result to be
            #    returned this time.  This is normally acceptable.
            if self.should_stale_item_be_fetched_synchronously(
                    delta, *args, **kwargs):
                logger.debug(
                    ("Job %s with key '%s' - STALE cache hit - running "
                     "synchronous refresh"),
                    self.class_path, key)
                result = self.refresh(*args, **kwargs)
                return self.process_result(
                    result, call=call, cache_status=self.STALE,
                    sync_fetch=True)

            else:
                logger.debug(
                    ("Job %s with key '%s' - STALE cache hit - triggering "
                     "async refresh and returning stale result"),
                    self.class_path, key)
                # We replace the item in the cache with a 'timeout' expiry - this
                # prevents cache hammering but guards against a 'limbo' situation
                # where the refresh task fails for some reason.
                timeout = self.timeout(*args, **kwargs)
                self.cache_set(key, timeout, data)
                self.async_refresh(*args, **kwargs)
                return self.process_result(
                    data, call=call, cache_status=self.STALE, sync_fetch=False)
        else:
            logger.debug("Job %s with key '%s' - cache HIT", self.class_path, key)
            return self.process_result(data, call=call, cache_status=self.HIT)

    def invalidate(self, *raw_args, **raw_kwargs):
        """
        Mark a cached item invalid and trigger an asynchronous
        job to refresh the cache
        """
        args = self.prepare_args(*raw_args)
        kwargs = self.prepare_kwargs(**raw_kwargs)
        key = self.key(*args, **kwargs)
        item = self.cache.get(key)
        if item is not None:
            expiry, data = item
            self.cache_set(key, self.timeout(*args, **kwargs), data)
            self.async_refresh(*args, **kwargs)

    def delete(self, *raw_args, **raw_kwargs):
        """
        Remove an item from the cache
        """
        args = self.prepare_args(*raw_args)
        kwargs = self.prepare_kwargs(**raw_kwargs)
        key = self.key(*args, **kwargs)
        item = self.cache.get(key)
        if item is not None:
            self.cache.delete(key)

    # --------------
    # HELPER METHODS
    # --------------

    def prepare_args(self, *args):
        return args

    def prepare_kwargs(self, **kwargs):
        return kwargs

    def cache_set(self, key, expiry, data):
        """
        Add a result to the cache

        :key: Cache key to use
        :expiry: The expiry timestamp after which the result is stale
        :data: The data to cache
        """
        self.cache.set(key, (expiry, data), self.cache_ttl)

        if getattr(settings, 'CACHEBACK_VERIFY_CACHE_WRITE', True):
            # We verify that the item was cached correctly.  This is to avoid a
            # Memcache problem where some values aren't cached correctly
            # without warning.
            __, cached_data = self.cache.get(key, (None, None))
            if data is not None and cached_data is None:
                raise RuntimeError(
                    "Unable to save data of type %s to cache" % (
                        type(data)))

    def refresh(self, *args, **kwargs):
        """
        Fetch the result SYNCHRONOUSLY and populate the cache
        """
        result = self.fetch(*args, **kwargs)
        self.cache_set(self.key(*args, **kwargs),
                       self.expiry(*args, **kwargs),
                       result)
        return result

    def async_refresh(self, *args, **kwargs):
        """
        Trigger an asynchronous job to refresh the cache
        """
        # We trigger the task with the class path to import as well as the
        # (a) args and kwargs for instantiating the class
        # (b) args and kwargs for calling the 'refresh' method

        try:
            enqueue_task(
                kwargs=dict(
                    klass_str=self.class_path,
                    obj_args=self.get_constructor_args(),
                    obj_kwargs=self.get_constructor_kwargs(),
                    call_args=args,
                    call_kwargs=kwargs
                ),
                **self.task_options
            )
        except Exception as e:
            # Handle exceptions from talking to RabbitMQ - eg connection
            # refused.  When this happens, we try to run the task
            # synchronously.
            logger.error("Unable to trigger task asynchronously - failing "
                         "over to synchronous refresh", exc_info=True)
            try:
                return self.refresh(*args, **kwargs)
            except Exception as e:
                # Something went wrong while running the task
                logger.error("Unable to refresh data synchronously: %s", e,
                             exc_info=True)
            else:
                logger.debug("Failover synchronous refresh completed successfully")

    def get_constructor_args(self):
        return ()

    def get_constructor_kwargs(self):
        """
        Return the kwargs that need to be passed to __init__ when
        reconstructing this class.
        """
        return {}

    @property
    def class_path(self):
        return '%s.%s' % (self.__module__, self.__class__.__name__)

    # Override these methods

    def empty(self):
        """
        Return the appropriate value for a cache MISS (and when we defer the
        repopulation of the cache)
        """
        return None

    def expiry(self, *args, **kwargs):
        """
        Return the expiry timestamp for this item.
        """
        return time.time() + self.lifetime

    def timeout(self, *args, **kwargs):
        """
        Return the refresh timeout for this item
        """
        return time.time() + self.refresh_timeout

    def should_missing_item_be_fetched_synchronously(self, *args, **kwargs):
        """
        Return whether to refresh an item synchronously when it is missing from
        the cache
        """
        return self.fetch_on_miss

    def should_item_be_fetched_synchronously(self, *args, **kwargs):
        import warnings
        warnings.warn(
            "The method 'should_item_be_fetched_synchronously' is deprecated "
            "and will be removed in 0.5.  Use "
            "'should_missing_item_be_fetched_synchronously' instead.",
            DeprecationWarning)
        return self.should_missing_item_be_fetched_synchronously(
            *args, **kwargs)

    def should_stale_item_be_fetched_synchronously(self, delta, *args, **kwargs):
        """
        Return whether to refresh an item synchronously when it is found in the
        cache but stale
        """
        if self.fetch_on_stale_threshold is None:
            return False
        return delta > (self.fetch_on_stale_threshold - self.lifetime)

    def key(self, *args, **kwargs):
        """
        Return the cache key to use.

        If you're passing anything but primitive types to the ``get`` method,
        it's likely that you'll need to override this method.
        """
        if not args and not kwargs:
            return self.class_path
        try:
            if args and not kwargs:
                return "%s:%s" % (self.class_path, self.hash(args))
            # The line might break if your passed values are un-hashable.  If
            # it does, you need to override this method and implement your own
            # key algorithm.
            return "%s:%s:%s:%s" % (self.class_path,
                                    self.hash(args),
                                    self.hash(tuple([k for k in sorted(kwargs)])),
                                    self.hash(tuple([kwargs[k] for k in sorted(kwargs)])))
        except TypeError:
            raise RuntimeError(
                "Unable to generate cache key due to unhashable"
                "args or kwargs - you need to implement your own"
                "key generation method to avoid this problem")

    def hash(self, value):
        """
        Generate a hash of the given tuple.

        This is for use in a cache key.
        """
        if isinstance(value, tuple):
            value = tuple(to_bytestring(v) for v in value)
        return hashlib.md5(six.b(':').join(value)).hexdigest()

    def fetch(self, *args, **kwargs):
        """
        Return the data for this job - this is where the expensive work should
        be done.
        """
        raise NotImplementedError()

    def process_result(self, result, call, cache_status, sync_fetch=None):
        """
        Transform the fetched data right before returning from .get(...)

        :param result: The result to be returned
        :param call: A named tuple with properties 'args' and 'kwargs that
                     holds the call args and kwargs
        :param cache_status: A status integrer, accessible as class constants
                             self.MISS, self.HIT, self.STALE
        :param sync_fetch: A boolean indicating whether a synchronous fetch was
                           performed. A value of None indicates that no fetch
                           was required (ie the result was a cache hit).
        """
        return result

    # --------------------
    # ASYNC HELPER METHODS
    # --------------------

    @classmethod
    def job_refresh(cls, klass_str, obj_args, obj_kwargs, call_args, call_kwargs):
        """
        Re-populate cache using the given job class.

        The job class is instantiated with the passed constructor args and the
        refresh method is called with the passed call args.  That is::

            data = klass(*obj_args, **obj_kwargs).refresh(
                *call_args, **call_kwargs)

        :klass_str: String repr of class (eg 'apps.twitter.jobs.FetchTweetsJob')
        :obj_args: Constructor args
        :obj_kwargs: Constructor kwargs
        :call_args: Refresh args
        :call_kwargs: Refresh kwargs
        """
        klass = get_job_class(klass_str)
        if klass is None:
            logger.error("Unable to construct %s with args %r and kwargs %r",
                         klass_str, obj_args, obj_kwargs)
            return

        logger.info("Using %s with constructor args %r and kwargs %r",
                    klass_str, obj_args, obj_kwargs)
        logger.info("Calling refresh with args %r and kwargs %r", call_args,
                    call_kwargs)
        start = time.time()
        try:
            klass(*obj_args, **obj_kwargs).refresh(
                *call_args, **call_kwargs)
        except Exception as e:
            logger.exception("Error running job: '%s'", e)
        else:
            duration = time.time() - start
            logger.info("Refreshed cache in %.6f seconds", duration)
<commit_msg>Refs #57 - added handling of Django Model instances out of the box<commit_after>import collections
import hashlib
import logging
import time

from django.conf import settings
from django.core.cache import DEFAULT_CACHE_ALIAS
from django.db.models import Model as DjangoModel
from django.utils import six

from cacheback.utils import enqueue_task, get_cache, get_job_class

logger = logging.getLogger('cacheback')

MEMCACHE_MAX_EXPIRATION = 2592000

# Container for call args (which makes things simpler to pass around)
Call = collections.namedtuple("Call", ['args', 'kwargs'])

def to_bytestring(value):
    """
    Encode an object as a UTF8 bytestring.  This function could be passed a
    bytestring, unicode string or object so must distinguish between them.

    :param value: object we want to transform into a bytestring
    :returns: a bytestring
    """
    if isinstance(value, DjangoModel):
        return ('%s:%s' % (value.__class__, hash(value))).encode('utf-8')
    if isinstance(value, six.text_type):
        return value.encode('utf8')
    if isinstance(value, six.binary_type):
        return value
    if six.PY2:
        return str(value)
    return bytes(str(value), 'utf8')

class Job(object):
    """
    A cached read job.

    This is the core class for the package which is intended to be subclassed
    to allow the caching behaviour to be customised.
    """
    # All items are stored in memcache as a tuple (expiry, data).  We don't use
    # the TTL functionality within memcache but implement on own.  If the
    # expiry value is None, this indicates that there is already a job created
    # for refreshing this item.

    #: Default cache lifetime is 10 minutes.  After this time, the result will
    #: be considered stale and requests will trigger a job to refresh it.
    lifetime = 600

    #: Timeout period during which no new tasks will be created for a
    #: single cache item.  This time should cover the normal time required to
    #: refresh the cache.
    refresh_timeout = 60

    #: Time to store items in the cache.  After this time, we will get a cache
    #: miss which can lead to synchronous refreshes if you have
    #: fetch_on_miss=True.
    cache_ttl = MEMCACHE_MAX_EXPIRATION

    #: Whether to perform a synchronous refresh when a result is missing from
    #: the cache.  Default behaviour is to do a synchronous fetch when the cache is empty.
    #: Stale results are generally ok, but not no results.
    fetch_on_miss = True

    #: Whether to perform a synchronous refresh when a result is in the cache
    #: but stale from. Default behaviour is never to do a synchronous fetch but
    #: there will be times when an item is _too_ stale to be returned.
    fetch_on_stale_threshold = None

    #: Overrides options for `refresh_cache.apply_async` (e.g. `queue`).
    task_options = None

    #: Cache statuses
    MISS, HIT, STALE = range(3)

    def __init__(self):
        self.cache_alias = getattr(settings, 'CACHEBACK_CACHE_ALIAS', DEFAULT_CACHE_ALIAS)
        self.cache = get_cache(self.cache_alias)
        self.task_options = self.task_options or {}

    # --------
    # MAIN API
    # --------

    def get(self, *raw_args, **raw_kwargs):
        """
        Return the data for this function (using the cache if possible).

        This method is not intended to be overidden
        """
        # We pass args and kwargs through a filter to allow them to be
        # converted into values that can be pickled.
        args = self.prepare_args(*raw_args)
        kwargs = self.prepare_kwargs(**raw_kwargs)

        # Build the cache key and attempt to fetch the cached item
        key = self.key(*args, **kwargs)
        item = self.cache.get(key)

        call = Call(args=raw_args, kwargs=raw_kwargs)

        if item is None:
            # Cache MISS - we can either:
            # a) fetch the data immediately, blocking execution until
            #    the fetch has finished, or
            # b) trigger an async refresh and return an empty result
            if self.should_missing_item_be_fetched_synchronously(*args, **kwargs):
                logger.debug(("Job %s with key '%s' - cache MISS - running "
                              "synchronous refresh"),
                             self.class_path, key)
                result = self.refresh(*args, **kwargs)
                return self.process_result(
                    result, call=call, cache_status=self.MISS, sync_fetch=True)

            else:
                logger.debug(("Job %s with key '%s' - cache MISS - triggering "
                              "async refresh and returning empty result"),
                             self.class_path, key)
                # To avoid cache hammering (ie lots of identical tasks
                # to refresh the same cache item), we reset the cache with an
                # empty result which will be returned until the cache is
                # refreshed.
                result = self.empty()
                self.cache_set(key, self.timeout(*args, **kwargs), result)
                self.async_refresh(*args, **kwargs)
                return self.process_result(
                    result, call=call, cache_status=self.MISS,
                    sync_fetch=False)

        expiry, data = item
        delta = time.time() - expiry
        if delta > 0:
            # Cache HIT but STALE expiry - we can either:
            # a) fetch the data immediately, blocking execution until
            #    the fetch has finished, or
            # b) trigger a refresh but allow the stale result to be
            #    returned this time.  This is normally acceptable.
            if self.should_stale_item_be_fetched_synchronously(
                    delta, *args, **kwargs):
                logger.debug(
                    ("Job %s with key '%s' - STALE cache hit - running "
                     "synchronous refresh"),
                    self.class_path, key)
                result = self.refresh(*args, **kwargs)
                return self.process_result(
                    result, call=call, cache_status=self.STALE,
                    sync_fetch=True)

            else:
                logger.debug(
                    ("Job %s with key '%s' - STALE cache hit - triggering "
                     "async refresh and returning stale result"),
                    self.class_path, key)
                # We replace the item in the cache with a 'timeout' expiry - this
                # prevents cache hammering but guards against a 'limbo' situation
                # where the refresh task fails for some reason.
                timeout = self.timeout(*args, **kwargs)
                self.cache_set(key, timeout, data)
                self.async_refresh(*args, **kwargs)
                return self.process_result(
                    data, call=call, cache_status=self.STALE, sync_fetch=False)
        else:
            logger.debug("Job %s with key '%s' - cache HIT", self.class_path, key)
            return self.process_result(data, call=call, cache_status=self.HIT)

    def invalidate(self, *raw_args, **raw_kwargs):
        """
        Mark a cached item invalid and trigger an asynchronous
        job to refresh the cache
        """
        args = self.prepare_args(*raw_args)
        kwargs = self.prepare_kwargs(**raw_kwargs)
        key = self.key(*args, **kwargs)
        item = self.cache.get(key)
        if item is not None:
            expiry, data = item
            self.cache_set(key, self.timeout(*args, **kwargs), data)
            self.async_refresh(*args, **kwargs)

    def delete(self, *raw_args, **raw_kwargs):
        """
        Remove an item from the cache
        """
        args = self.prepare_args(*raw_args)
        kwargs = self.prepare_kwargs(**raw_kwargs)
        key = self.key(*args, **kwargs)
        item = self.cache.get(key)
        if item is not None:
            self.cache.delete(key)

    # --------------
    # HELPER METHODS
    # --------------

    def prepare_args(self, *args):
        return args

    def prepare_kwargs(self, **kwargs):
        return kwargs

    def cache_set(self, key, expiry, data):
        """
        Add a result to the cache

        :key: Cache key to use
        :expiry: The expiry timestamp after which the result is stale
        :data: The data to cache
        """
        self.cache.set(key, (expiry, data), self.cache_ttl)

        if getattr(settings, 'CACHEBACK_VERIFY_CACHE_WRITE', True):
            # We verify that the item was cached correctly.  This is to avoid a
            # Memcache problem where some values aren't cached correctly
            # without warning.
            __, cached_data = self.cache.get(key, (None, None))
            if data is not None and cached_data is None:
                raise RuntimeError(
                    "Unable to save data of type %s to cache" % (
                        type(data)))

    def refresh(self, *args, **kwargs):
        """
        Fetch the result SYNCHRONOUSLY and populate the cache
        """
        result = self.fetch(*args, **kwargs)
        self.cache_set(self.key(*args, **kwargs),
                       self.expiry(*args, **kwargs),
                       result)
        return result

    def async_refresh(self, *args, **kwargs):
        """
        Trigger an asynchronous job to refresh the cache
        """
        # We trigger the task with the class path to import as well as the
        # (a) args and kwargs for instantiating the class
        # (b) args and kwargs for calling the 'refresh' method

        try:
            enqueue_task(
                kwargs=dict(
                    klass_str=self.class_path,
                    obj_args=self.get_constructor_args(),
                    obj_kwargs=self.get_constructor_kwargs(),
                    call_args=args,
                    call_kwargs=kwargs
                ),
                **self.task_options
            )
        except Exception as e:
            # Handle exceptions from talking to RabbitMQ - eg connection
            # refused.  When this happens, we try to run the task
            # synchronously.
            logger.error("Unable to trigger task asynchronously - failing "
                         "over to synchronous refresh", exc_info=True)
            try:
                return self.refresh(*args, **kwargs)
            except Exception as e:
                # Something went wrong while running the task
                logger.error("Unable to refresh data synchronously: %s", e,
                             exc_info=True)
            else:
                logger.debug("Failover synchronous refresh completed successfully")

    def get_constructor_args(self):
        return ()

    def get_constructor_kwargs(self):
        """
        Return the kwargs that need to be passed to __init__ when
        reconstructing this class.
        """
        return {}

    @property
    def class_path(self):
        return '%s.%s' % (self.__module__, self.__class__.__name__)

    # Override these methods

    def empty(self):
        """
        Return the appropriate value for a cache MISS (and when we defer the
        repopulation of the cache)
        """
        return None

    def expiry(self, *args, **kwargs):
        """
        Return the expiry timestamp for this item.
        """
        return time.time() + self.lifetime

    def timeout(self, *args, **kwargs):
        """
        Return the refresh timeout for this item
        """
        return time.time() + self.refresh_timeout

    def should_missing_item_be_fetched_synchronously(self, *args, **kwargs):
        """
        Return whether to refresh an item synchronously when it is missing from
        the cache
        """
        return self.fetch_on_miss

    def should_item_be_fetched_synchronously(self, *args, **kwargs):
        import warnings
        warnings.warn(
            "The method 'should_item_be_fetched_synchronously' is deprecated "
            "and will be removed in 0.5.  Use "
            "'should_missing_item_be_fetched_synchronously' instead.",
            DeprecationWarning)
        return self.should_missing_item_be_fetched_synchronously(
            *args, **kwargs)

    def should_stale_item_be_fetched_synchronously(self, delta, *args, **kwargs):
        """
        Return whether to refresh an item synchronously when it is found in the
        cache but stale
        """
        if self.fetch_on_stale_threshold is None:
            return False
        return delta > (self.fetch_on_stale_threshold - self.lifetime)

    def key(self, *args, **kwargs):
        """
        Return the cache key to use.

        If you're passing anything but primitive types to the ``get`` method,
        it's likely that you'll need to override this method.
        """
        if not args and not kwargs:
            return self.class_path
        try:
            if args and not kwargs:
                return "%s:%s" % (self.class_path, self.hash(args))
            # The line might break if your passed values are un-hashable.  If
            # it does, you need to override this method and implement your own
            # key algorithm.
            return "%s:%s:%s:%s" % (self.class_path,
                                    self.hash(args),
                                    self.hash(tuple([k for k in sorted(kwargs)])),
                                    self.hash(tuple([kwargs[k] for k in sorted(kwargs)])))
        except TypeError:
            raise RuntimeError(
                "Unable to generate cache key due to unhashable"
                "args or kwargs - you need to implement your own"
                "key generation method to avoid this problem")

    def hash(self, value):
        """
        Generate a hash of the given tuple.

        This is for use in a cache key.
        """
        if isinstance(value, tuple):
            value = tuple(to_bytestring(v) for v in value)
        return hashlib.md5(six.b(':').join(value)).hexdigest()

    def fetch(self, *args, **kwargs):
        """
        Return the data for this job - this is where the expensive work should
        be done.
        """
        raise NotImplementedError()

    def process_result(self, result, call, cache_status, sync_fetch=None):
        """
        Transform the fetched data right before returning from .get(...)

        :param result: The result to be returned
        :param call: A named tuple with properties 'args' and 'kwargs that
                     holds the call args and kwargs
        :param cache_status: A status integrer, accessible as class constants
                             self.MISS, self.HIT, self.STALE
        :param sync_fetch: A boolean indicating whether a synchronous fetch was
                           performed. A value of None indicates that no fetch
                           was required (ie the result was a cache hit).
        """
        return result

    # --------------------
    # ASYNC HELPER METHODS
    # --------------------

    @classmethod
    def job_refresh(cls, klass_str, obj_args, obj_kwargs, call_args, call_kwargs):
        """
        Re-populate cache using the given job class.

        The job class is instantiated with the passed constructor args and the
        refresh method is called with the passed call args.  That is::

            data = klass(*obj_args, **obj_kwargs).refresh(
                *call_args, **call_kwargs)

        :klass_str: String repr of class (eg 'apps.twitter.jobs.FetchTweetsJob')
        :obj_args: Constructor args
        :obj_kwargs: Constructor kwargs
        :call_args: Refresh args
        :call_kwargs: Refresh kwargs
        """
        klass = get_job_class(klass_str)
        if klass is None:
            logger.error("Unable to construct %s with args %r and kwargs %r",
                         klass_str, obj_args, obj_kwargs)
            return

        logger.info("Using %s with constructor args %r and kwargs %r",
                    klass_str, obj_args, obj_kwargs)
        logger.info("Calling refresh with args %r and kwargs %r", call_args,
                    call_kwargs)
        start = time.time()
        try:
            klass(*obj_args, **obj_kwargs).refresh(
                *call_args, **call_kwargs)
        except Exception as e:
            logger.exception("Error running job: '%s'", e)
        else:
            duration = time.time() - start
            logger.info("Refreshed cache in %.6f seconds", duration)
<|endoftext|>


<commit_before># Copyright 2015 Jared Rodriguez (jared at blacknode dot net)
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import logging
import time

import msgpack
import zmq

from mercury.agent.configuration import agent_configuration
from mercury.common.exceptions import MercuryCritical
from mercury.inspector.inspectors.interfaces import get_interface_by_name
from mercury.inspector.inspectors.routes import find_default_route

log = logging.getLogger(__name__)

def get_dhcp_ip(device_info, method='simple'):
    """
    Get the ip address provided by the DHCP server. Initial operation will use this IP to register with the rpc
     service. Plans are in the works for providing multiple avenues for selecting/generating(ipv6) addresses. Some
     options are:
        Command line argument
        Kernel command line arguments
        Link layer addressing, sans DHCP (48bit, IPV6 only : RFC2373)
    :param device_info: mercury.agent.inspector device_info
    :param method: Possible methods are:
                    simple - Find the interface used for the default route, select this interfaces primary (first)
                    address
                    udhcpc - A mercury_ghost and mercury.agent.inspector construct. The ghost (ephemeral) os that carries
                    the agent and inspector codes contains a script that is called during udhcpc operation. This
                    script will configured the interface as normal. In addition the script will write a file,
                    mercury_dhcp_info.sh, containing DHCP options recieved from the server (including the dhcp_ip).
    :return: ip address
    """
    if method == 'simple':
        routes = device_info['routes']
        interfaces = device_info['interfaces']
        default_route = find_default_route(routes)
        if not default_route:
            raise MercuryCritical('The is no default route defined, cannot use simple method')
        if 'src' in default_route:
            return default_route['src']

        # If the route does not contain a src definition, we take the first address on the 'default' dev
        default_interface = get_interface_by_name(interfaces, default_route['dev']) or dict()
        address_info = default_interface.get('address_info')
        if not address_info:
            raise MercuryCritical('Failed to determine dhcp address')

        log.debug('address_info: %s' % address_info)
        return address_info[0]['addr']

def register(mercury_id, local_ip, local_ip6, capabilities):
    rpc_backend = agent_configuration.get('remote', {}).get('rpc_service')

    if not rpc_backend:
        raise MercuryCritical('Missing rpc backend in local configuration')

    # There is still some confusion regarding how best to determine what ip
    # to publish. Current wisdom suggest that we find the default gateway,
    # determine the interface used for the default route, and take whatever
    # address is configured. This 'should' be the DHCP address right?
    # Relying solely on dhclient (in the initrd) might be an option and would
    # allow for parsing dhclient.leases.
    # Currently, the preboot environment uses udhcpc, so modifying the script
    # udhcpc calls on renew/bound to publish a cookie with vars set by the dhcp
    # server is an option as well

    # Either process should be configurable. ie,
    #  default_ip_source: simple # use default route
    #                     udhcpc # script in preconfig leaves cookie, parse the cookie
    #                     dhclient # parse /var/lib/dhclient.leases

    # And then there is ipv6, where we'd need to DHCP to get a route-able address
    # subnet, and then heuristically generate an ip address in the subnet, forgoing
    # the explicit need for publishing the address
    _caps = capabilities.copy()
    for capability in _caps:
        del _caps[capability]['entry']

    payload = {
        'mercury_id': mercury_id,
        'rpc_address': local_ip,
        'rpc_address6': local_ip6,
        'rpc_port': agent_configuration.get('rpc_port', 9003),
        'ping_port': agent_configuration.get('ping_port', 9004),
        'localtime': time.time(),
        'capabilities': _caps
    }
    packed = msgpack.packb(dict(action='register', client_info=payload))

    ctx = zmq.Context.instance()

    # noinspection PyUnresolvedReferences
    socket = ctx.socket(zmq.REQ)
    socket.connect(rpc_backend)
    socket.send(packed)

    return msgpack.unpackb(socket.recv())

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    from mercury.inspector import inspect
    from mercury.agent.capabilities import runtime_capabilities
    collection = inspect.inspect()

    _mercury_id = collection['mercury_id']
    _local_ip = get_dhcp_ip(collection)
    _local_ip6 = ''

    response = register(_mercury_id, _local_ip, _local_ip6, runtime_capabilities)
<commit_msg>deep copy<commit_after># Copyright 2015 Jared Rodriguez (jared at blacknode dot net)
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import logging
import time

import msgpack
import zmq

from mercury.agent.configuration import agent_configuration
from mercury.common.exceptions import MercuryCritical
from mercury.inspector.inspectors.interfaces import get_interface_by_name
from mercury.inspector.inspectors.routes import find_default_route

log = logging.getLogger(__name__)

def get_dhcp_ip(device_info, method='simple'):
    """
    Get the ip address provided by the DHCP server. Initial operation will use this IP to register with the rpc
     service. Plans are in the works for providing multiple avenues for selecting/generating(ipv6) addresses. Some
     options are:
        Command line argument
        Kernel command line arguments
        Link layer addressing, sans DHCP (48bit, IPV6 only : RFC2373)
    :param device_info: mercury.agent.inspector device_info
    :param method: Possible methods are:
                    simple - Find the interface used for the default route, select this interfaces primary (first)
                    address
                    udhcpc - A mercury_ghost and mercury.agent.inspector construct. The ghost (ephemeral) os that carries
                    the agent and inspector codes contains a script that is called during udhcpc operation. This
                    script will configured the interface as normal. In addition the script will write a file,
                    mercury_dhcp_info.sh, containing DHCP options recieved from the server (including the dhcp_ip).
    :return: ip address
    """
    if method == 'simple':
        routes = device_info['routes']
        interfaces = device_info['interfaces']
        default_route = find_default_route(routes)
        if not default_route:
            raise MercuryCritical('The is no default route defined, cannot use simple method')
        if 'src' in default_route:
            return default_route['src']

        # If the route does not contain a src definition, we take the first address on the 'default' dev
        default_interface = get_interface_by_name(interfaces, default_route['dev']) or dict()
        address_info = default_interface.get('address_info')
        if not address_info:
            raise MercuryCritical('Failed to determine dhcp address')

        log.debug('address_info: %s' % address_info)
        return address_info[0]['addr']

def _serialize_capabilities(capabilities):
    _d = {}
    for c in capabilities:
        temp_d = capabilities[c].copy()
        temp_d['entry'] = temp_d['entry'].__name__
        _d[c] = temp_d
    return _d

def register(mercury_id, local_ip, local_ip6, capabilities):
    rpc_backend = agent_configuration.get('remote', {}).get('rpc_service')

    if not rpc_backend:
        raise MercuryCritical('Missing rpc backend in local configuration')

    # There is still some confusion regarding how best to determine what ip
    # to publish. Current wisdom suggest that we find the default gateway,
    # determine the interface used for the default route, and take whatever
    # address is configured. This 'should' be the DHCP address right?
    # Relying solely on dhclient (in the initrd) might be an option and would
    # allow for parsing dhclient.leases.
    # Currently, the preboot environment uses udhcpc, so modifying the script
    # udhcpc calls on renew/bound to publish a cookie with vars set by the dhcp
    # server is an option as well

    # Either process should be configurable. ie,
    #  default_ip_source: simple # use default route
    #                     udhcpc # script in preconfig leaves cookie, parse the cookie
    #                     dhclient # parse /var/lib/dhclient.leases

    # And then there is ipv6, where we'd need to DHCP to get a route-able address
    # subnet, and then heuristically generate an ip address in the subnet, forgoing
    # the explicit need for publishing the address

    payload = {
        'mercury_id': mercury_id,
        'rpc_address': local_ip,
        'rpc_address6': local_ip6,
        'rpc_port': agent_configuration.get('rpc_port', 9003),
        'ping_port': agent_configuration.get('ping_port', 9004),
        'localtime': time.time(),
        'capabilities': _serialize_capabilities(capabilities)
    }
    packed = msgpack.packb(dict(action='register', client_info=payload))

    ctx = zmq.Context.instance()

    # noinspection PyUnresolvedReferences
    socket = ctx.socket(zmq.REQ)
    socket.connect(rpc_backend)
    socket.send(packed)

    return msgpack.unpackb(socket.recv())

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    from mercury.inspector import inspect
    from mercury.agent.capabilities import runtime_capabilities
    collection = inspect.inspect()

    _mercury_id = collection['mercury_id']
    _local_ip = get_dhcp_ip(collection)
    _local_ip6 = ''

    response = register(_mercury_id, _local_ip, _local_ip6, runtime_capabilities)
<|endoftext|>


<commit_before>#http://pypi.python.org/pypi/cql/1.0.4
#http://code.google.com/a/apache-extras.org/p/cassandra-dbapi2 /
#http://cassandra.apache.org/doc/cql/CQL.html

from collections import namedtuple
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement, Statement

try:
    import Queue as queue
except ImportError:
    # python 3
    import queue

import logging

from cqlengine.exceptions import CQLEngineException, UndefinedKeyspaceException
from cassandra import ConsistencyLevel
from cqlengine.statements import BaseCQLStatement
from cassandra.query import dict_factory

LOG = logging.getLogger('cqlengine.cql')

class CQLConnectionError(CQLEngineException): pass

Host = namedtuple('Host', ['name', 'port'])

cluster = None
session = None
lazy_connect_args = None
default_consistency_level = None

def setup(
        hosts,
        default_keyspace,
        consistency=ConsistencyLevel.ONE,
        lazy_connect=False,
        **kwargs):
    """
    Records the hosts and connects to one of them

    :param hosts: list of hosts, see http://datastax.github.io/python-driver/api/cassandra/cluster.html
    :type hosts: list
    :param default_keyspace: The default keyspace to use
    :type default_keyspace: str
    :param consistency: The global consistency level
    :type consistency: int
    :param lazy_connect: True if should not connect until first use
    :type lazy_connect: bool
    """
    global cluster, session, default_consistency_level, lazy_connect_args

    if 'username' in kwargs or 'password' in kwargs:
        raise CQLEngineException("Username & Password are now handled by using the native driver's auth_provider")

    if not default_keyspace:
        raise UndefinedKeyspaceException()

    from cqlengine import models
    models.DEFAULT_KEYSPACE = default_keyspace

    default_consistency_level = consistency
    if lazy_connect:
        lazy_connect_args = (hosts, default_keyspace, consistency, kwargs)
        return

    cluster = Cluster(hosts, **kwargs)
    session = cluster.connect()
    session.row_factory = dict_factory

def execute(query, params=None, consistency_level=None):

    if not session:
        raise CQLEngineException("It is required to setup() cqlengine before executing queries")

    handle_lazy_connect()

    if consistency_level is None:
        consistency_level = default_consistency_level

    if isinstance(query, Statement):
        pass

    elif isinstance(query, BaseCQLStatement):
        params = query.get_context()
        query = str(query)
        query = SimpleStatement(query, consistency_level=consistency_level)

    elif isinstance(query, basestring):
        query = SimpleStatement(query, consistency_level=consistency_level)

    params = params or {}
    result = session.execute(query, params)

    return result

def get_session():
    handle_lazy_connect()
    return session

def get_cluster():
    handle_lazy_connect()
    return cluster

def handle_lazy_connect():
    global lazy_connect_args
    if lazy_connect_args:
        hosts, default_keyspace, consistency, kwargs = lazy_connect_args
        lazy_connect_args = None
        setup(hosts, default_keyspace, consistency, **kwargs)
<commit_msg>Fix lazy_connect, call handle_lazy_connect before checking session existence<commit_after>#http://pypi.python.org/pypi/cql/1.0.4
#http://code.google.com/a/apache-extras.org/p/cassandra-dbapi2 /
#http://cassandra.apache.org/doc/cql/CQL.html

from collections import namedtuple
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement, Statement

try:
    import Queue as queue
except ImportError:
    # python 3
    import queue

import logging

from cqlengine.exceptions import CQLEngineException, UndefinedKeyspaceException
from cassandra import ConsistencyLevel
from cqlengine.statements import BaseCQLStatement
from cassandra.query import dict_factory

LOG = logging.getLogger('cqlengine.cql')

class CQLConnectionError(CQLEngineException): pass

Host = namedtuple('Host', ['name', 'port'])

cluster = None
session = None
lazy_connect_args = None
default_consistency_level = None

def setup(
        hosts,
        default_keyspace,
        consistency=ConsistencyLevel.ONE,
        lazy_connect=False,
        **kwargs):
    """
    Records the hosts and connects to one of them

    :param hosts: list of hosts, see http://datastax.github.io/python-driver/api/cassandra/cluster.html
    :type hosts: list
    :param default_keyspace: The default keyspace to use
    :type default_keyspace: str
    :param consistency: The global consistency level
    :type consistency: int
    :param lazy_connect: True if should not connect until first use
    :type lazy_connect: bool
    """
    global cluster, session, default_consistency_level, lazy_connect_args

    if 'username' in kwargs or 'password' in kwargs:
        raise CQLEngineException("Username & Password are now handled by using the native driver's auth_provider")

    if not default_keyspace:
        raise UndefinedKeyspaceException()

    from cqlengine import models
    models.DEFAULT_KEYSPACE = default_keyspace

    default_consistency_level = consistency
    if lazy_connect:
        lazy_connect_args = (hosts, default_keyspace, consistency, kwargs)
        return

    cluster = Cluster(hosts, **kwargs)
    session = cluster.connect()
    session.row_factory = dict_factory

def execute(query, params=None, consistency_level=None):

    handle_lazy_connect()

    if not session:
        raise CQLEngineException("It is required to setup() cqlengine before executing queries")

    if consistency_level is None:
        consistency_level = default_consistency_level

    if isinstance(query, Statement):
        pass

    elif isinstance(query, BaseCQLStatement):
        params = query.get_context()
        query = str(query)
        query = SimpleStatement(query, consistency_level=consistency_level)

    elif isinstance(query, basestring):
        query = SimpleStatement(query, consistency_level=consistency_level)

    params = params or {}
    result = session.execute(query, params)

    return result

def get_session():
    handle_lazy_connect()
    return session

def get_cluster():
    handle_lazy_connect()
    return cluster

def handle_lazy_connect():
    global lazy_connect_args
    if lazy_connect_args:
        hosts, default_keyspace, consistency, kwargs = lazy_connect_args
        lazy_connect_args = None
        setup(hosts, default_keyspace, consistency, **kwargs)
<|endoftext|>


<commit_before><commit_msg>[FIX] point_of_sale: use config sequence on order line creation<commit_after><|endoftext|>