<commit_before>from filecmp import demo
from django.conf.app_template import admin
from django.contrib.auth.decorators import login_required
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import render
from django.utils.datetime_safe import datetime
from django.utils.decorators import method_decorator
from django.views.generic import ListView
from django.http import HttpResponseRedirect, JsonResponse
from buzzit_messaging.views import __send_system__message__
from buzzit_models.models import *
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse, reverse_lazy
import django.contrib.messages as messages
from django.views.generic import ListView, DetailView
from buzzit_models.models import *
from django.contrib.auth.decorators import login_required
import json
from django.core.mail import send_mail

@login_required
def report_user(request,user_id):
    """
    current user report other user, and gives the reason which should not be empty
    :param request:
    :param user_id:
    :return:
    """
    if request.method == "POST":
        try:
            reported_user = User.objects.get(pk=user_id)
        except ObjectDoesNotExist:
            messages.error(request,"Der Benutzer existiert nicht.")
            return HttpResponseRedirect(reverse_lazy("home"))
        if request.method == "POST":
            report_message = UserReport()
            report_text = request.POST.get("text",False)
            try:
                if report_text:
                    report_message.text=report_text
            except ObjectDoesNotExist:
                messages.error(request,"Fehler")

            if len(report_message.text) < 1:
                messages.error(request,"Text zum Benutzermelden ist zu geben")
                return HttpResponseRedirect(reverse_lazy("home"))

            report_message.creator=request.user
            report_message.created=datetime.now()
            report_message.reported_user=reported_user
            report_message.save()
            messages.info(request, "Sie haben den <User:%s> Benutzer gemeldet" %reported_user)

            return HttpResponseRedirect(reverse_lazy('home'))
    else:
        try:
            reported_profile = Profile.objects.get(pk=user_id)
        except ObjectDoesNotExist:
            messages.error(request,"Der Benutzer existiert nicht.")
            return HttpResponseRedirect(reverse_lazy("home"))
        return render(request, "logged_in/report_user.html", {"profile":reported_profile})

class UserReportDetailsView(SuccessMessageMixin,ListView):
    """
    display the report text and reported user
    """
    model = UserReport
    template_name = "logged_in/user_report_deatails"

    def get_queryset(self):
        try:
            reported_user = self.kwargs.get["user_id"]
        except ObjectDoesNotExist:
            messages.error(self.request,"Benutzer existiert nicht")
            return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
        return UserReport.objects.filter(reported_user=reported_user).order_by("created")

    def get_context_data(self, **kwargs):
        try:
            reported_user=self.kwargs.get["user_id"]
        except ObjectDoesNotExist:
            messages.error(self.request,"Benutzer existiert nicht")
            return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

        reported_user_profile = reported_user.profile
        report = UserReport.objects.get(reported_user=reported_user)
        context=super(UserReportDetailsView, self).get_context_data(**kwargs)
        context["reported_user_profile"]= reported_user_profile
        context["report_text"]=report.text

        return context
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(UserReportDetailsView, self).dispatch(request,args,kwargs)

class AdminFrontpageView():
    pass

@login_required
def adminFrontPage(request):
     """
     show all userreports and
     postreports
     :param request:
     :return:
     """
     if request.user.is_superuser:
        userreports =[]
        postreports=[]
        userreports = UserReport.objects.all()
        postreports =CircleMessageReport.objects.all()

        return render(request,"logged_in/admin_dashboard.html",{"user_reports":userreports,"post_reports":postreports})
     else:
         messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
         return HttpResponseRedirect(reverse("home"))

class MessageReportDetailsView(DetailView):
    model = CircleMessageReport
    slug_field = "id"
    template_name = "logged_in/post_report_details.html"

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(MessageReportDetailsView, self).dispatch(request, args, kwargs)

@login_required
def AdminOverviewView(request):
    if request.user.is_superuser:
        adminlist = []
        adminlist = User.objects.filter(is_superuser = True)
        return render(request, "logged_in/admin_list.html", {"userlist": adminlist})
    else:
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))

@login_required
def delete_reported_post(request, report_id):
    """
    delete reported message from admin, check if the message has answers,
    reported message with all answers would be delete, else delete only message
    TODO was ist, wenn eine Nachricht rebuzzed wurde
    :param request:
    :param message_id:
    :return:
    """
    try:
        report = CircleMessageReport.objects.get(pk = report_id)
    except ObjectDoesNotExist:
        messages.error(request,"Der Report existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
    #if the reported post has anwsers, delete all
    if not(request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    post_to_del=report.reported_message
    answers=Circle_message.objects.filter(answer_to=post_to_del)
    answers.delete()
    post_to_del.delete()
    report.issuer = request.user
    report.valid = True
    report.closed = True
    messages.success(request,"Die Nachrichte wurde erfolgreich geloescht")
    return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

@login_required
def promote_user_to_admin(request, user_id):
    """
    check if user exists, then check if user is active
    :param request:
    :param user_id:
    :return:
    """
    try:
        admin_user=User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request,"Der Benuzer existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
    if not(request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))

    if not(admin_user.is_active):
        messages.info(request,"Der Benutzer ist deaktiviert")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    admin_user.is_superuser=True
    admin_user.save()
    messages.info(request,"Der Benutzer %s ist als AdminUser hinzugefuegt" % (admin_user.username,))
    return HttpResponseRedirect(reverse_lazy("admins_overview"))

@login_required
def demote_admin_to_user(request, user_id):
    """
    check if user exists, check if user is adminUser
    :param request:
    :param user_id:
    :return:
    """
    try:
        demote_user=User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request,"Der Benutzer existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    if not(request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    if not(demote_user.is_superuser):
        messages.error(request,"Der Benutzer ist kein Admin ")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    demote_user.is_superuser=False
    demote_user.save()
    messages.info(request,"Die Adminrechte von dem Benutzer wird entziehen")
    return HttpResponseRedirect(reverse_lazy("admins_overview"))

@login_required
def report_message(request, message_id):
    """
    Report a circlemessage with given <message_id>, if that exists.
    If that does not exist, then an error for the user is returned and he gets redirected to home.
    If that message exists,
        then the report will be created, if an reason (report.text) was given.
            The report is saved then.
        if there is no reason, an error will be created and the user is redirected to home.
    :param request:
    :param message_id:
    :return:
    """
    try:
        reported_message = Circle_message.objects.get(pk=message_id)
    except Exception:
        messages.error(request, "Die Nachricht existiert nicht")
        return HttpResponseRedirect(reverse("home"))
    if request.method == "POST":
        report = CircleMessageReport()
        report.reported_message = reported_message
        report.text = request.POST.get("text", False)
        if not report.text or len(report.text)<1:
            messages.error(request, "Keine Begruendung angegeben")
            return HttpResponseRedirect(reverse("home"))
        report.creator = request.user
        report.created = datetime.now()
        report.save()
        messages.success(request, "Nachricht wurde gemeldet")
        return HttpResponseRedirect(reverse("home"))
    reported_profile = Profile.objects.get(pk = reported_message.creator.pk)
    return render(request, "logged_in/report_post.html", {"profile" : reported_profile, "circlemessage" : reported_message})

@login_required
def ban_user(request, user_id):
    """
    set ban user and send email to him with reason,TODO provides ban user information to contact with admin user
    :param request:
    :param user_id:
    :return:
    """
    try:
        user_to_be_ban=User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request,"Der Benutzer existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    if not(request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    if not(user_to_be_ban.is_active):
        messages.info(request,"Der Benutzer ist bereits deaktiviert")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    message_for_ban=request.GET.get["text",False]
    user_to_be_ban.is_active=False
    send_mail("Deaktivieren dein Account", message= "Grund zum Deaktivieren: '%s'" % message_for_ban,
                      html_message="
um Deinen Account zu wieder aktivieren, kontaktieren Sie bitte :
" +
                                   "Klicke hier um den Account wieder zu aktivieren!."  +
                                   "" ,from_email="AccountAktivierung@vps146949.ovh.net", recipient_list=(user_to_be_ban.email,))
    messages.info(request,"Der Benutzer ist deaktiviert")
    return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
<commit_msg>report message läuft<commit_after>from filecmp import demo
from django.conf.app_template import admin
from django.contrib.auth.decorators import login_required
from django.contrib.messages.views import SuccessMessageMixin
from django.shortcuts import render
from django.utils.datetime_safe import datetime
from django.utils.decorators import method_decorator
from django.views.generic import ListView
from django.http import HttpResponseRedirect, JsonResponse
from buzzit_messaging.views import __send_system__message__
from buzzit_models.models import *
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse, reverse_lazy
import django.contrib.messages as messages
from django.views.generic import ListView, DetailView
from buzzit_models.models import *
from django.contrib.auth.decorators import login_required
import json
from django.core.mail import send_mail

@login_required
def report_user(request, user_id):
    """
    current user report other user, and gives the reason which should not be empty
    :param request:
    :param user_id:
    :return:
    """
    if request.method == "POST":
        try:
            reported_user = User.objects.get(pk=user_id)
        except ObjectDoesNotExist:
            messages.error(request, "Der Benutzer existiert nicht.")
            return HttpResponseRedirect(reverse_lazy("home"))
        if request.method == "POST":
            report_message = UserReport()
            report_text = request.POST.get("text", False)
            try:
                if report_text:
                    report_message.text = report_text
            except ObjectDoesNotExist:
                messages.error(request, "Fehler")

            if len(report_message.text) < 1:
                messages.error(request, "Text zum Benutzermelden ist zu geben")
                return HttpResponseRedirect(reverse_lazy("home"))

            report_message.creator = request.user
            report_message.created = datetime.now()
            report_message.reported_user = reported_user
            report_message.save()
            messages.info(request, "Sie haben den <User:%s> Benutzer gemeldet" % reported_user)

            return HttpResponseRedirect(reverse_lazy('home'))
    else:
        try:
            reported_profile = Profile.objects.get(pk=user_id)
        except ObjectDoesNotExist:
            messages.error(request, "Der Benutzer existiert nicht.")
            return HttpResponseRedirect(reverse_lazy("home"))
        return render(request, "logged_in/report_user.html", {"profile": reported_profile})

class UserReportDetailsView(SuccessMessageMixin, ListView):
    """
    display the report text and reported user
    """
    model = UserReport
    template_name = "logged_in/user_report_deatails"

    def get_queryset(self):
        try:
            reported_user = self.kwargs.get["user_id"]
        except ObjectDoesNotExist:
            messages.error(self.request, "Benutzer existiert nicht")
            return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
        return UserReport.objects.filter(reported_user=reported_user).order_by("created")

    def get_context_data(self, **kwargs):
        try:
            reported_user = self.kwargs.get["user_id"]
        except ObjectDoesNotExist:
            messages.error(self.request, "Benutzer existiert nicht")
            return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

        reported_user_profile = reported_user.profile
        report = UserReport.objects.get(reported_user=reported_user)
        context = super(UserReportDetailsView, self).get_context_data(**kwargs)
        context["reported_user_profile"] = reported_user_profile
        context["report_text"] = report.text

        return context

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(UserReportDetailsView, self).dispatch(request, args, kwargs)

class AdminFrontpageView():
    pass

@login_required
def adminFrontPage(request):
    """
    show all userreports and
    postreports
    :param request:
    :return:
    """
    if request.user.is_superuser:
        userreports = []
        postreports = []
        userreports = UserReport.objects.all()
        postreports = CircleMessageReport.objects.all()

        return render(request, "logged_in/admin_dashboard.html",
                      {"user_reports": userreports, "post_reports": postreports})
    else:
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))

class MessageReportDetailsView(DetailView):
    model = CircleMessageReport
    slug_field = "id"
    template_name = "logged_in/post_report_details.html"

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(MessageReportDetailsView, self).dispatch(request, args, kwargs)

@login_required
def AdminOverviewView(request):
    if request.user.is_superuser:
        adminlist = []
        adminlist = User.objects.filter(is_superuser=True)
        return render(request, "logged_in/admin_list.html", {"userlist": adminlist})
    else:
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))

@login_required
def delete_reported_post(request, report_id):
    """
    delete reported message from admin, check if the message has answers,
    reported message with all answers would be delete, else delete only message
    TODO was ist, wenn eine Nachricht rebuzzed wurde
    :param request:
    :param message_id:
    :return:
    """
    try:
        report = CircleMessageReport.objects.get(pk=report_id)
    except ObjectDoesNotExist:
        messages.error(request, "Der Report existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
    # if the reported post has anwsers, delete all
    if not (request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    post_to_del = report.reported_message
    answers = Circle_message.objects.filter(answer_to=post_to_del)
    answers.delete()
    post_to_del.delete()
    report.issuer = request.user
    report.valid = True
    report.closed = True
    messages.success(request, "Die Nachrichte wurde erfolgreich geloescht")
    return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

@login_required
def promote_user_to_admin(request, user_id):
    """
    check if user exists, then check if user is active
    :param request:
    :param user_id:
    :return:
    """
    try:
        admin_user = User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request, "Der Benuzer existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))
    if not (request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))

    if not (admin_user.is_active):
        messages.info(request, "Der Benutzer ist deaktiviert")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    admin_user.is_superuser = True
    admin_user.save()
    messages.info(request, "Der Benutzer %s ist als AdminUser hinzugefuegt" % (admin_user.username,))
    return HttpResponseRedirect(reverse_lazy("admins_overview"))

@login_required
def demote_admin_to_user(request, user_id):
    """
    check if user exists, check if user is adminUser
    :param request:
    :param user_id:
    :return:
    """
    try:
        demote_user = User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request, "Der Benutzer existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    if not (request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    if not (demote_user.is_superuser):
        messages.error(request, "Der Benutzer ist kein Admin ")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    demote_user.is_superuser = False
    demote_user.save()
    messages.info(request, "Die Adminrechte von dem Benutzer wird entziehen")
    return HttpResponseRedirect(reverse_lazy("admins_overview"))

@login_required
def report_message(request, message_id):
    """
    Report a circlemessage with given <message_id>, if that exists.
    If that does not exist, then an error for the user is returned and he gets redirected to home.
    If that message exists,
        then the report will be created, if an reason (report.text) was given.
            The report is saved then.
        if there is no reason, an error will be created and the user is redirected to home.
    :param request:
    :param message_id:
    :return:
    """
    try:
        reported_message = Circle_message.objects.get(pk=message_id)
    except Exception:
        messages.error(request, "Die Nachricht existiert nicht")
        return HttpResponseRedirect(reverse("home"))
    if request.method == "POST":
        report = CircleMessageReport()
        report.reported_message = reported_message
        report.text = request.POST.get("text", False)
        if not report.text or len(report.text) < 1:
            messages.error(request, "Keine Begruendung angegeben")
            return HttpResponseRedirect(reverse("home"))
        report.creator = request.user
        report.created = datetime.now()
        report.save()
        messages.success(request, "Nachricht wurde gemeldet")
        return HttpResponseRedirect(reverse("home"))
    reported_profile = Profile.objects.get(pk=reported_message.creator.pk)
    return render(request, "logged_in/report_post.html",
                  {"profile": reported_profile, "circlemessage": reported_message})

@login_required
def ban_user(request, user_id):
    """
    set ban user and send email to him with reason,TODO provides ban user information to contact with admin user
    :param request:
    :param user_id:
    :return:
    """
    try:
        user_to_be_ban = User.objects.get(pk=user_id)
    except ObjectDoesNotExist:
        messages.error(request, "Der Benutzer existiert nicht")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    if not (request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    if not (user_to_be_ban.is_active):
        messages.info(request, "Der Benutzer ist bereits deaktiviert")
        return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

    message_for_ban = request.GET.get["text", False]
    user_to_be_ban.is_active = False
    send_mail("Deaktivieren dein Account", message="Grund zum Deaktivieren: '%s'" % message_for_ban,
              html_message="
um Deinen Account zu wieder aktivieren, kontaktieren Sie bitte :
" +
                           "Klicke hier um den Account wieder zu aktivieren!." +
                           "", from_email="AccountAktivierung@vps146949.ovh.net",
              recipient_list=(user_to_be_ban.email,))
    messages.info(request, "Der Benutzer ist deaktiviert")
    return HttpResponseRedirect(reverse_lazy("admin_frontpage"))

@login_required
def setIgnoreReport(request, report_id):
    if not (request.user.is_superuser):
        messages.error(request, "Sie haben nicht die nötigen Zugangsrechte!")
        return HttpResponseRedirect(reverse("home"))
    try:
        report = Report.objects.get(pk=report_id)
    except:
        messages.error(request, "Report existiert nicht")
        return HttpResponseRedirect(reverse("admin_frontpage"))
    report.closed = True
    report.valid = False
    report.issuer = request.user
    report.save()
    messages.success(request, "Report wurde ignoriert")
    return HttpResponseRedirect(reverse("admin_frontpage"))
<|endoftext|>


<commit_before>import os.path
import StringIO
import sh
import sys
import threading

def null_dest( files ):
    for f in files:
        pass

def debug_dest( files ):
    for f in files:
        print f.name
        if hasattr( f, 'mode' ):
            print 'Mode: ', f.mode
        print '-' * len( f.name )
        print f.read()
        print

def debug_pipe( files ):
    for f in files:
        print '+', f.name

        yield f

def read_all_to( f, to, chunk_size = None ):
    while True:
        chunk = f.read( *( [ chunk_size ] if chunk_size else [] ) )

        if chunk:
            to( chunk )
        else:
            break

def read_all( f, chunk_size = None ):
    ret = ''
    while True:
        chunk = f.read( *( [ chunk_size ] if chunk_size else [] ) )

        if chunk:
            ret += chunk
        else:
            return ret

class lazy_file( object ):
    def __init__( self, name, *a, **kw ):
        self.name = name
        self._name = name
        self.file = None
        self.size = 0
        self.a = a
        self.kw = kw
        self.delete = False

    def open( self ):
        self.file = open( self._name, *self.a, **self.kw )
        self.size = os.stat( self._name ).st_size

    def read( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.read( *a, **kw )

    def write( self, data, *a, **kw ):
        if not self.file:
            self.open()

        self.size += len( data )

        return self.file.write( data, *a, **kw )

    def seek( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.seek( *a, **kw )

    def tell( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.tell( *a, **kw )

    def rename( self, name ):
        print '>', name
        self.name = name

        return self

class pipe:
    class _reader( object ):
        def __init__( self, pipe ):
            self.pipe = pipe
            self.pos = 0
            self.delete = False

        def read( self, *a, **kw ):
            self.pipe.need_data.set()
            self.pipe.has_data.wait()
            with self.pipe.chunks_lock:
                chunk = self.pipe.chunks.pop( 0 )

                if isinstance( chunk, Exception ):
                    raise chunk
                elif chunk:
                    self.pos += len( chunk )
                    self.pipe.chunk_size -= len( chunk )

                    if self.pipe.chunk_size < self.pipe.chunk_max:
                        self.pipe.not_full.set()

                if not len( self.pipe.chunks ):
                    self.pipe.has_data.clear()

                return chunk

        @property
        def name(self):
            return self.pipe.name

        def rename( self, name ):
            self.pipe.rename( name )
            return self

        def flush( self ):
            pass

    class _writer( object ):
        def __init__( self, pipe ):
            self.pipe = pipe
            self.pos = 0
            self.delete = False

        def write( self, chunk ):
            self.pipe.not_full.wait()
            with self.pipe.chunks_lock:
                self.pipe.chunks.append( chunk )

                if isinstance( chunk, Exception ):
                    pass
                elif chunk:
                    self.pos += len( chunk )
                    self.pipe.chunk_size += len( chunk )

                    if self.pipe.chunk_size >= self.pipe.chunk_max:
                        print 'full!'
                        self.pipe.not_full.clear()

                self.pipe.has_data.set()

        def close( self ):
            self.write( '' )

        @property
        def name(self):
            return self.pipe.name

        def rename( self, name ):
            self.pipe.rename( name )
            return self

        def flush( self ):
            pass

    def __init__( self, name, chunk_max = 8 * 1024 * 1024 ):
        self.chunks = []
        self.chunks_lock = threading.Lock()
        self.chunk_size = 0
        self.chunk_max = chunk_max
        self.need_data = threading.Event()
        self.has_data = threading.Event()
        self.not_full = threading.Event()
        self.not_full.set()
        self.r = self._reader( self )
        self.w = self._writer( self )
        self.name = name
        self.delete = False

    def rename( self, name ):
        print '>', name
        self.name = name

        return self

def example_splitter( files ):
    for f in files:
        if f.delete:
            yield f

        if f.name in [ __file__, 'targets.py' ] or f.name.endswith( '.pyc' ):
            pass
        elif f.name.endswith( '.txt' ):
            yield uppercase( f )
        elif f.name.endswith( '.less' ):
            yield lessc( f ).rename( f.name.replace( '.less', '.css' ) )
        else:
            yield f

def dest_select( files, targets ):
    if len( sys.argv ) > 1:
        return getattr( targets, sys.argv[1] )( files )
    else:
        return targets.default( files )

def uppercase( f ):
    p = pipe( f.name )

    def run():
        read_all( f, lambda chunk: p.w.write( chunk.upper() ) )
        p.w.close()

    threading.Thread( target = run ).start()

    return p.r

def lessc( f, *a, **kw ):
    print 'B', f.name
    p = pipe( f.name )

    def run():
        sh.lessc( '-', *a, _in = f, _out = p.w, **kw )
        p.w.close()

    threading.Thread( target = run ).start()

    return p.r

def uglifyjs( file_paths ):
    print 'B', ' + '.join( file_paths )
    p = pipe( '' )

    def run():
        sh.uglifyjs( file_paths, _out = p.w )
        p.w.close()

    threading.Thread( target = run ).start()

    return p.r
<commit_msg>allow passing arguments to uglifyjs as well<commit_after>import os.path
import StringIO
import sh
import sys
import threading

def null_dest( files ):
    for f in files:
        pass

def debug_dest( files ):
    for f in files:
        print f.name
        if hasattr( f, 'mode' ):
            print 'Mode: ', f.mode
        print '-' * len( f.name )
        print f.read()
        print

def debug_pipe( files ):
    for f in files:
        print '+', f.name

        yield f

def read_all_to( f, to, chunk_size = None ):
    while True:
        chunk = f.read( *( [ chunk_size ] if chunk_size else [] ) )

        if chunk:
            to( chunk )
        else:
            break

def read_all( f, chunk_size = None ):
    ret = ''
    while True:
        chunk = f.read( *( [ chunk_size ] if chunk_size else [] ) )

        if chunk:
            ret += chunk
        else:
            return ret

class lazy_file( object ):
    def __init__( self, name, *a, **kw ):
        self.name = name
        self._name = name
        self.file = None
        self.size = 0
        self.a = a
        self.kw = kw
        self.delete = False

    def open( self ):
        self.file = open( self._name, *self.a, **self.kw )
        self.size = os.stat( self._name ).st_size

    def read( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.read( *a, **kw )

    def write( self, data, *a, **kw ):
        if not self.file:
            self.open()

        self.size += len( data )

        return self.file.write( data, *a, **kw )

    def seek( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.seek( *a, **kw )

    def tell( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.tell( *a, **kw )

    def rename( self, name ):
        print '>', name
        self.name = name

        return self

class pipe:
    class _reader( object ):
        def __init__( self, pipe ):
            self.pipe = pipe
            self.pos = 0
            self.delete = False

        def read( self, *a, **kw ):
            self.pipe.need_data.set()
            self.pipe.has_data.wait()
            with self.pipe.chunks_lock:
                chunk = self.pipe.chunks.pop( 0 )

                if isinstance( chunk, Exception ):
                    raise chunk
                elif chunk:
                    self.pos += len( chunk )
                    self.pipe.chunk_size -= len( chunk )

                    if self.pipe.chunk_size < self.pipe.chunk_max:
                        self.pipe.not_full.set()

                if not len( self.pipe.chunks ):
                    self.pipe.has_data.clear()

                return chunk

        @property
        def name(self):
            return self.pipe.name

        def rename( self, name ):
            self.pipe.rename( name )
            return self

        def flush( self ):
            pass

    class _writer( object ):
        def __init__( self, pipe ):
            self.pipe = pipe
            self.pos = 0
            self.delete = False

        def write( self, chunk ):
            self.pipe.not_full.wait()
            with self.pipe.chunks_lock:
                self.pipe.chunks.append( chunk )

                if isinstance( chunk, Exception ):
                    pass
                elif chunk:
                    self.pos += len( chunk )
                    self.pipe.chunk_size += len( chunk )

                    if self.pipe.chunk_size >= self.pipe.chunk_max:
                        print 'full!'
                        self.pipe.not_full.clear()

                self.pipe.has_data.set()

        def close( self ):
            self.write( '' )

        @property
        def name(self):
            return self.pipe.name

        def rename( self, name ):
            self.pipe.rename( name )
            return self

        def flush( self ):
            pass

    def __init__( self, name, chunk_max = 8 * 1024 * 1024 ):
        self.chunks = []
        self.chunks_lock = threading.Lock()
        self.chunk_size = 0
        self.chunk_max = chunk_max
        self.need_data = threading.Event()
        self.has_data = threading.Event()
        self.not_full = threading.Event()
        self.not_full.set()
        self.r = self._reader( self )
        self.w = self._writer( self )
        self.name = name
        self.delete = False

    def rename( self, name ):
        print '>', name
        self.name = name

        return self

def example_splitter( files ):
    for f in files:
        if f.delete:
            yield f

        if f.name in [ __file__, 'targets.py' ] or f.name.endswith( '.pyc' ):
            pass
        elif f.name.endswith( '.txt' ):
            yield uppercase( f )
        elif f.name.endswith( '.less' ):
            yield lessc( f ).rename( f.name.replace( '.less', '.css' ) )
        else:
            yield f

def dest_select( files, targets ):
    if len( sys.argv ) > 1:
        return getattr( targets, sys.argv[1] )( files )
    else:
        return targets.default( files )

def uppercase( f ):
    p = pipe( f.name )

    def run():
        read_all( f, lambda chunk: p.w.write( chunk.upper() ) )
        p.w.close()

    threading.Thread( target = run ).start()

    return p.r

def lessc( f, *a, **kw ):
    print 'B', f.name
    p = pipe( f.name )

    def run():
        sh.lessc( '-', *a, _in = f, _out = p.w, **kw )
        p.w.close()

    threading.Thread( target = run ).start()

    return p.r

def uglifyjs( file_paths, *a, **kw ):
    print 'B', ' + '.join( file_paths )
    p = pipe( '' )

    def run():
        sh.uglifyjs( file_paths, *a, _out = p.w, **kw )
        p.w.close()

    threading.Thread( target = run ).start()

    return p.r
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

def add_legacy_name(apps, schema_editor):
    ContentType = apps.get_model('contenttypes', 'ContentType')
    for ct in ContentType.objects.all():
        try:
            ct.name = apps.get_model(ct.app_label, ct.model)._meta.object_name
        except:
            ct.name = ct.model
        ct.save()

class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='contenttype',
            options={'verbose_name': 'content type', 'verbose_name_plural': 'content types'},
        ),
        migrations.AlterField(
            model_name='contenttype',
            name='name',
            field=models.CharField(max_length=100, null=True),
        ),
        migrations.RunPython(
            migrations.RunPython.noop,
            add_legacy_name,
        ),
        migrations.RemoveField(
            model_name='contenttype',
            name='name',
        ),
    ]
<commit_msg>Explicitly checked for LookupError in contenttypes migration<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

def add_legacy_name(apps, schema_editor):
    ContentType = apps.get_model('contenttypes', 'ContentType')
    for ct in ContentType.objects.all():
        try:
            ct.name = apps.get_model(ct.app_label, ct.model)._meta.object_name
        except LookupError:
            ct.name = ct.model
        ct.save()

class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='contenttype',
            options={'verbose_name': 'content type', 'verbose_name_plural': 'content types'},
        ),
        migrations.AlterField(
            model_name='contenttype',
            name='name',
            field=models.CharField(max_length=100, null=True),
        ),
        migrations.RunPython(
            migrations.RunPython.noop,
            add_legacy_name,
        ),
        migrations.RemoveField(
            model_name='contenttype',
            name='name',
        ),
    ]
<|endoftext|>


<commit_before>from cytoolz.dicttoolz import valmap
from IPython.display import HTML, display_html
import os
import uuid
import pandas as pd
import networkx as nx
from networkx.readwrite import json_graph
import numpy as np
from IPython.display import HTML
import codecs
import jinja2
import os
import json
import copy

def _load_template(filename):
    if filename in ('base.js', 'base.html', 'scaffold.js'):
        filename = '{0}/templates/{1}'.format(SRC_DIR, filename)

    with open(filename, 'r') as f:
        code = f.read()
        return code

SRC_DIR = os.path.dirname(os.path.realpath(__file__))
env = jinja2.environment.Environment()
#env.loader = jinja2.FileSystemLoader([SRC_DIR + '/templates', SRC_DIR + '/libs'])
env.loader = jinja2.FunctionLoader(_load_template)

# based on http://stackoverflow.com/questions/3488934/simplejson-and-numpy-array
class MattaJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, np.generic):
            return obj.item()
        elif isinstance(obj, nx.Graph) or isinstance(obj, nx.DiGraph):
            if nx.is_tree(obj):
                # NOTE: the root must be the first node added. otherwise it won't work
                return json_graph.tree_data(obj, obj.nodes_iter().next())
            else:
                return json_graph.node_link_data(obj)
        elif isinstance(obj, pd.DataFrame):
            return obj.to_dict(outtype='records')
        return json.JSONEncoder.default(self, obj)

_dump_json = lambda x: json.dumps(x, cls=MattaJSONEncoder)

class sketch(object):
    def __init__(self, **kwargs):
        if not 'container_type' in kwargs:
            raise('need to define a container element')

        if not 'data' in kwargs or kwargs['data'] is None:
            raise Exception('you need to define at least one data variable')

        self.configuration = kwargs.copy()

        if not 'visualization_css' in kwargs:
            self.configuration['visualization_css'] =  '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
        if not 'visualization_js' in kwargs:
            self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])

        self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
        self.configuration['variables'] = valmap(_dump_json, self.configuration['variables'])
        self.configuration['__data_variables__'] = self.configuration['data'].keys()
        self.configuration['data'] = _dump_json(self.configuration['data'])

    def _render_(self, template_name):
        repr_args = self.configuration.copy()

        if self.configuration['visualization_css']:
            try:
                repr_args['visualization_css'] = env.get_template(self.configuration['visualization_css']).render(**repr_args)
            except IOError:
                repr_args['visualization_css'] = None

        if self.configuration['visualization_js']:
            repr_args['visualization_js'] = env.get_template(self.configuration['visualization_js']).render(**repr_args)
        else:
            raise Exception('Empty Visualization code!')

        template = env.get_template(template_name)

        if not 'figure_id' in repr_args or not repr_args['figure_id']:
            repr_args['figure_id'] = 'fig-{0}'.format(uuid.uuid4())

        if not 'vis_uuid' in repr_args or not repr_args['vis_uuid']:
            repr_args['vis_uuid'] = 'matta-vis-{0}'.format(uuid.uuid4())

        return template.render(**repr_args)

    def _ipython_display_(self):
        rendered = self._render_('base.html')
        display_html(HTML(rendered))

    def scaffold(self, filename=None, style=None, append=False):
        rendered = self._render_('scaffold.js')

        if filename is None:
            return rendered

        with open(filename, 'w') as f:
            f.write(rendered)

        if style is not None:
            mode = 'a' if append is True else 'w'
            with open(style, mode) as f:
                f.write(env.get_template(self.configuration['visualization_css']).render(**self.configuration))

def build_sketch(defaults, opt_process=None):
    def sketch_fn(**kwargs):
        sketch_args = copy.deepcopy(defaults)

        for key, value in kwargs.iteritems():
            if key in sketch_args:
                sketch_args[key] = value
            if key in sketch_args['data']:
                sketch_args['data'][key] = value
            elif key in sketch_args['options']:
                if type(sketch_args['options'][key]) == dict:
                    sketch_args['options'][key].update(value)
                else:
                    sketch_args['options'][key] = value
            elif key in sketch_args['variables']:
                if type(sketch_args['variables'][key]) == dict:
                    sketch_args['variables'][key].update(value)
                else:
                    sketch_args['variables'][key] = value
            elif key == 'figure_id':
                sketch_args[key] = value
            else:
                raise Exception('invalid argument: {0}'.format(key))

        if callable(opt_process):
            opt_process(sketch_args)

        return sketch(**sketch_args)

    return sketch_fn<commit_msg>using correct parameter name in DataFrame.to_dict<commit_after>from cytoolz.dicttoolz import valmap
from IPython.display import HTML, display_html
import os
import uuid
import pandas as pd
import networkx as nx
from networkx.readwrite import json_graph
import numpy as np
from IPython.display import HTML
import codecs
import jinja2
import os
import json
import copy

def _load_template(filename):
    if filename in ('base.js', 'base.html', 'scaffold.js'):
        filename = '{0}/templates/{1}'.format(SRC_DIR, filename)

    with open(filename, 'r') as f:
        code = f.read()
        return code

SRC_DIR = os.path.dirname(os.path.realpath(__file__))
env = jinja2.environment.Environment()
#env.loader = jinja2.FileSystemLoader([SRC_DIR + '/templates', SRC_DIR + '/libs'])
env.loader = jinja2.FunctionLoader(_load_template)

# based on http://stackoverflow.com/questions/3488934/simplejson-and-numpy-array
class MattaJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, np.generic):
            return obj.item()
        elif isinstance(obj, nx.Graph) or isinstance(obj, nx.DiGraph):
            if nx.is_tree(obj):
                # NOTE: the root must be the first node added. otherwise it won't work
                return json_graph.tree_data(obj, obj.nodes_iter().next())
            else:
                return json_graph.node_link_data(obj)
        elif isinstance(obj, pd.DataFrame):
            return obj.to_dict(orient='records')
        return json.JSONEncoder.default(self, obj)

_dump_json = lambda x: json.dumps(x, cls=MattaJSONEncoder)

class sketch(object):
    def __init__(self, **kwargs):
        if not 'container_type' in kwargs:
            raise('need to define a container element')

        if not 'data' in kwargs or kwargs['data'] is None:
            raise Exception('you need to define at least one data variable')

        self.configuration = kwargs.copy()

        if not 'visualization_css' in kwargs:
            self.configuration['visualization_css'] =  '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
        if not 'visualization_js' in kwargs:
            self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])

        self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
        self.configuration['variables'] = valmap(_dump_json, self.configuration['variables'])
        self.configuration['__data_variables__'] = self.configuration['data'].keys()
        self.configuration['data'] = _dump_json(self.configuration['data'])

    def _render_(self, template_name):
        repr_args = self.configuration.copy()

        if self.configuration['visualization_css']:
            try:
                repr_args['visualization_css'] = env.get_template(self.configuration['visualization_css']).render(**repr_args)
            except IOError:
                repr_args['visualization_css'] = None

        if self.configuration['visualization_js']:
            repr_args['visualization_js'] = env.get_template(self.configuration['visualization_js']).render(**repr_args)
        else:
            raise Exception('Empty Visualization code!')

        template = env.get_template(template_name)

        if not 'figure_id' in repr_args or not repr_args['figure_id']:
            repr_args['figure_id'] = 'fig-{0}'.format(uuid.uuid4())

        if not 'vis_uuid' in repr_args or not repr_args['vis_uuid']:
            repr_args['vis_uuid'] = 'matta-vis-{0}'.format(uuid.uuid4())

        return template.render(**repr_args)

    def _ipython_display_(self):
        rendered = self._render_('base.html')
        display_html(HTML(rendered))

    def scaffold(self, filename=None, style=None, append=False):
        rendered = self._render_('scaffold.js')

        if filename is None:
            return rendered

        with open(filename, 'w') as f:
            f.write(rendered)

        if style is not None:
            mode = 'a' if append is True else 'w'
            with open(style, mode) as f:
                f.write(env.get_template(self.configuration['visualization_css']).render(**self.configuration))

def build_sketch(defaults, opt_process=None):
    def sketch_fn(**kwargs):
        sketch_args = copy.deepcopy(defaults)

        for key, value in kwargs.iteritems():
            if key in sketch_args:
                sketch_args[key] = value
            if key in sketch_args['data']:
                sketch_args['data'][key] = value
            elif key in sketch_args['options']:
                if type(sketch_args['options'][key]) == dict:
                    sketch_args['options'][key].update(value)
                else:
                    sketch_args['options'][key] = value
            elif key in sketch_args['variables']:
                if type(sketch_args['variables'][key]) == dict:
                    sketch_args['variables'][key].update(value)
                else:
                    sketch_args['variables'][key] = value
            elif key == 'figure_id':
                sketch_args[key] = value
            else:
                raise Exception('invalid argument: {0}'.format(key))

        if callable(opt_process):
            opt_process(sketch_args)

        return sketch(**sketch_args)

    return sketch_fn<|endoftext|>


<commit_before>#! /usr/bin/env python

"""

Copyright [1999-2016] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License")
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

"""

	Please email comments or questions to the public Ensembl
	developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

	Questions may also be sent to the Ensembl help desk at
	<http://www.ensembl.org/Help/Contact>.

"""
import os.path
import pybedtools

def overlap_snps_to_bed(snps, bed):
	'''
		Find overlaps between SNP elements and annotated Bed file
		Args:
		* [ SNP ]
		* string (location of bed file)
		Returntype: pybedtools Interval iterator

	'''
	snps = list(snps)

	if (len(snps)==0):
		return []

	max_pos = max([x.pos + 1 for x in snps])
	min_pos = min([x.pos for x in snps])
	chrom = snps[0].chrom
	SNP_bt = snps_to_bt(snps)
	Annotation_bt_indexed = bed_to_bt_indexed(bed)

	intersection = Annotation_bt_indexed.tabix_intervals('{}:{}-{}'.format(chrom,min_pos,max_pos)).intersect(SNP_bt, wa=True, wb=True)
	return intersection

def closest(snps, bed):
	SNP_string = "\n".join("\t".join((snp.chrom, str(snp.pos), str(snp.pos+1), snp.rsID)) for snp in sorted(snps, key=lambda X: (X.chrom, X.pos)))
	SNP_bt = pybedtools.BedTool(SNP_string, from_string=True)
	Annotation_bt_indexed = bed_to_bt_indexed(bed)
	return SNP_bt.closest(Annotation_bt_indexed, wa=True, wb=True)

def snps_to_bt(snps):
	SNP_string = "\n".join(( "\t".join((snp.chrom, str(snp.pos), str(snp.pos+1), snp.rsID)) for snp in snps ))
	return pybedtools.BedTool(SNP_string, from_string=True)

def bed_to_bt_indexed(bed):
	if not os.path.isfile(bed + '.gz') or not os.path.isfile(bed + '.gz.tbi'):
		Annotation_bt = pybedtools.BedTool(bed)
		return Annotation_bt.tabix()
	else:
		return pybedtools.BedTool(bed + '.gz')
<commit_msg>Take into account difference between Ensembl 1-based and UCSC 0-based coords<commit_after>#! /usr/bin/env python

"""

Copyright [1999-2016] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License")
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

"""

	Please email comments or questions to the public Ensembl
	developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

	Questions may also be sent to the Ensembl help desk at
	<http://www.ensembl.org/Help/Contact>.

"""
import os.path
import pybedtools

def overlap_snps_to_bed(snps, bed):
	'''
		Find overlaps between SNP elements and annotated Bed file
		Args:
		* [ SNP ]
		* string (location of bed file)
		Returntype: pybedtools Interval iterator

	'''
	snps = list(snps)

	if (len(snps)==0):
		return []

	max_pos = max([x.pos for x in snps])
	min_pos = min([x.pos-1 for x in snps])
	chrom = snps[0].chrom
	SNP_bt = snps_to_bt(snps)
	Annotation_bt_indexed = bed_to_bt_indexed(bed)

	intersection = Annotation_bt_indexed.tabix_intervals('{}:{}-{}'.format(chrom,min_pos,max_pos)).intersect(SNP_bt, wa=True, wb=True)
	return intersection

def closest(snps, bed):
	SNP_string = "\n".join("\t".join((snp.chrom, str(snp.pos-1), str(snp.pos), snp.rsID)) for snp in sorted(snps, key=lambda X: (X.chrom, X.pos)))
	SNP_bt = pybedtools.BedTool(SNP_string, from_string=True)
	Annotation_bt_indexed = bed_to_bt_indexed(bed)
	return SNP_bt.closest(Annotation_bt_indexed, wa=True, wb=True)

def snps_to_bt(snps):
	SNP_string = "\n".join(( "\t".join((snp.chrom, str(snp.pos-1), str(snp.pos), snp.rsID)) for snp in snps ))
	return pybedtools.BedTool(SNP_string, from_string=True)

def bed_to_bt_indexed(bed):
	if not os.path.isfile(bed + '.gz') or not os.path.isfile(bed + '.gz.tbi'):
		Annotation_bt = pybedtools.BedTool(bed)
		return Annotation_bt.tabix()
	else:
		return pybedtools.BedTool(bed + '.gz')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""ROS has a parameter server, we have files.

The parameter store is a persistent key value store, implemented as a directory with a writer lock.
On Android, we store params under params_dir = /data/params. The writer lock is a file
"<params_dir>/.lock" taken using flock(), and data is stored in a directory symlinked to by
"<params_dir>/d".

Each key, value pair is stored as a file with named <key> with contents <value>, located in
  <params_dir>/d/<key>

Readers of a single key can just open("<params_dir>/d/<key>") and read the file contents.
Readers who want a consistent snapshot of multiple keys should take the lock.

Writers should take the lock before modifying anything. Writers should also leave the DB in a
consistent state after a crash. The implementation below does this by copying all params to a temp
directory <params_dir>/<tmp>, then atomically symlinking <params_dir>/<d> to <params_dir>/<tmp>
before deleting the old <params_dir>/<d> directory.

Writers that only modify a single key can simply take the lock, then swap the corresponding value
file in place without messing with <params_dir>/d.
"""
import time
import os
import errno
import shutil
import fcntl
import tempfile
import threading
from enum import Enum
from common.basedir import PARAMS

def mkdirs_exists_ok(path):
  try:
    os.makedirs(path)
  except OSError:
    if not os.path.isdir(path):
      raise

class TxType(Enum):
  PERSISTENT = 1
  CLEAR_ON_MANAGER_START = 2
  CLEAR_ON_PANDA_DISCONNECT = 3

class UnknownKeyName(Exception):
  pass

keys = {
  "AccessToken": [TxType.CLEAR_ON_MANAGER_START],
  "AthenadPid": [TxType.PERSISTENT],
  "CalibrationParams": [TxType.PERSISTENT],
  "CarParams": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "CarParamsCache": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "CarVin": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "CommunityFeaturesToggle": [TxType.PERSISTENT],
  "CompletedTrainingVersion": [TxType.PERSISTENT],
  "ControlsParams": [TxType.PERSISTENT],
  "DisablePowerDown": [TxType.PERSISTENT],
  "DisableUpdates": [TxType.PERSISTENT],
  "DoUninstall": [TxType.CLEAR_ON_MANAGER_START],
  "DongleId": [TxType.PERSISTENT],
  "GitBranch": [TxType.PERSISTENT],
  "GitCommit": [TxType.PERSISTENT],
  "GitRemote": [TxType.PERSISTENT],
  "GithubSshKeys": [TxType.PERSISTENT],
  "HasAcceptedTerms": [TxType.PERSISTENT],
  "HasCompletedSetup": [TxType.PERSISTENT],
  "IsDriverViewEnabled": [TxType.CLEAR_ON_MANAGER_START],
  "IsLdwEnabled": [TxType.PERSISTENT],
  "IsGeofenceEnabled": [TxType.PERSISTENT],
  "IsMetric": [TxType.PERSISTENT],
  "IsOffroad": [TxType.CLEAR_ON_MANAGER_START],
  "IsRHD": [TxType.PERSISTENT],
  "IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
  "IsUpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
  "IsUploadRawEnabled": [TxType.PERSISTENT],
  "LastAthenaPingTime": [TxType.PERSISTENT],
  "LastUpdateTime": [TxType.PERSISTENT],
  "LimitSetSpeed": [TxType.PERSISTENT],
  "LimitSetSpeedNeural": [TxType.PERSISTENT],
  "LiveParameters": [TxType.PERSISTENT],
  "LongitudinalControl": [TxType.PERSISTENT],
  "OpenpilotEnabledToggle": [TxType.PERSISTENT],
  "LaneChangeEnabled": [TxType.PERSISTENT],
  "PandaFirmware": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "PandaFirmwareHex": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "PandaDongleId": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "Passive": [TxType.PERSISTENT],
  "RecordFront": [TxType.PERSISTENT],
  "ReleaseNotes": [TxType.PERSISTENT],
  "ShouldDoUpdate": [TxType.CLEAR_ON_MANAGER_START],
  "SpeedLimitOffset": [TxType.PERSISTENT],
  "SubscriberInfo": [TxType.PERSISTENT],
  "TermsVersion": [TxType.PERSISTENT],
  "TrainingVersion": [TxType.PERSISTENT],
  "UpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
  "UpdateFailedCount": [TxType.CLEAR_ON_MANAGER_START],
  "Version": [TxType.PERSISTENT],
  "Offroad_ChargeDisabled": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "Offroad_ConnectivityNeeded": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_ConnectivityNeededPrompt": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_TemperatureTooHigh": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_PandaFirmwareMismatch": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "Offroad_InvalidTime": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_NeosUpdate": [TxType.CLEAR_ON_MANAGER_START],
}

def fsync_dir(path):
  fd = os.open(path, os.O_RDONLY)
  try:
    os.fsync(fd)
  finally:
    os.close(fd)

class FileLock():
  def __init__(self, path, create):
    self._path = path
    self._create = create
    self._fd = None

  def acquire(self):
    self._fd = os.open(self._path, os.O_CREAT if self._create else 0)
    fcntl.flock(self._fd, fcntl.LOCK_EX)

  def release(self):
    if self._fd is not None:
      os.close(self._fd)
      self._fd = None

class DBAccessor():
  def __init__(self, path):
    self._path = path
    self._vals = None

  def keys(self):
    self._check_entered()
    return self._vals.keys()

  def get(self, key):
    self._check_entered()

    if self._vals is None:
      return None

    try:
      return self._vals[key]
    except KeyError:
      return None

  def _get_lock(self, create):
    lock = FileLock(os.path.join(self._path, ".lock"), create)
    lock.acquire()
    return lock

  def _read_values_locked(self):
    """Callers should hold a lock while calling this method."""
    vals = {}
    try:
      data_path = self._data_path()
      keys = os.listdir(data_path)
      for key in keys:
        with open(os.path.join(data_path, key), "rb") as f:
          vals[key] = f.read()
    except (OSError, IOError) as e:
      # Either the DB hasn't been created yet, or somebody wrote a bug and left the DB in an
      # inconsistent state. Either way, return empty.
      if e.errno == errno.ENOENT:
        return {}

    return vals

  def _data_path(self):
    return os.path.join(self._path, "d")

  def _check_entered(self):
    if self._vals is None:
      raise Exception("Must call __enter__ before using DB")

class DBReader(DBAccessor):
  def __enter__(self):
    try:
      lock = self._get_lock(False)
    except OSError as e:
      # Do not create lock if it does not exist.
      if e.errno == errno.ENOENT:
        self._vals = {}
        return self

    try:
      # Read everything.
      self._vals = self._read_values_locked()
      return self
    finally:
      lock.release()

  def __exit__(self, exc_type, exc_value, traceback):
    pass

class DBWriter(DBAccessor):
  def __init__(self, path):
    super(DBWriter, self).__init__(path)
    self._lock = None
    self._prev_umask = None

  def put(self, key, value):
    self._vals[key] = value

  def delete(self, key):
    self._vals.pop(key, None)

  def __enter__(self):
    mkdirs_exists_ok(self._path)

    # Make sure we can write and that permissions are correct.
    self._prev_umask = os.umask(0)

    try:
      os.chmod(self._path, 0o777)
      self._lock = self._get_lock(True)
      self._vals = self._read_values_locked()
    except Exception:
      os.umask(self._prev_umask)
      self._prev_umask = None
      raise

    return self

  def __exit__(self, exc_type, exc_value, traceback):
    self._check_entered()

    try:
      # data_path refers to the externally used path to the params. It is a symlink.
      # old_data_path is the path currently pointed to by data_path.
      # tempdir_path is a path where the new params will go, which the new data path will point to.
      # new_data_path is a temporary symlink that will atomically overwrite data_path.
      #
      # The current situation is:
      #   data_path -> old_data_path
      # We're going to write params data to tempdir_path
      #   tempdir_path -> params data
      # Then point new_data_path to tempdir_path
      #   new_data_path -> tempdir_path
      # Then atomically overwrite data_path with new_data_path
      #   data_path -> tempdir_path
      old_data_path = None
      new_data_path = None
      tempdir_path = tempfile.mkdtemp(prefix=".tmp", dir=self._path)

      try:
        # Write back all keys.
        os.chmod(tempdir_path, 0o777)
        for k, v in self._vals.items():
          with open(os.path.join(tempdir_path, k), "wb") as f:
            f.write(v)
            f.flush()
            os.fsync(f.fileno())
        fsync_dir(tempdir_path)

        data_path = self._data_path()
        try:
          old_data_path = os.path.join(self._path, os.readlink(data_path))
        except (OSError, IOError):
          # NOTE(mgraczyk): If other DB implementations have bugs, this could cause
          #                 copies to be left behind, but we still want to overwrite.
          pass

        new_data_path = "{}.link".format(tempdir_path)
        os.symlink(os.path.basename(tempdir_path), new_data_path)
        os.rename(new_data_path, data_path)
        fsync_dir(self._path)
      finally:
        # If the rename worked, we can delete the old data. Otherwise delete the new one.
        success = new_data_path is not None and os.path.exists(data_path) and (
          os.readlink(data_path) == os.path.basename(tempdir_path))

        if success:
          if old_data_path is not None:
            shutil.rmtree(old_data_path)
        else:
          shutil.rmtree(tempdir_path)

        # Regardless of what happened above, there should be no link at new_data_path.
        if new_data_path is not None and os.path.islink(new_data_path):
          os.remove(new_data_path)
    finally:
      os.umask(self._prev_umask)
      self._prev_umask = None

      # Always release the lock.
      self._lock.release()
      self._lock = None

def read_db(params_path, key):
  path = "%s/d/%s" % (params_path, key)
  try:
    with open(path, "rb") as f:
      return f.read()
  except IOError:
    return None

def write_db(params_path, key, value):
  if isinstance(value, str):
    value = value.encode('utf8')

  prev_umask = os.umask(0)
  lock = FileLock(params_path + "/.lock", True)
  lock.acquire()

  try:
    tmp_path = tempfile.mktemp(prefix=".tmp", dir=params_path)
    with open(tmp_path, "wb") as f:
      f.write(value)
      f.flush()
      os.fsync(f.fileno())

    path = "%s/d/%s" % (params_path, key)
    os.rename(tmp_path, path)
    fsync_dir(os.path.dirname(path))
  finally:
    os.umask(prev_umask)
    lock.release()

class Params():
  def __init__(self, db=PARAMS):
    self.db = db

    # create the database if it doesn't exist...
    if not os.path.exists(self.db + "/d"):
      with self.transaction(write=True):
        pass

  def clear_all(self):
    shutil.rmtree(self.db, ignore_errors=True)
    with self.transaction(write=True):
      pass

  def transaction(self, write=False):
    if write:
      return DBWriter(self.db)
    else:
      return DBReader(self.db)

  def _clear_keys_with_type(self, tx_type):
    with self.transaction(write=True) as txn:
      for key in keys:
        if tx_type in keys[key]:
          txn.delete(key)

  def manager_start(self):
    self._clear_keys_with_type(TxType.CLEAR_ON_MANAGER_START)

  def panda_disconnect(self):
    self._clear_keys_with_type(TxType.CLEAR_ON_PANDA_DISCONNECT)

  def delete(self, key):
    with self.transaction(write=True) as txn:
      txn.delete(key)

  def get(self, key, block=False, encoding=None):
    if key not in keys:
      raise UnknownKeyName(key)

    while 1:
      ret = read_db(self.db, key)
      if not block or ret is not None:
        break
      # is polling really the best we can do?
      time.sleep(0.05)

    if ret is not None and encoding is not None:
      ret = ret.decode(encoding)

    return ret

  def put(self, key, dat):
    """
    Warning: This function blocks until the param is written to disk!
    In very rare cases this can take over a second, and your code will hang.

    Use the put_nonblocking helper function in time sensitive code, but
    in general try to avoid writing params as much as possible.
    """

    if key not in keys:
      raise UnknownKeyName(key)

    write_db(self.db, key, dat)

def put_nonblocking(key, val):
  def f(key, val):
    params = Params()
    params.put(key, val)

  t = threading.Thread(target=f, args=(key, val))
  t.start()
  return t
<commit_msg>Fix insecure temporary file creation (#1890)<commit_after>#!/usr/bin/env python3
"""ROS has a parameter server, we have files.

The parameter store is a persistent key value store, implemented as a directory with a writer lock.
On Android, we store params under params_dir = /data/params. The writer lock is a file
"<params_dir>/.lock" taken using flock(), and data is stored in a directory symlinked to by
"<params_dir>/d".

Each key, value pair is stored as a file with named <key> with contents <value>, located in
  <params_dir>/d/<key>

Readers of a single key can just open("<params_dir>/d/<key>") and read the file contents.
Readers who want a consistent snapshot of multiple keys should take the lock.

Writers should take the lock before modifying anything. Writers should also leave the DB in a
consistent state after a crash. The implementation below does this by copying all params to a temp
directory <params_dir>/<tmp>, then atomically symlinking <params_dir>/<d> to <params_dir>/<tmp>
before deleting the old <params_dir>/<d> directory.

Writers that only modify a single key can simply take the lock, then swap the corresponding value
file in place without messing with <params_dir>/d.
"""
import time
import os
import errno
import shutil
import fcntl
import tempfile
import threading
from enum import Enum
from common.basedir import PARAMS

def mkdirs_exists_ok(path):
  try:
    os.makedirs(path)
  except OSError:
    if not os.path.isdir(path):
      raise

class TxType(Enum):
  PERSISTENT = 1
  CLEAR_ON_MANAGER_START = 2
  CLEAR_ON_PANDA_DISCONNECT = 3

class UnknownKeyName(Exception):
  pass

keys = {
  "AccessToken": [TxType.CLEAR_ON_MANAGER_START],
  "AthenadPid": [TxType.PERSISTENT],
  "CalibrationParams": [TxType.PERSISTENT],
  "CarParams": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "CarParamsCache": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "CarVin": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "CommunityFeaturesToggle": [TxType.PERSISTENT],
  "CompletedTrainingVersion": [TxType.PERSISTENT],
  "ControlsParams": [TxType.PERSISTENT],
  "DisablePowerDown": [TxType.PERSISTENT],
  "DisableUpdates": [TxType.PERSISTENT],
  "DoUninstall": [TxType.CLEAR_ON_MANAGER_START],
  "DongleId": [TxType.PERSISTENT],
  "GitBranch": [TxType.PERSISTENT],
  "GitCommit": [TxType.PERSISTENT],
  "GitRemote": [TxType.PERSISTENT],
  "GithubSshKeys": [TxType.PERSISTENT],
  "HasAcceptedTerms": [TxType.PERSISTENT],
  "HasCompletedSetup": [TxType.PERSISTENT],
  "IsDriverViewEnabled": [TxType.CLEAR_ON_MANAGER_START],
  "IsLdwEnabled": [TxType.PERSISTENT],
  "IsGeofenceEnabled": [TxType.PERSISTENT],
  "IsMetric": [TxType.PERSISTENT],
  "IsOffroad": [TxType.CLEAR_ON_MANAGER_START],
  "IsRHD": [TxType.PERSISTENT],
  "IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
  "IsUpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
  "IsUploadRawEnabled": [TxType.PERSISTENT],
  "LastAthenaPingTime": [TxType.PERSISTENT],
  "LastUpdateTime": [TxType.PERSISTENT],
  "LimitSetSpeed": [TxType.PERSISTENT],
  "LimitSetSpeedNeural": [TxType.PERSISTENT],
  "LiveParameters": [TxType.PERSISTENT],
  "LongitudinalControl": [TxType.PERSISTENT],
  "OpenpilotEnabledToggle": [TxType.PERSISTENT],
  "LaneChangeEnabled": [TxType.PERSISTENT],
  "PandaFirmware": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "PandaFirmwareHex": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "PandaDongleId": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "Passive": [TxType.PERSISTENT],
  "RecordFront": [TxType.PERSISTENT],
  "ReleaseNotes": [TxType.PERSISTENT],
  "ShouldDoUpdate": [TxType.CLEAR_ON_MANAGER_START],
  "SpeedLimitOffset": [TxType.PERSISTENT],
  "SubscriberInfo": [TxType.PERSISTENT],
  "TermsVersion": [TxType.PERSISTENT],
  "TrainingVersion": [TxType.PERSISTENT],
  "UpdateAvailable": [TxType.CLEAR_ON_MANAGER_START],
  "UpdateFailedCount": [TxType.CLEAR_ON_MANAGER_START],
  "Version": [TxType.PERSISTENT],
  "Offroad_ChargeDisabled": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "Offroad_ConnectivityNeeded": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_ConnectivityNeededPrompt": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_TemperatureTooHigh": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_PandaFirmwareMismatch": [TxType.CLEAR_ON_MANAGER_START, TxType.CLEAR_ON_PANDA_DISCONNECT],
  "Offroad_InvalidTime": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_IsTakingSnapshot": [TxType.CLEAR_ON_MANAGER_START],
  "Offroad_NeosUpdate": [TxType.CLEAR_ON_MANAGER_START],
}

def fsync_dir(path):
  fd = os.open(path, os.O_RDONLY)
  try:
    os.fsync(fd)
  finally:
    os.close(fd)

class FileLock():
  def __init__(self, path, create):
    self._path = path
    self._create = create
    self._fd = None

  def acquire(self):
    self._fd = os.open(self._path, os.O_CREAT if self._create else 0)
    fcntl.flock(self._fd, fcntl.LOCK_EX)

  def release(self):
    if self._fd is not None:
      os.close(self._fd)
      self._fd = None

class DBAccessor():
  def __init__(self, path):
    self._path = path
    self._vals = None

  def keys(self):
    self._check_entered()
    return self._vals.keys()

  def get(self, key):
    self._check_entered()

    if self._vals is None:
      return None

    try:
      return self._vals[key]
    except KeyError:
      return None

  def _get_lock(self, create):
    lock = FileLock(os.path.join(self._path, ".lock"), create)
    lock.acquire()
    return lock

  def _read_values_locked(self):
    """Callers should hold a lock while calling this method."""
    vals = {}
    try:
      data_path = self._data_path()
      keys = os.listdir(data_path)
      for key in keys:
        with open(os.path.join(data_path, key), "rb") as f:
          vals[key] = f.read()
    except (OSError, IOError) as e:
      # Either the DB hasn't been created yet, or somebody wrote a bug and left the DB in an
      # inconsistent state. Either way, return empty.
      if e.errno == errno.ENOENT:
        return {}

    return vals

  def _data_path(self):
    return os.path.join(self._path, "d")

  def _check_entered(self):
    if self._vals is None:
      raise Exception("Must call __enter__ before using DB")

class DBReader(DBAccessor):
  def __enter__(self):
    try:
      lock = self._get_lock(False)
    except OSError as e:
      # Do not create lock if it does not exist.
      if e.errno == errno.ENOENT:
        self._vals = {}
        return self

    try:
      # Read everything.
      self._vals = self._read_values_locked()
      return self
    finally:
      lock.release()

  def __exit__(self, exc_type, exc_value, traceback):
    pass

class DBWriter(DBAccessor):
  def __init__(self, path):
    super(DBWriter, self).__init__(path)
    self._lock = None
    self._prev_umask = None

  def put(self, key, value):
    self._vals[key] = value

  def delete(self, key):
    self._vals.pop(key, None)

  def __enter__(self):
    mkdirs_exists_ok(self._path)

    # Make sure we can write and that permissions are correct.
    self._prev_umask = os.umask(0)

    try:
      os.chmod(self._path, 0o777)
      self._lock = self._get_lock(True)
      self._vals = self._read_values_locked()
    except Exception:
      os.umask(self._prev_umask)
      self._prev_umask = None
      raise

    return self

  def __exit__(self, exc_type, exc_value, traceback):
    self._check_entered()

    try:
      # data_path refers to the externally used path to the params. It is a symlink.
      # old_data_path is the path currently pointed to by data_path.
      # tempdir_path is a path where the new params will go, which the new data path will point to.
      # new_data_path is a temporary symlink that will atomically overwrite data_path.
      #
      # The current situation is:
      #   data_path -> old_data_path
      # We're going to write params data to tempdir_path
      #   tempdir_path -> params data
      # Then point new_data_path to tempdir_path
      #   new_data_path -> tempdir_path
      # Then atomically overwrite data_path with new_data_path
      #   data_path -> tempdir_path
      old_data_path = None
      new_data_path = None
      tempdir_path = tempfile.mkdtemp(prefix=".tmp", dir=self._path)

      try:
        # Write back all keys.
        os.chmod(tempdir_path, 0o777)
        for k, v in self._vals.items():
          with open(os.path.join(tempdir_path, k), "wb") as f:
            f.write(v)
            f.flush()
            os.fsync(f.fileno())
        fsync_dir(tempdir_path)

        data_path = self._data_path()
        try:
          old_data_path = os.path.join(self._path, os.readlink(data_path))
        except (OSError, IOError):
          # NOTE(mgraczyk): If other DB implementations have bugs, this could cause
          #                 copies to be left behind, but we still want to overwrite.
          pass

        new_data_path = "{}.link".format(tempdir_path)
        os.symlink(os.path.basename(tempdir_path), new_data_path)
        os.rename(new_data_path, data_path)
        fsync_dir(self._path)
      finally:
        # If the rename worked, we can delete the old data. Otherwise delete the new one.
        success = new_data_path is not None and os.path.exists(data_path) and (
          os.readlink(data_path) == os.path.basename(tempdir_path))

        if success:
          if old_data_path is not None:
            shutil.rmtree(old_data_path)
        else:
          shutil.rmtree(tempdir_path)

        # Regardless of what happened above, there should be no link at new_data_path.
        if new_data_path is not None and os.path.islink(new_data_path):
          os.remove(new_data_path)
    finally:
      os.umask(self._prev_umask)
      self._prev_umask = None

      # Always release the lock.
      self._lock.release()
      self._lock = None

def read_db(params_path, key):
  path = "%s/d/%s" % (params_path, key)
  try:
    with open(path, "rb") as f:
      return f.read()
  except IOError:
    return None

def write_db(params_path, key, value):
  if isinstance(value, str):
    value = value.encode('utf8')

  prev_umask = os.umask(0)
  lock = FileLock(params_path + "/.lock", True)
  lock.acquire()

  try:
    tmp_path = tempfile.NamedTemporaryFile(mode="wb", prefix=".tmp", dir=params_path, delete=False)
    with tmp_path as f:
      f.write(value)
      f.flush()
      os.fsync(f.fileno())

    path = "%s/d/%s" % (params_path, key)
    os.rename(tmp_path.name, path)
    fsync_dir(os.path.dirname(path))
  finally:
    os.umask(prev_umask)
    lock.release()

class Params():
  def __init__(self, db=PARAMS):
    self.db = db

    # create the database if it doesn't exist...
    if not os.path.exists(self.db + "/d"):
      with self.transaction(write=True):
        pass

  def clear_all(self):
    shutil.rmtree(self.db, ignore_errors=True)
    with self.transaction(write=True):
      pass

  def transaction(self, write=False):
    if write:
      return DBWriter(self.db)
    else:
      return DBReader(self.db)

  def _clear_keys_with_type(self, tx_type):
    with self.transaction(write=True) as txn:
      for key in keys:
        if tx_type in keys[key]:
          txn.delete(key)

  def manager_start(self):
    self._clear_keys_with_type(TxType.CLEAR_ON_MANAGER_START)

  def panda_disconnect(self):
    self._clear_keys_with_type(TxType.CLEAR_ON_PANDA_DISCONNECT)

  def delete(self, key):
    with self.transaction(write=True) as txn:
      txn.delete(key)

  def get(self, key, block=False, encoding=None):
    if key not in keys:
      raise UnknownKeyName(key)

    while 1:
      ret = read_db(self.db, key)
      if not block or ret is not None:
        break
      # is polling really the best we can do?
      time.sleep(0.05)

    if ret is not None and encoding is not None:
      ret = ret.decode(encoding)

    return ret

  def put(self, key, dat):
    """
    Warning: This function blocks until the param is written to disk!
    In very rare cases this can take over a second, and your code will hang.

    Use the put_nonblocking helper function in time sensitive code, but
    in general try to avoid writing params as much as possible.
    """

    if key not in keys:
      raise UnknownKeyName(key)

    write_db(self.db, key, dat)

def put_nonblocking(key, val):
  def f(key, val):
    params = Params()
    params.put(key, val)

  t = threading.Thread(target=f, args=(key, val))
  t.start()
  return t
<|endoftext|>