<commit_before>import shutil, tempfile, zipfile, os.path

from daisyproducer.documents.external import DaisyPipeline, SBSForm, StandardLargePrint
from daisyproducer.documents.forms import PartialDocumentForm, PartialVersionForm, PartialAttachmentForm, OCRForm, MarkupForm, SBSFormForm, RTFForm, EPUBForm, TextOnlyFilesetForm, DTBForm, SalePDFForm
from daisyproducer.documents.models import Document, Version, Attachment, LargePrintProfileForm
from daisyproducer.documents.views.utils import render_to_mimetype_response
from django.contrib.auth.decorators import login_required
from django.core.files.base import ContentFile
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models import Q
from django.forms.models import model_to_dict
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.views.generic.list_detail import object_list
from lxml import etree

@login_required
def index(request):
    """Show all the documents that are relevant for the groups that
    the user has and group them by action"""
    # FIXME: this can be simplified in a more modern version of django
    # (see http://docs.djangoproject.com/en/dev/ref/models/querysets/#in)
    # to the following:
    # user_groups = request.user.groups
    user_groups = request.user.groups.values('pk').query
    response = object_list(
        request,
        queryset = Document.objects.select_related('state').filter(
            # only show documents that aren't assigned to anyone else,
            Q(assigned_to=request.user) | Q(assigned_to__isnull=True),
            # and which are in a state for which my group is responsible
            state__responsible__in=user_groups
            ).order_by('state','title'),
        template_name = 'documents/todo_index.html'
    )
    return response

@login_required
def detail(request, document_id):
    try:
        document = Document.objects.select_related('state').get(pk=document_id)
    except Document.DoesNotExist:
        raise Http404

    versionForm = PartialVersionForm()
    attachmentForm = PartialAttachmentForm()
    documentForm = PartialDocumentForm()
    documentForm.limitChoicesToValidStates(document)
    return render_to_response('documents/todo_detail.html', locals(),
                              context_instance=RequestContext(request))

@login_required
@transaction.commit_on_success
def add_attachment(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method != 'POST':
        return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

    form = PartialAttachmentForm(request.POST, request.FILES)
    if not form.is_valid():
        versionForm = PartialVersionForm()
        attachmentForm = form
        documentForm = PartialDocumentForm()
        documentForm.limitChoicesToValidStates(document)
        return render_to_response('documents/todo_detail.html', locals(),
                                  context_instance=RequestContext(request))

    # this is a bit of a hack as we need to create (and save) an
    # attachment before the id is known. We need to know the id before we
    # can save the content file under /document_id/attachments/file_name
    attachment = Attachment.objects.create(
        comment=form.cleaned_data['comment'],
        document=document,
        mime_type=form.content_type,
        created_by=request.user)
    content_file = request.FILES['content']
    attachment.content.save(content_file.name, content_file)

    return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

@login_required
@transaction.commit_on_success
def add_version(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method != 'POST':
        return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

    form = PartialVersionForm(request.POST, request.FILES,
                              contentMetaData=model_to_dict(document))
    if not form.is_valid():
        versionForm = form
        attachmentForm = PartialAttachmentForm()
        documentForm = PartialDocumentForm()
        documentForm.limitChoicesToValidStates(document)
        return render_to_response('documents/todo_detail.html', locals(),
                                  context_instance=RequestContext(request))

    # this is a bit of a hack as we need to create (and save) a
    # version before the id is known. We need to know the id before we
    # can save the content file under /document_id/versions/version_id
    version = Version.objects.create(
        comment=form.cleaned_data['comment'],
        document=document,
        created_by=request.user)
    content_file = request.FILES['content']
    version.content.save(content_file.name, content_file)

    return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

@login_required
def transition(request, document_id):
    document = Document.objects.get(pk=document_id)

    if request.method != 'POST':
        return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

    form = PartialDocumentForm(request.POST)
    if not form.is_valid():
        versionForm = PartialVersionForm()
        attachmentForm = PartialAttachmentForm()
        documentForm = form
        return render_to_response('documents/todo_detail.html', locals(),
                                  context_instance=RequestContext(request))

    document.transitionTo(form.cleaned_data['state'])
    return HttpResponseRedirect(reverse('todo_index'))

@login_required
def ocr(request, document_id):
    document = Document.objects.get(pk=document_id)
    if request.method == 'POST':
        form = OCRForm(request.POST, request.FILES)
        if form.is_valid():
            # do whatever is needed here
            return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))
    else:
        form = OCRForm()

    return render_to_response('documents/todo_ocr.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def markup(request, document_id):
    document = Document.objects.select_related('state').get(pk=document_id)
    if request.method == 'POST':
        form = MarkupForm(request.POST,
                          document=document, user=request.user)
        if form.is_valid():
            # create a new version with the given form content
            form.save()
            return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))
        else:
            form.has_errors = True
    else:
        file_field = document.latest_version().content
        file_field.open()
        content = file_field.read()
        file_field.close()
        form = MarkupForm({'data' : content, 'comment' : "Insert a comment here"})
        # This is a bit of a cheap hack to avoid expensive validation
        # on initial display of the form
        form.has_errors = False

    return render_to_response('documents/todo_markup.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_xhtml(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    inputFile = document.latest_version().content.path
    outputFile = "/tmp/%s.xhtml" % document_id
    params = {}
    DaisyPipeline.dtbook2xhtml(inputFile, outputFile, **params)

    return render_to_mimetype_response('text/html',
                                       document.title.encode('utf-8'), outputFile)

@login_required
def preview_sbsform(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = SBSFormForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.sbsform" % document_id
            SBSForm.dtbook2sbsform(inputFile, outputFile, **form.cleaned_data)
            contraction = form.cleaned_data['contraction']
            contraction_to_mimetype_mapping = {0 : 'text/x-sbsform-g0',
                                               1 : 'text/x-sbsform-g1',
                                               2 : 'text/x-sbsform-g2'}
            mimetype = contraction_to_mimetype_mapping.get(contraction, 'text/x-sbsform-g2')
            return render_to_mimetype_response(mimetype, document.title.encode('utf-8'), outputFile)
    else:
        form = SBSFormForm()

    return render_to_response('documents/todo_sbsform.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_pdf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = LargePrintProfileForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.pdf" % document_id
            DaisyPipeline.dtbook2pdf(inputFile, outputFile, **form.cleaned_data)
            return render_to_mimetype_response('application/pdf',
                                               document.title.encode('utf-8'), outputFile)
    else:
        form = LargePrintProfileForm()

    return render_to_response('documents/todo_pdf.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_library_pdf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    inputFile = document.latest_version().content.path
    outputFile = "/tmp/%s.pdf" % document_id
    StandardLargePrint.dtbook2pdf(inputFile, outputFile)
    filename = document.title.encode('utf-8') + " 17pt"
    return render_to_mimetype_response('application/pdf', filename, outputFile)

@login_required
def preview_sale_pdf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = SalePDFForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.pdf" % document_id
            StandardLargePrint.dtbook2pdf(inputFile, outputFile, **form.cleaned_data)
            filename = document.title.encode('utf-8') + " " + form.cleaned_data['font_size']
            return render_to_mimetype_response('application/pdf', filename, outputFile)
    else:
        form = SalePDFForm()

    return render_to_response('documents/todo_sale_pdf.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_rtf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = RTFForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.rtf" % document_id
            DaisyPipeline.dtbook2rtf(inputFile, outputFile, **form.cleaned_data)
            return render_to_mimetype_response('application/rtf',
                                               document.title.encode('utf-8'), outputFile)
    else:
        form = RTFForm()

    return render_to_response('documents/todo_rtf.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_epub(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = EPUBForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.epub" % document_id
            defaults = {
                "dctitle" : document.title,
                "dcidentifier" : document.identifier,
                "dclanguage" : document.language,
                "dccreator" : document.author,
                "dcpublisher" : document.publisher,
                "dcdate" : document.date}
            defaults.update(form.cleaned_data)
            DaisyPipeline.dtbook2epub(inputFile, outputFile, **defaults)
            return render_to_mimetype_response('application/epub+zip',
                                               document.title.encode('utf-8'), outputFile)
    else:
        form = EPUBForm()

    return render_to_response('documents/todo_epub.html', locals(),
                              context_instance=RequestContext(request))

def zipDirectory(directory, zipFileName, document_title):
    outputFile = zipfile.ZipFile(zipFileName, 'w')
    cwd = os.getcwd()
    os.chdir(directory)
    for dirpath, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            # zipFile support in Python has a few weak spots: Older
            # Pythons die if the filename or the arcname that is
            # passed to ZipFile.write is not in the right encoding
            # FIXME: remove the encode("latin-1") workaround once we
            # upgrade to 2.6.2
            outputFile.write(
                os.path.join(dirpath, filename).encode("latin-1"),
                os.path.join(document_title, dirpath, filename).encode("latin-1"))
    outputFile.close()
    os.chdir(cwd)

@login_required
def preview_text_only_fileset(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = TextOnlyFilesetForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputDir = tempfile.mkdtemp(prefix="daisyproducer-")

            DaisyPipeline.dtbook2text_only_fileset(inputFile, outputDir, **form.cleaned_data)

            ignore, zipFileName = tempfile.mkstemp(suffix='.zip', prefix=document_id)
            zipDirectory(outputDir, zipFileName, document.title)
            shutil.rmtree(outputDir)

            return render_to_mimetype_response('application/zip',
                                               document.title.encode('utf-8'), zipFileName)
    else:
        form = TextOnlyFilesetForm()

    return render_to_response('documents/todo_text_only_fileset.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_dtb(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = DTBForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputDir = tempfile.mkdtemp(prefix="daisyproducer-")
            DaisyPipeline.dtbook2dtb(inputFile, outputDir, **form.cleaned_data)

            ignore, zipFileName = tempfile.mkstemp(suffix='.zip', prefix=document_id)
            zipDirectory(outputDir, zipFileName, document.title)
            # shutil.rmtree(outputDir)

            return render_to_mimetype_response('application/zip',
                                               document.title.encode('utf-8'), zipFileName)
    else:
        form = DTBForm()

    return render_to_response('documents/todo_dtb.html', locals(),
                              context_instance=RequestContext(request))

<commit_msg>Fix an UnicodeEncodeError<commit_after>import shutil, tempfile, zipfile, os.path

from daisyproducer.documents.external import DaisyPipeline, SBSForm, StandardLargePrint
from daisyproducer.documents.forms import PartialDocumentForm, PartialVersionForm, PartialAttachmentForm, OCRForm, MarkupForm, SBSFormForm, RTFForm, EPUBForm, TextOnlyFilesetForm, DTBForm, SalePDFForm
from daisyproducer.documents.models import Document, Version, Attachment, LargePrintProfileForm
from daisyproducer.documents.views.utils import render_to_mimetype_response
from django.contrib.auth.decorators import login_required
from django.core.files.base import ContentFile
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models import Q
from django.forms.models import model_to_dict
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.views.generic.list_detail import object_list
from lxml import etree

@login_required
def index(request):
    """Show all the documents that are relevant for the groups that
    the user has and group them by action"""
    # FIXME: this can be simplified in a more modern version of django
    # (see http://docs.djangoproject.com/en/dev/ref/models/querysets/#in)
    # to the following:
    # user_groups = request.user.groups
    user_groups = request.user.groups.values('pk').query
    response = object_list(
        request,
        queryset = Document.objects.select_related('state').filter(
            # only show documents that aren't assigned to anyone else,
            Q(assigned_to=request.user) | Q(assigned_to__isnull=True),
            # and which are in a state for which my group is responsible
            state__responsible__in=user_groups
            ).order_by('state','title'),
        template_name = 'documents/todo_index.html'
    )
    return response

@login_required
def detail(request, document_id):
    try:
        document = Document.objects.select_related('state').get(pk=document_id)
    except Document.DoesNotExist:
        raise Http404

    versionForm = PartialVersionForm()
    attachmentForm = PartialAttachmentForm()
    documentForm = PartialDocumentForm()
    documentForm.limitChoicesToValidStates(document)
    return render_to_response('documents/todo_detail.html', locals(),
                              context_instance=RequestContext(request))

@login_required
@transaction.commit_on_success
def add_attachment(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method != 'POST':
        return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

    form = PartialAttachmentForm(request.POST, request.FILES)
    if not form.is_valid():
        versionForm = PartialVersionForm()
        attachmentForm = form
        documentForm = PartialDocumentForm()
        documentForm.limitChoicesToValidStates(document)
        return render_to_response('documents/todo_detail.html', locals(),
                                  context_instance=RequestContext(request))

    # this is a bit of a hack as we need to create (and save) an
    # attachment before the id is known. We need to know the id before we
    # can save the content file under /document_id/attachments/file_name
    attachment = Attachment.objects.create(
        comment=form.cleaned_data['comment'],
        document=document,
        mime_type=form.content_type,
        created_by=request.user)
    content_file = request.FILES['content']
    attachment.content.save(content_file.name, content_file)

    return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

@login_required
@transaction.commit_on_success
def add_version(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method != 'POST':
        return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

    form = PartialVersionForm(request.POST, request.FILES,
                              contentMetaData=model_to_dict(document))
    if not form.is_valid():
        versionForm = form
        attachmentForm = PartialAttachmentForm()
        documentForm = PartialDocumentForm()
        documentForm.limitChoicesToValidStates(document)
        return render_to_response('documents/todo_detail.html', locals(),
                                  context_instance=RequestContext(request))

    # this is a bit of a hack as we need to create (and save) a
    # version before the id is known. We need to know the id before we
    # can save the content file under /document_id/versions/version_id
    version = Version.objects.create(
        comment=form.cleaned_data['comment'],
        document=document,
        created_by=request.user)
    content_file = request.FILES['content']
    version.content.save(content_file.name, content_file)

    return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

@login_required
def transition(request, document_id):
    document = Document.objects.get(pk=document_id)

    if request.method != 'POST':
        return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))

    form = PartialDocumentForm(request.POST)
    if not form.is_valid():
        versionForm = PartialVersionForm()
        attachmentForm = PartialAttachmentForm()
        documentForm = form
        return render_to_response('documents/todo_detail.html', locals(),
                                  context_instance=RequestContext(request))

    document.transitionTo(form.cleaned_data['state'])
    return HttpResponseRedirect(reverse('todo_index'))

@login_required
def ocr(request, document_id):
    document = Document.objects.get(pk=document_id)
    if request.method == 'POST':
        form = OCRForm(request.POST, request.FILES)
        if form.is_valid():
            # do whatever is needed here
            return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))
    else:
        form = OCRForm()

    return render_to_response('documents/todo_ocr.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def markup(request, document_id):
    document = Document.objects.select_related('state').get(pk=document_id)
    if request.method == 'POST':
        form = MarkupForm(request.POST,
                          document=document, user=request.user)
        if form.is_valid():
            # create a new version with the given form content
            form.save()
            return HttpResponseRedirect(reverse('todo_detail', args=[document_id]))
        else:
            form.has_errors = True
    else:
        file_field = document.latest_version().content
        file_field.open()
        content = file_field.read()
        file_field.close()
        form = MarkupForm({'data' : content, 'comment' : "Insert a comment here"})
        # This is a bit of a cheap hack to avoid expensive validation
        # on initial display of the form
        form.has_errors = False

    return render_to_response('documents/todo_markup.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_xhtml(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    inputFile = document.latest_version().content.path
    outputFile = "/tmp/%s.xhtml" % document_id
    params = {}
    DaisyPipeline.dtbook2xhtml(inputFile, outputFile, **params)

    return render_to_mimetype_response('text/html',
                                       document.title.encode('utf-8'), outputFile)

@login_required
def preview_sbsform(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = SBSFormForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.sbsform" % document_id
            SBSForm.dtbook2sbsform(inputFile, outputFile, **form.cleaned_data)
            contraction = form.cleaned_data['contraction']
            contraction_to_mimetype_mapping = {0 : 'text/x-sbsform-g0',
                                               1 : 'text/x-sbsform-g1',
                                               2 : 'text/x-sbsform-g2'}
            mimetype = contraction_to_mimetype_mapping.get(contraction, 'text/x-sbsform-g2')
            return render_to_mimetype_response(mimetype, document.title.encode('utf-8'), outputFile)
    else:
        form = SBSFormForm()

    return render_to_response('documents/todo_sbsform.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_pdf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = LargePrintProfileForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.pdf" % document_id
            DaisyPipeline.dtbook2pdf(inputFile, outputFile, **form.cleaned_data)
            return render_to_mimetype_response('application/pdf',
                                               document.title.encode('utf-8'), outputFile)
    else:
        form = LargePrintProfileForm()

    return render_to_response('documents/todo_pdf.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_library_pdf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    inputFile = document.latest_version().content.path
    outputFile = "/tmp/%s.pdf" % document_id
    StandardLargePrint.dtbook2pdf(inputFile, outputFile)
    filename = document.title + u" 17pt"
    return render_to_mimetype_response('application/pdf', filename.encode('utf-8'), outputFile)

@login_required
def preview_sale_pdf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = SalePDFForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.pdf" % document_id
            StandardLargePrint.dtbook2pdf(inputFile, outputFile, **form.cleaned_data)
            filename = document.title + u" " + form.cleaned_data['font_size']
            return render_to_mimetype_response('application/pdf',
                                               filename.encode('utf-8'), outputFile)
    else:
        form = SalePDFForm()

    return render_to_response('documents/todo_sale_pdf.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_rtf(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = RTFForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.rtf" % document_id
            DaisyPipeline.dtbook2rtf(inputFile, outputFile, **form.cleaned_data)
            return render_to_mimetype_response('application/rtf',
                                               document.title.encode('utf-8'), outputFile)
    else:
        form = RTFForm()

    return render_to_response('documents/todo_rtf.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_epub(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = EPUBForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputFile = "/tmp/%s.epub" % document_id
            defaults = {
                "dctitle" : document.title,
                "dcidentifier" : document.identifier,
                "dclanguage" : document.language,
                "dccreator" : document.author,
                "dcpublisher" : document.publisher,
                "dcdate" : document.date}
            defaults.update(form.cleaned_data)
            DaisyPipeline.dtbook2epub(inputFile, outputFile, **defaults)
            return render_to_mimetype_response('application/epub+zip',
                                               document.title.encode('utf-8'), outputFile)
    else:
        form = EPUBForm()

    return render_to_response('documents/todo_epub.html', locals(),
                              context_instance=RequestContext(request))

def zipDirectory(directory, zipFileName, document_title):
    outputFile = zipfile.ZipFile(zipFileName, 'w')
    cwd = os.getcwd()
    os.chdir(directory)
    for dirpath, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            # zipFile support in Python has a few weak spots: Older
            # Pythons die if the filename or the arcname that is
            # passed to ZipFile.write is not in the right encoding
            # FIXME: remove the encode("latin-1") workaround once we
            # upgrade to 2.6.2
            outputFile.write(
                os.path.join(dirpath, filename).encode("latin-1"),
                os.path.join(document_title, dirpath, filename).encode("latin-1"))
    outputFile.close()
    os.chdir(cwd)

@login_required
def preview_text_only_fileset(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = TextOnlyFilesetForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputDir = tempfile.mkdtemp(prefix="daisyproducer-")

            DaisyPipeline.dtbook2text_only_fileset(inputFile, outputDir, **form.cleaned_data)

            ignore, zipFileName = tempfile.mkstemp(suffix='.zip', prefix=document_id)
            zipDirectory(outputDir, zipFileName, document.title)
            shutil.rmtree(outputDir)

            return render_to_mimetype_response('application/zip',
                                               document.title.encode('utf-8'), zipFileName)
    else:
        form = TextOnlyFilesetForm()

    return render_to_response('documents/todo_text_only_fileset.html', locals(),
                              context_instance=RequestContext(request))

@login_required
def preview_dtb(request, document_id):
    document = get_object_or_404(Document, pk=document_id)

    if request.method == 'POST':
        form = DTBForm(request.POST)
        if form.is_valid():
            inputFile = document.latest_version().content.path
            outputDir = tempfile.mkdtemp(prefix="daisyproducer-")
            DaisyPipeline.dtbook2dtb(inputFile, outputDir, **form.cleaned_data)

            ignore, zipFileName = tempfile.mkstemp(suffix='.zip', prefix=document_id)
            zipDirectory(outputDir, zipFileName, document.title)
            # shutil.rmtree(outputDir)

            return render_to_mimetype_response('application/zip',
                                               document.title.encode('utf-8'), zipFileName)
    else:
        form = DTBForm()

    return render_to_response('documents/todo_dtb.html', locals(),
                              context_instance=RequestContext(request))

<|endoftext|>


<commit_before># coding: utf-8
import json
import threading
from twisted.internet.protocol import Protocol as TwistedProtocol
from ..utils import stars, second2time, music_symbol, Color as color

class Protocol(TwistedProtocol):
    def __init__(self):
        self.input_thread = threading.Thread(target=self.input)
        self.input_thread.setDaemon(True)

    def connectionMade(self):
        print(color.green('Connected'))
        if not self.input_thread.isAlive():
            self.input_thread.start()

    def input(self):
        while True:
            self.transport.write(raw_input())

    def dataReceived(self, data):
        for row in data.split('\n'):
            if row:
                try:
                    data = json.loads(row)
                    if len(data) == 1:
                        getattr(self, 'on_' + data[0])()
                    else:
                        getattr(self, 'on_' + data[0])(data[1])
                except Exception as e:
                    print(color.red('%sMessage processing failed: %s' % e.message))

    def on_error(self, message):
        print(color.red('Server error: %s' % message))

    def on_user(self, user):
        if user:
            print('%s <%s>' % (color.cyan(user['user_name']), user['email']))
        else:
            print('☺')

    def on_song(self, song):
        self.song = song
        print('%s%s - %s（%s）%s\n  %s（%s）\n  %s, %s\n  %s' % (
            music_symbol() + ' ',
            color.yellow(song['artist']),
            color.green(song['title']),
            second2time(song['length']),
            color.red(['♡', '♥'][song['like']]),
            color.cyan(song['albumtitle']),
            'http://music.douban.com' + song['album'],
            song['company'],
            song['public_time'],
            color.magenta(stars(song['rating_avg'])),
        ))

    def on_play(self, song):
        self.on_song(song)

    def on_skip(self):
        print(color.yellow('Skip'))

    def on_like(self):
        print(color.red('♥'))

    def on_unlike(self):
        print(color.white('♡'))

    def on_remove(self):
        print(color.red('Remove'))

    def on_pause(self):
        print(color.blue('Pause'))

    def on_resume(self):
        print(color.green('Resume'))

    def on_login_success(self, user):
        print(color.green('Login success'))
        self.on_user(user)

    def on_login_failed(self, message):
        print(color.red('Login failed: %s' % message))

    def on_kbps(self, kbps):
        print(color.magenta('%sKBps' % kbps))

    def on_channel(self, channel_id):
        self.channel_id = int(channel_id)
        if hasattr(self, 'channels'):
            for channel in self.channels:
                if channel_id == channel['channel_id']:
                    print(color.magenta(channel['name']))
                    return

    def on_channels(self, channels):
        self.channels = channels
        print(color.yellow('Channels:'))
        for channel in channels:
            active = color.cyan
            if hasattr(self, 'channel_id') and channel['channel_id'] == self.channel_id:
                active = color.magenta
            print('  %s（%s）' % (active(channel['name']), channel['channel_id']))

    def on_playlist(self, playlist):
        print(color.cyan('Playlist:'))
        for song in playlist:
            current = ' '
            if hasattr(self, 'song') and self.song == song:
                current = '>'
            print('%s %s - %s <%s>' % (
                current,
                color.yellow(song['artist']),
                color.green(song['title']),
                song['albumtitle']))

    def on_state(self, state):
        if state == 'playing':
            print(color.green('Playing'))

        if state == 'paused':
            print(color.yellow('Paused'))

        if state == 'stoped':
            print(color.red('Stoped'))

    def on_logout(self):
        print(color.yellow('Logout'))

    def on_volume(self, volume):
        print('Volume %s' % color.magenta(str(float(volume) * 100) + '%'))
<commit_msg>修正 state 输出<commit_after># coding: utf-8
import json
import threading
from twisted.internet.protocol import Protocol as TwistedProtocol
from ..utils import stars, second2time, music_symbol, Color as color

class Protocol(TwistedProtocol):
    def __init__(self):
        print('Starting...')
        self.input_thread = threading.Thread(target=self.input)
        self.input_thread.setDaemon(True)

    def connectionMade(self):
        print(color.green('Connected'))
        if not self.input_thread.isAlive():
            self.input_thread.start()

    def input(self):
        while True:
            self.transport.write(raw_input())

    def dataReceived(self, data):
        for row in data.split('\n'):
            if row:
                try:
                    data = json.loads(row)
                    if len(data) == 1:
                        getattr(self, 'on_' + data[0])()
                    else:
                        getattr(self, 'on_' + data[0])(data[1])
                except Exception as e:
                    print(color.red('%sMessage processing failed: %s' % e.message))

    def on_error(self, message):
        print(color.red('Server error: %s' % message))

    def on_user(self, user):
        if user:
            print('%s <%s>' % (color.cyan(user['user_name']), user['email']))
        else:
            print('☺')

    def on_song(self, song):
        self.song = song
        print('%s%s - %s（%s）%s\n  %s（%s）\n  %s, %s\n  %s' % (
            music_symbol() + ' ',
            color.yellow(song['artist']),
            color.green(song['title']),
            second2time(song['length']),
            color.red(['♡', '♥'][song['like']]),
            color.cyan(song['albumtitle']),
            'http://music.douban.com' + song['album'],
            song['company'],
            song['public_time'],
            color.magenta(stars(song['rating_avg'])),
        ))

    def on_play(self, song):
        self.on_song(song)

    def on_skip(self):
        print(color.yellow('Skip'))

    def on_like(self):
        print(color.red('♥'))

    def on_unlike(self):
        print(color.white('♡'))

    def on_remove(self):
        print(color.red('Remove'))

    def on_pause(self):
        print(color.blue('Pause'))

    def on_resume(self):
        print(color.green('Resume'))

    def on_login_success(self, user):
        print(color.green('Login success'))
        self.on_user(user)

    def on_login_failed(self, message):
        print(color.red('Login failed: %s' % message))

    def on_kbps(self, kbps):
        print(color.magenta('%sKBps' % kbps))

    def on_channel(self, channel_id):
        self.channel_id = int(channel_id)
        if hasattr(self, 'channels'):
            for channel in self.channels:
                if channel_id == channel['channel_id']:
                    print(color.magenta(channel['name']))
                    return

    def on_channels(self, channels):
        self.channels = channels
        print(color.yellow('Channels:'))
        for channel in channels:
            active = color.cyan
            if hasattr(self, 'channel_id') and channel['channel_id'] == self.channel_id:
                active = color.magenta
            print('  %s（%s）' % (active(channel['name']), channel['channel_id']))

    def on_playlist(self, playlist):
        print(color.cyan('Playlist:'))
        for song in playlist:
            current = ' '
            if hasattr(self, 'song') and self.song == song:
                current = '>'
            print('%s %s - %s <%s>' % (
                current,
                color.yellow(song['artist']),
                color.green(song['title']),
                song['albumtitle']))

    def on_state(self, state):
        if state == 'playing':
            print(color.green('Playing'))

        if state == 'paused':
            print(color.yellow('Paused'))

        if state == 'null':
            print(color.red('Stoped'))

        if state == 'ready':
            print(color.cyan('Ready'))

    def on_logout(self):
        print(color.yellow('Logout'))

    def on_volume(self, volume):
        print('Volume %s' % color.magenta(str(float(volume) * 100) + '%'))
<|endoftext|>


<commit_before>from models import *
from rest_framework import viewsets
from serializers import *
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import *
from rest_framework.views import Request, Response
from rest_framework.parsers import JSONParser

# Resource routes
class EAVariableDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EAVariableDescriptionSerializer
    filter_fields = ('number', 'name',)
    queryset = EAVariableDescription.objects.all()

class EAVariableCodeDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EAVariableCodeDescriptionSerializer
    filter_fields = ('variable', 'code', 'description',)
    queryset = EAVariableCodeDescription.objects.all()

# Can filter by code, code__variable, or society
class EAVariableCodedValueViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EAVariableCodedValueSerializer
    filter_fields = ('variable','coded_value','code','society','code',)
    # Avoid additional database trips by select_related for foreign keys
    queryset = EAVariableCodedValue.objects.select_related('variable').select_related('code').all()

class SocietyViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = SocietySerializer
    queryset = Society.objects.all()

class ISOCodeViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = ISOCodeSerializer
    filter_fields = ('iso_code',)
    queryset = ISOCode.objects.all()

class EnvironmentalViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalSerializer
    filter_fields = ('society', 'iso_code',)
    queryset = Environmental.objects.all()

class LanguageClassViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageClassSerializer
    filter_fields = ('level', 'parent', 'name',)
    model = LanguageClass
    def get_queryset(self):
        queryset = LanguageClass.objects.all()
        level = self.request.QUERY_PARAMS.get('level', None)
        if level is not None:
            queryset = queryset.filter(level=level)
        else:
            queryset = queryset.filter(level=1)
        return queryset

class LanguageFamilyViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageFamilySerializer
    filter_fields = ('name',)
    queryset = LanguageFamily.objects.all()

# Need an API to get classifications / languages for a class

class LanguageClassificationViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageClassificationSerializer
    filter_fields = ('scheme', 'language', 'name', 'family', 'class_family', 'class_subfamily', 'class_subsubfamily',)
    queryset = LanguageClassification.objects.all()

class LanguageViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageSerializer
    filter_fields = ('name', 'iso_code', 'society',)
    queryset = Language.objects.all()

# search/filter APIs
@api_view(['GET'])
@permission_classes((IsAuthenticatedOrReadOnly,))
def find_societies(request):
    """
    View to find the societies that match an input request.  Currently expects
    { language_class_ids: [1,2,3...], ea_variable_codes: [4,5,6...] }
    """
    language_class_ids = request.QUERY_PARAMS.getlist('language_class_ids')
    language_class_ids = [int(x) for x in language_class_ids]
    # Loop over the language class IDs to get classes
    language_classes = LanguageClass.objects.filter(pk__in=language_class_ids)
    # Classifications are related to classes
    language_classifications = []
    for language_class in language_classes:
        language_classifications += language_class.languages1.all()
        language_classifications += language_class.languages2.all()
        language_classifications += language_class.languages3.all()
    # Now get languages from classifications
    iso_codes = []
    for language_classification in language_classifications:
        iso_codes.append(language_classification.language.iso_code)
    # now get societies from ISO codes
    language_societies = Society.objects.filter(iso_code__in=iso_codes)

    # Now get the societies from EA Variables
    ea_variable_code_ids = request.QUERY_PARAMS.getlist('ea_variable_codes')
    ea_variable_code_ids = [int(x) for x in ea_variable_code_ids]
    codes = EAVariableCodeDescription.objects.filter(pk__in=ea_variable_code_ids) # returns a queryset
    coded_value_ids = []
    # Aggregate all the coded values for each selected code
    for code in codes:
        coded_value_ids += code.eavariablecodedvalue_set.values_list('id', flat=True)
    # Coded values have a FK to society.  Aggregate the societies from each value
    ea_variable_societies = Society.objects.filter(eavariablecodedvalue__in=coded_value_ids)

    # Combine the querysets
    societies = language_societies | ea_variable_societies
    return Response(SocietySerializer(societies).data)

<commit_msg>Properly concatenate Society QuerySets<commit_after>from models import *
from rest_framework import viewsets
from serializers import *
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import *
from rest_framework.views import Request, Response
from rest_framework.parsers import JSONParser
from itertools import chain

# Resource routes
class EAVariableDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EAVariableDescriptionSerializer
    filter_fields = ('number', 'name',)
    queryset = EAVariableDescription.objects.all()

class EAVariableCodeDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EAVariableCodeDescriptionSerializer
    filter_fields = ('variable', 'code', 'description',)
    queryset = EAVariableCodeDescription.objects.all()

# Can filter by code, code__variable, or society
class EAVariableCodedValueViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EAVariableCodedValueSerializer
    filter_fields = ('variable','coded_value','code','society','code',)
    # Avoid additional database trips by select_related for foreign keys
    queryset = EAVariableCodedValue.objects.select_related('variable').select_related('code').all()

class SocietyViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = SocietySerializer
    queryset = Society.objects.all()

class ISOCodeViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = ISOCodeSerializer
    filter_fields = ('iso_code',)
    queryset = ISOCode.objects.all()

class EnvironmentalViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalSerializer
    filter_fields = ('society', 'iso_code',)
    queryset = Environmental.objects.all()

class LanguageClassViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageClassSerializer
    filter_fields = ('level', 'parent', 'name',)
    model = LanguageClass
    def get_queryset(self):
        queryset = LanguageClass.objects.all()
        level = self.request.QUERY_PARAMS.get('level', None)
        if level is not None:
            queryset = queryset.filter(level=level)
        else:
            queryset = queryset.filter(level=1)
        return queryset

class LanguageFamilyViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageFamilySerializer
    filter_fields = ('name',)
    queryset = LanguageFamily.objects.all()

# Need an API to get classifications / languages for a class

class LanguageClassificationViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageClassificationSerializer
    filter_fields = ('scheme', 'language', 'name', 'family', 'class_family', 'class_subfamily', 'class_subsubfamily',)
    queryset = LanguageClassification.objects.all()

class LanguageViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageSerializer
    filter_fields = ('name', 'iso_code', 'society',)
    queryset = Language.objects.all()

# search/filter APIs
@api_view(['GET'])
@permission_classes((IsAuthenticatedOrReadOnly,))
def find_societies(request):
    """
    View to find the societies that match an input request.  Currently expects
    { language_class_ids: [1,2,3...], ea_variable_codes: [4,5,6...] }
    """
    language_class_ids = request.QUERY_PARAMS.getlist('language_class_ids')
    language_class_ids = [int(x) for x in language_class_ids]
    # Loop over the language class IDs to get classes
    language_classes = LanguageClass.objects.filter(pk__in=language_class_ids)
    # Classifications are related to classes
    language_classifications = []
    for language_class in language_classes:
        language_classifications += language_class.languages1.all()
        language_classifications += language_class.languages2.all()
        language_classifications += language_class.languages3.all()
    # Now get languages from classifications
    iso_codes = []
    for language_classification in language_classifications:
        iso_codes.append(language_classification.language.iso_code)
    # now get societies from ISO codes
    language_societies = Society.objects.filter(iso_code__in=iso_codes)

    # Now get the societies from EA Variables
    ea_variable_code_ids = request.QUERY_PARAMS.getlist('ea_variable_codes')
    ea_variable_code_ids = [int(x) for x in ea_variable_code_ids]
    codes = EAVariableCodeDescription.objects.filter(pk__in=ea_variable_code_ids) # returns a queryset
    coded_value_ids = []
    # Aggregate all the coded values for each selected code
    for code in codes:
        coded_value_ids += code.eavariablecodedvalue_set.values_list('id', flat=True)
    # Coded values have a FK to society.  Aggregate the societies from each value
    ea_variable_societies = Society.objects.filter(eavariablecodedvalue__in=coded_value_ids)

    # Combine the querysets
    societies = list(chain(language_societies, ea_variable_societies))
    return Response(SocietySerializer(societies).data)

<|endoftext|>


<commit_before>import csv
import zipfile

from rest_framework import renderers
from six import StringIO, text_type

# TODO: add in 'How to cite' here
CSV_PREAMBLE = """
Research that uses data from D-PLACE should cite both the original source(s) of
the data and this paper (e.g., research using cultural data from the Binford
Hunter-Gatherer dataset: "Binford (2001); Binford and Johnson (2006); Kirby et
al. Submitted)." The reference list should include the date data were accessed
and URL for D-PLACE (http://d-place.org), in addition to the full references
for Binford (2001) and Binford and Johnson (2006).
""".replace("\n", " ").strip().lstrip()

class DPLACECSVResults(object):
    def __init__(self, data):
        self.data = data
        self.field_map = dict()
        self.field_names = [
            'Society name',
            'Society source',
            'Longitude',
            'Latitude',
            'ISO code',
            'Language name',
            'Language family']
        self.rows = []
        self.parse()
        self.encode_field_names()
        self.flatten()

    def field_names_for_cultural_variable(self, variable):
        v = "%s %s" % (variable['label'], variable['name'])
        return {
            'code': "Code: %s" % v,
            'description': "Description: %s" % v,
            'comments': "Comment: %s" % v,
            'focal_year': "Focal Year: %s" % v,
            'sources': "References: %s" % v,
        }

    def field_names_for_environmental_variable(self, variable):
        return {'name': "%s (%s)" % (variable['name'], variable['units'])}

    def parse(self):
        if 'geographic_regions' in self.data:
            if len(self.data['geographic_regions']) > 0:
                self.field_names.append('Continent')
                self.field_names.append('Region name')

        if 'variable_descriptions' in self.data:
            self.field_map['variable_descriptions'] = dict()
            for v in self.data['variable_descriptions']:
                field_names = self.field_names_for_cultural_variable(v['variable'])
                self.field_map['variable_descriptions'][v['variable']['id']] = field_names
                self.field_names.append(field_names['focal_year'])
                self.field_names.append(field_names['code'])
                self.field_names.append(field_names['description'])
                self.field_names.append(field_names['comments'])
                self.field_names.append(field_names['sources'])

        if 'environmental_variables' in self.data:
            self.field_map['environmental_variables'] = dict()
            for v in self.data['environmental_variables']:
                field_names = self.field_names_for_environmental_variable(v)
                self.field_map['environmental_variables'][v['id']] = field_names
                self.field_names.append(field_names['name'])

    def encode_field_names(self):
        # Field names must also be utf-8 encoded
        self.field_names = [
            field.encode("utf-8") for field in self.field_names
        ]

    def flatten(self):
        # data is a dictionary with a list of societies
        for item in self.data.get('societies', []):
            row = dict()
            # Merge in society data
            society = item['society']
            row['Society name'] = society['name']
            row['Society source'] = society['source']['name']
            row['Longitude'] = "" if society['location'] is None \
                else society['location']['coordinates'][0]
            row['Latitude'] = "" if society['location'] is None \
                else society['location']['coordinates'][1]

            if society['language'] is not None and 'name' in society['language']:
                row['ISO code'] = society['language']['iso_code']
                row['Language name'] = society['language']['name']
                if 'family' in society['language'] and 'name' in society['language']['family']:
                    row['Language family'] = society['language']['family']['name']
            else:
                row['Language name'] = ""
                row['Language family'] = ""

            # geographic - only one
            geographic_regions = item['geographic_regions']
            if len(geographic_regions) == 1:
                geographic_region = geographic_regions[0]
                row['Continent'] = geographic_region['continent']
                row['Region name'] = geographic_region['region_nam']

            # cultural
            cultural_trait_values = item['variable_coded_values']
            extra_rows = [] # Binford societies may have multiple values for one variable
            for cultural_trait_value in cultural_trait_values:
                # Figure out the column name
                variable_id = cultural_trait_value['variable']
                field_names = self.field_map['variable_descriptions'][variable_id]
                if field_names['code'] in row:
                    if 'code_description' in cultural_trait_value:
                        try:
                            description = cultural_trait_value['code_description']['description']
                        except:
                            description = ""
                    else:
                        description = ""
                    extra_rows.append(dict({
                        'Society name': society['name'],
                        'Society source': society['source']['name'],
                        field_names['code']: cultural_trait_value['coded_value'],
                        field_names['description']: description,
                        field_names['focal_year']: cultural_trait_value['focal_year'],
                        field_names['comments']: cultural_trait_value['comment'],
                        field_names['sources']: ''.join([x['author'] + '(' + x['year'] + '); ' for x in cultural_trait_value['references']])
                    }))
                    continue

                row[field_names['code']] = cultural_trait_value['coded_value']
                row[field_names['focal_year']] = cultural_trait_value['focal_year']
                if 'code_description' in cultural_trait_value:
                    try:
                        row[field_names['description']] = \
                            cultural_trait_value['code_description']['description']
                    except:
                        row[field_names['description']] = ''
                row[field_names['comments']] = cultural_trait_value['comment']
                row[field_names['sources']] = ''.join([
                    x['author'] + '(' + x['year'] + '); '
                    for x in cultural_trait_value['references']])
            # environmental
            environmental_values = item['environmental_values']
            for environmental_value in environmental_values:
                variable_id = environmental_value['variable']
                field_names = self.field_map['environmental_variables'][variable_id]
                row[field_names['name']] = environmental_value['value']
            # language - already have
            #
            self.rows.append(row)
            for extra in extra_rows:
                self.rows.append(extra)

def encode_if_text(val):
    return val.encode('utf-8') if isinstance(val, text_type) else val

def encode_rowdict(rowdict):
    encoded = dict()
    for k in rowdict:
        encoded_k = encode_if_text(k)
        elem = rowdict[k]
        encoded[encoded_k] = encode_if_text(elem)
    return encoded

class DPLACECSVRenderer(renderers.BaseRenderer):
    media_type = 'text/csv'
    format = 'csv'

    def render(self, data, accepted_media_type=None, renderer_context=None):
        "Renders a list of SocietyResultSets to CSV"
        if data is None:
            return ''
        results = DPLACECSVResults(data)
        csv_buffer = StringIO()
        csv_writer = csv.DictWriter(csv_buffer, results.field_names)
        cite_writer = csv.writer(csv_buffer)
        cite_writer.writerow([CSV_PREAMBLE])
        csv_writer.writeheader()
        for row in results.rows:
            csv_writer.writerow(encode_rowdict(row))

        return csv_buffer.getvalue()

class ZipRenderer(renderers.BaseRenderer):
    media_type = 'application/zip'
    format = 'zip'

    def render(self, data, accepted_media_type=None, renderer_context=None):
        """Renders zip file for phylogeny download"""
        if data is None:
            return ''

        s = StringIO()
        zf = zipfile.ZipFile(s, "w")
        try:
            if 'legends' in data:
                for l in data['legends'] or []:
                    if 'svg' in l:
                        if 'name' in l:
                            zf.writestr(
                                l['name'].encode('utf-8'),
                                l['svg'].encode('utf-8')
                            )
            if 'tree' in data:
                if 'name' in data:
                    zf.writestr(
                        (data['name'] or 'x').encode('utf-8'),
                        (data['tree'] or '').encode('utf-8'))
                else:
                    zf.writestr('tree.svg', (data['tree'] or '').encode('utf-8'))
        finally:
            zf.close()
        return s.getvalue()
<commit_msg>Adding subcase to CSV download<commit_after>import csv
import zipfile

from rest_framework import renderers
from six import StringIO, text_type

# TODO: add in 'How to cite' here
CSV_PREAMBLE = """
Research that uses data from D-PLACE should cite both the original source(s) of
the data and this paper (e.g., research using cultural data from the Binford
Hunter-Gatherer dataset: "Binford (2001); Binford and Johnson (2006); Kirby et
al. Submitted)." The reference list should include the date data were accessed
and URL for D-PLACE (http://d-place.org), in addition to the full references
for Binford (2001) and Binford and Johnson (2006).
""".replace("\n", " ").strip().lstrip()

class DPLACECSVResults(object):
    def __init__(self, data):
        self.data = data
        self.field_map = dict()
        self.field_names = [
            'Society name',
            'Society source',
            'Longitude',
            'Latitude',
            'ISO code',
            'Language name',
            'Language family']
        self.rows = []
        self.parse()
        self.encode_field_names()
        self.flatten()

    def field_names_for_cultural_variable(self, variable):
        v = "%s %s" % (variable['label'], variable['name'])
        return {
            'code': "Code: %s" % v,
            'description': "Description: %s" % v,
            'comments': "Comment: %s" % v,
            'focal_year': "Focal Year: %s" % v,
            'sources': "References: %s" % v,
        }

    def field_names_for_environmental_variable(self, variable):
        return {'name': "%s (%s)" % (variable['name'], variable['units'])}

    def parse(self):
        if 'geographic_regions' in self.data:
            if len(self.data['geographic_regions']) > 0:
                self.field_names.append('Continent')
                self.field_names.append('Region name')

        if 'variable_descriptions' in self.data:
            self.field_map['variable_descriptions'] = dict()
            for v in self.data['variable_descriptions']:
                field_names = self.field_names_for_cultural_variable(v['variable'])
                self.field_map['variable_descriptions'][v['variable']['id']] = field_names
                self.field_names.append(field_names['focal_year'])
                self.field_names.append(field_names['code'])
                self.field_names.append(field_names['description'])
                self.field_names.append(field_names['comments'])
                self.field_names.append(field_names['sources'])

        if 'environmental_variables' in self.data:
            self.field_map['environmental_variables'] = dict()
            for v in self.data['environmental_variables']:
                field_names = self.field_names_for_environmental_variable(v)
                self.field_map['environmental_variables'][v['id']] = field_names
                self.field_names.append(field_names['name'])

    def encode_field_names(self):
        # Field names must also be utf-8 encoded
        self.field_names = [
            field.encode("utf-8") for field in self.field_names
        ]

    def flatten(self):
        # data is a dictionary with a list of societies
        for item in self.data.get('societies', []):
            row = dict()
            # Merge in society data
            society = item['society']
            row['Society name'] = society['name']
            row['Society source'] = society['source']['name']
            row['Longitude'] = "" if society['location'] is None \
                else society['location']['coordinates'][0]
            row['Latitude'] = "" if society['location'] is None \
                else society['location']['coordinates'][1]

            if society['language'] is not None and 'name' in society['language']:
                row['ISO code'] = society['language']['iso_code']
                row['Language name'] = society['language']['name']
                if 'family' in society['language'] and 'name' in society['language']['family']:
                    row['Language family'] = society['language']['family']['name']
            else:
                row['Language name'] = ""
                row['Language family'] = ""

            # geographic - only one
            geographic_regions = item['geographic_regions']
            if len(geographic_regions) == 1:
                geographic_region = geographic_regions[0]
                row['Continent'] = geographic_region['continent']
                row['Region name'] = geographic_region['region_nam']

            # cultural
            cultural_trait_values = item['variable_coded_values']
            extra_rows = [] # Binford societies may have multiple values for one variable
            for cultural_trait_value in cultural_trait_values:
                # Figure out the column name
                variable_id = cultural_trait_value['variable']
                field_names = self.field_map['variable_descriptions'][variable_id]
                if field_names['code'] in row:
                    if 'code_description' in cultural_trait_value:
                        try:
                            description = cultural_trait_value['code_description']['description']
                        except:
                            description = ""
                    else:
                        description = ""
                    extra_rows.append(dict({
                        'Society name': society['name'],
                        'Society source': society['source']['name'],
                        field_names['code']: cultural_trait_value['coded_value'],
                        field_names['description']: description,
                        field_names['focal_year']: cultural_trait_value['focal_year'],
                        field_names['comments']: cultural_trait_value['subcase'] + '; ' + cultural_trait_value['comment'],
                        field_names['sources']: ''.join([x['author'] + '(' + x['year'] + '); ' for x in cultural_trait_value['references']])
                    }))
                    continue

                row[field_names['code']] = cultural_trait_value['coded_value']
                row[field_names['focal_year']] = cultural_trait_value['focal_year']
                if 'code_description' in cultural_trait_value:
                    try:
                        row[field_names['description']] = \
                            cultural_trait_value['code_description']['description']
                    except:
                        row[field_names['description']] = ''
                row[field_names['comments']] = cultural_trait_value['subcase'] + '; '+ cultural_trait_value['comment']
                row[field_names['sources']] = ''.join([
                    x['author'] + '(' + x['year'] + '); '
                    for x in cultural_trait_value['references']])
            # environmental
            environmental_values = item['environmental_values']
            for environmental_value in environmental_values:
                variable_id = environmental_value['variable']
                field_names = self.field_map['environmental_variables'][variable_id]
                row[field_names['name']] = environmental_value['value']
            # language - already have
            #
            self.rows.append(row)
            for extra in extra_rows:
                self.rows.append(extra)

def encode_if_text(val):
    return val.encode('utf-8') if isinstance(val, text_type) else val

def encode_rowdict(rowdict):
    encoded = dict()
    for k in rowdict:
        encoded_k = encode_if_text(k)
        elem = rowdict[k]
        encoded[encoded_k] = encode_if_text(elem)
    return encoded

class DPLACECSVRenderer(renderers.BaseRenderer):
    media_type = 'text/csv'
    format = 'csv'

    def render(self, data, accepted_media_type=None, renderer_context=None):
        "Renders a list of SocietyResultSets to CSV"
        if data is None:
            return ''
        results = DPLACECSVResults(data)
        csv_buffer = StringIO()
        csv_writer = csv.DictWriter(csv_buffer, results.field_names)
        cite_writer = csv.writer(csv_buffer)
        cite_writer.writerow([CSV_PREAMBLE])
        csv_writer.writeheader()
        for row in results.rows:
            csv_writer.writerow(encode_rowdict(row))

        return csv_buffer.getvalue()

class ZipRenderer(renderers.BaseRenderer):
    media_type = 'application/zip'
    format = 'zip'

    def render(self, data, accepted_media_type=None, renderer_context=None):
        """Renders zip file for phylogeny download"""
        if data is None:
            return ''

        s = StringIO()
        zf = zipfile.ZipFile(s, "w")
        try:
            if 'legends' in data:
                for l in data['legends'] or []:
                    if 'svg' in l:
                        if 'name' in l:
                            zf.writestr(
                                l['name'].encode('utf-8'),
                                l['svg'].encode('utf-8')
                            )
            if 'tree' in data:
                if 'name' in data:
                    zf.writestr(
                        (data['name'] or 'x').encode('utf-8'),
                        (data['tree'] or '').encode('utf-8'))
                else:
                    zf.writestr('tree.svg', (data['tree'] or '').encode('utf-8'))
        finally:
            zf.close()
        return s.getvalue()
<|endoftext|>