<commit_before># Copyright 2007-2017 Brian Masney <masneyb@onstation.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import os
import os.path
from zipfile import ZipFile
import csv
from django.http import HttpResponseRedirect, Http404
import cavedb.coord_transform
from cavedb.docgen_composite import Composite
import cavedb.docgen_dvd
import cavedb.docgen_entrance_csv
import cavedb.docgen_gis_maps
import cavedb.docgen_gpx
import cavedb.docgen_kml
import cavedb.docgen_latex_letter_bw
import cavedb.docgen_latex_letter_color
import cavedb.docgen_latex_letter_draft
import cavedb.docgen_mapserver_mapfile
import cavedb.docgen_mxf
import cavedb.docgen_shp
import cavedb.docgen_text
import cavedb.docgen_todo_txt
import cavedb.models
import cavedb.perms
import cavedb.settings
import cavedb.utils

# Documents that span a single bulletin
def create_bulletin_docgen_classes(bulletin):
    return Composite([cavedb.docgen_dvd.create_for_bulletin(bulletin),
                      cavedb.docgen_entrance_csv.create_for_bulletin(bulletin),
                      cavedb.docgen_gpx.create_for_bulletin(bulletin),
                      cavedb.docgen_kml.create_for_bulletin(bulletin),
                      cavedb.docgen_mxf.create_for_bulletin(bulletin),
                      cavedb.docgen_text.Text(bulletin),
                      cavedb.docgen_todo_txt.TodoTxt(bulletin),

                      # The SHP and mapserver files need to be created before the GisMaps.
                      cavedb.docgen_shp.create_for_bulletin(bulletin),
                      cavedb.docgen_mapserver_mapfile.MapserverMapfile(bulletin),
                      cavedb.docgen_gis_maps.GisMaps(bulletin),

                      # Create the LaTeX PDFs last since they depend on other resources.
                      cavedb.docgen_latex_letter_draft.LatexLetterDraft(bulletin),
                      cavedb.docgen_latex_letter_bw.LatexLetterBW(bulletin),
                      cavedb.docgen_latex_letter_color.LatexLetterColor(bulletin),
                     ])

# Documents that span all bulletins
def create_global_docgen_classes():
    return Composite([cavedb.docgen_entrance_csv.create_for_global(),
                      cavedb.docgen_gpx.create_for_global(),
                      cavedb.docgen_kml.create_for_global(),
                      cavedb.docgen_mxf.create_for_global(),
                      cavedb.docgen_shp.create_for_global(),
                     ])

def get_bulletin_download_links(bulletin):
    docgen = create_bulletin_docgen_classes(bulletin)
    return docgen.create_html_download_urls()

def write_bulletin_files(bulletin):
    outputter = create_bulletin_docgen_classes(bulletin)

    all_regions_gis_hash = get_all_regions_gis_hash(bulletin.id)

    outputter.open(all_regions_gis_hash)

    outputter.indexed_terms(get_indexed_terms(bulletin))

    write_gis_sections(bulletin.id, outputter)

    outputter.begin_regions()

    for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin.id):
        write_region(region, outputter)

    outputter.end_regions()

    outputter.close()

    write_build_scripts(bulletin.id, outputter)

def write_global_bulletin_files():
    outputter = create_global_docgen_classes()

    outputter.open(None)

    outputter.indexed_terms([])

    # No GIS section. Only write that on the bulletin.

    outputter.begin_regions()

    for bulletin in cavedb.models.Bulletin.objects.all():
        for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin.id):
            write_region(region, outputter)

    outputter.end_regions()

    outputter.close()

    write_build_scripts(cavedb.utils.GLOBAL_BULLETIN_ID, outputter)

def write_build_scripts(bulletin_id, outputter):
    # Write two different build scripts. The first is a wrapper for the second
    # script. The first one is not run with -e so that the lockfile is properly
    # cleaned up. The second script is ran with -e so that the script will fail
    # if any commands fail. The web interface checks for the presence of this
    # lockfile to see if the bulletin documents are currently being generated.

    build_script_wrapper_file = cavedb.utils.get_build_script_wrapper(bulletin_id)
    build_lock_file = cavedb.utils.get_build_lock_file(bulletin_id)
    build_script_file = cavedb.utils.get_build_script(bulletin_id)
    build_log_file = cavedb.utils.get_build_log_filename(bulletin_id)

    with open(build_script_wrapper_file, 'w') as output:
        output.write('#!/bin/bash\n')
        output.write('touch %s\n' % (build_lock_file))
        output.write('%s > %s 2>&1\n' % (build_script_file, build_log_file))
        output.write('rm -f %s\n' % (build_lock_file))
    os.chmod(build_script_wrapper_file, 0755)

    build_script = outputter.generate_buildscript()

    with open(build_script_file, 'w') as output:
        output.write('#!/bin/bash -ev\n')
        output.write(build_script)
    os.chmod(build_script_file, 0755)

def write_region(region, outputter):
    gis_region_hash = get_region_gis_hash(region.id)

    outputter.begin_region(region, gis_region_hash)

    for feature in cavedb.models.Feature.objects.filter(bulletin_region__id=region.id):
        write_feature(feature, outputter)

    outputter.end_region()

def add_gis_lineplot(lineplot, gisdir, lineplot_type, outputter):
    zipfile_name = '%s/%s' % (cavedb.settings.MEDIA_ROOT, lineplot.attach_zip)

    if not os.path.isdir(gisdir):
        os.makedirs(gisdir)

    zipfile = ZipFile(zipfile_name, "r")
    for name in zipfile.namelist():
        if name.endswith('/'):
            continue

        base = os.path.basename(name)
        if base == '':
            base = name

        gisfile_name = '%s/%s' % (gisdir, base)
        gisfile = open(gisfile_name, 'w')
        gisfile.write(zipfile.read(name))
        gisfile.close()

    zipfile.close()

    outputter.gis_lineplot(lineplot, lineplot_type, '%s/%s' % (gisdir, lineplot.shp_filename))

def write_gis_sections(bulletin_id, outputter):
    outputter.begin_gis_maps()
    for gismap in cavedb.models.GisMap.objects.all():
        outputter.gis_map(gismap)
    outputter.end_gis_maps()

    outputter.begin_gis_layers()

    for layer in cavedb.models.GisLayer.objects.all():
        outputter.gis_layer(layer)

    lineplots_dir = '%s/lineplots' % \
                    (cavedb.docgen_gis_common.get_bulletin_shp_directory(bulletin_id))
    if not os.path.isdir(lineplots_dir):
        os.makedirs(lineplots_dir)

    # Write out a CSV file that contains a description of all of the lineplots
    # These lineplots will be included with the SHP ZIP file.
    csvfile = open('%s/lineplots.csv' % (lineplots_dir), 'w')
    csvwriter = csv.writer(csvfile, delimiter=',')
    csvwriter.writerow(['id', 'type', 'description', 'datum', 'coordinate_system'])

    # Expand cave and surface lineplots
    for lineplot in cavedb.models.BulletinGisLineplot.objects.filter(bulletin__id=bulletin_id):
        gisdir = '%s/bulletin_lineplot_%s' % (lineplots_dir, lineplot.id)
        add_gis_lineplot(lineplot, gisdir, 'surface', outputter)

        csvwriter.writerow(['bulletin_%s' % (lineplot.id), 'surface', lineplot.description, \
                            lineplot.datum, lineplot.coord_sys])

    for lineplot in cavedb.models.FeatureGisLineplot.objects \
       .filter(feature__bulletin_region__bulletin__id=bulletin_id):

        gisdir = '%s/feature_lineplot_%s' % (lineplots_dir, lineplot.id)
        add_gis_lineplot(lineplot, gisdir, 'underground', outputter)

        csvwriter.writerow(['feature_%s' % (lineplot.id), 'underground', lineplot.description, \
                            lineplot.datum, lineplot.coord_sys])

    csvfile.close()

    outputter.end_gis_layers()

class FeatureTodoAnalyzer(object):
    def __init__(self, feature):
        self.missing_str = ''
        self.missing_coord = False
        self.missing_ele = False
        self.saw_entrance = False
        self.has_map = False

        self.todo_descr = feature.todo_descr
        self.todo_enum = feature.todo_enum
        if not self.todo_enum:
            self.todo_enum = 'minor_computer_work'

        if not feature.survey_id:
            self.missing_str += ' ID'
        if not feature.description:
            self.missing_str += ' description'

    def process_entrance(self, ent):
        self.saw_entrance = True
        if not ent.elevation_ft:
            self.missing_ele = True

        if not ent.utmeast and not ent.longitude:
            self.missing_coord = True

    def process_photo(self, photo):
        if photo.type == 'map':
            self.has_map = True

    def process_referenced_map(self):
        self.has_map = True

    def write_todo(self, feature, outputter):
        if self.missing_coord or not self.saw_entrance:
            self.missing_str += ' GPS'
            self.todo_enum = 'minor_field_work'
        elif self.missing_ele:
            self.missing_str += ' elevation'

        if self.missing_str:
            self.missing_str = 'The following fields are missing: (%s ).' % (self.missing_str)

            if not self.todo_descr:
                self.todo_descr = self.missing_str
            else:
                self.todo_descr = '%s %s' % (self.todo_descr, self.missing_str)

        if self.todo_descr:
            outputter.feature_todo(feature, self.todo_enum, self.todo_descr)

def write_feature(feature, outputter):
    todo = FeatureTodoAnalyzer(feature)

    outputter.begin_feature(feature)

    for ent in cavedb.models.FeatureEntrance.objects.filter(feature=feature.id):
        if not ent.publish_location:
            continue

        coordinates = cavedb.coord_transform.transform_coordinate(ent)

        outputter.feature_entrance(feature, ent, coordinates)
        todo.process_entrance(ent)

    for attachment in cavedb.models.FeatureAttachment.objects.filter(feature__id=feature.id):
        outputter.feature_attachment(feature, attachment)

    for photo in cavedb.models.FeaturePhoto.objects.filter(feature__id=feature.id):
        outputter.feature_photo(feature, photo)
        todo.process_photo(photo)

    for refmap in cavedb.models.FeatureReferencedMap.objects\
       .filter(feature__id=feature.id, map__show_in_pdf=True):

        outputter.feature_referenced_map(feature, refmap)
        todo.process_referenced_map()

    for ref in cavedb.models.FeatureReference.objects.filter(feature=feature.id):
        outputter.feature_reference(feature, ref)

    todo.write_todo(feature, outputter)
    outputter.end_feature()

def get_region_gis_hash(region_id):
    has_entrances = False

    md5hash = hashlib.md5()
    for feature in cavedb.models.Feature.objects.filter(bulletin_region__id=region_id):
        for entrance in cavedb.models.FeatureEntrance.objects.filter(feature=feature.id):
            if not entrance.publish_location:
                continue

            coordinates = cavedb.coord_transform.transform_coordinate(entrance)
            wgs84_lon_lat = coordinates.get_lon_lat_wgs84()

            if not wgs84_lon_lat[0] or not wgs84_lon_lat[1]:
                continue

            nad83_utm = coordinates.get_utm_nad83()

            has_entrances = True
            entranceinfo = '%s,%s,%s,%s,%s,%s,%s,%s,%s' % \
                           (feature.name, feature.feature_type, feature.is_significant, \
                            entrance.entrance_name, entrance.utmzone, \
                            nad83_utm[1], nad83_utm[0], wgs84_lon_lat[1], wgs84_lon_lat[0])
            md5hash.update(entranceinfo)

    return md5hash.hexdigest() if has_entrances else None

def get_all_regions_gis_hash(bulletin_id):
    has_entrances = False

    md5 = hashlib.md5()
    for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin_id):
        gis_region_hash = get_region_gis_hash(region.id)
        if gis_region_hash:
            md5.update(gis_region_hash)
            has_entrances = True

    return md5.hexdigest() if has_entrances else None

def get_indexed_terms(bulletin):
    indexed_terms = []

    if bulletin.indexed_terms:
        for search_term in cavedb.utils.newline_split(bulletin.indexed_terms):
            indexed_terms.append(search_term)

    for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin.id):
        for feature in cavedb.models.Feature.objects.filter(bulletin_region__id=region.id):
            indexed_terms.append(feature.name.strip())

            for alias in cavedb.utils.get_all_feature_alt_names(feature):
                indexed_terms.append(alias)

    return indexed_terms

def run_buildscript_wrapper(bulletin_id):
    #pylint: disable=protected-access

    build_script_wrapper_file = cavedb.utils.get_build_script_wrapper(bulletin_id)

    basedir = '%s/bulletins/bulletin_%s' % (cavedb.settings.MEDIA_ROOT, bulletin_id)
    os.chdir(basedir)
    os.system('%s' % (build_script_wrapper_file))

def generate_bulletin(request, bulletin_id):
    #pylint: disable=unused-argument
    if not cavedb.perms.is_bulletin_generation_allowed(bulletin_id):
        raise Http404

    bulletin = cavedb.models.Bulletin.objects.get(pk=bulletin_id)
    if bulletin is None:
        raise Http404

    with open(cavedb.settings.WORKER_FIFO, 'w') as output:
        output.write('%s\n' % (bulletin_id))

    return HttpResponseRedirect('%sadmin/cavedb/bulletin/' % (cavedb.settings.CONTEXT_PATH))
<commit_msg>use tee to write the build output to stdout and to the build log file<commit_after># Copyright 2007-2017 Brian Masney <masneyb@onstation.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import os
import os.path
from zipfile import ZipFile
import csv
from django.http import HttpResponseRedirect, Http404
import cavedb.coord_transform
from cavedb.docgen_composite import Composite
import cavedb.docgen_dvd
import cavedb.docgen_entrance_csv
import cavedb.docgen_gis_maps
import cavedb.docgen_gpx
import cavedb.docgen_kml
import cavedb.docgen_latex_letter_bw
import cavedb.docgen_latex_letter_color
import cavedb.docgen_latex_letter_draft
import cavedb.docgen_mapserver_mapfile
import cavedb.docgen_mxf
import cavedb.docgen_shp
import cavedb.docgen_text
import cavedb.docgen_todo_txt
import cavedb.models
import cavedb.perms
import cavedb.settings
import cavedb.utils

# Documents that span a single bulletin
def create_bulletin_docgen_classes(bulletin):
    return Composite([cavedb.docgen_dvd.create_for_bulletin(bulletin),
                      cavedb.docgen_entrance_csv.create_for_bulletin(bulletin),
                      cavedb.docgen_gpx.create_for_bulletin(bulletin),
                      cavedb.docgen_kml.create_for_bulletin(bulletin),
                      cavedb.docgen_mxf.create_for_bulletin(bulletin),
                      cavedb.docgen_text.Text(bulletin),
                      cavedb.docgen_todo_txt.TodoTxt(bulletin),

                      # The SHP and mapserver files need to be created before the GisMaps.
                      cavedb.docgen_shp.create_for_bulletin(bulletin),
                      cavedb.docgen_mapserver_mapfile.MapserverMapfile(bulletin),
                      cavedb.docgen_gis_maps.GisMaps(bulletin),

                      # Create the LaTeX PDFs last since they depend on other resources.
                      cavedb.docgen_latex_letter_draft.LatexLetterDraft(bulletin),
                      cavedb.docgen_latex_letter_bw.LatexLetterBW(bulletin),
                      cavedb.docgen_latex_letter_color.LatexLetterColor(bulletin),
                     ])

# Documents that span all bulletins
def create_global_docgen_classes():
    return Composite([cavedb.docgen_entrance_csv.create_for_global(),
                      cavedb.docgen_gpx.create_for_global(),
                      cavedb.docgen_kml.create_for_global(),
                      cavedb.docgen_mxf.create_for_global(),
                      cavedb.docgen_shp.create_for_global(),
                     ])

def get_bulletin_download_links(bulletin):
    docgen = create_bulletin_docgen_classes(bulletin)
    return docgen.create_html_download_urls()

def write_bulletin_files(bulletin):
    outputter = create_bulletin_docgen_classes(bulletin)

    all_regions_gis_hash = get_all_regions_gis_hash(bulletin.id)

    outputter.open(all_regions_gis_hash)

    outputter.indexed_terms(get_indexed_terms(bulletin))

    write_gis_sections(bulletin.id, outputter)

    outputter.begin_regions()

    for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin.id):
        write_region(region, outputter)

    outputter.end_regions()

    outputter.close()

    write_build_scripts(bulletin.id, outputter)

def write_global_bulletin_files():
    outputter = create_global_docgen_classes()

    outputter.open(None)

    outputter.indexed_terms([])

    # No GIS section. Only write that on the bulletin.

    outputter.begin_regions()

    for bulletin in cavedb.models.Bulletin.objects.all():
        for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin.id):
            write_region(region, outputter)

    outputter.end_regions()

    outputter.close()

    write_build_scripts(cavedb.utils.GLOBAL_BULLETIN_ID, outputter)

def write_build_scripts(bulletin_id, outputter):
    # Write two different build scripts. The first is a wrapper for the second
    # script. The first one is not run with -e so that the lockfile is properly
    # cleaned up. The second script is ran with -e so that the script will fail
    # if any commands fail. The web interface checks for the presence of this
    # lockfile to see if the bulletin documents are currently being generated.

    build_script_wrapper_file = cavedb.utils.get_build_script_wrapper(bulletin_id)
    build_lock_file = cavedb.utils.get_build_lock_file(bulletin_id)
    build_script_file = cavedb.utils.get_build_script(bulletin_id)
    build_log_file = cavedb.utils.get_build_log_filename(bulletin_id)

    with open(build_script_wrapper_file, 'w') as output:
        output.write('#!/bin/bash\n')
        output.write('touch %s\n' % (build_lock_file))
        output.write('%s 2>&1 | tee %s\n' % (build_script_file, build_log_file))
        output.write('rm -f %s\n' % (build_lock_file))
    os.chmod(build_script_wrapper_file, 0755)

    build_script = outputter.generate_buildscript()

    with open(build_script_file, 'w') as output:
        output.write('#!/bin/bash -ev\n')
        output.write(build_script)
    os.chmod(build_script_file, 0755)

def write_region(region, outputter):
    gis_region_hash = get_region_gis_hash(region.id)

    outputter.begin_region(region, gis_region_hash)

    for feature in cavedb.models.Feature.objects.filter(bulletin_region__id=region.id):
        write_feature(feature, outputter)

    outputter.end_region()

def add_gis_lineplot(lineplot, gisdir, lineplot_type, outputter):
    zipfile_name = '%s/%s' % (cavedb.settings.MEDIA_ROOT, lineplot.attach_zip)

    if not os.path.isdir(gisdir):
        os.makedirs(gisdir)

    zipfile = ZipFile(zipfile_name, "r")
    for name in zipfile.namelist():
        if name.endswith('/'):
            continue

        base = os.path.basename(name)
        if base == '':
            base = name

        gisfile_name = '%s/%s' % (gisdir, base)
        gisfile = open(gisfile_name, 'w')
        gisfile.write(zipfile.read(name))
        gisfile.close()

    zipfile.close()

    outputter.gis_lineplot(lineplot, lineplot_type, '%s/%s' % (gisdir, lineplot.shp_filename))

def write_gis_sections(bulletin_id, outputter):
    outputter.begin_gis_maps()
    for gismap in cavedb.models.GisMap.objects.all():
        outputter.gis_map(gismap)
    outputter.end_gis_maps()

    outputter.begin_gis_layers()

    for layer in cavedb.models.GisLayer.objects.all():
        outputter.gis_layer(layer)

    lineplots_dir = '%s/lineplots' % \
                    (cavedb.docgen_gis_common.get_bulletin_shp_directory(bulletin_id))
    if not os.path.isdir(lineplots_dir):
        os.makedirs(lineplots_dir)

    # Write out a CSV file that contains a description of all of the lineplots
    # These lineplots will be included with the SHP ZIP file.
    csvfile = open('%s/lineplots.csv' % (lineplots_dir), 'w')
    csvwriter = csv.writer(csvfile, delimiter=',')
    csvwriter.writerow(['id', 'type', 'description', 'datum', 'coordinate_system'])

    # Expand cave and surface lineplots
    for lineplot in cavedb.models.BulletinGisLineplot.objects.filter(bulletin__id=bulletin_id):
        gisdir = '%s/bulletin_lineplot_%s' % (lineplots_dir, lineplot.id)
        add_gis_lineplot(lineplot, gisdir, 'surface', outputter)

        csvwriter.writerow(['bulletin_%s' % (lineplot.id), 'surface', lineplot.description, \
                            lineplot.datum, lineplot.coord_sys])

    for lineplot in cavedb.models.FeatureGisLineplot.objects \
       .filter(feature__bulletin_region__bulletin__id=bulletin_id):

        gisdir = '%s/feature_lineplot_%s' % (lineplots_dir, lineplot.id)
        add_gis_lineplot(lineplot, gisdir, 'underground', outputter)

        csvwriter.writerow(['feature_%s' % (lineplot.id), 'underground', lineplot.description, \
                            lineplot.datum, lineplot.coord_sys])

    csvfile.close()

    outputter.end_gis_layers()

class FeatureTodoAnalyzer(object):
    def __init__(self, feature):
        self.missing_str = ''
        self.missing_coord = False
        self.missing_ele = False
        self.saw_entrance = False
        self.has_map = False

        self.todo_descr = feature.todo_descr
        self.todo_enum = feature.todo_enum
        if not self.todo_enum:
            self.todo_enum = 'minor_computer_work'

        if not feature.survey_id:
            self.missing_str += ' ID'
        if not feature.description:
            self.missing_str += ' description'

    def process_entrance(self, ent):
        self.saw_entrance = True
        if not ent.elevation_ft:
            self.missing_ele = True

        if not ent.utmeast and not ent.longitude:
            self.missing_coord = True

    def process_photo(self, photo):
        if photo.type == 'map':
            self.has_map = True

    def process_referenced_map(self):
        self.has_map = True

    def write_todo(self, feature, outputter):
        if self.missing_coord or not self.saw_entrance:
            self.missing_str += ' GPS'
            self.todo_enum = 'minor_field_work'
        elif self.missing_ele:
            self.missing_str += ' elevation'

        if self.missing_str:
            self.missing_str = 'The following fields are missing: (%s ).' % (self.missing_str)

            if not self.todo_descr:
                self.todo_descr = self.missing_str
            else:
                self.todo_descr = '%s %s' % (self.todo_descr, self.missing_str)

        if self.todo_descr:
            outputter.feature_todo(feature, self.todo_enum, self.todo_descr)

def write_feature(feature, outputter):
    todo = FeatureTodoAnalyzer(feature)

    outputter.begin_feature(feature)

    for ent in cavedb.models.FeatureEntrance.objects.filter(feature=feature.id):
        if not ent.publish_location:
            continue

        coordinates = cavedb.coord_transform.transform_coordinate(ent)

        outputter.feature_entrance(feature, ent, coordinates)
        todo.process_entrance(ent)

    for attachment in cavedb.models.FeatureAttachment.objects.filter(feature__id=feature.id):
        outputter.feature_attachment(feature, attachment)

    for photo in cavedb.models.FeaturePhoto.objects.filter(feature__id=feature.id):
        outputter.feature_photo(feature, photo)
        todo.process_photo(photo)

    for refmap in cavedb.models.FeatureReferencedMap.objects\
       .filter(feature__id=feature.id, map__show_in_pdf=True):

        outputter.feature_referenced_map(feature, refmap)
        todo.process_referenced_map()

    for ref in cavedb.models.FeatureReference.objects.filter(feature=feature.id):
        outputter.feature_reference(feature, ref)

    todo.write_todo(feature, outputter)
    outputter.end_feature()

def get_region_gis_hash(region_id):
    has_entrances = False

    md5hash = hashlib.md5()
    for feature in cavedb.models.Feature.objects.filter(bulletin_region__id=region_id):
        for entrance in cavedb.models.FeatureEntrance.objects.filter(feature=feature.id):
            if not entrance.publish_location:
                continue

            coordinates = cavedb.coord_transform.transform_coordinate(entrance)
            wgs84_lon_lat = coordinates.get_lon_lat_wgs84()

            if not wgs84_lon_lat[0] or not wgs84_lon_lat[1]:
                continue

            nad83_utm = coordinates.get_utm_nad83()

            has_entrances = True
            entranceinfo = '%s,%s,%s,%s,%s,%s,%s,%s,%s' % \
                           (feature.name, feature.feature_type, feature.is_significant, \
                            entrance.entrance_name, entrance.utmzone, \
                            nad83_utm[1], nad83_utm[0], wgs84_lon_lat[1], wgs84_lon_lat[0])
            md5hash.update(entranceinfo)

    return md5hash.hexdigest() if has_entrances else None

def get_all_regions_gis_hash(bulletin_id):
    has_entrances = False

    md5 = hashlib.md5()
    for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin_id):
        gis_region_hash = get_region_gis_hash(region.id)
        if gis_region_hash:
            md5.update(gis_region_hash)
            has_entrances = True

    return md5.hexdigest() if has_entrances else None

def get_indexed_terms(bulletin):
    indexed_terms = []

    if bulletin.indexed_terms:
        for search_term in cavedb.utils.newline_split(bulletin.indexed_terms):
            indexed_terms.append(search_term)

    for region in cavedb.models.BulletinRegion.objects.filter(bulletin__id=bulletin.id):
        for feature in cavedb.models.Feature.objects.filter(bulletin_region__id=region.id):
            indexed_terms.append(feature.name.strip())

            for alias in cavedb.utils.get_all_feature_alt_names(feature):
                indexed_terms.append(alias)

    return indexed_terms

def run_buildscript_wrapper(bulletin_id):
    #pylint: disable=protected-access

    build_script_wrapper_file = cavedb.utils.get_build_script_wrapper(bulletin_id)

    basedir = '%s/bulletins/bulletin_%s' % (cavedb.settings.MEDIA_ROOT, bulletin_id)
    os.chdir(basedir)
    os.system('%s' % (build_script_wrapper_file))

def generate_bulletin(request, bulletin_id):
    #pylint: disable=unused-argument
    if not cavedb.perms.is_bulletin_generation_allowed(bulletin_id):
        raise Http404

    bulletin = cavedb.models.Bulletin.objects.get(pk=bulletin_id)
    if bulletin is None:
        raise Http404

    with open(cavedb.settings.WORKER_FIFO, 'w') as output:
        output.write('%s\n' % (bulletin_id))

    return HttpResponseRedirect('%sadmin/cavedb/bulletin/' % (cavedb.settings.CONTEXT_PATH))
<|endoftext|>


<commit_before>"""celery.backends.amqp"""
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Consumer, Publisher
from celery.backends.base import BaseBackend

RESULTSTORE_EXCHANGE = "celres"

class Backend(BaseBackend):
    """AMQP backend. Publish results by sending messages to the broker
    using the task id as routing key.

    Note that results published using this backend is read-once only.
    After the result has been read, the result is deleted.

    """

    capabilities = ["ResultStore"]

    def __init__(self, *args, **kwargs):
        super(Backend, self).__init__(*args, **kwargs)
        self._cache = {}

    def _publisher_for_task_id(self, task_id, connection):
        routing_key = task_id.replace("-", "")
        return Publisher(connection, exchange=RESULTSTORE_EXCHANGE,
                         exchange_type="direct",
                         routing_key="%s" % routing_key)

    def _consumer_for_task_id(self, task_id, connection):
        routing_key = task_id.replace("-", "")
        return Consumer(connection, queue=task_id,
                        exchange=RESULTSTORE_EXCHANGE,
                        exchange_type="direct",
                        routing_key="%s" % routing_key)

    def store_result(self, task_id, result, status):
        """Mark task as done (executed)."""
        if status == "DONE":
            result = self.prepare_result(result)
        elif status == "FAILURE":
            result = self.prepare_exception(result)

        meta = {"task_id": task_id,
                "result": result,
                "status": status}

        connection = DjangoBrokerConnection()
        publisher = self._publisher_for_task_id(task_id, connection)
        consumer = self._consumer_for_task_id(task_id, connection)
        c.fetch()
        publisher.send(meta, serializer="pickle", immediate=False)
        publisher.close()
        connection.close()

        return result

    def is_done(self, task_id):
        """Returns ``True`` if task with ``task_id`` has been executed."""
        return self.get_status(task_id) == "DONE"

    def get_status(self, task_id):
        """Get the status of a task."""
        return self._get_task_meta_for(task_id)["status"]

    def _get_task_meta_for(self, task_id):
        results = []

        def callback(message_data, message):
            results.append(message_data)
            message.ack()

        routing_key = task_id.replace("-", "")

        connection = DjangoBrokerConnection()
        consumer = self._consumer_for_task_id(task_id, connection)
        consumer.register_callback(callback)

        try:
            consumer.iterconsume().next()
        finally:
            consumer.close()
            connection.close()

        return results[0]

    def get_result(self, task_id):
        """Get the result for a task."""
        result = self._get_task_meta_for(task_id)
        if result["status"] == "FAILURE":
            return self.exception_to_python(result["result"])
        else:
            return result["result"]

    def cleanup(self):
        """Delete expired metadata."""
        pass
<commit_msg>AMQP result-store backend is now working (and it's fast!)<commit_after>"""celery.backends.amqp"""
from carrot.connection import DjangoBrokerConnection
from carrot.messaging import Consumer, Publisher
from celery.backends.base import BaseBackend

RESULTSTORE_EXCHANGE = "celeryresults"

class Backend(BaseBackend):
    """AMQP backend. Publish results by sending messages to the broker
    using the task id as routing key.

    **NOTE:** Results published using this backend is read-once only.
    After the result has been read, the result is deleted. (however, it's
    still cached locally by the backend instance).

    """

    capabilities = ["ResultStore"]

    def __init__(self, *args, **kwargs):
        super(Backend, self).__init__(*args, **kwargs)
        self.connection = DjangoBrokerConnection()
        self._cache = {}

    def _declare_queue(self, task_id, connection):
        routing_key = task_id.replace("-", "")
        backend = connection.create_backend()
        backend.queue_declare(queue=routing_key, durable=True,
                                exclusive=False, auto_delete=True)
        backend.exchange_declare(exchange=RESULTSTORE_EXCHANGE,
                                 type="direct",
                                 durable=True,
                                 auto_delete=False)
        backend.queue_bind(queue=routing_key, exchange=RESULTSTORE_EXCHANGE,
                           routing_key=routing_key)
        backend.close()

    def _publisher_for_task_id(self, task_id, connection):
        routing_key = task_id.replace("-", "")
        self._declare_queue(task_id, connection)
        p = Publisher(connection, exchange=RESULTSTORE_EXCHANGE,
                      exchange_type="direct",
                      routing_key=routing_key)
        return p

    def _consumer_for_task_id(self, task_id, connection):
        routing_key = task_id.replace("-", "")
        self._declare_queue(task_id, connection)
        return Consumer(connection, queue=routing_key,
                        exchange=RESULTSTORE_EXCHANGE,
                        exchange_type="direct",
                        no_ack=False, auto_ack=False,
                        auto_delete=True,
                        routing_key=routing_key)

    def store_result(self, task_id, result, status):
        """Send task return value and status."""
        if status == "DONE":
            result = self.prepare_result(result)
        elif status == "FAILURE":
            result = self.prepare_exception(result)

        meta = {"task_id": task_id,
                "result": result,
                "status": status}

        connection = self.connection
        publisher = self._publisher_for_task_id(task_id, connection)
        publisher.send(meta, serializer="pickle")
        publisher.close()

        print("SENT %s RESULT: %s TO %s" % (
            status, result, task_id.replace("-", "")))
        return result

    def is_done(self, task_id):
        """Returns ``True`` if task with ``task_id`` has been executed."""
        return self.get_status(task_id) == "DONE"

    def get_status(self, task_id):
        """Get the status of a task."""
        return self._get_task_meta_for(task_id)["status"]

    def _get_task_meta_for(self, task_id):

        if task_id in self._cache:
            return self._cache[task_id]

        results = []

        def callback(message_data, message):
            results.append(message_data)
            message.ack()

        routing_key = task_id.replace("-", "")

        connection = self.connection
        consumer = self._consumer_for_task_id(task_id, connection)
        consumer.register_callback(callback)

        try:
            consumer.iterconsume().next()
        finally:
            consumer.backend.channel.queue_delete(routing_key)
            consumer.close()

        self._cache[task_id] = results[0]
        return results[0]

    def get_result(self, task_id):
        """Get the result for a task."""
        result = self._get_task_meta_for(task_id)
        if result["status"] == "FAILURE":
            return self.exception_to_python(result["result"])
        else:
            return result["result"]
<|endoftext|>


<commit_before>"""celery.backends.base"""
from celery.timer import TimeoutTimer
try:
    import cPickle as pickle
except ImportError:
    import pickle

def find_nearest_pickleable_exception(exc):
    """With an exception instance, iterate over its super classes (by mro)
    and find the first super exception that is pickleable. It does
    not go below :exc:`Exception` (i.e. it skips :exc:`Exception`,
    :class:`BaseException` and :class:`object`). If that happens
    you should use :exc:`UnpickleableException` instead.

    :param exc: An exception instance.

    :returns: the nearest exception if it's not :exc:`Exception` or below,
        if it is it returns ``None``.

    :rtype: :exc:`Exception`

    """
    for supercls in exc.__class__.mro():
        if supercls is Exception:
            # only BaseException and object, from here on down,
            # we don't care about these.
            return None
        try:
            superexc = supercls(*exc.args)
            pickle.dumps(superexc)
        except:
            pass
        else:
            return superexc
    return None

class UnpickleableExceptionWrapper(Exception):
    """Wraps unpickleable exceptions.

    :param exc_module: see :attr:`exc_module`.

    :param exc_cls_name: see :attr:`exc_cls_name`.

    :param exc_args: see :attr:`exc_args`

    .. attribute:: exc_module

        The module of the original exception.

    .. attribute:: exc_cls_name

        The name of the original exception class.

    .. attribute:: exc_args

        The arguments for the original exception.

    Example

        >>> try:
        ...     something_raising_unpickleable_exc()
        >>> except Exception, e:
        ...     exc = UnpickleableException(e.__class__.__module__,
        ...                                 e.__class__.__name__,
        ...                                 e.args)
        ...     pickle.dumps(exc) # Works fine.

    """

    def __init__(self, exc_module, exc_cls_name, exc_args):
        self.exc_module = exc_module
        self.exc_cls_name = exc_cls_name
        self.exc_args = exc_args
        super(Exception, self).__init__(exc_module, exc_cls_name, exc_args)

class BaseBackend(object):
    """The base backend class. All backends should inherit from this."""

    capabilities = []
    UnpickleableExceptionWrapper = UnpickleableExceptionWrapper

    def store_result(self, task_id, result, status):
        """Store the result and status of a task."""
        raise NotImplementedError(
                "store_result is not supported by this backend.")

    def mark_as_done(self, task_id, result):
        """Mark task as successfully executed."""
        return self.store_result(task_id, result, status="DONE")

    def mark_as_failure(self, task_id, exc):
        """Mark task as executed with failure. Stores the execption."""
        return self.store_result(task_id, exc, status="FAILURE")

    def create_exception_cls(self, name, module, parent=None):
        """Dynamically create an exception class."""
        if not parent:
            parent = Exception
        return type(name, (parent, ), {"__module__": module})

    def prepare_exception(self, exc):
        """Prepare exception for serialization."""
        nearest = find_nearest_pickleable_exception(exc)
        if nearest:
            return nearest

        try:
            pickle.dumps(exc)
        except pickle.PickleError:
            excwrapper = UnpickleableExceptionWrapper(
                            exc.__class__.__module__,
                            exc.__class__.__name__,
                            exc.args)
            return excwrapper
        else:
            return exc

    def exception_to_python(self, exc):
        """Convert serialized exception to Python exception."""
        if isinstance(exc, UnpickleableExceptionWrapper):
            exc_cls = self.create_exception_cls(exc.exc_cls_name,
                                                exc.exc_module)
            return exc_cls(*exc.exc_args)
        return exc

    def mark_as_retry(self, task_id, exc):
        """Mark task for retry."""
        return self.store_result(task_id, exc, status="RETRY")

    def get_status(self, task_id):
        """Get the status of a task."""
        raise NotImplementedError(
                "get_status is not supported by this backend.")

    def prepare_result(self, result):
        """Prepare result for storage."""
        if result is None:
            return True
        return result

    def get_result(self, task_id):
        """Get the result of a task."""
        raise NotImplementedError(
                "get_result is not supported by this backend.")

    def is_done(self, task_id):
        """Returns ``True`` if the task was successfully executed."""
        return self.get_status(task_id) == "DONE"

    def cleanup(self):
        """Backend cleanup. Is run by
        :class:`celery.task.DeleteExpiredTaskMetaTask`."""
        pass

    def wait_for(self, task_id, timeout=None):
        """Wait for task and return its result.

        If the task raises an exception, this exception
        will be re-raised by :func:`wait_for`.

        If ``timeout`` is not ``None``, this raises the
        :class:`celery.timer.TimeoutError` exception if the operation takes
        longer than ``timeout`` seconds.

        """
        timeout_timer = TimeoutTimer(timeout)
        while True:
            status = self.get_status(task_id)
            if status == "DONE":
                return self.get_result(task_id)
            elif status == "FAILURE":
                raise self.get_result(task_id)
            timeout_timer.tick()

    def cleanup_process(self):
        """Cleanup actions to do at the end of a task worker process.

        See :func:`celery.worker.jail`.

        """
        pass
<commit_msg>Rename cleanup_process -> process_cleanup<commit_after>"""celery.backends.base"""
from celery.timer import TimeoutTimer
try:
    import cPickle as pickle
except ImportError:
    import pickle

def find_nearest_pickleable_exception(exc):
    """With an exception instance, iterate over its super classes (by mro)
    and find the first super exception that is pickleable. It does
    not go below :exc:`Exception` (i.e. it skips :exc:`Exception`,
    :class:`BaseException` and :class:`object`). If that happens
    you should use :exc:`UnpickleableException` instead.

    :param exc: An exception instance.

    :returns: the nearest exception if it's not :exc:`Exception` or below,
        if it is it returns ``None``.

    :rtype: :exc:`Exception`

    """
    for supercls in exc.__class__.mro():
        if supercls is Exception:
            # only BaseException and object, from here on down,
            # we don't care about these.
            return None
        try:
            superexc = supercls(*exc.args)
            pickle.dumps(superexc)
        except:
            pass
        else:
            return superexc
    return None

class UnpickleableExceptionWrapper(Exception):
    """Wraps unpickleable exceptions.

    :param exc_module: see :attr:`exc_module`.

    :param exc_cls_name: see :attr:`exc_cls_name`.

    :param exc_args: see :attr:`exc_args`

    .. attribute:: exc_module

        The module of the original exception.

    .. attribute:: exc_cls_name

        The name of the original exception class.

    .. attribute:: exc_args

        The arguments for the original exception.

    Example

        >>> try:
        ...     something_raising_unpickleable_exc()
        >>> except Exception, e:
        ...     exc = UnpickleableException(e.__class__.__module__,
        ...                                 e.__class__.__name__,
        ...                                 e.args)
        ...     pickle.dumps(exc) # Works fine.

    """

    def __init__(self, exc_module, exc_cls_name, exc_args):
        self.exc_module = exc_module
        self.exc_cls_name = exc_cls_name
        self.exc_args = exc_args
        super(Exception, self).__init__(exc_module, exc_cls_name, exc_args)

class BaseBackend(object):
    """The base backend class. All backends should inherit from this."""

    capabilities = []
    UnpickleableExceptionWrapper = UnpickleableExceptionWrapper

    def store_result(self, task_id, result, status):
        """Store the result and status of a task."""
        raise NotImplementedError(
                "store_result is not supported by this backend.")

    def mark_as_done(self, task_id, result):
        """Mark task as successfully executed."""
        return self.store_result(task_id, result, status="DONE")

    def mark_as_failure(self, task_id, exc):
        """Mark task as executed with failure. Stores the execption."""
        return self.store_result(task_id, exc, status="FAILURE")

    def create_exception_cls(self, name, module, parent=None):
        """Dynamically create an exception class."""
        if not parent:
            parent = Exception
        return type(name, (parent, ), {"__module__": module})

    def prepare_exception(self, exc):
        """Prepare exception for serialization."""
        nearest = find_nearest_pickleable_exception(exc)
        if nearest:
            return nearest

        try:
            pickle.dumps(exc)
        except pickle.PickleError:
            excwrapper = UnpickleableExceptionWrapper(
                            exc.__class__.__module__,
                            exc.__class__.__name__,
                            exc.args)
            return excwrapper
        else:
            return exc

    def exception_to_python(self, exc):
        """Convert serialized exception to Python exception."""
        if isinstance(exc, UnpickleableExceptionWrapper):
            exc_cls = self.create_exception_cls(exc.exc_cls_name,
                                                exc.exc_module)
            return exc_cls(*exc.exc_args)
        return exc

    def mark_as_retry(self, task_id, exc):
        """Mark task for retry."""
        return self.store_result(task_id, exc, status="RETRY")

    def get_status(self, task_id):
        """Get the status of a task."""
        raise NotImplementedError(
                "get_status is not supported by this backend.")

    def prepare_result(self, result):
        """Prepare result for storage."""
        if result is None:
            return True
        return result

    def get_result(self, task_id):
        """Get the result of a task."""
        raise NotImplementedError(
                "get_result is not supported by this backend.")

    def is_done(self, task_id):
        """Returns ``True`` if the task was successfully executed."""
        return self.get_status(task_id) == "DONE"

    def cleanup(self):
        """Backend cleanup. Is run by
        :class:`celery.task.DeleteExpiredTaskMetaTask`."""
        pass

    def wait_for(self, task_id, timeout=None):
        """Wait for task and return its result.

        If the task raises an exception, this exception
        will be re-raised by :func:`wait_for`.

        If ``timeout`` is not ``None``, this raises the
        :class:`celery.timer.TimeoutError` exception if the operation takes
        longer than ``timeout`` seconds.

        """
        timeout_timer = TimeoutTimer(timeout)
        while True:
            status = self.get_status(task_id)
            if status == "DONE":
                return self.get_result(task_id)
            elif status == "FAILURE":
                raise self.get_result(task_id)
            timeout_timer.tick()

    def process_cleanup(self):
        """Cleanup actions to do at the end of a task worker process.

        See :func:`celery.worker.jail`.

        """
        pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
hyper/http20/response
~~~~~~~~~~~~~~~~~~~~~

Contains the HTTP/2 equivalent of the HTTPResponse object defined in
httplib/http.client.
"""
import logging
import zlib

from .util import pop_from_key_value_set

log = logging.getLogger(__name__)

class DeflateDecoder(object):
    """
    This is a decoding object that wraps ``zlib`` and is used for decoding
    deflated content.

    This rationale for the existence of this object is pretty unpleasant.
    The HTTP RFC specifies that 'deflate' is a valid content encoding. However,
    the spec _meant_ the zlib encoding form. Unfortunately, people who didn't
    read the RFC very carefully actually implemented a different form of
    'deflate'. Insanely, ``zlib`` handles them using two wbits values. This is
    such a mess it's hard to adequately articulate.

    This class was lovingly borrowed from the excellent urllib3 library under
    license: see NOTICES. If you ever see @shazow, you should probably buy him
    a drink or something.
    """
    def __init__(self):
        self._first_try = True
        self._data = b''
        self._obj = zlib.decompressobj(zlib.MAX_WBITS)

    def __getattr__(self, name):
        return getattr(self._obj, name)

    def decompress(self, data):
        if not self._first_try:
            return self._obj.decompress(data)

        self._data += data
        try:
            return self._obj.decompress(data)
        except zlib.error:
            self._first_try = False
            self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
            try:
                return self.decompress(self._data)
            finally:
                self._data = None

class Headers(object):
    def __init__(self, pairs):
        # This conversion to dictionary is unwise, as there may be repeated
        # keys, but it's acceptable for an early alpha.
        self._headers = dict(pairs)
        self._strip_headers()

    def getheader(self, name, default=None):
        return self._headers.get(name, default)

    def getheaders(self):
        return list(self._headers.items())

    def items(self):
        return self._headers.items()

    def merge(self, headers):
        for n, v in headers:
            self._headers[n] = v
        self._strip_headers()

    def _strip_headers(self):
        """
        Strips the headers attached to the instance of any header beginning
        with a colon that ``hyper`` doesn't understand. This method logs at
        warning level about the deleted headers, for discoverability.
        """
        # Convert to list to ensure that we don't mutate the headers while
        # we iterate over them.
        for name in list(self._headers.keys()):
            if name.startswith(':'):
                val = self._headers.pop(name)
                log.warning("Unknown reserved header: %s: %s", name, val)

class HTTP20Response(object):
    """
    An ``HTTP20Response`` wraps the HTTP/2 response from the server. It
    provides access to the response headers and the entity body. The response
    is an iterable object and can be used in a with statement (though due to
    the persistent connections used in HTTP/2 this has no effect, and is done
    soley for compatibility).
    """
    def __init__(self, headers, stream):
        #: The reason phrase returned by the server. This is not used in
        #: HTTP/2, and so is always the empty string.
        self.reason = ''

        status = pop_from_key_value_set(headers, ':status')[0]

        #: The status code returned by the server.
        self.status = int(status)

        # The response headers. These are determined upon creation, assigned
        # once, and never assigned again.
        self._headers = Headers(headers)

        # The response trailers. These are always intially ``None``.
        self._trailers = None

        # The stream this response is being sent over.
        self._stream = stream

        # We always read in one-data-frame increments from the stream, so we
        # may need to buffer some for incomplete reads.
        self._data_buffer = b''

        # This object is used for decompressing gzipped request bodies. Right
        # now we only support gzip because that's all the RFC mandates of us.
        # Later we'll add support for more encodings.
        # This 16 + MAX_WBITS nonsense is to force gzip. See this
        # Stack Overflow answer for more:
        # http://stackoverflow.com/a/2695466/1401686
        if self._headers.getheader('content-encoding') == 'gzip':
            self._decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
        elif self._headers.getheader('content-encoding') == 'deflate':
            self._decompressobj = DeflateDecoder()
        else:
            self._decompressobj = None

    def read(self, amt=None, decode_content=True):
        """
        Reads the response body, or up to the next ``amt`` bytes.

        :param amt: (optional) The amount of data to read. If not provided, all
            the data will be read from the response.
        :param decode_content: (optional) If ``True``, will transparently
            decode the response data.
        :returns: The read data. Note that if ``decode_content`` is set to
            ``True``, the actual amount of data returned may be different to
            the amount requested.
        """
        if amt is not None and amt <= len(self._data_buffer):
            data = self._data_buffer[:amt]
            self._data_buffer = self._data_buffer[amt:]
            response_complete = False
        elif amt is not None:
            read_amt = amt - len(self._data_buffer)
            self._data_buffer += self._stream._read(read_amt)
            data = self._data_buffer[:amt]
            self._data_buffer = self._data_buffer[amt:]
            response_complete = len(data) < amt
        else:
            data = b''.join([self._data_buffer, self._stream._read()])
            response_complete = True

        # We may need to decode the body.
        if decode_content and self._decompressobj and data:
            data = self._decompressobj.decompress(data)

        # If we're at the end of the request, we have some cleaning up to do.
        # Close the stream, and if necessary flush the buffer.
        if response_complete:
            if decode_content and self._decompressobj:
                data += self._decompressobj.flush()

            if self._stream.response_headers:
                self._headers.merge(self._stream.response_headers)

        # We're at the end. Close the connection.
        if not data:
            self.close()

        return data

    def getheader(self, name, default=None):
        """
        Return the value of the header ``name``, or ``default`` if there is no
        header matching ``name``. If there is more than one header with the
        value ``name``, return all of the values joined by ', '. If ``default``
        is any iterable other than a single string, its elements are similarly
        returned joined by commas.

        :param name: The name of the header to get the value of.
        :param default: (optional) The return value if the header wasn't sent.
        :returns: The value of the header.
        """
        return self._headers.getheader(name, default)

    def getheaders(self):
        """
        Get all the headers sent on the response.

        :returns: A list of ``(header, value)`` tuples.
        """
        return list(self._headers.items())

    def gettrailer(self, name, default=None):
        """
        Return the value of the trailer ``name``, or ``default`` if there is no
        trailer matching ``name``. If there is more than one trailer with the
        value ``name``, return all of the values joined by ', '. If ``default``
        is any iterable other than a single string, its elements are similarly
        returned joined by commas.

        .. warning:: Note that this method requires that the stream is
                     totally exhausted. This means that, if you have not
                     completely read from the stream, all stream data will be
                     read into memory.

        :param name: The name of the trailer to get the value of.
        :param default: (optional) The return value if the trailer wasn't sent.
        :returns: The value of the trailer.
        """
        # We need to get the trailers.
        if self._trailers is None:
            trailers = self.stream.gettrailers() or []
            self._trailers = Headers(trailers)

        return self._trailers.getheader(name, default)

    def gettrailers(self):
        """
        Get all the trailers sent on the response.

        .. warning:: Note that this method requires that the stream is
                     totally exhausted. This means that, if you have not
                     completely read from the stream, all stream data will be
                     read into memory.

        :returns: A list of ``(header, value)`` tuples.
        """
        # We need to get the trailers.
        if self._trailers is None:
            trailers = self.stream.gettrailers() or []
            self._trailers = Headers(trailers)

        return list(self._trailers.items())

    def fileno(self):
        """
        Return the ``fileno`` of the underlying socket. This function is
        currently not implemented.
        """
        raise NotImplementedError("Not currently implemented.")

    def close(self):
        """
        Close the response. In effect this closes the backing HTTP/2 stream.

        :returns: Nothing.
        """
        self._stream.close()

    # The following methods implement the context manager protocol.
    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()
        return False  # Never swallow exceptions.

class HTTP20Push(object):
    """
    Represents a request-response pair sent by the server through the server
    push mechanism.
    """
    def __init__(self, request_headers, stream):
        scheme, method, authority, path = (
            pop_from_key_value_set(request_headers,
                                   ':scheme', ':method', ':authority', ':path')
        )
        #: The scheme of the simulated request
        self.scheme = scheme
        #: The method of the simulated request (must be safe and cacheable, e.g. GET)
        self.method = method
        #: The authority of the simulated request (usually host:port)
        self.authority = authority
        #: The path of the simulated request
        self.path = path

        self._request_headers = Headers(request_headers)
        self._stream = stream

    def getrequestheader(self, name, default=None):
        """
        Return the value of the simulated request header ``name``, or ``default``
        if there is no header matching ``name``. If there is more than one header
        with the value ``name``, return all of the values joined by ``', '``.
        If ``default`` is any iterable other than a single string, its elements
        are similarly returned joined by commas.

        :param name: The name of the header to get the value of.
        :param default: (optional) The return value if the header wasn't sent.
        :returns: The value of the header.
        """
        return self._request_headers.getheader(name, default)

    def getrequestheaders(self):
        """
        Get all the simulated request headers.

        :returns: A list of ``(header, value)`` tuples.
        """
        return self._request_headers.getheaders()

    def getresponse(self):
        """
        Get the pushed response provided by the server.

        :returns: A :class:`HTTP20Response <hyper.HTTP20Response>` object
            representing the pushed response.
        """
        return HTTP20Response(self._stream.getheaders(), self._stream)

    def cancel(self):
        """
        Cancel the pushed response and close the stream.

        :returns: Nothing.
        """
        self._stream.close(8) # CANCEL
<commit_msg>HTTP20Response has no attribute stream.<commit_after># -*- coding: utf-8 -*-
"""
hyper/http20/response
~~~~~~~~~~~~~~~~~~~~~

Contains the HTTP/2 equivalent of the HTTPResponse object defined in
httplib/http.client.
"""
import logging
import zlib

from .util import pop_from_key_value_set

log = logging.getLogger(__name__)

class DeflateDecoder(object):
    """
    This is a decoding object that wraps ``zlib`` and is used for decoding
    deflated content.

    This rationale for the existence of this object is pretty unpleasant.
    The HTTP RFC specifies that 'deflate' is a valid content encoding. However,
    the spec _meant_ the zlib encoding form. Unfortunately, people who didn't
    read the RFC very carefully actually implemented a different form of
    'deflate'. Insanely, ``zlib`` handles them using two wbits values. This is
    such a mess it's hard to adequately articulate.

    This class was lovingly borrowed from the excellent urllib3 library under
    license: see NOTICES. If you ever see @shazow, you should probably buy him
    a drink or something.
    """
    def __init__(self):
        self._first_try = True
        self._data = b''
        self._obj = zlib.decompressobj(zlib.MAX_WBITS)

    def __getattr__(self, name):
        return getattr(self._obj, name)

    def decompress(self, data):
        if not self._first_try:
            return self._obj.decompress(data)

        self._data += data
        try:
            return self._obj.decompress(data)
        except zlib.error:
            self._first_try = False
            self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
            try:
                return self.decompress(self._data)
            finally:
                self._data = None

class Headers(object):
    def __init__(self, pairs):
        # This conversion to dictionary is unwise, as there may be repeated
        # keys, but it's acceptable for an early alpha.
        self._headers = dict(pairs)
        self._strip_headers()

    def getheader(self, name, default=None):
        return self._headers.get(name, default)

    def getheaders(self):
        return list(self._headers.items())

    def items(self):
        return self._headers.items()

    def merge(self, headers):
        for n, v in headers:
            self._headers[n] = v
        self._strip_headers()

    def _strip_headers(self):
        """
        Strips the headers attached to the instance of any header beginning
        with a colon that ``hyper`` doesn't understand. This method logs at
        warning level about the deleted headers, for discoverability.
        """
        # Convert to list to ensure that we don't mutate the headers while
        # we iterate over them.
        for name in list(self._headers.keys()):
            if name.startswith(':'):
                val = self._headers.pop(name)
                log.warning("Unknown reserved header: %s: %s", name, val)

class HTTP20Response(object):
    """
    An ``HTTP20Response`` wraps the HTTP/2 response from the server. It
    provides access to the response headers and the entity body. The response
    is an iterable object and can be used in a with statement (though due to
    the persistent connections used in HTTP/2 this has no effect, and is done
    soley for compatibility).
    """
    def __init__(self, headers, stream):
        #: The reason phrase returned by the server. This is not used in
        #: HTTP/2, and so is always the empty string.
        self.reason = ''

        status = pop_from_key_value_set(headers, ':status')[0]

        #: The status code returned by the server.
        self.status = int(status)

        # The response headers. These are determined upon creation, assigned
        # once, and never assigned again.
        self._headers = Headers(headers)

        # The response trailers. These are always intially ``None``.
        self._trailers = None

        # The stream this response is being sent over.
        self._stream = stream

        # We always read in one-data-frame increments from the stream, so we
        # may need to buffer some for incomplete reads.
        self._data_buffer = b''

        # This object is used for decompressing gzipped request bodies. Right
        # now we only support gzip because that's all the RFC mandates of us.
        # Later we'll add support for more encodings.
        # This 16 + MAX_WBITS nonsense is to force gzip. See this
        # Stack Overflow answer for more:
        # http://stackoverflow.com/a/2695466/1401686
        if self._headers.getheader('content-encoding') == 'gzip':
            self._decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
        elif self._headers.getheader('content-encoding') == 'deflate':
            self._decompressobj = DeflateDecoder()
        else:
            self._decompressobj = None

    def read(self, amt=None, decode_content=True):
        """
        Reads the response body, or up to the next ``amt`` bytes.

        :param amt: (optional) The amount of data to read. If not provided, all
            the data will be read from the response.
        :param decode_content: (optional) If ``True``, will transparently
            decode the response data.
        :returns: The read data. Note that if ``decode_content`` is set to
            ``True``, the actual amount of data returned may be different to
            the amount requested.
        """
        if amt is not None and amt <= len(self._data_buffer):
            data = self._data_buffer[:amt]
            self._data_buffer = self._data_buffer[amt:]
            response_complete = False
        elif amt is not None:
            read_amt = amt - len(self._data_buffer)
            self._data_buffer += self._stream._read(read_amt)
            data = self._data_buffer[:amt]
            self._data_buffer = self._data_buffer[amt:]
            response_complete = len(data) < amt
        else:
            data = b''.join([self._data_buffer, self._stream._read()])
            response_complete = True

        # We may need to decode the body.
        if decode_content and self._decompressobj and data:
            data = self._decompressobj.decompress(data)

        # If we're at the end of the request, we have some cleaning up to do.
        # Close the stream, and if necessary flush the buffer.
        if response_complete:
            if decode_content and self._decompressobj:
                data += self._decompressobj.flush()

            if self._stream.response_headers:
                self._headers.merge(self._stream.response_headers)

        # We're at the end. Close the connection.
        if not data:
            self.close()

        return data

    def getheader(self, name, default=None):
        """
        Return the value of the header ``name``, or ``default`` if there is no
        header matching ``name``. If there is more than one header with the
        value ``name``, return all of the values joined by ', '. If ``default``
        is any iterable other than a single string, its elements are similarly
        returned joined by commas.

        :param name: The name of the header to get the value of.
        :param default: (optional) The return value if the header wasn't sent.
        :returns: The value of the header.
        """
        return self._headers.getheader(name, default)

    def getheaders(self):
        """
        Get all the headers sent on the response.

        :returns: A list of ``(header, value)`` tuples.
        """
        return list(self._headers.items())

    def gettrailer(self, name, default=None):
        """
        Return the value of the trailer ``name``, or ``default`` if there is no
        trailer matching ``name``. If there is more than one trailer with the
        value ``name``, return all of the values joined by ', '. If ``default``
        is any iterable other than a single string, its elements are similarly
        returned joined by commas.

        .. warning:: Note that this method requires that the stream is
                     totally exhausted. This means that, if you have not
                     completely read from the stream, all stream data will be
                     read into memory.

        :param name: The name of the trailer to get the value of.
        :param default: (optional) The return value if the trailer wasn't sent.
        :returns: The value of the trailer.
        """
        # We need to get the trailers.
        if self._trailers is None:
            trailers = self._stream.gettrailers() or []
            self._trailers = Headers(trailers)

        return self._trailers.getheader(name, default)

    def gettrailers(self):
        """
        Get all the trailers sent on the response.

        .. warning:: Note that this method requires that the stream is
                     totally exhausted. This means that, if you have not
                     completely read from the stream, all stream data will be
                     read into memory.

        :returns: A list of ``(header, value)`` tuples.
        """
        # We need to get the trailers.
        if self._trailers is None:
            trailers = self._stream.gettrailers() or []
            self._trailers = Headers(trailers)

        return list(self._trailers.items())

    def fileno(self):
        """
        Return the ``fileno`` of the underlying socket. This function is
        currently not implemented.
        """
        raise NotImplementedError("Not currently implemented.")

    def close(self):
        """
        Close the response. In effect this closes the backing HTTP/2 stream.

        :returns: Nothing.
        """
        self._stream.close()

    # The following methods implement the context manager protocol.
    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()
        return False  # Never swallow exceptions.

class HTTP20Push(object):
    """
    Represents a request-response pair sent by the server through the server
    push mechanism.
    """
    def __init__(self, request_headers, stream):
        scheme, method, authority, path = (
            pop_from_key_value_set(request_headers,
                                   ':scheme', ':method', ':authority', ':path')
        )
        #: The scheme of the simulated request
        self.scheme = scheme
        #: The method of the simulated request (must be safe and cacheable, e.g. GET)
        self.method = method
        #: The authority of the simulated request (usually host:port)
        self.authority = authority
        #: The path of the simulated request
        self.path = path

        self._request_headers = Headers(request_headers)
        self._stream = stream

    def getrequestheader(self, name, default=None):
        """
        Return the value of the simulated request header ``name``, or ``default``
        if there is no header matching ``name``. If there is more than one header
        with the value ``name``, return all of the values joined by ``', '``.
        If ``default`` is any iterable other than a single string, its elements
        are similarly returned joined by commas.

        :param name: The name of the header to get the value of.
        :param default: (optional) The return value if the header wasn't sent.
        :returns: The value of the header.
        """
        return self._request_headers.getheader(name, default)

    def getrequestheaders(self):
        """
        Get all the simulated request headers.

        :returns: A list of ``(header, value)`` tuples.
        """
        return self._request_headers.getheaders()

    def getresponse(self):
        """
        Get the pushed response provided by the server.

        :returns: A :class:`HTTP20Response <hyper.HTTP20Response>` object
            representing the pushed response.
        """
        return HTTP20Response(self._stream.getheaders(), self._stream)

    def cancel(self):
        """
        Cancel the pushed response and close the stream.

        :returns: Nothing.
        """
        self._stream.close(8) # CANCEL
<|endoftext|>


<commit_before>
import os
import unicodecsv

from iatidq import app

YEAR_MAP_FILE = '2012_2013_organisation_mapping.csv'
OLD_YEAR = '2012'
NEW_YEAR = '2013'
OLD_FIELD_ID = OLD_YEAR + '_id'
NEW_FIELD_ID = NEW_YEAR + '_id'
OLD_INDICATORS_FILE = OLD_YEAR + '_indicators.csv'
OLD_RESULTS_FILE = OLD_YEAR + '_results.csv'
NEW_INDICATOR_NAME = NEW_YEAR + '_indicator_name'

def get_old_organisation_id(organisation_code='GB-1'):
    path = app.config["DATA_STORAGE_DIR"]
    old_organisation_file = os.path.join(path, YEAR_MAP_FILE)

    old_organisation_data = unicodecsv.DictReader(file(old_organisation_file))
    for row in old_organisation_data:
        if row[NEW_FIELD_ID] == organisation_code:
            return row[OLD_FIELD_ID]

def get_old_indicators():
    path = app.config["DATA_STORAGE_DIR"]
    old_indicators_file = os.path.join(path, OLD_INDICATORS_FILE)
    old_indicators_data = unicodecsv.DictReader(file(old_indicators_file))

    indicator_data = {}
    for row in old_indicators_data:
        if ((row["question_number"]) and (row[NEW_INDICATOR_NAME])):
            indicator_data[int(row["question_number"])] = row[NEW_INDICATOR_NAME]
    return indicator_data

# read in(!) a CSV file of last year's results; it must have fields

# id              - not used
# question_id     - the indicator id of the question in the old data
# target_id       - the organisation ID in the old data
# result          - used in templates
# result_evidence - used in templates
# result_comments - used in templates
# result_review   - not used

def get_organisation_results(organisation_code, newindicators):
    old_organisation_id = get_old_organisation_id(organisation_code)
    old_indicators = get_old_indicators()

    path = app.config["DATA_STORAGE_DIR"]

    old_results_file = os.path.join(path, OLD_RESULTS_FILE)
    old_results_data = unicodecsv.DictReader(file(old_results_file))

    data = {}

    for d in old_results_data:
        if d["target_id"] == old_organisation_id:
            assert "question_id" in d
            question_id = int(d["question_id"])
            if question_id in old_indicators:
                data[old_indicators[question_id]] = d

    for indicator_name in newindicators:
        if indicator_name not in data:
            data[indicator_name] = { 'result': '' }

    return data
<commit_msg>add comment<commit_after>
import os
import unicodecsv

from iatidq import app

YEAR_MAP_FILE = '2012_2013_organisation_mapping.csv'
OLD_YEAR = '2012'
NEW_YEAR = '2013'
OLD_FIELD_ID = OLD_YEAR + '_id'
NEW_FIELD_ID = NEW_YEAR + '_id'
OLD_INDICATORS_FILE = OLD_YEAR + '_indicators.csv'
OLD_RESULTS_FILE = OLD_YEAR + '_results.csv'
NEW_INDICATOR_NAME = NEW_YEAR + '_indicator_name'

def get_old_organisation_id(organisation_code='GB-1'):
    path = app.config["DATA_STORAGE_DIR"]
    old_organisation_file = os.path.join(path, YEAR_MAP_FILE)

    old_organisation_data = unicodecsv.DictReader(file(old_organisation_file))
    for row in old_organisation_data:
        if row[NEW_FIELD_ID] == organisation_code:
            return row[OLD_FIELD_ID]

def get_old_indicators():
    path = app.config["DATA_STORAGE_DIR"]
    old_indicators_file = os.path.join(path, OLD_INDICATORS_FILE)
    old_indicators_data = unicodecsv.DictReader(file(old_indicators_file))

    indicator_data = {}
    for row in old_indicators_data:
        if ((row["question_number"]) and (row[NEW_INDICATOR_NAME])):
            indicator_data[int(row["question_number"])] = row[NEW_INDICATOR_NAME]
    return indicator_data

# read in(!) a CSV file of last year's results; it must have fields

# id              - not used
# question_id     - the indicator id of the question in the old data
# target_id       - the organisation ID in the old data
# result          - used in templates
# result_evidence - used in templates
# result_comments - used in templates
# result_review   - not used

def get_organisation_results(organisation_code, newindicators):
    old_organisation_id = get_old_organisation_id(organisation_code)
    old_indicators = get_old_indicators()

    path = app.config["DATA_STORAGE_DIR"]

    old_results_file = os.path.join(path, OLD_RESULTS_FILE)
    old_results_data = unicodecsv.DictReader(file(old_results_file))

    data = {}

    # old_indicators is a map from int -> string, e.g., 1 -> "foia"

    for d in old_results_data:
        if d["target_id"] == old_organisation_id:
            assert "question_id" in d
            question_id = int(d["question_id"])
            if question_id in old_indicators:
                data[old_indicators[question_id]] = d

    for indicator_name in newindicators:
        if indicator_name not in data:
            data[indicator_name] = { 'result': '' }

    return data
<|endoftext|>