<commit_before>import logging

logger = logging.getLogger(__name__)

import os
import copy
import bisect
import pysam
import pybedtools
import vcf

svs_of_interest = ["DEL", "INS", "DUP", "DUP:TANDEM", "INV"]
sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller", "BreakDancer", "CNVnator"] # order is important!
precise_sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller"]
sv_sources_to_type = {"Pindel": "SR", "BreakSeq": "JM", "BreakDancer": "RP", "CNVnator": "RD", "HaplotypeCaller": "AS"}

mydir = os.path.dirname(os.path.realpath(__file__))
gaps_b37 = os.path.join(mydir, "resources/b37.gaps.bed")
gaps_hg19 = os.path.join(mydir, "resources/hg19.gaps.bed")

def get_gaps_file(contig_names):
    hg19_major = set(["chr%d" % i for i in xrange(1, 23)] + ["chrX", "chrY", "chrM"])
    b37_major = set([str(i) for i in xrange(1, 23)] + ["X", "Y", "MT"])
    if set(contig_names) & hg19_major: return gaps_hg19
    if set(contig_names) & b37_major: return gaps_b37

    logger.warn("Could not guess gaps file for reference. No gap filtering will be done.")
    return None

class SVInterval:
    def __init__(self, chrom=None, start=0, end=0, name=None, sv_type=None, length=0, sources=set(), gt="./1", wiggle=0,
                 info=None, cipos=[], ciend=[], native_sv=None):
        self.chrom = chrom
        self.start = start
        self.end = end
        self.name = name
        self.sv_type = sv_type
        self.length = length
        self.info = info
        self.sources = sources
        self.gt = gt
        self.wiggle = wiggle
        self.sub_intervals = []
        self.is_precise = False
        self.is_validated = False
        self.validating_interval = None
        self.cipos = cipos
        self.ciend = ciend
        self.native_sv = native_sv

    def merge(self, interval):
        self.start = min(self.start, interval.start - interval.wiggle)
        self.end = max(self.end, interval.end + interval.wiggle)
        self.length = max(self.length, interval.length)
        self.name = self.name + "," + interval.name
        self.sub_intervals.append(interval)
        self.sources |= interval.sources

    def set_merged(self, interval1, interval2):
        self.chrom = interval1.chrom
        self.start = min(interval1.start - interval1.wiggle, interval2.start - interval2.wiggle)
        self.end = max(interval1.end + interval1.wiggle, interval2.end + interval2.wiggle)
        self.length = max(interval1.length, interval2.length)
        self.name = interval1.name + "," + interval2.name
        self.sv_type = interval1.sv_type
        self.info = None
        self.sub_intervals = [interval1, interval2]
        self.sources = interval1.sources | interval2.sources
        self.gt = interval1.gt

    def __lt__(self, other):
        if self.chrom != other.chrom: return self.chrom < other.chrom
        if self.start != other.start: return self.start < other.start
        return self.end < other.end

    def __str__(self):
        if self.sub_intervals:
            return ",".join([str(interval) for interval in self.sub_intervals])
        return "%s-%d-%d-%d-%s" % (self.chrom, self.start, self.end, self.length, ",".join(list(self.sources)))

    def __repr__(self):
        return "<" + self.__class__.__name__ + " " + str(self.__dict__) + ">"

    def overlaps(self, other, min_fraction_self=1e-9, min_fraction_other=1e-9, min_overlap_length_self=1,
                 min_overlap_length_other=1):
        if self.chrom != other.chrom:
            return False
        if max(self.start - self.wiggle, other.start - other.wiggle) \
                >= min(self.end + self.wiggle, other.end + other.wiggle):
            return False

        self_length = float(self.end - self.start + 2 * self.wiggle)
        other_length = float(other.end - other.start + 2 * other.wiggle)
        overlap_length = min(self.end + self.wiggle, other.end + other.wiggle) - max(self.start - self.wiggle,
                                                                                other.start - other.wiggle)
        return float(overlap_length) >= max(min_fraction_self * self_length,
                                            min_fraction_other * other_length) and overlap_length >= max(
            min_fraction_self, min_overlap_length_other)

    def is_adjacent(self, other, gap=0):
        if self.chrom != other.chrom:
            return False
        return (self.end + gap >= other.start and self.end + gap < other.end) or (
        other.end + gap >= self.start and other.end + gap < self.end)

    def get_start(self):
        if not self.sub_intervals:
            return self.start

        return min([interval.get_start() for interval in self.sub_intervals])

    def get_end(self):
        if not self.sub_intervals:
            return self.end

        return max([interval.get_end() for interval in self.sub_intervals])

    def do_validation(self, overlap_ratio=0.5):
        self.start = self.get_start()
        self.end = self.get_end()

        if self.sv_type not in svs_of_interest:
            return

        if not self.sub_intervals:
            # This interval did not overlap anything
            self.is_precise = list(self.sources)[0] in precise_sv_sources
            # if self.sv_type == "INV" and self.length >= 1000: self.is_validated = True
            return

        lists = {source: [] for source in sv_sources}

        precise_merged = merge_intervals(
            [interval for interval in self.sub_intervals if list(interval.sources)[0] in precise_sv_sources])

        for interval in self.sub_intervals:
            # logger.debug("Examining subinterval " + str(interval))
            lists[list(interval.sources)[0]].append(interval)

        for source in sv_sources:
            # If the SV is called multiple times by the same tool, don't validate
            if len(lists[source]) == 1:
                if self.overlaps(lists[source][0], overlap_ratio, overlap_ratio) \
                        or interval_overlaps_interval_list(lists[source][0], precise_merged, overlap_ratio,
                                                           overlap_ratio):
                    self.is_validated = len(self.sources) > 1
                    self.validating_interval = lists[source][0]
                    self.is_precise = source in precise_sv_sources
                    break # This break makes the order of sv_sources important

        if not self.is_validated:
            return

        if self.is_precise and self.validating_interval is not None:
            self.start = self.validating_interval.start
            self.end = self.validating_interval.end
            self.length = self.validating_interval.length
            self.gt = self.validating_interval.gt
            self.info = self.validating_interval.info

            if self.length == 0:
                for source in sv_sources:
                    if source in self.sources:
                        if lists[source] and lists[source][0].length > 0:
                            self.length = lists[source][0].length
                            break

    def fix_pos(self):
        if self.sv_type == "INS":
            mid = (self.start + self.end) / 2
            if not self.is_precise:
                self.end = self.start
                self.cipos = ["0", "%d" % (self.end - self.start)]
                # self.cipos = ["%d" % (mid - self.start), "%d" % (mid - self.end)]
            else:
                self.start = mid
                self.end = mid

    def to_vcf_record(self, fasta_handle=None, sample = ""):
        if self.start <= 0: return None
        if self.sv_type not in svs_of_interest: return None

        # ignore private haplotype caller calls
        if ((not self.sub_intervals) or len(self.sources) == 1) and list(self.sources)[0] == "HaplotypeCaller":
            return None

        # formulate the INFO field
        info = {}
        if not self.sub_intervals:
            info.update(self.info)
        else:
            if self.info:
                info.update(self.info)
            for interval in self.sub_intervals:
                # TODO: this will just overwrite the other dict entries... this should be ok for pass variants
                # TODO: kind of strange
                if interval.info:
                    info.update(interval.info)
        svmethods = [sv_sources_to_type[tool] for tool in self.sources]
        svmethods.sort()
        sv_len = -self.length if self.sv_type == "DEL" else self.length
        info.update({"SVLEN": sv_len,
                "SVTYPE": self.sv_type,
                "SVMETHOD": ",".join(svmethods)})
        if self.sv_type == "DEL" or self.sv_type == "DUP":
            info["END"] = self.end

        if not self.is_precise:
            info.update({"IMPRECISE": None})

        info.update({"VT":"SV"})
        info.update({"SVTOOL":"MetaSVMerge"})
        info.update({"NUM_SVMETHODS":len(self.sources)})
        info.update({"SOURCES":str(self)})
        if self.cipos:
            info.update({"CIPOS": (",".join([str(a) for a in self.cipos]))})

        vcf_record = vcf.model._Record(self.chrom,
                                       self.start - 1,
                                       ".",
                                       fasta_handle.fetch(self.chrom, max(0, self.start - 2), max(1, self.start - 1)) if fasta_handle else "N",
                                       ["<%s>" % (self.sv_type)],
                                       ".",
                                       "PASS" if self.is_validated else "LowQual",
                                       info,
                                       "GT",
                                       [0],
                                       [vcf.model._Call(None, sample, vcf.model.make_calldata_tuple("GT")(GT="1/1"))])
        return vcf_record

    def to_bed_interval(self, sample_name):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        # if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        #if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None
        end = self.end if self.sv_type != "INS" else (self.end + 1)

        return pybedtools.Interval(self.chrom, self.start, end, name="%s,%d,%s" % (
        self.sv_type, self.length, ";".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))),
                                   score=str(len(self.sources)))

    def to_svp_record(self, sample_name, id_num):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None

        start_outer = self.start
        start_inner = self.start
        end_inner = self.end
        end_outer = self.end
        type_of_computational_approach = ",".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))

        return "%s\t%d\t%d\t%d\t%d\t%s\t%d\t%s\t%s\t%s\t%s\tMetaSV\t%d" % (
        self.chrom, start_outer, start_inner, end_inner, end_outer, self.sv_type, self.length, "BWA", "Illumina",
        sample_name, type_of_computational_approach, id_num)

def interval_overlaps_interval_list(interval, interval_list, min_fraction_self=1e-9, min_fraction_other=1e-9):
    index = bisect.bisect_left(interval_list, interval)
    if index > 0 and interval.overlaps(interval_list[index - 1], min_fraction_self, min_fraction_other,
                                       min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    if index < len(interval_list) and interval.overlaps(interval_list[index], min_fraction_self, min_fraction_other,
                                                        min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    return False

def merge_intervals(interval_list):
    interval_list.sort()
    merged_intervals = []
    if not interval_list:
        return []

    current_merged_interval = copy.deepcopy(interval_list[0])
    for i in xrange(len(interval_list) - 1):
        next_interval = interval_list[i + 1]

        if current_merged_interval.overlaps(next_interval) or current_merged_interval.is_adjacent(next_interval):
            if current_merged_interval.sub_intervals:
                current_merged_interval.merge(next_interval)
            else:
                new_merged_interval = SVInterval()
                # logger.debug("Merging %s with %s" % (repr(current_merged_interval), repr(next_interval)))
                new_merged_interval.set_merged(current_merged_interval, next_interval)
                current_merged_interval = new_merged_interval
        else:
            merged_intervals.append(current_merged_interval)
            current_merged_interval = copy.deepcopy(next_interval)

    merged_intervals.append(current_merged_interval)
    merged_intervals.sort()
    return merged_intervals
<commit_msg>formatting<commit_after>import logging

logger = logging.getLogger(__name__)

import os
import copy
import bisect
import pysam
import pybedtools
import vcf

svs_of_interest = ["DEL", "INS", "DUP", "DUP:TANDEM", "INV"]
sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller", "BreakDancer", "CNVnator"] # order is important!
precise_sv_sources = ["Pindel", "BreakSeq", "HaplotypeCaller"]
sv_sources_to_type = {"Pindel": "SR", "BreakSeq": "JM", "BreakDancer": "RP", "CNVnator": "RD", "HaplotypeCaller": "AS"}

mydir = os.path.dirname(os.path.realpath(__file__))
gaps_b37 = os.path.join(mydir, "resources/b37.gaps.bed")
gaps_hg19 = os.path.join(mydir, "resources/hg19.gaps.bed")

def get_gaps_file(contig_names):
    hg19_major = set(["chr%d" % i for i in xrange(1, 23)] + ["chrX", "chrY", "chrM"])
    b37_major = set([str(i) for i in xrange(1, 23)] + ["X", "Y", "MT"])
    if set(contig_names) & hg19_major: return gaps_hg19
    if set(contig_names) & b37_major: return gaps_b37

    logger.warn("Could not guess gaps file for reference. No gap filtering will be done.")
    return None

class SVInterval:
    def __init__(self, chrom=None, start=0, end=0, name=None, sv_type=None, length=0, sources=set(), gt="./1", wiggle=0,
                 info=None, cipos=[], ciend=[], native_sv=None):
        self.chrom = chrom
        self.start = start
        self.end = end
        self.name = name
        self.sv_type = sv_type
        self.length = length
        self.info = info
        self.sources = sources
        self.gt = gt
        self.wiggle = wiggle
        self.sub_intervals = []
        self.is_precise = False
        self.is_validated = False
        self.validating_interval = None
        self.cipos = cipos
        self.ciend = ciend
        self.native_sv = native_sv

    def merge(self, interval):
        self.start = min(self.start, interval.start - interval.wiggle)
        self.end = max(self.end, interval.end + interval.wiggle)
        self.length = max(self.length, interval.length)
        self.name = self.name + "," + interval.name
        self.sub_intervals.append(interval)
        self.sources |= interval.sources

    def set_merged(self, interval1, interval2):
        self.chrom = interval1.chrom
        self.start = min(interval1.start - interval1.wiggle, interval2.start - interval2.wiggle)
        self.end = max(interval1.end + interval1.wiggle, interval2.end + interval2.wiggle)
        self.length = max(interval1.length, interval2.length)
        self.name = interval1.name + "," + interval2.name
        self.sv_type = interval1.sv_type
        self.info = None
        self.sub_intervals = [interval1, interval2]
        self.sources = interval1.sources | interval2.sources
        self.gt = interval1.gt

    def __lt__(self, other):
        if self.chrom != other.chrom: return self.chrom < other.chrom
        if self.start != other.start: return self.start < other.start
        return self.end < other.end

    def __str__(self):
        if self.sub_intervals:
            return ",".join([str(interval) for interval in self.sub_intervals])
        return "%s-%d-%d-%d-%s" % (self.chrom, self.start, self.end, self.length, ",".join(list(self.sources)))

    def __repr__(self):
        return "<" + self.__class__.__name__ + " " + str(self.__dict__) + ">"

    def overlaps(self, other, min_fraction_self=1e-9, min_fraction_other=1e-9, min_overlap_length_self=1,
                 min_overlap_length_other=1):
        if self.chrom != other.chrom:
            return False
        if max(self.start - self.wiggle, other.start - other.wiggle) \
                >= min(self.end + self.wiggle, other.end + other.wiggle):
            return False

        self_length = float(self.end - self.start + 2 * self.wiggle)
        other_length = float(other.end - other.start + 2 * other.wiggle)
        overlap_length = min(self.end + self.wiggle, other.end + other.wiggle) - max(self.start - self.wiggle,
                                                                                other.start - other.wiggle)
        return float(overlap_length) >= max(min_fraction_self * self_length,
                                            min_fraction_other * other_length) and overlap_length >= max(
            min_fraction_self, min_overlap_length_other)

    def is_adjacent(self, other, gap=0):
        if self.chrom != other.chrom:
            return False
        return (self.end + gap >= other.start and self.end + gap < other.end) or (
        other.end + gap >= self.start and other.end + gap < self.end)

    def get_start(self):
        if not self.sub_intervals:
            return self.start

        return min([interval.get_start() for interval in self.sub_intervals])

    def get_end(self):
        if not self.sub_intervals:
            return self.end

        return max([interval.get_end() for interval in self.sub_intervals])

    def do_validation(self, overlap_ratio=0.5):
        self.start = self.get_start()
        self.end = self.get_end()

        if self.sv_type not in svs_of_interest:
            return

        if not self.sub_intervals:
            # This interval did not overlap anything
            self.is_precise = list(self.sources)[0] in precise_sv_sources
            # if self.sv_type == "INV" and self.length >= 1000: self.is_validated = True
            return

        lists = {source: [] for source in sv_sources}

        precise_merged = merge_intervals(
            [interval for interval in self.sub_intervals if list(interval.sources)[0] in precise_sv_sources])

        for interval in self.sub_intervals:
            # logger.debug("Examining subinterval " + str(interval))
            lists[list(interval.sources)[0]].append(interval)

        for source in sv_sources:
            # If the SV is called multiple times by the same tool, don't validate
            if len(lists[source]) == 1:
                if self.overlaps(lists[source][0], overlap_ratio, overlap_ratio) \
                        or interval_overlaps_interval_list(lists[source][0], precise_merged, overlap_ratio,
                                                           overlap_ratio):
                    self.is_validated = len(self.sources) > 1
                    self.validating_interval = lists[source][0]
                    self.is_precise = source in precise_sv_sources
                    break  # This break makes the order of sv_sources important

        if not self.is_validated:
            return

        if self.is_precise and self.validating_interval is not None:
            self.start = self.validating_interval.start
            self.end = self.validating_interval.end
            self.length = self.validating_interval.length
            self.gt = self.validating_interval.gt
            self.info = self.validating_interval.info

            if self.length == 0:
                for source in sv_sources:
                    if source in self.sources:
                        if lists[source] and lists[source][0].length > 0:
                            self.length = lists[source][0].length
                            break

    def fix_pos(self):
        if self.sv_type == "INS":
            mid = (self.start + self.end) / 2
            if not self.is_precise:
                self.end = self.start
                self.cipos = ["0", "%d" % (self.end - self.start)]
                # self.cipos = ["%d" % (mid - self.start), "%d" % (mid - self.end)]
            else:
                self.start = mid
                self.end = mid

    def to_vcf_record(self, fasta_handle=None, sample = ""):
        if self.start <= 0: return None
        if self.sv_type not in svs_of_interest: return None

        # ignore private haplotype caller calls
        if ((not self.sub_intervals) or len(self.sources) == 1) and list(self.sources)[0] == "HaplotypeCaller":
            return None

        # formulate the INFO field
        info = {}
        if not self.sub_intervals:
            info.update(self.info)
        else:
            if self.info:
                info.update(self.info)
            for interval in self.sub_intervals:
                # TODO: this will just overwrite the other dict entries... this should be ok for pass variants
                # TODO: kind of strange
                if interval.info:
                    info.update(interval.info)
        svmethods = [sv_sources_to_type[tool] for tool in self.sources]
        svmethods.sort()
        sv_len = -self.length if self.sv_type == "DEL" else self.length
        info.update({"SVLEN": sv_len,
                "SVTYPE": self.sv_type,
                "SVMETHOD": ",".join(svmethods)})
        if self.sv_type == "DEL" or self.sv_type == "DUP":
            info["END"] = self.end

        if not self.is_precise:
            info.update({"IMPRECISE": None})

        info.update({"VT":"SV"})
        info.update({"SVTOOL":"MetaSVMerge"})
        info.update({"NUM_SVMETHODS":len(self.sources)})
        info.update({"SOURCES":str(self)})
        if self.cipos:
            info.update({"CIPOS": (",".join([str(a) for a in self.cipos]))})

        vcf_record = vcf.model._Record(self.chrom,
                                       self.start - 1,
                                       ".",
                                       fasta_handle.fetch(self.chrom, max(0, self.start - 2), max(1, self.start - 1)) if fasta_handle else "N",
                                       ["<%s>" % (self.sv_type)],
                                       ".",
                                       "PASS" if self.is_validated else "LowQual",
                                       info,
                                       "GT",
                                       [0],
                                       [vcf.model._Call(None, sample, vcf.model.make_calldata_tuple("GT")(GT="1/1"))])
        return vcf_record

    def to_bed_interval(self, sample_name):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        # if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        #if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None
        end = self.end if self.sv_type != "INS" else (self.end + 1)

        return pybedtools.Interval(self.chrom, self.start, end, name="%s,%d,%s" % (
        self.sv_type, self.length, ";".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))),
                                   score=str(len(self.sources)))

    def to_svp_record(self, sample_name, id_num):
        if self.start <= 0: return None
        if self.sv_type not in ["DEL", "INS", "INV"]: return None
        if not self.sub_intervals and list(self.sources)[0] == "HaplotypeCaller": return None
        if len(self.sources) == 1 and list(self.sources)[0] == "HaplotypeCaller": return None

        start_outer = self.start
        start_inner = self.start
        end_inner = self.end
        end_outer = self.end
        type_of_computational_approach = ",".join(sorted([sv_sources_to_type[tool] for tool in self.sources]))

        return "%s\t%d\t%d\t%d\t%d\t%s\t%d\t%s\t%s\t%s\t%s\tMetaSV\t%d" % (
        self.chrom, start_outer, start_inner, end_inner, end_outer, self.sv_type, self.length, "BWA", "Illumina",
        sample_name, type_of_computational_approach, id_num)

def interval_overlaps_interval_list(interval, interval_list, min_fraction_self=1e-9, min_fraction_other=1e-9):
    index = bisect.bisect_left(interval_list, interval)
    if index > 0 and interval.overlaps(interval_list[index - 1], min_fraction_self, min_fraction_other,
                                       min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    if index < len(interval_list) and interval.overlaps(interval_list[index], min_fraction_self, min_fraction_other,
                                                        min_overlap_length_self=1, min_overlap_length_other=1):
        return True
    return False

def merge_intervals(interval_list):
    interval_list.sort()
    merged_intervals = []
    if not interval_list:
        return []

    current_merged_interval = copy.deepcopy(interval_list[0])
    for i in xrange(len(interval_list) - 1):
        next_interval = interval_list[i + 1]

        if current_merged_interval.overlaps(next_interval) or current_merged_interval.is_adjacent(next_interval):
            if current_merged_interval.sub_intervals:
                current_merged_interval.merge(next_interval)
            else:
                new_merged_interval = SVInterval()
                # logger.debug("Merging %s with %s" % (repr(current_merged_interval), repr(next_interval)))
                new_merged_interval.set_merged(current_merged_interval, next_interval)
                current_merged_interval = new_merged_interval
        else:
            merged_intervals.append(current_merged_interval)
            current_merged_interval = copy.deepcopy(next_interval)

    merged_intervals.append(current_merged_interval)
    merged_intervals.sort()
    return merged_intervals
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging

from utils import mongo
from utils import config
from utils import entity_resolver

class PartyFundingParser():
    def __init__(self):
        self._logger = logging.getLogger('spud')

    def run(self):
        self.db = mongo.MongoInterface()
        self.resolver = entity_resolver.MasterEntitiesResolver()
        self.lords_titles = config.lords_titles

        _all_entries = self.db.fetch_all('scraped_party_funding', paged=False)
        for doc in _all_entries:
            parsed = {}
            parsed["recipient"] = self._get_recipient(
                doc["recipient"], doc["donee_type"], doc["recipient_type"]
            )
            parsed["donor_name"] = self._get_donor(
                doc["donor_name"], doc["donor_type"]
            )
            parsed["donor_type"] = doc["donor_type"]
            parsed["donee_type"] = doc["donee_type"]
            parsed["recipient_type"] = doc["recipient_type"]
            parsed["donation_type"] = doc["donation_type"]
            parsed["value"] = self._remove_broken_pound(doc["value"])
            parsed["purpose"] = doc["purpose"]
            parsed["nature_provision"] = doc["nature_provision"]
            parsed["ec_reference"] = doc["ec_reference"]
            parsed["company_reg"] = doc["company_reg"]
            parsed["is_sponsorship"] = doc["is_sponsorship"]
            parsed["6212"] = doc["6212"]
            parsed["recd_by"] = doc["recd_by"]
            parsed["received_date"] = doc["received_date"]
            parsed["reported_date"] = doc["reported_date"]
            parsed["accepted_date"] = doc["accepted_date"]
            if not parsed["recipient"]:
                self._print_dic(parsed)
                self._logger.debug("---\n")
            else:
                self._print_out("recipient", parsed["recipient"])
                self._print_out("donee_type", parsed["donee_type"])
                self._print_out("donor_name", doc["donor_name"])
                self._print_out("found_name", parsed["donor_name"])
                self._print_out("donor_type", parsed["donor_type"])
                self._print_out("value", parsed["value"])
                self._logger.debug("---\n")
                self.db.save('parsed_party_funding', parsed)

    def _get_recipient(self, entry, entry_type, recipient_type):
        result = self._remove_illegal_chars(entry)
        if entry_type == "MP - Member of Parliament":
            result = self.resolver.find_mp(result)
        elif entry_type == "Political Party" or \
                recipient_type == "Political Party":
            result = self.resolver.find_party(result)
        else:
            result = self.resolver.get_entities(result)
            if result and isinstance(result, list):
                result = result[0]
        if not result:
            return entry
        else:
            return result

    def _get_donor(self, entry, entry_type):
        result = self._remove_illegal_chars(entry)
        if entry_type == "Individual":
            title = entry.split(" ")[0]
            if title in self.lords_titles:
                result = self.resolver.find_lord(result)
            else:
                result = self.resolver.get_entities(result)
        else:
            result = self.resolver.find_donor(
                entry, delimiter=",", fuzzy_delimit=False
            )
        if result and isinstance(result, list):
            result = result[0]
        if not result:
            return entry
        else:
            return result

    @staticmethod
    def _remove_broken_pound(text):
        return text.replace(u"��", u"£")

    @staticmethod
    def _remove_illegal_chars(text):
        text = text.replace("\\", " ")
        return text

    def _print_out(self, key, value):
        self._logger.debug("  %-30s%-20s" % (key, value))

    def _print_dic(self, dictionary):
        for keys, values in dictionary.items():
            self._logger.debug(" %-20s:%-25s" % (keys, values))
<commit_msg>Removed "Rt Hon " to improve fuzzy matching<commit_after># -*- coding: utf-8 -*-
import logging

from utils import mongo
from utils import config
from utils import entity_resolver

class PartyFundingParser():
    def __init__(self):
        self._logger = logging.getLogger('spud')

    def run(self):
        self.db = mongo.MongoInterface()
        self.resolver = entity_resolver.MasterEntitiesResolver()
        self.lords_titles = config.lords_titles

        _all_entries = self.db.fetch_all('scraped_party_funding', paged=False)
        for doc in _all_entries:
            parsed = {}
            parsed["recipient"] = self._get_recipient(
                doc["recipient"], doc["donee_type"], doc["recipient_type"]
            )
            parsed["donor_name"] = self._get_donor(
                doc["donor_name"], doc["donor_type"]
            )
            parsed["donor_type"] = doc["donor_type"]
            parsed["donee_type"] = doc["donee_type"]
            parsed["recipient_type"] = doc["recipient_type"]
            parsed["donation_type"] = doc["donation_type"]
            parsed["value"] = self._remove_broken_pound(doc["value"])
            parsed["purpose"] = doc["purpose"]
            parsed["nature_provision"] = doc["nature_provision"]
            parsed["ec_reference"] = doc["ec_reference"]
            parsed["company_reg"] = doc["company_reg"]
            parsed["is_sponsorship"] = doc["is_sponsorship"]
            parsed["6212"] = doc["6212"]
            parsed["recd_by"] = doc["recd_by"]
            parsed["received_date"] = doc["received_date"]
            parsed["reported_date"] = doc["reported_date"]
            parsed["accepted_date"] = doc["accepted_date"]
            if not parsed["recipient"]:
                self._print_dic(parsed)
                self._logger.debug("---\n")
            else:
                self._print_out("recipient", parsed["recipient"])
                self._print_out("donee_type", parsed["donee_type"])
                self._print_out("donor_name", doc["donor_name"])
                self._print_out("found_name", parsed["donor_name"])
                self._print_out("donor_type", parsed["donor_type"])
                self._print_out("value", parsed["value"])
                self._logger.debug("---\n")
                self.db.save('parsed_party_funding', parsed)

    def _get_recipient(self, entry, entry_type, recipient_type):
        result = self._remove_extraneous(entry)
        if entry_type == "MP - Member of Parliament":
            result = self.resolver.find_mp(result)
        elif entry_type == "Political Party" or \
                recipient_type == "Political Party":
            result = self.resolver.find_party(result)
        else:
            result = self.resolver.get_entities(result)
            if result and isinstance(result, list):
                result = result[0]
        if not result:
            return entry
        else:
            return result

    def _get_donor(self, entry, entry_type):
        result = self._remove_extraneous(entry)
        if entry_type == "Individual":
            title = entry.split(" ")[0]
            if title in self.lords_titles:
                result = self.resolver.find_lord(result)
            else:
                result = self.resolver.get_entities(result)
        else:
            result = self.resolver.find_donor(
                entry, delimiter=",", fuzzy_delimit=False
            )
        if result and isinstance(result, list):
            result = result[0]
        if not result:
            return entry
        else:
            return result

    @staticmethod
    def _remove_broken_pound(text):
        return text.replace(u"��", u"£")

    @staticmethod
    def _remove_extraneous(text):
        text = text.replace("\\", " ")
        text = text.replace('"', ' ')
        text = text.replace('The Rt Hon ', '')
        return text

    def _print_out(self, key, value):
        self._logger.debug("  %-30s%-20s" % (key, value))

    def _print_dic(self, dictionary):
        for keys, values in dictionary.items():
            self._logger.debug(" %-20s:%-25s" % (keys, values))
<|endoftext|>


<commit_before>import json
import os.path
import pickle

from flask import Flask, request, redirect
from flask_cors import CORS

from settings import DEBUG
from constants import GITHUB_REPOSITORY_LINK, DEFAULT_THRESHOLD, THRESHOLD_DELTA
from utils import update_dict, check_prediction

app = Flask(__name__)
CORS(app)

@app.route("/update-user", methods=['GET'])
def main():
	# get data from args
	gcm_id = request.args.get("gcm_id")
	new_data = request.args.get("user_data")

	file_path = "user_data/" + gcm_id

	if os.path.isfile(file_path):
		user_history = pickle.load(open(file_path, "rb"))

		existing_data = user_history['data']
		threshold = user_history['threshold']

		data_dict = update_dict(existing_data, new_data)
	else:
		data_dict = update_dict({}, new_data)
		threshold = DEFAULT_THRESHOLD

	data_dict = check_prediction(data_dict, threshold)

	user_json = {
		'data' : data_dict,
		'threshold' : threshold,
	}

	pickle.dump(user_json, open(file_path, "wb"))

	return json.dumps(data_dict)

@app.route("/change-threshold", methods=['GET'])
def change_threshold():
	gcm_id = request.args.get("gcm_id")
	delta_flag = request.args.get("flag")

	file_path = "user_data/" + gcm_id

	if os.path.isfile(file_path):
		user_history = pickle.load(open(file_path, "rb"))

		existing_data = user_history['data']
		threshold = user_history['threshold']

		if delta_flag == 0:
			new_threshold = threshold - THRESHOLD_DELTA
		else:
			new_threshold = threshold + THRESHOLD_DELTA

		user_json = {
			'data' : existing_data,
			'threshold' : new_threshold,
		}

		pickle.dump(user_json, open(file_path, "wb"))

		return True

	return False

@app.errorhandler(404)
def page_not_found(error):
	message = "Invalid URL, Please refer to the docs for correct URL"

	response = {
		'message' : message,
		'docs_url' : GITHUB_REPOSITORY_LINK,
	}

	return json.dumps(response)

if __name__ == "__main__":
	app.run(debug=True)
<commit_msg>change response headers to application/json<commit_after>import os.path
import pickle

from flask import Flask, request, redirect, jsonify
from flask_cors import CORS

from settings import DEBUG
from constants import GITHUB_REPOSITORY_LINK, DEFAULT_THRESHOLD, THRESHOLD_DELTA
from utils import update_dict, check_prediction

app = Flask(__name__)
CORS(app)

@app.route("/update-user", methods=['GET'])
def main():
	# get data from args
	gcm_id = request.args.get("gcm_id")
	new_data = request.args.get("user_data")

	file_path = "user_data/" + gcm_id

	if os.path.isfile(file_path):
		user_history = pickle.load(open(file_path, "rb"))

		existing_data = user_history['data']
		threshold = user_history['threshold']

		data_dict = update_dict(existing_data, new_data)
	else:
		data_dict = update_dict({}, new_data)
		threshold = DEFAULT_THRESHOLD

	data_dict = check_prediction(data_dict, threshold)

	user_json = {
		'data' : data_dict,
		'threshold' : threshold,
	}

	pickle.dump(user_json, open(file_path, "wb"))

	return jsonify(**data_dict)

@app.route("/change-threshold", methods=['GET'])
def change_threshold():
	gcm_id = request.args.get("gcm_id")
	delta_flag = request.args.get("flag")

	file_path = "user_data/" + gcm_id

	if os.path.isfile(file_path):
		user_history = pickle.load(open(file_path, "rb"))

		existing_data = user_history['data']
		threshold = user_history['threshold']

		if delta_flag == 0:
			new_threshold = threshold - THRESHOLD_DELTA
		else:
			new_threshold = threshold + THRESHOLD_DELTA

		user_json = {
			'data' : existing_data,
			'threshold' : new_threshold,
		}

		pickle.dump(user_json, open(file_path, "wb"))

		return True

	return False

@app.errorhandler(404)
def page_not_found(error):
	message = "Invalid URL, Please refer to the docs for correct URL"

	response = {
		'message' : message,
		'docs_url' : GITHUB_REPOSITORY_LINK,
	}

	return jsonify(**response)

if __name__ == "__main__":
	app.run(debug=True)
<|endoftext|>


<commit_before>from enum import Enum
class Couleur(Enum):
	"""
		Enumération utilisée pour la couleur des feux
	"""
	rouge = 0
	vert = 1

class feu:
		"""
			Modélise un feu de signalisation.
				# intersection : Intersection sur laquelle agit le feu
				# temps_vert : temps durant lequel l'axe passé en parametre reste vert en secondes
				# temps_cycle : temps du cycle en secondes
				# axe : 0 si axe 1-3, 1 si axe 2-4
				# /!\ Voir dessin Bonfante si questions /!\
		"""

	def __init__(self, intersection, temps_vert=10, temps_cycle = 20, axe=0):
		# Intersection ou se trouve le feu
		self.intersection = intersection

		# Temps que l'axe passé en parametre reste vert
		self.temps_vert = temps_vert

		# Temps de la durée d'un cycle de feu
		self.temps_cycle = temps_cycle

		# Axe que l'on contrôle
		# 0 si axe 1-3
		# 1 si axe 2-4
		self.axe = axe

		# Couleur actuel de l'axe passé en parametre
		self.couleur_axe = Couleur.vert

	def notifie_temps(self, temps, simulation_manager):
		"""
			Methode appelée lorsque le simulateur augmente le temps
		"""
		self.change_couleur(temps, simulation_manger)

	def change_couleur(self,temps, simulation_manger):
		"""
			Change la couleur du feu en fonction du temps
		"""
		# Recupere le temps absolu dans le simulation manager
		temps_absolu = simulation_manager.get_temps_absolu()
		# Choisir la bonne couleur du feu
		temp = temps_absolu%self.temps_cycle
		if(temp<self.temps_vert):
			self.couleur_axe = Couleur.vert
		else:
			self.couleur_axe = Couleur.rouge

	def estPassant(self):
		return self.couleur_axe == Couleur.vert<commit_msg>Correction des feux<commit_after>from enum import Enum
class Couleur(Enum):
	"""
		Enumération utilisée pour la couleur des feux
	"""
	rouge = 0
	vert = 1

class feu(controleAcces):
		"""
			Modélise un feu de signalisation.
				# intersection : Intersection sur laquelle agit le feu
				# temps_vert : temps durant lequel l'axe passé en parametre reste vert en secondes
				# temps_cycle : temps du cycle en secondes
				# @author : Bonfante
		"""

	def __init__(self, intersection, temps_vert=10, temps_cycle = 20):
		# Intersection ou se trouve le feu
		self.intersection = intersection

		# Temps que l'axe passé en parametre reste vert
		self.temps_vert = temps_vert

		# Temps de la durée d'un cycle de feu
		self.temps_cycle = temps_cycle

		# Couleur actuel de l'axe passé en parametre
		self.couleur_axe = Couleur.vert

	def notifie_temps(self, temps, simulation_manager):
		"""
			Methode appelée lorsque le simulateur augmente le temps
		"""
		self.change_couleur(temps, simulation_manger)

	def change_couleur(self,temps, simulation_manger):
		"""
			Change la couleur du feu en fonction du temps
		"""
		# Recupere le temps absolu dans le simulation manager
		temps_absolu = simulation_manager.get_temps_absolu()
		# Choisir la bonne couleur du feu
		temp = temps_absolu%self.temps_cycle
		if(temp<self.temps_vert):
			self.couleur_axe = Couleur.vert
		else:
			self.couleur_axe = Couleur.rouge

	def est_passant(self):
		return self.couleur_axe == Couleur.vert<|endoftext|>


<commit_before>import logging
__version__ = '1.1.11'

gearauthsite = "http://gearauth.netpie.io:8080"
gearauthrequesttokenendpoint = gearauthsite+"/api/rtoken"
gearauthaccesstokenendpoint = gearauthsite+"/api/atoken"

mgrev = "PY11k"
gearkey = None
gearsecret = None
gearalias = None
appid = None
gearname = None
accesstoken = None
requesttoken = None
client = None
scope = ""
gearexaddress = None
gearexport = None
mqtt_client = None
logger = logging.getLogger("python-microgear")
<commit_msg>change gearauthsite to ga.netpie.io<commit_after>import logging
__version__ = '1.1.12'

gearauthsite = "http://ga.netpie.io:8080"
gearauthrequesttokenendpoint = gearauthsite+"/api/rtoken"
gearauthaccesstokenendpoint = gearauthsite+"/api/atoken"

mgrev = "PY11k"
gearkey = None
gearsecret = None
gearalias = None
appid = None
gearname = None
accesstoken = None
requesttoken = None
client = None
scope = ""
gearexaddress = None
gearexport = None
mqtt_client = None
logger = logging.getLogger("python-microgear")
<|endoftext|>


<commit_before>from matplotlib.dates import DayLocator, WeekdayLocator, MonthLocator, YearLocator

def parse_break_str(txt):
    "parses '10 weeks' into tuple (10, week)."
    txt = txt.strip()
    if len(txt.split()) == 2:
        n, units = txt.split()
    else:
        n,units = 1, txt
    units = units.rstrip('s') # e.g. weeks => week
    n = int(n)
    return n, units

# matplotlib's YearLocator uses different named
# arguments than the others
LOCATORS = {
    'day': DayLocator,
    'week': WeekdayLocator,
    'month': MonthLocator,
    'year': lambda interval: YearLocator(base=interval)
}

def date_breaks(width):
    """
    "Regularly spaced dates."

    width:
        an interval specification. must be one of [day, week, month, year]
    usage:
        date_breaks(width = '1 year')
        date_breaks(width = '6 weeks')
        date_breaks('months')
    """
    period, units = parse_break_str(width)
    Locator = LOCATORS.get(units)
    locator = Locator(interval=period)
    return locator
<commit_msg>Add more granular date locators<commit_after>from matplotlib.dates import MinuteLocator, HourLocator, DayLocator
from matplotlib.dates import WeekdayLocator, MonthLocator, YearLocator

def parse_break_str(txt):
    "parses '10 weeks' into tuple (10, week)."
    txt = txt.strip()
    if len(txt.split()) == 2:
        n, units = txt.split()
    else:
        n,units = 1, txt
    units = units.rstrip('s') # e.g. weeks => week
    n = int(n)
    return n, units

# matplotlib's YearLocator uses different named
# arguments than the others
LOCATORS = {
    'minute': MinuteLocator,
    'hour': HourLocator,
    'day': DayLocator,
    'week': WeekdayLocator,
    'month': MonthLocator,
    'year': lambda interval: YearLocator(base=interval)
}

def date_breaks(width):
    """
    "Regularly spaced dates."

    width:
        an interval specification. must be one of [minute, hour, day, week, month, year]
    usage:
        date_breaks(width = '1 year')
        date_breaks(width = '6 weeks')
        date_breaks('months')
    """
    period, units = parse_break_str(width)
    Locator = LOCATORS.get(units)
    locator = Locator(interval=period)
    return locator
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

from __future__ import division

import warnings
import numpy as np

from .widget import Widget
from ...util.np_backport import nanmean

from vispy.ext.cassowary import (SimplexSolver, expression,
                                 Variable, STRONG, MEDIUM, WEAK, REQUIRED)

class Grid(Widget):
    """
    Widget that automatically sets the position and size of child Widgets to
    proportionally divide its internal area into a grid.

    Parameters
    ----------
    spacing : int
        Spacing between widgets.
    **kwargs : dict
        Keyword arguments to pass to `Widget`.
    """
    def __init__(self, spacing=6, **kwargs):
        from .viewbox import ViewBox
        self._next_cell = [0, 0]  # row, col
        self._cells = {}
        self._grid_widgets = {}
        self.spacing = spacing
        self._n_added = 0
        self._default_class = ViewBox  # what to add when __getitem__ is used
        Widget.__init__(self, **kwargs)

    def __getitem__(self, idxs):
        """Return an item or create it if the location is available"""
        if not isinstance(idxs, tuple):
            idxs = (idxs,)
        if len(idxs) == 1:
            idxs = idxs + (slice(None),)
        elif len(idxs) != 2:
            raise ValueError('Incorrect index: %s' % (idxs,))
        lims = np.empty((2, 2), int)
        for ii, idx in enumerate(idxs):
            if isinstance(idx, int):
                idx = slice(idx, idx + 1, None)
            if not isinstance(idx, slice):
                raise ValueError('indices must be slices or integers, not %s'
                                 % (type(idx),))
            if idx.step is not None and idx.step != 1:
                raise ValueError('step must be one or None, not %s' % idx.step)
            start = 0 if idx.start is None else idx.start
            end = self.grid_size[ii] if idx.stop is None else idx.stop
            lims[ii] = [start, end]
        layout = self.layout_array
        existing = layout[lims[0, 0]:lims[0, 1], lims[1, 0]:lims[1, 1]] + 1
        if existing.any():
            existing = set(list(existing.ravel()))
            ii = list(existing)[0] - 1
            if len(existing) != 1 or ((layout == ii).sum() !=
                                      np.prod(np.diff(lims))):
                raise ValueError('Cannot add widget (collision)')
            return self._grid_widgets[ii][-1]
        spans = np.diff(lims)[:, 0]
        item = self.add_widget(self._default_class(),
                               row=lims[0, 0], col=lims[1, 0],
                               row_span=spans[0], col_span=spans[1])
        return item

    def add_widget(self, widget=None, row=None, col=None, row_span=1,
                   col_span=1):
        """
        Add a new widget to this grid. This will cause other widgets in the
        grid to be resized to make room for the new widget.

        Parameters
        ----------
        widget : Widget
            The Widget to add
        row : int
            The row in which to add the widget (0 is the topmost row)
        col : int
            The column in which to add the widget (0 is the leftmost column)
        row_span : int
            The number of rows to be occupied by this widget. Default is 1.
        col_span : int
            The number of columns to be occupied by this widget. Default is 1.

        Notes
        -----
        The widget's parent is automatically set to this grid, and all other
        parent(s) are removed.
        """
        if row is None:
            row = self._next_cell[0]
        if col is None:
            col = self._next_cell[1]

        if widget is None:
            widget = Widget()

        _row = self._cells.setdefault(row, {})
        _row[col] = widget
        self._grid_widgets[self._n_added] = (row, col, row_span, col_span,
                                             widget)
        self._n_added += 1
        widget.parent = self

        self._next_cell = [row, col+col_span]

        # update stretch based on colspan/rowspan
        stretch = list(widget.stretch)
        stretch[0] = col_span if stretch[0] is None else stretch[0]
        stretch[1] = row_span if stretch[1] is None else stretch[1]
        widget.stretch = stretch

        self._update_child_widgets()
        return widget

    def add_grid(self, row=None, col=None, row_span=1, col_span=1,
                 **kwargs):
        """
        Create a new Grid and add it as a child widget.

        Parameters
        ----------
        row : int
            The row in which to add the widget (0 is the topmost row)
        col : int
            The column in which to add the widget (0 is the leftmost column)
        row_span : int
            The number of rows to be occupied by this widget. Default is 1.
        col_span : int
            The number of columns to be occupied by this widget. Default is 1.
        **kwargs : dict
            Keyword arguments to pass to the new `Grid`.
        """
        from .grid import Grid
        grid = Grid(**kwargs)
        return self.add_widget(grid, row, col, row_span, col_span)

    def add_view(self, row=None, col=None, row_span=1, col_span=1,
                 **kwargs):
        """
        Create a new ViewBox and add it as a child widget.

        Parameters
        ----------
        row : int
            The row in which to add the widget (0 is the topmost row)
        col : int
            The column in which to add the widget (0 is the leftmost column)
        row_span : int
            The number of rows to be occupied by this widget. Default is 1.
        col_span : int
            The number of columns to be occupied by this widget. Default is 1.
        **kwargs : dict
            Keyword arguments to pass to `ViewBox`.
        """
        from .viewbox import ViewBox
        view = ViewBox(**kwargs)
        return self.add_widget(view, row, col, row_span, col_span)

    def next_row(self):
        self._next_cell = [self._next_cell[0] + 1, 0]

    @property
    def grid_size(self):
        rvals = [widget[0]+widget[2] for widget in self._grid_widgets.values()]
        cvals = [widget[1]+widget[3] for widget in self._grid_widgets.values()]
        return max(rvals + [0]), max(cvals + [0])

    @property
    def layout_array(self):
        locs = -1 * np.ones(self.grid_size, int)
        for key in self._grid_widgets.keys():
            r, c, rs, cs = self._grid_widgets[key][:4]
            locs[r:r + rs, c:c + cs] = key
        return locs

    def __repr__(self):
        return (('<Grid at %s:\n' % hex(id(self))) +
                str(self.layout_array + 1) + '>')

    # MODIFIED VERSION
    def _update_child_widgets(self):
        # Resize all widgets in this grid to share space.
        n_rows, n_cols = self.grid_size
        if n_rows == 0 or n_cols == 0:
            return
        print("---")

        grid_layout = np.array([[None for _ in range(0, n_cols)]
                                for _ in range(0, n_rows)])

        for key, value in self._grid_widgets.items():
            (row, col, rspan, cspan, widget) = value
            grid_layout[row:row+rspan, col:col+cspan] = widget

        print("grid layout:\n%s" % grid_layout)

        rect = self.rect.padded(self.padding + self.margin)
        solver = SimplexSolver()

        # x, width constraints ------

        for nrow, row in enumerate(grid_layout):
            stretch_lhs = None
            prev_widget = None
            total_w_expr = expression.Expression()

            if len(row) == 0:
                continue

            for ncol, widget in enumerate(row):
                if prev_widget == widget or widget is None:
                    continue

                var_w = Variable("width(%s, %s)" % (nrow, ncol))
                widget.var_w = var_w

                widget.var_x = Variable("x(%s, %s)" % (nrow, ncol))

                # add the width the solver
                total_w_expr.add_expression(var_w)

                solver.add_constraint(widget.var_w >= 0, strength=REQUIRED)
                solver.add_constraint(widget.var_x >= 0, strength=REQUIRED)

                if widget.min_width is not None:
                    solver.add_constraint(var_w >= widget.min_width,
                                          strength=STRONG)

                if widget.max_width is not None:
                    solver.add_constraint(var_w <= widget.max_width,
                                          strength=STRONG)

                (stretch_w, _) = widget.stretch

                if stretch_w is not None:
                    if stretch_lhs is None:
                        stretch_lhs = var_w / stretch_w
                    else:
                        solver.add_constraint(stretch_lhs == var_w / stretch_w,
                                              strength=MEDIUM)

                if prev_widget is not None:
                    solver.add_constraint(widget.var_x + widget.var_w * 0.5 -
                                          (prev_widget.var_w  * 0.5 + prev_widget.var_x) ==
                                          widget.var_w * 0.5 + prev_widget.var_w * 0.5, strength=REQUIRED)
                else:
                    widget.var_x.value = 0
                    solver.add_stay(widget.var_x)
                prev_widget = widget

            # total width = width
            if len(total_w_expr.terms) > 0:
                solver.add_constraint(total_w_expr == rect.width)

        # height constraints ---------
        print("\n\ntranspose: %s" % grid_layout.transpose())

        for ncol, col in enumerate(grid_layout.transpose()):
            print("ncol %s : %s" % (ncol, col))
            stretch_lhs = None
            prev_widget = None
            total_h_expr = expression.Expression()

            if len(col) == 0:
                continue

            for nrow, widget in enumerate(col):
                if prev_widget == widget or widget is None:
                    continue

                var_h = Variable("height %s (%s, %s)" % (widget, nrow, ncol))
                widget.var_h = var_h  # store to set the height after solving

                widget.var_y = Variable("y %s (%s, %s)" % (widget, nrow, ncol))

                # add the height the solver
                total_h_expr.add_expression(var_h)

                solver.add_constraint(widget.var_h >= 0, strength=REQUIRED)
                solver.add_constraint(widget.var_y >= 0, strength=REQUIRED)

                if widget.min_height is not None:
                    solver.add_constraint(var_h >= widget.min_height,
                                          strength=STRONG)

                if widget.max_height is not None:
                    solver.add_constraint(var_h <= widget.max_height,
                                          strength=STRONG)

                (_, stretch_h) = widget.stretch

                if stretch_h is not None:
                    if stretch_lhs is None:
                        stretch_lhs = var_h / stretch_h
                    else:
                        solver.add_constraint(stretch_lhs == var_h / stretch_h,
                                              strength=WEAK)

                if prev_widget is not None:
                   solver.add_constraint(widget.var_y + widget.var_h * 0.5 -
                                          (prev_widget.var_h  * 0.5 + prev_widget.var_y) ==
                                          widget.var_h * 0.5 + prev_widget.var_h * 0.5, strength=REQUIRED)
                else:
                    widget.var_y.value = 0
                    solver.add_stay(widget.var_y)

                prev_widget = widget

            # total height = height
            if len(total_h_expr.terms) > 0:
                solver.add_constraint(total_h_expr == rect.height)

        solver.resolve()

        # copy solution
        for (_, val) in self._grid_widgets.items():
            (_, _, _, _, widget) = val
            widget.width = widget.var_w.value
            widget.height = widget.var_h.value
            widget.pos = widget.var_x.value, widget.var_y.value
        return

    # def _update_child_widgets(self):
    #     # Resize all widgets in this grid to share space.
    #     n_rows, n_cols = self.grid_size
    #     if n_rows == 0 or n_cols == 0:
    #         return

    #     # 1. Collect information about occupied cells and their contents
    #     occupied = np.zeros((n_rows, n_cols), dtype=bool)
    #     stretch = np.zeros((n_rows, n_cols, 2), dtype=float)
    #     stretch[:] = np.nan
    #     #minsize = np.zeros((n_rows, n_cols, 2), dtype=float)
    #     for key, val in self._grid_widgets.items():
    #         w = val[4]
    #         row, col, rspan, cspan, ch = self._grid_widgets[key]
    #         occupied[row:row+rspan, col:col+cspan] = True
    #         stretch[row:row+rspan, col:col+cspan] = (np.array(w.stretch) /
    #                                                  [cspan, rspan])
    #     row_occ = occupied.sum(axis=1) > 0
    #     col_occ = occupied.sum(axis=0) > 0
    #     with warnings.catch_warnings(record=True):  # mean of empty slice
    #         row_stretch = nanmean(stretch[..., 1], axis=1)
    #         col_stretch = nanmean(stretch[..., 0], axis=0)
    #     row_stretch[np.isnan(row_stretch)] = 0
    #     col_stretch[np.isnan(col_stretch)] = 0

    #     # 2. Decide width of each row/col
    #     rect = self.rect.padded(self.padding + self.margin)
    #     n_cols = col_occ.sum()
    #     colspace = rect.width - (n_cols-1) * self.spacing
    #     colsizes = col_stretch * colspace / col_stretch.sum()
    #     colsizes[colsizes == 0] = -self.spacing
    #     n_rows = row_occ.sum()
    #     rowspace = rect.height - (n_rows-1) * self.spacing
    #     rowsizes = row_stretch * rowspace / row_stretch.sum()
    #     rowsizes[rowsizes == 0] = -self.spacing

    #     # 3. Decide placement of row/col edges
    #     colend = np.cumsum(colsizes) + self.spacing * np.arange(len(colsizes))
    #     colstart = colend - colsizes
    #     rowend = np.cumsum(rowsizes) + self.spacing * np.arange(len(rowsizes))
    #     rowstart = rowend - rowsizes

    #     # snap to pixel boundaries to avoid drawing artifacts
    #     colstart = np.round(colstart) + self.margin
    #     colend = np.round(colend) + self.margin
    #     rowstart = np.round(rowstart) + self.margin
    #     rowend = np.round(rowend) + self.margin

    #     for key in self._grid_widgets.keys():
    #         row, col, rspan, cspan, ch = self._grid_widgets[key]

    #         # Translate the origin of the node to the corner of the area
    #         # ch.transform.reset()
    #         # ch.transform.translate((colstart[col], rowstart[row]))
    #         ch.pos = colstart[col], rowstart[row]

    #         # ..and set the size to match.
    #         w = colend[col+cspan-1]-colstart[col]
    #         h = rowend[row+rspan-1]-rowstart[row]
    #         ch.size = w, h
<commit_msg>neater impl of same algorithm<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.

from __future__ import division

import warnings
import numpy as np

from .widget import Widget
from ...util.np_backport import nanmean

from vispy.ext.cassowary import (SimplexSolver, expression,
                                 Variable, STRONG, MEDIUM, WEAK, REQUIRED)

class Grid(Widget):
    """
    Widget that automatically sets the position and size of child Widgets to
    proportionally divide its internal area into a grid.

    Parameters
    ----------
    spacing : int
        Spacing between widgets.
    **kwargs : dict
        Keyword arguments to pass to `Widget`.
    """
    def __init__(self, spacing=6, **kwargs):
        from .viewbox import ViewBox
        self._next_cell = [0, 0]  # row, col
        self._cells = {}
        self._grid_widgets = {}
        self.spacing = spacing
        self._n_added = 0
        self._default_class = ViewBox  # what to add when __getitem__ is used
        Widget.__init__(self, **kwargs)

    def __getitem__(self, idxs):
        """Return an item or create it if the location is available"""
        if not isinstance(idxs, tuple):
            idxs = (idxs,)
        if len(idxs) == 1:
            idxs = idxs + (slice(None),)
        elif len(idxs) != 2:
            raise ValueError('Incorrect index: %s' % (idxs,))
        lims = np.empty((2, 2), int)
        for ii, idx in enumerate(idxs):
            if isinstance(idx, int):
                idx = slice(idx, idx + 1, None)
            if not isinstance(idx, slice):
                raise ValueError('indices must be slices or integers, not %s'
                                 % (type(idx),))
            if idx.step is not None and idx.step != 1:
                raise ValueError('step must be one or None, not %s' % idx.step)
            start = 0 if idx.start is None else idx.start
            end = self.grid_size[ii] if idx.stop is None else idx.stop
            lims[ii] = [start, end]
        layout = self.layout_array
        existing = layout[lims[0, 0]:lims[0, 1], lims[1, 0]:lims[1, 1]] + 1
        if existing.any():
            existing = set(list(existing.ravel()))
            ii = list(existing)[0] - 1
            if len(existing) != 1 or ((layout == ii).sum() !=
                                      np.prod(np.diff(lims))):
                raise ValueError('Cannot add widget (collision)')
            return self._grid_widgets[ii][-1]
        spans = np.diff(lims)[:, 0]
        item = self.add_widget(self._default_class(),
                               row=lims[0, 0], col=lims[1, 0],
                               row_span=spans[0], col_span=spans[1])
        return item

    def add_widget(self, widget=None, row=None, col=None, row_span=1,
                   col_span=1):
        """
        Add a new widget to this grid. This will cause other widgets in the
        grid to be resized to make room for the new widget.

        Parameters
        ----------
        widget : Widget
            The Widget to add
        row : int
            The row in which to add the widget (0 is the topmost row)
        col : int
            The column in which to add the widget (0 is the leftmost column)
        row_span : int
            The number of rows to be occupied by this widget. Default is 1.
        col_span : int
            The number of columns to be occupied by this widget. Default is 1.

        Notes
        -----
        The widget's parent is automatically set to this grid, and all other
        parent(s) are removed.
        """
        if row is None:
            row = self._next_cell[0]
        if col is None:
            col = self._next_cell[1]

        if widget is None:
            widget = Widget()

        _row = self._cells.setdefault(row, {})
        _row[col] = widget
        self._grid_widgets[self._n_added] = (row, col, row_span, col_span,
                                             widget)
        self._n_added += 1
        widget.parent = self

        self._next_cell = [row, col+col_span]

        # update stretch based on colspan/rowspan
        stretch = list(widget.stretch)
        stretch[0] = col_span if stretch[0] is None else stretch[0]
        stretch[1] = row_span if stretch[1] is None else stretch[1]
        widget.stretch = stretch

        self._update_child_widgets()
        return widget

    def _prepare_draw(self, view):
        self._update_child_widgets()
        print("grid update: %s" % self)

    def add_grid(self, row=None, col=None, row_span=1, col_span=1,
                 **kwargs):
        """
        Create a new Grid and add it as a child widget.

        Parameters
        ----------
        row : int
            The row in which to add the widget (0 is the topmost row)
        col : int
            The column in which to add the widget (0 is the leftmost column)
        row_span : int
            The number of rows to be occupied by this widget. Default is 1.
        col_span : int
            The number of columns to be occupied by this widget. Default is 1.
        **kwargs : dict
            Keyword arguments to pass to the new `Grid`.
        """
        from .grid import Grid
        grid = Grid(**kwargs)
        return self.add_widget(grid, row, col, row_span, col_span)

    def add_view(self, row=None, col=None, row_span=1, col_span=1,
                 **kwargs):
        """
        Create a new ViewBox and add it as a child widget.

        Parameters
        ----------
        row : int
            The row in which to add the widget (0 is the topmost row)
        col : int
            The column in which to add the widget (0 is the leftmost column)
        row_span : int
            The number of rows to be occupied by this widget. Default is 1.
        col_span : int
            The number of columns to be occupied by this widget. Default is 1.
        **kwargs : dict
            Keyword arguments to pass to `ViewBox`.
        """
        from .viewbox import ViewBox
        view = ViewBox(**kwargs)
        return self.add_widget(view, row, col, row_span, col_span)

    def next_row(self):
        self._next_cell = [self._next_cell[0] + 1, 0]

    @property
    def grid_size(self):
        rvals = [widget[0]+widget[2] for widget in self._grid_widgets.values()]
        cvals = [widget[1]+widget[3] for widget in self._grid_widgets.values()]
        return max(rvals + [0]), max(cvals + [0])

    @property
    def layout_array(self):
        locs = -1 * np.ones(self.grid_size, int)
        for key in self._grid_widgets.keys():
            r, c, rs, cs = self._grid_widgets[key][:4]
            locs[r:r + rs, c:c + cs] = key
        return locs

    def __repr__(self):
        return (('<Grid at %s:\n' % hex(id(self))) +
                str(self.layout_array + 1) + '>')

    # MODIFIED VERSION
    def _update_child_widgets(self):
        # Resize all widgets in this grid to share space.
        n_rows, n_cols = self.grid_size
        if n_rows == 0 or n_cols == 0:
            return
        print("---")

        grid_layout = np.array([[None for _ in range(0, n_cols)]
                                for _ in range(0, n_rows)])

        for key, value in self._grid_widgets.items():
            (row, col, rspan, cspan, widget) = value
            grid_layout[row:row+rspan, col:col+cspan] = widget

        print("grid layout:\n--------\n%s" % grid_layout)

        rect = self.rect.padded(self.padding + self.margin)
        solver = SimplexSolver()

        # x, width constraints ------
        total_w_eqns = [expression.Expression() for _ in range(0, n_cols)]
        total_h_eqns = [expression.Expression() for _ in range(0, n_rows)]

        stretch_w_eqns = []
        stretch_h_eqns = []

        for key, value in self._grid_widgets.items():
            (row, col, rspan, cspan, widget) = value

            widget.var_w = Variable("w-(%s, %s)" % (row, col))
            widget.var_h = Variable("h-(%s, %s)" % (row, col))

            widget.var_x = Variable("x-(%s, %s)" % (row, col))
            widget.var_y = Variable("y-(%s, %s)" % (row, col))

            # position - row
            if row == 0:
                widget.var_y.value = 0
                solver.add_stay(widget.var_y)
            else:
                prev_widget = grid_layout[row - 1][col]
                if prev_widget is not None:
                    solver.add_constraint(widget.var_y == prev_widget.var_y + prev_widget.var_h )

            # position - col
            if col == 0:
                widget.var_x.value = 0
                solver.add_stay(widget.var_x)
            else:
                prev_widget = grid_layout[row][col - 1]
                if prev_widget is not None:
                    solver.add_constraint(widget.var_x == prev_widget.var_x + prev_widget.var_w )

            # dimensions
            for h_eqn in total_h_eqns[col:col+cspan]:
                    h_eqn.add_expression(widget.var_h)

            for w_eqn in total_w_eqns[row:row+rspan]:
                    w_eqn.add_expression(widget.var_w)

            if widget.min_width is not None:
                solver.add_constraint(widget.var_w >= widget.min_width)
            if widget.max_width is not None:
                solver.add_constraint(widget.var_w <= widget.max_width)

            if widget.min_height is not None:
                solver.add_constraint(widget.var_w >= widget.min_height)
            if widget.max_height is not None:
                solver.add_constraint(widget.var_w <= widget.max_height)

            if widget.stretch[0] is not None:
                stretch_w_eqns.append(widget.var_w / widget.stretch[0])

            if widget.stretch[1] is not None:
                stretch_h_eqns.append(widget.var_h / widget.stretch[1])

        print("total w: %s" % (total_w_eqns))
        print("total h: %s" % (total_h_eqns))
        # set total width, height eqns
        for w_eqn in total_w_eqns:
            if len(w_eqn.terms) > 0:
                solver.add_constraint(w_eqn == rect.width, strength=REQUIRED)
        for h_eqn in total_h_eqns:
            if len(h_eqn.terms) > 0:
                solver.add_constraint(h_eqn == rect.height, strength=REQUIRED)

        # add stretch eqns
        if len(stretch_w_eqns) > 1:
            for eqn in stretch_w_eqns[1:]:
                solver.add_constraint(eqn == stretch_w_eqns[0], strength=WEAK)

        if len(stretch_h_eqns) > 1:
            for eqn in stretch_h_eqns[1:]:
                solver.add_constraint(eqn == stretch_h_eqns[0], strength=WEAK)

        solver.solve()

        # copy solution
        for (_, val) in self._grid_widgets.items():
            (row, col, rspan, cspan, widget) = val
            widget.width = widget.var_w.value
            widget.height = widget.var_h.value
            widget.pos = (widget.var_x.value, widget.var_y.value)

            print("----")
            print("row, col: (%s, %s)" % (row, col))
            print("x, y: (%s, %s)" % widget.pos)
            print("w, h: (%s, %s)" % (widget.width, widget.height))
        return

    # def _update_child_widgets(self):
    #     # Resize all widgets in this grid to share space.
    #     n_rows, n_cols = self.grid_size
    #     if n_rows == 0 or n_cols == 0:
    #         return

    #     # 1. Collect information about occupied cells and their contents
    #     occupied = np.zeros((n_rows, n_cols), dtype=bool)
    #     stretch = np.zeros((n_rows, n_cols, 2), dtype=float)
    #     stretch[:] = np.nan
    #     #minsize = np.zeros((n_rows, n_cols, 2), dtype=float)
    #     for key, val in self._grid_widgets.items():
    #         w = val[4]
    #         row, col, rspan, cspan, ch = self._grid_widgets[key]
    #         occupied[row:row+rspan, col:col+cspan] = True
    #         stretch[row:row+rspan, col:col+cspan] = (np.array(w.stretch) /
    #                                                  [cspan, rspan])
    #     row_occ = occupied.sum(axis=1) > 0
    #     col_occ = occupied.sum(axis=0) > 0
    #     with warnings.catch_warnings(record=True):  # mean of empty slice
    #         row_stretch = nanmean(stretch[..., 1], axis=1)
    #         col_stretch = nanmean(stretch[..., 0], axis=0)
    #     row_stretch[np.isnan(row_stretch)] = 0
    #     col_stretch[np.isnan(col_stretch)] = 0

    #     # 2. Decide width of each row/col
    #     rect = self.rect.padded(self.padding + self.margin)
    #     n_cols = col_occ.sum()
    #     colspace = rect.width - (n_cols-1) * self.spacing
    #     colsizes = col_stretch * colspace / col_stretch.sum()
    #     colsizes[colsizes == 0] = -self.spacing
    #     n_rows = row_occ.sum()
    #     rowspace = rect.height - (n_rows-1) * self.spacing
    #     rowsizes = row_stretch * rowspace / row_stretch.sum()
    #     rowsizes[rowsizes == 0] = -self.spacing

    #     # 3. Decide placement of row/col edges
    #     colend = np.cumsum(colsizes) + self.spacing * np.arange(len(colsizes))
    #     colstart = colend - colsizes
    #     rowend = np.cumsum(rowsizes) + self.spacing * np.arange(len(rowsizes))
    #     rowstart = rowend - rowsizes

    #     # snap to pixel boundaries to avoid drawing artifacts
    #     colstart = np.round(colstart) + self.margin
    #     colend = np.round(colend) + self.margin
    #     rowstart = np.round(rowstart) + self.margin
    #     rowend = np.round(rowend) + self.margin

    #     for key in self._grid_widgets.keys():
    #         row, col, rspan, cspan, ch = self._grid_widgets[key]

    #         # Translate the origin of the node to the corner of the area
    #         # ch.transform.reset()
    #         # ch.transform.translate((colstart[col], rowstart[row]))
    #         ch.pos = colstart[col], rowstart[row]

    #         # ..and set the size to match.
    #         w = colend[col+cspan-1]-colstart[col]
    #         h = rowend[row+rspan-1]-rowstart[row]
    #         ch.size = w, h
<|endoftext|>


<commit_before>from Cookie import SimpleCookie
from pprint import pformat
from urllib import urlencode
import datastructures
from django.conf.settings import DEFAULT_MIME_TYPE

class HttpRequest(object): # needs to be new-style class because subclasses define "property"s
    "A basic HTTP request"
    def __init__(self):
        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
        self.path = ''

    def __repr__(self):
        return '<HttpRequest\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
            (pformat(self.GET), pformat(self.POST), pformat(self.COOKIES),
            pformat(self.META))

    def __getitem__(self, key):
        for d in (self.POST, self.GET):
            if d.has_key(key):
                return d[key]
        raise KeyError, "%s not found in either POST or GET" % key

    def get_full_path(self):
        return ''

def parse_file_upload(header_dict, post_data):
    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    import email, email.Message
    from cgi import parse_header
    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    raw_message += '\r\n\r\n' + post_data
    msg = email.message_from_string(raw_message)
    POST = datastructures.MultiValueDict()
    FILES = datastructures.MultiValueDict()
    for submessage in msg.get_payload():
        if isinstance(submessage, email.Message.Message):
            name_dict = parse_header(submessage['Content-Disposition'])[1]
            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
            # or {'name': 'blah'} for POST fields
            # We assume all uploaded files have a 'filename' set.
            if name_dict.has_key('filename'):
                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
                if not name_dict['filename'].strip():
                    continue
                # IE submits the full path, so trim everything but the basename.
                # (We can't use os.path.basename because it expects Linux paths.)
                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
                FILES.appendlist(name_dict['name'], {
                    'filename': filename,
                    'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
                    'content': submessage.get_payload(),
                })
            else:
                POST.appendlist(name_dict['name'], submessage.get_payload())
    return POST, FILES

class QueryDict(datastructures.MultiValueDict):
    """A specialized MultiValueDict that takes a query string when initialized.
    This is immutable unless you create a copy of it."""
    def __init__(self, query_string):
        try:
            from mod_python.util import parse_qsl
        except ImportError:
            from cgi import parse_qsl
        if not query_string:
            self.data = {}
            self._keys = []
        else:
            self.data = {}
            self._keys = []
            for name, value in parse_qsl(query_string, True): # keep_blank_values=True
                if name in self.data:
                    self.data[name].append(value)
                else:
                    self.data[name] = [value]
                if name not in self._keys:
                    self._keys.append(name)
        self._mutable = False

    def __setitem__(self, key, value):
        if not self._mutable:
            raise AttributeError, "This QueryDict instance is immutable"
        else:
            self.data[key] = [value]
            if not key in self._keys:
                self._keys.append(key)

    def setlist(self, key, list_):
        if not self._mutable:
            raise AttributeError, "This QueryDict instance is immutable"
        else:
            self.data[key] = list_
            if not key in self._keys:
                self._keys.append(key)

    def copy(self):
        "Returns a mutable copy of this object"
        cp = datastructures.MultiValueDict.copy(self)
        cp._mutable = True
        return cp

    def assert_synchronized(self):
        assert(len(self._keys) == len(self.data.keys())), \
            "QueryDict data structure is out of sync: %s %s" % (str(self._keys), str(self.data))

    def items(self):
        "Respect order preserved by self._keys"
        self.assert_synchronized()
        items = []
        for key in self._keys:
            if key in self.data:
                items.append((key, self.data[key][0]))
        return items

    def keys(self):
        self.assert_synchronized()
        return self._keys

    def urlencode(self):
        output = []
        for k, list_ in self.data.items():
            output.extend([urlencode({k: v}) for v in list_])
        return '&'.join(output)

def parse_cookie(cookie):
    if cookie == '':
        return {}
    c = SimpleCookie()
    c.load(cookie)
    cookiedict = {}
    for key in c.keys():
        cookiedict[key] = c.get(key).value
    return cookiedict

class HttpResponse:
    "A basic HTTP response, with content and dictionary-accessed headers"
    def __init__(self, content='', mimetype=DEFAULT_MIME_TYPE):
        self.content = content
        self.headers = {'Content-Type':mimetype}
        self.cookies = SimpleCookie()
        self.status_code = 200

    def __str__(self):
        "Full HTTP message, including headers"
        return '\n'.join(['%s: %s' % (key, value)
            for key, value in self.headers.items()]) \
            + '\n\n' + self.content

    def __setitem__(self, header, value):
        self.headers[header] = value

    def __delitem__(self, header):
        try:
            del self.headers[header]
        except KeyError:
            pass

    def __getitem__(self, header):
        return self.headers[header]

    def has_header(self, header):
        "Case-insensitive check for a header"
        header = header.lower()
        for key in self.headers.keys():
            if key.lower() == header:
                return True
        return False

    def set_cookie(self, key, value='', max_age=None, path='/', domain=None, secure=None):
        self.cookies[key] = value
        for var in ('max_age', 'path', 'domain', 'secure'):
            val = locals()[var]
            if val is not None:
                self.cookies[key][var.replace('_', '-')] = val

    def delete_cookie(self, key):
        try:
            self.cookies[key]['max_age'] = 0
        except KeyError:
            pass

    def get_content_as_string(self, encoding):
        """
        Returns the content as a string, encoding it from a Unicode object if
        necessary.
        """
        if isinstance(self.content, unicode):
            return self.content.encode(encoding)
        return self.content

    # The remaining methods partially implement the file-like object interface.
    # See http://docs.python.org/lib/bltin-file-objects.html
    def write(self, content):
        self.content += content

    def flush(self):
        pass

    def tell(self):
        return len(self.content)

class HttpResponseRedirect(HttpResponse):
    def __init__(self, redirect_to):
        HttpResponse.__init__(self)
        self['Location'] = redirect_to
        self.status_code = 302

class HttpResponseNotModified(HttpResponse):
    def __init__(self):
        HttpResponse.__init__(self)
        self.status_code = 304

class HttpResponseNotFound(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 404

class HttpResponseForbidden(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 403

class HttpResponseGone(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 410

class HttpResponseServerError(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 500
<commit_msg>Fixed #543 -- Fixed broken mod_python support due to an import in the wrong place<commit_after>from Cookie import SimpleCookie
from pprint import pformat
from urllib import urlencode
import datastructures

class HttpRequest(object): # needs to be new-style class because subclasses define "property"s
    "A basic HTTP request"
    def __init__(self):
        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
        self.path = ''

    def __repr__(self):
        return '<HttpRequest\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
            (pformat(self.GET), pformat(self.POST), pformat(self.COOKIES),
            pformat(self.META))

    def __getitem__(self, key):
        for d in (self.POST, self.GET):
            if d.has_key(key):
                return d[key]
        raise KeyError, "%s not found in either POST or GET" % key

    def get_full_path(self):
        return ''

def parse_file_upload(header_dict, post_data):
    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    import email, email.Message
    from cgi import parse_header
    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    raw_message += '\r\n\r\n' + post_data
    msg = email.message_from_string(raw_message)
    POST = datastructures.MultiValueDict()
    FILES = datastructures.MultiValueDict()
    for submessage in msg.get_payload():
        if isinstance(submessage, email.Message.Message):
            name_dict = parse_header(submessage['Content-Disposition'])[1]
            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
            # or {'name': 'blah'} for POST fields
            # We assume all uploaded files have a 'filename' set.
            if name_dict.has_key('filename'):
                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
                if not name_dict['filename'].strip():
                    continue
                # IE submits the full path, so trim everything but the basename.
                # (We can't use os.path.basename because it expects Linux paths.)
                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
                FILES.appendlist(name_dict['name'], {
                    'filename': filename,
                    'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
                    'content': submessage.get_payload(),
                })
            else:
                POST.appendlist(name_dict['name'], submessage.get_payload())
    return POST, FILES

class QueryDict(datastructures.MultiValueDict):
    """A specialized MultiValueDict that takes a query string when initialized.
    This is immutable unless you create a copy of it."""
    def __init__(self, query_string):
        try:
            from mod_python.util import parse_qsl
        except ImportError:
            from cgi import parse_qsl
        if not query_string:
            self.data = {}
            self._keys = []
        else:
            self.data = {}
            self._keys = []
            for name, value in parse_qsl(query_string, True): # keep_blank_values=True
                if name in self.data:
                    self.data[name].append(value)
                else:
                    self.data[name] = [value]
                if name not in self._keys:
                    self._keys.append(name)
        self._mutable = False

    def __setitem__(self, key, value):
        if not self._mutable:
            raise AttributeError, "This QueryDict instance is immutable"
        else:
            self.data[key] = [value]
            if not key in self._keys:
                self._keys.append(key)

    def setlist(self, key, list_):
        if not self._mutable:
            raise AttributeError, "This QueryDict instance is immutable"
        else:
            self.data[key] = list_
            if not key in self._keys:
                self._keys.append(key)

    def copy(self):
        "Returns a mutable copy of this object"
        cp = datastructures.MultiValueDict.copy(self)
        cp._mutable = True
        return cp

    def assert_synchronized(self):
        assert(len(self._keys) == len(self.data.keys())), \
            "QueryDict data structure is out of sync: %s %s" % (str(self._keys), str(self.data))

    def items(self):
        "Respect order preserved by self._keys"
        self.assert_synchronized()
        items = []
        for key in self._keys:
            if key in self.data:
                items.append((key, self.data[key][0]))
        return items

    def keys(self):
        self.assert_synchronized()
        return self._keys

    def urlencode(self):
        output = []
        for k, list_ in self.data.items():
            output.extend([urlencode({k: v}) for v in list_])
        return '&'.join(output)

def parse_cookie(cookie):
    if cookie == '':
        return {}
    c = SimpleCookie()
    c.load(cookie)
    cookiedict = {}
    for key in c.keys():
        cookiedict[key] = c.get(key).value
    return cookiedict

class HttpResponse:
    "A basic HTTP response, with content and dictionary-accessed headers"
    def __init__(self, content='', mimetype=None):
        if not mimetype:
            from django.conf.settings import DEFAULT_MIME_TYPE
            mimetype = DEFAULT_MIME_TYPE
        self.content = content
        self.headers = {'Content-Type':mimetype}
        self.cookies = SimpleCookie()
        self.status_code = 200

    def __str__(self):
        "Full HTTP message, including headers"
        return '\n'.join(['%s: %s' % (key, value)
            for key, value in self.headers.items()]) \
            + '\n\n' + self.content

    def __setitem__(self, header, value):
        self.headers[header] = value

    def __delitem__(self, header):
        try:
            del self.headers[header]
        except KeyError:
            pass

    def __getitem__(self, header):
        return self.headers[header]

    def has_header(self, header):
        "Case-insensitive check for a header"
        header = header.lower()
        for key in self.headers.keys():
            if key.lower() == header:
                return True
        return False

    def set_cookie(self, key, value='', max_age=None, path='/', domain=None, secure=None):
        self.cookies[key] = value
        for var in ('max_age', 'path', 'domain', 'secure'):
            val = locals()[var]
            if val is not None:
                self.cookies[key][var.replace('_', '-')] = val

    def delete_cookie(self, key):
        try:
            self.cookies[key]['max_age'] = 0
        except KeyError:
            pass

    def get_content_as_string(self, encoding):
        """
        Returns the content as a string, encoding it from a Unicode object if
        necessary.
        """
        if isinstance(self.content, unicode):
            return self.content.encode(encoding)
        return self.content

    # The remaining methods partially implement the file-like object interface.
    # See http://docs.python.org/lib/bltin-file-objects.html
    def write(self, content):
        self.content += content

    def flush(self):
        pass

    def tell(self):
        return len(self.content)

class HttpResponseRedirect(HttpResponse):
    def __init__(self, redirect_to):
        HttpResponse.__init__(self)
        self['Location'] = redirect_to
        self.status_code = 302

class HttpResponseNotModified(HttpResponse):
    def __init__(self):
        HttpResponse.__init__(self)
        self.status_code = 304

class HttpResponseNotFound(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 404

class HttpResponseForbidden(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 403

class HttpResponseGone(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 410

class HttpResponseServerError(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 500
<|endoftext|>


<commit_before>"""
Sets up the path and stuff for anything running this.

You can run "python -i run.py" to have a sort of console.
"""
import os
import sys

import dev_appserver
dev_appserver.fix_sys_path()

sys.path.insert(1, os.path.join(os.path.abspath('.'), 'gaenv'))
import app #pylint: disable=W0611
<commit_msg>Oh, didn't need that<commit_after>"""
Sets up the path and stuff for anything running this.

You can run "python -i run.py" to have a sort of console.
"""
import os
import sys

sys.path.insert(1, os.path.join(os.path.abspath('.'), 'gaenv'))
import app #pylint: disable=W0611
<|endoftext|>