<commit_before>#!/usr/bin/env python3
# coding: utf-8
"""
EXPERIMENTAL

Regular expression rule checker for Khan Academy translations.

Instructions:
 - Download https://crowdin.com/download/project/khanacademy.zip
 - Unzip the 'de' folder.
 - From the directory where the 'de' folder is located, run this script.
"""
import polib
import re
import operator
import json
import itertools
import os
import os.path
import urllib
import shutil
import htmlmin
import datetime
import collections
from multiprocessing import Pool
from ansicolor import red, black, blue
from jinja2 import Environment, FileSystemLoader
from UpdateAllFiles import getTranslationFilemapCache
from Rules import Severity, importRulesForLanguage
from LintReport import readLintCSV
from compressinja.html import HtmlCompressor

def writeToFile(filename, s):
    "Utility function to write a string to a file identified by its filename"
    with open(filename, "w") as outfile:
        outfile.write(s)

def readPOFiles(directory):
    """
    Read all PO files from a given directory and return
    a dictionary path -> PO object.

    Also supports using a single file as argument.
    """
    if os.path.isfile(directory): #Single file>=
        poFilenames = [directory]
    else:
        poFilenames = []
        #Recursively iterate directory, ignore everythin except *.po
        for (curdir, _, files) in os.walk(directory):
            for f in files:
                #Ignore non-PO files
                if not f.endswith(".po") and not f.endswith(".pot"): continue
                #Add to list of files to process
                poFilenames.append(os.path.join(curdir, f))
    # Parsing is computationally expensive.
    # Distribute processing amongst distinct processing
    #  if there is a significant number of files
    if len(poFilenames) > 10:
        pool = Pool(None) #As many as CPUs
        parsedFiles = pool.map(polib.pofile, poFilenames)
        return {path: parsedFile
                   for path, parsedFile
                   in zip(poFilenames, parsedFiles)}
    else: #Only a small number of files, process directly
        return {path: polib.pofile(path) for path in poFilenames}

_multiSpace = re.compile(r"\s+")

def genCrowdinSearchString(entry):
    s = entry.msgstr[:100].replace('*', ' ')
    s = s.replace('$', ' ').replace('\\', ' ').replace(',', ' ')
    s = s.replace('.', ' ').replace('?', ' ').replace('!', ' ')
    s = s.replace("-", " ").replace(":", " ")
    #Remove consecutive spaces
    s = _multiSpace.sub(" ", s)
    return urllib.parse.quote(s.replace('☃', ' ').replace("|", " "))

# Minification takes half the space but too much CPU time
#def minifyHTML(html):
#    return htmlmin.minify(html, remove_empty_space=True)
def minifyHTML(html):
    return html

class HTMLHitRenderer(object):
    """
    A state container for the code which applies rules and generates HTML.
    """
    def __init__(self, outdir, lang="de"):
        self.outdir = outdir
        self.lang = lang
        #Import rules by language
        rules = importRulesForLanguage(lang)
        self.rules = sorted(rules, reverse=True)
        #Initialize template engine
        self.env = Environment(loader=FileSystemLoader('templates'), trim_blocks=True, lstrip_blocks=True, extensions=[HtmlCompressor])
        self.ruleTemplate = self.env.get_template("template.html")
        self.indexTemplate = self.env.get_template("index.html")
        self.lintTemplate = self.env.get_template("lint.html")
        # Get timestamp
        self.timestamp = datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
        #Process lastdownload date (copied to the templated)
        lastdownloadPath = os.path.join("cache", "lastdownload-{0}.txt".format(lang))
        if os.path.isfile(lastdownloadPath):
            with open(lastdownloadPath) as infile:
                self.downloadTimestamp = infile.read().strip()
        else:
            self.downloadTimestamp = None
        # Initialize translation ID/URL map
        translationFilemapCache = getTranslationFilemapCache()
        self.translationURLs = {
            "{0}/{1}".format(lang, v["path"]):
                "https://crowdin.com/translate/khanacademy/{0}/enus-{1}".format(v["id"], lang)
            for v in translationFilemapCache.values()
        }
    def filepath_to_url(self, filename):
        return filename.replace("/", "_")
    def computeRuleHits(self, po, filename="[unknown filename]"):
        """
        Compute all rule hits for a single parsed PO file
        """
        unsorted = {
            rule: list(rule.apply_to_po(po, filename=filename))
            for rule in self.rules
        }
        return collections.OrderedDict(sorted(unsorted.items(), key=operator.itemgetter(0)))
    def computeRuleHitsForFileSet(self, poFiles):
        """
        For each file in the given filename -> PO object dictionary,
        compute the Rule -> Hits dictonary.

        Stores the information in the current instance.
        Does not return anything
        """
        # Compute dict with sorted & prettified filenames
        files = {filename: self.filepath_to_url(filename) for filename in poFiles.keys()}
        self.files = collections.OrderedDict(sorted(files.items()))
        # Apply rules
        self.fileRuleHits = {
            filename: self.computeRuleHits(po, filename)
            for filename, po in poFiles.items()
        }
        # Compute total stats by file
        self.statsByFile = {
            filename: {"hits": self.countRuleHitsAboveSeverity(ruleHits, Severity.standard),
               "warnings": self.countRuleHitsAboveSeverity(ruleHits, Severity.warning),
               "errors": self.countRuleHitsAboveSeverity(ruleHits, Severity.dangerous),
               "infos": self.countRuleHitsAboveSeverity(ruleHits, Severity.info),
               "notices": self.countRuleHitsAboveSeverity(ruleHits, Severity.notice),
               "link": self.filepath_to_url(filename)}
            for filename, ruleHits in self.fileRuleHits.items()
        }
        # Compute by-rule stats per file
        self.statsByFileAndRule = {
            filename: {rule: len(hits) for rule, hits in ruleHits.items()}
            for filename, ruleHits in self.fileRuleHits.items()
        }
        # Compute total stats per rule
        self.totalStatsByRule = {
            rule: sum((stat[rule] for stat in self.statsByFileAndRule.values()))
            for rule in self.rules
        }
    def countRuleHitsAboveSeverity(self, ruleHits, severity):
        """In a rule -> hitlist mapping, count the total number of hits above a given severity"""
        return sum((len(hits) for rule, hits in ruleHits.items() if rule.severity >= severity))
    def countRuleHitsAtSeverity(self, ruleHits, severity):
        """In a rule -> hitlist mapping, count the total number of hits above a given severity"""
        return sum((len(hits) for rule, hits in ruleHits.items() if rule.severity == severity))
    def writeStatsJSON(self):
        """
        Write a statistics-by-filename JSON to outdir/filestats.sjon
        """
        # Write file
        with open(os.path.join(args.outdir, "filestats.json"), "w") as outfile:
            stats = {
                filename: {"hits": self.countRuleHitsAboveSeverity(ruleHits, Severity.standard),
                           "warnings": self.countRuleHitsAboveSeverity(ruleHits, Severity.warning),
                           "errors": self.countRuleHitsAboveSeverity(ruleHits, Severity.dangerous),
                           "infos": self.countRuleHitsAboveSeverity(ruleHits, Severity.info),
                           "notices": self.countRuleHitsAboveSeverity(ruleHits, Severity.notice),
                           "link": self.filepath_to_url(filename)}
                for filename, ruleHits in self.fileRuleHits.items()
            }
            json.dump(stats, outfile)
    def _renderDirectory(self, ruleHits, ruleStats, directory, filename, filelist={}):
        # Generate output HTML for each rule
        for rule, hits in ruleHits.items():
            # Render hits for individual rule
            outfilePath = os.path.join(directory, rule.get_machine_name() + ".html")
            #Remove file (redirects to 404 file) if there are no hits
            if hits: # Render hits
                writeToFile(outfilePath,
                    minifyHTML(self.ruleTemplate.render(hits=hits, timestamp=self.timestamp, downloadTimestamp=self.downloadTimestamp, translationURLs=self.translationURLs, urllib=urllib, rule=rule, genCrowdinSearchString=genCrowdinSearchString)))
            else: # No hits
                if os.path.isfile(outfilePath):
                    os.remove(outfilePath)
        # Render file index page (no filelist)
        writeToFile(os.path.join(directory, "index.html"),
            minifyHTML(self.indexTemplate.render(rules=self.rules, timestamp=self.timestamp, files=filelist, statsByFile=self.statsByFile,
                          statsByRule=ruleStats, downloadTimestamp=self.downloadTimestamp, filename=filename, translationURLs=self.translationURLs)))
    def renderLintHTML(self):
        "Parse & render lint"
        lintFilename = os.path.join("cache", "{0}-lint.csv".format(self.lang))
        if os.path.isfile(lintFilename):
            print(black("Rendering lint...", bold=True))
            lintEntries = readLintCSV(lintFilename)
            writeToFile(os.path.join(self.outdir, "lint.html"),
                minifyHTML(self.lintTemplate.render(lintEntries=lintEntries)))
        else:
            print("Skipping lint (%s does not exist)" % lintFilename)
    def hitsToHTML(self):
        """
        Apply a rule and write a directory of output HTML files
        """
        for filename, ruleHits in self.fileRuleHits.items():
            filepath = self.filepath_to_url(filename)
            ruleStats = self.statsByFileAndRule[filename]
            # Ensure output directory is present
            directory = os.path.join(self.outdir, filepath)
            if not os.path.isdir(directory):
                os.mkdir(directory)
            # Perform rendering
            self._renderDirectory(ruleHits, ruleStats, directory, filename, {})
        #####################
        ## Render overview ##
        #####################
        # Compute global hits for every rule
        overviewHits = {
            rule: itertools.chain(*(fileHits[rule] for fileHits in self.fileRuleHits.values()))
            for rule in self.rules
        }
        self._renderDirectory(overviewHits, self.totalStatsByRule, self.outdir, filename="all files", filelist=self.files)
        # Copy static files
        shutil.copyfile("templates/katc.js", os.path.join(self.outdir, "katc.js"))
        shutil.copyfile("templates/katc.css", os.path.join(self.outdir, "katc.css"))
        shutil.copyfile("templates/404.html", os.path.join(self.outdir, "404.html"))

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--download', action='store_true', help='Download or update the directory')
    parser.add_argument('-l', '--language', default="de", help='The language directory to use/extract')
    parser.add_argument('outdir', nargs='?', default="output", help='The output directory to use')
    args = parser.parse_args()

    # Download / update if requested
    if args.download:
        download()

    # Create directory
    if not os.path.isdir(args.outdir):
        os.mkdir(args.outdir)

    # Import
    potDir = os.path.join("cache", args.language)
    print(black("Reading files from {0} folder...".format(potDir), bold=True))
    poFiles = readPOFiles(potDir)
    print(black("Read {0]} files".format(len(poFiles)), bold=True))

    # Compute hits
    print(black("Computing rules...", bold=True))
    renderer = HTMLHitRenderer(args.outdir)
    renderer.computeRuleHitsForFileSet(poFiles)
    # Ensure the HUGE po stuff goes out of scope ASAP
    poFiles = None

    # Generate HTML
    print(black("Rendering HTML...", bold=True))
    renderer.renderLintHTML()
    renderer.hitsToHTML()

    # Generate filestats.json
    print (black("Generating JSON API files...", bold=True))
    renderer.writeStatsJSON()

    # If data is present, generate subtitle information
    videosJSONPath = os.path.join("cache", "videos.json")
    if os.path.isfile(videosJSONPath):
        print (black("Rendering subtitles overview...", bold=True))
        with open(videosJSONPath) as infile:
            exercises = json.load(infile)
        subtitleTemplate = renderer.env.get_template("subtitles.html")
        writeToFile(os.path.join(args.outdir, "subtitles.html"), subtitleTemplate.render(exercises=exercises))
<commit_msg>Remove unused minifyHTML call<commit_after>#!/usr/bin/env python3
# coding: utf-8
"""
EXPERIMENTAL

Regular expression rule checker for Khan Academy translations.

Instructions:
 - Download https://crowdin.com/download/project/khanacademy.zip
 - Unzip the 'de' folder.
 - From the directory where the 'de' folder is located, run this script.
"""
import polib
import re
import operator
import json
import itertools
import os
import os.path
import urllib
import shutil
import datetime
import collections
from multiprocessing import Pool
from ansicolor import red, black, blue
from jinja2 import Environment, FileSystemLoader
from UpdateAllFiles import getTranslationFilemapCache
from Rules import Severity, importRulesForLanguage
from LintReport import readLintCSV
from compressinja.html import HtmlCompressor

def writeToFile(filename, s):
    "Utility function to write a string to a file identified by its filename"
    with open(filename, "w") as outfile:
        outfile.write(s)

def readPOFiles(directory):
    """
    Read all PO files from a given directory and return
    a dictionary path -> PO object.

    Also supports using a single file as argument.
    """
    if os.path.isfile(directory): #Single file>=
        poFilenames = [directory]
    else:
        poFilenames = []
        #Recursively iterate directory, ignore everythin except *.po
        for (curdir, _, files) in os.walk(directory):
            for f in files:
                #Ignore non-PO files
                if not f.endswith(".po") and not f.endswith(".pot"): continue
                #Add to list of files to process
                poFilenames.append(os.path.join(curdir, f))
    # Parsing is computationally expensive.
    # Distribute processing amongst distinct processing
    #  if there is a significant number of files
    if len(poFilenames) > 10:
        pool = Pool(None) #As many as CPUs
        parsedFiles = pool.map(polib.pofile, poFilenames)
        return {path: parsedFile
                   for path, parsedFile
                   in zip(poFilenames, parsedFiles)}
    else: #Only a small number of files, process directly
        return {path: polib.pofile(path) for path in poFilenames}

_multiSpace = re.compile(r"\s+")

def genCrowdinSearchString(entry):
    s = entry.msgstr[:100].replace('*', ' ')
    s = s.replace('$', ' ').replace('\\', ' ').replace(',', ' ')
    s = s.replace('.', ' ').replace('?', ' ').replace('!', ' ')
    s = s.replace("-", " ").replace(":", " ")
    #Remove consecutive spaces
    s = _multiSpace.sub(" ", s)
    return urllib.parse.quote(s.replace('☃', ' ').replace("|", " "))

class HTMLHitRenderer(object):
    """
    A state container for the code which applies rules and generates HTML.
    """
    def __init__(self, outdir, lang="de"):
        self.outdir = outdir
        self.lang = lang
        #Import rules by language
        rules = importRulesForLanguage(lang)
        self.rules = sorted(rules, reverse=True)
        #Initialize template engine
        self.env = Environment(loader=FileSystemLoader('templates'), trim_blocks=True, lstrip_blocks=True, extensions=[HtmlCompressor])
        self.ruleTemplate = self.env.get_template("template.html")
        self.indexTemplate = self.env.get_template("index.html")
        self.lintTemplate = self.env.get_template("lint.html")
        # Get timestamp
        self.timestamp = datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
        #Process lastdownload date (copied to the templated)
        lastdownloadPath = os.path.join("cache", "lastdownload-{0}.txt".format(lang))
        if os.path.isfile(lastdownloadPath):
            with open(lastdownloadPath) as infile:
                self.downloadTimestamp = infile.read().strip()
        else:
            self.downloadTimestamp = None
        # Initialize translation ID/URL map
        translationFilemapCache = getTranslationFilemapCache()
        self.translationURLs = {
            "{0}/{1}".format(lang, v["path"]):
                "https://crowdin.com/translate/khanacademy/{0}/enus-{1}".format(v["id"], lang)
            for v in translationFilemapCache.values()
        }
    def filepath_to_url(self, filename):
        return filename.replace("/", "_")
    def computeRuleHits(self, po, filename="[unknown filename]"):
        """
        Compute all rule hits for a single parsed PO file
        """
        unsorted = {
            rule: list(rule.apply_to_po(po, filename=filename))
            for rule in self.rules
        }
        return collections.OrderedDict(sorted(unsorted.items(), key=operator.itemgetter(0)))
    def computeRuleHitsForFileSet(self, poFiles):
        """
        For each file in the given filename -> PO object dictionary,
        compute the Rule -> Hits dictonary.

        Stores the information in the current instance.
        Does not return anything
        """
        # Compute dict with sorted & prettified filenames
        files = {filename: self.filepath_to_url(filename) for filename in poFiles.keys()}
        self.files = collections.OrderedDict(sorted(files.items()))
        # Apply rules
        self.fileRuleHits = {
            filename: self.computeRuleHits(po, filename)
            for filename, po in poFiles.items()
        }
        # Compute total stats by file
        self.statsByFile = {
            filename: {"hits": self.countRuleHitsAboveSeverity(ruleHits, Severity.standard),
               "warnings": self.countRuleHitsAboveSeverity(ruleHits, Severity.warning),
               "errors": self.countRuleHitsAboveSeverity(ruleHits, Severity.dangerous),
               "infos": self.countRuleHitsAboveSeverity(ruleHits, Severity.info),
               "notices": self.countRuleHitsAboveSeverity(ruleHits, Severity.notice),
               "link": self.filepath_to_url(filename)}
            for filename, ruleHits in self.fileRuleHits.items()
        }
        # Compute by-rule stats per file
        self.statsByFileAndRule = {
            filename: {rule: len(hits) for rule, hits in ruleHits.items()}
            for filename, ruleHits in self.fileRuleHits.items()
        }
        # Compute total stats per rule
        self.totalStatsByRule = {
            rule: sum((stat[rule] for stat in self.statsByFileAndRule.values()))
            for rule in self.rules
        }
    def countRuleHitsAboveSeverity(self, ruleHits, severity):
        """In a rule -> hitlist mapping, count the total number of hits above a given severity"""
        return sum((len(hits) for rule, hits in ruleHits.items() if rule.severity >= severity))
    def countRuleHitsAtSeverity(self, ruleHits, severity):
        """In a rule -> hitlist mapping, count the total number of hits above a given severity"""
        return sum((len(hits) for rule, hits in ruleHits.items() if rule.severity == severity))
    def writeStatsJSON(self):
        """
        Write a statistics-by-filename JSON to outdir/filestats.sjon
        """
        # Write file
        with open(os.path.join(args.outdir, "filestats.json"), "w") as outfile:
            stats = {
                filename: {"hits": self.countRuleHitsAboveSeverity(ruleHits, Severity.standard),
                           "warnings": self.countRuleHitsAboveSeverity(ruleHits, Severity.warning),
                           "errors": self.countRuleHitsAboveSeverity(ruleHits, Severity.dangerous),
                           "infos": self.countRuleHitsAboveSeverity(ruleHits, Severity.info),
                           "notices": self.countRuleHitsAboveSeverity(ruleHits, Severity.notice),
                           "link": self.filepath_to_url(filename)}
                for filename, ruleHits in self.fileRuleHits.items()
            }
            json.dump(stats, outfile)
    def _renderDirectory(self, ruleHits, ruleStats, directory, filename, filelist={}):
        # Generate output HTML for each rule
        for rule, hits in ruleHits.items():
            # Render hits for individual rule
            outfilePath = os.path.join(directory, rule.get_machine_name() + ".html")
            #Remove file (redirects to 404 file) if there are no hits
            if hits: # Render hits
                writeToFile(outfilePath,
                    self.ruleTemplate.render(hits=hits, timestamp=self.timestamp, downloadTimestamp=self.downloadTimestamp, translationURLs=self.translationURLs, urllib=urllib, rule=rule, genCrowdinSearchString=genCrowdinSearchString))
            else: # No hits
                if os.path.isfile(outfilePath):
                    os.remove(outfilePath)
        # Render file index page (no filelist)
        writeToFile(os.path.join(directory, "index.html"),
            self.indexTemplate.render(rules=self.rules, timestamp=self.timestamp, files=filelist, statsByFile=self.statsByFile,
                          statsByRule=ruleStats, downloadTimestamp=self.downloadTimestamp, filename=filename, translationURLs=self.translationURLs))
    def renderLintHTML(self):
        "Parse & render lint"
        lintFilename = os.path.join("cache", "{0}-lint.csv".format(self.lang))
        if os.path.isfile(lintFilename):
            print(black("Rendering lint...", bold=True))
            lintEntries = readLintCSV(lintFilename)
            writeToFile(os.path.join(self.outdir, "lint.html"),
                self.lintTemplate.render(lintEntries=lintEntries))
        else:
            print("Skipping lint (%s does not exist)" % lintFilename)
    def hitsToHTML(self):
        """
        Apply a rule and write a directory of output HTML files
        """
        for filename, ruleHits in self.fileRuleHits.items():
            filepath = self.filepath_to_url(filename)
            ruleStats = self.statsByFileAndRule[filename]
            # Ensure output directory is present
            directory = os.path.join(self.outdir, filepath)
            if not os.path.isdir(directory):
                os.mkdir(directory)
            # Perform rendering
            self._renderDirectory(ruleHits, ruleStats, directory, filename, {})
        #####################
        ## Render overview ##
        #####################
        # Compute global hits for every rule
        overviewHits = {
            rule: itertools.chain(*(fileHits[rule] for fileHits in self.fileRuleHits.values()))
            for rule in self.rules
        }
        self._renderDirectory(overviewHits, self.totalStatsByRule, self.outdir, filename="all files", filelist=self.files)
        # Copy static files
        shutil.copyfile("templates/katc.js", os.path.join(self.outdir, "katc.js"))
        shutil.copyfile("templates/katc.css", os.path.join(self.outdir, "katc.css"))
        shutil.copyfile("templates/404.html", os.path.join(self.outdir, "404.html"))

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--download', action='store_true', help='Download or update the directory')
    parser.add_argument('-l', '--language', default="de", help='The language directory to use/extract')
    parser.add_argument('outdir', nargs='?', default="output", help='The output directory to use')
    args = parser.parse_args()

    # Download / update if requested
    if args.download:
        download()

    # Create directory
    if not os.path.isdir(args.outdir):
        os.mkdir(args.outdir)

    # Import
    potDir = os.path.join("cache", args.language)
    print(black("Reading files from {0} folder...".format(potDir), bold=True))
    poFiles = readPOFiles(potDir)
    print(black("Read {0]} files".format(len(poFiles)), bold=True))

    # Compute hits
    print(black("Computing rules...", bold=True))
    renderer = HTMLHitRenderer(args.outdir)
    renderer.computeRuleHitsForFileSet(poFiles)
    # Ensure the HUGE po stuff goes out of scope ASAP
    poFiles = None

    # Generate HTML
    print(black("Rendering HTML...", bold=True))
    renderer.renderLintHTML()
    renderer.hitsToHTML()

    # Generate filestats.json
    print (black("Generating JSON API files...", bold=True))
    renderer.writeStatsJSON()

    # If data is present, generate subtitle information
    videosJSONPath = os.path.join("cache", "videos.json")
    if os.path.isfile(videosJSONPath):
        print (black("Rendering subtitles overview...", bold=True))
        with open(videosJSONPath) as infile:
            exercises = json.load(infile)
        subtitleTemplate = renderer.env.get_template("subtitles.html")
        writeToFile(os.path.join(args.outdir, "subtitles.html"), subtitleTemplate.render(exercises=exercises))
<|endoftext|>


<commit_before>"""Ensemble square-root filters for the 1-d Kuramoto-Sivashinsky eqn"""
import numpy as np
import sys
from KS import KS
from enkf import serial_ensrf, bulk_ensrf, etkf, letkf, etkf_modens,\
                 serial_ensrf_modens

if len(sys.argv) < 3:
    msg="""python KSensrf.py covinflate covlocal method

all variables are observed, assimilation interval is 3 time units (6*dt)
8 ensemble members, observation error standard deviation = 0.1.

time mean error and spread stats printed to standard output.

covinflate:  inflation parameter (if >= 1 constant covariance inflation
applied to prior, if < 1 relaxation to prior variance applied to posterior).

covlocal:  localization distance (distance at which Gaspari-Cohn polynomial goes
to zero).

method:  =0 for serial Potter method
         =1 for bulk Potter method (all obs at once)
         =2 for ETKF (no localization applied)
         =3 for LETKF (using observation localization)
         =4 for serial Potter method with localization via modulation ensemble
         =5 for ETKF with modulation ensemble
         =6 for serial Potter method using sqrt of localized Pb ensemble"""
    raise SystemExit(msg)

covinflate1 = float(sys.argv[1])
covinflate2 = float(sys.argv[2])
corrl = float(sys.argv[3])
method = int(sys.argv[4])

ntstart = 1000 # time steps to spin up truth run
ntimes = 11000 # ob times
nens = 10 # ensemble members
oberrstdev = 1.0; oberrvar = oberrstdev**2 # ob error
verbose = False # print error stats every time if True
dtassim = 0.3  # assimilation interval
smooth_len = 0 # smoothing interval for H operator (0 or identity obs).
gaussian = True # Gaussian or running average smoothing in H.
thresh = 0.99 # threshold for modulated ensemble eigenvalue truncation.
# other model parameters...
dt = 0.1; diffusion = 0.05; exponent = 16; npts = 128

np.random.seed(42) # fix random seed for reproducibility

# model instance for truth (nature) run
model = KS(N=npts,dt=dt,exponent=exponent,diffusion=diffusion)
# mode instance for ensemble
ensemble = KS(N=npts,members=nens,dt=dt,exponent=exponent,diffusion=diffusion)
for nt in range(ntstart): # spinup truth run
    model.advance()

# sample obs from truth, compute climo stats for model.
xx = []; tt = []
for nt in range(ntimes):
    model.advance()
    xx.append(model.x[0]) # single member
    tt.append(float(nt)*model.dt)
xtruth = np.array(xx,np.float)
timetruth = np.array(tt,np.float)
xtruth_mean = xtruth.mean()
xprime = xtruth - xtruth_mean
xvar = np.sum(xprime**2,axis=0)/(ntimes-1)
xtruth_stdev = np.sqrt(xvar.mean())
if verbose:
    print 'climo for truth run:'
    print 'x mean =',xtruth_mean
    print 'x stdev =',xtruth_stdev
# forward operator.
# identity obs.
ndim = ensemble.n
h = np.eye(ndim)
# smoothing in forward operator
# gaussian or heaviside kernel.
if smooth_len > 0:
    for j in range(ndim):
        for i in range(ndim):
            rr = float(i-j)
            if i-j < -(ndim/2): rr = float(ndim-j+i)
            if i-j > (ndim/2): rr = float(i-ndim-j)
            r = np.fabs(rr)/smooth_len
            if gaussian:
                h[j,i] = np.exp(-r**2/0.15) # Gaussian
            else: # running average (heaviside kernel)
                if r <= 1:
                    h[j,i] = 1.
                else:
                    h[j,i] = 0.
        # normalize H so sum of weight is 1
        h[j,:] = h[j,:]/h[j,:].sum()
obs = np.empty(xtruth.shape, xtruth.dtype)
for nt in range(xtruth.shape[0]):
    obs[nt] = np.dot(h,xtruth[nt])
obs = obs + oberrstdev*np.random.standard_normal(size=obs.shape)

# spinup ensemble
ntot = xtruth.shape[0]
nspinup = ntstart
for n in range(ntstart):
    ensemble.advance()

nsteps = int(dtassim/model.dt) # time steps in assimilation interval
if verbose:
    print 'ntstart, nspinup, ntot, nsteps =',ntstart,nspinup,ntot,nsteps
if nsteps % 1  != 0:
    raise ValueError, 'assimilation interval must be an integer number of model time steps'
else:
    nsteps = int(nsteps)

def ensrf(ensemble,xmean,xprime,h,obs,oberrvar,covlocal,method=1,z=None):
    if method == 0: # ensrf with obs one at time
        return serial_ensrf(xmean,xprime,h,obs,oberrvar,covlocal,covlocal)
    elif method == 1: # ensrf with all obs at once
        return bulk_ensrf(xmean,xprime,h,obs,oberrvar,covlocal)
    elif method == 2: # etkf (no localization)
        return etkf(xmean,xprime,h,obs,oberrvar)
    elif method == 3: # letkf
        return letkf(xmean,xprime,h,obs,oberrvar,covlocal)
    elif method == 4: # serial ensrf using 'modulated' ensemble
        return serial_ensrf_modens(xmean,xprime,h,obs,oberrvar,covlocal,z)
    elif method == 5: # etkf using 'modulated' ensemble
        return etkf_modens(xmean,xprime,h,obs,oberrvar,covlocal,z)
    elif method == 6: # serial ensrf using sqrt of localized Pb
        return serial_ensrf_modens(xmean,xprime,h,obs,oberrvar,covlocal,None)
    else:
        raise ValueError('illegal value for enkf method flag')

# define localization matrix.
covlocal = np.eye(ndim)
if corrl < 2*ndim:
    for j in range(ndim):
        for i in range(ndim):
            rr = float(i-j)
            if i-j < -(ndim/2): rr = float(ndim-j+i)
            if i-j > (ndim/2): rr = float(i-ndim-j)
            r = np.fabs(rr)/corrl
            #if r < 1.: # Bohman taper
            #    taper = (1.-r)*np.cos(np.pi*r) + np.sin(np.pi*r)/np.pi
            #taper = np.exp(-(r**2/0.15)) # Gaussian
            # Gaspari-Cohn polynomial.
            rr = 2.*r
            taper = 0.
            if r <= 0.5:
                taper = ((( -0.25*rr +0.5 )*rr +0.625 )*rr -5.0/3.0 )*rr**2+1.
            elif r > 0.5 and r < 1.:
                taper = (((( rr/12.0 -0.5 )*rr +0.625 )*rr +5.0/3.0 )*rr -5.0 )*rr \
                        + 4.0 - 2.0 / (3.0 * rr)
            covlocal[j,i]=taper

# compute square root of covlocal
if method in [4,5]:
    evals, eigs = np.linalg.eigh(covlocal)
    evals = np.where(evals > 1.e-10, evals, 1.e-10)
    evalsum = evals.sum(); neig = 0
    frac = 0.0
    while frac < thresh:
        frac = evals[ndim-neig-1:ndim].sum()/evalsum
        neig += 1
    #print 'neig = ',neig
    zz = (eigs*np.sqrt(evals/frac)).T
    z = zz[ndim-neig:ndim,:]
else:
    neig = 0
    z = None

# run assimilation.
fcsterr = []
fcstsprd = []
analerr = []
analsprd = []
diverged = False
fsprdmean = np.zeros(ndim,np.float)
asprdmean = np.zeros(ndim,np.float)
ferrmean = np.zeros(ndim,np.float)
aerrmean = np.zeros(ndim,np.float)
for nassim in range(0,ntot,nsteps):
    # assimilate obs
    xmean = ensemble.x.mean(axis=0)
    xmean_b = xmean.copy()
    xprime = ensemble.x - xmean
    # calculate background error, sprd stats.
    ferr = (xmean - xtruth[nassim])**2
    if np.isnan(ferr.mean()):
        diverged = True
        break
    fsprd = (xprime**2).sum(axis=0)/(ensemble.members-1)
    if nassim >= nspinup:
        fsprdmean = fsprdmean + fsprd
        ferrmean = ferrmean + ferr
        fcsterr.append(ferr.mean()); fcstsprd.append(fsprd.mean())
    # update state estimate.
    # use 'bulk' ensrf for first half of spinup period.
    if nassim < nspinup/2:
        xmean,xprime =\
        ensrf(ensemble,xmean,xprime,h,obs[nassim,:],oberrvar,covlocal,method=1,z=z)
    else:
        xmean,xprime =\
        ensrf(ensemble,xmean,xprime,h,obs[nassim,:],oberrvar,covlocal,method=method,z=z)
    # calculate analysis error, sprd stats.
    aerr = (xmean - xtruth[nassim])**2
    asprd = (xprime**2).sum(axis=0)/(ensemble.members-1)
    if nassim >= nspinup:
        asprdmean = asprdmean + asprd
        aerrmean = aerrmean + aerr
        analerr.append(aerr.mean()); analsprd.append(asprd.mean())
    if verbose:
        print nassim,timetruth[nassim],np.sqrt(ferr.mean()),np.sqrt(fsprd.mean()),np.sqrt(aerr.mean()),np.sqrt(asprd.mean())
    # RTPS posterior inflation (Whitaker and Hamill)
    if covinflate1 < 0.:
        fstdev = np.sqrt(fsprd); astdev = np.sqrt(asprd)
        inf_fact = 1.-covinflate1*(fstdev-astdev)/astdev
    else:
        # posterior inflation (Hodyss and Campbell)
        nminus1 = ensemble.members
        inc = xmean - xmean_b
        inf_fact = covinflate1*asprd + (fsprd/nminus1)*(asprd/fsprd)**2 +\
                   covinflate2*(2.*inc**2/nminus1)*(asprd/fsprd)**2
        inf_fact = np.sqrt(inf_fact/asprd)
    xprime *= inf_fact
    # run forecast model.
    ensemble.x = xmean + xprime
    for n in range(nsteps):
        ensemble.advance() # perfect model

# print out time mean stats.
# error and spread are normalized by observation error.
if diverged:
    print method,len(fcsterr),corrl,covinflate1,covinflate2,np.nan,np.nan,neig
else:
    fcsterr = np.array(fcsterr)
    fcstsprd = np.array(fcstsprd)
    analerr = np.array(analerr)
    analsprd = np.array(analsprd)
    fstdev = np.sqrt(fcstsprd.mean())
    astdev = np.sqrt(analsprd.mean())
    asprdmean = asprdmean/len(fcstsprd)
    aerrmean = aerrmean/len(analerr)
    #import matplotlib.pyplot as plt
    #plt.plot(np.arange(ndim),asprdmean,color='b',label='error')
    #plt.plot(np.arange(ndim),aerrmean,color='r',label='spread')
    #plt.legend()
    #plt.show()
    print method,len(fcsterr),corrl,covinflate1,covinflate2,oberrstdev,np.sqrt(fcsterr.mean()),fstdev,\
          np.sqrt(analerr.mean()),astdev,neig
<commit_msg>update<commit_after>"""Ensemble square-root filters for the 1-d Kuramoto-Sivashinsky eqn"""
import numpy as np
import sys
from KS import KS
from enkf import serial_ensrf, bulk_ensrf, etkf, letkf, etkf_modens,\
                 serial_ensrf_modens

if len(sys.argv) < 3:
    msg="""python KSensrf.py covinflate covlocal method

all variables are observed, assimilation interval is 3 time units (6*dt)
8 ensemble members, observation error standard deviation = 0.1.

time mean error and spread stats printed to standard output.

covinflate:  inflation parameter (if >= 1 constant covariance inflation
applied to prior, if < 1 relaxation to prior variance applied to posterior).

covlocal:  localization distance (distance at which Gaspari-Cohn polynomial goes
to zero).

method:  =0 for serial Potter method
         =1 for bulk Potter method (all obs at once)
         =2 for ETKF (no localization applied)
         =3 for LETKF (using observation localization)
         =4 for serial Potter method with localization via modulation ensemble
         =5 for ETKF with modulation ensemble
         =6 for serial Potter method using sqrt of localized Pb ensemble"""
    raise SystemExit(msg)

covinflate1 = float(sys.argv[1])
covinflate2 = float(sys.argv[2])
corrl = float(sys.argv[3])
method = int(sys.argv[4])

ntstart = 1000 # time steps to spin up truth run
ntimes = 11000 # ob times
nens = 10 # ensemble members
oberrstdev = 1.0; oberrvar = oberrstdev**2 # ob error
verbose = False # print error stats every time if True
dtassim = 0.3  # assimilation interval
smooth_len = 0 # smoothing interval for H operator (0 or identity obs).
gaussian = True # Gaussian or running average smoothing in H.
thresh = 0.99 # threshold for modulated ensemble eigenvalue truncation.
# other model parameters...
dt = 0.1; diffusion = 0.05; exponent = 16; npts = 128

np.random.seed(42) # fix random seed for reproducibility

# model instance for truth (nature) run
model = KS(N=npts,dt=dt,exponent=exponent,diffusion=diffusion)
# mode instance for ensemble
ensemble = KS(N=npts,members=nens,dt=dt,exponent=exponent,diffusion=diffusion)
for nt in range(ntstart): # spinup truth run
    model.advance()

# sample obs from truth, compute climo stats for model.
xx = []; tt = []
for nt in range(ntimes):
    model.advance()
    xx.append(model.x[0]) # single member
    tt.append(float(nt)*model.dt)
xtruth = np.array(xx,np.float)
timetruth = np.array(tt,np.float)
xtruth_mean = xtruth.mean()
xprime = xtruth - xtruth_mean
xvar = np.sum(xprime**2,axis=0)/(ntimes-1)
xtruth_stdev = np.sqrt(xvar.mean())
if verbose:
    print 'climo for truth run:'
    print 'x mean =',xtruth_mean
    print 'x stdev =',xtruth_stdev
# forward operator.
# identity obs.
ndim = ensemble.n
h = np.eye(ndim)
# smoothing in forward operator
# gaussian or heaviside kernel.
if smooth_len > 0:
    for j in range(ndim):
        for i in range(ndim):
            rr = float(i-j)
            if i-j < -(ndim/2): rr = float(ndim-j+i)
            if i-j > (ndim/2): rr = float(i-ndim-j)
            r = np.fabs(rr)/smooth_len
            if gaussian:
                h[j,i] = np.exp(-r**2/0.15) # Gaussian
            else: # running average (heaviside kernel)
                if r <= 1:
                    h[j,i] = 1.
                else:
                    h[j,i] = 0.
        # normalize H so sum of weight is 1
        h[j,:] = h[j,:]/h[j,:].sum()
obs = np.empty(xtruth.shape, xtruth.dtype)
for nt in range(xtruth.shape[0]):
    obs[nt] = np.dot(h,xtruth[nt])
obs = obs + oberrstdev*np.random.standard_normal(size=obs.shape)

# spinup ensemble
ntot = xtruth.shape[0]
nspinup = ntstart
for n in range(ntstart):
    ensemble.advance()

nsteps = int(dtassim/model.dt) # time steps in assimilation interval
if verbose:
    print 'ntstart, nspinup, ntot, nsteps =',ntstart,nspinup,ntot,nsteps
if nsteps % 1  != 0:
    raise ValueError, 'assimilation interval must be an integer number of model time steps'
else:
    nsteps = int(nsteps)

def ensrf(ensemble,xmean,xprime,h,obs,oberrvar,covlocal,method=1,z=None):
    if method == 0: # ensrf with obs one at time
        return serial_ensrf(xmean,xprime,h,obs,oberrvar,covlocal,covlocal)
    elif method == 1: # ensrf with all obs at once
        return bulk_ensrf(xmean,xprime,h,obs,oberrvar,covlocal)
    elif method == 2: # etkf (no localization)
        return etkf(xmean,xprime,h,obs,oberrvar)
    elif method == 3: # letkf
        return letkf(xmean,xprime,h,obs,oberrvar,covlocal)
    elif method == 4: # serial ensrf using 'modulated' ensemble
        return serial_ensrf_modens(xmean,xprime,h,obs,oberrvar,covlocal,z)
    elif method == 5: # etkf using 'modulated' ensemble
        return etkf_modens(xmean,xprime,h,obs,oberrvar,covlocal,z)
    elif method == 6: # serial ensrf using sqrt of localized Pb
        return serial_ensrf_modens(xmean,xprime,h,obs,oberrvar,covlocal,None)
    else:
        raise ValueError('illegal value for enkf method flag')

# define localization matrix.
covlocal = np.eye(ndim)
if corrl < 2*ndim:
    for j in range(ndim):
        for i in range(ndim):
            rr = float(i-j)
            if i-j < -(ndim/2): rr = float(ndim-j+i)
            if i-j > (ndim/2): rr = float(i-ndim-j)
            r = np.fabs(rr)/corrl
            #if r < 1.: # Bohman taper
            #    taper = (1.-r)*np.cos(np.pi*r) + np.sin(np.pi*r)/np.pi
            #taper = np.exp(-(r**2/0.15)) # Gaussian
            # Gaspari-Cohn polynomial.
            rr = 2.*r
            taper = 0.
            if r <= 0.5:
                taper = ((( -0.25*rr +0.5 )*rr +0.625 )*rr -5.0/3.0 )*rr**2+1.
            elif r > 0.5 and r < 1.:
                taper = (((( rr/12.0 -0.5 )*rr +0.625 )*rr +5.0/3.0 )*rr -5.0 )*rr \
                        + 4.0 - 2.0 / (3.0 * rr)
            covlocal[j,i]=taper

# compute square root of covlocal
if method in [4,5]:
    evals, eigs = np.linalg.eigh(covlocal)
    evals = np.where(evals > 1.e-10, evals, 1.e-10)
    evalsum = evals.sum(); neig = 0
    frac = 0.0
    while frac < thresh:
        frac = evals[ndim-neig-1:ndim].sum()/evalsum
        neig += 1
    #print 'neig = ',neig
    zz = (eigs*np.sqrt(evals/frac)).T
    z = zz[ndim-neig:ndim,:]
else:
    neig = 0
    z = None

# run assimilation.
fcsterr = []
fcstsprd = []
analerr = []
analsprd = []
diverged = False
fsprdmean = np.zeros(ndim,np.float)
asprdmean = np.zeros(ndim,np.float)
ferrmean = np.zeros(ndim,np.float)
aerrmean = np.zeros(ndim,np.float)
for nassim in range(0,ntot,nsteps):
    # assimilate obs
    xmean = ensemble.x.mean(axis=0)
    xmean_b = xmean.copy()
    xprime = ensemble.x - xmean
    # calculate background error, sprd stats.
    ferr = (xmean - xtruth[nassim])**2
    if np.isnan(ferr.mean()):
        diverged = True
        break
    fsprd = (xprime**2).sum(axis=0)/(ensemble.members-1)
    if nassim >= nspinup:
        fsprdmean = fsprdmean + fsprd
        ferrmean = ferrmean + ferr
        fcsterr.append(ferr.mean()); fcstsprd.append(fsprd.mean())
    # update state estimate.
    # use 'bulk' ensrf for first half of spinup period.
    if nassim < nspinup/2:
        xmean,xprime =\
        ensrf(ensemble,xmean,xprime,h,obs[nassim,:],oberrvar,covlocal,method=1,z=z)
    else:
        xmean,xprime =\
        ensrf(ensemble,xmean,xprime,h,obs[nassim,:],oberrvar,covlocal,method=method,z=z)
    # calculate analysis error, sprd stats.
    aerr = (xmean - xtruth[nassim])**2
    asprd = (xprime**2).sum(axis=0)/(ensemble.members-1)
    if nassim >= nspinup:
        asprdmean = asprdmean + asprd
        aerrmean = aerrmean + aerr
        analerr.append(aerr.mean()); analsprd.append(asprd.mean())
    if verbose:
        print nassim,timetruth[nassim],np.sqrt(ferr.mean()),np.sqrt(fsprd.mean()),np.sqrt(aerr.mean()),np.sqrt(asprd.mean())
    # RTPS posterior inflation (Whitaker and Hamill)
    if covinflate1 < 0.:
        fstdev = np.sqrt(fsprd); astdev = np.sqrt(asprd)
        inf_fact = 1.-covinflate1*(fstdev-astdev)/astdev
    else:
        # posterior inflation (Hodyss and Campbell)
        nmem = ensemble.members; inc = xmean - xmean_b
        inf_fact = covinflate1*asprd + \
        (asprd/fsprd)**2*((fsprd/nmem) + covinflate2*(2.*inc**2/(nmem-1)))
        inf_fact = np.sqrt(inf_fact/asprd)
    xprime *= inf_fact
    # run forecast model.
    ensemble.x = xmean + xprime
    for n in range(nsteps):
        ensemble.advance() # perfect model

# print out time mean stats.
# error and spread are normalized by observation error.
if diverged:
    print method,len(fcsterr),corrl,covinflate1,covinflate2,np.nan,np.nan,neig
else:
    fcsterr = np.array(fcsterr)
    fcstsprd = np.array(fcstsprd)
    analerr = np.array(analerr)
    analsprd = np.array(analsprd)
    fstdev = np.sqrt(fcstsprd.mean())
    astdev = np.sqrt(analsprd.mean())
    asprdmean = asprdmean/len(fcstsprd)
    aerrmean = aerrmean/len(analerr)
    #import matplotlib.pyplot as plt
    #plt.plot(np.arange(ndim),asprdmean,color='b',label='error')
    #plt.plot(np.arange(ndim),aerrmean,color='r',label='spread')
    #plt.legend()
    #plt.show()
    print method,len(fcsterr),corrl,covinflate1,covinflate2,oberrstdev,np.sqrt(fcsterr.mean()),fstdev,\
          np.sqrt(analerr.mean()),astdev,neig
<|endoftext|>


<commit_before>import threading, urllib
from lxml import etree
import re

from molly.apps.places.providers import BaseMapsProvider

class ACISLiveMapsProvider(BaseMapsProvider):
    """
    Populates bus stop entities with real time departure metadata using the ACIS
    Live interface
    """

    ACISLIVE_URLS = {
        # The key is the atco regional prefix - the value is a tuple consisting
        # of the base URL for the ACIS Live instance, a function to determine
        # the identifier to use for the live bus times board, and a function to
        # determine the identifier to use for the bus stop messages board
        # 010: Bristol
        # 040: Buckinghamshire
        # 050: Cambridgeshire
        # 160: Gloucestershire
        # 240: Kent
        # 250: Lancashire
        #
        '340': ('http://www.oxontime.com/',
                lambda entity: entity.identifiers.get('naptan'),
                lambda entity: entity.identifiers.get('atco'),
               ), # Oxfordshire
        '370': ('http://sypte.acislive.com/',
                lambda entity: entity.identifiers.get('atco')[:3] + entity.identifiers.get('atco')[4:],
                lambda entity: entity.identifiers.get('atco')[:3] + entity.identifiers.get('atco')[4:],
                ), # South Yorkshire
        # 440: Wessex
        # 450: West Yorkshire
        # 571: Cardiff
    }

    def get_acislive_base(self, entity):
        """
        Gets the ACIS live information for this entity

        @return: A tuple of base URL, identifier for realtime info, identifier
                 for messages
        """
        if entity.identifiers['atco'][:3] in self.ACISLIVE_URLS:
            base, departures, messages = self.ACISLIVE_URLS[entity.identifiers['atco'][:3]]
            return base, departures(entity), messages(entity)
        else:
            raise NoACISLiveInstanceException

    def get_realtime_url(self, entity):
        """
        Gets the appropriate URL for the realtime departure board for that
        entity.

        @return: A string of the URL for that stop departure board, or None if
                 there is no known departure board for that stop
        """
        base, departures, messages = self.get_acislive_base(entity)
        return base + 'pip/stop.asp?naptan=%s&textonly=1' % departures

    def get_messages_url(self, entity):
        """
        Gets the URL for the bus stop messages associated for that entity

        @return: A string of the URL for that stop departure board, or None if
                 there is no known messages URL for that stop
        """
        base, departures, messages = self.get_acislive_base(entity)
        return base + 'pip/stop_simulator_message.asp?NaPTAN=%s' % messages

    def augment_metadata(self, entities):
        threads = []
        for entity in entities:
            # Ignore non-bus stop entities
            if entity.identifiers.get('atco') is None:
                continue

            thread = threading.Thread(target=self.get_times, args=[entity])
            thread.start()
            threads.append(thread)

        for thread in threads:
            thread.join()

    def get_times(self, entity):

        try:
            realtime_url = self.get_realtime_url(entity)
            xml = etree.parse(urllib.urlopen(realtime_url),
                              parser = etree.HTMLParser())
        except (TypeError, IOError, NoACISLiveInstanceException):
            rows = []
            pip_info = None
        else:
            try:
                cells = xml.find('.//table').findall('td')
                rows = [cells[i:i+4] for i in range(0, len(cells), 4)]
            except AttributeError:
                rows = []

            # Get the messages associated with that bus stop
            try:
                messages_url = self.get_messages_url(entity)
                if messages_url != None:
                    messages_page = urllib.urlopen(messages_url).read()
                    pip_info = re.findall(r'msgs\[\d+\] = "(?P<message>[^"]+)"',
                                          messages_page)
                    pip_info = filter(lambda pip: pip != ' ', pip_info)
                else:
                    pip_info = None
            except:
                pip_info = None

        services = {}
        for row in rows:
            service, destination, proximity = [row[i].text.encode('utf8').replace('\xc2\xa0', '') for i in range(3)]

            if not service in services:
                services[service] = (destination, proximity, [])
            else:
                services[service][2].append(proximity)

        services = [(s[0], s[1][0], s[1][1], s[1][2]) for s in services.items()]
        services.sort(key= lambda x: ( ' '*(5-len(x[0]) + (1 if x[0][-1].isalpha() else 0)) + x[0] ))
        services.sort(key= lambda x: 0 if x[2]=='DUE' else int(x[2].split(' ')[0]))

        services = [{
            'service': s[0],
            'destination': s[1],
            'next': s[2],
            'following': s[3],
        } for s in services]

        entity.metadata['real_time_information'] = {
            'services': services,
            'pip_info': pip_info,
        }
        entity.metadata['meta_refresh'] = 60

class NoACISLiveInstanceException(Exception):
    """
    An exception to indicate that there is no ACIS Live instance to determine
    real-time bus information for the passed in entity
    """
    pass<commit_msg>West Yorkshire support<commit_after>import threading, urllib
from lxml import etree
import re

from molly.apps.places.providers import BaseMapsProvider

class ACISLiveMapsProvider(BaseMapsProvider):
    """
    Populates bus stop entities with real time departure metadata using the ACIS
    Live interface
    """

    ACISLIVE_URLS = {
        # The key is the atco regional prefix - the value is a tuple consisting
        # of the base URL for the ACIS Live instance, a function to determine
        # the identifier to use for the live bus times board, and a function to
        # determine the identifier to use for the bus stop messages board
        # 010: Bristol
        # 040: Buckinghamshire
        # 050: Cambridgeshire
        # 160: Gloucestershire
        # 240: Kent
        # 250: Lancashire
        #
        '340': ('http://www.oxontime.com/',
                lambda entity: entity.identifiers.get('naptan'),
                lambda entity: entity.identifiers.get('atco'),
               ), # Oxfordshire
        '370': ('http://sypte.acislive.com/',
                lambda entity: entity.identifiers.get('atco')[:3] + entity.identifiers.get('atco')[4:],
                lambda entity: entity.identifiers.get('atco')[:3] + entity.identifiers.get('atco')[4:],
                ), # South Yorkshire
        # 440: Wessex
        '450': ('http://wymetro.acislive.com/',
                lambda entity: entity.identifiers.get('plate'),
                lambda entity: entity.identifiers.get('plate'),
               ), #West Yorkshire
        # 571: Cardiff
    }

    def get_acislive_base(self, entity):
        """
        Gets the ACIS live information for this entity

        @return: A tuple of base URL, identifier for realtime info, identifier
                 for messages
        """
        if entity.identifiers['atco'][:3] in self.ACISLIVE_URLS:
            base, departures, messages = self.ACISLIVE_URLS[entity.identifiers['atco'][:3]]
            return base, departures(entity), messages(entity)
        else:
            raise NoACISLiveInstanceException

    def get_realtime_url(self, entity):
        """
        Gets the appropriate URL for the realtime departure board for that
        entity.

        @return: A string of the URL for that stop departure board, or None if
                 there is no known departure board for that stop
        """
        base, departures, messages = self.get_acislive_base(entity)
        return base + 'pip/stop.asp?naptan=%s&textonly=1' % departures

    def get_messages_url(self, entity):
        """
        Gets the URL for the bus stop messages associated for that entity

        @return: A string of the URL for that stop departure board, or None if
                 there is no known messages URL for that stop
        """
        base, departures, messages = self.get_acislive_base(entity)
        return base + 'pip/stop_simulator_message.asp?NaPTAN=%s' % messages

    def augment_metadata(self, entities):
        threads = []
        for entity in entities:
            # Ignore non-bus stop entities
            if entity.identifiers.get('atco') is None:
                continue

            thread = threading.Thread(target=self.get_times, args=[entity])
            thread.start()
            threads.append(thread)

        for thread in threads:
            thread.join()

    def get_times(self, entity):

        try:
            realtime_url = self.get_realtime_url(entity)
            xml = etree.parse(urllib.urlopen(realtime_url),
                              parser = etree.HTMLParser())
        except (TypeError, IOError, NoACISLiveInstanceException):
            rows = []
            pip_info = None
        else:
            try:
                cells = xml.find('.//table').findall('td')
                rows = [cells[i:i+4] for i in range(0, len(cells), 4)]
            except AttributeError:
                rows = []

            # Get the messages associated with that bus stop
            try:
                messages_url = self.get_messages_url(entity)
                if messages_url != None:
                    messages_page = urllib.urlopen(messages_url).read()
                    pip_info = re.findall(r'msgs\[\d+\] = "(?P<message>[^"]+)"',
                                          messages_page)
                    pip_info = filter(lambda pip: pip != ' ', pip_info)
                else:
                    pip_info = None
            except:
                pip_info = None

        services = {}
        for row in rows:
            service, destination, proximity = [row[i].text.encode('utf8').replace('\xc2\xa0', '') for i in range(3)]

            if not service in services:
                services[service] = (destination, proximity, [])
            else:
                services[service][2].append(proximity)

        services = [(s[0], s[1][0], s[1][1], s[1][2]) for s in services.items()]
        services.sort(key= lambda x: ( ' '*(5-len(x[0]) + (1 if x[0][-1].isalpha() else 0)) + x[0] ))
        services.sort(key= lambda x: 0 if x[2]=='DUE' else int(x[2].split(' ')[0]))

        services = [{
            'service': s[0],
            'destination': s[1],
            'next': s[2],
            'following': s[3],
        } for s in services]

        entity.metadata['real_time_information'] = {
            'services': services,
            'pip_info': pip_info,
        }
        entity.metadata['meta_refresh'] = 60

class NoACISLiveInstanceException(Exception):
    """
    An exception to indicate that there is no ACIS Live instance to determine
    real-time bus information for the passed in entity
    """
    pass<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from rest_framework import status
from rest_framework.test import APITestCase

from decimal import Decimal

from mortgageinsurance.models import Monthly, Upfront

class MonthlyTest(APITestCase):
    def populate_db(self):
        """ Prepopulate DB with dummy data. """

        m_ins1_1 = Monthly(insurer='INS1',min_ltv=Decimal("30.001"),max_ltv=Decimal("70"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("1.4"))
        m_ins1_1.save()
        m_ins2_1 = Monthly(insurer='INS2',min_ltv=Decimal("30.001"),max_ltv=Decimal("70"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("1.6"))
        m_ins2_1.save()
        m_ins1_2 = Monthly(insurer='INS1',min_ltv=Decimal("20.001"),max_ltv=Decimal("30"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("0.523"))
        m_ins1_2.save()
        m_ins2_2 = Monthly(insurer='INS2',min_ltv=Decimal("20.001"),max_ltv=Decimal("30"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("0.642"))
        m_ins2_2.save()
        m_fha = Monthly(insurer='FHA',min_ltv=Decimal("20.001"),max_ltv=Decimal("70"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("0.85"))
        m_fha.save()

        u_fha = Upfront(loan_type='FHA',min_ltv=Decimal("0"), max_ltv=Decimal("80"),premium=Decimal("3.75"))
        u_fha.save()
        u_va_disabled = Upfront(loan_type='VA',va_status='DISABLED',min_ltv=Decimal("0"), max_ltv=Decimal("80"),premium=Decimal("1.5"))
        u_va_disabled.save()
        u_va_regular_y = Upfront(loan_type='VA',va_status='REGULAR',va_first_use=1,min_ltv=Decimal("0"),max_ltv=Decimal("80"),premium=Decimal("2.0"))
        u_va_regular_y.save()

    def setUp(self):
        self.url = '/oah-api/mortgage-insurance/'
        self.populate_db()

    def test_mortgage_insurance__no_args(self):
        """ ... when no parameters provided """
        response = self.client.get(self.url, {})
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('price'), [u'This field is required.'])
        self.assertEqual(response.data.get('loan_amount'), [u'This field is required.'])
        self.assertEqual(response.data.get('minfico'), [u'This field is required.'])
        self.assertEqual(response.data.get('maxfico'), [u'This field is required.'])
        self.assertEqual(response.data.get('loan_term'), [u'This field is required.'])
        self.assertEqual(response.data.get('loan_type'), [u'This field is required.'])
        self.assertEqual(response.data.get('rate_structure'), [u'This field is required.'])
        self.assertEqual(len(response.data), 7)

    # Tests with invalid arguments #

    def test_mortgage_insurance__invalid_no_match(self):
        """ ... when parameters are valid but no matching data in database """

        response_fixed = self.client.get(self.url,
            {
                'price': 400000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertTrue(response_fixed.data.get('data').get('monthly')is None)
        self.assertTrue(response_fixed.data.get('data').get('upfront') is None)

    def test_mortgage_insurance__invalid_price(self):
        """ ... when price is 0 """
        response = self.client.get(self.url,
            {
                'price': 0,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('price'), [u'price needs to be greater than 0.'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_no_arm_type(self):
        """ ...missing arm_type when rate_structure is arm """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'arm_type is required if rate_structure is ARM'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_arm_type_3_1(self):
        """ ... when arm_type is 3/1 """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
                'arm_type': '3-1',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'No mortgage insurance data for 3/1 ARM'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_no_va_status(self):
        """ ... missing va_status when loan_type is va """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va',
                'rate_structure': 'fixed',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'va_status is required if loan_type is VA or VA-HB'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_no_va_first_use(self):
        """ ... missing va_first_use when loan_type is va and va_status is ! disabled """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va-hb',
                'rate_structure': 'fixed',
                'va_status': 'REGULAR',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'va_first_use is required if va_status is not DISABLED'])
        self.assertEqual(len(response.data), 1)

    # Tests with valid arguments #
    def test_mortgage_insurance__valid_fixed_not_VA_FHA(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type != VA or FHA """

        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertEqual(response_fixed.data.get('data').get('monthly'), 1.5)
        self.assertTrue(response_fixed.data.get('data').get('upfront') is None)

    def test_mortgage_insurance__valid_arm_not_VA_FHA(self):
        """ ... when Valid Rate Structure = arm, Loan_Type != VA or FHA """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
                'arm_type': '7-1'
            })

        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertEqual(response_fixed.data.get('data').get('monthly'), 1.5)
        self.assertTrue(response_fixed.data.get('data').get('upfront') is None)

    def test_mortgage_insurance__valid_fixed_FHA(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type = FHA """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'fha',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertEqual(response_fixed.data.get('data').get('monthly'), 0.85)
        self.assertEqual(response_fixed.data.get('data').get('upfront'), 3.75)

    def test_mortgage_insurance__valid_fixed_va_disabled(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type = VA, VA_Status = DISABLED """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va-hb',
                'va_status': 'disabled',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertTrue(response_fixed.data.get('data').get('monthly') is None)
        self.assertEqual(response_fixed.data.get('data').get('upfront'), 1.5)

    def test_mortgage_insurance__valid_fixed_va_not_disabled(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type = VA, VA_Status != DISABLED """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va',
                'va_status': 'regular',
                'va_first_use': 1,
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertTrue(response_fixed.data.get('data').get('monthly') is None)
        self.assertEqual(response_fixed.data.get('data').get('upfront'), 2.0)

<commit_msg>Add invalid tests for va_first_use<commit_after>from django.core.urlresolvers import reverse
from rest_framework import status
from rest_framework.test import APITestCase

from decimal import Decimal

from mortgageinsurance.models import Monthly, Upfront

class MonthlyTest(APITestCase):
    def populate_db(self):
        """ Prepopulate DB with dummy data. """

        m_ins1_1 = Monthly(insurer='INS1',min_ltv=Decimal("30.001"),max_ltv=Decimal("70"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("1.4"))
        m_ins1_1.save()
        m_ins2_1 = Monthly(insurer='INS2',min_ltv=Decimal("30.001"),max_ltv=Decimal("70"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("1.6"))
        m_ins2_1.save()
        m_ins1_2 = Monthly(insurer='INS1',min_ltv=Decimal("20.001"),max_ltv=Decimal("30"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("0.523"))
        m_ins1_2.save()
        m_ins2_2 = Monthly(insurer='INS2',min_ltv=Decimal("20.001"),max_ltv=Decimal("30"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("0.642"))
        m_ins2_2.save()
        m_fha = Monthly(insurer='FHA',min_ltv=Decimal("20.001"),max_ltv=Decimal("70"),min_fico=700,max_fico=740,loan_term=30,pmt_type='FIXED',min_loan_amt=Decimal("400000"),max_loan_amt=Decimal("1000000"),premium=Decimal("0.85"))
        m_fha.save()

        u_fha = Upfront(loan_type='FHA',min_ltv=Decimal("0"), max_ltv=Decimal("80"),premium=Decimal("3.75"))
        u_fha.save()
        u_va_disabled = Upfront(loan_type='VA',va_status='DISABLED',min_ltv=Decimal("0"), max_ltv=Decimal("80"),premium=Decimal("1.5"))
        u_va_disabled.save()
        u_va_regular_y = Upfront(loan_type='VA',va_status='REGULAR',va_first_use=1,min_ltv=Decimal("0"),max_ltv=Decimal("80"),premium=Decimal("2.0"))
        u_va_regular_y.save()

    def setUp(self):
        self.url = '/oah-api/mortgage-insurance/'
        self.populate_db()

    def test_mortgage_insurance__no_args(self):
        """ ... when no parameters provided """
        response = self.client.get(self.url, {})
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('price'), [u'This field is required.'])
        self.assertEqual(response.data.get('loan_amount'), [u'This field is required.'])
        self.assertEqual(response.data.get('minfico'), [u'This field is required.'])
        self.assertEqual(response.data.get('maxfico'), [u'This field is required.'])
        self.assertEqual(response.data.get('loan_term'), [u'This field is required.'])
        self.assertEqual(response.data.get('loan_type'), [u'This field is required.'])
        self.assertEqual(response.data.get('rate_structure'), [u'This field is required.'])
        self.assertEqual(len(response.data), 7)

    # Tests with invalid arguments #

    def test_mortgage_insurance__invalid_no_match(self):
        """ ... when parameters are valid but no matching data in database """

        response_fixed = self.client.get(self.url,
            {
                'price': 400000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertTrue(response_fixed.data.get('data').get('monthly')is None)
        self.assertTrue(response_fixed.data.get('data').get('upfront') is None)

    def test_mortgage_insurance__invalid_price(self):
        """ ... when price is 0 """
        response = self.client.get(self.url,
            {
                'price': 0,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('price'), [u'price needs to be greater than 0.'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_no_arm_type(self):
        """ ...missing arm_type when rate_structure is arm """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'arm_type is required if rate_structure is ARM'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_arm_type_3_1(self):
        """ ... when arm_type is 3/1 """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
                'arm_type': '3-1',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'No mortgage insurance data for 3/1 ARM'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_no_va_status(self):
        """ ... missing va_status when loan_type is va """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va',
                'rate_structure': 'fixed',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'va_status is required if loan_type is VA or VA-HB'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_no_va_first_use(self):
        """ ... missing va_first_use when loan_type is va and va_status is ! disabled """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va-hb',
                'rate_structure': 'fixed',
                'va_status': 'REGULAR',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'va_first_use is required if va_status is not DISABLED'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_string_va_first_use(self):
        """ ... invalid va_first_use in string when loan_type is va and va_status is ! disabled """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va-hb',
                'rate_structure': 'fixed',
                'va_status': 'REGULAR',
                'va_first_use': 'Y',
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('va_first_use'), [u'Enter a whole number.'])
        self.assertEqual(len(response.data), 1)

    def test_mortgage_insurance__invalid_number_va_first_use(self):
        """ ... invalid va_first_use in string when loan_type is va and va_status is ! disabled """
        response = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va-hb',
                'rate_structure': 'fixed',
                'va_status': 'REGULAR',
                'va_first_use': 2,
            })

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertEqual(response.data.get('non_field_errors'), [u'va_first_use needs to be 0 or 1.'])
        self.assertEqual(len(response.data), 1)

    # Tests with valid arguments #
    def test_mortgage_insurance__valid_fixed_not_VA_FHA(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type != VA or FHA """

        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertEqual(response_fixed.data.get('data').get('monthly'), 1.5)
        self.assertTrue(response_fixed.data.get('data').get('upfront') is None)

    def test_mortgage_insurance__valid_arm_not_VA_FHA(self):
        """ ... when Valid Rate Structure = arm, Loan_Type != VA or FHA """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'conf',
                'rate_structure': 'arm',
                'arm_type': '7-1'
            })

        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertEqual(response_fixed.data.get('data').get('monthly'), 1.5)
        self.assertTrue(response_fixed.data.get('data').get('upfront') is None)

    def test_mortgage_insurance__valid_fixed_FHA(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type = FHA """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'fha',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertEqual(response_fixed.data.get('data').get('monthly'), 0.85)
        self.assertEqual(response_fixed.data.get('data').get('upfront'), 3.75)

    def test_mortgage_insurance__valid_fixed_va_disabled(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type = VA, VA_Status = DISABLED """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va-hb',
                'va_status': 'disabled',
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertTrue(response_fixed.data.get('data').get('monthly') is None)
        self.assertEqual(response_fixed.data.get('data').get('upfront'), 1.5)

    def test_mortgage_insurance__valid_fixed_va_not_disabled(self):
        """ ... when Valid Rate Structure = Fixed, Loan_Type = VA, VA_Status != DISABLED """
        response_fixed = self.client.get(self.url,
            {
                'price': 700000,
                'loan_amount': 420000,
                'minfico': 700,
                'maxfico': 740,
                'loan_term': 30,
                'loan_type': 'va',
                'va_status': 'regular',
                'va_first_use': 1,
                'rate_structure': 'fixed',
            })
        self.assertEqual(response_fixed.status_code, status.HTTP_200_OK)
        self.assertFalse(response_fixed.data.get('data') is None)
        self.assertTrue(response_fixed.data.get('data').get('monthly') is None)
        self.assertEqual(response_fixed.data.get('data').get('upfront'), 2.0)

<|endoftext|>