<commit_before>#!/usr/bin/env python
#
# download and unpack a riot-web tarball.
#
# Allows `bundles` to be extracted to a common directory, and a link to
# config.json to be added.

from __future__ import print_function

import argparse
import os
import os.path
import subprocess
import sys
import tarfile
import shutil
import glob

try:
    # python3
    from urllib.request import urlretrieve
except ImportError:
    # python2
    from urllib import urlretrieve

class DeployException(Exception):
    pass

def create_relative_symlink(linkname, target):
    relpath = os.path.relpath(target, os.path.dirname(linkname))
    print ("Symlink %s -> %s" % (linkname, relpath))
    os.symlink(relpath, linkname)

def move_bundles(source, dest):
    """Move the contents of the 'bundles' directory to a common dir

    We check that we will not be overwriting anything before we proceed.

    Args:
        source (str): path to 'bundles' within the extracted tarball
        dest (str): target common directory
    """

    if not os.path.isdir(dest):
        os.mkdir(dest)

    # build a map from source to destination, checking for non-existence as we go.
    renames = {}
    for f in os.listdir(source):
        dst = os.path.join(dest, f)
        if os.path.exists(dst):
            print (
                "Skipping bundle. The bundle includes '%s' which we have previously deployed."
                % f
            )
        else:
            renames[os.path.join(source, f)] = dst

    for (src, dst) in renames.iteritems():
        print ("Move %s -> %s" % (src, dst))
        os.rename(src, dst)

class Deployer:
    def __init__(self):
        self.packages_path = "."
        self.bundles_path = None
        self.should_clean = False
        # filename -> symlink path e.g 'config.localhost.json' => '../localhost/config.json'
        self.symlink_paths = {}
        self.verify_signature = True

    def deploy(self, tarball, extract_path):
        """Download a tarball if necessary, and unpack it

        Returns:
            (str) the path to the unpacked deployment
        """
        print("Deploying %s to %s" % (tarball, extract_path))

        name_str = os.path.basename(tarball).replace(".tar.gz", "")
        extracted_dir = os.path.join(extract_path, name_str)
        if os.path.exists(extracted_dir):
            raise DeployException('Cannot unpack %s: %s already exists' % (
                tarball, extracted_dir))

        downloaded = False
        if tarball.startswith("http://") or tarball.startswith("https://"):
            tarball = self.download_and_verify(tarball)
            print("Downloaded file: %s" % tarball)
            downloaded = True

        try:
            with tarfile.open(tarball) as tar:
                tar.extractall(extract_path)
        finally:
            if self.should_clean and downloaded:
                os.remove(tarball)

        print ("Extracted into: %s" % extracted_dir)

        if self.symlink_paths:
            for link_path, file_path in self.symlink_paths.iteritems():
                create_relative_symlink(
                    target=file_path,
                    linkname=os.path.join(extracted_dir, link_path)
                )

        if self.bundles_path:
            extracted_bundles = os.path.join(extracted_dir, 'bundles')
            move_bundles(source=extracted_bundles, dest=self.bundles_path)

            # replace the extracted_bundles dir (which may not be empty if some
            # bundles were skipped) with a symlink to the common dir.
            shutil.rmtree(extracted_bundles)
            create_relative_symlink(
                target=self.bundles_path,
                linkname=extracted_bundles,
            )
        return extracted_dir

    def download_and_verify(self, url):
        tarball = self.download_file(url)

        if self.verify_signature:
            sigfile = self.download_file(url + ".asc")
            subprocess.check_call(["gpg", "--verify", sigfile, tarball])

        return tarball

    def download_file(self, url):
        if not os.path.isdir(self.packages_path):
            os.mkdir(self.packages_path)
        local_filename = os.path.join(self.packages_path,
                                      url.split('/')[-1])
        sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
        sys.stdout.flush()
        urlretrieve(url, local_filename)
        print ("Done")
        return local_filename

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
    parser.add_argument(
        "-p", "--packages-dir", default="./packages", help=(
            "The directory to download the tarball into. (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-e", "--extract-path", default="./deploys", help=(
            "The location to extract .tar.gz files to. (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-b", "--bundles-dir", nargs='?', default="./bundles", help=(
            "A directory to move the contents of the 'bundles' directory to. A \
            symlink to the bundles directory will also be written inside the \
            extracted tarball. Example: './bundles'. \
            (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-c", "--clean", action="store_true", default=False, help=(
            "Remove .tar.gz files after they have been downloaded and extracted. \
            (Default: %(default)s)"
        )
    )
    parser.add_argument(
        "--include", nargs='*', default='./config*.json', help=(
            "Symlink these files into the root of the deployed tarball. \
             Useful for config files and home pages. Supports glob syntax. \
             (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "tarball", help=(
            "filename of tarball, or URL to download."
        ),
    )

    args = parser.parse_args()

    deployer = Deployer()
    deployer.packages_path = args.packages_dir
    deployer.bundles_path = args.bundles_dir
    deployer.should_clean = args.clean

    deployer.symlink_paths = {}

    for include in args.include:
        deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })

    print("%r", (deployer.symlink_paths,))

    deployer.deploy(args.tarball, args.extract_path)
<commit_msg>oops, leftover debug<commit_after>#!/usr/bin/env python
#
# download and unpack a riot-web tarball.
#
# Allows `bundles` to be extracted to a common directory, and a link to
# config.json to be added.

from __future__ import print_function

import argparse
import os
import os.path
import subprocess
import sys
import tarfile
import shutil
import glob

try:
    # python3
    from urllib.request import urlretrieve
except ImportError:
    # python2
    from urllib import urlretrieve

class DeployException(Exception):
    pass

def create_relative_symlink(linkname, target):
    relpath = os.path.relpath(target, os.path.dirname(linkname))
    print ("Symlink %s -> %s" % (linkname, relpath))
    os.symlink(relpath, linkname)

def move_bundles(source, dest):
    """Move the contents of the 'bundles' directory to a common dir

    We check that we will not be overwriting anything before we proceed.

    Args:
        source (str): path to 'bundles' within the extracted tarball
        dest (str): target common directory
    """

    if not os.path.isdir(dest):
        os.mkdir(dest)

    # build a map from source to destination, checking for non-existence as we go.
    renames = {}
    for f in os.listdir(source):
        dst = os.path.join(dest, f)
        if os.path.exists(dst):
            print (
                "Skipping bundle. The bundle includes '%s' which we have previously deployed."
                % f
            )
        else:
            renames[os.path.join(source, f)] = dst

    for (src, dst) in renames.iteritems():
        print ("Move %s -> %s" % (src, dst))
        os.rename(src, dst)

class Deployer:
    def __init__(self):
        self.packages_path = "."
        self.bundles_path = None
        self.should_clean = False
        # filename -> symlink path e.g 'config.localhost.json' => '../localhost/config.json'
        self.symlink_paths = {}
        self.verify_signature = True

    def deploy(self, tarball, extract_path):
        """Download a tarball if necessary, and unpack it

        Returns:
            (str) the path to the unpacked deployment
        """
        print("Deploying %s to %s" % (tarball, extract_path))

        name_str = os.path.basename(tarball).replace(".tar.gz", "")
        extracted_dir = os.path.join(extract_path, name_str)
        if os.path.exists(extracted_dir):
            raise DeployException('Cannot unpack %s: %s already exists' % (
                tarball, extracted_dir))

        downloaded = False
        if tarball.startswith("http://") or tarball.startswith("https://"):
            tarball = self.download_and_verify(tarball)
            print("Downloaded file: %s" % tarball)
            downloaded = True

        try:
            with tarfile.open(tarball) as tar:
                tar.extractall(extract_path)
        finally:
            if self.should_clean and downloaded:
                os.remove(tarball)

        print ("Extracted into: %s" % extracted_dir)

        if self.symlink_paths:
            for link_path, file_path in self.symlink_paths.iteritems():
                create_relative_symlink(
                    target=file_path,
                    linkname=os.path.join(extracted_dir, link_path)
                )

        if self.bundles_path:
            extracted_bundles = os.path.join(extracted_dir, 'bundles')
            move_bundles(source=extracted_bundles, dest=self.bundles_path)

            # replace the extracted_bundles dir (which may not be empty if some
            # bundles were skipped) with a symlink to the common dir.
            shutil.rmtree(extracted_bundles)
            create_relative_symlink(
                target=self.bundles_path,
                linkname=extracted_bundles,
            )
        return extracted_dir

    def download_and_verify(self, url):
        tarball = self.download_file(url)

        if self.verify_signature:
            sigfile = self.download_file(url + ".asc")
            subprocess.check_call(["gpg", "--verify", sigfile, tarball])

        return tarball

    def download_file(self, url):
        if not os.path.isdir(self.packages_path):
            os.mkdir(self.packages_path)
        local_filename = os.path.join(self.packages_path,
                                      url.split('/')[-1])
        sys.stdout.write("Downloading %s -> %s..." % (url, local_filename))
        sys.stdout.flush()
        urlretrieve(url, local_filename)
        print ("Done")
        return local_filename

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Deploy a Riot build on a web server.")
    parser.add_argument(
        "-p", "--packages-dir", default="./packages", help=(
            "The directory to download the tarball into. (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-e", "--extract-path", default="./deploys", help=(
            "The location to extract .tar.gz files to. (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-b", "--bundles-dir", nargs='?', default="./bundles", help=(
            "A directory to move the contents of the 'bundles' directory to. A \
            symlink to the bundles directory will also be written inside the \
            extracted tarball. Example: './bundles'. \
            (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "-c", "--clean", action="store_true", default=False, help=(
            "Remove .tar.gz files after they have been downloaded and extracted. \
            (Default: %(default)s)"
        )
    )
    parser.add_argument(
        "--include", nargs='*', default='./config*.json', help=(
            "Symlink these files into the root of the deployed tarball. \
             Useful for config files and home pages. Supports glob syntax. \
             (Default: '%(default)s')"
        )
    )
    parser.add_argument(
        "tarball", help=(
            "filename of tarball, or URL to download."
        ),
    )

    args = parser.parse_args()

    deployer = Deployer()
    deployer.packages_path = args.packages_dir
    deployer.bundles_path = args.bundles_dir
    deployer.should_clean = args.clean

    deployer.symlink_paths = {}

    for include in args.include:
        deployer.symlink_paths.update({ os.path.basename(pth): pth for pth in glob.iglob(include) })

    deployer.deploy(args.tarball, args.extract_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

"""
Evaluates the performance of the detecting and fixing syntax errors.
"""

import csv
import tempfile
import math
import sys
import json
from pathlib import Path
from typing import (
    Iterable, Iterator, List, NamedTuple, Optional, Sequence, TextIO, Tuple,
    cast
)

from tqdm import tqdm

from sensibility import (
    SourceFile,
    Edit, Insertion, Deletion, Substitution,
    SourceVector,
    Corpus, Vectors,
    Agreement, Vind,
    vectorize_tokens, vocabulary,
)
from sensibility.tokenize_js import tokenize_file, check_syntax_file
from sensibility.mutations import Mutations
from sensibility.predictions import Predictions, Contexts
from sensibility._paths import VECTORS_PATH, SOURCES_PATH

# TODO: make a big FixResult object -> allow it to be encoded as JSON.
# TODO: make an IndexResult object including:
#   - cosine distance
#   - harmonic mean
#   - sum
#   - Euclidean distance
#   - indexed sum
#   - cosine distance + indexed sum
#   - rank?
# TODO: try cosine distance      -,
# TODO: try indexing and adding  -'

class SensibilityForEvaluation:
    """
    Detects and fixes syntax errors in JavaScript files.
    """
    context_length = 20

    def __init__(self, fold: int) -> None:
        self.predictions = Predictions(fold)

    def rank_and_fix(self, filename: str, k: int=4) -> 'FixResult':
        """
        Rank the syntax error location (in token number) and returns a possible
        fix for the given filename.
        """

        # Get file vector for the incorrect file.
        with cast(TextIO, open(filename, 'rt', encoding='UTF-8')) as script:
            tokens = tokenize_file(script)
        file_vector = SourceVector(vectorize_tokens(tokens))
        assert len(file_vector) > 0

        padding = self.context_length

        # Holds the lowest agreement at each point in the file.
        least_agreements: List[Agreement] = []

        # These will hold the TOP predictions at a given point.
        forwards_predictions: List[Agreement] = []
        backwards_predictions: List[Agreement] = []
        contexts = enumerate(self.contexts(file_vector))

        for index, ((prefix, token), (suffix, _)) in contexts:
            assert token == file_vector[index], (
                str(token) + ' ' + str(file_vector[index])
            )

            # Fetch predictions.
            prefix_pred = self.predictions.predict_forwards(prefix)
            suffix_pred = self.predictions.predict_backwards(suffix)

            # It should be a categorical distribution.
            assert math.isclose(sum(prefix_pred), 1.0, rel_tol=0.01)  # type: ignore
            assert math.isclose(sum(suffix_pred), 1.0, rel_tol=0.01)  # type: ignore

            # Store the TOP prediction from both models.
            top_next_prediction = index_of_max(prefix_pred)
            forwards_predictions.append(top_next_prediction)
            top_prev_prediction = index_of_max(suffix_pred)
            backwards_predictions.append(top_prev_prediction)
            assert top_next_prediction == forwards_predictions[index]

            agreement = Agreement(
                squared_error_agreement(prefix_pred, suffix_pred),
                index
            )
            least_agreements.append(agreement)

        fixes = Fixes(file_vector)

        # For the top disagreements, synthesize fixes.
        least_agreements.sort()
        for disagreement in least_agreements[:k]:
            pos = disagreement.index

            # Assume an addition. Let's try removing some tokens.
            fixes.try_delete(pos)

            likely_next: Vind = forwards_predictions[pos]
            likely_prev: Vind = backwards_predictions[pos]

            # Assume a deletion. Let's try inserting some tokens.
            if likely_next is not None:
                fixes.try_insert(pos, likely_next)
            if likely_prev is not None:
                fixes.try_insert(pos, likely_prev)

            # Assume it's a substitution. Let's try swapping the token.
            if likely_next is not None:
                fixes.try_substitute(pos, likely_next)
            if likely_prev is not None:
                fixes.try_substitute(pos, likely_prev)

        return FixResult(
            ranks=least_agreements,
            fixes=[] if not fixes else tuple(fixes)
        )

    def contexts(self, file_vector: SourceVector) -> Contexts:
        """
        Yield every context (prefix, suffix) in the given file vector.
        """
        return self.predictions.contexts(cast(Sequence[Vind], file_vector))

    @staticmethod
    def is_okay(filename: str) -> bool:
        """
        Check if the syntax is okay.
        """
        with open(filename, 'rt') as source_file:
            return check_syntax_file(cast(TextIO, source_file))

class FixResult(NamedTuple):
    # The results of detecting and fixing syntax errors.
    ranks: Sequence[Agreement]
    fixes: Sequence[Edit]

def to_text(token: Optional[Vind]) -> Optional[str]:
    return None if token is None else vocabulary.to_text(token)

class Evaluation:
    FIELDS = '''
        fold filehash n.lines n.tokens
        m.kind m.loc m.token m.old
        correct.line line.top.rank rank.correct.line
        fixed true.fix
        f.kind f.loc f.token f.old
    '''.split()

    def __init__(self, fold: int) -> None:
        assert 0 <= fold < 5
        self.fold = fold
        self._filename = f'results.{fold}.csv'

    def __enter__(self) -> None:
        self._file = open(self._filename, 'w')
        self._writer = csv.DictWriter(self._file, fieldnames=self.FIELDS)
        self._writer.writeheader()

    def __exit__(self, *exc_info) -> None:
        self._file.close()

    def write(self, *,
              program: SourceFile,
              mutation: Edit,
              fix: Optional[Edit],
              line_of_top_rank: int,
              rank_correct_line: int) -> None:

        kind, loc, new_tok, old_tok = mutation.serialize()
        row = {
            "fold": fold,
            "filehash": program.file_hash,
            "n.lines": program.sloc,
            "n.tokens": len(program.source_tokens),
            "m.kind": kind,
            "m.loc": loc,
            "m.token": to_text(new_tok),
            "m.old": to_text(old_tok),
        }

        if fix is None:
            row.update({
                "fixed": False,
                "true.fix": None,
                "f.kind": None,
                "f.loc": None,
                "f.token": None,
                "f.old": None,
            })
        else:
            kind, loc, new_tok, old_tok = fix.serialize()
            row.update({
                "fixed": True,
                "true.fix": fix == mutation.additive_inverse(),
                "f.kind": kind,
                "f.loc": loc,
                "f.token": to_text(new_tok),
                "f.old": to_text(old_tok),
            })

        self._writer.writerow(row)
        self._file.flush()

    def run(self) -> None:
        """
        Run the evaluation.
        """
        SourceFile.vectors = Vectors.connect_to(VECTORS_PATH)
        SourceFile.corpus = Corpus.connect_to(SOURCES_PATH)

        with self, Mutations() as mutations:
            for program, mutation in tqdm(mutations.for_fold(self.fold)):
                self.evaluate_mutant(program, mutation)

    def evaluate_mutant(self, program: SourceFile, mutation: Edit) -> None:
        """
        Evaluate one particular mutant.
        """
        # Figure out the line of the mutation in the original file.
        correct_line = program.line_of_token(mutation.index)

        # Apply the original mutation.
        mutant = mutation.apply(program.vector)
        with temporary_program(mutant) as mutated_file:
            # Do the (canned) prediction...
            ranked_locations, fix = rank_and_fix(fold, mutated_file)

        if not ranked_locations:
            # rank_and_fix() can occasional return a zero-sized list.
            # In which case, return early.
            return

        # Figure out the rank of the actual mutation.
        top_error_index = ranked_locations[0].index
        line_of_top_location = program.source_tokens[top_error_index].line
        rank_correct_line = first_with_line_no(ranked_locations, mutation,
                                               correct_line, program)

        self.write(program=program,
                   mutation=mutation, fix=fix,
                   line_of_top_rank=line_of_top_location,
                   rank_correct_line=rank_correct_line)

def temporary_program(program: SourceVector) -> TextIO:
    """
    Returns a temporary file that with the contents of the file written to it.
    """
    raw_file = tempfile.NamedTemporaryFile(mode='w+t', encoding='UTF-8')
    mutated_file = cast(TextIO, raw_file)
    try:
        program.print(file=mutated_file)
        mutated_file.flush()
        mutated_file.seek(0)
    except IOError as error:
        raw_file.close()
        raise error
    return mutated_file

# TODO: Move this to a different file, probably
class Fixes(Iterable[Edit]):
    def __init__(self, vector: SourceVector) -> None:
        self.vector = vector
        self.fixes: List[Edit] = []

    def try_insert(self, index: int, token: Vind) -> None:
        self._try_edit(Insertion.create_mutation(self.vector, index, token))

    def try_delete(self, index: int) -> None:
        self._try_edit(Deletion.create_mutation(self.vector, index))

    def try_substitute(self, index: int, token: Vind) -> None:
        edit = Substitution.create_mutation(self.vector, index, token)
        self._try_edit(edit)

    def _try_edit(self, edit: Edit) -> None:
        """
        Actually apply the edit to the file. Add it to the fixes if it works.
        """
        with temporary_program(edit.apply(self.vector)) as mutant_file:
            if check_syntax_file(mutant_file):
                self.fixes.append(edit)

    def __bool__(self) -> bool:
        return len(self.fixes) > 0

    def __iter__(self) -> Iterator[Edit]:
        return iter(self.fixes)

def fix_zeros(preds, epsilon=sys.float_info.epsilon):
    """
    Replace zeros with really small values..
    """
    for i, pred in enumerate(preds):
        if math.isclose(pred, 0.0):
            preds[i] = epsilon

def harmonic_mean_agreement(prefix_pred, suffix_pred):
    # Avoid NaNs
    fix_zeros(prefix_pred)
    fix_zeros(suffix_pred)

    mean = harmonic_mean(prefix_pred, suffix_pred)
    # Rank the values from lowest to top (???)
    paired_rankings = rank(mean)
    # Get the value with the minimum probability (???)
    _, min_prob = paired_rankings[0]

    return min_prob

def squared_error_agreement(prefix_pred, suffix_pred):
    """
    Return the agreement (probability) of this token using sum of squared
    errors.
    """
    # Pretend the sum of squared error is like the cross-entropy of
    # prefix and suffix.
    entropy = ((prefix_pred - suffix_pred) ** 2).sum()
    return -entropy

def index_of_max(seq: Iterable) -> int:
    return max(enumerate(seq), key=lambda t: t[1])[0]

def rank_and_fix(fold: int, mutated_file: TextIO):
    sensibility = SensibilityForEvaluation(fold)
    return sensibility.rank_and_fix(mutated_file.name)

def first_with_line_no(ranked_locations: List[Agreement],
                       mutation: Edit,
                       correct_line: int,
                       program: SourceFile) -> int:
    """
    Return the first result with the given coorect line number.
    """
    for rank, location in enumerate(ranked_locations, start=1):
        if program.line_of_token(location.index, mutation) == correct_line:
            return rank
    raise ValueError(f'Could not find any location on {correct_line}')

if __name__ == '__main__':
    fold = int(sys.argv[1])
    Evaluation(fold).run()
<commit_msg>Create IndexResult for more fine-grained information.<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

"""
Evaluates the performance of the detecting and fixing syntax errors.
"""

import csv
import tempfile
import math
import sys
import json
from pathlib import Path
from typing import (
    Iterable, Iterator, SupportsFloat, Sequence,
    List, Optional,
    NamedTuple, Tuple,
    TextIO, cast,
)

import numpy as np
from numpy.linalg import norm
from tqdm import tqdm

from sensibility import (
    SourceFile,
    Edit, Insertion, Deletion, Substitution,
    SourceVector,
    Corpus, Vectors,
    Vind,
    vectorize_tokens, vocabulary,
)
from sensibility.tokenize_js import tokenize_file, check_syntax_file
from sensibility.mutations import Mutations
from sensibility.predictions import Predictions, Contexts
from sensibility._paths import VECTORS_PATH, SOURCES_PATH

# TODO: make an IndexResult object including:
#   - cosine distance
#   - harmonic mean
#   - sum
#   - Euclidean distance
#   - indexed sum
#   - cosine distance + indexed sum
#   - rank?

def is_normalized_vector(x: np.ndarray, p: int=2, tolerance=0.01) -> bool:
    """
    Returns whether the vector is normalized

    >>> is_normalized_vector(np.array([ 0.,  0.,  1.]))
    True
    >>> is_normalized_vector(np.array([ 0.5 ,  0.25,  0.25]))
    False
    >>> is_normalized_vector(np.array([ 0.5 ,  0.25,  0.25]), p=1)
    True
    """
    return math.isclose(norm(x, p), 1.0, rel_tol=tolerance)

class IndexResult(SupportsFloat):
    """
    Provides results for EACH INDIVIDUAL INDEX in a file.
    """
    __slots__ = (
        'index',
        'cosine_similarity', 'squared_euclidean_distance',
        'harmonic_mean', 'indexed_sum'
    )

    def __init__(self, index: int, program: SourceVector,
                 a: np.ndarray, b: np.ndarray) -> None:
        assert 0 <= index < len(program)
        self.index = index

        # Categorical distributions MUST have |x|_1 == 1.0
        assert is_normalized_vector(a, p=1) and is_normalized_vector(b, p=1)

        self.harmonic_mean = ...
        self.squared_euclidean_distance = ((a - b) ** 2).sum()

        # How probable is the token at this position?
        # 2.0 == both models absoultely think this token should be here.
        # 1.0 == lukewarm---possibly one model thinks this token should be here
        # 0.0 == both models are perplexed by this token.
        self.indexed_sum = a[program[index]] + b[program[index]]

        # How similar are the two vectors?
        self.cosine_similarity = (a @ b) / (norm(a) * norm(b))

    def __float__(self) -> float:
        """
        Returns the similariy between the two elements.
        """
        # We can tweak λ to weigh local and global factors differently,
        # but for now, weigh them equally.
        # TODO: use sklearn's Lasso regression to find this coefficient?
        # TODO:                 `-> fit_intercept=False
        λ = 0.5
        return λ * self.indexed_sum / 2.0 + (1.0 - λ) * self.cosine_similarity

class SensibilityForEvaluation:
    """
    Detects and fixes syntax errors in JavaScript files.
    """
    context_length = 20

    def __init__(self, fold: int) -> None:
        self.predictions = Predictions(fold)

    def rank_and_fix(self, filename: str, k: int=4) -> 'FixResult':
        """
        Rank the syntax error location (in token number) and returns a possible
        fix for the given filename.
        """

        # Get file vector for the incorrect file.
        with cast(TextIO, open(filename, 'rt', encoding='UTF-8')) as script:
            tokens = tokenize_file(script)
        file_vector = SourceVector(vectorize_tokens(tokens))
        assert len(file_vector) > 0

        padding = self.context_length

        # Holds the lowest agreement at each point in the file.
        results: List[IndexResult] = []

        # These will hold the TOP predictions at a given point.
        forwards_predictions: List[Vind] = []
        backwards_predictions: List[Vind] = []
        contexts = enumerate(self.contexts(file_vector))

        for index, ((prefix, token), (suffix, _)) in contexts:
            assert token == file_vector[index], (
                str(token) + ' ' + str(file_vector[index])
            )

            # Fetch predictions.
            prefix_pred = np.array(self.predictions.predict_forwards(prefix))
            suffix_pred = np.array(self.predictions.predict_backwards(suffix))

            result = IndexResult(index, file_vector, prefix_pred, suffix_pred)
            results.append(result)

            # Store the TOP prediction from each model.
            # TODO: document corner cases!
            top_next_prediction = prefix_pred[prefix_pred.argmax()]
            forwards_predictions.append(cast(Vind, top_next_prediction))
            top_prev_prediction = suffix_pred[suffix_pred.argmax()]
            backwards_predictions.append(cast(Vind, top_prev_prediction))
            assert top_next_prediction == forwards_predictions[index]

        # Rank the results by some metric of similarity defined by IndexResult
        # (the top rank will be LEAST similar).
        ranked_results = tuple(sorted(results, key=float))

        # For the top disagreements, synthesize fixes.
        fixes = Fixes(file_vector)
        for disagreement in ranked_results[:k]:
            pos = disagreement.index

            # Assume an addition. Let's try removing the offensive token.
            fixes.try_delete(pos)

            likely_next: Vind = forwards_predictions[pos]
            likely_prev: Vind = backwards_predictions[pos]

            # Assume a deletion. Let's try inserting some tokens.
            fixes.try_insert(pos, likely_next)
            fixes.try_insert(pos, likely_prev)

            # Assume it's a substitution. Let's try swapping the token.
            if likely_next is not None:
                fixes.try_substitute(pos, likely_next)
            if likely_prev is not None:
                fixes.try_substitute(pos, likely_prev)

        return FixResult(
            ranks=ranked_results,
            fixes=[] if not fixes else tuple(fixes)
        )

    def contexts(self, file_vector: SourceVector) -> Contexts:
        """
        Yield every context (prefix, suffix) in the given file vector.
        """
        return self.predictions.contexts(cast(Sequence[Vind], file_vector))

    @staticmethod
    def is_okay(filename: str) -> bool:
        """
        Check if the syntax is okay.
        """
        with open(filename, 'rt') as source_file:
            return check_syntax_file(cast(TextIO, source_file))

class FixResult(NamedTuple):
    # The results of detecting and fixing syntax errors.
    ranks: Sequence[IndexResult]
    fixes: Sequence[Edit]

def to_text(token: Optional[Vind]) -> Optional[str]:
    """
    Converts the token to its textual representation, if it exists.
    """
    return None if token is None else vocabulary.to_text(token)

class Evaluation:
    FIELDS = '''
        fold filehash n.lines n.tokens
        m.kind m.loc m.token m.old
        correct.line line.top.rank rank.correct.line
        fixed true.fix
        f.kind f.loc f.token f.old
    '''.split()

    def __init__(self, fold: int) -> None:
        assert 0 <= fold < 5
        self.fold = fold
        self._filename = f'results.{fold}.csv'

    def __enter__(self) -> None:
        self._file = open(self._filename, 'w')
        self._writer = csv.DictWriter(self._file, fieldnames=self.FIELDS)
        self._writer.writeheader()

    def __exit__(self, *exc_info) -> None:
        self._file.close()

    def write(self, *,
              program: SourceFile,
              mutation: Edit,
              fix: Optional[Edit],
              line_of_top_rank: int,
              rank_correct_line: int) -> None:

        kind, loc, new_tok, old_tok = mutation.serialize()
        row = {
            "fold": fold,
            "filehash": program.file_hash,
            "n.lines": program.sloc,
            "n.tokens": len(program.source_tokens),
            "m.kind": kind,
            "m.loc": loc,
            "m.token": to_text(new_tok),
            "m.old": to_text(old_tok),
        }

        if fix is None:
            row.update({
                "fixed": False,
                "true.fix": None,
                "f.kind": None,
                "f.loc": None,
                "f.token": None,
                "f.old": None,
            })
        else:
            kind, loc, new_tok, old_tok = fix.serialize()
            row.update({
                "fixed": True,
                "true.fix": fix == mutation.additive_inverse(),
                "f.kind": kind,
                "f.loc": loc,
                "f.token": to_text(new_tok),
                "f.old": to_text(old_tok),
            })

        self._writer.writerow(row)
        self._file.flush()

    def run(self) -> None:
        """
        Run the evaluation.
        """
        SourceFile.vectors = Vectors.connect_to(VECTORS_PATH)
        SourceFile.corpus = Corpus.connect_to(SOURCES_PATH)

        with self, Mutations() as mutations:
            for program, mutation in tqdm(mutations.for_fold(self.fold)):
                self.evaluate_mutant(program, mutation)

    def evaluate_mutant(self, program: SourceFile, mutation: Edit) -> None:
        """
        Evaluate one particular mutant.
        """
        # Figure out the line of the mutation in the original file.
        correct_line = program.line_of_token(mutation.index)

        # Apply the original mutation.
        mutant = mutation.apply(program.vector)
        with temporary_program(mutant) as mutated_file:
            # Do the (canned) prediction...
            ranked_locations, fix = rank_and_fix(fold, mutated_file)

        if not ranked_locations:
            # rank_and_fix() can occasional return a zero-sized list.
            # In which case, return early.
            return

        # Figure out the rank of the actual mutation.
        top_error_index = ranked_locations[0].index
        line_of_top_location = program.source_tokens[top_error_index].line
        rank_correct_line = first_with_line_no(ranked_locations, mutation,
                                               correct_line, program)

        self.write(program=program,
                   mutation=mutation, fix=fix,
                   line_of_top_rank=line_of_top_location,
                   rank_correct_line=rank_correct_line)

def temporary_program(program: SourceVector) -> TextIO:
    """
    Returns a temporary file that with the contents of the file written to it.
    """
    raw_file = tempfile.NamedTemporaryFile(mode='w+t', encoding='UTF-8')
    mutated_file = cast(TextIO, raw_file)
    try:
        program.print(file=mutated_file)
        mutated_file.flush()
        mutated_file.seek(0)
    except IOError as error:
        raw_file.close()
        raise error
    return mutated_file

# TODO: Move this to a different file, probably
class Fixes(Iterable[Edit]):
    def __init__(self, vector: SourceVector) -> None:
        self.vector = vector
        self.fixes: List[Edit] = []

    def try_insert(self, index: int, token: Vind) -> None:
        self._try_edit(Insertion.create_mutation(self.vector, index, token))

    def try_delete(self, index: int) -> None:
        self._try_edit(Deletion.create_mutation(self.vector, index))

    def try_substitute(self, index: int, token: Vind) -> None:
        edit = Substitution.create_mutation(self.vector, index, token)
        self._try_edit(edit)

    def _try_edit(self, edit: Edit) -> None:
        """
        Actually apply the edit to the file. Add it to the fixes if it works.
        """
        with temporary_program(edit.apply(self.vector)) as mutant_file:
            if check_syntax_file(mutant_file):
                self.fixes.append(edit)

    def __bool__(self) -> bool:
        return len(self.fixes) > 0

    def __iter__(self) -> Iterator[Edit]:
        return iter(self.fixes)

def fix_zeros(preds, epsilon=sys.float_info.epsilon):
    """
    Replace zeros with really small values..
    """
    for i, pred in enumerate(preds):
        if math.isclose(pred, 0.0):
            preds[i] = epsilon

def harmonic_mean_agreement(prefix_pred, suffix_pred):
    # Avoid NaNs
    fix_zeros(prefix_pred)
    fix_zeros(suffix_pred)

    mean = harmonic_mean(prefix_pred, suffix_pred)
    # Rank the values from lowest to top (???)
    paired_rankings = rank(mean)
    # Get the value with the minimum probability (???)
    _, min_prob = paired_rankings[0]

    return min_prob

def rank_and_fix(fold: int, mutated_file: TextIO):
    sensibility = SensibilityForEvaluation(fold)
    return sensibility.rank_and_fix(mutated_file.name)

def first_with_line_no(ranked_locations: Sequence[IndexResult],
                       mutation: Edit,
                       correct_line: int,
                       program: SourceFile) -> int:
    """
    Return the first result with the given coorect line number.
    """
    for rank, location in enumerate(ranked_locations, start=1):
        if program.line_of_token(location.index, mutation) == correct_line:
            return rank
    raise ValueError(f'Could not find any location on {correct_line}')

if __name__ == '__main__':
    fold = int(sys.argv[1])
    Evaluation(fold).run()
<|endoftext|>


<commit_before>"""
Standardize names of data files on Ohio Secretary of State and
save to mappings/filenames.json

File-name conventions on OH site vary widely according to election, but typically there is a single precinct file, a race-wide (county) file
and additional files for absentee and provisional ballots. Earlier election results are in HTML and have no precinct files. This example is from
the 2012 election:

    general election
        precinct:            yyyymmddprecinct.xlsx
        county:              FinalResults.xlsx
        provisional:         provisional.xlsx
        absentee:            absentee.xlsx

    primary election
        precinct:            2012precinct.xlsx
        county:              [per-race csv files]
        provisional:         provisional.xlsx
        absentee:            absentee.xlsx

    Exceptions: 2000 & 2002 are HTML pages with no precinct-level results.

The elections object created from the Dashboard API includes a portal link to the main page of results (needed for scraping results links) and a
direct link to the most detailed data for that election (precinct-level, if available, for general and primary elections or county level otherwise).
If precinct-level results file is available, grab that. If not, run the _url_paths function to load details about the location and scope of HTML (aspx)
files. Use the path attribute and the base_url to construct the full raw results URLs.
"""
import os
from os.path import join
import re
import json
import unicodecsv
import urlparse
import scrapelib
from bs4 import BeautifulSoup

from openelex.api import elections as elec_api
from openelex.base.datasource import BaseDatasource

class Datasource(BaseDatasource):

    # PUBLIC INTERFACE
    def mappings(self, year=None):
        """Return array of dicts containing source url and
        standardized filename for raw results file, along
        with other pieces of metadata
        """
        mappings = []
        for yr, elecs in self.elections(year).items():
            mappings.extend(self._build_metadata(yr, elecs))
        return mappings

    def target_urls(self, year=None):
        "Get list of source data urls, optionally filtered by year"
        return [item['raw_url'] for item in self.mappings(year)]

    def filename_url_pairs(self, year=None):
        return [(item['generated_filename'], item['raw_url'])
                for item in self.mappings(year)]

    def elections(self, year=None):
        # Fetch all elections initially and stash on instance
        if not hasattr(self, '_elections'):
            # Store elections by year
            self._elections = {}
            for elec in elec_api.find(self.state):
                rtype = elec['race_type'].lower()
                elec['slug'] = "-".join((self.state, elec['start_date'], rtype))
                yr = int(elec['start_date'][:4])
                self._elections.setdefault(yr, []).append(elec)
        if year:
            year_int = int(year)
            return {year_int: self._elections[year_int]}
        return self._elections

    # PRIVATE METHODS

    def _build_metadata(self, year, elections):
        meta = []
        year_int = int(year)
        # if precinct-level files available, just grab those - general and primary only
        precinct_elections = [e for e in elections if e['precinct_level'] == True]
        other_elections = [e for e in elections if e['precinct_level'] == False]
        if precinct_elections:
            meta.append(self._precinct_meta(year, precinct_elections))
        if other_elections:
            for election in other_elections:
                results = [x for x in self._url_paths() if x['date'] == election['start_date']]
                for result in results:
                    meta.append({
                        "generated_filename": self._generate_office_filename(election['direct_link'], election['start_date'], election['race_type'], result),
                        "raw_url": self._build_raw_url(year, result['path']),
                        "ocd_id": 'ocd-division/country:us/state:oh',
                        "name": 'Ohio',
                        "election": election['slug']
                    })
        return meta

    def _build_raw_url(self, year, path):
        return "http://www.sos.state.oh.us/sos/elections/Research/electResultsMain/%sElectionsResults/%s" % (year, path)

    def _precinct_meta(self, year, elections):
        payload = []
        meta = {
            'ocd_id': 'ocd-division/country:us/state:oh/precinct:all',
            'name': 'Ohio',
        }

        try:
            general = [e for e in elections if e['race_type'] == 'general'][0]
        except:
            general = None

        try:
            primary = [e for e in elections if e['race_type'] == 'primary'][0]
        except:
            primary = None

        if general:
            # Add General meta to payload
            general_url = general['direct_link']
            general_filename = self._generate_precinct_filename(general_url, general['start_date'], 'general')
            gen_meta = meta.copy()
            gen_meta.update({
                'raw_url': general_url,
                'generated_filename': general_filename,
                'election': general['slug']
            })
            payload.append(gen_meta)

        # Add Primary meta to payload
        if primary and int(year) > 2000:
            pri_meta = meta.copy()
            primary_url = primary['direct_link']
            primary_filename = self._generate_precinct_filename(primary_url, primary['start_date'], 'primary')
            pri_meta.update({
                'raw_url': primary_url,
                'generated_filename': primary_filename,
                'election': primary['slug']
            })
            payload.append(pri_meta)
        return payload

    def _generate_precinct_filename(self, url, start_date, election_type):
        # example: 20121106__oh__general__precincts.xlsx
        bits = [
            start_date.replace('-',''),
            self.state.lower(),
            election_type,
            'precincts'
        ]
        path = urlparse.urlparse(url).path
        ext = os.path.splitext(path)[1]
        name = "__".join(bits)+ ext
        return name

    def _generate_office_filename(self, url, start_date, election_type, result):
        # example: 20021105__oh__general__gov.aspx
        if result['district'] == '':
            office = result['office']
        else:
            office = result['office'] + '__' + result['district']
        if result['special'] == '1':
            election_type = election_type + '__special'
        if result['race_type'] == 'general':
            bits = [
                start_date.replace('-',''),
                self.state.lower(),
                election_type,
                office
            ]
        else:
            bits = [
                start_date.replace('-',''),
                self.state.lower(),
                result['party'],
                election_type,
                office
            ]
        path = urlparse.urlparse(url).path
        ext = os.path.splitext(path)[1]
        name = "__".join(bits)+ ext
        return name

    # returns a list of html links from results page
    def _results_links(self, year, election):
        s = scrapelib.Scraper(requests_per_minute=10, follow_robots=True)
        html = s.urlopen(election['portal_link'])
        soup = BeautifulSoup(html)
        links = [x.get('href') for x in soup.find_all('a')]
        results = [l for l in links if 'xlsx' in str(l)]
        # filter on paths for election

    def _url_paths(self):
        "Returns a JSON array of url path mappings"
        filename = join(self.mappings_dir, 'url_paths.csv')
        with open(filename, 'rU') as csvfile:
            reader = unicodecsv.DictReader(csvfile)
            mappings = json.dumps([row for row in reader])
        return json.loads(mappings)

    def _jurisdictions(self):
        """Ohio counties"""
        m = self.jurisdiction_mappings()
        mappings = [x for x in m if x['results_name'] != ""]
        return mappings
<commit_msg>cleaned up<commit_after>"""
Standardize names of data files on Ohio Secretary of State and
save to mappings/filenames.json

File-name conventions on OH site vary widely according to election, but typically there is a single precinct file, a race-wide (county) file
and additional files for absentee and provisional ballots. Earlier election results are in HTML and have no precinct files. This example is from
the 2012 election:

    general election
        precinct:            yyyymmddprecinct.xlsx
        county:              FinalResults.xlsx
        provisional:         provisional.xlsx
        absentee:            absentee.xlsx

    primary election
        precinct:            2012precinct.xlsx
        county:              [per-race csv files]
        provisional:         provisional.xlsx
        absentee:            absentee.xlsx

    Exceptions: 2000 & 2002 are HTML pages with no precinct-level results.

The elections object created from the Dashboard API includes a portal link to the main page of results (needed for scraping results links) and a
direct link to the most detailed data for that election (precinct-level, if available, for general and primary elections or county level otherwise).
If precinct-level results file is available, grab that. If not, run the _url_paths function to load details about the location and scope of HTML (aspx)
files. Use the path attribute and the base_url to construct the full raw results URLs.
"""
import os
from os.path import join
import re
import json
import unicodecsv
import urlparse

from openelex.api import elections as elec_api
from openelex.base.datasource import BaseDatasource

class Datasource(BaseDatasource):

    # PUBLIC INTERFACE
    def mappings(self, year=None):
        """Return array of dicts containing source url and
        standardized filename for raw results file, along
        with other pieces of metadata
        """
        mappings = []
        for yr, elecs in self.elections(year).items():
            mappings.extend(self._build_metadata(yr, elecs))
        return mappings

    def target_urls(self, year=None):
        "Get list of source data urls, optionally filtered by year"
        return [item['raw_url'] for item in self.mappings(year)]

    def filename_url_pairs(self, year=None):
        return [(item['generated_filename'], item['raw_url'])
                for item in self.mappings(year)]

    def elections(self, year=None):
        # Fetch all elections initially and stash on instance
        if not hasattr(self, '_elections'):
            # Store elections by year
            self._elections = {}
            for elec in elec_api.find(self.state):
                rtype = elec['race_type'].lower()
                elec['slug'] = "-".join((self.state, elec['start_date'], rtype))
                yr = int(elec['start_date'][:4])
                self._elections.setdefault(yr, []).append(elec)
        if year:
            year_int = int(year)
            return {year_int: self._elections[year_int]}
        return self._elections

    # PRIVATE METHODS

    def _build_metadata(self, year, elections):
        meta = []
        year_int = int(year)
        # if precinct-level files available, just grab those - general and primary only
        precinct_elections = [e for e in elections if e['precinct_level'] == True]
        other_elections = [e for e in elections if e['precinct_level'] == False]
        if precinct_elections:
            meta.append(self._precinct_meta(year, precinct_elections))
        if other_elections:
            for election in other_elections:
                results = [x for x in self._url_paths() if x['date'] == election['start_date']]
                for result in results:
                    meta.append({
                        "generated_filename": self._generate_office_filename(election['direct_link'], election['start_date'], election['race_type'], result),
                        "raw_url": self._build_raw_url(year, result['path']),
                        "ocd_id": 'ocd-division/country:us/state:oh',
                        "name": 'Ohio',
                        "election": election['slug']
                    })
        return meta

    def _build_raw_url(self, year, path):
        return "http://www.sos.state.oh.us/sos/elections/Research/electResultsMain/%sElectionsResults/%s" % (year, path)

    def _precinct_meta(self, year, elections):
        payload = []
        meta = {
            'ocd_id': 'ocd-division/country:us/state:oh/precinct:all',
            'name': 'Ohio',
        }

        try:
            general = [e for e in elections if e['race_type'] == 'general'][0]
        except:
            general = None

        try:
            primary = [e for e in elections if e['race_type'] == 'primary'][0]
        except:
            primary = None

        if general:
            # Add General meta to payload
            general_url = general['direct_link']
            general_filename = self._generate_precinct_filename(general_url, general['start_date'], 'general')
            gen_meta = meta.copy()
            gen_meta.update({
                'raw_url': general_url,
                'generated_filename': general_filename,
                'election': general['slug']
            })
            payload.append(gen_meta)

        # Add Primary meta to payload
        if primary and int(year) > 2000:
            pri_meta = meta.copy()
            primary_url = primary['direct_link']
            primary_filename = self._generate_precinct_filename(primary_url, primary['start_date'], 'primary')
            pri_meta.update({
                'raw_url': primary_url,
                'generated_filename': primary_filename,
                'election': primary['slug']
            })
            payload.append(pri_meta)
        return payload

    def _generate_precinct_filename(self, url, start_date, election_type):
        # example: 20121106__oh__general__precincts.xlsx
        bits = [
            start_date.replace('-',''),
            self.state.lower(),
            election_type,
            'precincts'
        ]
        path = urlparse.urlparse(url).path
        ext = os.path.splitext(path)[1]
        name = "__".join(bits)+ ext
        return name

    def _generate_office_filename(self, url, start_date, election_type, result):
        # example: 20021105__oh__general__gov.aspx
        if result['district'] == '':
            office = result['office']
        else:
            office = result['office'] + '__' + result['district']
        if result['special'] == '1':
            election_type = election_type + '__special'
        if result['race_type'] == 'general':
            bits = [
                start_date.replace('-',''),
                self.state.lower(),
                election_type,
                office
            ]
        else:
            bits = [
                start_date.replace('-',''),
                self.state.lower(),
                result['party'],
                election_type,
                office
            ]
        path = urlparse.urlparse(url).path
        ext = os.path.splitext(path)[1]
        name = "__".join(bits)+ ext
        return name

    # returns a list of html links from results page
    def _results_links(self, year, election):
        s = scrapelib.Scraper(requests_per_minute=10, follow_robots=True)
        html = s.urlopen(election['portal_link'])
        soup = BeautifulSoup(html)
        links = [x.get('href') for x in soup.find_all('a')]
        results = [l for l in links if 'xlsx' in str(l)]
        # filter on paths for election

    def _url_paths(self):
        "Returns a JSON array of url path mappings"
        filename = join(self.mappings_dir, 'url_paths.csv')
        with open(filename, 'rU') as csvfile:
            reader = unicodecsv.DictReader(csvfile)
            mappings = json.dumps([row for row in reader])
        return json.loads(mappings)

    def _jurisdictions(self):
        """Ohio counties"""
        m = self.jurisdiction_mappings()
        mappings = [x for x in m if x['results_name'] != ""]
        return mappings
<|endoftext|>


<commit_before>import sys
import traceback
import threading
import functools

def call_task(task, func, args, kwargs):
    try:
        result = func(*args, **kwargs)
    except:
        exn = sys.exc_info()[1]
        tbstr = traceback.format_exc()
        task.set_failed(exn, tbstr)
    else:
        task.set_done(result)

def async_call_task(task, func, args, kwargs):
    thread = threading.Thread(target=call_task, args=(task, func, args, kwargs))
    thread.daemon = True
    thread.start()

class Scheduler(object):
    def do_call(self):
        raise NotImplementedError()

    def call(self, func, *args, **kwargs):
        self.do_call(func, args, kwargs)

    def async_call(self, func, *args, **kwargs):
        task = Task(self)
        async_call_task(task, func, args, kwargs)
        return task

WAITING, DONE, FAILED, CANCELLED = range(4)

class TaskNotReady(Exception):
    pass

class TaskAlreadySet(Exception):
    pass

class TaskCancelled(Exception):
    pass

class Task(object):
    def __init__(self, scheduler, success=None, failure=None):
        self.scheduler = scheduler
        self.__on_success = success
        self.__on_failure = failure
        self.__status = WAITING
        self.__result = None
        self.__traceback = ""
        self.__cond = threading.Condition()

    def set_done(self, result):
        with self.__cond:
            if self.__status == CANCELLED:
                return
            if self.__status != WAITING:
                raise TaskAlreadySet()
            self.__result = result
            self.__status = DONE
            if self.__on_success is not None:
                self.scheduler.call(self.__on_success, result)

    def set_failed(self, exn, traceback=""):
        with self.__cond:
            if self.__status == CANCELLED:
                return
            if self.__status != WAITING:
                raise TaskAlreadySet()
            self.__result = exn
            self.__traceback = traceback
            self.__status = FAILED
            if self.__on_failure is not None:
                self.scheduler.call(self.__on_failure, exn, traceback)
            else:
                if traceback:
                    print traceback
                else:
                    print "Error: Task Failed:", exn

    def wait(self):
        with self.__cond:
            while self.__status == WAITING:
                self.__cond.wait()
            if self.__status == DONE:
                return self.__result
            elif self.__status == FAILED:
                raise self.__result
            elif self.__status == CANCELLED:
                raise TaskCancelled()

    @property
    def ready(self):
        with self.__cond:
            return self.__status != WAITING

    def __check_ready(self):
        if self.__status == WAITING:
            raise TaskNotReady()
        if self.__status == CANCELLED:
            raise TaskCancelled()

    @property
    def failed(self):
        with self.__cond:
            self.__check_ready()
            return self.__status == FAILED

    @property
    def result(self):
        with self.__cond:
            self.__check_ready()
            return self.__result

    def success(self, success):
        with self.__cond:
            if self.__status == DONE:
                self.scheduler.call(success, self.__result)
            elif self.__status == WAITING:
                self.__on_success = success
            elif self.__status == CANCELLED:
                raise TaskCancelled()

    def failure(self, failure):
        with self.__cond:
            if self.__status == FAILED:
                self.scheduler.call(failure, self.__result, self.__traceback)
            elif self.__status == WAITING:
                self.__on_failure = failure
            elif self.__status == CANCELLED:
                raise TaskCancelled()

    success = property(fset=success)
    failure = property(fset=failure)

    def bind(self, success=None, failure=None):
        if success is not None:
            self.success = success
        if failure is not None:
            self.failure = failure

    def cancel(self):
        with self.__cond:
            if self.__status == WAITING:
                self.__status = CANCELLED

class CoroutineTask(Task):
    def __init__(self, scheduler, gen):
        Task.__init__(self, scheduler)
        self.__gen = gen
        self.__next(self.__gen.next)

    def __next(self, func, *args, **kwargs):
        try:
            ret = func(*args, **kwargs)
        except StopIteration:
            self.set_done(None)
        except Exception, exn:
            self.set_failed(exn, traceback.format_exc())
        else:
            if isinstance(ret, Task):
                ret.bind(self.on_success, self.on_failure)
            else:
                self.set_done(ret)

    def on_success(self, result):
        self.__next(self.__gen.send, result)

    def on_failure(self, exn, traceback):
        self.__next(self.__gen.throw, exn)

def coroutine(scheduler, func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        return CoroutineTask(scheduler, func(*args, **kwargs))
    return wrapper

def coroutine_func(func):
    @functools.wraps(func)
    def wrapper(scheduler, *args, **kwargs):
        return CoroutineTask(scheduler, func(*args, **kwargs))
    return wrapper

def coroutine_method(method):
    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        return CoroutineTask(self.scheduler, method(self, *args, **kwargs))
    return wrapper
<commit_msg>Renamed<commit_after>import sys
import traceback
import threading
import functools

def call_task(task, func, args, kwargs):
    try:
        result = func(*args, **kwargs)
    except:
        exn = sys.exc_info()[1]
        tbstr = traceback.format_exc()
        task.set_failed(exn, tbstr)
    else:
        task.set_done(result)

def async_call_task(task, func, args, kwargs):
    thread = threading.Thread(target=call_task, args=(task, func, args, kwargs))
    thread.daemon = True
    thread.start()

class Scheduler(object):
    def do_call(self):
        raise NotImplementedError()

    def call(self, func, *args, **kwargs):
        self.do_call(func, args, kwargs)

    def async_call(self, func, *args, **kwargs):
        task = Task(self)
        async_call_task(task, func, args, kwargs)
        return task

WAITING, DONE, FAILED, CANCELLED = range(4)

class TaskNotReady(Exception):
    pass

class TaskAlreadySet(Exception):
    pass

class TaskCancelled(Exception):
    pass

class Task(object):
    def __init__(self, scheduler, success=None, failure=None):
        self.scheduler = scheduler
        self.__on_success = success
        self.__on_failure = failure
        self.__status = WAITING
        self.__result = None
        self.__traceback = ""
        self.__cond = threading.Condition()

    def set_done(self, result):
        with self.__cond:
            if self.__status == CANCELLED:
                return
            if self.__status != WAITING:
                raise TaskAlreadySet()
            self.__result = result
            self.__status = DONE
            if self.__on_success is not None:
                self.scheduler.call(self.__on_success, result)

    def set_failed(self, exn, traceback=""):
        with self.__cond:
            if self.__status == CANCELLED:
                return
            if self.__status != WAITING:
                raise TaskAlreadySet()
            self.__result = exn
            self.__traceback = traceback
            self.__status = FAILED
            if self.__on_failure is not None:
                self.scheduler.call(self.__on_failure, exn, traceback)
            else:
                if traceback:
                    print traceback
                else:
                    print "Error: Task Failed:", exn

    def wait(self):
        with self.__cond:
            while self.__status == WAITING:
                self.__cond.wait()
            if self.__status == DONE:
                return self.__result
            elif self.__status == FAILED:
                raise self.__result
            elif self.__status == CANCELLED:
                raise TaskCancelled()

    @property
    def ready(self):
        with self.__cond:
            return self.__status != WAITING

    def __check_ready(self):
        if self.__status == WAITING:
            raise TaskNotReady()
        if self.__status == CANCELLED:
            raise TaskCancelled()

    @property
    def failed(self):
        with self.__cond:
            self.__check_ready()
            return self.__status == FAILED

    @property
    def result(self):
        with self.__cond:
            self.__check_ready()
            return self.__result

    def success(self, success):
        with self.__cond:
            if self.__status == DONE:
                self.scheduler.call(success, self.__result)
            elif self.__status == WAITING:
                self.__on_success = success
            elif self.__status == CANCELLED:
                raise TaskCancelled()

    def failure(self, failure):
        with self.__cond:
            if self.__status == FAILED:
                self.scheduler.call(failure, self.__result, self.__traceback)
            elif self.__status == WAITING:
                self.__on_failure = failure
            elif self.__status == CANCELLED:
                raise TaskCancelled()

    success = property(fset=success)
    failure = property(fset=failure)

    def bind(self, success=None, failure=None):
        if success is not None:
            self.success = success
        if failure is not None:
            self.failure = failure

    def cancel(self):
        with self.__cond:
            if self.__status == WAITING:
                self.__status = CANCELLED

class CoroutineTask(Task):
    def __init__(self, scheduler, gen):
        Task.__init__(self, scheduler)
        self.__gen = gen
        self.__next(self.__gen.next)

    def __next(self, func, *args, **kwargs):
        try:
            ret = func(*args, **kwargs)
        except StopIteration:
            self.set_done(None)
        except Exception, exn:
            self.set_failed(exn, traceback.format_exc())
        else:
            if isinstance(ret, Task):
                ret.bind(self.__success_next, self.__failure_next)
            else:
                self.set_done(ret)

    def __success_next(self, result):
        self.__next(self.__gen.send, result)

    def __failure_next(self, exn, traceback):
        self.__next(self.__gen.throw, exn)

def coroutine(scheduler, func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        return CoroutineTask(scheduler, func(*args, **kwargs))
    return wrapper

def coroutine_func(func):
    @functools.wraps(func)
    def wrapper(scheduler, *args, **kwargs):
        return CoroutineTask(scheduler, func(*args, **kwargs))
    return wrapper

def coroutine_method(method):
    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        return CoroutineTask(self.scheduler, method(self, *args, **kwargs))
    return wrapper
<|endoftext|>


<commit_before>
"""
Medtronic - openaps driver for Medtronic
"""
from openaps.uses.use import Use
from openaps.uses.registry import Registry
from openaps.configurable import Configurable
import decocare
import argparse
import json
from decocare import stick, session, link, commands, history
from datetime import datetime
from dateutil import relativedelta
from dateutil.parser import parse

def configure_use_app (app, parser):
  pass
  # parser.add_argument('foobar', help="LOOK AT ME")

def configure_add_app (app, parser):
  parser.add_argument('serial')

def configure_app (app, parser):
  if app.parent.name == 'add':
    """
    print "CONFIG INNER", app, app.parent.name, app.name
    """
def configure_parser (parser):
  pass
def main (args, app):
  """
  print "MEDTRONIC", args, app
  print "app commands", app.selected.name
  """

use = Registry( )

@use( )
class scan (Use):
  """ scan for usb stick """
  def configure_app (self, app, parser):
    pass
    # print "hahaha"
  def scanner (self):
    from decocare.scan import scan
    return scan( )
  def main (self, args, app):
    return self.scanner( )

import logging
import logging.handlers
class MedtronicTask (scan):
  requires_session = True
  save_session = True
  record_stats = True

  def before_main (self, args, app):
    self.setup_medtronic( )
    if self.requires_session:
      self.check_session(app)
    else:
      self.pump.setModel(number=self.device.fields.get('model', ''))

  def after_main (self, args, app):
    if self.save_session:
      self.device.store(app.config)
      app.config.save( )
    if self.uart:
      self.uart.close( )

  def get_session_info (self):
    expires = self.device.fields.get('expires', None)
    now = datetime.now( )
    out = dict(device=self.device.name
      , vendor=__name__
      , used=now
      )
    if expires is None or parse(expires) < now:
      fields = self.create_session( )
      out.update(**self.update_session_info(fields))
    else:
      out['expires'] = parse(expires)
      out['model'] = self.get_model( )
    return out

  def update_session_info (self, fields):
    out = { }
    self.device.add_option('expires', fields['expires'].isoformat( ))
    self.device.add_option('model', fields['model'])
    out['expires'] = fields['expires']
    out['model'] = fields['model']
    return out

  def create_session (self):
    minutes = int(self.device.fields.get('minutes', 28))
    now = datetime.now( )
    self.pump.power_control(minutes=minutes)
    model = self.get_model( )
    offset = relativedelta.relativedelta(minutes=minutes) + relativedelta.relativedelta(minutes=-1)
    out = dict(device=self.device.name
      , model=model
      , vendor=__name__
      , created_at=now
      , started=now
      , expires=now + offset
      )
    return out
  def check_session (self, app):
    self.session = self.get_session_info( )
    self.device.add_option('model', self.device.fields.get('model', self.get_model( )))
  def get_model (self):
    model = self.pump.read_model( ).getData( )
    return model
  def setup_medtronic (self):
    log = logging.getLogger(decocare.__name__)
    log.setLevel(logging.INFO)
    log.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))
    self.uart = stick.Stick(link.Link(self.scanner( )))
    self.uart.open( )
    serial = self.device.fields['serial']
    self.pump = session.Pump(self.uart, serial)
    stats = self.uart.interface_stats( )
  def main (self, args, app):
    return self.scanner( )

class Session (MedtronicTask):
  """ session for pump
  """
  requires_session = False
  def configure_parser (self, parser):
    parser.add_argument('--minutes', type=int, default='10')
  def setup_application (self):
    """
    """
  def main (self, args, app):
    info = self.create_session( )
    info.update(**self.update_session_info(info))
    return info

@use( )
class model (MedtronicTask):
  """ Get model number
  """
  def configure_app (self, app, parser):
    pass
  def main (self, args, app):
    model = self.pump.read_model( ).getData( )
    return model

@use( )
class read_status (MedtronicTask):
  """ Get pump status
  """
  def main (self, args, app):
    return self.pump.model.read_status( )

@use( )
class status (read_status):
  """ Get pump status (alias for read_status)
  """

@use( )
class reservoir (MedtronicTask):
  """ Get pump remaining insulin
  """
  def main (self, args, app):
    return self.pump.model.read_reservoir( )

@use( )
class settings (MedtronicTask):
  """ Get pump settings
  """
  def main (self, args, app):
    return self.pump.model.read_settings( )

@use( )
class mytest (MedtronicTask):
  """ Testing read_settings
  """
  requires_session = False
  def main (self, args, app):
    return self.pump.model.my_read_settings( )

@use( )
class read_clock (MedtronicTask):
  """ Read date/time of pump
  """
  def main (self, args, app):
    return self.pump.model.read_clock( )

class SameNameCommand (MedtronicTask):
  def main (self, args, app):
    name = self.__class__.__name__.split('.').pop( )
    return getattr(self.pump.model, name)(**self.get_params(args))

@use( )
class read_temp_basal (SameNameCommand):
  """ Read temporary basal rates. """

@use( )
class read_settings (SameNameCommand):
  """ Read settings. """

@use( )
class read_carb_ratios (SameNameCommand):
  """ Read carb_ratios. """

@use( )
class read_basal_profile_std (SameNameCommand):
  """ Read default basal profile. """

@use( )
class read_basal_profile_A (SameNameCommand):
  """ Read basal profile A. """

@use( )
class read_basal_profile_B (SameNameCommand):
  """ Read basal profile B. """

@use( )
class read_selected_basal_profile (SameNameCommand):
  """ Fetch the currently selected basal profile. """

@use( )
class read_current_glucose_pages (SameNameCommand):
  """ Read current glucose pages. """

@use( )
class read_current_history_pages (SameNameCommand):
  """ Read current history pages. """

@use( )
class suspend_pump (SameNameCommand):
  """ Suspend pumping. """
  def main (self, args, app):
    result = super(suspend_pump, self).main(args, app)
    result.update(enacted_at=datetime.now( ))
    return result

@use( )
class resume_pump (suspend_pump):
  """ resume pumping. """

@use( )
class read_battery_status (SameNameCommand):
  """ Check battery status. """

@use( )
class read_bg_targets (SameNameCommand):
  """ Read bg targets. """

@use( )
class read_insulin_sensitivies (SameNameCommand):
  """ Read insulin sensitivies. """

@use( )
class read_glucose_data (SameNameCommand):
  """ Read pump glucose page
  """
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def get_params (self, args):
    return dict(page=int(args.page))

class InputProgramRequired (MedtronicTask):
  def upload_program (self, program):
    raise NotImplementedError( )
  def get_params (self, args):
    return dict(input=args.input)
  def configure_app (self, app, parser):
    parser.add_argument('input', default='-')
  def get_program (self, args):
    params = self.get_params(args)
    program = json.load(argparse.FileType('r')(params.get('input')))
    return program
  def main (self, args, app):
    program = self.get_program(args)
    results = self.upload_program(program)
    program.update(timestamp=datetime.now( ), **results)
    return program

@use( )
class set_temp_basal (InputProgramRequired):
  """ Set temporary basal rates.
  """
  def upload_program (self, program):
    return self.pump.model.set_temp_basal(**program)

@use( )
class bolus (InputProgramRequired):
  """ Send bolus.
  """
  def upload_program (self, program):
    return self.pump.model.bolus(**program)

@use( )
class filter_glucose_date (SameNameCommand):
  """ Search for glucose pages including begin and end dates (iso 8601).
  """
  def get_params (self, args):
    return dict(begin=args.begin, end=args.end)
  def configure_app (self, app, parser):
    parser.add_argument('begin', default='')
    parser.add_argument('end', default='')

@use( )
class filter_isig_date (filter_glucose_date):
  """ Search for isig pages including begin and end dates (iso 8601).
  """

@use( )
class read_history_data (MedtronicTask):
  """ Read pump history page
  """
  def get_params (self, args):
    return dict(page=int(args.page))
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def main (self, args, app):
    history = self.pump.model.read_history_data(**self.get_params(args))
    return history

@use( )
class iter_glucose (MedtronicTask):
  """ Read latest 100 glucose records
  """
  def get_params (self, args):
    return dict(count=int(args.count))
  def configure_app (self, app, parser):
    parser.add_argument('count', type=int, default=99)
  def range (self):
    return self.pump.model.iter_glucose_pages( )
  maxCount = 99
  def main (self, args, app):
    self.maxCount = self.get_params(args).get('count', self.maxCount)
    num = 0
    records = [ ]
    for rec in self.range( ):
      records.append(rec)
      num = num + 1
      if num > self.maxCount:
        break
    return records

@use( )
class iter_pump (iter_glucose):
  """ Read latest 100 pump records
  """
  def range (self):
    return self.pump.model.iter_history_pages( )

@use ( )
class iter_glucose_hours (MedtronicTask):
  """ Read latest n hours of glucose data
  """
  def get_params (self, args):
    params = dict(hours=float(args.hours))

    if 'now' in args and args.now is not None:
      params.update(now=args.now)

    return params

  def configure_app (self, app, parser):
    parser.add_argument('hours', type=float, help='The number of hours of historical data to retrieve')
    parser.add_argument('--now', help='A read_clock report filename from which to offset the hours')

  def range (self):
    return self.pump.model.iter_glucose_pages( )

  def get_record_timestamp (self, record):
    return parse(record['date']) if 'date' in record else None

  def main (self, args, app):
    params = self.get_params(args)
    min_offset = relativedelta.relativedelta(hours=params['hours'])
    min_timestamp = parse(json.load(argparse.FileType('r')(params['now']))) - min_offset if 'now' in params else None

    records = [ ]
    for rec in self.range( ):
      timestamp = self.get_record_timestamp(rec)

      if timestamp is not None and min_timestamp is None:
        min_timestamp = timestamp - min_offset

      if timestamp is None or timestamp >= min_timestamp:
        records.append(rec)
      else:
        break
    return records

@use ( )
class iter_pump_hours (iter_glucose_hours):
  """ Read latest n hours of pump records
  """
  def range (self):
    return self.pump.model.iter_history_pages( )

  def get_record_timestamp (self, record):
    return parse(record['timestamp']) if 'timestamp' in record else None

def set_config (args, device):
  device.add_option('serial', args.serial)

def display_device (device):
  return ''

known_uses = [
  Session,
]
def get_uses (device, config):
  all_uses = known_uses[:] + use.get_uses(device, config)
  all_uses.sort(key=lambda usage: getattr(usage, 'sortOrder', usage.__name__))
  return all_uses

<commit_msg>eliminate duplicate logging<commit_after>
"""
Medtronic - openaps driver for Medtronic
"""
from openaps.uses.use import Use
from openaps.uses.registry import Registry
from openaps.configurable import Configurable
import decocare
import argparse
import json
from decocare import stick, session, link, commands, history
from datetime import datetime
from dateutil import relativedelta
from dateutil.parser import parse

def configure_use_app (app, parser):
  pass
  # parser.add_argument('foobar', help="LOOK AT ME")

def configure_add_app (app, parser):
  parser.add_argument('serial')

def configure_app (app, parser):
  if app.parent.name == 'add':
    """
    print "CONFIG INNER", app, app.parent.name, app.name
    """
def configure_parser (parser):
  pass
def main (args, app):
  """
  print "MEDTRONIC", args, app
  print "app commands", app.selected.name
  """

use = Registry( )

@use( )
class scan (Use):
  """ scan for usb stick """
  def configure_app (self, app, parser):
    pass
    # print "hahaha"
  def scanner (self):
    from decocare.scan import scan
    return scan( )
  def main (self, args, app):
    return self.scanner( )

import logging
import logging.handlers
class MedtronicTask (scan):
  requires_session = True
  save_session = True
  record_stats = True

  def before_main (self, args, app):
    self.setup_medtronic( )
    if self.requires_session:
      self.check_session(app)
    else:
      self.pump.setModel(number=self.device.fields.get('model', ''))

  def after_main (self, args, app):
    if self.save_session:
      self.device.store(app.config)
      app.config.save( )
    if self.uart:
      self.uart.close( )

  def get_session_info (self):
    expires = self.device.fields.get('expires', None)
    now = datetime.now( )
    out = dict(device=self.device.name
      , vendor=__name__
      , used=now
      )
    if expires is None or parse(expires) < now:
      fields = self.create_session( )
      out.update(**self.update_session_info(fields))
    else:
      out['expires'] = parse(expires)
      out['model'] = self.get_model( )
    return out

  def update_session_info (self, fields):
    out = { }
    self.device.add_option('expires', fields['expires'].isoformat( ))
    self.device.add_option('model', fields['model'])
    out['expires'] = fields['expires']
    out['model'] = fields['model']
    return out

  def create_session (self):
    minutes = int(self.device.fields.get('minutes', 3))
    now = datetime.now( )
    self.pump.power_control(minutes=minutes)
    model = self.get_model( )
    offset = relativedelta.relativedelta(minutes=minutes) + relativedelta.relativedelta(minutes=-1)
    out = dict(device=self.device.name
      , model=model
      , vendor=__name__
      , created_at=now
      , started=now
      , expires=now + offset
      )
    return out
  def check_session (self, app):
    self.session = self.get_session_info( )
    self.device.add_option('model', self.device.fields.get('model', self.get_model( )))
  def get_model (self):
    model = self.pump.read_model( ).getData( )
    return model
  def setup_medtronic (self):
    log = logging.getLogger(decocare.__name__)
    level = getattr(logging, self.device.fields.get('logLevel', 'WARN'))
    address = self.device.fields.get('logAddress', '/dev/log')
    log.setLevel(level)
    for previous in log.handlers[:]:
      log.removeHandler(previous)
    log.addHandler(logging.handlers.SysLogHandler(address=address))
    self.uart = stick.Stick(link.Link(self.scanner( )))
    self.uart.open( )
    serial = self.device.fields['serial']
    self.pump = session.Pump(self.uart, serial)
    stats = self.uart.interface_stats( )
  def main (self, args, app):
    return self.scanner( )

class Session (MedtronicTask):
  """ session for pump
  """
  requires_session = False
  def configure_parser (self, parser):
    parser.add_argument('--minutes', type=int, default='10')
  def setup_application (self):
    """
    """
  def main (self, args, app):
    info = self.create_session( )
    info.update(**self.update_session_info(info))
    return info

@use( )
class model (MedtronicTask):
  """ Get model number
  """
  def configure_app (self, app, parser):
    pass
  def main (self, args, app):
    model = self.pump.read_model( ).getData( )
    return model

@use( )
class read_status (MedtronicTask):
  """ Get pump status
  """
  def main (self, args, app):
    return self.pump.model.read_status( )

@use( )
class status (read_status):
  """ Get pump status (alias for read_status)
  """

@use( )
class reservoir (MedtronicTask):
  """ Get pump remaining insulin
  """
  def main (self, args, app):
    return self.pump.model.read_reservoir( )

@use( )
class settings (MedtronicTask):
  """ Get pump settings
  """
  def main (self, args, app):
    return self.pump.model.read_settings( )

@use( )
class mytest (MedtronicTask):
  """ Testing read_settings
  """
  requires_session = False
  def main (self, args, app):
    return self.pump.model.my_read_settings( )

@use( )
class read_clock (MedtronicTask):
  """ Read date/time of pump
  """
  def main (self, args, app):
    return self.pump.model.read_clock( )

class SameNameCommand (MedtronicTask):
  def main (self, args, app):
    name = self.__class__.__name__.split('.').pop( )
    return getattr(self.pump.model, name)(**self.get_params(args))

@use( )
class read_temp_basal (SameNameCommand):
  """ Read temporary basal rates. """

@use( )
class read_settings (SameNameCommand):
  """ Read settings. """

@use( )
class read_carb_ratios (SameNameCommand):
  """ Read carb_ratios. """

@use( )
class read_basal_profile_std (SameNameCommand):
  """ Read default basal profile. """

@use( )
class read_basal_profile_A (SameNameCommand):
  """ Read basal profile A. """

@use( )
class read_basal_profile_B (SameNameCommand):
  """ Read basal profile B. """

@use( )
class read_selected_basal_profile (SameNameCommand):
  """ Fetch the currently selected basal profile. """

@use( )
class read_current_glucose_pages (SameNameCommand):
  """ Read current glucose pages. """

@use( )
class read_current_history_pages (SameNameCommand):
  """ Read current history pages. """

@use( )
class suspend_pump (SameNameCommand):
  """ Suspend pumping. """
  def main (self, args, app):
    result = super(suspend_pump, self).main(args, app)
    result.update(enacted_at=datetime.now( ))
    return result

@use( )
class resume_pump (suspend_pump):
  """ resume pumping. """

@use( )
class read_battery_status (SameNameCommand):
  """ Check battery status. """

@use( )
class read_bg_targets (SameNameCommand):
  """ Read bg targets. """

@use( )
class read_insulin_sensitivies (SameNameCommand):
  """ Read insulin sensitivies. """

@use( )
class read_glucose_data (SameNameCommand):
  """ Read pump glucose page
  """
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def get_params (self, args):
    return dict(page=int(args.page))

class InputProgramRequired (MedtronicTask):
  def upload_program (self, program):
    raise NotImplementedError( )
  def get_params (self, args):
    return dict(input=args.input)
  def configure_app (self, app, parser):
    parser.add_argument('input', default='-')
  def get_program (self, args):
    params = self.get_params(args)
    program = json.load(argparse.FileType('r')(params.get('input')))
    return program
  def main (self, args, app):
    program = self.get_program(args)
    results = self.upload_program(program)
    program.update(timestamp=datetime.now( ), **results)
    return program

@use( )
class set_temp_basal (InputProgramRequired):
  """ Set temporary basal rates.
  """
  def upload_program (self, program):
    return self.pump.model.set_temp_basal(**program)

@use( )
class bolus (InputProgramRequired):
  """ Send bolus.
  """
  def upload_program (self, program):
    return self.pump.model.bolus(**program)

@use( )
class filter_glucose_date (SameNameCommand):
  """ Search for glucose pages including begin and end dates (iso 8601).
  """
  def get_params (self, args):
    return dict(begin=args.begin, end=args.end)
  def configure_app (self, app, parser):
    parser.add_argument('begin', default='')
    parser.add_argument('end', default='')

@use( )
class filter_isig_date (filter_glucose_date):
  """ Search for isig pages including begin and end dates (iso 8601).
  """

@use( )
class read_history_data (MedtronicTask):
  """ Read pump history page
  """
  def get_params (self, args):
    return dict(page=int(args.page))
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def main (self, args, app):
    history = self.pump.model.read_history_data(**self.get_params(args))
    return history

@use( )
class iter_glucose (MedtronicTask):
  """ Read latest 100 glucose records
  """
  def get_params (self, args):
    return dict(count=int(args.count))
  def configure_app (self, app, parser):
    parser.add_argument('count', type=int, default=99)
  def range (self):
    return self.pump.model.iter_glucose_pages( )
  maxCount = 99
  def main (self, args, app):
    self.maxCount = self.get_params(args).get('count', self.maxCount)
    num = 0
    records = [ ]
    for rec in self.range( ):
      records.append(rec)
      num = num + 1
      if num > self.maxCount:
        break
    return records

@use( )
class iter_pump (iter_glucose):
  """ Read latest 100 pump records
  """
  def range (self):
    return self.pump.model.iter_history_pages( )

@use ( )
class iter_glucose_hours (MedtronicTask):
  """ Read latest n hours of glucose data
  """
  def get_params (self, args):
    params = dict(hours=float(args.hours))

    if 'now' in args and args.now is not None:
      params.update(now=args.now)

    return params

  def configure_app (self, app, parser):
    parser.add_argument('hours', type=float, help='The number of hours of historical data to retrieve')
    parser.add_argument('--now', help='A read_clock report filename from which to offset the hours')

  def range (self):
    return self.pump.model.iter_glucose_pages( )

  def get_record_timestamp (self, record):
    return parse(record['date']) if 'date' in record else None

  def main (self, args, app):
    params = self.get_params(args)
    min_offset = relativedelta.relativedelta(hours=params['hours'])
    min_timestamp = parse(json.load(argparse.FileType('r')(params['now']))) - min_offset if 'now' in params else None

    records = [ ]
    for rec in self.range( ):
      timestamp = self.get_record_timestamp(rec)

      if timestamp is not None and min_timestamp is None:
        min_timestamp = timestamp - min_offset

      if timestamp is None or timestamp >= min_timestamp:
        records.append(rec)
      else:
        break
    return records

@use ( )
class iter_pump_hours (iter_glucose_hours):
  """ Read latest n hours of pump records
  """
  def range (self):
    return self.pump.model.iter_history_pages( )

  def get_record_timestamp (self, record):
    return parse(record['timestamp']) if 'timestamp' in record else None

def set_config (args, device):
  device.add_option('serial', args.serial)

def display_device (device):
  return ''

known_uses = [
  Session,
]
def get_uses (device, config):
  all_uses = known_uses[:] + use.get_uses(device, config)
  all_uses.sort(key=lambda usage: getattr(usage, 'sortOrder', usage.__name__))
  return all_uses

<|endoftext|>