<commit_before>#!/usr/bin/env python

"""
This module implements basic kinds of jobs for VASP runs.
"""

from __future__ import division

__author__ = "Shyue Ping Ong"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyuep@gmail.com"
__status__ = "Beta"
__date__ = "2/4/13"

import subprocess
import os
import shutil
import math
import logging

from pymatgen.io.vaspio.vasp_input import VaspInput, Incar
from pymatgen.io.vaspio.vasp_output import Outcar
from pymatgen.io.smartio import read_structure
from pymatgen.io.vaspio_set import MITVaspInputSet
from pymatgen.serializers.json_coders import PMGJSONDecoder
from monty.os.path import which

from custodian.ansible.interpreter import Modder
from custodian.ansible.actions import FileActions, DictActions
from custodian.custodian import Job, gzip_dir

VASP_INPUT_FILES = {"INCAR", "POSCAR", "POTCAR", "KPOINTS"}

VASP_OUTPUT_FILES = ['DOSCAR', 'INCAR', 'KPOINTS', 'POSCAR', 'PROCAR',
                     'vasprun.xml', 'CHGCAR', 'CHG', 'EIGENVAL', 'OSZICAR',
                     'WAVECAR', 'CONTCAR', 'IBZKPT', 'OUTCAR']

class VaspJob(Job):
    """
    A basic vasp job. Just runs whatever is in the directory. But conceivably
     can be a complex processing of inputs etc. with initialization.
    """

    def __init__(self, vasp_cmd, output_file="vasp.out", suffix="",
                 final=True, gzipped=False, backup=True,
                 default_vasp_input_set=MITVaspInputSet(), auto_npar=True,
                 auto_gamma=True, settings_override=None,
                 gamma_vasp_cmd=None, copy_magmom=False):
        """
        This constructor is necessarily complex due to the need for
        flexibility. For standard kinds of runs, it's often better to use one
        of the static constructors. The defaults are usually fine too.

        Args:
            vasp_cmd (str): Command to run vasp as a list of args. For example,
                if you are using mpirun, it can be something like
                ["mpirun", "pvasp.5.2.11"]
            output_file (str): Name of file to direct standard out to.
                Defaults to "vasp.out".
            suffix (str): A suffix to be appended to the final output. E.g.,
                to rename all VASP output from say vasp.out to
                vasp.out.relax1, provide ".relax1" as the suffix.
            final (bool): Indicating whether this is the final vasp job in a
                series. Defaults to True.
            backup (bool): Whether to backup the initial input files. If True,
                the INCAR, KPOINTS, POSCAR and POTCAR will be copied with a
                ".orig" appended. Defaults to True.
            gzipped (bool): Deprecated. Please use the Custodian class's
                gzipped_output option instead.
            default_vasp_input_set (VaspInputSet): Species the default input
                set (see pymatgen's documentation in pymatgen.io.vaspio_set to
                use for directories that do not contain full set of VASP
                input files. For example, if a directory contains only a
                POSCAR or a cif, the vasp input set will be used to generate
                the necessary input files for the run. If the directory already
                contain a full set of VASP input files,
                this input is ignored. Defaults to the MITVaspInputSet.
            auto_npar (bool): Whether to automatically tune NPAR to be sqrt(
                number of cores) as recommended by VASP for DFT calculations.
                Generally, this results in significant speedups. Defaults to
                True. Set to False for HF, GW and RPA calculations.
            auto_gamma (bool): Whether to automatically check if run is a
                Gamma 1x1x1 run, and whether a Gamma optimized version of
                VASP exists with ".gamma" appended to the name of the VASP
                executable (typical setup in many systems). If so, run the
                gamma optimized version of VASP instead of regular VASP. You
                can also specify the gamma vasp command using the
                gamma_vasp_cmd argument if the command is named differently.
            settings_override ([dict]): An ansible style list of dict to
                override changes. For example, to set ISTART=1 for subsequent
                runs and to copy the CONTCAR to the POSCAR, you will provide::

                    [{"dict": "INCAR", "action": {"_set": {"ISTART": 1}}},
                     {"filename": "CONTCAR",
                      "action": {"_file_copy": {"dest": "POSCAR"}}}]
            gamma_vasp_cmd (str): Command for gamma vasp version when
                auto_gamma is True. Should follow the list style of
                subprocess. Defaults to None, which means ".gamma" is added
                to the last argument of the standard vasp_cmd.
            copy_magmom (bool): Whether to copy the final magmom from the
                OUTCAR to the next INCAR. Useful for multi-relaxation runs
                where the CHGCAR and WAVECAR are sometimes deleted (due to
                changes in fft grid, etc.). Only applies to non-final runs.
        """
        self.vasp_cmd = vasp_cmd
        self.output_file = output_file
        self.final = final
        self.backup = backup
        self.gzipped = gzipped
        self.default_vis = default_vasp_input_set
        self.suffix = suffix
        self.settings_override = settings_override
        self.auto_npar = auto_npar
        self.auto_gamma = auto_gamma
        self.gamma_vasp_cmd = gamma_vasp_cmd
        self.copy_magmom = copy_magmom

    def setup(self):
        """
        Performs initial setup for VaspJob, including overriding any settings
        and backing up.
        """
        files = os.listdir(".")
        num_structures = 0
        if not set(files).issuperset(VASP_INPUT_FILES):
            for f in files:
                try:
                    struct = read_structure(f)
                    num_structures += 1
                except:
                    pass
            if num_structures != 1:
                raise RuntimeError("{} structures found. Unable to continue."
                                   .format(num_structures))
            else:
                self.default_vis.write_input(struct, ".")

        if self.backup:
            for f in VASP_INPUT_FILES:
                shutil.copy(f, "{}.orig".format(f))

        if self.auto_npar:
            try:
                vi = VaspInput.from_directory(".")
                incar = vi["INCAR"]
                #Only optimized NPAR for non-HF and non-RPA calculations.
                if not (incar.get("LHFCALC") or incar.get("LRPA") or
                        incar.get("LEPSILON")):
                    if incar.get("IBRION") in [5, 6, 7, 8]:
                        # NPAR should not be set for Hessian matrix
                        # calculations, whether in DFPT or otherwise.
                        del incar["NPAR"]
                    else:
                        import multiprocessing
                        ncores = multiprocessing.cpu_count()
                        for npar in range(int(round(math.sqrt(ncores))),
                                          ncores):
                            if ncores % npar == 0:
                                incar["NPAR"] = npar
                                break
                    incar.write_file("INCAR")
            except:
                pass

        if self.settings_override is not None:
            vi = VaspInput.from_directory(".")
            m = Modder([FileActions, DictActions])
            modified = []
            for a in self.settings_override:
                if "dict" in a:
                    modified.append(a["dict"])
                    vi[a["dict"]] = m.modify_object(a["action"],
                                                    vi[a["dict"]])
                elif "filename" in a:
                    m.modify(a["action"], a["filename"])
            for f in modified:
                vi[f].write_file(f)

    def run(self):
        """
        Perform the actual VASP run.

        Returns:
            (subprocess.Popen) Used for monitoring.
        """
        cmd = list(self.vasp_cmd)
        if self.auto_gamma:
            vi = VaspInput.from_directory(".")
            kpts = vi["KPOINTS"]
            if kpts.style == "Gamma" and tuple(kpts.kpts[0]) == (1, 1, 1):
                if self.gamma_vasp_cmd is not None and which(
                        self.gamma_vasp_cmd[-1]):
                    cmd = self.gamma_vasp_cmd
                elif which(cmd[-1] + ".gamma"):
                    cmd[-1] += ".gamma"
        logging.info("Running {}".format(" ".join(cmd)))
        with open(self.output_file, 'w') as f:
            p = subprocess.Popen(cmd, stdout=f)
        return p

    def postprocess(self):
        """
        Postprocessing includes renaming and gzipping where necessary.
        Also copies the magmom to the incar if necessary
        """
        for f in VASP_OUTPUT_FILES + [self.output_file]:
            if os.path.exists(f):
                if self.final and self.suffix != "":
                    shutil.move(f, "{}{}".format(f, self.suffix))
                elif self.suffix != "":
                    shutil.copy(f, "{}{}".format(f, self.suffix))

        if self.copy_magmom and not self.final:
            outcar = Outcar("OUTCAR")
            magmom = [m['tot'] for m in outcar.magnetization]
            incar = Incar.from_file("INCAR")
            incar['MAGMOM'] = magmom
            incar.write_file("INCAR")

        if self.gzipped:
            gzip_dir(".")

    @staticmethod
    def double_relaxation_run(vasp_cmd, gzipped=True):
        """
        Returns a list of two jobs corresponding to an AFLOW style double
        relaxation run.

        Args:
            vasp_cmd (str): Command to run vasp as a list of args. For example,
                if you are using mpirun, it can be something like
                ["mpirun", "pvasp.5.2.11"]

        Returns:
            List of two jobs corresponding to an AFLOW style run.
        """
        return [VaspJob(vasp_cmd, final=False, suffix=".relax1"),
                VaspJob(
                    vasp_cmd, final=True, backup=False,
                    suffix=".relax2", gzipped=gzipped,
                    settings_override=[
                        {"dict": "INCAR",
                         "action": {"_set": {"ISTART": 1}}},
                        {"filename": "CONTCAR",
                         "action": {"_file_copy": {"dest": "POSCAR"}}}])]

    @property
    def to_dict(self):
        d = dict(vasp_cmd=self.vasp_cmd,
                 output_file=self.output_file, suffix=self.suffix,
                 final=self.final, gzipped=self.gzipped, backup=self.backup,
                 default_vasp_input_set=self.default_vis.to_dict,
                 auto_npar=self.auto_npar, auto_gamma=self.auto_gamma,
                 settings_override=self.settings_override,
                 gamma_vasp_cmd=self.gamma_vasp_cmd
                 )
        d["@module"] = self.__class__.__module__
        d["@class"] = self.__class__.__name__
        return d

    @classmethod
    def from_dict(cls, d):
        vis = PMGJSONDecoder().process_decoded(d["default_vasp_input_set"])
        return VaspJob(
            vasp_cmd=d["vasp_cmd"], output_file=d["output_file"],
            suffix=d["suffix"], final=d["final"], gzipped=d["gzipped"],
            backup=d["backup"], default_vasp_input_set=vis,
            auto_npar=d['auto_npar'], auto_gamma=d['auto_gamma'],
            settings_override=d["settings_override"],
            gamma_vasp_cmd=d["gamma_vasp_cmd"])
<commit_msg>added try-catch for copy_magmom<commit_after>#!/usr/bin/env python

"""
This module implements basic kinds of jobs for VASP runs.
"""

from __future__ import division

__author__ = "Shyue Ping Ong"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyuep@gmail.com"
__status__ = "Beta"
__date__ = "2/4/13"

import subprocess
import os
import shutil
import math
import logging

from pymatgen.io.vaspio.vasp_input import VaspInput, Incar
from pymatgen.io.vaspio.vasp_output import Outcar
from pymatgen.io.smartio import read_structure
from pymatgen.io.vaspio_set import MITVaspInputSet
from pymatgen.serializers.json_coders import PMGJSONDecoder
from monty.os.path import which

from custodian.ansible.interpreter import Modder
from custodian.ansible.actions import FileActions, DictActions
from custodian.custodian import Job, gzip_dir

VASP_INPUT_FILES = {"INCAR", "POSCAR", "POTCAR", "KPOINTS"}

VASP_OUTPUT_FILES = ['DOSCAR', 'INCAR', 'KPOINTS', 'POSCAR', 'PROCAR',
                     'vasprun.xml', 'CHGCAR', 'CHG', 'EIGENVAL', 'OSZICAR',
                     'WAVECAR', 'CONTCAR', 'IBZKPT', 'OUTCAR']

class VaspJob(Job):
    """
    A basic vasp job. Just runs whatever is in the directory. But conceivably
     can be a complex processing of inputs etc. with initialization.
    """

    def __init__(self, vasp_cmd, output_file="vasp.out", suffix="",
                 final=True, gzipped=False, backup=True,
                 default_vasp_input_set=MITVaspInputSet(), auto_npar=True,
                 auto_gamma=True, settings_override=None,
                 gamma_vasp_cmd=None, copy_magmom=False):
        """
        This constructor is necessarily complex due to the need for
        flexibility. For standard kinds of runs, it's often better to use one
        of the static constructors. The defaults are usually fine too.

        Args:
            vasp_cmd (str): Command to run vasp as a list of args. For example,
                if you are using mpirun, it can be something like
                ["mpirun", "pvasp.5.2.11"]
            output_file (str): Name of file to direct standard out to.
                Defaults to "vasp.out".
            suffix (str): A suffix to be appended to the final output. E.g.,
                to rename all VASP output from say vasp.out to
                vasp.out.relax1, provide ".relax1" as the suffix.
            final (bool): Indicating whether this is the final vasp job in a
                series. Defaults to True.
            backup (bool): Whether to backup the initial input files. If True,
                the INCAR, KPOINTS, POSCAR and POTCAR will be copied with a
                ".orig" appended. Defaults to True.
            gzipped (bool): Deprecated. Please use the Custodian class's
                gzipped_output option instead.
            default_vasp_input_set (VaspInputSet): Species the default input
                set (see pymatgen's documentation in pymatgen.io.vaspio_set to
                use for directories that do not contain full set of VASP
                input files. For example, if a directory contains only a
                POSCAR or a cif, the vasp input set will be used to generate
                the necessary input files for the run. If the directory already
                contain a full set of VASP input files,
                this input is ignored. Defaults to the MITVaspInputSet.
            auto_npar (bool): Whether to automatically tune NPAR to be sqrt(
                number of cores) as recommended by VASP for DFT calculations.
                Generally, this results in significant speedups. Defaults to
                True. Set to False for HF, GW and RPA calculations.
            auto_gamma (bool): Whether to automatically check if run is a
                Gamma 1x1x1 run, and whether a Gamma optimized version of
                VASP exists with ".gamma" appended to the name of the VASP
                executable (typical setup in many systems). If so, run the
                gamma optimized version of VASP instead of regular VASP. You
                can also specify the gamma vasp command using the
                gamma_vasp_cmd argument if the command is named differently.
            settings_override ([dict]): An ansible style list of dict to
                override changes. For example, to set ISTART=1 for subsequent
                runs and to copy the CONTCAR to the POSCAR, you will provide::

                    [{"dict": "INCAR", "action": {"_set": {"ISTART": 1}}},
                     {"filename": "CONTCAR",
                      "action": {"_file_copy": {"dest": "POSCAR"}}}]
            gamma_vasp_cmd (str): Command for gamma vasp version when
                auto_gamma is True. Should follow the list style of
                subprocess. Defaults to None, which means ".gamma" is added
                to the last argument of the standard vasp_cmd.
            copy_magmom (bool): Whether to copy the final magmom from the
                OUTCAR to the next INCAR. Useful for multi-relaxation runs
                where the CHGCAR and WAVECAR are sometimes deleted (due to
                changes in fft grid, etc.). Only applies to non-final runs.
        """
        self.vasp_cmd = vasp_cmd
        self.output_file = output_file
        self.final = final
        self.backup = backup
        self.gzipped = gzipped
        self.default_vis = default_vasp_input_set
        self.suffix = suffix
        self.settings_override = settings_override
        self.auto_npar = auto_npar
        self.auto_gamma = auto_gamma
        self.gamma_vasp_cmd = gamma_vasp_cmd
        self.copy_magmom = copy_magmom

    def setup(self):
        """
        Performs initial setup for VaspJob, including overriding any settings
        and backing up.
        """
        files = os.listdir(".")
        num_structures = 0
        if not set(files).issuperset(VASP_INPUT_FILES):
            for f in files:
                try:
                    struct = read_structure(f)
                    num_structures += 1
                except:
                    pass
            if num_structures != 1:
                raise RuntimeError("{} structures found. Unable to continue."
                                   .format(num_structures))
            else:
                self.default_vis.write_input(struct, ".")

        if self.backup:
            for f in VASP_INPUT_FILES:
                shutil.copy(f, "{}.orig".format(f))

        if self.auto_npar:
            try:
                vi = VaspInput.from_directory(".")
                incar = vi["INCAR"]
                #Only optimized NPAR for non-HF and non-RPA calculations.
                if not (incar.get("LHFCALC") or incar.get("LRPA") or
                        incar.get("LEPSILON")):
                    if incar.get("IBRION") in [5, 6, 7, 8]:
                        # NPAR should not be set for Hessian matrix
                        # calculations, whether in DFPT or otherwise.
                        del incar["NPAR"]
                    else:
                        import multiprocessing
                        ncores = multiprocessing.cpu_count()
                        for npar in range(int(round(math.sqrt(ncores))),
                                          ncores):
                            if ncores % npar == 0:
                                incar["NPAR"] = npar
                                break
                    incar.write_file("INCAR")
            except:
                pass

        if self.settings_override is not None:
            vi = VaspInput.from_directory(".")
            m = Modder([FileActions, DictActions])
            modified = []
            for a in self.settings_override:
                if "dict" in a:
                    modified.append(a["dict"])
                    vi[a["dict"]] = m.modify_object(a["action"],
                                                    vi[a["dict"]])
                elif "filename" in a:
                    m.modify(a["action"], a["filename"])
            for f in modified:
                vi[f].write_file(f)

    def run(self):
        """
        Perform the actual VASP run.

        Returns:
            (subprocess.Popen) Used for monitoring.
        """
        cmd = list(self.vasp_cmd)
        if self.auto_gamma:
            vi = VaspInput.from_directory(".")
            kpts = vi["KPOINTS"]
            if kpts.style == "Gamma" and tuple(kpts.kpts[0]) == (1, 1, 1):
                if self.gamma_vasp_cmd is not None and which(
                        self.gamma_vasp_cmd[-1]):
                    cmd = self.gamma_vasp_cmd
                elif which(cmd[-1] + ".gamma"):
                    cmd[-1] += ".gamma"
        logging.info("Running {}".format(" ".join(cmd)))
        with open(self.output_file, 'w') as f:
            p = subprocess.Popen(cmd, stdout=f)
        return p

    def postprocess(self):
        """
        Postprocessing includes renaming and gzipping where necessary.
        Also copies the magmom to the incar if necessary
        """
        for f in VASP_OUTPUT_FILES + [self.output_file]:
            if os.path.exists(f):
                if self.final and self.suffix != "":
                    shutil.move(f, "{}{}".format(f, self.suffix))
                elif self.suffix != "":
                    shutil.copy(f, "{}{}".format(f, self.suffix))

        if self.copy_magmom and not self.final:
            try:
                outcar = Outcar("OUTCAR")
                magmom = [m['tot'] for m in outcar.magnetization]
                incar = Incar.from_file("INCAR")
                incar['MAGMOM'] = magmom
                incar.write_file("INCAR")
            except:
                logging.error('MAGMOM copy from OUTCAR to INCAR failed')

        if self.gzipped:
            gzip_dir(".")

    @staticmethod
    def double_relaxation_run(vasp_cmd, gzipped=True):
        """
        Returns a list of two jobs corresponding to an AFLOW style double
        relaxation run.

        Args:
            vasp_cmd (str): Command to run vasp as a list of args. For example,
                if you are using mpirun, it can be something like
                ["mpirun", "pvasp.5.2.11"]

        Returns:
            List of two jobs corresponding to an AFLOW style run.
        """
        return [VaspJob(vasp_cmd, final=False, suffix=".relax1"),
                VaspJob(
                    vasp_cmd, final=True, backup=False,
                    suffix=".relax2", gzipped=gzipped,
                    settings_override=[
                        {"dict": "INCAR",
                         "action": {"_set": {"ISTART": 1}}},
                        {"filename": "CONTCAR",
                         "action": {"_file_copy": {"dest": "POSCAR"}}}])]

    @property
    def to_dict(self):
        d = dict(vasp_cmd=self.vasp_cmd,
                 output_file=self.output_file, suffix=self.suffix,
                 final=self.final, gzipped=self.gzipped, backup=self.backup,
                 default_vasp_input_set=self.default_vis.to_dict,
                 auto_npar=self.auto_npar, auto_gamma=self.auto_gamma,
                 settings_override=self.settings_override,
                 gamma_vasp_cmd=self.gamma_vasp_cmd
                 )
        d["@module"] = self.__class__.__module__
        d["@class"] = self.__class__.__name__
        return d

    @classmethod
    def from_dict(cls, d):
        vis = PMGJSONDecoder().process_decoded(d["default_vasp_input_set"])
        return VaspJob(
            vasp_cmd=d["vasp_cmd"], output_file=d["output_file"],
            suffix=d["suffix"], final=d["final"], gzipped=d["gzipped"],
            backup=d["backup"], default_vasp_input_set=vis,
            auto_npar=d['auto_npar'], auto_gamma=d['auto_gamma'],
            settings_override=d["settings_override"],
            gamma_vasp_cmd=d["gamma_vasp_cmd"])
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function
import numbers
from datetime import date, datetime
from toolz import first

from ..compatibility import basestring
from ..expr.core import Expr
from ..expr import TableSymbol, eval_str, Union
from ..dispatch import dispatch

__all__ = ['compute', 'compute_one', 'drop', 'create_index']

base = (numbers.Real, str, date, datetime)

@dispatch(object, object)
def compute_one(a, b, **kwargs):
    raise NotImplementedError("Blaze does not know how to compute "
                              "expression of type `%s` on data of type `%s`"
                              % (type(a).__name__, type(b).__name__))

@dispatch(base)
def compute_one(a, **kwargs):
    return a

@dispatch((list, tuple))
def compute_one(seq, scope={}, **kwargs):
    return type(seq)(compute(item, scope, **kwargs) for item in seq)

@dispatch(Expr, object)
def compute(expr, o, **kwargs):
    """ Compute against single input

    Assumes that only one TableSymbol exists in expression

    >>> t = TableSymbol('t', '{name: string, balance: int}')
    >>> deadbeats = t[t['balance'] < 0]['name']

    >>> data = [['Alice', 100], ['Bob', -50], ['Charlie', -20]]
    >>> # list(compute(deadbeats, {t: data}))
    >>> list(compute(deadbeats, data))
    ['Bob', 'Charlie']
    """
    ts = set([x for x in expr.subterms() if isinstance(x, TableSymbol)])
    if len(ts) == 1:
        return compute(expr, {first(ts): o}, **kwargs)
    else:
        raise ValueError("Give compute dictionary input, got %s" % str(o))

@dispatch(object)
def compute_down(expr):
    """ Compute the expression on the entire inputs

    inputs match up to leaves of the expression
    """
    return expr

def V(d, expr):
    """ Processes an expression top-down then bottom-up """
    # Base case: expression is in dict, return associated data
    if expr in d:
        return d[expr]

    # See if we have a direct computation path
    if (hasattr(expr, 'leaves') and compute_down.resolve(
            (type(expr),) + tuple(type(d.get(leaf)) for leaf in expr.leaves()))):
        leaves = [d[leaf] for leaf in expr.leaves()]
        return compute_down(expr, *leaves)
    else:
        # Compute children of this expression
        children = ([V(d, child) for child in expr.inputs]
                    if hasattr(expr, 'inputs') else [])

        # Compute this expression given the children
        return compute_one(expr, *children, scope=d)

def bottom_up(d, expr):
    """
    Process an expression from the leaves upwards

    Parameters
    ----------

    d : dict mapping {TableSymbol: data}
        Maps expressions to data elements, likely at the leaves of the tree
    expr : Expr
        Expression to compute

    Helper function for ``compute``
    """
    # Base case: expression is in dict, return associated data
    if expr in d:
        return d[expr]

    # Compute children of this expression
    children = ([bottom_up(d, child) for child in expr.inputs]
                if hasattr(expr, 'inputs') else [])

    # Compute this expression given the children
    result = compute_one(expr, *children, scope=d)

    return result

@dispatch(Expr, dict)
def pre_compute(expr, d):
    """ Transform expr prior to calling ``compute`` """
    return expr

@dispatch(Expr, object, dict)
def post_compute(expr, result, d):
    """ Effects after the computation is complete """
    return result

@dispatch(Expr, dict)
def compute(expr, d):
    """ Compute expression against data sources

    >>> t = TableSymbol('t', '{name: string, balance: int}')
    >>> deadbeats = t[t['balance'] < 0]['name']

    >>> data = [['Alice', 100], ['Bob', -50], ['Charlie', -20]]
    >>> list(compute(deadbeats, {t: data}))
    ['Bob', 'Charlie']
    """
    expr = pre_compute(expr, d)
    result = V(d, expr)
    return post_compute(expr, result, d)

def columnwise_funcstr(t, variadic=True, full=False):
    """Build a string that can be eval'd to return a ``lambda`` expression.

    Parameters
    ----------
    t : ColumnWise
        An expression whose leaves (at each application of the returned
        expression) are all instances of ``ScalarExpression``.
        For example ::

            t.petal_length / max(t.petal_length)

        is **not** a valid ``ColumnWise``, since the expression ::

            max(t.petal_length)

        has a leaf ``t`` that is not a ``ScalarExpression``. A example of a
        valid ``ColumnWise`` expression is ::

            t.petal_length / 4

    Returns
    -------
    f : str
        A string that can be passed to ``eval`` and will return a function that
        operates on each row and applies a scalar expression to a subset of the
        columns in each row.

    Examples
    --------
    >>> t = TableSymbol('t', '{x: real, y: real, z: real}')
    >>> cw = t['x'] + t['z']
    >>> columnwise_funcstr(cw)
    'lambda x, z: x + z'

    >>> columnwise_funcstr(cw, variadic=False)
    'lambda (x, z): x + z'

    >>> columnwise_funcstr(cw, variadic=False, full=True)
    'lambda (x, y, z): x + z'
    """
    if full:
        columns = t.child.columns
    else:
        columns = t.active_columns()
    if variadic:
        prefix = 'lambda %s: '
    else:
        prefix = 'lambda (%s): '

    return prefix % ', '.join(map(str, columns)) + eval_str(t.expr)

@dispatch(Union, (list, tuple))
def compute_one(t, children, **kwargs):
    return compute_one(t, children[0], tuple(children))

@dispatch(object, basestring, (basestring, list, tuple))
def create_index(t, index_name, column_name_or_names):
    """Create an index on a column.

    Parameters
    ----------
    o : table-like
    index_name : str
        The name of the index to create
    column_name_or_names : basestring, list, tuple
        A column name to index on, or a list or tuple for a composite index

    Examples
    --------
    >>> # Using SQLite
    >>> from blaze import SQL
    >>> # create a table called 'tb', in memory
    >>> sql = SQL('sqlite:///:memory:', 'tb',
    ...           schema='{id: int64, value: float64, categ: string}')
    >>> data = [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
    >>> sql.extend(data)
    >>> # create an index on the 'id' column (for SQL we must provide a name)
    >>> sql.table.indexes
    set()
    >>> create_index(sql, 'id', name='id_index')
    >>> sql.table.indexes
    {Index('id_index', Column('id', BigInteger(), table=<tb>, nullable=False))}
    """
    raise NotImplementedError("create_index not implemented for type %r" %
                              type(t).__name__)

@dispatch(object)
def drop(rsrc):
    """Remove a resource.

    Parameters
    ----------
    rsrc : CSV, SQL, tables.Table, pymongo.Collection
        A resource that will be removed. For example, calling ``drop(csv)`` will
        delete the CSV file.

    Examples
    --------
    >>> # Using SQLite
    >>> from blaze import SQL, into
    >>> # create a table called 'tb', in memory
    >>> sql = SQL('sqlite:///:memory:', 'tb',
    ...           schema='{id: int64, value: float64, categ: string}')
    >>> data = [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
    >>> sql.extend(data)
    >>> into(list, sql)
    [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
    >>> sql.table.exists(sql.engine)
    True
    >>> drop(sql)
    >>> sql.table.exists(sql.engine)
    False
    """
    raise NotImplementedError("drop not implemented for type %r" %
                              type(rsrc).__name__)
<commit_msg>rename V to top-to-bottom<commit_after>from __future__ import absolute_import, division, print_function
import numbers
from datetime import date, datetime
from toolz import first

from ..compatibility import basestring
from ..expr.core import Expr
from ..expr import TableSymbol, eval_str, Union
from ..dispatch import dispatch

__all__ = ['compute', 'compute_one', 'drop', 'create_index']

base = (numbers.Real, str, date, datetime)

@dispatch(object, object)
def compute_one(a, b, **kwargs):
    raise NotImplementedError("Blaze does not know how to compute "
                              "expression of type `%s` on data of type `%s`"
                              % (type(a).__name__, type(b).__name__))

@dispatch(base)
def compute_one(a, **kwargs):
    return a

@dispatch((list, tuple))
def compute_one(seq, scope={}, **kwargs):
    return type(seq)(compute(item, scope, **kwargs) for item in seq)

@dispatch(Expr, object)
def compute(expr, o, **kwargs):
    """ Compute against single input

    Assumes that only one TableSymbol exists in expression

    >>> t = TableSymbol('t', '{name: string, balance: int}')
    >>> deadbeats = t[t['balance'] < 0]['name']

    >>> data = [['Alice', 100], ['Bob', -50], ['Charlie', -20]]
    >>> # list(compute(deadbeats, {t: data}))
    >>> list(compute(deadbeats, data))
    ['Bob', 'Charlie']
    """
    ts = set([x for x in expr.subterms() if isinstance(x, TableSymbol)])
    if len(ts) == 1:
        return compute(expr, {first(ts): o}, **kwargs)
    else:
        raise ValueError("Give compute dictionary input, got %s" % str(o))

@dispatch(object)
def compute_down(expr):
    """ Compute the expression on the entire inputs

    inputs match up to leaves of the expression
    """
    return expr

def top_to_bottom(d, expr):
    """ Processes an expression top-down then bottom-up """
    # Base case: expression is in dict, return associated data
    if expr in d:
        return d[expr]

    # See if we have a direct computation path
    if (hasattr(expr, 'leaves') and compute_down.resolve(
            (type(expr),) + tuple(type(d.get(leaf)) for leaf in expr.leaves()))):
        leaves = [d[leaf] for leaf in expr.leaves()]
        return compute_down(expr, *leaves)
    else:
        # Compute children of this expression
        children = ([top_to_bottom(d, child) for child in expr.inputs]
                    if hasattr(expr, 'inputs') else [])

        # Compute this expression given the children
        return compute_one(expr, *children, scope=d)

def bottom_up(d, expr):
    """
    Process an expression from the leaves upwards

    Parameters
    ----------

    d : dict mapping {TableSymbol: data}
        Maps expressions to data elements, likely at the leaves of the tree
    expr : Expr
        Expression to compute

    Helper function for ``compute``
    """
    # Base case: expression is in dict, return associated data
    if expr in d:
        return d[expr]

    # Compute children of this expression
    children = ([bottom_up(d, child) for child in expr.inputs]
                if hasattr(expr, 'inputs') else [])

    # Compute this expression given the children
    result = compute_one(expr, *children, scope=d)

    return result

@dispatch(Expr, dict)
def pre_compute(expr, d):
    """ Transform expr prior to calling ``compute`` """
    return expr

@dispatch(Expr, object, dict)
def post_compute(expr, result, d):
    """ Effects after the computation is complete """
    return result

@dispatch(Expr, dict)
def compute(expr, d):
    """ Compute expression against data sources

    >>> t = TableSymbol('t', '{name: string, balance: int}')
    >>> deadbeats = t[t['balance'] < 0]['name']

    >>> data = [['Alice', 100], ['Bob', -50], ['Charlie', -20]]
    >>> list(compute(deadbeats, {t: data}))
    ['Bob', 'Charlie']
    """
    expr = pre_compute(expr, d)
    result = top_to_bottom(d, expr)
    return post_compute(expr, result, d)

def columnwise_funcstr(t, variadic=True, full=False):
    """Build a string that can be eval'd to return a ``lambda`` expression.

    Parameters
    ----------
    t : ColumnWise
        An expression whose leaves (at each application of the returned
        expression) are all instances of ``ScalarExpression``.
        For example ::

            t.petal_length / max(t.petal_length)

        is **not** a valid ``ColumnWise``, since the expression ::

            max(t.petal_length)

        has a leaf ``t`` that is not a ``ScalarExpression``. A example of a
        valid ``ColumnWise`` expression is ::

            t.petal_length / 4

    Returns
    -------
    f : str
        A string that can be passed to ``eval`` and will return a function that
        operates on each row and applies a scalar expression to a subset of the
        columns in each row.

    Examples
    --------
    >>> t = TableSymbol('t', '{x: real, y: real, z: real}')
    >>> cw = t['x'] + t['z']
    >>> columnwise_funcstr(cw)
    'lambda x, z: x + z'

    >>> columnwise_funcstr(cw, variadic=False)
    'lambda (x, z): x + z'

    >>> columnwise_funcstr(cw, variadic=False, full=True)
    'lambda (x, y, z): x + z'
    """
    if full:
        columns = t.child.columns
    else:
        columns = t.active_columns()
    if variadic:
        prefix = 'lambda %s: '
    else:
        prefix = 'lambda (%s): '

    return prefix % ', '.join(map(str, columns)) + eval_str(t.expr)

@dispatch(Union, (list, tuple))
def compute_one(t, children, **kwargs):
    return compute_one(t, children[0], tuple(children))

@dispatch(object, basestring, (basestring, list, tuple))
def create_index(t, index_name, column_name_or_names):
    """Create an index on a column.

    Parameters
    ----------
    o : table-like
    index_name : str
        The name of the index to create
    column_name_or_names : basestring, list, tuple
        A column name to index on, or a list or tuple for a composite index

    Examples
    --------
    >>> # Using SQLite
    >>> from blaze import SQL
    >>> # create a table called 'tb', in memory
    >>> sql = SQL('sqlite:///:memory:', 'tb',
    ...           schema='{id: int64, value: float64, categ: string}')
    >>> data = [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
    >>> sql.extend(data)
    >>> # create an index on the 'id' column (for SQL we must provide a name)
    >>> sql.table.indexes
    set()
    >>> create_index(sql, 'id', name='id_index')
    >>> sql.table.indexes
    {Index('id_index', Column('id', BigInteger(), table=<tb>, nullable=False))}
    """
    raise NotImplementedError("create_index not implemented for type %r" %
                              type(t).__name__)

@dispatch(object)
def drop(rsrc):
    """Remove a resource.

    Parameters
    ----------
    rsrc : CSV, SQL, tables.Table, pymongo.Collection
        A resource that will be removed. For example, calling ``drop(csv)`` will
        delete the CSV file.

    Examples
    --------
    >>> # Using SQLite
    >>> from blaze import SQL, into
    >>> # create a table called 'tb', in memory
    >>> sql = SQL('sqlite:///:memory:', 'tb',
    ...           schema='{id: int64, value: float64, categ: string}')
    >>> data = [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
    >>> sql.extend(data)
    >>> into(list, sql)
    [(1, 2.0, 'a'), (2, 3.0, 'b'), (3, 4.0, 'c')]
    >>> sql.table.exists(sql.engine)
    True
    >>> drop(sql)
    >>> sql.table.exists(sql.engine)
    False
    """
    raise NotImplementedError("drop not implemented for type %r" %
                              type(rsrc).__name__)
<|endoftext|>


<commit_before>import datetime
from casexml.apps.case.models import CommCareCase
from custom.bihar.calculations import homevisit, pregnancy, postpartum, newborn, familyplanning, mortality
import fluff

A_DAY = datetime.timedelta(days=1)

class CareBiharFluff(fluff.IndicatorDocument):
    document_class = CommCareCase

    domains = ('care-bihar',)
    group_by = ['domain', 'owner_id']

    # home visit

    bp2 = homevisit.BPCalculator(days=75, n_visits=2)
    bp3 = homevisit.BPCalculator(days=45, n_visits=3)
    pnc = homevisit.VisitCalculator(schedule=(1, 3, 6), visit_type='pnc')
    ebf = homevisit.VisitCalculator(
        schedule=(14, 28, 60, 90, 120, 150),
        visit_type='eb',
    )
    cf = homevisit.VisitCalculator(
        schedule=[m * 30 for m in (6, 7, 8, 9, 12, 15, 18)],
        visit_type='cf',
    )

    upcoming_deliveries = homevisit.DueNextMonth()
    deliveries = homevisit.RecentDeliveryList()
    new_pregnancies = homevisit.RecentlyOpened()
    no_bp_counseling = homevisit.NoBPList()
    no_ifa_tablets = homevisit.NoIFAList()
    no_emergency_prep = homevisit.NoEmergencyPrep()
    no_newborn_prep = homevisit.NoNewbornPrep()
    no_postpartum_counseling = homevisit.NoPostpartumCounseling()
    no_family_planning = homevisit.NoFamilyPlanning()

    # pregnancy

    hd = pregnancy.VisitedQuicklyBirthPlace(at='home')
    idv = pregnancy.VisitedQuicklyBirthPlace(at=('private', 'public'))
    idnb = pregnancy.BreastFedBirthPlace(at=('private', 'public'))
    born_at_home = pregnancy.LiveBirthPlace(at='home')
    born_at_public_hospital = pregnancy.LiveBirthPlace(at='public')
    born_in_transit = pregnancy.LiveBirthPlace(at='transit')
    born_in_private_hospital = pregnancy.LiveBirthPlace(at='private')

    # postpartum

    comp1 = postpartum.Complications(days=1)
    comp3 = postpartum.Complications(days=3)
    comp5 = postpartum.Complications(days=5)
    comp7 = postpartum.Complications(days=7)

    # newborn

    ptlb = newborn.PretermNewborn()
    lt2kglb = newborn.LessThan2Kilos()
    visited_weak_ones = newborn.VisitedWeakNewborn()
    skin_to_skin = newborn.NoSkinToSkin()
    feed_vigour = newborn.NotBreastfeedingVigorously()

    # family planning

    interested_in_fp = familyplanning.FamilyPlanning()
    adopted_fp = familyplanning.AdoptedFP()
    exp_int_fp = familyplanning.InterestInFP()
    no_fp = familyplanning.NoFP()
    pregnant_fp = familyplanning.PregnantInterestInFP()

    # mortality

    mother_mortality = mortality.MMCalculator()
    infant_mortality = mortality.IMCalculator()
    still_birth_public = mortality.StillBirthPlace(at='public')
    still_birth_home = mortality.StillBirthPlace(at='home')
    live_birth = mortality.LiveBirth()

    class Meta:
        app_label = 'bihar'

CareBiharFluffPillow = CareBiharFluff.pillow()<commit_msg>pillow for dumping some form data to sql; useful for validation<commit_after>import datetime
from custom.bihar import getters
from pillowtop.listener import BasicPillow
from casexml.apps.case.models import CommCareCase
from couchforms.models import XFormInstance
from custom.bihar.calculations import homevisit, pregnancy, postpartum, newborn, familyplanning, mortality
from custom.bihar.calculations.utils.visits import VISIT_TYPES
import fluff
from django.db import models

A_DAY = datetime.timedelta(days=1)

class CareBiharFluff(fluff.IndicatorDocument):
    document_class = CommCareCase

    domains = ('care-bihar',)
    group_by = ['domain', 'owner_id']

    # home visit

    bp2 = homevisit.BPCalculator(days=75, n_visits=2)
    bp3 = homevisit.BPCalculator(days=45, n_visits=3)
    pnc = homevisit.VisitCalculator(schedule=(1, 3, 6), visit_type='pnc')
    ebf = homevisit.VisitCalculator(
        schedule=(14, 28, 60, 90, 120, 150),
        visit_type='eb',
    )
    cf = homevisit.VisitCalculator(
        schedule=[m * 30 for m in (6, 7, 8, 9, 12, 15, 18)],
        visit_type='cf',
    )

    upcoming_deliveries = homevisit.DueNextMonth()
    deliveries = homevisit.RecentDeliveryList()
    new_pregnancies = homevisit.RecentlyOpened()
    no_bp_counseling = homevisit.NoBPList()
    no_ifa_tablets = homevisit.NoIFAList()
    no_emergency_prep = homevisit.NoEmergencyPrep()
    no_newborn_prep = homevisit.NoNewbornPrep()
    no_postpartum_counseling = homevisit.NoPostpartumCounseling()
    no_family_planning = homevisit.NoFamilyPlanning()

    # pregnancy

    hd = pregnancy.VisitedQuicklyBirthPlace(at='home')
    idv = pregnancy.VisitedQuicklyBirthPlace(at=('private', 'public'))
    idnb = pregnancy.BreastFedBirthPlace(at=('private', 'public'))
    born_at_home = pregnancy.LiveBirthPlace(at='home')
    born_at_public_hospital = pregnancy.LiveBirthPlace(at='public')
    born_in_transit = pregnancy.LiveBirthPlace(at='transit')
    born_in_private_hospital = pregnancy.LiveBirthPlace(at='private')

    # postpartum

    comp1 = postpartum.Complications(days=1)
    comp3 = postpartum.Complications(days=3)
    comp5 = postpartum.Complications(days=5)
    comp7 = postpartum.Complications(days=7)

    # newborn

    ptlb = newborn.PretermNewborn()
    lt2kglb = newborn.LessThan2Kilos()
    visited_weak_ones = newborn.VisitedWeakNewborn()
    skin_to_skin = newborn.NoSkinToSkin()
    feed_vigour = newborn.NotBreastfeedingVigorously()

    # family planning

    interested_in_fp = familyplanning.FamilyPlanning()
    adopted_fp = familyplanning.AdoptedFP()
    exp_int_fp = familyplanning.InterestInFP()
    no_fp = familyplanning.NoFP()
    pregnant_fp = familyplanning.PregnantInterestInFP()

    # mortality

    mother_mortality = mortality.MMCalculator()
    infant_mortality = mortality.IMCalculator()
    still_birth_public = mortality.StillBirthPlace(at='public')
    still_birth_home = mortality.StillBirthPlace(at='home')
    live_birth = mortality.LiveBirth()

    class Meta:
        app_label = 'bihar'

CareBiharFluffPillow = CareBiharFluff.pillow()

class BirthPreparednessForm(models.Model):
    """Testing only"""
    _id = models.CharField(primary_key=True, max_length=40)

    date_modified = models.DateField()
    date_next_bp = models.DateField(null=True)
    days_visit_overdue = models.IntegerField(null=True)

    @classmethod
    def from_couch(cls, form):
        assert form.xmlns == VISIT_TYPES['bp']

        return cls(
            _id=form._id,
            date_modified=getters.date_modified(form),
            date_next_bp=getters.date_next_bp(form),
            days_visit_overdue=getters.days_visit_overdue(form),
        )

class CareBiharFormPillow(BasicPillow):
    """
    Run this as a way to validate/debug form data
    by saving it to SQL it validates a schema
    and lets you look through the data to make sure it seems reasonable
    """
    couch_filter = 'fluff_filter/domain_type'
    extra_args = {
        'domains': 'care-bihar',
        'doc_type': 'XFormInstance'
    }
    document_class = XFormInstance

    def change_transform(self, doc_dict):
        form = self.document_class.wrap(doc_dict)
        if form.xmlns == VISIT_TYPES['bp']:
            return BirthPreparednessForm.from_couch(form)

    def change_transport(self, form_model):
        form_model.save()
<|endoftext|>


<commit_before>"""
Graph visitors.
"""

class NoVisitor(Exception):
    def __init__(self, args):
        self.args = args
    def __str__(self):
        return 'No transformer for Node: %s' % repr(self.args)

#------------------------------------------------------------------------
# Pretty Printing
#------------------------------------------------------------------------

class ExprPrinter(object):

    def __init__(self):
        self.indent = 0

    def visit(self, tree):

        if tree.children:
            print ('\t'*self.indent) + tree.__class__.__name__
            self.indent += 1
            for node in [self.visit(i) for i in tree.children]:
                print node
            self.indent -= 1
        else:
            return ('\t'*self.indent) + tree.__class__.__name__

    def Unknown(self, tree):
        raise NoVisitor(tree)

#------------------------------------------------------------------------
# Transformers
#------------------------------------------------------------------------

# TODO: write transformer counterparts?

class BasicGraphVisitor(object):
    """
    Visit an aterm graph. Each term must have a handler or Unknown is called.
    """

    # TODO: more robust!
    def visit_node(self, tree):
        nodei = tree.__class__.__name__
        trans = getattr(self, nodei, False)
        if trans:
            return trans(tree)
        else:
            return self._unknown(tree)

    def visit(self, tree):
        if isinstance(tree, list):
            return [self.visit(i) for i in tree]
        else:
            return self.visit_node(tree)

    def _unknown(self, tree):
        return self.Unknown(tree)

    def Unknown(self, tree):
        raise NoVisitor(tree)

class GraphVisitor(BasicGraphVisitor):
    """
    Similar to BasicGraphVisitor, but visits the children of unhandled terms.
    """

    def _unknown(self, tree):
        return self.visitchildren(tree)

    def visitchildren(self, tree):
        for fieldname in tree._fields:
            field = getattr(tree, fieldname)
            self.visit(field)

class GraphTransformer(GraphVisitor):
    """
    Similar to GraphVisitor, but return values replace the terms in the graph.
    """

    def _unknown(self, tree):
        return self.visitchildren(tree)

    def visit(self, tree):
        if isinstance(tree, list):
            result = [self.visit(i) for i in tree]
            return [x for x in result if x is not None]
        else:
            return self.visit_node(tree)

    def visitchildren(self, tree):
        for fieldname in tree._fields:
            field = getattr(tree, fieldname)
            result = self.visit(field)
            setattr(tree, fieldname, result)

        return tree

class GraphTranslator(GraphTransformer):
    """
    Similar to GraphTransformer, but it allows easy translation of the graph
    to other forms of graph, while mutating the graph at the same time.

    Example:

        We want to translate a portion of our graph to a Python AST, and
        replace the translated part of the graph with a single graph node
        reflecting this transformation.

            1) In each translatable node, set 'self.result = PythonAstNode()'.
               Read results of children by reading 'self.result' after visiting
               the respective children.

            2) For every translatable non-root node, return None to delete
               the node.

            3) At the root of the translatable sub-graph, return the node we
               want in our graph.
    """

    def set_resultlist(self, results):
        if len(results) == 0:
            results = None
        elif len(results) == 1:
            results = results[0]

        self.result = results

    def visit(self, tree):
        self.result = None
        if isinstance(tree, list):
            children = []
            results = []
            for child in tree:
                child = self.visit(child)
                if child is not None:
                    children.append(child)
                if self.result is not None:
                    results.append(self.result)

            self.set_resultlist(results)
            return [c for c in children if c is not None]
        else:
            return self.visit_node(tree)

    def visitchildren(self, tree):
        results = []
        for fieldname in tree._fields:
            field = getattr(tree, fieldname)
            result = self.visit(field)
            setattr(tree, fieldname, result)

            if self.result is not None:
                results.append(self.result)

        self.set_resultlist(results)
        return tree

class MroVisitor(object):

    def visit(self, tree):
        if isinstance(tree, list):
            return [self.visit(i) for i in tree]
        else:
            fn = None
            # Will always fall back to object() if defined.
            for o in tree.__class__.mro():
                nodei = o.__name__
                trans = getattr(self, nodei, False)
                if trans:
                    fn = trans
                    break
                else:
                    continue

            if fn:
                return fn(tree)
            else:
                self.Unknown(tree)

    def Unknown(self, tree):
        raise NoVisitor(tree)
<commit_msg>One convention for unknown nodes.<commit_after>"""
Graph visitors.
"""

class NoVisitor(Exception):
    def __init__(self, args):
        self.args = args
    def __str__(self):
        return 'No transformer for Node: %s' % repr(self.args)

#------------------------------------------------------------------------
# Pretty Printing
#------------------------------------------------------------------------

class ExprPrinter(object):

    def __init__(self):
        self.indent = 0

    def visit(self, tree):

        if tree.children:
            print ('\t'*self.indent) + tree.__class__.__name__
            self.indent += 1
            for node in [self.visit(i) for i in tree.children]:
                print node
            self.indent -= 1
        else:
            return ('\t'*self.indent) + tree.__class__.__name__

#------------------------------------------------------------------------
# Transformers
#------------------------------------------------------------------------

# TODO: write transformer counterparts?

class BasicGraphVisitor(object):
    """
    Visit an aterm graph. Each term must have a handler or _unknown is called.
    """

    # TODO: more robust!
    def visit_node(self, tree):
        nodei = tree.__class__.__name__
        trans = getattr(self, nodei, False)
        if trans:
            return trans(tree)
        else:
            return self._unknown(tree)

    def visit(self, tree):
        if isinstance(tree, list):
            return [self.visit(i) for i in tree]
        else:
            return self.visit_node(tree)

    def _unknown(self, tree):
        raise NoVisitor(tree)

class GraphVisitor(BasicGraphVisitor):
    """
    Similar to BasicGraphVisitor, but visits the children of unhandled terms.
    """

    def _unknown(self, tree):
        return self.visitchildren(tree)

    def visitchildren(self, tree):
        for fieldname in tree._fields:
            field = getattr(tree, fieldname)
            self.visit(field)

class GraphTransformer(GraphVisitor):
    """
    Similar to GraphVisitor, but return values replace the terms in the graph.
    """

    def _unknown(self, tree):
        return self.visitchildren(tree)

    def visit(self, tree):
        if isinstance(tree, list):
            result = [self.visit(i) for i in tree]
            return [x for x in result if x is not None]
        else:
            return self.visit_node(tree)

    def visitchildren(self, tree):
        for fieldname in tree._fields:
            field = getattr(tree, fieldname)
            result = self.visit(field)
            setattr(tree, fieldname, result)

        return tree

class GraphTranslator(GraphTransformer):
    """
    Similar to GraphTransformer, but it allows easy translation of the graph
    to other forms of graph, while mutating the graph at the same time.

    Example:

        We want to translate a portion of our graph to a Python AST, and
        replace the translated part of the graph with a single graph node
        reflecting this transformation.

            1) In each translatable node, set 'self.result = PythonAstNode()'.
               Read results of children by reading 'self.result' after visiting
               the respective children.

            2) For every translatable non-root node, return None to delete
               the node.

            3) At the root of the translatable sub-graph, return the node we
               want in our graph.
    """

    def set_resultlist(self, results):
        if len(results) == 0:
            results = None
        elif len(results) == 1:
            results = results[0]

        self.result = results

    def visit(self, tree):
        self.result = None
        if isinstance(tree, list):
            children = []
            results = []
            for child in tree:
                child = self.visit(child)
                if child is not None:
                    children.append(child)
                if self.result is not None:
                    results.append(self.result)

            self.set_resultlist(results)
            return [c for c in children if c is not None]
        else:
            return self.visit_node(tree)

    def visitchildren(self, tree):
        results = []
        for fieldname in tree._fields:
            field = getattr(tree, fieldname)
            result = self.visit(field)
            setattr(tree, fieldname, result)

            if self.result is not None:
                results.append(self.result)

        self.set_resultlist(results)
        return tree

class MroVisitor(object):

    def visit(self, tree):
        if isinstance(tree, list):
            return [self.visit(i) for i in tree]
        else:
            fn = None
            # Will always fall back to object() if defined.
            for o in tree.__class__.mro():
                nodei = o.__name__
                trans = getattr(self, nodei, False)
                if trans:
                    fn = trans
                    break
                else:
                    continue

            if fn:
                return fn(tree)
            else:
                self._unknown(tree)
<|endoftext|>


<commit_before>import unittest

from perfkit.common import http_util

class MockRequest(object):
  values = {}

  def __init__(self, values):
    self.values = values

  def get(self, key, default=None):
    return self.values.get(key) or default

class DashboardTest(unittest.TestCase):

  def testGetString(self):
    expected_value = 'TEST_VALUE'

    params = MockRequest({
        'name': expected_value
    })

    actual_value = http_util.GetStringParam(params, 'name')
    self.assertEqual(expected_value, actual_value)

  def testGetStringNotProvided(self):
    expected_value = 'TEST_VALUE'

    params = MockRequest({
        'name': expected_value
    })

    actual_value = http_util.GetStringParam(params, 'name')
    self.assertEqual(expected_value, actual_value)

  def testGetStringDefault(self):
    expected_value = 'TEST_VALUE'

    params = MockRequest({})

    actual_value = http_util.GetStringParam(params, 'name',
                                           default=expected_value)
    self.assertEqual(expected_value, actual_value)

  def testGetStringOptional(self):
    expected_value = None

    params = MockRequest({})

    actual_value = http_util.GetStringParam(params, 'name', required=False)
    self.assertEqual(expected_value, actual_value)

  def testGetStringRequiredNotProvided(self):
    params = MockRequest({
        'existing': 'value'
    })

    self.assertRaises(http_util.ParameterError,
                      http_util.GetStringParam,
                      params, 'name', True)

  def testGetStringRequiredDefaultOverride(self):
    expected_value = 'TEST_VALUE'
    params = MockRequest({})

    self.assertEqual(expected_value,
                     http_util.GetStringParam(
                         params, 'name', True, expected_value))

  def testGetBooleanTrue(self):
    providedValue = '1'
    expected_value = True

    params = MockRequest({'isopen': providedValue})

    self.assertEquals(expected_value,
                      http_util.GetBoolParam(params, 'isopen'))

  def testGetBooleanFalse(self):
    providedValue = '0'
    expected_value = False

    params = MockRequest({'isopen': providedValue})

    self.assertEquals(expected_value,
                      http_util.GetBoolParam(params, 'isopen'))

  def testGetBooleanInvalid(self):
    providedValue = 'invalid'

    params = MockRequest({'isopen': providedValue})

    self.assertRaises(http_util.ParameterError,
                      http_util.GetBoolParam,
                      params, 'isopen')

  def testGetJSON(self):
    providedValue = '{"name": "value"}'
    expected_value = {'name': 'value'}

    params = MockRequest({'data': providedValue})

    self.assertEquals(expected_value,
                      http_util.GetJsonParam(params, 'data'))

  def testGetJSONInvalid(self):
    providedValue = 'invalid'

    params = MockRequest({'data': providedValue})

    self.assertRaises(http_util.ParameterError,
                      http_util.GetJsonParam,
                      params, 'data')
<commit_msg>=Fix indent.<commit_after>import unittest

from perfkit.common import http_util

class MockRequest(object):
  values = {}

  def __init__(self, values):
    self.values = values

  def get(self, key, default=None):
    return self.values.get(key) or default

class DashboardTest(unittest.TestCase):

  def testGetString(self):
    expected_value = 'TEST_VALUE'

    params = MockRequest({
        'name': expected_value
    })

    actual_value = http_util.GetStringParam(params, 'name')
    self.assertEqual(expected_value, actual_value)

  def testGetStringNotProvided(self):
    expected_value = 'TEST_VALUE'

    params = MockRequest({
        'name': expected_value
    })

    actual_value = http_util.GetStringParam(params, 'name')
    self.assertEqual(expected_value, actual_value)

  def testGetStringDefault(self):
    expected_value = 'TEST_VALUE'

    params = MockRequest({})

    actual_value = http_util.GetStringParam(params, 'name',
                                            default=expected_value)
    self.assertEqual(expected_value, actual_value)

  def testGetStringOptional(self):
    expected_value = None

    params = MockRequest({})

    actual_value = http_util.GetStringParam(params, 'name', required=False)
    self.assertEqual(expected_value, actual_value)

  def testGetStringRequiredNotProvided(self):
    params = MockRequest({
        'existing': 'value'
    })

    self.assertRaises(http_util.ParameterError,
                      http_util.GetStringParam,
                      params, 'name', True)

  def testGetStringRequiredDefaultOverride(self):
    expected_value = 'TEST_VALUE'
    params = MockRequest({})

    self.assertEqual(expected_value,
                     http_util.GetStringParam(
                         params, 'name', True, expected_value))

  def testGetBooleanTrue(self):
    providedValue = '1'
    expected_value = True

    params = MockRequest({'isopen': providedValue})

    self.assertEquals(expected_value,
                      http_util.GetBoolParam(params, 'isopen'))

  def testGetBooleanFalse(self):
    providedValue = '0'
    expected_value = False

    params = MockRequest({'isopen': providedValue})

    self.assertEquals(expected_value,
                      http_util.GetBoolParam(params, 'isopen'))

  def testGetBooleanInvalid(self):
    providedValue = 'invalid'

    params = MockRequest({'isopen': providedValue})

    self.assertRaises(http_util.ParameterError,
                      http_util.GetBoolParam,
                      params, 'isopen')

  def testGetJSON(self):
    providedValue = '{"name": "value"}'
    expected_value = {'name': 'value'}

    params = MockRequest({'data': providedValue})

    self.assertEquals(expected_value,
                      http_util.GetJsonParam(params, 'data'))

  def testGetJSONInvalid(self):
    providedValue = 'invalid'

    params = MockRequest({'data': providedValue})

    self.assertRaises(http_util.ParameterError,
                      http_util.GetJsonParam,
                      params, 'data')
<|endoftext|>


<commit_before>'''
Created on Feb 23, 2016

@author: Devin
'''
from plotly.offline import plot
from plotly.offline import download_plotlyjs, init_notebook_mode, iplot
import plotly.graph_objs as go
from openpyxl import load_workbook
import numpy as np

#wb = load_workbook(filename = 'Areas.xlsx')

#sheet = wb.get_sheet_by_name('Sarah')
#type(sheet)

#d = [sheet.cell(row = i, column = 2).value for i in range(1, 7)]

def close_enough(nextVal, currentVal, error = 1e-7):
    diff = nextVal - currentVal
    diff = abs(diff)
    maxdiff = max(diff)
    if maxdiff < error:
        return True
    else:
        return False

def calc(x, better = 3.0, muchbetter = 9.0):
    if(x == ">"):
        return 1./better
    elif(x == ">>"):
        return 1./muchbetter
    elif(x == "<"):
        return better
    elif(x == "<<"):
        return muchbetter
    elif(x == "E"):
        return 1
    else:
        return("Error, unknown value "+ x)

def single_stats(fname, sheetname, bars = False, better = 3,  muchbetter = 9, doppelganger = False):
    if bars:
        eigen = single_stats(fname, sheetname, bars = False, better = better, muchbetter = muchbetter, doppelganger = doppelganger)
        altnames = get_altnames(fname)
        data = go.Bar(x=altnames, y=eigen)
        layout = go.Layout(title = sheetname+"'s Priorities")
        return iplot(go.Figure(data = go.Data([data]), layout = layout))

    else:
        return Largest_eigen(getMatrix(fname, sheetname, better, muchbetter, doppelganger))

def group_stats(xlsxFname, listOfSheetnames = None, bars = False, better = 3,  muchbetter = 9, doppelganger = False):
    if listOfSheetnames is None:
        listOfSheetnames = get_sheetnames(xlsxFname)
    if bars:
        eigen = group_stats(xlsxFname, listOfSheetnames, bars = False, better = better,  muchbetter = muchbetter, doppelganger = doppelganger)
        altnames = get_altnames(xlsxFname)
        data = go.Bar(x=altnames, y=eigen)
        layout = go.Layout(title = "Group Priorities")
        # Do nifty stuff like in tutorial to create plotly chart
        return iplot(go.Figure(data = go.Data([data]), layout = layout))
    else:
        listOfMatrices = []
        for sheetName in listOfSheetnames:
            theMatrix = getMatrix(xlsxFname, sheetName, better = better,  muchbetter = muchbetter, doppelganger = doppelganger)
            listOfMatrices.append(theMatrix)
        geomAvg = geometric_avg(listOfMatrices)
        return Largest_eigen(geomAvg)

def Largest_eigen(x, error = 1e-7):
    size = x.shape[0]
    currentVal = np.array([1./size for i in range(size)])
    while True:
        nextVal = np.matmul(x, currentVal)
        nextVal = nextVal / sum(nextVal)
        #print "Working Still"
        if close_enough(nextVal, currentVal, error = 1e-7):
            return nextVal
        currentVal = nextVal
    return currentVal

def get_altnames(xlsxfile):
    wb = load_workbook(filename = xlsxfile)
    firstsheet = wb.get_sheet_names()[0]
    sheet = wb.get_sheet_by_name(firstsheet)
    nrows = sheet.max_row
    firstcol = [sheet.cell(row = i, column = 1).value for i in range(1, nrows)]
    thirdcol = [sheet.cell(row = i, column = 3).value for i in range(1, nrows)]

    rval = list()
    for row in range(0, len(firstcol)):
        if thirdcol[row] != None:
            if (firstcol[row] != None) and (firstcol[row] not in rval):
                rval.append(firstcol[row])
            if (thirdcol[row] not in rval):
                rval.append(thirdcol[row])
    return rval

def getMatrix(fname, sheetName, better = 3, muchbetter = 9, doppelganger = False):
    #First we need to know the alternatives in this sheet
    altnames = get_altnames(fname)
    #The number of alternatives is the size of the return matrix
    matSize = len(altnames)
    #The return matrix starts off with 1's on the diagonal, and zeroes elsewhere
    returnMatrix = np.identity(matSize)
    #Load up the excel spreadsheet and get the actual sheet
    wb = load_workbook(filename = fname)
    xsheet = wb.get_sheet_by_name(sheetName)
    #Loop over the rows in this sheet
    for row in range(xsheet.max_row):
        #Get the first column val, which is the row of the comparison
        val1 = xsheet.cell(row = row+1, column = 1).value
        #Get the second column val, which is the value of the comparison
        val2 = xsheet.cell(row = row+1, column = 2).value
        #Get the third column value, which is the COLUMN of the comparison
        val3 = xsheet.cell(row = row+1, column = 3).value
        #If third column value == None, that means this wasn't a pairwise comparison entry
        #Maybe it was a demographic bit (FavColor)
        if (val3 != None):
            #Get the index of the alt name in the list of altnames
            #This tells us where to place the vote in the resulting matrix
            rowIndex = altnames.index(val1)
            colIndex = altnames.index(val3)
            #Get the numeric value of the vote from the stringy vote
            voteValue = calc(val2, better, muchbetter)
            #Lastly place the vote in, and it's reciprocal on the other side
            #of the diagonal
            returnMatrix[rowIndex,colIndex] = voteValue
            returnMatrix[colIndex, rowIndex] = 1./voteValue
#            returnMatrix[index3, index1] =
    if doppelganger:
        returnMatrix = np.transpose(returnMatrix)

    return returnMatrix

#def getBars(stats, barnames, title, xlsxfile, sheetname):
    #altnames = get_altnames(xlsxfile)
    #data = go.Bar(x= altnames, y= single_stats(xlsxfile, sheetname))
    ##layout = go.Layout(title = sheetname+ "'s Priorities")
    #iplot(go.Figure(data = go.Data([data]), layout = layout))

def geometric_avg(listOfMats):
    #Create rval
    rval = np.zeros_like(listOfMats[0])
    nrows = rval.shape[0]
    ncols = rval.shape[1]
    for row in range(nrows):
        for col in range(ncols):
            val = 1
            count = 0
            for mat in listOfMats:
                if mat[row, col]!=0:
                    val *= mat[row,col]
                    count+=1
            if count != 0:
                val = pow(val, 1.0/count)
            #Finally have the value, put it in
            rval[row,col] = val
    return rval
print(group_stats('Areas.xlsx', ['Sarah', 'Drew']))
#print getMatrix('Areas.xlsx', 'Sarah')

def get_sheetnames(xlsxFname):
    wb = load_workbook(filename = xlsxFname)
    return wb.get_sheet_names()

<commit_msg>Stopping reading in example data and printing in devahp.py<commit_after>'''
Created on Feb 23, 2016

@author: Devin
'''
from plotly.offline import plot
from plotly.offline import download_plotlyjs, init_notebook_mode, iplot
import plotly.graph_objs as go
from openpyxl import load_workbook
import numpy as np

#wb = load_workbook(filename = 'Areas.xlsx')

#sheet = wb.get_sheet_by_name('Sarah')
#type(sheet)

#d = [sheet.cell(row = i, column = 2).value for i in range(1, 7)]

def close_enough(nextVal, currentVal, error = 1e-7):
    diff = nextVal - currentVal
    diff = abs(diff)
    maxdiff = max(diff)
    if maxdiff < error:
        return True
    else:
        return False

def calc(x, better = 3.0, muchbetter = 9.0):
    if(x == ">"):
        return 1./better
    elif(x == ">>"):
        return 1./muchbetter
    elif(x == "<"):
        return better
    elif(x == "<<"):
        return muchbetter
    elif(x == "E"):
        return 1
    else:
        return("Error, unknown value "+ x)

def single_stats(fname, sheetname, bars = False, better = 3,  muchbetter = 9, doppelganger = False):
    if bars:
        eigen = single_stats(fname, sheetname, bars = False, better = better, muchbetter = muchbetter, doppelganger = doppelganger)
        altnames = get_altnames(fname)
        data = go.Bar(x=altnames, y=eigen)
        layout = go.Layout(title = sheetname+"'s Priorities")
        return iplot(go.Figure(data = go.Data([data]), layout = layout))

    else:
        return Largest_eigen(getMatrix(fname, sheetname, better, muchbetter, doppelganger))

def group_stats(xlsxFname, listOfSheetnames = None, bars = False, better = 3,  muchbetter = 9, doppelganger = False):
    if listOfSheetnames is None:
        listOfSheetnames = get_sheetnames(xlsxFname)
    if bars:
        eigen = group_stats(xlsxFname, listOfSheetnames, bars = False, better = better,  muchbetter = muchbetter, doppelganger = doppelganger)
        altnames = get_altnames(xlsxFname)
        data = go.Bar(x=altnames, y=eigen)
        layout = go.Layout(title = "Group Priorities")
        # Do nifty stuff like in tutorial to create plotly chart
        return iplot(go.Figure(data = go.Data([data]), layout = layout))
    else:
        listOfMatrices = []
        for sheetName in listOfSheetnames:
            theMatrix = getMatrix(xlsxFname, sheetName, better = better,  muchbetter = muchbetter, doppelganger = doppelganger)
            listOfMatrices.append(theMatrix)
        geomAvg = geometric_avg(listOfMatrices)
        return Largest_eigen(geomAvg)

def Largest_eigen(x, error = 1e-7):
    size = x.shape[0]
    currentVal = np.array([1./size for i in range(size)])
    while True:
        nextVal = np.matmul(x, currentVal)
        nextVal = nextVal / sum(nextVal)
        #print "Working Still"
        if close_enough(nextVal, currentVal, error = 1e-7):
            return nextVal
        currentVal = nextVal
    return currentVal

def get_altnames(xlsxfile):
    wb = load_workbook(filename = xlsxfile)
    firstsheet = wb.get_sheet_names()[0]
    sheet = wb.get_sheet_by_name(firstsheet)
    nrows = sheet.max_row
    firstcol = [sheet.cell(row = i, column = 1).value for i in range(1, nrows)]
    thirdcol = [sheet.cell(row = i, column = 3).value for i in range(1, nrows)]

    rval = list()
    for row in range(0, len(firstcol)):
        if thirdcol[row] != None:
            if (firstcol[row] != None) and (firstcol[row] not in rval):
                rval.append(firstcol[row])
            if (thirdcol[row] not in rval):
                rval.append(thirdcol[row])
    return rval

def getMatrix(fname, sheetName, better = 3, muchbetter = 9, doppelganger = False):
    #First we need to know the alternatives in this sheet
    altnames = get_altnames(fname)
    #The number of alternatives is the size of the return matrix
    matSize = len(altnames)
    #The return matrix starts off with 1's on the diagonal, and zeroes elsewhere
    returnMatrix = np.identity(matSize)
    #Load up the excel spreadsheet and get the actual sheet
    wb = load_workbook(filename = fname)
    xsheet = wb.get_sheet_by_name(sheetName)
    #Loop over the rows in this sheet
    for row in range(xsheet.max_row):
        #Get the first column val, which is the row of the comparison
        val1 = xsheet.cell(row = row+1, column = 1).value
        #Get the second column val, which is the value of the comparison
        val2 = xsheet.cell(row = row+1, column = 2).value
        #Get the third column value, which is the COLUMN of the comparison
        val3 = xsheet.cell(row = row+1, column = 3).value
        #If third column value == None, that means this wasn't a pairwise comparison entry
        #Maybe it was a demographic bit (FavColor)
        if (val3 != None):
            #Get the index of the alt name in the list of altnames
            #This tells us where to place the vote in the resulting matrix
            rowIndex = altnames.index(val1)
            colIndex = altnames.index(val3)
            #Get the numeric value of the vote from the stringy vote
            voteValue = calc(val2, better, muchbetter)
            #Lastly place the vote in, and it's reciprocal on the other side
            #of the diagonal
            returnMatrix[rowIndex,colIndex] = voteValue
            returnMatrix[colIndex, rowIndex] = 1./voteValue
#            returnMatrix[index3, index1] =
    if doppelganger:
        returnMatrix = np.transpose(returnMatrix)

    return returnMatrix

#def getBars(stats, barnames, title, xlsxfile, sheetname):
    #altnames = get_altnames(xlsxfile)
    #data = go.Bar(x= altnames, y= single_stats(xlsxfile, sheetname))
    ##layout = go.Layout(title = sheetname+ "'s Priorities")
    #iplot(go.Figure(data = go.Data([data]), layout = layout))

def geometric_avg(listOfMats):
    #Create rval
    rval = np.zeros_like(listOfMats[0])
    nrows = rval.shape[0]
    ncols = rval.shape[1]
    for row in range(nrows):
        for col in range(ncols):
            val = 1
            count = 0
            for mat in listOfMats:
                if mat[row, col]!=0:
                    val *= mat[row,col]
                    count+=1
            if count != 0:
                val = pow(val, 1.0/count)
            #Finally have the value, put it in
            rval[row,col] = val
    return rval
#print(group_stats('Areas.xlsx', ['Sarah', 'Drew']))
#print getMatrix('Areas.xlsx', 'Sarah')

def get_sheetnames(xlsxFname):
    wb = load_workbook(filename = xlsxFname)
    return wb.get_sheet_names()

<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*-*- encoding: utf-8 -*-*-
#
# Copyright (C) 2012 onwards University of Deusto
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# This software consists of contributions made by many individuals,
# listed below:
#
# Author: Luis Rodriguez Gil <luis.rodriguezgil@deusto.es>
#
from flask import make_response, request, jsonify
from weblab.core.wl import weblab_api

import uuid
import json
import requests
import traceback

# # PROTOCOL
#
# ## ACTION: Create a new job
# METHOD: POST to /compiserv/queue/armc
# PARAMETERS: Can receive the file as a multipart file, or as raw data.
#
# RETURNS:
# A JSON object:
# { result: <result>, // <result> is 'accepted' or 'denied'
#   uid: <uid>, // only if result is 'accepted'
# }
#
#
# ## ACTION: Check the state of a job
# METHOD: GET to /compiserv/queue/<uid>
# PARAMETERS: The <uid>
#
# RETURNS:
# A JSON object:
# { state: <state>, // <state> is 'finished' if the task is done or failed, 'unfinished' if it is reportedly in progress or
#                      in queue, and 'not_found' if it could not be found.
#
# }
#

JOBS = {}
BASE_URL = "http://llcompilerservice.azurewebsites.net/CompilerGeneratorService.svc"
POST_URL = BASE_URL + "/PutCompilerTask/uvision"
GET_URL = BASE_URL + "/GetCompilerTask/uvision/{0}/{1}"

@weblab_api.route_web('/compiserv/')
def compiserve():
    msg = "Welcome to the Compiler Service. This is not yet implemented."
    data = {"msg": msg}
    contents = json.dumps(data, indent = 4)
    response = make_response(contents)
    response.content_type = 'application/json'
    return response

@weblab_api.route_web('/compiserv/queue/armc', methods=["POST"])
def compiserve_queue_armc_post():
    """
    Enqueues an ARMC synthesization job. This can be done by any client.
    :return:
    """

    response = {
        "result": ""  # accepted or denied
    }

    file_contents = None

    if request.data is not None:
        file_contents = request.data
    elif request.files is not None and len(request.files) > 0:
        file_contents = request.files['file']
        # TO-DO

    if file_contents is None:
        response['result'] = 'denied'
        response['reason'] = 'no_file_sent'

    else:
        # Send to AZURE SERVICE
        resp = requests.post(POST_URL, file_contents)
        json_resp = resp.json()
        generated_date = json_resp["GeneratedDate"]
        id = json_resp["ID"]
        token = json_resp["TokenID"]

        response['result'] = 'accepted'
        uid = "{0}+{1}".format(id, token)
        response['uid'] = uid

        # Store the JOB in the queue.
        JOBS[uid] = {'state': 'queued'}

    contents = json.dumps(response, indent=4)
    response = make_response(contents)
    response.content_type = 'application/json'
    return response

@weblab_api.route_web('/compiserv/queue/armc/<uid>', methods=["GET"])
def compiserve_queue_get(uid):
    """
    Enquiries about the status of a specific job. This can be done by any client.
    :return:
    """

    try:

        print("Received UID is: " + uid)

        result = {
            "state": ""
        }

        if uid not in JOBS:
            return jsonify(state='not_found')

        job = JOBS[uid]

        # Split the UID into its components.
        id, tokenid = uid.split("+", 1)

        # Retrieve the state of the remote JOB
        resp = requests.get(GET_URL.format(id, tokenid))
        jsresp = resp.json()

        # BinaryFile, CompletedDate, LogFile, State
        state = jsresp['State'].lower()

        if state == 'finished':
            binary_file = jsresp['BinaryFile']
            completed_date = jsresp['CompletedDate']
            log_file = jsresp['LogFile']

            # Store the files internally. TODO: DO THIS PROPERLY. For now we store them in memory.
            job["binary_file"] = binary_file
            job["completed_date"] = completed_date
            job["log_file"] = log_file

            result['state'] = 'done'

        elif state.startswith('unfinished'):
            splits = state.split(":")
            number = int(splits[1].strip())
            result['state'] = 'queued'
            result['position'] = number

        # TODO: How are failures reported?

        else:
            raise Exception("Unrecognized job state: " + state)

        contents = json.dumps(result, indent=4)
        response = make_response(contents)
        response.content_type = 'application/json'
        return response

    except Exception as ex:
        tb = traceback.format_exc()
        return jsonify(state='error', traceback=tb)

@weblab_api.route_web('/compiserv/result/<uid>/outputfile', methods=["GET"])
def compiserve_result_outputfile(uid):
    """
    Requests the result (outputfile) of a job. This is only to be called INTERNALLY by the Experiment Servers
    and is protected by a token.
    :return:
    """

    if uid not in JOBS:
        result = {
            'result': 'error',
            'msg': "job not found"
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"
        return response

    # Find the job
    job = JOBS[uid]

    if "binary_file" in job:  # TODO: IF FILE IS INDEED READY
        file_contents = job["binary_file"]
        response = make_response(file_contents)
        response.headers["Content-Disposition"] = "attachment; filename=result.bin"
        response.headers["Content-Type"] = "application/octet-stream"
    else:  #
        result = {
            'result': 'error',
            'msg': 'result not found'
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"

    return response

@weblab_api.route_web('/compiserv/result/<uid>/logfile', methods=["GET"])
def compiserve_result_logfile(uid):
    """
    Requests the result (logfile) of a job. This is only to be called INTERNALLY by the Experiment Servers
    and is protected by a token.
    :param uid:
    :return:
    """

    if uid not in JOBS:
        result = {
            'result': 'error',
            'msg': "job not found"
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"
        return response

    # Find the job
    job = JOBS[uid]

    if "log_file" in job:  # TODO: IF FILE IS INDEED READY
        # TODO: Check the state of the job. Do not assume it is finished.
        file_contents = job["log_file"]
        response = make_response(file_contents)
        response.headers["Content-Disposition"] = "attachment; filename=logfile.bin"
        response.headers["Content-Type"] = "application/octet-stream"
    else:  #
        result = {
            'result': 'error',
            'msg': 'result not found'
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"

    return response
<commit_msg>[compiserv]: Yet another minor bugfix<commit_after>#!/usr/bin/env python
#-*-*- encoding: utf-8 -*-*-
#
# Copyright (C) 2012 onwards University of Deusto
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# This software consists of contributions made by many individuals,
# listed below:
#
# Author: Luis Rodriguez Gil <luis.rodriguezgil@deusto.es>
#
from flask import make_response, request, jsonify
from weblab.core.wl import weblab_api

import uuid
import json
import requests
import traceback

# # PROTOCOL
#
# ## ACTION: Create a new job
# METHOD: POST to /compiserv/queue/armc
# PARAMETERS: Can receive the file as a multipart file, or as raw data.
#
# RETURNS:
# A JSON object:
# { result: <result>, // <result> is 'accepted' or 'denied'
#   uid: <uid>, // only if result is 'accepted'
# }
#
#
# ## ACTION: Check the state of a job
# METHOD: GET to /compiserv/queue/<uid>
# PARAMETERS: The <uid>
#
# RETURNS:
# A JSON object:
# { state: <state>, // <state> is 'finished' if the task is done or failed, 'unfinished' if it is reportedly in progress or
#                      in queue, and 'not_found' if it could not be found.
#
# }
#

JOBS = {}
BASE_URL = "http://llcompilerservice.azurewebsites.net/CompilerGeneratorService.svc"
POST_URL = BASE_URL + "/PutCompilerTask/uvision"
GET_URL = BASE_URL + "/GetCompilerTask/{0}/{1}"

@weblab_api.route_web('/compiserv/')
def compiserve():
    msg = "Welcome to the Compiler Service. This is not yet implemented."
    data = {"msg": msg}
    contents = json.dumps(data, indent = 4)
    response = make_response(contents)
    response.content_type = 'application/json'
    return response

@weblab_api.route_web('/compiserv/queue/armc', methods=["POST"])
def compiserve_queue_armc_post():
    """
    Enqueues an ARMC synthesization job. This can be done by any client.
    :return:
    """

    response = {
        "result": ""  # accepted or denied
    }

    file_contents = None

    if request.data is not None:
        file_contents = request.data
    elif request.files is not None and len(request.files) > 0:
        file_contents = request.files['file']
        # TO-DO

    if file_contents is None:
        response['result'] = 'denied'
        response['reason'] = 'no_file_sent'

    else:
        # Send to AZURE SERVICE
        resp = requests.post(POST_URL, file_contents)
        json_resp = resp.json()
        generated_date = json_resp["GeneratedDate"]
        id = json_resp["ID"]
        token = json_resp["TokenID"]

        response['result'] = 'accepted'
        uid = "{0}+{1}".format(id, token)
        response['uid'] = uid

        # Store the JOB in the queue.
        JOBS[uid] = {'state': 'queued'}

    contents = json.dumps(response, indent=4)
    response = make_response(contents)
    response.content_type = 'application/json'
    return response

@weblab_api.route_web('/compiserv/queue/armc/<uid>', methods=["GET"])
def compiserve_queue_get(uid):
    """
    Enquiries about the status of a specific job. This can be done by any client.
    :return:
    """

    try:

        print("Received UID is: " + uid)

        result = {
            "state": ""
        }

        if uid not in JOBS:
            return jsonify(state='not_found')

        job = JOBS[uid]

        # Split the UID into its components.
        id, tokenid = uid.split("+", 1)

        # Retrieve the state of the remote JOB
        resp = requests.get(GET_URL.format(id, tokenid))
        jsresp = resp.json()

        # BinaryFile, CompletedDate, LogFile, State
        state = jsresp['State'].lower()

        if state == 'finished':
            binary_file = jsresp['BinaryFile']
            completed_date = jsresp['CompletedDate']
            log_file = jsresp['LogFile']

            # Store the files internally. TODO: DO THIS PROPERLY. For now we store them in memory.
            job["binary_file"] = binary_file
            job["completed_date"] = completed_date
            job["log_file"] = log_file

            result['state'] = 'done'

        elif state.startswith('unfinished'):
            splits = state.split(":")
            number = int(splits[1].strip())
            result['state'] = 'queued'
            result['position'] = number

        # TODO: How are failures reported?

        else:
            raise Exception("Unrecognized job state: " + state)

        contents = json.dumps(result, indent=4)
        response = make_response(contents)
        response.content_type = 'application/json'
        return response

    except Exception as ex:
        tb = traceback.format_exc()
        return jsonify(state='error', traceback=tb)

@weblab_api.route_web('/compiserv/result/<uid>/outputfile', methods=["GET"])
def compiserve_result_outputfile(uid):
    """
    Requests the result (outputfile) of a job. This is only to be called INTERNALLY by the Experiment Servers
    and is protected by a token.
    :return:
    """

    if uid not in JOBS:
        result = {
            'result': 'error',
            'msg': "job not found"
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"
        return response

    # Find the job
    job = JOBS[uid]

    if "binary_file" in job:  # TODO: IF FILE IS INDEED READY
        file_contents = job["binary_file"]
        response = make_response(file_contents)
        response.headers["Content-Disposition"] = "attachment; filename=result.bin"
        response.headers["Content-Type"] = "application/octet-stream"
    else:  #
        result = {
            'result': 'error',
            'msg': 'result not found'
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"

    return response

@weblab_api.route_web('/compiserv/result/<uid>/logfile', methods=["GET"])
def compiserve_result_logfile(uid):
    """
    Requests the result (logfile) of a job. This is only to be called INTERNALLY by the Experiment Servers
    and is protected by a token.
    :param uid:
    :return:
    """

    if uid not in JOBS:
        result = {
            'result': 'error',
            'msg': "job not found"
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"
        return response

    # Find the job
    job = JOBS[uid]

    if "log_file" in job:  # TODO: IF FILE IS INDEED READY
        # TODO: Check the state of the job. Do not assume it is finished.
        file_contents = job["log_file"]
        response = make_response(file_contents)
        response.headers["Content-Disposition"] = "attachment; filename=logfile.bin"
        response.headers["Content-Type"] = "application/octet-stream"
    else:  #
        result = {
            'result': 'error',
            'msg': 'result not found'
        }
        result = json.dumps(result, indent=4)
        response = make_response(result)
        response.headers["Content-Type"] = "application/json"

    return response
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess;
import sys;

#Declaring the variables for the program
count = 0;
results = [];

#loop to check and get all the lines of output from nomos for all the files.
for arg in sys.argv:
        if(count > 0):
                results.insert(pos,subprocess.check_output(["dosocs2","oneshot","%s"%(arg,)]));
        count += 1;

#Just printing all the output that is stored.
for match in results:
        print match;

<commit_msg>Delete DoSOCSprogram.py<commit_after><|endoftext|>