<commit_before>#!/usr/bin/env python

import copy
import os
import shutil
import sys
import tempfile
import unittest

import xiosim_driver as xd
import xiosim_stat as xs

def CreateDriver():
    PIN_ROOT = os.environ["PIN_ROOT"]
    XIOSIM_INSTALL = os.environ["XIOSIM_INSTALL"]
    XIOSIM_TREE = os.environ["XIOSIM_TREE"]
    xio = xd.XIOSimDriver(PIN_ROOT, XIOSIM_INSTALL, XIOSIM_TREE)
    return xio

class XIOSimTest(unittest.TestCase):
    ''' Test fixtures for XIOSim end-to-end tests.'''
    def setUp(self):
        ''' Set up a driver with common options and a temp run directory.'''
        self.xio = CreateDriver()
        self.run_dir = tempfile.mkdtemp()
        self.clean_run_dir = ("LEAVE_TEST_DIR" not in os.environ)
        self.setDriverParams()
        self.expected_vals = []

    def tearDown(self):
        if self.clean_run_dir:
            shutil.rmtree(self.xio.GetRunDir())

    def runAndValidate(self):
        ''' Execute XIOSimDriver and check output for golden values of stats.'''
        out_file = os.path.join(self.run_dir, "harness.out")
        err_file = os.path.join(self.run_dir, "harness.err")
        ret = self.xio.Exec(stdout_file=out_file, stderr_file=err_file, cwd=self.run_dir)
        self.assertEqual(ret, 0, "XIOSim run failed (errcode %d)" % ret)

        for re, golden_val in self.expected_vals:
            val = xs.GetStat(self.xio.GetSimOut(), re)
            res = xs.ValidateStat(val, golden_val)
            self.assertEqual(res, True, "%s: expected %.2f, got %.2f" %
                                        (re, golden_val, val))

    def writeTestBmkConfig(self, bmk, num_copies=1):
        ''' Create a temp benchmark config file in the test run directory.'''
        cfg_file = self.xio.GenerateTestBmkConfig(bmk, num_copies)
        bmk_cfg = os.path.join(self.run_dir, "%s.cfg" % bmk)
        with open(bmk_cfg, "w") as f:
            for l in cfg_file:
                f.write(l)
        return bmk_cfg

    def writeTestConfig(self, start_config, changes):
        ''' Create a temp config file in the test run directory,
        starting with @start_config and applying a dictionary or
        paramter changes. '''
        cfg_file = os.path.join(self.run_dir, os.path.basename(start_config))
        cfg_contents = self.xio.GenerateConfigFile(start_config, changes)
        with open(cfg_file, "w") as f:
            for l in cfg_contents:
                f.write(l)
        return cfg_file

class Fib1Test(XIOSimTest):
    ''' End-to-end test with a single binary running from start to end.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1Test, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 118369.0))

    def runTest(self):
        self.runAndValidate()

class NoneTest(XIOSimTest):
    ''' End-to-end test with the 'none' core model.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "none.cfg"))

    def setUp(self):
        super(NoneTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 118369.0))

    def runTest(self):
        self.runAndValidate()

class Fib1LengthTest(XIOSimTest):
    ''' End-to-end test with a single binary running for X instructions.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddInstLength(10000)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1LengthTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 9900.0))

    def runTest(self):
        self.runAndValidate()

class Fib1SkipTest(XIOSimTest):
    ''' End-to-end test with a single binary skipping first X instructions.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddSkipInst(50000)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1SkipTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 68369.0))

    def runTest(self):
        self.runAndValidate()

class Fib1PinPointTest(XIOSimTest):
    ''' End-to-end test with a single binary and one pinpoint.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        ppfile = os.path.join(self.xio.GetTreeDir(), "tests",
                              "fib..pintool.1.pp")
        self.xio.AddPinPointFile(ppfile)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1PinPointTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 30000.0))

    def runTest(self):
        self.runAndValidate()

class Fib1PinPointsTest(XIOSimTest):
    ''' End-to-end test with a single binary and multiple pinpoints.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        ppfile = os.path.join(self.xio.GetTreeDir(), "tests",
                              "fib..pintool.2.pp")
        self.xio.AddPinPointFile(ppfile)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1PinPointsTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 10100.0))

    def runTest(self):
        self.runAndValidate()

class ROITest(XIOSimTest):
    ''' End-to-end test with a single binary with ROI hooks.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("roi")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddROIOptions()
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(ROITest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 217000.0))

    def runTest(self):
        self.runAndValidate()

class ReplaceTest(XIOSimTest):
    ''' End-to-end test where we replace a function call with a NOP.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("repl")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddReplaceOptions("fib_repl")
        # 30K cycles for the magic instruction that replaces fib_repl()
        # 30K is high enough that the test can catch if we didn't add it,
        # yet low enough that it doesn't tickle the deadlock threshold (50K)
        repl = {
            "core_cfg.exec_cfg.exeu magic.latency" : "30000",
            "core_cfg.exec_cfg.exeu magic.rate" : "30000",
        }
        test_cfg = self.writeTestConfig(os.path.join(self.xio.GetTreeDir(),
                                                     "config", "N.cfg"),
                                        repl)
        self.xio.AddZestoOptions(test_cfg)

    def setUp(self):
        super(ReplaceTest, self).setUp()
        # repl is ~1M instructions
        # when we correctly ignore the middle call, we expect ~450K
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 447000.0))
        # repl when we just ignore the middle call takes ~397K cycles + 30K magic
        self.expected_vals.append((xs.PerfStatRE("sim_cycle"), 426000.0))

    def runTest(self):
        self.runAndValidate()

class PowerTest(XIOSimTest):
    ''' End-to-end test that calculates power.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)

        repl = {
            "system_cfg.simulate_power" : "true"
        }
        test_cfg = self.writeTestConfig(os.path.join(self.xio.GetTreeDir(),
                                                     "config", "A.cfg"),
                                        repl)
        self.xio.AddZestoOptions(test_cfg)

    def setUp(self):
        super(PowerTest, self).setUp()
        self.expected_vals.append((xs.PowerStatRE("  Runtime Dynamic"), 0.69))
        self.expected_vals.append((xs.PowerStatRE("  Total Leakage"), 0.48))

    def runTest(self):
        self.runAndValidate()

#TODO(skanev): Add a power trace test

class DFSTest(XIOSimTest):
    ''' End-to-end test that checks that "sample" DFS modifies frequencies.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("step")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)

        repl = {
            "system_cfg.simulate_power" : "true",
            "uncore_cfg.dvfs_cfg.config" : "\"sample\"",
            "uncore_cfg.dvfs_cfg.interval" : "20000",
        }
        test_cfg = self.writeTestConfig(os.path.join(self.xio.GetTreeDir(),
                                                     "config", "A.cfg"),
                                        repl)
        self.xio.AddZestoOptions(test_cfg)

    def setUp(self):
        super(DFSTest, self).setUp()
        # Average freq ~901 MHz = 1600 * 2668273.0 / 4736546.0
        self.expected_vals.append((xs.PerfStatRE("c0.sim_cycle"), 2668273.0))
        self.expected_vals.append((xs.PerfStatRE("sim_cycle"), 4736546.0))
        # XXX: The dynamic number appears a little low, but that's more of a
        # validation issue, not a "hey, DFS is working" issue
        self.expected_vals.append((xs.PowerStatRE("  Runtime Dynamic"), 0.275))
        self.expected_vals.append((xs.PowerStatRE("  Total Leakage"), 0.48))

    def runTest(self):
        self.runAndValidate()

class Fib2Test(XIOSimTest):
    ''' End-to-end test with multiple programs running from start to end.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib", num_copies=2)
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib2Test, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 236483.00))

    def runTest(self):
        self.runAndValidate()

class REPTest(XIOSimTest):
    ''' End-to-end test for REP instructions.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("rep")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddROIOptions()
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "none.cfg"))

    def setUp(self):
        super(REPTest, self).setUp()
        # 917,504 = 3.5 * 256 * 1024 -- see rep.cpp
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 917504.0))

    def runTest(self):
        self.runAndValidate()

if __name__ == "__main__":
    unittest.main()
<commit_msg>Add segf to integration tests.<commit_after>#!/usr/bin/env python

import copy
import os
import shutil
import sys
import tempfile
import unittest

import xiosim_driver as xd
import xiosim_stat as xs

def CreateDriver():
    PIN_ROOT = os.environ["PIN_ROOT"]
    XIOSIM_INSTALL = os.environ["XIOSIM_INSTALL"]
    XIOSIM_TREE = os.environ["XIOSIM_TREE"]
    xio = xd.XIOSimDriver(PIN_ROOT, XIOSIM_INSTALL, XIOSIM_TREE)
    return xio

class XIOSimTest(unittest.TestCase):
    ''' Test fixtures for XIOSim end-to-end tests.'''
    def setUp(self):
        ''' Set up a driver with common options and a temp run directory.'''
        self.xio = CreateDriver()
        self.run_dir = tempfile.mkdtemp()
        self.clean_run_dir = ("LEAVE_TEST_DIR" not in os.environ)
        self.setDriverParams()
        self.expected_vals = []

    def tearDown(self):
        if self.clean_run_dir:
            shutil.rmtree(self.xio.GetRunDir())

    def runAndValidate(self):
        ''' Execute XIOSimDriver and check output for golden values of stats.'''
        out_file = os.path.join(self.run_dir, "harness.out")
        err_file = os.path.join(self.run_dir, "harness.err")
        ret = self.xio.Exec(stdout_file=out_file, stderr_file=err_file, cwd=self.run_dir)
        self.assertEqual(ret, 0, "XIOSim run failed (errcode %d)" % ret)

        for re, golden_val in self.expected_vals:
            val = xs.GetStat(self.xio.GetSimOut(), re)
            res = xs.ValidateStat(val, golden_val)
            self.assertEqual(res, True, "%s: expected %.2f, got %.2f" %
                                        (re, golden_val, val))

    def writeTestBmkConfig(self, bmk, num_copies=1):
        ''' Create a temp benchmark config file in the test run directory.'''
        cfg_file = self.xio.GenerateTestBmkConfig(bmk, num_copies)
        bmk_cfg = os.path.join(self.run_dir, "%s.cfg" % bmk)
        with open(bmk_cfg, "w") as f:
            for l in cfg_file:
                f.write(l)
        return bmk_cfg

    def writeTestConfig(self, start_config, changes):
        ''' Create a temp config file in the test run directory,
        starting with @start_config and applying a dictionary or
        paramter changes. '''
        cfg_file = os.path.join(self.run_dir, os.path.basename(start_config))
        cfg_contents = self.xio.GenerateConfigFile(start_config, changes)
        with open(cfg_file, "w") as f:
            for l in cfg_contents:
                f.write(l)
        return cfg_file

class Fib1Test(XIOSimTest):
    ''' End-to-end test with a single binary running from start to end.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1Test, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 118369.0))

    def runTest(self):
        self.runAndValidate()

class NoneTest(XIOSimTest):
    ''' End-to-end test with the 'none' core model.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "none.cfg"))

    def setUp(self):
        super(NoneTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 118369.0))

    def runTest(self):
        self.runAndValidate()

class Fib1LengthTest(XIOSimTest):
    ''' End-to-end test with a single binary running for X instructions.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddInstLength(10000)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1LengthTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 9900.0))

    def runTest(self):
        self.runAndValidate()

class Fib1SkipTest(XIOSimTest):
    ''' End-to-end test with a single binary skipping first X instructions.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddSkipInst(50000)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1SkipTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 68369.0))

    def runTest(self):
        self.runAndValidate()

class Fib1PinPointTest(XIOSimTest):
    ''' End-to-end test with a single binary and one pinpoint.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        ppfile = os.path.join(self.xio.GetTreeDir(), "tests",
                              "fib..pintool.1.pp")
        self.xio.AddPinPointFile(ppfile)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1PinPointTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 30000.0))

    def runTest(self):
        self.runAndValidate()

class Fib1PinPointsTest(XIOSimTest):
    ''' End-to-end test with a single binary and multiple pinpoints.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        ppfile = os.path.join(self.xio.GetTreeDir(), "tests",
                              "fib..pintool.2.pp")
        self.xio.AddPinPointFile(ppfile)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib1PinPointsTest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 10100.0))

    def runTest(self):
        self.runAndValidate()

class ROITest(XIOSimTest):
    ''' End-to-end test with a single binary with ROI hooks.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("roi")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddROIOptions()
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(ROITest, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 217000.0))

    def runTest(self):
        self.runAndValidate()

class ReplaceTest(XIOSimTest):
    ''' End-to-end test where we replace a function call with a NOP.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("repl")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddReplaceOptions("fib_repl")
        # 30K cycles for the magic instruction that replaces fib_repl()
        # 30K is high enough that the test can catch if we didn't add it,
        # yet low enough that it doesn't tickle the deadlock threshold (50K)
        repl = {
            "core_cfg.exec_cfg.exeu magic.latency" : "30000",
            "core_cfg.exec_cfg.exeu magic.rate" : "30000",
        }
        test_cfg = self.writeTestConfig(os.path.join(self.xio.GetTreeDir(),
                                                     "config", "N.cfg"),
                                        repl)
        self.xio.AddZestoOptions(test_cfg)

    def setUp(self):
        super(ReplaceTest, self).setUp()
        # repl is ~1M instructions
        # when we correctly ignore the middle call, we expect ~450K
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 447000.0))
        # repl when we just ignore the middle call takes ~397K cycles + 30K magic
        self.expected_vals.append((xs.PerfStatRE("sim_cycle"), 426000.0))

    def runTest(self):
        self.runAndValidate()

class PowerTest(XIOSimTest):
    ''' End-to-end test that calculates power.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)

        repl = {
            "system_cfg.simulate_power" : "true"
        }
        test_cfg = self.writeTestConfig(os.path.join(self.xio.GetTreeDir(),
                                                     "config", "A.cfg"),
                                        repl)
        self.xio.AddZestoOptions(test_cfg)

    def setUp(self):
        super(PowerTest, self).setUp()
        self.expected_vals.append((xs.PowerStatRE("  Runtime Dynamic"), 0.69))
        self.expected_vals.append((xs.PowerStatRE("  Total Leakage"), 0.48))

    def runTest(self):
        self.runAndValidate()

#TODO(skanev): Add a power trace test

class DFSTest(XIOSimTest):
    ''' End-to-end test that checks that "sample" DFS modifies frequencies.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("step")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)

        repl = {
            "system_cfg.simulate_power" : "true",
            "uncore_cfg.dvfs_cfg.config" : "\"sample\"",
            "uncore_cfg.dvfs_cfg.interval" : "20000",
        }
        test_cfg = self.writeTestConfig(os.path.join(self.xio.GetTreeDir(),
                                                     "config", "A.cfg"),
                                        repl)
        self.xio.AddZestoOptions(test_cfg)

    def setUp(self):
        super(DFSTest, self).setUp()
        # Average freq ~901 MHz = 1600 * 2668273.0 / 4736546.0
        self.expected_vals.append((xs.PerfStatRE("c0.sim_cycle"), 2668273.0))
        self.expected_vals.append((xs.PerfStatRE("sim_cycle"), 4736546.0))
        # XXX: The dynamic number appears a little low, but that's more of a
        # validation issue, not a "hey, DFS is working" issue
        self.expected_vals.append((xs.PowerStatRE("  Runtime Dynamic"), 0.275))
        self.expected_vals.append((xs.PowerStatRE("  Total Leakage"), 0.48))

    def runTest(self):
        self.runAndValidate()

class Fib2Test(XIOSimTest):
    ''' End-to-end test with multiple programs running from start to end.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("fib", num_copies=2)
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(Fib2Test, self).setUp()
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 236483.00))

    def runTest(self):
        self.runAndValidate()

class REPTest(XIOSimTest):
    ''' End-to-end test for REP instructions.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("rep")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddROIOptions()
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "none.cfg"))

    def setUp(self):
        super(REPTest, self).setUp()
        # 917,504 = 3.5 * 256 * 1024 -- see rep.cpp
        self.expected_vals.append((xs.PerfStatRE("all_insn"), 917504.0))

    def runTest(self):
        self.runAndValidate()

class SegfTest(XIOSimTest):
    ''' End-to-end test for faults on speculative path.'''
    def setDriverParams(self):
        bmk_cfg = self.writeTestBmkConfig("segf")
        self.xio.AddBmks(bmk_cfg)

        self.xio.AddPinOptions()
        self.xio.AddPintoolOptions(num_cores=1)
        self.xio.AddZestoOptions(os.path.join(self.xio.GetTreeDir(),
                                              "config", "N.cfg"))

    def setUp(self):
        super(SegfTest, self).setUp()

    def runTest(self):
        self.runAndValidate()

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals

import json

import frappe
from frappe import _, throw
from frappe.utils import add_days, cstr, date_diff, get_link_to_form, getdate
from frappe.utils.nestedset import NestedSet

class CircularReferenceError(frappe.ValidationError): pass
class EndDateCannotBeGreaterThanProjectEndDateError(frappe.ValidationError): pass

class Task(NestedSet):
	nsm_parent_field = 'parent_task'

	def get_feed(self):
		return '{0}: {1}'.format(_(self.status), self.subject)

	def get_project_details(self):
		return {
			"project": self.project
		}

	def get_customer_details(self):
		cust = frappe.db.sql("select customer_name from `tabCustomer` where name=%s", self.customer)
		if cust:
			ret = {'customer_name': cust and cust[0][0] or ''}
			return ret

	def validate(self):
		self.validate_dates()
		self.validate_progress()
		self.validate_status()
		self.update_depends_on()

	def validate_dates(self):
		if self.exp_start_date and self.exp_end_date and getdate(self.exp_start_date) > getdate(self.exp_end_date):
			frappe.throw(_("'Expected Start Date' can not be greater than 'Expected End Date'"))

		if self.act_start_date and self.act_end_date and getdate(self.act_start_date) > getdate(self.act_end_date):
			frappe.throw(_("'Actual Start Date' can not be greater than 'Actual End Date'"))

	def validate_status(self):
		if self.status!=self.get_db_value("status") and self.status == "Completed":
			for d in self.depends_on:
				if frappe.db.get_value("Task", d.task, "status") != "Completed":
					frappe.throw(_("Cannot close task {0} as its dependant task {1} is not closed.").format(frappe.bold(self.name), frappe.bold(d.task)))

			from frappe.desk.form.assign_to import clear
			clear(self.doctype, self.name)

	def validate_progress(self):
		if (self.progress or 0) > 100:
			frappe.throw(_("Progress % for a task cannot be more than 100."))

		if self.progress == 100:
			self.status = 'Completed'

		if self.status == 'Completed':
			self.progress = 100

	def update_depends_on(self):
		depends_on_tasks = self.depends_on_tasks or ""
		for d in self.depends_on:
			if d.task and not d.task in depends_on_tasks:
				depends_on_tasks += d.task + ","
		self.depends_on_tasks = depends_on_tasks

	def update_nsm_model(self):
		frappe.utils.nestedset.update_nsm(self)

	def on_update(self):
		self.update_nsm_model()
		self.check_recursion()
		self.reschedule_dependent_tasks()
		self.update_project()
		self.unassign_todo()
		self.populate_depends_on()

	def unassign_todo(self):
		if self.status in ("Completed", "Cancelled"):
			from frappe.desk.form.assign_to import clear
			clear(self.doctype, self.name)

	def update_total_expense_claim(self):
		self.total_expense_claim = frappe.db.sql("""select sum(total_sanctioned_amount) from `tabExpense Claim`
			where project = %s and task = %s and docstatus=1""",(self.project, self.name))[0][0]

	def update_time_and_costing(self):
		tl = frappe.db.sql("""select min(from_time) as start_date, max(to_time) as end_date,
			sum(billing_amount) as total_billing_amount, sum(costing_amount) as total_costing_amount,
			sum(hours) as time from `tabTimesheet Detail` where task = %s and docstatus=1"""
			,self.name, as_dict=1)[0]
		if self.status == "Open":
			self.status = "Working"
		self.total_costing_amount= tl.total_costing_amount
		self.total_billing_amount= tl.total_billing_amount
		self.actual_time= tl.time
		self.act_start_date= tl.start_date
		self.act_end_date= tl.end_date

	def update_project(self):
		if self.project and not self.flags.from_project:
			frappe.get_doc("Project", self.project).update_project()

	def check_recursion(self):
		if self.flags.ignore_recursion_check: return
		check_list = [['task', 'parent'], ['parent', 'task']]
		for d in check_list:
			task_list, count = [self.name], 0
			while (len(task_list) > count ):
				tasks = frappe.db.sql(" select %s from `tabTask Depends On` where %s = %s " %
					(d[0], d[1], '%s'), cstr(task_list[count]))
				count = count + 1
				for b in tasks:
					if b[0] == self.name:
						frappe.throw(_("Circular Reference Error"), CircularReferenceError)
					if b[0]:
						task_list.append(b[0])

				if count == 15:
					break

	def reschedule_dependent_tasks(self):
		end_date = self.exp_end_date or self.act_end_date
		if end_date:
			for task_name in frappe.db.sql("""
				select name from `tabTask` as parent
				where parent.project = %(project)s
					and parent.name in (
						select parent from `tabTask Depends On` as child
						where child.task = %(task)s and child.project = %(project)s)
			""", {'project': self.project, 'task':self.name }, as_dict=1):
				task = frappe.get_doc("Task", task_name.name)
				if task.exp_start_date and task.exp_end_date and task.exp_start_date < getdate(end_date) and task.status == "Open":
					task_duration = date_diff(task.exp_end_date, task.exp_start_date)
					task.exp_start_date = add_days(end_date, 1)
					task.exp_end_date = add_days(task.exp_start_date, task_duration)
					task.flags.ignore_recursion_check = True
					task.save()

	def has_webform_permission(self):
		project_user = frappe.db.get_value("Project User", {"parent": self.project, "user":frappe.session.user} , "user")
		if project_user:
			return True

	def populate_depends_on(self):
		if self.parent_task:
			parent = frappe.get_doc('Task', self.parent_task)
			if not self.name in [row.task for row in parent.depends_on]:
				parent.append("depends_on", {
					"doctype": "Task Depends On",
					"task": self.name,
					"subject": self.subject
				})
				parent.save()

	def on_trash(self):
		if check_if_child_exists(self.name):
			throw(_("Child Task exists for this Task. You can not delete this Task."))

		if self.project:
			tasks = (frappe.get_doc('Project', self.project)).tasks
			for task in tasks:
				if (task.get('task_id') == self.name):
					frappe.delete_doc('Project Task', task.name)

		self.update_nsm_model()

	def update_status(self):
		if self.status not in ('Cancelled', 'Completed') and self.exp_end_date:
			from datetime import datetime
			if self.exp_end_date < datetime.now().date():
				self.db_set('status', 'Overdue')
				self.update_project()

@frappe.whitelist()
def check_if_child_exists(name):
	child_tasks = frappe.get_all("Task", filters={"parent_task": name})
	child_tasks = [get_link_to_form("Task", task.name) for task in child_tasks]
	return child_tasks

def get_project(doctype, txt, searchfield, start, page_len, filters):
	from erpnext.controllers.queries import get_match_cond
	return frappe.db.sql(""" select name from `tabProject`
			where %(key)s like %(txt)s
				%(mcond)s
			order by name
			limit %(start)s, %(page_len)s""" % {
				'key': searchfield,
				'txt': frappe.db.escape('%' + txt + '%'),
				'mcond':get_match_cond(doctype),
				'start': start,
				'page_len': page_len
			})

@frappe.whitelist()
def set_multiple_status(names, status):
	names = json.loads(names)
	for name in names:
		task = frappe.get_doc("Task", name)
		task.status = status
		task.save()

def set_tasks_as_overdue():
	tasks = frappe.get_all("Task", filters={'status':['not in',['Cancelled', 'Completed']]})
	for task in tasks:
		frappe.get_doc("Task", task.name).update_status()

@frappe.whitelist()
def get_children(doctype, parent, task=None, project=None, is_root=False):

	filters = [['docstatus', '<', '2']]

	if task:
		filters.append(['parent_task', '=', task])
	elif parent and not is_root:
		# via expand child
		filters.append(['parent_task', '=', parent])
	else:
		filters.append(['ifnull(`parent_task`, "")', '=', ''])

	if project:
		filters.append(['project', '=', project])

	tasks = frappe.get_list(doctype, fields=[
		'name as value',
		'subject as title',
		'is_group as expandable'
	], filters=filters, order_by='name')

	# return tasks
	return tasks

@frappe.whitelist()
def add_node():
	from frappe.desk.treeview import make_tree_args
	args = frappe.form_dict
	args.update({
		"name_field": "subject"
	})
	args = make_tree_args(**args)

	if args.parent_task == 'All Tasks' or args.parent_task == args.project:
		args.parent_task = None

	frappe.get_doc(args).insert()

@frappe.whitelist()
def add_multiple_tasks(data, parent):
	data = json.loads(data)
	new_doc = {'doctype': 'Task', 'parent_task': parent if parent!="All Tasks" else ""}
	new_doc['project'] = frappe.db.get_value('Task', {"name": parent}, 'project') or ""

	for d in data:
		if not d.get("subject"): continue
		new_doc['subject'] = d.get("subject")
		new_task = frappe.get_doc(new_doc)
		new_task.insert()

def on_doctype_update():
	frappe.db.add_index("Task", ["lft", "rgt"])
<commit_msg>fix: Remove parens<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals

import json

import frappe
from frappe import _, throw
from frappe.utils import add_days, cstr, date_diff, get_link_to_form, getdate
from frappe.utils.nestedset import NestedSet

class CircularReferenceError(frappe.ValidationError): pass
class EndDateCannotBeGreaterThanProjectEndDateError(frappe.ValidationError): pass

class Task(NestedSet):
	nsm_parent_field = 'parent_task'

	def get_feed(self):
		return '{0}: {1}'.format(_(self.status), self.subject)

	def get_project_details(self):
		return {
			"project": self.project
		}

	def get_customer_details(self):
		cust = frappe.db.sql("select customer_name from `tabCustomer` where name=%s", self.customer)
		if cust:
			ret = {'customer_name': cust and cust[0][0] or ''}
			return ret

	def validate(self):
		self.validate_dates()
		self.validate_progress()
		self.validate_status()
		self.update_depends_on()

	def validate_dates(self):
		if self.exp_start_date and self.exp_end_date and getdate(self.exp_start_date) > getdate(self.exp_end_date):
			frappe.throw(_("'Expected Start Date' can not be greater than 'Expected End Date'"))

		if self.act_start_date and self.act_end_date and getdate(self.act_start_date) > getdate(self.act_end_date):
			frappe.throw(_("'Actual Start Date' can not be greater than 'Actual End Date'"))

	def validate_status(self):
		if self.status!=self.get_db_value("status") and self.status == "Completed":
			for d in self.depends_on:
				if frappe.db.get_value("Task", d.task, "status") != "Completed":
					frappe.throw(_("Cannot close task {0} as its dependant task {1} is not closed.").format(frappe.bold(self.name), frappe.bold(d.task)))

			from frappe.desk.form.assign_to import clear
			clear(self.doctype, self.name)

	def validate_progress(self):
		if (self.progress or 0) > 100:
			frappe.throw(_("Progress % for a task cannot be more than 100."))

		if self.progress == 100:
			self.status = 'Completed'

		if self.status == 'Completed':
			self.progress = 100

	def update_depends_on(self):
		depends_on_tasks = self.depends_on_tasks or ""
		for d in self.depends_on:
			if d.task and not d.task in depends_on_tasks:
				depends_on_tasks += d.task + ","
		self.depends_on_tasks = depends_on_tasks

	def update_nsm_model(self):
		frappe.utils.nestedset.update_nsm(self)

	def on_update(self):
		self.update_nsm_model()
		self.check_recursion()
		self.reschedule_dependent_tasks()
		self.update_project()
		self.unassign_todo()
		self.populate_depends_on()

	def unassign_todo(self):
		if self.status in ("Completed", "Cancelled"):
			from frappe.desk.form.assign_to import clear
			clear(self.doctype, self.name)

	def update_total_expense_claim(self):
		self.total_expense_claim = frappe.db.sql("""select sum(total_sanctioned_amount) from `tabExpense Claim`
			where project = %s and task = %s and docstatus=1""",(self.project, self.name))[0][0]

	def update_time_and_costing(self):
		tl = frappe.db.sql("""select min(from_time) as start_date, max(to_time) as end_date,
			sum(billing_amount) as total_billing_amount, sum(costing_amount) as total_costing_amount,
			sum(hours) as time from `tabTimesheet Detail` where task = %s and docstatus=1"""
			,self.name, as_dict=1)[0]
		if self.status == "Open":
			self.status = "Working"
		self.total_costing_amount= tl.total_costing_amount
		self.total_billing_amount= tl.total_billing_amount
		self.actual_time= tl.time
		self.act_start_date= tl.start_date
		self.act_end_date= tl.end_date

	def update_project(self):
		if self.project and not self.flags.from_project:
			frappe.get_doc("Project", self.project).update_project()

	def check_recursion(self):
		if self.flags.ignore_recursion_check: return
		check_list = [['task', 'parent'], ['parent', 'task']]
		for d in check_list:
			task_list, count = [self.name], 0
			while (len(task_list) > count ):
				tasks = frappe.db.sql(" select %s from `tabTask Depends On` where %s = %s " %
					(d[0], d[1], '%s'), cstr(task_list[count]))
				count = count + 1
				for b in tasks:
					if b[0] == self.name:
						frappe.throw(_("Circular Reference Error"), CircularReferenceError)
					if b[0]:
						task_list.append(b[0])

				if count == 15:
					break

	def reschedule_dependent_tasks(self):
		end_date = self.exp_end_date or self.act_end_date
		if end_date:
			for task_name in frappe.db.sql("""
				select name from `tabTask` as parent
				where parent.project = %(project)s
					and parent.name in (
						select parent from `tabTask Depends On` as child
						where child.task = %(task)s and child.project = %(project)s)
			""", {'project': self.project, 'task':self.name }, as_dict=1):
				task = frappe.get_doc("Task", task_name.name)
				if task.exp_start_date and task.exp_end_date and task.exp_start_date < getdate(end_date) and task.status == "Open":
					task_duration = date_diff(task.exp_end_date, task.exp_start_date)
					task.exp_start_date = add_days(end_date, 1)
					task.exp_end_date = add_days(task.exp_start_date, task_duration)
					task.flags.ignore_recursion_check = True
					task.save()

	def has_webform_permission(self):
		project_user = frappe.db.get_value("Project User", {"parent": self.project, "user":frappe.session.user} , "user")
		if project_user:
			return True

	def populate_depends_on(self):
		if self.parent_task:
			parent = frappe.get_doc('Task', self.parent_task)
			if not self.name in [row.task for row in parent.depends_on]:
				parent.append("depends_on", {
					"doctype": "Task Depends On",
					"task": self.name,
					"subject": self.subject
				})
				parent.save()

	def on_trash(self):
		if check_if_child_exists(self.name):
			throw(_("Child Task exists for this Task. You can not delete this Task."))

		if self.project:
			tasks = frappe.get_doc('Project', self.project).tasks
			for task in tasks:
				if task.get('task_id') == self.name:
					frappe.delete_doc('Project Task', task.name)

		self.update_nsm_model()

	def update_status(self):
		if self.status not in ('Cancelled', 'Completed') and self.exp_end_date:
			from datetime import datetime
			if self.exp_end_date < datetime.now().date():
				self.db_set('status', 'Overdue')
				self.update_project()

@frappe.whitelist()
def check_if_child_exists(name):
	child_tasks = frappe.get_all("Task", filters={"parent_task": name})
	child_tasks = [get_link_to_form("Task", task.name) for task in child_tasks]
	return child_tasks

def get_project(doctype, txt, searchfield, start, page_len, filters):
	from erpnext.controllers.queries import get_match_cond
	return frappe.db.sql(""" select name from `tabProject`
			where %(key)s like %(txt)s
				%(mcond)s
			order by name
			limit %(start)s, %(page_len)s""" % {
				'key': searchfield,
				'txt': frappe.db.escape('%' + txt + '%'),
				'mcond':get_match_cond(doctype),
				'start': start,
				'page_len': page_len
			})

@frappe.whitelist()
def set_multiple_status(names, status):
	names = json.loads(names)
	for name in names:
		task = frappe.get_doc("Task", name)
		task.status = status
		task.save()

def set_tasks_as_overdue():
	tasks = frappe.get_all("Task", filters={'status':['not in',['Cancelled', 'Completed']]})
	for task in tasks:
		frappe.get_doc("Task", task.name).update_status()

@frappe.whitelist()
def get_children(doctype, parent, task=None, project=None, is_root=False):

	filters = [['docstatus', '<', '2']]

	if task:
		filters.append(['parent_task', '=', task])
	elif parent and not is_root:
		# via expand child
		filters.append(['parent_task', '=', parent])
	else:
		filters.append(['ifnull(`parent_task`, "")', '=', ''])

	if project:
		filters.append(['project', '=', project])

	tasks = frappe.get_list(doctype, fields=[
		'name as value',
		'subject as title',
		'is_group as expandable'
	], filters=filters, order_by='name')

	# return tasks
	return tasks

@frappe.whitelist()
def add_node():
	from frappe.desk.treeview import make_tree_args
	args = frappe.form_dict
	args.update({
		"name_field": "subject"
	})
	args = make_tree_args(**args)

	if args.parent_task == 'All Tasks' or args.parent_task == args.project:
		args.parent_task = None

	frappe.get_doc(args).insert()

@frappe.whitelist()
def add_multiple_tasks(data, parent):
	data = json.loads(data)
	new_doc = {'doctype': 'Task', 'parent_task': parent if parent!="All Tasks" else ""}
	new_doc['project'] = frappe.db.get_value('Task', {"name": parent}, 'project') or ""

	for d in data:
		if not d.get("subject"): continue
		new_doc['subject'] = d.get("subject")
		new_task = frappe.get_doc(new_doc)
		new_task.insert()

def on_doctype_update():
	frappe.db.add_index("Task", ["lft", "rgt"])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
dotfiles.core
~~~~~~~~~~~~~

This module provides the basic functionality of dotfiles.
"""

import os
import shutil

__version__ = '0.2.0'
__author__ = "Jon Bernard"
__license__ = "GPL"

class Dotfile(object):

    def __init__(self, name, target):
        if name.startswith('/'):
            self.name = name
        else:
            self.name = os.path.expanduser('~/.%s' % name.strip('.'))
        self.basename = os.path.basename(self.name)
        self.target = target.rstrip('/')
        self.status = ''
        if not os.path.lexists(self.name):
            self.status = 'missing'
        elif os.path.realpath(self.name) != self.target:
            self.status = 'unmanaged'

    def sync(self, force):
        if self.status == 'missing':
            os.symlink(self.target, self.name)
        elif self.status == 'unmanaged':
            if not force:
                print "Skipping \"%s\", use --force to override" % self.basename
                return
            os.remove(self.name)
            os.symlink(self.target, self.name)

    def add(self):
        if self.status == 'missing':
            print "Skipping \"%s\", file not found" % self.basename
            return
        if self.status == '':
            print "Skipping \"%s\", already managed" % self.basename
            return
        shutil.move(self.name, self.target)
        os.symlink(self.target, self.name)

    def remove(self):
        if self.status != '':
            print "Skipping \"%s\", file is %s" % (self.basename, self.status)
            return
        os.remove(self.name)
        shutil.move(self.target, self.name)

    def __str__(self):
        return '%-18s %-s' % (self.name.split('/')[-1], self.status)

class Dotfiles(object):

    def __init__(self, location, prefix, ignore, externals, force):
        self.location = location
        self.prefix = prefix
        self.force = force
        self.dotfiles = []
        contents = [x for x in os.listdir(self.location) if x not in ignore]
        for file in contents:
            self.dotfiles.append(Dotfile(file[len(prefix):],
                os.path.join(self.location, file)))
        for file in externals.keys():
            self.dotfiles.append(Dotfile(file, externals[file]))

    def list(self, **kwargs):
        for dotfile in sorted(self.dotfiles, key=lambda dotfile: dotfile.name):
            if dotfile.status or kwargs.get('verbose', True):
                print dotfile

    def check(self, **kwargs):
        self.list(verbose=False)

    def sync(self, **kwargs):
        for dotfile in self.dotfiles:
            dotfile.sync(self.force)

    def add(self, **kwargs):
        for file in kwargs.get('files', None):
            if os.path.basename(file).startswith('.'):
                Dotfile(file,
                        os.path.join(self.location,
                            self.prefix + os.path.basename(file).strip('.'))).add()
            else:
                print "Skipping \"%s\", not a dotfile" % file

    def remove(self, **kwargs):
        for file in kwargs.get('files', None):
            if os.path.basename(file).startswith('.'):
                Dotfile(file,
                        os.path.join(self.location,
                            self.prefix + os.path.basename(file).strip('.'))).remove()
            else:
                print "Skipping \"%s\", not a dotfile" % file
<commit_msg>Bump version number for version 0.3.0 release<commit_after># -*- coding: utf-8 -*-

"""
dotfiles.core
~~~~~~~~~~~~~

This module provides the basic functionality of dotfiles.
"""

import os
import shutil

__version__ = '0.3.0'
__author__ = "Jon Bernard"
__license__ = "GPL"

class Dotfile(object):

    def __init__(self, name, target):
        if name.startswith('/'):
            self.name = name
        else:
            self.name = os.path.expanduser('~/.%s' % name.strip('.'))
        self.basename = os.path.basename(self.name)
        self.target = target.rstrip('/')
        self.status = ''
        if not os.path.lexists(self.name):
            self.status = 'missing'
        elif os.path.realpath(self.name) != self.target:
            self.status = 'unmanaged'

    def sync(self, force):
        if self.status == 'missing':
            os.symlink(self.target, self.name)
        elif self.status == 'unmanaged':
            if not force:
                print "Skipping \"%s\", use --force to override" % self.basename
                return
            os.remove(self.name)
            os.symlink(self.target, self.name)

    def add(self):
        if self.status == 'missing':
            print "Skipping \"%s\", file not found" % self.basename
            return
        if self.status == '':
            print "Skipping \"%s\", already managed" % self.basename
            return
        shutil.move(self.name, self.target)
        os.symlink(self.target, self.name)

    def remove(self):
        if self.status != '':
            print "Skipping \"%s\", file is %s" % (self.basename, self.status)
            return
        os.remove(self.name)
        shutil.move(self.target, self.name)

    def __str__(self):
        return '%-18s %-s' % (self.name.split('/')[-1], self.status)

class Dotfiles(object):

    def __init__(self, location, prefix, ignore, externals, force):
        self.location = location
        self.prefix = prefix
        self.force = force
        self.dotfiles = []
        contents = [x for x in os.listdir(self.location) if x not in ignore]
        for file in contents:
            self.dotfiles.append(Dotfile(file[len(prefix):],
                os.path.join(self.location, file)))
        for file in externals.keys():
            self.dotfiles.append(Dotfile(file, externals[file]))

    def list(self, **kwargs):
        for dotfile in sorted(self.dotfiles, key=lambda dotfile: dotfile.name):
            if dotfile.status or kwargs.get('verbose', True):
                print dotfile

    def check(self, **kwargs):
        self.list(verbose=False)

    def sync(self, **kwargs):
        for dotfile in self.dotfiles:
            dotfile.sync(self.force)

    def add(self, **kwargs):
        for file in kwargs.get('files', None):
            if os.path.basename(file).startswith('.'):
                Dotfile(file,
                        os.path.join(self.location,
                            self.prefix + os.path.basename(file).strip('.'))).add()
            else:
                print "Skipping \"%s\", not a dotfile" % file

    def remove(self, **kwargs):
        for file in kwargs.get('files', None):
            if os.path.basename(file).startswith('.'):
                Dotfile(file,
                        os.path.join(self.location,
                            self.prefix + os.path.basename(file).strip('.'))).remove()
            else:
                print "Skipping \"%s\", not a dotfile" % file
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import subprocess
import os.path as op
import yaml

def main(target_dataset):
    """Download dataset from OpenNeuro or OSF.

    Parameters
    ----------
    target_dataset : {'preprocessed', 'partially', 'fully'}
        Which dataset to download. See project README for more info.

    """
    with open(op.join(op.dirname(op.realpath(__file__)), 'config.yml')) as f:
        config = yaml.safe_load(f)
    deriv_folder = op.join(config['DATA_DIR'], 'derivatives')
    print(f"Using {config['DATA_DIR']} as data root directory.")
    if target_dataset == 'fully':
        print("Downloading fully-processed data.")
        subprocess.call(["curl", "-O", "-J", "-L", "https://osf.io/djak4/download"])
        subprocess.call(["tar", "xf", "sfp_fully_processed_data.tar.gz"])
        subprocess.call(["rsync", "-avPLuz", "derivatives/", f"{deriv_folder}/"])
        subprocess.call(["rm", "-r", "derivatives/"])
        subprocess.call(["rm", "sfp_fully_processed_data.tar.gz"])
    elif target_dataset == 'extra':
        print("Downloading fully-processed data.")
        subprocess.call(["curl", "-O", "-J", "-L", "https://osf.io/b3cdj/download"])
        subprocess.call(["tar", "xf", "sfp_extra_data.tar.gz"])
        subprocess.call(["rsync", "-avPLuz", "derivatives/", f"{deriv_folder}/"])
        subprocess.call(["rm", "-r", "derivatives/"])
        subprocess.call(["rm", "sfp_extra_data.tar.gz"])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=("Download the spatial frequency preferences dataset, to reproduce the results.")
    )
    parser.add_argument("target_dataset", choices=['preprocessed', 'partially', 'fully', 'extra'],
                        help="Which dataset to download, see project README for details.")
    args = vars(parser.parse_args())
    main(**args)
<commit_msg>copies stimuli<commit_after>#!/usr/bin/env python3

import argparse
import subprocess
import os.path as op
import yaml

def main(target_dataset):
    """Download dataset from OpenNeuro or OSF.

    Parameters
    ----------
    target_dataset : {'preprocessed', 'partially', 'fully'}
        Which dataset to download. See project README for more info.

    """
    with open(op.join(op.dirname(op.realpath(__file__)), 'config.yml')) as f:
        config = yaml.safe_load(f)
    deriv_folder = op.join(config['DATA_DIR'], 'derivatives')
    stim_folder = op.join(config['DATA_DIR'], 'stimuli')
    print(f"Using {config['DATA_DIR']} as data root directory.")
    if target_dataset == 'fully':
        print("Downloading fully-processed data.")
        subprocess.call(["curl", "-O", "-J", "-L", "https://osf.io/djak4/download"])
        subprocess.call(["tar", "xf", "sfp_fully_processed_data.tar.gz"])
        subprocess.call(["rsync", "-avPLuz", "derivatives/", f"{deriv_folder}/"])
        subprocess.call(["rm", "-r", "derivatives/"])
        subprocess.call(["rm", "sfp_fully_processed_data.tar.gz"])
    elif target_dataset == 'extra':
        print("Downloading fully-processed data.")
        subprocess.call(["curl", "-O", "-J", "-L", "https://osf.io/b3cdj/download"])
        subprocess.call(["tar", "xf", "sfp_extra_data.tar.gz"])
        subprocess.call(["rsync", "-avPLuz", "derivatives/", f"{deriv_folder}/"])
        subprocess.call(["rsync", "-avPLuz", "stimuli/", f"{stim_folder}/"])
        subprocess.call(["rm", "-r", "derivatives/"])
        subprocess.call(["rm", "sfp_extra_data.tar.gz"])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=("Download the spatial frequency preferences dataset, to reproduce the results.")
    )
    parser.add_argument("target_dataset", choices=['preprocessed', 'partially', 'fully', 'extra'],
                        help="Which dataset to download, see project README for details.")
    args = vars(parser.parse_args())
    main(**args)
<|endoftext|>


<commit_before>import json

import jsonschema

import eva_cttv_pipeline.config as config
import eva_cttv_pipeline.utilities as utilities
from eva_cttv_pipeline import efo_term

__author__ = 'Javier Lopez: javild@gmail.com'

utilities.check_for_local_schema()
GEN_SCHEMA_FILE = utilities.get_resource_file(__package__, config.local_schema + "/src/genetics.json")
SOM_SCHEMA_FILE = utilities.get_resource_file(__package__, config.local_schema + "/src/literature_curated.json")

class CTTVEvidenceString(dict):
    def __init__(self, a_dictionary):
        dict.__init__(self, a_dictionary)

    def add_unique_association_field(self, key, value):
        self['unique_association_fields'][key] = value

    def set_target(self, id, activity):
        self['target']['id'].append(id)
        self['target']['activity'] = activity

    @property
    def disease(self):
        return efo_term.EFOTerm(self['disease']['id'][0])

    @disease.setter
    def disease(self, id):
        self['disease']['id'].append(id)

    @property
    def evidence_codes(self):
        return self['evidence']['evidence_codes']

    @evidence_codes.setter
    def evidence_codes(self, ev_code_list):
        self['evidence']['evidence_codes'] = ev_code_list

    @property
    def top_level_literature(self):
        return self['literature']['references']

    @top_level_literature.setter
    def top_level_literature(self, reference_list):
        self['literature'] = {'references': [{'lit_id': reference} for reference in reference_list]}

class CTTVGeneticsEvidenceString(CTTVEvidenceString):
    schema = json.loads(open(GEN_SCHEMA_FILE, 'r').read())

    def __init__(self):

        CTTVEvidenceString.__init__(self,
                                    {'sourceID': 'eva',
                                     'validated_against_schema_version': '1.2.2',
                                     'type': 'genetic_association',
                                     'access_level': 'public',
                                     'unique_association_fields': {},
                                     'target': {
                                         'id': [],  # ENSEMBL ID
                                         'target_type': 'http://identifiers.org/cttv.target/gene_variant',
                                         'activity': None  # http://identifiers.org/cttv.activity/predicted_damaging
                                     },
                                     'variant': {
                                         'id': [],  # rs
                                         'type': None  # snp single
                                     },
                                     'evidence': {
                                         'gene2variant': {
                                             'provenance_type': {
                                                 'expert': {
                                                     'status': True,
                                                     'statement': 'Primary submitter of data'
                                                 },
                                                 'database': {
                                                     'dbxref': {
                                                         'id': 'http://identifiers.org/clinvar',
                                                         'url': None,
                                                         'version': '2015-04',
                                                     },
                                                     'id': 'EVA',
                                                     'version': '1.0'

                                                 }
                                             },
                                             'is_associated': True,
                                             'date_asserted': None,
                                             'evidence_codes': [],
                                             'functional_consequence': None,
                                             'urls': [
                                                 {
                                                     'nice_name': 'Further details in ClinVar database',
                                                     'url': None  # http://www.ncbi.nlm.nih.gov/clinvar/RCV000038347
                                                 }
                                             ]
                                         },
                                         'variant2disease': {
                                             'resource_score': {
                                                 'type': 'pvalue',
                                                 'value': 0.0000001,
                                                 'method': {
                                                     'url': '',
                                                     'description': 'Not provided by data supplier'
                                                 }
                                             },
                                             'provenance_type': {
                                                 'expert': {
                                                     'status': True,
                                                     'statement': 'Primary submitter of data'
                                                 },
                                                 'database': {
                                                     'dbxref': {
                                                         'id': 'http://identifiers.org/clinvar',
                                                         'url': None,
                                                         'version': '2015-04',
                                                     },
                                                     'id': 'EVA',
                                                     'version': '1.0'

                                                 }
                                             },
                                             'unique_experiment_reference': 'http://europepmc.org/abstract/MED/0',
                                             'is_associated': True,
                                             'date_asserted': None,
                                             'evidence_codes': [
                                                 'http://purl.obolibrary.org/obo/ECO_0000205'
                                             ],
                                             'urls': [
                                                 {
                                                     'nice_name': 'Further details in ClinVar database',
                                                     'url': None  # http://www.ncbi.nlm.nih.gov/clinvar/RCV000038347
                                                 }
                                             ]
                                         },
                                     },
                                     'disease': {
                                         'id': [],  # EFO terms
                                     }
                                     }
                                    )

    @property
    def db_xref_url(self):
        if self['evidence']['gene2variant']['provenance_type']['database']['dbxref']['url'] \
                == self['evidence']['variant2disease']['provenance_type']['database']['dbxref']['url']:
            return self['evidence']['variant2disease']['provenance_type']['database']['dbxref']['url']
        else:
            raise Exception("db_xref_url attributes different")

    @db_xref_url.setter
    def db_xref_url(self, url):
        self['evidence']['gene2variant']['provenance_type']['database']['dbxref']['url'] = url
        self['evidence']['variant2disease']['provenance_type']['database']['dbxref']['url'] = url

    @property
    def url(self):
        if self['evidence']['gene2variant']['urls'][0]['url'] \
                == self['evidence']['variant2disease']['urls'][0]['url']:
            return self['evidence']['gene2variant']['urls'][0]['url']
        else:
            raise Exception("url attributes different")

    @url.setter
    def url(self, url):
        self['evidence']['gene2variant']['urls'][0]['url'] = url
        self['evidence']['variant2disease']['urls'][0]['url'] = url

    @property
    def gene_2_var_ev_codes(self):
        return self['evidence']['gene2variant']['evidence_codes']

    @gene_2_var_ev_codes.setter
    def gene_2_var_ev_codes(self, gene_2_var_ev_codes):
        self['evidence']['gene2variant']['evidence_codes'] = gene_2_var_ev_codes

    @property
    def gene_2_var_func_consequence(self):
        return self['evidence']['gene2variant']['functional_consequence']

    @gene_2_var_func_consequence.setter
    def gene_2_var_func_consequence(self, so_term):
        self['evidence']['gene2variant']['functional_consequence'] = so_term

    def set_var_2_disease_literature(self, ref_list):
        self['evidence']['variant2disease']['provenance_type']['literature'] = {'references': [{'lit_id': reference} for reference in ref_list]}

    @property
    def association(self):
        if self['evidence']['gene2variant']['is_associated'] \
                == self['evidence']['variant2disease']['is_associated']:
            return self['evidence']['gene2variant']['is_associated']
        else:
            raise Exception("association attributes different")

    @association.setter
    def association(self, is_associated):
        self['evidence']['gene2variant']['is_associated'] = is_associated
        self['evidence']['variant2disease']['is_associated'] = is_associated

    def validate(self):
        # try:
        jsonschema.validate(self, CTTVGeneticsEvidenceString.schema, format_checker=jsonschema.FormatChecker())
        # except Exception as e:
        #     print(str(self))
        #     print(e)
        #     traceback.print_stack()
        #     sys.exit()
        self.disease.is_obsolete()

    def set_variant(self, id, type):
        self['variant']['id'].append(id)
        self['variant']['type'] = type

    @property
    def unique_reference(self):
        return self['evidence']['variant2disease']['unique_experiment_reference']

    @unique_reference.setter
    def unique_reference(self, reference):
        self['evidence']['variant2disease']['unique_experiment_reference'] = reference

    @property
    def date(self):
        if self['evidence']['gene2variant']['date_asserted'] \
                == self['evidence']['variant2disease']['date_asserted']:
            return self['evidence']['gene2variant']['date_asserted']
        else:
            raise Exception("date attributes have different values")

    @date.setter
    def date(self, date_string):
        self['evidence']['gene2variant']['date_asserted'] = date_string
        self['evidence']['variant2disease']['date_asserted'] = date_string

class CTTVSomaticEvidenceString(CTTVEvidenceString):
    schema = json.loads(open(SOM_SCHEMA_FILE, 'r').read())

    def __init__(self):

        CTTVEvidenceString.__init__(self,
                                    {'sourceID': 'eva_somatic',
                                     'validated_against_schema_version': '1.2.2',
                                     'type': 'somatic_mutation',
                                     'access_level': 'public',
                                     'unique_association_fields': {},
                                     'target': {
                                         'id': [],  # ENSEMBL ID
                                         'target_type': 'http://identifiers.org/cttv.target/gene_variant',
                                         'activity': None  # http://identifiers.org/cttv.activity/predicted_damaging
                                     },
                                     'evidence': {
                                         'resource_score': {
                                             'type': 'probability',
                                             'value': 1
                                         },
                                         'provenance_type': {
                                             'expert': {
                                                 'status': True,
                                                 'statement': 'Primary submitter of data'
                                             },
                                             'database': {
                                                 'dbxref': {
                                                     'id': 'http://identifiers.org/clinvar',
                                                     'url': None,
                                                     'version': '2015-04',
                                                 },
                                                 'id': 'EVA',
                                                 'version': '1.0'

                                             }
                                         },
                                         'is_associated': True,
                                         'date_asserted': None,
                                         'evidence_codes': [
                                             'http://purl.obolibrary.org/obo/ECO_0000205'
                                         ],
                                         'known_mutations': [
                                         ],
                                         'urls': [
                                             {
                                                 'nice_name': 'Further details in ClinVar database',
                                                 'url': None  # http://www.ncbi.nlm.nih.gov/clinvar/RCV000038347
                                             }
                                         ]
                                     },

                                     'disease': {
                                         'id': [],  # EFO terms
                                     }
                                     }
                                    )

    @property
    def db_xref_url(self):
        return self['evidence']['provenance_type']['database']['dbxref']['url']

    @db_xref_url.setter
    def db_xref_url(self, url):
        self['evidence']['provenance_type']['database']['dbxref']['url'] = url

    @property
    def url(self):
        return self['evidence']['urls'][0]['url']

    @url.setter
    def url(self, url):
        self['evidence']['urls'][0]['url'] = url

    @property
    def evidence_literature(self):
        return self['evidence']['provenance_type']['literature']['references']

    @evidence_literature.setter
    def evidence_literature(self, ref_list):
        self['evidence']['provenance_type']['literature'] = {'references': [{'lit_id': reference} for reference in ref_list]}

    @property
    def association(self):
        return self['evidence']['is_associated']

    @association.setter
    def association(self, is_associated):
        self['evidence']['is_associated'] = is_associated

    def validate(self):
        jsonschema.validate(self, CTTVSomaticEvidenceString.schema, format_checker=jsonschema.FormatChecker())
        self.disease.is_obsolete()

    @property
    def date(self):
        return self['evidence']['date_asserted']

    @date.setter
    def date(self, date_string):
        self['evidence']['date_asserted'] = date_string

    def add_known_mutation(self, new_functional_consequence, so_name):
        new_known_mutation = {'functional_consequence': new_functional_consequence, 'preferred_name': so_name}
        self['evidence']['known_mutations'].append(new_known_mutation)

    def set_known_mutations(self, consequence_type):
        for so_term in consequence_type.so_terms:
            so_name = so_term.so_name
            if so_term.accession:
                new_functional_consequence = "http://purl.obolibrary.org/obo/" + so_term.accession.replace(':', '_')
            else:
                new_functional_consequence = 'http://targetvalidation.org/sequence/' + so_term.so_name
            self.add_known_mutation(new_functional_consequence, so_name)
<commit_msg>Changed calls to super classes in evidence_strings.<commit_after>import json

import jsonschema

import eva_cttv_pipeline.config as config
import eva_cttv_pipeline.utilities as utilities
from eva_cttv_pipeline import efo_term

__author__ = 'Javier Lopez: javild@gmail.com'

utilities.check_for_local_schema()
GEN_SCHEMA_FILE = utilities.get_resource_file(__package__, config.local_schema + "/src/genetics.json")
SOM_SCHEMA_FILE = utilities.get_resource_file(__package__, config.local_schema + "/src/literature_curated.json")

class CTTVEvidenceString(dict):
    def __init__(self, a_dictionary):
        super(CTTVEvidenceString, self).__init__(a_dictionary)
        # dict.__init__(a_dictionary)

    def add_unique_association_field(self, key, value):
        self['unique_association_fields'][key] = value

    def set_target(self, id, activity):
        self['target']['id'].append(id)
        self['target']['activity'] = activity

    @property
    def disease(self):
        return efo_term.EFOTerm(self['disease']['id'][0])

    @disease.setter
    def disease(self, id):
        self['disease']['id'].append(id)

    @property
    def evidence_codes(self):
        return self['evidence']['evidence_codes']

    @evidence_codes.setter
    def evidence_codes(self, ev_code_list):
        self['evidence']['evidence_codes'] = ev_code_list

    @property
    def top_level_literature(self):
        return self['literature']['references']

    @top_level_literature.setter
    def top_level_literature(self, reference_list):
        self['literature'] = {'references': [{'lit_id': reference} for reference in reference_list]}

class CTTVGeneticsEvidenceString(CTTVEvidenceString):
    schema = json.loads(open(GEN_SCHEMA_FILE, 'r').read())

    def __init__(self):

        a_dictionary = {'sourceID': 'eva',
                                     'validated_against_schema_version': '1.2.2',
                                     'type': 'genetic_association',
                                     'access_level': 'public',
                                     'unique_association_fields': {},
                                     'target': {
                                         'id': [],  # ENSEMBL ID
                                         'target_type': 'http://identifiers.org/cttv.target/gene_variant',
                                         'activity': None  # http://identifiers.org/cttv.activity/predicted_damaging
                                     },
                                     'variant': {
                                         'id': [],  # rs
                                         'type': None  # snp single
                                     },
                                     'evidence': {
                                         'gene2variant': {
                                             'provenance_type': {
                                                 'expert': {
                                                     'status': True,
                                                     'statement': 'Primary submitter of data'
                                                 },
                                                 'database': {
                                                     'dbxref': {
                                                         'id': 'http://identifiers.org/clinvar',
                                                         'url': None,
                                                         'version': '2015-04',
                                                     },
                                                     'id': 'EVA',
                                                     'version': '1.0'

                                                 }
                                             },
                                             'is_associated': True,
                                             'date_asserted': None,
                                             'evidence_codes': [],
                                             'functional_consequence': None,
                                             'urls': [
                                                 {
                                                     'nice_name': 'Further details in ClinVar database',
                                                     'url': None  # http://www.ncbi.nlm.nih.gov/clinvar/RCV000038347
                                                 }
                                             ]
                                         },
                                         'variant2disease': {
                                             'resource_score': {
                                                 'type': 'pvalue',
                                                 'value': 0.0000001,
                                                 'method': {
                                                     'url': '',
                                                     'description': 'Not provided by data supplier'
                                                 }
                                             },
                                             'provenance_type': {
                                                 'expert': {
                                                     'status': True,
                                                     'statement': 'Primary submitter of data'
                                                 },
                                                 'database': {
                                                     'dbxref': {
                                                         'id': 'http://identifiers.org/clinvar',
                                                         'url': None,
                                                         'version': '2015-04',
                                                     },
                                                     'id': 'EVA',
                                                     'version': '1.0'

                                                 }
                                             },
                                             'unique_experiment_reference': 'http://europepmc.org/abstract/MED/0',
                                             'is_associated': True,
                                             'date_asserted': None,
                                             'evidence_codes': [
                                                 'http://purl.obolibrary.org/obo/ECO_0000205'
                                             ],
                                             'urls': [
                                                 {
                                                     'nice_name': 'Further details in ClinVar database',
                                                     'url': None  # http://www.ncbi.nlm.nih.gov/clinvar/RCV000038347
                                                 }
                                             ]
                                         },
                                     },
                                     'disease': {
                                         'id': [],  # EFO terms
                                     }
                                     }

        # CTTVEvidenceString.__init__(self, a_dictionary)
        super(CTTVGeneticsEvidenceString, self).__init__(a_dictionary)

    @property
    def db_xref_url(self):
        if self['evidence']['gene2variant']['provenance_type']['database']['dbxref']['url'] \
                == self['evidence']['variant2disease']['provenance_type']['database']['dbxref']['url']:
            return self['evidence']['variant2disease']['provenance_type']['database']['dbxref']['url']
        else:
            raise Exception("db_xref_url attributes different")

    @db_xref_url.setter
    def db_xref_url(self, url):
        self['evidence']['gene2variant']['provenance_type']['database']['dbxref']['url'] = url
        self['evidence']['variant2disease']['provenance_type']['database']['dbxref']['url'] = url

    @property
    def url(self):
        if self['evidence']['gene2variant']['urls'][0]['url'] \
                == self['evidence']['variant2disease']['urls'][0]['url']:
            return self['evidence']['gene2variant']['urls'][0]['url']
        else:
            raise Exception("url attributes different")

    @url.setter
    def url(self, url):
        self['evidence']['gene2variant']['urls'][0]['url'] = url
        self['evidence']['variant2disease']['urls'][0]['url'] = url

    @property
    def gene_2_var_ev_codes(self):
        return self['evidence']['gene2variant']['evidence_codes']

    @gene_2_var_ev_codes.setter
    def gene_2_var_ev_codes(self, gene_2_var_ev_codes):
        self['evidence']['gene2variant']['evidence_codes'] = gene_2_var_ev_codes

    @property
    def gene_2_var_func_consequence(self):
        return self['evidence']['gene2variant']['functional_consequence']

    @gene_2_var_func_consequence.setter
    def gene_2_var_func_consequence(self, so_term):
        self['evidence']['gene2variant']['functional_consequence'] = so_term

    def set_var_2_disease_literature(self, ref_list):
        self['evidence']['variant2disease']['provenance_type']['literature'] = {'references': [{'lit_id': reference} for reference in ref_list]}

    @property
    def association(self):
        if self['evidence']['gene2variant']['is_associated'] \
                == self['evidence']['variant2disease']['is_associated']:
            return self['evidence']['gene2variant']['is_associated']
        else:
            raise Exception("association attributes different")

    @association.setter
    def association(self, is_associated):
        self['evidence']['gene2variant']['is_associated'] = is_associated
        self['evidence']['variant2disease']['is_associated'] = is_associated

    def validate(self):
        # try:
        jsonschema.validate(self, CTTVGeneticsEvidenceString.schema, format_checker=jsonschema.FormatChecker())
        # except Exception as e:
        #     print(str(self))
        #     print(e)
        #     traceback.print_stack()
        #     sys.exit()
        self.disease.is_obsolete()

    def set_variant(self, id, type):
        self['variant']['id'].append(id)
        self['variant']['type'] = type

    @property
    def unique_reference(self):
        return self['evidence']['variant2disease']['unique_experiment_reference']

    @unique_reference.setter
    def unique_reference(self, reference):
        self['evidence']['variant2disease']['unique_experiment_reference'] = reference

    @property
    def date(self):
        if self['evidence']['gene2variant']['date_asserted'] \
                == self['evidence']['variant2disease']['date_asserted']:
            return self['evidence']['gene2variant']['date_asserted']
        else:
            raise Exception("date attributes have different values")

    @date.setter
    def date(self, date_string):
        self['evidence']['gene2variant']['date_asserted'] = date_string
        self['evidence']['variant2disease']['date_asserted'] = date_string

class CTTVSomaticEvidenceString(CTTVEvidenceString):
    schema = json.loads(open(SOM_SCHEMA_FILE, 'r').read())

    def __init__(self):
        a_dictionary = {'sourceID': 'eva_somatic',
                                     'validated_against_schema_version': '1.2.2',
                                     'type': 'somatic_mutation',
                                     'access_level': 'public',
                                     'unique_association_fields': {},
                                     'target': {
                                         'id': [],  # ENSEMBL ID
                                         'target_type': 'http://identifiers.org/cttv.target/gene_variant',
                                         'activity': None  # http://identifiers.org/cttv.activity/predicted_damaging
                                     },
                                     'evidence': {
                                         'resource_score': {
                                             'type': 'probability',
                                             'value': 1
                                         },
                                         'provenance_type': {
                                             'expert': {
                                                 'status': True,
                                                 'statement': 'Primary submitter of data'
                                             },
                                             'database': {
                                                 'dbxref': {
                                                     'id': 'http://identifiers.org/clinvar',
                                                     'url': None,
                                                     'version': '2015-04',
                                                 },
                                                 'id': 'EVA',
                                                 'version': '1.0'

                                             }
                                         },
                                         'is_associated': True,
                                         'date_asserted': None,
                                         'evidence_codes': [
                                             'http://purl.obolibrary.org/obo/ECO_0000205'
                                         ],
                                         'known_mutations': [
                                         ],
                                         'urls': [
                                             {
                                                 'nice_name': 'Further details in ClinVar database',
                                                 'url': None  # http://www.ncbi.nlm.nih.gov/clinvar/RCV000038347
                                             }
                                         ]
                                     },

                                     'disease': {
                                         'id': [],  # EFO terms
                                     }
                                     }
        # CTTVEvidenceString.__init__(self,a_dictionary)

        super(CTTVSomaticEvidenceString, self).__init__(a_dictionary)

    @property
    def db_xref_url(self):
        return self['evidence']['provenance_type']['database']['dbxref']['url']

    @db_xref_url.setter
    def db_xref_url(self, url):
        self['evidence']['provenance_type']['database']['dbxref']['url'] = url

    @property
    def url(self):
        return self['evidence']['urls'][0]['url']

    @url.setter
    def url(self, url):
        self['evidence']['urls'][0]['url'] = url

    @property
    def evidence_literature(self):
        return self['evidence']['provenance_type']['literature']['references']

    @evidence_literature.setter
    def evidence_literature(self, ref_list):
        self['evidence']['provenance_type']['literature'] = {'references': [{'lit_id': reference} for reference in ref_list]}

    @property
    def association(self):
        return self['evidence']['is_associated']

    @association.setter
    def association(self, is_associated):
        self['evidence']['is_associated'] = is_associated

    def validate(self):
        jsonschema.validate(self, CTTVSomaticEvidenceString.schema, format_checker=jsonschema.FormatChecker())
        self.disease.is_obsolete()

    @property
    def date(self):
        return self['evidence']['date_asserted']

    @date.setter
    def date(self, date_string):
        self['evidence']['date_asserted'] = date_string

    def add_known_mutation(self, new_functional_consequence, so_name):
        new_known_mutation = {'functional_consequence': new_functional_consequence, 'preferred_name': so_name}
        self['evidence']['known_mutations'].append(new_known_mutation)

    def set_known_mutations(self, consequence_type):
        for so_term in consequence_type.so_terms:
            so_name = so_term.so_name
            if so_term.accession:
                new_functional_consequence = "http://purl.obolibrary.org/obo/" + so_term.accession.replace(':', '_')
            else:
                new_functional_consequence = 'http://targetvalidation.org/sequence/' + so_term.so_name
            self.add_known_mutation(new_functional_consequence, so_name)
<|endoftext|>


<commit_before>import csv
import json
import copy
from global_config import config
import math

def total_person_hours(project):
    '''
    Calculates the total estimated and actual person hours of effort expended in a project `project'.
    '''
    total_estimated = project.total_estimated_effort
    total_actual = project.actual_budget() / config["developer_hourly_cost"]
    return (int(total_estimated), int(total_actual))

def report_table_line(team, module, size, estimate, actual, cost, wall, productive):
    '''
    Formats a line of the endgame report dump.
    '''
    s = ""
    s += team + (" " * (15 - (len(team))))
    s += module + (" " * (13 - len(module)))
    s += str(size) + (" " * (6 - (len(str(size)))))
    s += str(estimate) + (" " * (16 - (len(str(estimate)))))
    s += str(actual) + (" " * (15 - (len(str(actual)))))
    s += str(cost) + (" " * (10 - (len(str(cost)))))
    s += str(wall) + (" " * (13 - len(str(wall))))
    s += str(productive)
    return s

def generate_report(project):
    '''
    Generates endgame report for the project `project'.
    '''
    report = {}
    report["score"] = project.game_score()
    report["total_time"] = str(project.current_time - project.start_time)
    report["nominal_end_time"] = str(project.delivery_date)
    report["actual_end_time"] = str(project.current_time)
    report["days_behind_schedule"] = project.days_behind_schedule()
    report["expected_budget"] = project.expected_budget()
    report["actual_budget"] = project.actual_budget()
    report["expected_revenue"] = project.expected_revenue()
    report["actual_revenue"] = project.actual_revenue()
    report["endgame_cash"] = project.game_score()

    # Generate table to compare estimated/actual effort broken down by module
    effort_table = []
    effort_table.append(['Team', 'Module', 'Team', 'Expected Effort', 'Actual Effort  ', 'Module', 'Wall clock', 'Staff Time'])
    effort_table.append(['Name', 'Name',   'Size', '(man hrs)',       '(man hrs)  ',     'Cost $', 'time (hrs)', 'time (hrs)'])
    total_estimated = 0
    total_actual = 0
    for location in project.locations:
        for team in location.teams:
            for module in team.completed_modules:
                expected = int(module.expected_cost)
                actual = int(module.actual_cost)
                wall = module.wall_clock_time()
                productive = module.productive_time_on_task() * team.size
                dollars = int(productive * config["developer_hourly_cost"] )
                effort_table.append([team.name, module.name, team.size, expected, actual, dollars, wall, productive])
                total_estimated += expected
                total_actual += actual
    # Add totals row
    # effort_table.append(["Total", "Total", total_estimated, total_actual])

    report["effort_table"] = effort_table

    return report

def write_endgame_json(report):
    '''
    Writes the endgame report, given a report `report'

    @untestable - This function is just a call to the python standard library and thus makes no sense to test.
    '''
    end_report = copy.deepcopy(report)
    table = end_report["effort_table"]
    del end_report["effort_table"]
    outfile = open('report.txt', 'w')
    outfile.write(json.dumps(end_report, indent=4))
    outfile.write('\n')

    writer = csv.writer(outfile, delimiter=',')
    writer.writerows(table)

<commit_msg>file format for report reverted to pure json<commit_after>import json
import copy
from global_config import config

def total_person_hours(project):
    '''
    Calculates the total estimated and actual person hours of effort expended in a project `project'.
    '''
    total_estimated = project.total_estimated_effort
    total_actual = project.actual_budget() / config["developer_hourly_cost"]
    return (int(total_estimated), int(total_actual))

def report_table_line(team, module, size, estimate, actual, cost, wall, productive):
    '''
    Formats a line of the endgame report dump.
    '''
    s = ""
    s += team + (" " * (15 - (len(team))))
    s += module + (" " * (13 - len(module)))
    s += str(size) + (" " * (6 - (len(str(size)))))
    s += str(estimate) + (" " * (16 - (len(str(estimate)))))
    s += str(actual) + (" " * (15 - (len(str(actual)))))
    s += str(cost) + (" " * (10 - (len(str(cost)))))
    s += str(wall) + (" " * (13 - len(str(wall))))
    s += str(productive)
    return s

def generate_report(project):
    '''
    Generates endgame report for the project `project'.
    '''
    report = {}
    report["score"] = project.game_score()
    report["total_time"] = str(project.current_time - project.start_time)
    report["nominal_end_time"] = str(project.delivery_date)
    report["actual_end_time"] = str(project.current_time)
    report["days_behind_schedule"] = project.days_behind_schedule()
    report["expected_budget"] = project.expected_budget()
    report["actual_budget"] = project.actual_budget()
    report["expected_revenue"] = project.expected_revenue()
    report["actual_revenue"] = project.actual_revenue()
    report["endgame_cash"] = project.game_score()

    # Generate table to compare estimated/actual effort broken down by module
    effort_table = []
    effort_table.append(['Team', 'Module', 'Team', 'Expected Effort', 'Actual Effort  ', 'Module', 'Wall clock', 'Staff Time'])
    effort_table.append(['Name', 'Name',   'Size', '(man hrs)',       '(man hrs)  ',     'Cost $', 'time (hrs)', 'time (hrs)'])
    total_estimated = 0
    total_actual = 0
    for location in project.locations:
        for team in location.teams:
            for module in team.completed_modules:
                expected = int(module.expected_cost)
                actual = int(module.actual_cost)
                wall = module.wall_clock_time()
                productive = module.productive_time_on_task() * team.size
                dollars = int(productive * config["developer_hourly_cost"] )
                effort_table.append([team.name, module.name, team.size, expected, actual, dollars, wall, productive])
                total_estimated += expected
                total_actual += actual
    # Add totals row
    # effort_table.append(["Total", "Total", total_estimated, total_actual])

    report["effort_table"] = effort_table

    return report

def write_endgame_json(report):
    '''
    Writes the endgame report, given a report `report'

    @untestable - This function is just a call to the python standard library and thus makes no sense to test.
    '''

    outfile = open('report.json', 'w')
    outfile.write(json.dumps(report, indent=4))

<|endoftext|>