<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
'''
Created on 24 Feb 2010

@author: filo
'''
from nipype.interfaces.base import BaseInterface,\
    traits, TraitedSpec, File, InputMultiPath, OutputMultiPath
from nipype.utils.misc import isdefined
import nipype.externals.pynifti as nifti
import numpy as np
from math import floor, ceil
from scipy.ndimage.morphology import grey_dilation
import os
from nipype.utils.filemanip import fname_presuffix, split_filename

class PickAtlasInputSpec(TraitedSpec):
    atlas = File(exists=True, desc="Location of the atlas that will be used.", compulsory=True)
    labels = traits.Either(traits.Int, traits.List(traits.Int),
                           desc="Labels of regions that will be included in the mask. Must be \
                                compatible with the atlas used.", compulsory=True)
    hemi = traits.Enum('both','left','right', desc="Restrict the mask to only one hemisphere: left or right", usedefault=True)
    dilation_size = traits.Int(desc="Defines how much the mask will be dilated (expanded in 3D).", usedefault = True)
    output_file = File(desc="Where to store the output mask.")

class PickAtlasOutputSpec(TraitedSpec):
    mask_file = File(exists=True, desc="output mask file")

class PickAtlas(BaseInterface):
    '''
    Returns ROI masks given an atlas and a list of labels. Supports dilation
    and left right masking (assuming the atlas is properly aligned).
    '''
    input_spec = PickAtlasInputSpec
    output_spec = PickAtlasOutputSpec

    def _run_interface(self, runtime):
        nim = self._get_brodmann_area()
        nifti.save(nim, self._gen_output_filename())

        runtime.returncode = 0
        return runtime

    def _gen_output_filename(self):
        if not isdefined(self.inputs.output_file):
            output = fname_presuffix(fname=self.inputs.atlas, suffix = "_mask",
                                     newpath= os.getcwd(), use_ext = True)
        else:
            output = self.inputs.output_file
        return output

    def _get_brodmann_area(self):
        nii = nifti.load(self.inputs.atlas)
        origdata = nii.get_data()
        newdata = np.zeros(origdata.shape)

        if not isinstance(self.inputs.labels, list):
            labels = [self.inputs.labels]
        else:
            labels = self.inputs.labels
        for label in labels:
            newdata[origdata == label] = 1
        if self.inputs.hemi == 'right':
            newdata[floor(float(origdata.shape[0]) / 2):, :, :] = 0
        elif self.inputs.hemi == 'left':
            newdata[:ceil(float(origdata.shape[0]) / 2), :, : ] = 0

        if self.inputs.dilation_size != 0:
            newdata = grey_dilation(newdata , (2 * self.inputs.dilation_size + 1,
                                               2 * self.inputs.dilation_size + 1,
                                               2 * self.inputs.dilation_size + 1))

        return nifti.Nifti1Image(newdata, nii.get_affine(), nii.get_header())

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['mask_file'] = self._gen_output_filename()
        return outputs

class SimpleThresholdInputSpec(TraitedSpec):
    volumes = InputMultiPath(File(exists=True), desc='volumes to be thresholded', mandatory=True)
    threshold = traits.Float(desc='volumes to be thresholdedeverything below this value will be set to zero', mandatory=True)

class SimpleThresholdOutputSpec(TraitedSpec):
    thresholded_volumes = OutputMultiPath(File(exists=True), desc="thresholded volumes")

class SimpleThreshold(BaseInterface):
    input_spec = SimpleThresholdInputSpec
    output_spec = SimpleThresholdOutputSpec

    def _run_interface(self, runtime):
        for fname in self.inputs.volumes:
            img = nifti.load(fname)
            data = np.array(img.get_data())

            active_map = data > self.inputs.threshold

            thresholded_map = np.zeros(data.shape)
            thresholded_map[active_map] = data[active_map]

            new_img = nifti.Nifti1Image(thresholded_map, img.get_affine(), img.get_header())
            _, base, _ = split_filename(fname)
            nifti.save(new_img, base + '_thresholded.nii')

        runtime.returncode=0
        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs["thresholded_volumes"] = []
        for fname in self.inputs.volumes:
            _, base, _ = split_filename(fname)
            outputs["thresholded_volumes"].append(base + '_thresholded.nii')
        return outputs

<commit_msg>indentation fixes<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
'''
Created on 24 Feb 2010

@author: filo
'''
from nipype.interfaces.base import BaseInterface,\
    traits, TraitedSpec, File, InputMultiPath, OutputMultiPath
from nipype.utils.misc import isdefined
import nipype.externals.pynifti as nifti
import numpy as np
from math import floor, ceil
from scipy.ndimage.morphology import grey_dilation
import os
from nipype.utils.filemanip import fname_presuffix, split_filename

class PickAtlasInputSpec(TraitedSpec):
    atlas = File(exists=True, desc="Location of the atlas that will be used.", compulsory=True)
    labels = traits.Either(traits.Int, traits.List(traits.Int),
                           desc="Labels of regions that will be included in the mask. Must be \
compatible with the atlas used.", compulsory=True)
    hemi = traits.Enum('both','left','right', desc="Restrict the mask to only one hemisphere: left or right", usedefault=True)
    dilation_size = traits.Int(desc="Defines how much the mask will be dilated (expanded in 3D).", usedefault = True)
    output_file = File(desc="Where to store the output mask.")

class PickAtlasOutputSpec(TraitedSpec):
    mask_file = File(exists=True, desc="output mask file")

class PickAtlas(BaseInterface):
    '''
    Returns ROI masks given an atlas and a list of labels. Supports dilation
    and left right masking (assuming the atlas is properly aligned).
    '''
    input_spec = PickAtlasInputSpec
    output_spec = PickAtlasOutputSpec

    def _run_interface(self, runtime):
        nim = self._get_brodmann_area()
        nifti.save(nim, self._gen_output_filename())

        runtime.returncode = 0
        return runtime

    def _gen_output_filename(self):
        if not isdefined(self.inputs.output_file):
            output = fname_presuffix(fname=self.inputs.atlas, suffix = "_mask",
                                     newpath= os.getcwd(), use_ext = True)
        else:
            output = self.inputs.output_file
        return output

    def _get_brodmann_area(self):
        nii = nifti.load(self.inputs.atlas)
        origdata = nii.get_data()
        newdata = np.zeros(origdata.shape)

        if not isinstance(self.inputs.labels, list):
            labels = [self.inputs.labels]
        else:
            labels = self.inputs.labels
        for label in labels:
            newdata[origdata == label] = 1
        if self.inputs.hemi == 'right':
            newdata[floor(float(origdata.shape[0]) / 2):, :, :] = 0
        elif self.inputs.hemi == 'left':
            newdata[:ceil(float(origdata.shape[0]) / 2), :, : ] = 0

        if self.inputs.dilation_size != 0:
            newdata = grey_dilation(newdata , (2 * self.inputs.dilation_size + 1,
                                               2 * self.inputs.dilation_size + 1,
                                               2 * self.inputs.dilation_size + 1))

        return nifti.Nifti1Image(newdata, nii.get_affine(), nii.get_header())

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs['mask_file'] = self._gen_output_filename()
        return outputs

class SimpleThresholdInputSpec(TraitedSpec):
    volumes = InputMultiPath(File(exists=True), desc='volumes to be thresholded', mandatory=True)
    threshold = traits.Float(desc='volumes to be thresholdedeverything below this value will be set to zero', mandatory=True)

class SimpleThresholdOutputSpec(TraitedSpec):
    thresholded_volumes = OutputMultiPath(File(exists=True), desc="thresholded volumes")

class SimpleThreshold(BaseInterface):
    input_spec = SimpleThresholdInputSpec
    output_spec = SimpleThresholdOutputSpec

    def _run_interface(self, runtime):
        for fname in self.inputs.volumes:
            img = nifti.load(fname)
            data = np.array(img.get_data())

            active_map = data > self.inputs.threshold

            thresholded_map = np.zeros(data.shape)
            thresholded_map[active_map] = data[active_map]

            new_img = nifti.Nifti1Image(thresholded_map, img.get_affine(), img.get_header())
            _, base, _ = split_filename(fname)
            nifti.save(new_img, base + '_thresholded.nii')

        runtime.returncode=0
        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        outputs["thresholded_volumes"] = []
        for fname in self.inputs.volumes:
            _, base, _ = split_filename(fname)
            outputs["thresholded_volumes"].append(base + '_thresholded.nii')
        return outputs

<|endoftext|>


<commit_before># Copyright 2017 Eficent Business and IT Consulting Services, S.L.
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).

{
    "name": "Stock Request Purchase",
    "summary": "Internal request for stock",
    "version": "12.0.1.1.0",
    "license": "LGPL-3",
    "website": "https://github.com/stock-logistics-warehouse",
    "author": "Eficent, "
              "Odoo Community Association (OCA)",
    "category": "Warehouse Management",
    "depends": [
        "stock_request",
        "purchase_stock",
    ],
    "data": [
        "security/ir.model.access.csv",
        "views/stock_request_views.xml",
        "views/stock_request_order_views.xml",
        "views/purchase_order_views.xml",
    ],
    "installable": True,
    'auto_install': True,
}
<commit_msg>stock_request_purchase 12.0.1.1.1<commit_after># Copyright 2017 Eficent Business and IT Consulting Services, S.L.
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).

{
    "name": "Stock Request Purchase",
    "summary": "Internal request for stock",
    "version": "12.0.1.1.1",
    "license": "LGPL-3",
    "website": "https://github.com/stock-logistics-warehouse",
    "author": "Eficent, "
              "Odoo Community Association (OCA)",
    "category": "Warehouse Management",
    "depends": [
        "stock_request",
        "purchase_stock",
    ],
    "data": [
        "security/ir.model.access.csv",
        "views/stock_request_views.xml",
        "views/stock_request_order_views.xml",
        "views/purchase_order_views.xml",
    ],
    "installable": True,
    'auto_install': True,
}
<|endoftext|>


<commit_before>"""About and help services. (help browser anyone?)"""

import os

import pkg_resources
from gi.repository import GdkPixbuf
from gi.repository import Gtk

from gaphor import __version__
from gaphor.core import _, inject, action, build_action_group
from gaphor.abc import Service, ActionProvider

class HelpService(Service, ActionProvider):

    menu_xml = """
      <ui>
        <menubar name="mainwindow">
          <menu action="help">
            <placeholder name="ternary">
              <menuitem action="help-about" />
            </placeholder>
          </menu>
        </menubar>
      </ui>
    """

    main_window = inject("main_window")

    def __init__(self):
        self.action_group = build_action_group(self)

    def init(self, app):
        pass

    def shutdown(self):
        pass

    @action(name="help-about", stock_id="gtk-about")
    def about(self):
        logo_file = os.path.join(
            pkg_resources.get_distribution("gaphor").location,
            "gaphor",
            "ui",
            "pixmaps",
            "logo.png",
        )
        logo = GdkPixbuf.Pixbuf.new_from_file(logo_file)
        version = __version__
        about = Gtk.Dialog.new()
        about.set_title(_("About Gaphor"))
        about.set_modal(True)
        about.set_transient_for(self.main_window.window)
        about.add_buttons(Gtk.STOCK_OK, Gtk.ResponseType.OK)
        about.set_default_response(Gtk.ResponseType.OK)
        vbox = about.vbox

        image = Gtk.Image()
        image.set_from_pixbuf(logo)
        vbox.pack_start(image, True, True, 0)

        notebook = Gtk.Notebook()
        notebook.set_scrollable(True)
        notebook.set_border_width(4)
        notebook.set_tab_pos(Gtk.PositionType.BOTTOM)
        vbox.pack_start(notebook, True, True, 0)

        tab_vbox = Gtk.VBox()

        def add_label(text, padding_x=0, padding_y=0):
            label = Gtk.Label(label=text)
            label.set_property("use-markup", True)
            label.set_padding(padding_x, padding_y)
            label.set_justify(Gtk.Justification.CENTER)
            tab_vbox.pack_start(label, True, True, 0)

        add_label('Version %s' % version)
        add_label(
            'Gaphor is the simple modeling tool written in Python',
            8,
            8,
        )
        add_label(
            'Copyright (c) 2001-2019 Arjan J. Molenaar and Dan Yeaw',
            8,
            8,
        )

        notebook.append_page(tab_vbox, Gtk.Label(label=_("About")))

        tab_vbox = Gtk.VBox()

        add_label(
            "This software is published\n"
            "under the terms of the\n"
            'Apache Software License v2.\n'
            "See the LICENSE.txt file for details.",
            0,
            8,
        )
        notebook.append_page(tab_vbox, Gtk.Label(label=_("License")))

        tab_vbox = Gtk.VBox()

        add_label(
            "Thanks to all the wonderful people that have contributed:\n\n"
            "Arjan Molenaar\n"
            "Artur Wroblewski\n"
            "Jeroen Vloothuis\n"
            "Dan Yeaw\n"
            "Enno Groeper\n"
            "Adam Boduch\n"
            "Jordi Mallach\n"
            "Ygor Mutti\n"
            "Alexis Howells\n"
            "Encolpe Degoute\n"
            "Melis Dogan"
        )
        add_label("")
        notebook.append_page(tab_vbox, Gtk.Label(label=_("Contributors")))

        vbox.show_all()
        about.run()
        about.destroy()
<commit_msg>Use new DI for HelpService<commit_after>"""About and help services. (help browser anyone?)"""

import os

import pkg_resources
from gi.repository import GdkPixbuf
from gi.repository import Gtk

from gaphor import __version__
from gaphor.core import _, inject, action, build_action_group
from gaphor.abc import Service, ActionProvider

class HelpService(Service, ActionProvider):

    menu_xml = """
      <ui>
        <menubar name="mainwindow">
          <menu action="help">
            <placeholder name="ternary">
              <menuitem action="help-about" />
            </placeholder>
          </menu>
        </menubar>
      </ui>
    """

    def __init__(self, main_window):
        self.main_window = main_window
        self.action_group = build_action_group(self)

    def init(self, app):
        pass

    def shutdown(self):
        pass

    @action(name="help-about", stock_id="gtk-about")
    def about(self):
        logo_file = os.path.join(
            pkg_resources.get_distribution("gaphor").location,
            "gaphor",
            "ui",
            "pixmaps",
            "logo.png",
        )
        logo = GdkPixbuf.Pixbuf.new_from_file(logo_file)
        version = __version__
        about = Gtk.Dialog.new()
        about.set_title(_("About Gaphor"))
        about.set_modal(True)
        about.set_transient_for(self.main_window.window)
        about.add_buttons(Gtk.STOCK_OK, Gtk.ResponseType.OK)
        about.set_default_response(Gtk.ResponseType.OK)
        vbox = about.vbox

        image = Gtk.Image()
        image.set_from_pixbuf(logo)
        vbox.pack_start(image, True, True, 0)

        notebook = Gtk.Notebook()
        notebook.set_scrollable(True)
        notebook.set_border_width(4)
        notebook.set_tab_pos(Gtk.PositionType.BOTTOM)
        vbox.pack_start(notebook, True, True, 0)

        tab_vbox = Gtk.VBox()

        def add_label(text, padding_x=0, padding_y=0):
            label = Gtk.Label(label=text)
            label.set_property("use-markup", True)
            label.set_padding(padding_x, padding_y)
            label.set_justify(Gtk.Justification.CENTER)
            tab_vbox.pack_start(label, True, True, 0)

        add_label('Version %s' % version)
        add_label(
            'Gaphor is the simple modeling tool written in Python',
            8,
            8,
        )
        add_label(
            'Copyright (c) 2001-2019 Arjan J. Molenaar and Dan Yeaw',
            8,
            8,
        )

        notebook.append_page(tab_vbox, Gtk.Label(label=_("About")))

        tab_vbox = Gtk.VBox()

        add_label(
            "This software is published\n"
            "under the terms of the\n"
            'Apache Software License v2.\n'
            "See the LICENSE.txt file for details.",
            0,
            8,
        )
        notebook.append_page(tab_vbox, Gtk.Label(label=_("License")))

        tab_vbox = Gtk.VBox()

        add_label(
            "Thanks to all the wonderful people that have contributed:\n\n"
            "Arjan Molenaar\n"
            "Artur Wroblewski\n"
            "Jeroen Vloothuis\n"
            "Dan Yeaw\n"
            "Enno Groeper\n"
            "Adam Boduch\n"
            "Jordi Mallach\n"
            "Ygor Mutti\n"
            "Alexis Howells\n"
            "Encolpe Degoute\n"
            "Melis Dogan"
        )
        add_label("")
        notebook.append_page(tab_vbox, Gtk.Label(label=_("Contributors")))

        vbox.show_all()
        about.run()
        about.destroy()
<|endoftext|>


<commit_before>def test_make_science_data():

    import numpy as np
    import pickle

    def gaussian(amp, fwhm, mean):
        return lambda x: amp * np.exp(-(x - mean) ** 2 / 4.0 / (fwhm / 2.355) ** 2)

    # Data properties
    RMS = 0.05
    NCOMPS = 3
    NCHANNELS = 512
    NSPECTRA = 10
    TRAINING_SET = False
    FILENAME = "agd_data_science.pickle"

    # Component properties
    AMP_lims = [0.5, 4.0]
    FWHM_lims = [20, 80]  # channels
    MEAN_lims = [0.25 * NCHANNELS, 0.75 * NCHANNELS]  # channels

    # Initialize
    agd_data = {}
    chan = np.arange(NCHANNELS)
    errors = chan * 0.0 + RMS  # Constant noise for all spectra

    # Begin populating data
    for i in range(NSPECTRA):
        spectrum_i = np.random.randn(NCHANNELS) * RMS

        amps = []
        fwhms = []
        means = []

        for comp in range(NCOMPS):
            # Select random values for components within specified ranges
            a = np.random.uniform(AMP_lims[0], AMP_lims[1])
            w = np.random.uniform(FWHM_lims[0], FWHM_lims[1])
            m = np.random.uniform(MEAN_lims[0], MEAN_lims[1])

            # Add Gaussian profile with the above random parameters to the spectrum
            spectrum_i += gaussian(a, w, m)(chan)

            # Append the parameters to initialized lists for storing
            amps.append(a)
            fwhms.append(w)
            means.append(m)

        # Enter results into AGD dataset
        agd_data["data_list"] = agd_data.get("data_list", []) + [spectrum_i]
        agd_data["x_values"] = agd_data.get("x_values", []) + [chan]
        agd_data["errors"] = agd_data.get("errors", []) + [errors]

        # If training data, keep answers
        if TRAINING_SET:
            agd_data["amplitudes"] = agd_data.get("amplitudes", []) + [amps]
            agd_data["fwhms"] = agd_data.get("fwhms", []) + [fwhms]
            agd_data["means"] = agd_data.get("means", []) + [means]

    pickle.dump(agd_data, open(FILENAME, "wb"))
    print("Created: ", FILENAME)

def test_make_train_data():

    import numpy as np
    import pickle

    def gaussian(amp, fwhm, mean):
        return lambda x: amp * np.exp(-(x - mean) ** 2 / 4.0 / (fwhm / 2.355) ** 2)

    # Data properties
    RMS = 0.05
    NCOMPS = 3
    NCHANNELS = 512
    NSPECTRA = 30
    TRAINING_SET = True
    FILENAME = "agd_data_train.pickle"

    # Component properties
    AMP_lims = [0.5, 4.0]
    FWHM_lims = [20, 80]  # channels
    MEAN_lims = [0.25 * NCHANNELS, 0.75 * NCHANNELS]  # channels

    # Initialize
    agd_data = {}
    chan = np.arange(NCHANNELS)
    errors = np.ones(NCHANNELS) * RMS

    # Begin populating data
    for i in range(NSPECTRA):
        spectrum_i = np.random.randn(NCHANNELS) * RMS

        # Sample random components:
        amps = np.random.rand(NCOMPS) * (AMP_lims[1] - AMP_lims[0]) + AMP_lims[0]
        fwhms = np.random.rand(NCOMPS) * (FWHM_lims[1] - FWHM_lims[0]) + FWHM_lims[0]
        means = np.random.rand(NCOMPS) * (MEAN_lims[1] - MEAN_lims[0]) + MEAN_lims[0]

        # Create spectrum
        for a, w, m in zip(amps, fwhms, means):
            spectrum_i += gaussian(a, w, m)(chan)

        # Enter results into AGD dataset
        agd_data["data_list"] = agd_data.get("data_list", []) + [spectrum_i]
        agd_data["x_values"] = agd_data.get("x_values", []) + [chan]
        agd_data["errors"] = agd_data.get("errors", []) + [errors]

        # If training data, keep answers
        if TRAINING_SET:
            agd_data["amplitudes"] = agd_data.get("amplitudes", []) + [amps]
            agd_data["fwhms"] = agd_data.get("fwhms", []) + [fwhms]
            agd_data["means"] = agd_data.get("means", []) + [means]

    pickle.dump(agd_data, open(FILENAME, "wb"))
    print("Created: ", FILENAME)

def test_train():

    import numpy as np
    import gausspy.gp as gp

    imp.reload(gp)

    TRAINING_DATA = "agd_data_train.pickle"

    g = gp.GaussianDecomposer()
    g.load_training_data(TRAINING_DATA)

    # One phase training
    g.set("phase", "one")
    g.set("SNR_thresh", 5.0)

    g.train(alpha1_initial=2.0)

    # Two phase training
    # g.set('phase', 'two')
    # g.set('SNR_thresh', 5.)
    # g.set('SNR2_thresh', 5.)

    # g.train(alpha1_initial = 1., alpha2_initial = 3.)

def test_decompose():

    import numpy as np
    import gausspy.gp as gp
    import time
    import pickle

    imp.reload(gp)

    SCIENCE_DATA = "agd_data_science.pickle"

    g = gp.GaussianDecomposer()

    # Two phase
    g.set("phase", "two")
    g.set("SNR_thresh", 5.0)
    g.set("SNR2_thresh", 5.0)
    g.set("alpha1", 1.02)
    g.set("alpha2", 2.22)

    t0 = time.time()
    new_data = g.batch_decomposition(SCIENCE_DATA)
    print("Elapsed time [s]: ", int(time.time() - t0))

    pickle.dump(new_data, open("agd_data_science_decomposed.pickle", "wb"))

def test_remove_files():

    import os

    files_to_remove = [
        "agd_data_science.pickle",
        "agd_data_train.pickle",
        "agd_data_science_decomposed.pickle",
        "batchdecomp_temp.pickle",
    ]

    for file_to_remove in files_to_remove:
        os.system("rm -rf " + file_to_remove)
<commit_msg>Fixed imports<commit_after>import imp

def test_make_science_data():

    import numpy as np
    import pickle

    def gaussian(amp, fwhm, mean):
        return lambda x: amp * np.exp(-(x - mean) ** 2 / 4.0 / (fwhm / 2.355) ** 2)

    # Data properties
    RMS = 0.05
    NCOMPS = 3
    NCHANNELS = 512
    NSPECTRA = 10
    TRAINING_SET = False
    FILENAME = "agd_data_science.pickle"

    # Component properties
    AMP_lims = [0.5, 4.0]
    FWHM_lims = [20, 80]  # channels
    MEAN_lims = [0.25 * NCHANNELS, 0.75 * NCHANNELS]  # channels

    # Initialize
    agd_data = {}
    chan = np.arange(NCHANNELS)
    errors = chan * 0.0 + RMS  # Constant noise for all spectra

    # Begin populating data
    for i in range(NSPECTRA):
        spectrum_i = np.random.randn(NCHANNELS) * RMS

        amps = []
        fwhms = []
        means = []

        for comp in range(NCOMPS):
            # Select random values for components within specified ranges
            a = np.random.uniform(AMP_lims[0], AMP_lims[1])
            w = np.random.uniform(FWHM_lims[0], FWHM_lims[1])
            m = np.random.uniform(MEAN_lims[0], MEAN_lims[1])

            # Add Gaussian profile with the above random parameters to the spectrum
            spectrum_i += gaussian(a, w, m)(chan)

            # Append the parameters to initialized lists for storing
            amps.append(a)
            fwhms.append(w)
            means.append(m)

        # Enter results into AGD dataset
        agd_data["data_list"] = agd_data.get("data_list", []) + [spectrum_i]
        agd_data["x_values"] = agd_data.get("x_values", []) + [chan]
        agd_data["errors"] = agd_data.get("errors", []) + [errors]

        # If training data, keep answers
        if TRAINING_SET:
            agd_data["amplitudes"] = agd_data.get("amplitudes", []) + [amps]
            agd_data["fwhms"] = agd_data.get("fwhms", []) + [fwhms]
            agd_data["means"] = agd_data.get("means", []) + [means]

    pickle.dump(agd_data, open(FILENAME, "wb"))
    print("Created: ", FILENAME)

def test_make_train_data():

    import numpy as np
    import pickle

    def gaussian(amp, fwhm, mean):
        return lambda x: amp * np.exp(-(x - mean) ** 2 / 4.0 / (fwhm / 2.355) ** 2)

    # Data properties
    RMS = 0.05
    NCOMPS = 3
    NCHANNELS = 512
    NSPECTRA = 30
    TRAINING_SET = True
    FILENAME = "agd_data_train.pickle"

    # Component properties
    AMP_lims = [0.5, 4.0]
    FWHM_lims = [20, 80]  # channels
    MEAN_lims = [0.25 * NCHANNELS, 0.75 * NCHANNELS]  # channels

    # Initialize
    agd_data = {}
    chan = np.arange(NCHANNELS)
    errors = np.ones(NCHANNELS) * RMS

    # Begin populating data
    for i in range(NSPECTRA):
        spectrum_i = np.random.randn(NCHANNELS) * RMS

        # Sample random components:
        amps = np.random.rand(NCOMPS) * (AMP_lims[1] - AMP_lims[0]) + AMP_lims[0]
        fwhms = np.random.rand(NCOMPS) * (FWHM_lims[1] - FWHM_lims[0]) + FWHM_lims[0]
        means = np.random.rand(NCOMPS) * (MEAN_lims[1] - MEAN_lims[0]) + MEAN_lims[0]

        # Create spectrum
        for a, w, m in zip(amps, fwhms, means):
            spectrum_i += gaussian(a, w, m)(chan)

        # Enter results into AGD dataset
        agd_data["data_list"] = agd_data.get("data_list", []) + [spectrum_i]
        agd_data["x_values"] = agd_data.get("x_values", []) + [chan]
        agd_data["errors"] = agd_data.get("errors", []) + [errors]

        # If training data, keep answers
        if TRAINING_SET:
            agd_data["amplitudes"] = agd_data.get("amplitudes", []) + [amps]
            agd_data["fwhms"] = agd_data.get("fwhms", []) + [fwhms]
            agd_data["means"] = agd_data.get("means", []) + [means]

    pickle.dump(agd_data, open(FILENAME, "wb"))
    print("Created: ", FILENAME)

def test_train():

    import gausspy.gp as gp

    imp.reload(gp)

    TRAINING_DATA = "agd_data_train.pickle"

    g = gp.GaussianDecomposer()
    g.load_training_data(TRAINING_DATA)

    # One phase training
    g.set("phase", "one")
    g.set("SNR_thresh", 5.0)

    g.train(alpha1_initial=2.0)

    # Two phase training
    # g.set('phase', 'two')
    # g.set('SNR_thresh', 5.)
    # g.set('SNR2_thresh', 5.)

    # g.train(alpha1_initial = 1., alpha2_initial = 3.)

def test_decompose():

    import gausspy.gp as gp
    import time
    import pickle

    imp.reload(gp)

    SCIENCE_DATA = "agd_data_science.pickle"

    g = gp.GaussianDecomposer()

    # Two phase
    g.set("phase", "two")
    g.set("SNR_thresh", 5.0)
    g.set("SNR2_thresh", 5.0)
    g.set("alpha1", 1.02)
    g.set("alpha2", 2.22)

    t0 = time.time()
    new_data = g.batch_decomposition(SCIENCE_DATA)
    print("Elapsed time [s]: ", int(time.time() - t0))

    pickle.dump(new_data, open("agd_data_science_decomposed.pickle", "wb"))

def test_remove_files():

    import os

    files_to_remove = [
        "agd_data_science.pickle",
        "agd_data_train.pickle",
        "agd_data_science_decomposed.pickle",
        "batchdecomp_temp.pickle",
    ]

    for file_to_remove in files_to_remove:
        os.system("rm -rf " + file_to_remove)
<|endoftext|>


<commit_before>#!/usr/bin/python
import datetime, openpyxl as xl, os
from argparse import Namespace
import code
import operator, collections, re, argparse

from django.core.management.base import BaseCommand, CommandError
import utils.sms_utils as sms
import backend.models as back

class Command(BaseCommand):

    help = 'Parse and import SMS data bank'

    def add_arguments(self,parser):
        # code.interact(local=locals())
        parser.add_argument('-d','--dir',help='directory to find translations in default (translations)',default='translations')
        subparsers = parser.add_subparsers(help='sms bank commands')

        # The cmd argument is required for django.core.management.base.CommandParser
        make_parser = subparsers.add_parser('make',cmd=parser.cmd,help='make final translations or todos')
        make_parser.add_argument('type',type=todo_or_final,help='type of make. one of (t)do or (f)inal')
        make_parser.add_argument('-c','--copy',help='make copy of todo messages',action='store_true',default=False)
        make_parser.set_defaults(action='make_messages')

        check_parser = subparsers.add_parser('check',cmd=parser.cmd,help='check and print stats for final translations')
        check_parser.set_defaults(action='check_messages')

        clean_parser = subparsers.add_parser('clean',cmd=parser.cmd,help='clean messages bank')
        clean_parser.set_defaults(action='clean_messages')

        import_parser = subparsers.add_parser('import',cmd=parser.cmd,help='import messages to backend')
        import_parser.add_argument('-d','--done',default=False,action='store_true',help='only import messages marked as done')
        import_parser.set_defaults(action='import_messages')

        test_parser = subparsers.add_parser('test',cmd=parser.cmd,help='test message row creation')
        test_parser.set_defaults(action='test')

        custom_parser = subparsers.add_parser('custom',cmd=parser.cmd,help='run custom command')
        custom_parser.set_defaults(action='custom')

    def handle(self,*args,**options):

        self.stdout.write( 'Action: {} Root: {}'.format(options['action'],options['dir']) )

        self.paths = argparse.Namespace(
            bank = os.path.join(options['dir'],'bank.xlsx'),

            todo_swahili = os.path.join(options['dir'],'todo_swahili.xlsx'),
            todo_luo = os.path.join(options['dir'],'todo_luo.xlsx'),

            done_swahili = os.path.join(options['dir'],'done_swahili.xlsx'),
            done_luo = os.path.join(options['dir'],'done_luo.xlsx'),

            final = os.path.join(options['dir'],'translations.xlsx')
        )

        self.options = options
        getattr(self,options['action'])()

    ########################################
    # Commands
    ########################################

    def custom(self):
        sms_bank = xl.load_workbook(self.paths.bank)
        new_wb = xl.workbook.workbook()
        ws = new_wb.active

        for row in sms_bank.get_sheet_by_name('anc').rows:
            if row[0].value == '#':
                ws.append(row)
            else:
                msg = sms.MessageBankRow(row)
                ws.append(msg.get_bank_row())

    def make_messages(self):
        self.stdout.write( 'Making Messages: {}'.format(self.options['type'].capitalize()) )
        if self.options['type'] == 'todo':
            self.make_todo_messages()
        elif self.options['type'] == 'final':
            self.make_final_messages()

    def make_todo_messages(self):
        ''' Take a formated SMS bank and create language specific todo files'''
        sms_bank = xl.load_workbook(self.paths.bank)
        try:
            final_wb = xl.load_workbook(self.paths.final)
            old_translations = sms.message_dict(final_wb.active,sms.FinalRow)
        except IOError as e:
            old_translations = None

        translations = sms.read_sms_bank(sms_bank,old_translations,'anc','special')

        print "Total Found: {} Todo: {}".format( len(translations),
            len([t for t in translations if t.is_todo()]) )

        swahili_wb = make_language_todos(translations,'swahili')
        luo_wb = make_language_todos(translations,'luo')

        swahili_wb.save(self.paths.todo_swahili)
        luo_wb.save(self.paths.todo_luo)

        if self.options['copy']:
            # TODO: finish this
            pass

    def make_final_messages(self):
        '''Take language specific todo files and done files and make a final translations file '''

        # Load todo messages or fail
        try:
            todo_swahili_wb = xl.load_workbook(self.paths.todo_swahili)
            todo_luo_wb = xl.load_workbook(self.paths.todo_luo)
        except IOError as e:
            self.stderr.write('Error: Swahili and or Lue todo files not found')
            return

        todo_swahili_bank = sms.message_dict(todo_swahili_wb.active,sms.TranslationRow,language_name='swahili')
        todo_luo_bank = sms.message_dict(todo_luo_wb.active,sms.TranslationRow,language_name='luo')

        # Load done messages or fail
        try:
            done_swahili_wb = xl.load_workbook(self.paths.done_swahili)
            done_luo_wb = xl.load_workbook(self.paths.done_luo)
        except IOError as e:
            self.stderr.write('Error: Swahili and or Lue done files not found')
            return
        done_swahili_bank = sms.message_dict(done_swahili_wb.active,sms.TranslationRow,language_name='swahili')
        done_luo_bank = sms.message_dict(done_luo_wb.active,sms.TranslationRow,language_name='luo')

        # Load sms bank
        sms_bank = xl.load_workbook(self.paths.bank)
        translations = sms.read_sms_bank(sms_bank,None,'anc','special')

        # Create final wb and add header
        final_wb = xl.workbook.Workbook()
        new_ws = final_wb.active
        new_ws.append(sms.FinalRow.header)

        for msg in translations:
            description = msg.description()

            todo_swahili , todo_luo = todo_swahili_bank[description] , todo_luo_bank[description]
            done_swahili , done_luo = done_swahili_bank[description] , done_luo_bank[description]
            msg.swahili , msg.luo , msg.english = done_swahili.swahili , done_luo.luo , msg.new

            # Determine done status
            is_done_swahili = not done_swahili.is_todo() or todo_swahili.swahili != done_swahili.swahili
            is_done_luo = not done_luo.is_todo() or todo_luo.luo != done_luo.luo
            if not done_luo.is_todo() and not done_swahili.is_todo():
                msg.status = 'clean'
            elif is_done_swahili and is_done_luo:
                msg.status = 'done'
            elif is_done_swahili:
                msg.status = 'swahili'
            elif is_done_luo:
                msg.status = 'luo'
            else:
                msg.status = 'todo'

            msg.configure_variables()
            new_ws.append(msg.get_final_row())

            # Wrap text on english, swahili, luo columns
            last = new_ws.rows[-1]
            for i in [6,7,8]:
                last[i].alignment = WRAP_TEXT

            new_ws.freeze_panes = 'A2'
            new_ws.auto_filter.ref = 'A1:I1'

            column_widths = {'A':4,'D':6,'E':6,'F':6,'G':50,'H':50,'I':50}
            for col_letter, width in column_widths.items():
                new_ws.column_dimensions[col_letter].width = width

        self.stdout.write('{} translations. {} todo.'.format( len(translations),
            len([m for m in translations if m.is_todo()])
        ) )

        final_wb.save(self.paths.final)

    def check_messages(self):

        sms_wb = xl.load_workbook(self.paths.final)
        messages = sms.parse_messages(sms_wb.active,sms.FinalRow)

        stats = recursive_dd()
        for msg in messages:
            condition_hiv = stats[ '{}_HIV_{}'.format(msg.track,msg.get_hiv_messaging_str()) ]
            condition_hiv.default_factory = list

            base_group = condition_hiv[ '{}_{}'.format(msg.send_base,msg.group) ]
            base_group.append(msg)

        for condition_hiv, base_groups in stats.items():
            self.stdout.write( '{}'.format(condition_hiv) )
            for base_group,items in base_groups.items():
                count = len(items)
                todo = len(filter(lambda m: m.is_todo(),items))
                self.stdout.write( '\t{}: {}({}) {}'.format(
                    base_group,count,todo,
                    sorted([i.offset for i in items])
                ) )

        # self.options['ascii_msg'] = 'Warning: non-ascii chars found: {count}'
        # non_ascii_dict = self.non_ascii_count()
        #

    def clean_messages(self):
        self.stderr.write('Clean function not implemented!')

    def import_messages(self):
        sms_bank = xl.load_workbook(self.paths.final)
        do_all = not self.options.get('done',False)

        todo,total = 0,0
        for row in sms_bank.active.rows[1:]:
            msg = sms.FinalRow(row)
            total += 1
            if do_all or msg.status == 'done':
                auto = back.AutomatedMessage.objects.from_excel(msg)
                if msg.is_todo():
                    self.stdout.write('Warning: message {} still todo'.format(msg.description()))
                    todo += 1
        self.stdout.write('Messages Imported: {} Messages Todo: {}'.format(total,todo))

    def non_ascii_count(self):
        sms_bank = xl.load_workbook(self.options['bank'])
        non_ascii = collections.defaultdict(int)

        # Count non ascii characters (ord value greater than 127)
        for row in sms_bank.active.rows[1:]:
            msg = sms.MessageRow(row,translation=True)

            for c in msg.english + msg.swahili + msg.luo:
                if ord(c) > 127:
                    non_ascii[c] += 1

        # Print non ascii counts
        print self.options.get('ascii_msg','non-ascii count: {count}').format(count=len(non_ascii))
        for c,count in non_ascii.items():
            print u'({0!r} {0!s}) => {1}'.format(c,count)

        return len(non_ascii) > 0

    def test(self):

        final_wb = xl.load_workbook(self.paths.final)
        old_translations = sms.message_dict(final_wb.active,sms.FinalRow)

        print 'Count: {} Example: {!r}'.format(len(old_translations),old_translations.get(old_translations.keys()[0]))

########################################
#  Utility Functions
########################################

WRAP_TEXT = xl.styles.Alignment(wrap_text=True,vertical='top')

def recursive_dd():
    return collections.defaultdict(recursive_dd)

def bank_or_translation(option):
    if option.startswith('b'):
        return 'bank'
    elif options.startswith('t'):
        return 'translation'
    raise argparse.ArgumentTypeError('type must be either bank or translation')

def todo_or_final(option):
    if option.startswith('t'):
        return 'todo'
    elif option.startswith('f'):
        return 'final'
    raise argparse.ArgumentTypeError('type must be either todo or translation')

def make_language_todos(translations,language):

    wb = xl.workbook.Workbook()
    ws = wb.active

    ws.append(sms.TranslationRow.header(language))

    for msg in translations:
        ws.append(msg.get_translation_row(language))
        last = ws.rows[-1]
        for i in [6,7,8]:
            last[i].alignment = WRAP_TEXT

    ws.freeze_panes = 'A2'
    ws.auto_filter.ref = 'A1:I1'

    column_widths = {'A':4,'D':6,'E':6,'F':6,'G':50,'H':50,'I':50}
    for col_letter, width in column_widths.items():
        ws.column_dimensions[col_letter].width = width

    return wb
<commit_msg>feat(sms_bank) add copy option for make todos<commit_after>#!/usr/bin/python
import datetime, openpyxl as xl, os
from argparse import Namespace
import code
import operator, collections, re, argparse

from django.core.management.base import BaseCommand, CommandError
import utils.sms_utils as sms
import backend.models as back

class Command(BaseCommand):

    help = 'Parse and import SMS data bank'

    def add_arguments(self,parser):
        # code.interact(local=locals())
        parser.add_argument('-d','--dir',help='directory to find translations in default (translations)',default='translations')
        subparsers = parser.add_subparsers(help='sms bank commands')

        # The cmd argument is required for django.core.management.base.CommandParser
        make_parser = subparsers.add_parser('make',cmd=parser.cmd,help='make final translations or todos')
        make_parser.add_argument('type',type=todo_or_final,help='type of make. one of (t)do or (f)inal')
        make_parser.add_argument('-c','--copy',help='make copy of todo messages',action='store_true',default=False)
        make_parser.set_defaults(action='make_messages')

        check_parser = subparsers.add_parser('check',cmd=parser.cmd,help='check and print stats for final translations')
        check_parser.set_defaults(action='check_messages')

        clean_parser = subparsers.add_parser('clean',cmd=parser.cmd,help='clean messages bank')
        clean_parser.set_defaults(action='clean_messages')

        import_parser = subparsers.add_parser('import',cmd=parser.cmd,help='import messages to backend')
        import_parser.add_argument('-d','--done',default=False,action='store_true',help='only import messages marked as done')
        import_parser.set_defaults(action='import_messages')

        test_parser = subparsers.add_parser('test',cmd=parser.cmd,help='test message row creation')
        test_parser.set_defaults(action='test')

        custom_parser = subparsers.add_parser('custom',cmd=parser.cmd,help='run custom command')
        custom_parser.set_defaults(action='custom')

    def handle(self,*args,**options):

        self.stdout.write( 'Action: {} Root: {}'.format(options['action'],options['dir']) )

        self.paths = argparse.Namespace(
            bank = os.path.join(options['dir'],'bank.xlsx'),

            todo_swahili = os.path.join(options['dir'],'todo_swahili.xlsx'),
            todo_luo = os.path.join(options['dir'],'todo_luo.xlsx'),

            done_swahili = os.path.join(options['dir'],'done_swahili.xlsx'),
            done_luo = os.path.join(options['dir'],'done_luo.xlsx'),

            final = os.path.join(options['dir'],'translations.xlsx')
        )

        self.options = options
        getattr(self,options['action'])()

    ########################################
    # Commands
    ########################################

    def custom(self):
        sms_bank = xl.load_workbook(self.paths.bank)
        new_wb = xl.workbook.workbook()
        ws = new_wb.active

        for row in sms_bank.get_sheet_by_name('anc').rows:
            if row[0].value == '#':
                ws.append(row)
            else:
                msg = sms.MessageBankRow(row)
                ws.append(msg.get_bank_row())

    def make_messages(self):
        self.stdout.write( 'Making Messages: {}'.format(self.options['type'].capitalize()) )
        if self.options['type'] == 'todo':
            self.make_todo_messages()
        elif self.options['type'] == 'final':
            self.make_final_messages()

    def make_todo_messages(self):
        ''' Take a formated SMS bank and create language specific todo files'''
        sms_bank = xl.load_workbook(self.paths.bank)
        try:
            final_wb = xl.load_workbook(self.paths.final)
            old_translations = sms.message_dict(final_wb.active,sms.FinalRow)
        except IOError as e:
            old_translations = None

        translations = sms.read_sms_bank(sms_bank,old_translations,'anc','special')

        print "Total Found: {} Todo: {}".format( len(translations),
            len([t for t in translations if t.is_todo()]) )

        swahili_wb = make_language_todos(translations,'swahili')
        luo_wb = make_language_todos(translations,'luo')

        swahili_wb.save(self.paths.todo_swahili)
        luo_wb.save(self.paths.todo_luo)

        if self.options['copy']:
            swahili_copy_path = path_with_date(self.paths.todo_swahili)
            luo_copy_path = path_with_date(self.paths.todo_luo)
            swahili_wb.save(swahili_copy_path)
            luo_wb.save(luo_copy_path)

    def make_final_messages(self):
        '''Take language specific todo files and done files and make a final translations file '''

        # Load todo messages or fail
        try:
            todo_swahili_wb = xl.load_workbook(self.paths.todo_swahili)
            todo_luo_wb = xl.load_workbook(self.paths.todo_luo)
        except IOError as e:
            self.stderr.write('Error: Swahili and or Lue todo files not found')
            return

        todo_swahili_bank = sms.message_dict(todo_swahili_wb.active,sms.TranslationRow,language_name='swahili')
        todo_luo_bank = sms.message_dict(todo_luo_wb.active,sms.TranslationRow,language_name='luo')

        # Load done messages or fail
        try:
            done_swahili_wb = xl.load_workbook(self.paths.done_swahili)
            done_luo_wb = xl.load_workbook(self.paths.done_luo)
        except IOError as e:
            self.stderr.write('Error: Swahili and or Lue done files not found')
            return
        done_swahili_bank = sms.message_dict(done_swahili_wb.active,sms.TranslationRow,language_name='swahili')
        done_luo_bank = sms.message_dict(done_luo_wb.active,sms.TranslationRow,language_name='luo')

        # Load sms bank
        sms_bank = xl.load_workbook(self.paths.bank)
        translations = sms.read_sms_bank(sms_bank,None,'anc','special')

        # Create final wb and add header
        final_wb = xl.workbook.Workbook()
        new_ws = final_wb.active
        new_ws.append(sms.FinalRow.header)

        for msg in translations:
            description = msg.description()

            todo_swahili , todo_luo = todo_swahili_bank[description] , todo_luo_bank[description]
            done_swahili , done_luo = done_swahili_bank[description] , done_luo_bank[description]
            msg.swahili , msg.luo , msg.english = done_swahili.swahili , done_luo.luo , msg.new

            # Determine done status
            is_done_swahili = not done_swahili.is_todo() or todo_swahili.swahili != done_swahili.swahili
            is_done_luo = not done_luo.is_todo() or todo_luo.luo != done_luo.luo
            if not done_luo.is_todo() and not done_swahili.is_todo():
                msg.status = 'clean'
            elif is_done_swahili and is_done_luo:
                msg.status = 'done'
            elif is_done_swahili:
                msg.status = 'swahili'
            elif is_done_luo:
                msg.status = 'luo'
            else:
                msg.status = 'todo'

            msg.configure_variables()
            new_ws.append(msg.get_final_row())

            # Wrap text on english, swahili, luo columns
            last = new_ws.rows[-1]
            for i in [6,7,8]:
                last[i].alignment = WRAP_TEXT

            new_ws.freeze_panes = 'A2'
            new_ws.auto_filter.ref = 'A1:I1'

            column_widths = {'A':4,'D':6,'E':6,'F':6,'G':50,'H':50,'I':50}
            for col_letter, width in column_widths.items():
                new_ws.column_dimensions[col_letter].width = width

        self.stdout.write('{} translations. {} todo.'.format( len(translations),
            len([m for m in translations if m.is_todo()])
        ) )

        final_wb.save(self.paths.final)

    def check_messages(self):

        sms_wb = xl.load_workbook(self.paths.final)
        messages = sms.parse_messages(sms_wb.active,sms.FinalRow)

        stats = recursive_dd()
        for msg in messages:
            condition_hiv = stats[ '{}_HIV_{}'.format(msg.track,msg.get_hiv_messaging_str()) ]
            condition_hiv.default_factory = list

            base_group = condition_hiv[ '{}_{}'.format(msg.send_base,msg.group) ]
            base_group.append(msg)

        for condition_hiv, base_groups in stats.items():
            self.stdout.write( '{}'.format(condition_hiv) )
            for base_group,items in base_groups.items():
                count = len(items)
                todo = len(filter(lambda m: m.is_todo(),items))
                self.stdout.write( '\t{}: {}({}) {}'.format(
                    base_group,count,todo,
                    sorted([i.offset for i in items])
                ) )

        # self.options['ascii_msg'] = 'Warning: non-ascii chars found: {count}'
        # non_ascii_dict = self.non_ascii_count()
        #

    def clean_messages(self):
        self.stderr.write('Clean function not implemented!')

    def import_messages(self):
        sms_bank = xl.load_workbook(self.paths.final)
        do_all = not self.options.get('done',False)

        todo,total = 0,0
        for row in sms_bank.active.rows[1:]:
            msg = sms.FinalRow(row)
            total += 1
            if do_all or msg.status == 'done':
                auto = back.AutomatedMessage.objects.from_excel(msg)
                if msg.is_todo():
                    self.stdout.write('Warning: message {} still todo'.format(msg.description()))
                    todo += 1
        self.stdout.write('Messages Imported: {} Messages Todo: {}'.format(total,todo))

    def non_ascii_count(self):
        sms_bank = xl.load_workbook(self.options['bank'])
        non_ascii = collections.defaultdict(int)

        # Count non ascii characters (ord value greater than 127)
        for row in sms_bank.active.rows[1:]:
            msg = sms.MessageRow(row,translation=True)

            for c in msg.english + msg.swahili + msg.luo:
                if ord(c) > 127:
                    non_ascii[c] += 1

        # Print non ascii counts
        print self.options.get('ascii_msg','non-ascii count: {count}').format(count=len(non_ascii))
        for c,count in non_ascii.items():
            print u'({0!r} {0!s}) => {1}'.format(c,count)

        return len(non_ascii) > 0

    def test(self):

        final_wb = xl.load_workbook(self.paths.final)
        old_translations = sms.message_dict(final_wb.active,sms.FinalRow)

        print 'Count: {} Example: {!r}'.format(len(old_translations),old_translations.get(old_translations.keys()[0]))

########################################
#  Utility Functions
########################################

WRAP_TEXT = xl.styles.Alignment(wrap_text=True,vertical='top')

def recursive_dd():
    return collections.defaultdict(recursive_dd)

def bank_or_translation(option):
    if option.startswith('b'):
        return 'bank'
    elif options.startswith('t'):
        return 'translation'
    raise argparse.ArgumentTypeError('type must be either bank or translation')

def todo_or_final(option):
    if option.startswith('t'):
        return 'todo'
    elif option.startswith('f'):
        return 'final'
    raise argparse.ArgumentTypeError('type must be either todo or translation')

def make_language_todos(translations,language):

    wb = xl.workbook.Workbook()
    ws = wb.active

    ws.append(sms.TranslationRow.header(language))

    for msg in translations:
        ws.append(msg.get_translation_row(language))
        last = ws.rows[-1]
        for i in [6,7,8]:
            last[i].alignment = WRAP_TEXT

    ws.freeze_panes = 'A2'
    ws.auto_filter.ref = 'A1:I1'

    column_widths = {'A':4,'D':6,'E':6,'F':6,'G':50,'H':50,'I':50}
    for col_letter, width in column_widths.items():
        ws.column_dimensions[col_letter].width = width

    return wb

def path_with_date(path):
    root , ext = os.path.splitext(path)
    date_str = datetime.date.today().strftime('%Y-%m-%d')
    return "{}_{}{}".format(root,date_str,ext)
<|endoftext|>


<commit_before>import datetime
from itertools import chain, imap
from django.contrib.contenttypes.models import ContentType
from django.utils.html import linebreaks

from valuenetwork.valueaccounting.models import Commitment, Process

def split_thousands(n, sep=','):
    s = str(n)
    if len(s) <= 3: return s
    return split_thousands(s[:-3], sep) + sep + s[-3:]

def dfs(node, all_nodes, depth):
    """
    Performs a recursive depth-first search starting at ``node``.
    """
    to_return = [node,]
    for subnode in all_nodes:
        if subnode.parent and subnode.parent.id == node.id:
            to_return.extend(dfs(subnode, all_nodes, depth+1))
    return to_return

def flattened_children(node, all_nodes, to_return):
     to_return.append(node)
     for subnode in all_nodes:
         if subnode.parent and subnode.parent.id == node.id:
             flattened_children(subnode, all_nodes, to_return)
     return to_return

class Edge(object):
    def __init__(self, from_node, to_node, label):
        self.from_node = from_node
        self.to_node = to_node
        self.label = label
        self.width = 1

def project_graph(producers):
    nodes = []
    edges = []
    #import pdb; pdb.set_trace()
    for p in producers:
        for rt in p.produced_resource_type_relationships():
            for pt in rt.resource_type.consuming_process_type_relationships():
                if p.project and pt.process_type.project:
                    if p.project != pt.process_type.project:
                        nodes.extend([p.project, pt.process_type.project, rt.resource_type])
                        edges.append(Edge(p.project, rt.resource_type, rt.relationship.name))
                        edges.append(Edge(rt.resource_type, pt.process_type.project, pt.inverse_label()))
    return [nodes, edges]

def explode(process_type_relationship, nodes, edges, depth, depth_limit):
    if depth > depth_limit:
        return
    #if process_type_relationship.process_type.name.startswith('Q'):
    #    return
    nodes.append(process_type_relationship.process_type)
    edges.append(Edge(
        process_type_relationship.process_type,
        process_type_relationship.resource_type,
        process_type_relationship.relationship.name
    ))
    for rtr in process_type_relationship.process_type.consumed_resource_type_relationships():
        nodes.append(rtr.resource_type)
        edges.append(Edge(rtr.resource_type, process_type_relationship.process_type, rtr.inverse_label()))
        for art in rtr.resource_type.producing_agent_relationships():
            nodes.append(art.agent)
            edges.append(Edge(art.agent, rtr.resource_type, art.relationship.name))
        for pt in rtr.resource_type.producing_process_type_relationships():
            explode(pt, nodes, edges, depth+1, depth_limit)

def graphify(focus, depth_limit):
    nodes = [focus]
    edges = []
    for art in focus.consuming_agent_relationships():
        nodes.append(art.agent)
        edges.append(Edge(focus, art.agent, art.relationship.name))
    for ptr in focus.producing_process_type_relationships():
        explode(ptr, nodes, edges, 0, depth_limit)
    return [nodes, edges]

def project_network():
    producers = [p for p in ProcessType.objects.all() if p.produced_resource_types()]
    nodes = []
    edges = []
    for p in producers:
        for rt in p.produced_resource_types():
            for pt in rt.consuming_process_types():
                if p.project != pt.project:
                    nodes.extend([p.project, pt.project, rt])
                    edges.append(Edge(p.project, rt))
                    edges.append(Edge(rt, pt.project))
    return [nodes, edges]

class TimelineEvent(object):
    def __init__(self, node, start, end, title, link, description):
         self.node = node
         self.start = start
         self.end = end
         self.title = title
         self.link = link
         self.description = description

    def dictify(self):
        descrip = ""
        if self.description:
            descrip = self.description
        d = {
            "start": self.start.strftime("%b %e %Y 00:00:00 GMT-0600"),
            "title": self.title,
            "description": linebreaks(descrip),
        }
        if self.end:
            d["end"] = self.end.strftime("%b %e %Y 00:00:00 GMT-0600")
            d["durationEvent"] = True
        else:
            d["durationEvent"] = False
        if self.link:
            d["link"] = self.link
        mrq = []
        for mreq in self.node.material_requirements():
            abbrev = mreq.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(mreq.quantity),
                abbrev,
                mreq.resource_type.name])
            mrq.append(label)
        d["materialReqmts"] = mrq
        trq = []
        for treq in self.node.tool_requirements():
            abbrev = treq.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(treq.quantity),
                abbrev,
                treq.resource_type.name])
            trq.append(label)
        d["toolReqmts"] = trq
        wrq = []
        for wreq in self.node.work_requirements():
            abbrev = wreq.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(wreq.quantity),
                abbrev,
                wreq.resource_type.name])
            wrq.append(label)
        d["workReqmts"] = wrq
        items = []
        for item in self.node.order_items():
            abbrev = item.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(item.quantity),
                abbrev,
                item.resource_type.name])
            items.append(label)
        d["orderItems"] = items
        return d

def create_events(orders, processes, events):
    for order in orders:
        te = TimelineEvent(
            order,
            order.due_date,
            "",
            order.timeline_title(),
            order.get_absolute_url(),
            order.timeline_description(),
        )
        events['events'].append(te.dictify())
    for process in processes:
        #if process.id == 15:
        #    import pdb; pdb.set_trace()
        te = TimelineEvent(
            process,
            process.start_date,
            process.end_date,
            process.timeline_title(),
            process.get_absolute_url(),
            process.timeline_description(),
        )
        events['events'].append(te.dictify())

def explode_events(resource_type, backsked_date, events):
    for art in resource_type.producing_agent_relationships():
        order_date = backsked_date - datetime.timedelta(days=art.lead_time)
        te = TimelineEvent(
            art,
            order_date,
            "",
            art.timeline_title(),
            resource_type.url,
            resource_type.description,
        )
        events['events'].append(te.dictify())
    for pp in resource_type.producing_process_types():
        start_date = backsked_date - datetime.timedelta(days=(pp.estimated_duration/1440))
        ppte = TimelineEvent(
            pp,
            start_date,
            backsked_date,
            pp.timeline_title(),
            pp.url,
            pp.description,
        )
        events['events'].append(ppte.dictify())
        for crt in pp.consumed_resource_types():
            explode_events(crt, start_date, events)

def backschedule_process_types(commitment, process_type,events):
    lead_time=1
    arts = None
    if commitment.from_agent:
        arts = commitment.from_agent.resource_types.filter(resource_type=commitment.resource_type)
    if arts:
        lead_time = arts[0].lead_time
    end_date = commitment.due_date - datetime.timedelta(days=lead_time)
    start_date = end_date - datetime.timedelta(days=(process_type.estimated_duration/1440))
    ppte = TimelineEvent(
        process_type,
        start_date,
        end_date,
        process_type.timeline_title(),
        process_type.url,
        process_type.description,
    )
    events['events'].append(ppte.dictify())
    for crt in process_type.consumed_resource_types():
        explode_events(crt, start_date, events)

def backschedule_process(order, process, events):
    te = TimelineEvent(
        process,
        process.start_date,
        process.end_date,
        process.timeline_title(),
        process.url,
        process.notes,
    )
    events['events'].append(te.dictify())
    for ic in process.incoming_commitments():
        te = TimelineEvent(
            ic,
            ic.due_date,
            "",
            ic.timeline_title(),
            ic.url,
            ic.description,
        )
        events['events'].append(te.dictify())
        resource_type = ic.resource_type
        pcs = resource_type.producing_commitments()
        if pcs:
            for pc in pcs:
                if pc.independent_demand == order:
                    te = TimelineEvent(
                        pc,
                        pc.due_date,
                        "",
                        pc.timeline_title(),
                        pc.url,
                        pc.description,
                    )
                    events['events'].append(te.dictify())
                    backschedule_process(order, pc.process, events)

    return events

def backschedule_order(order, events):
    te = TimelineEvent(
        order,
        order.due_date,
        "",
        order.timeline_title(),
        "",
        order.description,
    )
    events['events'].append(te.dictify())
    for pc in order.producing_commitments():
        te = TimelineEvent(
            pc,
            pc.due_date,
            "",
            pc.timeline_title(),
            pc.url,
            pc.description,
        )
        events['events'].append(te.dictify())
        backschedule_process(order, pc.process, events)

def recursively_explode_demands(process, order, user):
    pt = process.process_type
    output = process.main_outgoing_commitment()
    for ptrt in pt.consumed_resource_type_relationships():
        commitment = Commitment(
            independent_demand=order,
            event_type=ptrt.relationship.event_type,
            relationship=ptrt.relationship,
            due_date=process.start_date,
            resource_type=ptrt.resource_type,
            process=process,
            project=pt.project,
            quantity=output.quantity * ptrt.quantity,
            unit_of_quantity=ptrt.resource_type.unit,
            created_by=user,
        )
        commitment.save()
        pptr = ptrt.resource_type.main_producing_process_type_relationship()
        if pptr:
            next_pt = pptr.process_type
            start_date = process.start_date - datetime.timedelta(minutes=next_pt.estimated_duration)
            next_process = Process(
                name=next_pt.name,
                process_type=next_pt,
                project=next_pt.project,
                url=next_pt.url,
                end_date=process.start_date,
                start_date=start_date,
            )
            next_process.save()
            next_commitment = Commitment(
                independent_demand=order,
                event_type=pptr.relationship.event_type,
                relationship=pptr.relationship,
                due_date=process.start_date,
                resource_type=pptr.resource_type,
                process=next_process,
                project=next_pt.project,
                quantity=output.quantity * pptr.quantity,
                unit_of_quantity=pptr.resource_type.unit,
                created_by=user,
            )
            next_commitment.save()
            recursively_explode_demands(next_process, order, user)

class XbillNode(object):
    def __init__(self, node, depth):
         self.node = node
         self.depth = depth
         self.open = False
         self.close = []
         self.xbill_class = self.node.xbill_class()

    def xbill_object(self):
        return self.node.xbill_child_object()

    def xbill_label(self):
        return self.node.xbill_label()

    def xbill_explanation(self):
        return self.node.xbill_explanation()

    #def xbill_class(self):
    #    ct = ContentType.objects.get_for_model(self.xbill_object().__class__)
    #    return "-".join(ct.name.split())

    def category(self):
        return self.node.xbill_category()

def xbill_dfs(node, all_nodes, depth):
    """
    Performs a recursive depth-first search starting at ``node``.
    """
    to_return = [XbillNode(node,depth),]
    #print "+created node:+", node, depth
    #import pdb; pdb.set_trace()
    for subnode in all_nodes:
        parents = subnode.xbill_parent_object().xbill_parents()
        if not subnode is node:
            if parents and node in parents:
                #print "*active node:*", node, "*depth:*", depth, "*subnode:*", subnode, "*parent_object:*", subnode.xbill_parent_object(), "*parents:*", parents
                #import pdb; pdb.set_trace()
                to_return.extend(xbill_dfs(subnode, all_nodes, depth+1))
    return to_return

def explode_xbill_children(node, nodes):
    nodes.append(node)
    #import pdb; pdb.set_trace()
    for kid in node.xbill_child_object().xbill_children():
        explode_xbill_children(kid, nodes)

def generate_xbill(resource_type):
    nodes = []
    for kid in resource_type.xbill_children():
        explode_xbill_children(kid, nodes)
    nodes = list(set(nodes))
    #import pdb; pdb.set_trace()
    to_return = []
    for kid in resource_type.xbill_children():
        to_return.extend(xbill_dfs(kid, nodes, 1))
    annotate_tree_properties(to_return)
    #to_return.sort(lambda x, y: cmp(x.xbill_object().name,
    #                                y.xbill_object().name))
    return to_return

#adapted from threaded_comments.util
def annotate_tree_properties(nodes):
    """
    iterate through nodes and adds some magic properties to each of them
    representing opening list of children and closing it
    """
    if not nodes:
        return

    it = iter(nodes)

    # get the first item, this will fail if no items !
    old = it.next()

    # first item starts a new thread
    old.open = True
    for c in it:

        # increase the depth
        if c.depth > old.depth:
            c.open = True

        else: # c.depth <= old.depth
            # close some depths
            old.close = range(old.depth - c.depth)

        # iterate
        old = c

    old.close = range(old.depth)

<commit_msg>shortcut recipe cycles to avoid infinite loops<commit_after>import datetime
from itertools import chain, imap
from django.contrib.contenttypes.models import ContentType
from django.utils.html import linebreaks

from valuenetwork.valueaccounting.models import Commitment, Process

def split_thousands(n, sep=','):
    s = str(n)
    if len(s) <= 3: return s
    return split_thousands(s[:-3], sep) + sep + s[-3:]

def dfs(node, all_nodes, depth):
    """
    Performs a recursive depth-first search starting at ``node``.
    """
    to_return = [node,]
    for subnode in all_nodes:
        if subnode.parent and subnode.parent.id == node.id:
            to_return.extend(dfs(subnode, all_nodes, depth+1))
    return to_return

def flattened_children(node, all_nodes, to_return):
     to_return.append(node)
     for subnode in all_nodes:
         if subnode.parent and subnode.parent.id == node.id:
             flattened_children(subnode, all_nodes, to_return)
     return to_return

class Edge(object):
    def __init__(self, from_node, to_node, label):
        self.from_node = from_node
        self.to_node = to_node
        self.label = label
        self.width = 1

def project_graph(producers):
    nodes = []
    edges = []
    #import pdb; pdb.set_trace()
    for p in producers:
        for rt in p.produced_resource_type_relationships():
            for pt in rt.resource_type.consuming_process_type_relationships():
                if p.project and pt.process_type.project:
                    if p.project != pt.process_type.project:
                        nodes.extend([p.project, pt.process_type.project, rt.resource_type])
                        edges.append(Edge(p.project, rt.resource_type, rt.relationship.name))
                        edges.append(Edge(rt.resource_type, pt.process_type.project, pt.inverse_label()))
    return [nodes, edges]

def explode(process_type_relationship, nodes, edges, depth, depth_limit):
    if depth > depth_limit:
        return
    #if process_type_relationship.process_type.name.startswith('Q'):
    #    return
    nodes.append(process_type_relationship.process_type)
    edges.append(Edge(
        process_type_relationship.process_type,
        process_type_relationship.resource_type,
        process_type_relationship.relationship.name
    ))
    for rtr in process_type_relationship.process_type.consumed_resource_type_relationships():
        nodes.append(rtr.resource_type)
        edges.append(Edge(rtr.resource_type, process_type_relationship.process_type, rtr.inverse_label()))
        for art in rtr.resource_type.producing_agent_relationships():
            nodes.append(art.agent)
            edges.append(Edge(art.agent, rtr.resource_type, art.relationship.name))
        for pt in rtr.resource_type.producing_process_type_relationships():
            explode(pt, nodes, edges, depth+1, depth_limit)

def graphify(focus, depth_limit):
    nodes = [focus]
    edges = []
    for art in focus.consuming_agent_relationships():
        nodes.append(art.agent)
        edges.append(Edge(focus, art.agent, art.relationship.name))
    for ptr in focus.producing_process_type_relationships():
        explode(ptr, nodes, edges, 0, depth_limit)
    return [nodes, edges]

def project_network():
    producers = [p for p in ProcessType.objects.all() if p.produced_resource_types()]
    nodes = []
    edges = []
    for p in producers:
        for rt in p.produced_resource_types():
            for pt in rt.consuming_process_types():
                if p.project != pt.project:
                    nodes.extend([p.project, pt.project, rt])
                    edges.append(Edge(p.project, rt))
                    edges.append(Edge(rt, pt.project))
    return [nodes, edges]

class TimelineEvent(object):
    def __init__(self, node, start, end, title, link, description):
         self.node = node
         self.start = start
         self.end = end
         self.title = title
         self.link = link
         self.description = description

    def dictify(self):
        descrip = ""
        if self.description:
            descrip = self.description
        d = {
            "start": self.start.strftime("%b %e %Y 00:00:00 GMT-0600"),
            "title": self.title,
            "description": linebreaks(descrip),
        }
        if self.end:
            d["end"] = self.end.strftime("%b %e %Y 00:00:00 GMT-0600")
            d["durationEvent"] = True
        else:
            d["durationEvent"] = False
        if self.link:
            d["link"] = self.link
        mrq = []
        for mreq in self.node.material_requirements():
            abbrev = mreq.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(mreq.quantity),
                abbrev,
                mreq.resource_type.name])
            mrq.append(label)
        d["materialReqmts"] = mrq
        trq = []
        for treq in self.node.tool_requirements():
            abbrev = treq.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(treq.quantity),
                abbrev,
                treq.resource_type.name])
            trq.append(label)
        d["toolReqmts"] = trq
        wrq = []
        for wreq in self.node.work_requirements():
            abbrev = wreq.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(wreq.quantity),
                abbrev,
                wreq.resource_type.name])
            wrq.append(label)
        d["workReqmts"] = wrq
        items = []
        for item in self.node.order_items():
            abbrev = item.unit_of_quantity.abbrev or ""
            label = " ".join([
                str(item.quantity),
                abbrev,
                item.resource_type.name])
            items.append(label)
        d["orderItems"] = items
        return d

def create_events(orders, processes, events):
    for order in orders:
        te = TimelineEvent(
            order,
            order.due_date,
            "",
            order.timeline_title(),
            order.get_absolute_url(),
            order.timeline_description(),
        )
        events['events'].append(te.dictify())
    for process in processes:
        te = TimelineEvent(
            process,
            process.start_date,
            process.end_date,
            process.timeline_title(),
            process.get_absolute_url(),
            process.timeline_description(),
        )
        events['events'].append(te.dictify())

def explode_events(resource_type, backsked_date, events):
    for art in resource_type.producing_agent_relationships():
        order_date = backsked_date - datetime.timedelta(days=art.lead_time)
        te = TimelineEvent(
            art,
            order_date,
            "",
            art.timeline_title(),
            resource_type.url,
            resource_type.description,
        )
        events['events'].append(te.dictify())
    for pp in resource_type.producing_process_types():
        start_date = backsked_date - datetime.timedelta(days=(pp.estimated_duration/1440))
        ppte = TimelineEvent(
            pp,
            start_date,
            backsked_date,
            pp.timeline_title(),
            pp.url,
            pp.description,
        )
        events['events'].append(ppte.dictify())
        for crt in pp.consumed_resource_types():
            explode_events(crt, start_date, events)

def backschedule_process_types(commitment, process_type,events):
    lead_time=1
    arts = None
    if commitment.from_agent:
        arts = commitment.from_agent.resource_types.filter(resource_type=commitment.resource_type)
    if arts:
        lead_time = arts[0].lead_time
    end_date = commitment.due_date - datetime.timedelta(days=lead_time)
    start_date = end_date - datetime.timedelta(days=(process_type.estimated_duration/1440))
    ppte = TimelineEvent(
        process_type,
        start_date,
        end_date,
        process_type.timeline_title(),
        process_type.url,
        process_type.description,
    )
    events['events'].append(ppte.dictify())
    for crt in process_type.consumed_resource_types():
        explode_events(crt, start_date, events)

def backschedule_process(order, process, events):
    te = TimelineEvent(
        process,
        process.start_date,
        process.end_date,
        process.timeline_title(),
        process.url,
        process.notes,
    )
    events['events'].append(te.dictify())
    for ic in process.incoming_commitments():
        te = TimelineEvent(
            ic,
            ic.due_date,
            "",
            ic.timeline_title(),
            ic.url,
            ic.description,
        )
        events['events'].append(te.dictify())
        resource_type = ic.resource_type
        pcs = resource_type.producing_commitments()
        if pcs:
            for pc in pcs:
                if pc.independent_demand == order:
                    te = TimelineEvent(
                        pc,
                        pc.due_date,
                        "",
                        pc.timeline_title(),
                        pc.url,
                        pc.description,
                    )
                    events['events'].append(te.dictify())
                    backschedule_process(order, pc.process, events)

    return events

def backschedule_order(order, events):
    te = TimelineEvent(
        order,
        order.due_date,
        "",
        order.timeline_title(),
        "",
        order.description,
    )
    events['events'].append(te.dictify())
    for pc in order.producing_commitments():
        te = TimelineEvent(
            pc,
            pc.due_date,
            "",
            pc.timeline_title(),
            pc.url,
            pc.description,
        )
        events['events'].append(te.dictify())
        backschedule_process(order, pc.process, events)

def recursively_explode_demands(process, order, user):
    pt = process.process_type
    output = process.main_outgoing_commitment()
    for ptrt in pt.consumed_resource_type_relationships():
        commitment = Commitment(
            independent_demand=order,
            event_type=ptrt.relationship.event_type,
            relationship=ptrt.relationship,
            due_date=process.start_date,
            resource_type=ptrt.resource_type,
            process=process,
            project=pt.project,
            quantity=output.quantity * ptrt.quantity,
            unit_of_quantity=ptrt.resource_type.unit,
            created_by=user,
        )
        commitment.save()
        pptr = ptrt.resource_type.main_producing_process_type_relationship()
        if pptr:
            next_pt = pptr.process_type
            start_date = process.start_date - datetime.timedelta(minutes=next_pt.estimated_duration)
            next_process = Process(
                name=next_pt.name,
                process_type=next_pt,
                project=next_pt.project,
                url=next_pt.url,
                end_date=process.start_date,
                start_date=start_date,
            )
            next_process.save()
            next_commitment = Commitment(
                independent_demand=order,
                event_type=pptr.relationship.event_type,
                relationship=pptr.relationship,
                due_date=process.start_date,
                resource_type=pptr.resource_type,
                process=next_process,
                project=next_pt.project,
                quantity=output.quantity * pptr.quantity,
                unit_of_quantity=pptr.resource_type.unit,
                created_by=user,
            )
            next_commitment.save()
            recursively_explode_demands(next_process, order, user)

class XbillNode(object):
    def __init__(self, node, depth):
         self.node = node
         self.depth = depth
         self.open = False
         self.close = []
         self.xbill_class = self.node.xbill_class()

    def xbill_object(self):
        return self.node.xbill_child_object()

    def xbill_label(self):
        return self.node.xbill_label()

    def xbill_explanation(self):
        return self.node.xbill_explanation()

    #def xbill_class(self):
    #    ct = ContentType.objects.get_for_model(self.xbill_object().__class__)
    #    return "-".join(ct.name.split())

    def category(self):
        return self.node.xbill_category()

def xbill_dfs(node, all_nodes, visited, depth):
    """
    Performs a recursive depth-first search starting at ``node``.
    """
    to_return = []
    if node not in visited:
        visited.append(node)
        #to_return = [XbillNode(node,depth),]
        to_return.append(XbillNode(node,depth))
        print "+created node:+", node, depth
        for subnode in all_nodes:
            parents = subnode.xbill_parent_object().xbill_parents()
            xclass = subnode.xbill_class()
            if not subnode is node:
                if parents and node in parents:
                    print "*active node:*", node, "*depth:*", depth, "*subnode:*", subnode, "*parent_object:*", subnode.xbill_parent_object(), "*parents:*", parents
                    #import pdb; pdb.set_trace()
                    to_return.extend(xbill_dfs(subnode, all_nodes, visited, depth+1))
    return to_return

def explode_xbill_children(node, nodes, exploded):
    if node not in nodes:
        nodes.append(node)
        #import pdb; pdb.set_trace()
        xclass = node.xbill_class()
        explode = True
        if xclass == 'process-type':
            #import pdb; pdb.set_trace()
            pt = node.process_type
            if pt in exploded:
                explode = False
            else:
                exploded.append(pt)
        if explode:
            for kid in node.xbill_child_object().xbill_children():
                explode_xbill_children(kid, nodes, exploded)

def generate_xbill(resource_type):
    nodes = []
    exploded = []
    for kid in resource_type.xbill_children():
        explode_xbill_children(kid, nodes, exploded)
    nodes = list(set(nodes))
    #import pdb; pdb.set_trace()
    to_return = []
    visited = []
    for kid in resource_type.xbill_children():
        to_return.extend(xbill_dfs(kid, nodes, visited, 1))
    annotate_tree_properties(to_return)
    #to_return.sort(lambda x, y: cmp(x.xbill_object().name,
    #                                y.xbill_object().name))
    return to_return

#adapted from threaded_comments.util
def annotate_tree_properties(nodes):
    """
    iterate through nodes and adds some magic properties to each of them
    representing opening list of children and closing it
    """
    if not nodes:
        return

    it = iter(nodes)

    # get the first item, this will fail if no items !
    old = it.next()

    # first item starts a new thread
    old.open = True
    for c in it:

        # increase the depth
        if c.depth > old.depth:
            c.open = True

        else: # c.depth <= old.depth
            # close some depths
            old.close = range(old.depth - c.depth)

        # iterate
        old = c

    old.close = range(old.depth)

<|endoftext|>


<commit_before>class Landscape(object):
    def __init__(self, filename):
        self.landscape = self.load_landscape(filename)
        self.dry_squares = self.find_dry_squares()

    def load_landscape(self, filename):
        print('Loading landscape')
        return []

    def find_dry_squares(self):
        print('calculating number of dry squares')
        return []

class Population(object):
    def __init__(self, Landscape, birth, death, diffusion, min_ro, max_ro):
        self.min_ro = min_ro
        self.max_ro = max_ro
        self.birth = birth
        self.death = death
        self.diffusion = diffusion
        self.density = self.random_density(Landscape)

    def random_density(self, Landscape):
        min_ro = self.min_ro
        max_ro = self.max_ro
        grid = Landscape.landscape
        print('Random distribution')

class PumaPopulation(Population):
    def __init__(self, Landscape, birth=1.01, death=2.02, diffusion=3.03,
                 min_ro=0, max_ro=5):
        self.kind = 'PumaPopulation'
        super(PumaPopulation, self).__init__(Landscape, birth, death,
                                             diffusion, min_ro, max_ro)
        print('Puma population created')

    # alternatively
    # def update_density(self, puma_pop, hare_pop):
    # which is slightly more messy to implement as you need to keep track of
    # arguments and their order but is arguably more efficient (how much?)
    def update_density(self, populations):
        puma_pop = next((pop for pop in populations if
                    pop.kind=='PumaPopulation'), None)
        print(puma_pop.kind)
        hare_pop = next((pop for pop in populations if
                    pop.kind=='HarePopulation'), None) # None is default value
        # set X_pop.density to zeroes if no pop in populations
        print(self.kind +' density updated')
        # set ro=0 if ro<0

# class Environment() ?
class Simulation():
    def __init__(self, Landscape, *args):
        # create populations list but ignore args which are not populations
        self.populations = [pop for pop in args if isinstance(pop, Population)]
        print('Running simulation')

    # not sure is this function required
    def add_population(self, pop):
        return self.populations.append(pop)

    # not sure is this function required
    def remove_population(self, pop):
        print(pop.kind + ' removed')

    def update(self):
        for pop in self.populations:
            pop.update_density(self.populations)

# create new landscape from the file 'my_land'
env = Landscape('my_land')

# create new population using default values
puma_pop = PumaPopulation(env)
print(puma_pop.birth)
print(puma_pop.death)
print(puma_pop.max_ro)
print('#######')

# create new population using specific values (config file...)
puma_pop = PumaPopulation(env, death=9.999, max_ro=55)
print(puma_pop.birth)
print(puma_pop.death)
print(puma_pop.max_ro)

# create new simulation with the landscape env and puma population
sim = Simulation(env, puma_pop)
# update one step
sim.update()
<commit_msg>Example class structure<commit_after>import numpy as np
class Landscape(object):
    def __init__(self, filename):
        self.landscape = self.load_landscape(filename)
        self.dry_squares = self.find_dry_squares()

    def load_landscape(self, filename):
        print('Loading landscape')
        return []

    def find_dry_squares(self):
        print('calculating number of dry squares')
        return []

class Population(object):
    def __init__(self, Landscape, birth, death, diffusion, min_ro, max_ro):
        self.min_ro = min_ro
        self.max_ro = max_ro
        self.birth = birth
        self.death = death
        self.diffusion = diffusion
        self.density = self.random_density(Landscape)

    def random_density(self, Landscape):
        min_ro = self.min_ro
        max_ro = self.max_ro
        grid = Landscape.landscape
        print('Random distribution')
        return np.array([[1,2], [3,4]])

class PumaPopulation(Population):
    def __init__(self, Landscape, birth=1.01, death=2.02, diffusion=3.03,
                 min_ro=0, max_ro=5):
        self.kind = 'PumaPopulation'
        super(PumaPopulation, self).__init__(Landscape, birth, death,
                                             diffusion, min_ro, max_ro)
        print('Puma population created')

    # alternatively
    # def update_density(self, puma_pop, hare_pop):
    # which is slightly more messy to implement as you need to keep track of
    # arguments and their order but is arguably more efficient (how much?)
    def update_density(self, populations):
        # set X_pop.density to zeroes if no pop in populations
        # this should allow computation to proceed (verify that)
        zero_density_arr = np.zeros(self.density.shape)
        puma_pop = next((pop.density for pop in populations if
                    pop.kind=='PumaPopulation'), zero_density_arr)
        hare_pop = next((pop.density for pop in populations if
                    pop.kind=='HarePopulation'), zero_density_arr)
        print(hare_pop)
        print(puma_pop)
        print(self.kind +' density updated')
        # set ro=0 if ro<0

# class Environment() ?
class Simulation():
    def __init__(self, Landscape, *args):
        # create populations list but ignore args which are not populations
        self.populations = [pop for pop in args if isinstance(pop, Population)]
        print('Running simulation')

    # not sure is this function required
    def add_population(self, pop):
        return self.populations.append(pop)

    # not sure is this function required
    def remove_population(self, pop):
        print(pop.kind + ' removed')

    def update(self):
        for pop in self.populations:
            pop.update_density(self.populations)

# create new landscape from the file 'my_land'
env = Landscape('my_land')

# create new population using default values
puma_pop = PumaPopulation(env)
print(puma_pop.birth)
print(puma_pop.death)
print(puma_pop.max_ro)
print('#######')

# create new population using specific values (config file...)
puma_pop = PumaPopulation(env, death=9.999, max_ro=55)
print(puma_pop.birth)
print(puma_pop.death)
print(puma_pop.max_ro)

# create new simulation with the landscape env and puma population
sim = Simulation(env, puma_pop)
# update one step
sim.update()
<|endoftext|>