<commit_before># Copyright (C) 2014 Linaro Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Container for all the import related functions."""

import os
import pymongo

from datetime import datetime

from models import DB_NAME
from models.defconfig import (
    DEFCONFIG_ACCEPTED_FILES,
    DefConfigDocument,
)
from models.job import JobDocument
from utils.db import save
from utils.utc import utc

BASE_PATH = '/var/www/images/kernel-ci'

def import_job_from_json(
        json_obj, database, base_path=BASE_PATH, callback=None):
    """Import a job based on the provided JSON object.

    The provided JSON object, a dict-like object, should contain at least the
    `job' and `kernel' keys. These keys will be used to traverse the directory
    structure found at the `/job/kernel' path (starting from the default
    location).

    :param json_obj: A dict-like object, that should contain the keys `job' and
                     `kernel'.
    :param database: The database where to save the parsed data.
    :param base_path: The base path where to start constructing the traverse
                      directory. It defaults to: /var/www/images/kernel-ci.
    :param callback: Optional function that will be called at the end of
                     execution.
    """
    job_dir = json_obj['job']
    kernel_dir = json_obj['kernel']

    return_code = _import_job(job_dir, kernel_dir, database, base_path)

    if callback:
        callback(return_code)

def _import_job(job, kernel, database, base_path=BASE_PATH):
    """Traverse the job dir and create the documenst to save.

    :param job: The name of the job.
    :param kernel: The name of the kernel.
    :param database: The databse where to save the parsed data.
    :param base_path: The base path where to strat the traversing.
    :return Return a status code based on the save operation.
    """
    job_dir = os.path.join(base_path, job, kernel)
    job_id = JobDocument.JOB_ID_FORMAT % (job, kernel)

    docs = []
    doc = JobDocument(job_id, job=job, kernel=kernel)
    doc.created = datetime.now(tz=utc).isoformat()

    docs.append(doc)

    if os.path.isdir(job_dir):
        docs.extend(_traverse_defconf_dir(job_dir, job_id))

    return save(database, docs)

def _traverse_defconf_dir(kernel_dir, job_id):
    """Traverse the defconfing directories.

    :param kernel_dir: The kernel dir where to start.
    :param job_id: The id of the JobDocument the defconfing will be linked to.
    :return A list of documents.
    """
    defconf_docs = []
    for defconf_dir in os.listdir(kernel_dir):
        defconf_doc = DefConfigDocument(defconf_dir)
        defconf_doc.job_id = job_id

        for dirname, subdirs, files in os.walk(
                os.path.join(kernel_dir, defconf_dir)):
            # Consider only the actual directory and its files.
            subdirs[:] = []
            for key, val in DEFCONFIG_ACCEPTED_FILES.iteritems():
                if key in files:
                    setattr(defconf_doc, val, os.path.join(dirname, key))
        defconf_docs.append(defconf_doc)
    return defconf_docs

def _import_all(base_path=BASE_PATH):
    """This function is used only to trigger the import from the command line.

    Do not use it elsewhere.
    :param base_path: Where to start traversing directories. Defaults to:
                      /var/www/images/kernel-ci.
    :return The docs to save. All docs are subclasses of BaseDocument.
    """

    docs = []

    for job_dir in os.listdir(base_path):
        job_id = job_dir
        job_dir = os.path.join(base_path, job_dir)

        for kernel_dir in os.listdir(job_dir):
            doc_id = JobDocument.JOB_ID_FORMAT % (job_id, kernel_dir)
            job_doc = JobDocument(doc_id, job=job_id, kernel=kernel_dir)
            job_doc.created = datetime.now(tz=utc).isoformat()
            docs.append(job_doc)

            kernel_dir = os.path.join(job_dir, kernel_dir)

            docs.extend(_traverse_defconf_dir(kernel_dir, doc_id))

    return docs

if __name__ == '__main__':
    connection = pymongo.MongoClient()
    database = connection[DB_NAME]

    documents = _import_all()
    save(database, documents)

    connection.disconnect()
<commit_msg>Remove callback arguments, make functions return docs.<commit_after># Copyright (C) 2014 Linaro Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Container for all the import related functions."""

import os
import pymongo

from datetime import datetime

from models import DB_NAME
from models.defconfig import (
    DEFCONFIG_ACCEPTED_FILES,
    DefConfigDocument,
)
from models.job import JobDocument
from utils.db import save
from utils.utc import utc

BASE_PATH = '/var/www/images/kernel-ci'

def import_job_from_json(json_obj, base_path=BASE_PATH):
    """Import a job based on the provided JSON object.

    The provided JSON object, a dict-like object, should contain at least the
    `job' and `kernel' keys. These keys will be used to traverse the directory
    structure found at the `/job/kernel' path (starting from the default
    location).

    :param json_obj: A dict-like object, that should contain the keys `job' and
                     `kernel'.
    :param base_path: The base path where to start constructing the traverse
                      directory. It defaults to: /var/www/images/kernel-ci.
    :return The documents to be save.
    """
    job_dir = json_obj['job']
    kernel_dir = json_obj['kernel']

    return _import_job(job_dir, kernel_dir, base_path)

def _import_job(job, kernel, base_path=BASE_PATH):
    """Traverse the job dir and create the documenst to save.

    :param job: The name of the job.
    :param kernel: The name of the kernel.
    :param base_path: The base path where to strat the traversing.
    :return The documents to be save.
    """
    job_dir = os.path.join(base_path, job, kernel)
    job_id = JobDocument.JOB_ID_FORMAT % (job, kernel)

    docs = []
    doc = JobDocument(job_id, job=job, kernel=kernel)
    doc.created = datetime.now(tz=utc).isoformat()

    docs.append(doc)

    if os.path.isdir(job_dir):
        docs.extend(_traverse_defconf_dir(job_dir, job_id))

    return docs

def _traverse_defconf_dir(kernel_dir, job_id):
    """Traverse the defconfing directories.

    :param kernel_dir: The kernel dir where to start.
    :param job_id: The id of the JobDocument the defconfing will be linked to.
    :return A list of documents.
    """
    defconf_docs = []
    for defconf_dir in os.listdir(kernel_dir):
        defconf_doc = DefConfigDocument(defconf_dir)
        defconf_doc.job_id = job_id

        for dirname, subdirs, files in os.walk(
                os.path.join(kernel_dir, defconf_dir)):
            # Consider only the actual directory and its files.
            subdirs[:] = []
            for key, val in DEFCONFIG_ACCEPTED_FILES.iteritems():
                if key in files:
                    setattr(defconf_doc, val, os.path.join(dirname, key))
        defconf_docs.append(defconf_doc)

    return defconf_docs

def _import_all(base_path=BASE_PATH):
    """This function is used only to trigger the import from the command line.

    Do not use it elsewhere.
    :param base_path: Where to start traversing directories. Defaults to:
                      /var/www/images/kernel-ci.
    :return The docs to save. All docs are subclasses of BaseDocument.
    """

    docs = []

    for job_dir in os.listdir(base_path):
        job_id = job_dir
        job_dir = os.path.join(base_path, job_dir)

        for kernel_dir in os.listdir(job_dir):
            doc_id = JobDocument.JOB_ID_FORMAT % (job_id, kernel_dir)
            job_doc = JobDocument(doc_id, job=job_id, kernel=kernel_dir)
            job_doc.created = datetime.now(tz=utc).isoformat()
            docs.append(job_doc)

            kernel_dir = os.path.join(job_dir, kernel_dir)

            docs.extend(_traverse_defconf_dir(kernel_dir, doc_id))

    return docs

if __name__ == '__main__':
    connection = pymongo.MongoClient()
    database = connection[DB_NAME]

    documents = _import_all()
    save(database, documents)

    connection.disconnect()
<|endoftext|>


<commit_before>"""
Handle the control command.
"""

__author__     = "Jon Bringhurst <jon@bringhurst.org>"
__copyright__  = "Copyright 2013 Los Alamos National Security, LLC."
__license__    = "Apache License, Version 2.0"

import shlex

from ..protocol import HavenProtocol

class CommandControl():
    """Haven control command handler."""

    def handle(self, ctl, line):
        args = shlex.split(line)
        uuid = None
        path = None

        if len(args) < 1:
            print("The control command requires arguments. Please see 'help control'.")
            return

        opts = {
            "attach":   HavenProtocol.HVN_CLNT_PROTO_CTRL_ATTACH,
            "destroy":  HavenProtocol.HVN_CLNT_PROTO_CTRL_DESTROY,
            "exit":     HavenProtocol.HVN_CLNT_PROTO_CTRL_EXIT,
            "follower": HavenProtocol.HVN_CLNT_PROTO_CTRL_FOLLOWER,
            "leader":   HavenProtocol.HVN_CLNT_PROTO_CTRL_LEADER,
            "location": HavenProtocol.HVN_CLNT_PROTO_CTRL_LOCATION,
            "proxy":    HavenProtocol.HVN_CLNT_PROTO_CTRL_PROXY,
            "router":   HavenProtocol.HVN_CLNT_PROTO_CTRL_ROUTER
        }

        if args[0].lower() in opts.keys():
            protocol_key = opts[args[0].lower()]
        else:
            print("Command `" + str(args[0]) + "' not recognized.")
            print("Please see 'help control'.")
            return

        if protocol_key == HavenProtocol.HVN_CLNT_PROTO_CTRL_FOLLOWER:
            if len(args) < 2:
                print("The follower subcommand requires a uuid argument.")
                print("Please see 'help control'.")
                return
            uuid = args[1]

        if protocol_key == HavenProtocol.HVN_CLNT_PROTO_CTRL_LEADER:
            if len(args) < 2:
                print("The leader subcommand requires a path argument.")
                print("Please see 'help control'.")
                return
            path = args[1]

        if ctl.conn.is_connected:
            ctl.proto.send_control(ctl.conn, protocol_key, uuid, path)
            ctl.proto.recv_control(ctl.conn)
        else:
            print("Connection to server failed.")

# EOF
<commit_msg>Recognized UUID on control attach command.<commit_after>"""
Handle the control command.
"""

__author__     = "Jon Bringhurst <jon@bringhurst.org>"
__copyright__  = "Copyright 2013 Los Alamos National Security, LLC."
__license__    = "Apache License, Version 2.0"

import shlex

from ..protocol import HavenProtocol

class CommandControl():
    """Haven control command handler."""

    def handle(self, ctl, line):
        args = shlex.split(line)
        uuid = None
        path = None

        if len(args) < 1:
            print("The control command requires arguments. Please see 'help control'.")
            return

        opts = {
            "attach":   HavenProtocol.HVN_CLNT_PROTO_CTRL_ATTACH,
            "destroy":  HavenProtocol.HVN_CLNT_PROTO_CTRL_DESTROY,
            "exit":     HavenProtocol.HVN_CLNT_PROTO_CTRL_EXIT,
            "follower": HavenProtocol.HVN_CLNT_PROTO_CTRL_FOLLOWER,
            "leader":   HavenProtocol.HVN_CLNT_PROTO_CTRL_LEADER,
            "location": HavenProtocol.HVN_CLNT_PROTO_CTRL_LOCATION,
            "proxy":    HavenProtocol.HVN_CLNT_PROTO_CTRL_PROXY,
            "router":   HavenProtocol.HVN_CLNT_PROTO_CTRL_ROUTER
        }

        if args[0].lower() in opts.keys():
            protocol_key = opts[args[0].lower()]
        else:
            print("Command `" + str(args[0]) + "' not recognized.")
            print("Please see 'help control'.")
            return

        if protocol_key == HavenProtocol.HVN_CLNT_PROTO_CTRL_ATTACH:
            if len(args) < 2:
                print("The attach subcommand requires a uuid argument.")
                print("Please see 'help control'.")
                return
            uuid = args[1]

        if protocol_key == HavenProtocol.HVN_CLNT_PROTO_CTRL_FOLLOWER:
            if len(args) < 2:
                print("The follower subcommand requires a uuid argument.")
                print("Please see 'help control'.")
                return
            uuid = args[1]

        if protocol_key == HavenProtocol.HVN_CLNT_PROTO_CTRL_LEADER:
            if len(args) < 2:
                print("The leader subcommand requires a path argument.")
                print("Please see 'help control'.")
                return
            path = args[1]

        if ctl.conn.is_connected:
            ctl.proto.send_control(ctl.conn, protocol_key, uuid, path)
            ctl.proto.recv_control(ctl.conn)
        else:
            print("Connection to server failed.")

# EOF
<|endoftext|>


<commit_before>from qtpy.QtWidgets import QTableWidgetItem, QHBoxLayout, QMainWindow
from qtpy.QtGui import QColor
from qtpy import QtCore
import numpy as np

from .. import load_ui
from src.iBeatles.step6.initialization import Initialization

class StrainMappingLauncher(object):

    def __init__(self, parent=None):
        self.parent = parent

        # if self.parent.strain_mapping_ui is None:
        strain_mapping_window = StrainMappingWindow(parent=parent)
        strain_mapping_window.show()
        self.parent.strain_mapping_ui = strain_mapping_window
        # else:
        #     self.parent.strain_mapping_ui.setFocus()
        #     self.parent.strain_mapping_ui.activateWindow()

class StrainMappingWindow(QMainWindow):
    colorscale_nbr_row = 15
    colorscale_cell_size = {'width': 50,
                            'height': 30}

    def __init__(self, parent=None):

        self.parent = parent
        QMainWindow.__init__(self, parent=parent)
        self.ui = load_ui('ui_strainMapping.ui', baseinstance=self)
        self.setWindowTitle("6. Strain Mapping")

        o_init = Initialization(parent=self, grand_parent=self.parent)
        o_init.all()

        # self.init_labels()
        # # self.init_widgets()
        # self.display_images_and_selection()

    def display_images_and_selection(self):
        self.display_images()
        self.display_selections()

    def get_min_max(self, table=[], item='strain_mapping'):
        if item == 'strain_mapping':
            item = 'd_spacing'

        min_value = 1e6
        max_value = -1e6
        # is_first_value = True
        for _index in table:
            _entry = table[_index]

            value = _entry[item]['val']
            if min_value > value:
                min_value = value
            elif max_value < value:
                max_value = value

        return [min_value, max_value]

    def clear_colorscale_table(self, item='strain_mapping', table_ui=None):
        nbr_row = table_ui.rowCount()
        for _row in np.arange(nbr_row):
            table_ui.removeRow(0)

    def initialize_colorscale_table(self, min_value=0,
                                    max_value=0,
                                    item='strain_mapping'):

        if item == 'strain_mapping':
            _table_ui = self.ui.strain_mapping_colorbar
        elif item == 'sigma':
            _table_ui = self.ui.alpha_colorbar
        else:
            _table_ui = self.ui.sigma_colorbar

        self.clear_colorscale_table(item=item, table_ui=_table_ui)

        nbr_row = self.colorscale_nbr_row
        step = (float(max_value) - float(min_value)) / (nbr_row - 1)
        mid_point = np.int(nbr_row / 2)
        _row = 0

        if min_value == max_value:
            nbr_row = 1

        if np.isnan(step):
            nbr_row = 1

        for _index in np.arange(nbr_row - 1, -1, -1):
            _table_ui.insertRow(_row)
            _table_ui.setRowHeight(_row, self.colorscale_cell_size['height'])
            _table_ui.setColumnWidth(_row, self.colorscale_cell_size['width'])
            if np.isnan(step):
                _value = np.NaN
            else:
                _value = min_value + _index * step
            _color = self.get_color_for_this_value(min_value=min_value,
                                                   max_value=max_value,
                                                   value=_value)

            if np.isnan(_value):
                _item = QTableWidgetItem("nan")
            else:
                _item = QTableWidgetItem("{:04.2f}".format(_value))
            _item.setBackgroundColor(_color)
            _item.setTextAlignment(QtCore.Qt.AlignRight)
            if (_row < mid_point) and (nbr_row != 1):
                # font should be changd from black to white
                _foreground_color = QColor(255, 255, 255, alpha=255)
                _item.setTextColor(_foreground_color)

            _table_ui.setItem(_row, 0, _item)
            _row += 1

    def get_color_for_this_value(self, min_value=0, max_value=1, value=0):
        if np.isnan(value):
            return QColor(255, 255, 255, alpha=100)  # white
        elif max_value == min_value:
            return QColor(250, 0, 0, alpha=255)  # red

        _ratio = (1 - (float(value) - float(min_value)) / (float(max_value) - float(min_value)))
        return QColor(0, _ratio * 255, 0, alpha=255)

    def display_selections(self, item='strain_mapping'):

        # strain mapping
        table_dictionary = self.parent.table_dictionary

        [min_value, max_value] = self.get_min_max(table=table_dictionary,
                                                  item=item)

        print("min_value: {} and max_value: {}".format(min_value, max_value))

        self.initialize_colorscale_table(min_value=min_value,
                                         max_value=max_value,
                                         item=item)

        for _index in table_dictionary:
            _entry = table_dictionary[_index]
            x0 = _entry['bin_coordinates']['x0']
            x1 = _entry['bin_coordinates']['x1']
            y0 = _entry['bin_coordinates']['y0']
            y1 = _entry['bin_coordinates']['y1']

            rect = pg.QtGui.QGraphicsRectItem(x0, y0, x1 - x0, y1 - y0)
            rect.setPen(pg.mkPen(None))
            rect.setBrush(pg.mkBrush('r'))
            self.ui.strain_mapping_image.addItem(rect)
            return

    def display_images(self):
        _data = self.parent.data_metadata['normalized']['data_live_selection']
        if not _data == []:
            self.ui.strain_mapping_image.setImage(_data)
            self.ui.sigma_image.setImage(_data)
            self.ui.alpha_image.setImage(_data)

    def set_tab_widgets(self):
        image_view = pg.ImageView()
        image_view.ui.roiBtn.hide()
        image_view.ui.menuBtn.hide()
        hori_layout = QHBoxLayout()
        hori_layout.addWidget(image_view)

        return {'layout': hori_layout,
                'image_view': image_view,
                }

    def init_labels(self):
        self.ui.d0_label.setText(u"d0")
        self.ui.d0_units_label.setText(u"\u212B")

    def closeEvent(self, event=None):
        if self.parent.strain_mapping_ui:
            self.parent.strain_mapping_ui.close()
        self.parent.strain_mapping_ui = None
<commit_msg>window can be launched only if fitting window exists. this refs #94<commit_after>from qtpy.QtWidgets import QTableWidgetItem, QHBoxLayout, QMainWindow
from qtpy.QtGui import QColor
from qtpy import QtCore
import numpy as np

from .. import load_ui
from src.iBeatles.step6.initialization import Initialization
from src.iBeatles.utilities.status_message_config import StatusMessageStatus, show_status_message

class StrainMappingLauncher(object):

    def __init__(self, parent=None):
        self.parent = parent

        if self.parent.fitting_ui is None:
            show_status_message(parent=self.parent,
                                message="Strain Mapping requiere to first launch the fitting window!",
                                status=StatusMessageStatus.error,
                                duration_s=10)
        else:
            strain_mapping_window = StrainMappingWindow(parent=parent)
            strain_mapping_window.show()
            self.parent.strain_mapping_ui = strain_mapping_window

class StrainMappingWindow(QMainWindow):
    colorscale_nbr_row = 15
    colorscale_cell_size = {'width': 50,
                            'height': 30}

    def __init__(self, parent=None):

        self.parent = parent
        QMainWindow.__init__(self, parent=parent)
        self.ui = load_ui('ui_strainMapping.ui', baseinstance=self)
        self.setWindowTitle("6. Strain Mapping")

        o_init = Initialization(parent=self, grand_parent=self.parent)
        o_init.all()

        # self.init_labels()
        # # self.init_widgets()
        # self.display_images_and_selection()

    def display_images_and_selection(self):
        self.display_images()
        self.display_selections()

    def get_min_max(self, table=[], item='strain_mapping'):
        if item == 'strain_mapping':
            item = 'd_spacing'

        min_value = 1e6
        max_value = -1e6
        # is_first_value = True
        for _index in table:
            _entry = table[_index]

            value = _entry[item]['val']
            if min_value > value:
                min_value = value
            elif max_value < value:
                max_value = value

        return [min_value, max_value]

    def clear_colorscale_table(self, item='strain_mapping', table_ui=None):
        nbr_row = table_ui.rowCount()
        for _row in np.arange(nbr_row):
            table_ui.removeRow(0)

    def initialize_colorscale_table(self, min_value=0,
                                    max_value=0,
                                    item='strain_mapping'):

        if item == 'strain_mapping':
            _table_ui = self.ui.strain_mapping_colorbar
        elif item == 'sigma':
            _table_ui = self.ui.alpha_colorbar
        else:
            _table_ui = self.ui.sigma_colorbar

        self.clear_colorscale_table(item=item, table_ui=_table_ui)

        nbr_row = self.colorscale_nbr_row
        step = (float(max_value) - float(min_value)) / (nbr_row - 1)
        mid_point = np.int(nbr_row / 2)
        _row = 0

        if min_value == max_value:
            nbr_row = 1

        if np.isnan(step):
            nbr_row = 1

        for _index in np.arange(nbr_row - 1, -1, -1):
            _table_ui.insertRow(_row)
            _table_ui.setRowHeight(_row, self.colorscale_cell_size['height'])
            _table_ui.setColumnWidth(_row, self.colorscale_cell_size['width'])
            if np.isnan(step):
                _value = np.NaN
            else:
                _value = min_value + _index * step
            _color = self.get_color_for_this_value(min_value=min_value,
                                                   max_value=max_value,
                                                   value=_value)

            if np.isnan(_value):
                _item = QTableWidgetItem("nan")
            else:
                _item = QTableWidgetItem("{:04.2f}".format(_value))
            _item.setBackgroundColor(_color)
            _item.setTextAlignment(QtCore.Qt.AlignRight)
            if (_row < mid_point) and (nbr_row != 1):
                # font should be changd from black to white
                _foreground_color = QColor(255, 255, 255, alpha=255)
                _item.setTextColor(_foreground_color)

            _table_ui.setItem(_row, 0, _item)
            _row += 1

    def get_color_for_this_value(self, min_value=0, max_value=1, value=0):
        if np.isnan(value):
            return QColor(255, 255, 255, alpha=100)  # white
        elif max_value == min_value:
            return QColor(250, 0, 0, alpha=255)  # red

        _ratio = (1 - (float(value) - float(min_value)) / (float(max_value) - float(min_value)))
        return QColor(0, _ratio * 255, 0, alpha=255)

    def display_selections(self, item='strain_mapping'):

        # strain mapping
        table_dictionary = self.parent.table_dictionary

        [min_value, max_value] = self.get_min_max(table=table_dictionary,
                                                  item=item)

        print("min_value: {} and max_value: {}".format(min_value, max_value))

        self.initialize_colorscale_table(min_value=min_value,
                                         max_value=max_value,
                                         item=item)

        for _index in table_dictionary:
            _entry = table_dictionary[_index]
            x0 = _entry['bin_coordinates']['x0']
            x1 = _entry['bin_coordinates']['x1']
            y0 = _entry['bin_coordinates']['y0']
            y1 = _entry['bin_coordinates']['y1']

            rect = pg.QtGui.QGraphicsRectItem(x0, y0, x1 - x0, y1 - y0)
            rect.setPen(pg.mkPen(None))
            rect.setBrush(pg.mkBrush('r'))
            self.ui.strain_mapping_image.addItem(rect)
            return

    def display_images(self):
        _data = self.parent.data_metadata['normalized']['data_live_selection']
        if not _data == []:
            self.ui.strain_mapping_image.setImage(_data)
            self.ui.sigma_image.setImage(_data)
            self.ui.alpha_image.setImage(_data)

    def set_tab_widgets(self):
        image_view = pg.ImageView()
        image_view.ui.roiBtn.hide()
        image_view.ui.menuBtn.hide()
        hori_layout = QHBoxLayout()
        hori_layout.addWidget(image_view)

        return {'layout': hori_layout,
                'image_view': image_view,
                }

    def init_labels(self):
        self.ui.d0_label.setText(u"d0")
        self.ui.d0_units_label.setText(u"\u212B")

    def closeEvent(self, event=None):
        if self.parent.strain_mapping_ui:
            self.parent.strain_mapping_ui.close()
        self.parent.strain_mapping_ui = None
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

from collections import defaultdict, namedtuple
import random
import sys
import kevlar
from kevlar.call import Variant

# Mappings for SNVs
nucl_to_index = {'A': 0, 'C': 1, 'G': 2, 'T': 3}
index_to_nucl = {0: 'A', 1: 'C', 2: 'G', 3: 'T'}

"""
Viable inheritance scenarios

Each scenario is a 3-tuple showing the genotype of the trio for a particular
variant (child, mother, father).
- 0 refers to homozygous reference (0/0)
- 1 refers to heterzygous (0/1 or 1/0)
- 2 refers to homozygous alternate (1/1)

Combinations of genotypes that are not valid inheritance scenarios are
excluded.
"""
inheritance_scenarios = [

    (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 0, 1), (1, 0, 2), (1, 1, 0),
    (1, 1, 1), (1, 1, 2), (1, 2, 0), (1, 2, 1), (2, 1, 1), (2, 1, 2),
    (2, 2, 1), (2, 2, 2),
]

def weighted_choice(values, weights, rng=random.Random()):
    """Stolen shamelessly from https://stackoverflow.com/a/3679747/459780."""
    assert len(values) == len(weights)
    total = sum(weights)
    r = rng.uniform(0, total)
    cumsum = 0
    for v, w in zip(values, weights):
        if cumsum + w >= r:
            return v
        cumsum += w
    assert False  # pragma: no cover

def mutate_snv(sequence, position, offset, ksize=31):
    orignucl = sequence[position]
    nuclindex = nucl_to_index[orignucl]
    newindex = (nuclindex + offset) % 4
    newnucl = index_to_nucl[newindex]

    windowstart = max(position - ksize + 1, 0)
    windowend = min(position + ksize, len(sequence))
    refrwindow = sequence[windowstart:windowend]
    altwindow = '{:s}{:s}{:s}'.format(
        sequence[windowstart:position], newnucl,
        sequence[position+1:windowend]
    )

    return orignucl, newnucl, refrwindow, altwindow

def mutate_insertion(sequence, position, length, duplpos, ksize=31):
    duplseq = sequence[duplpos:duplpos + length]
    refrseq = sequence[position - 1]
    altseq = refrseq + duplseq

    windowstart = max(position - ksize + 1, 0)
    windowend = min(position + ksize - 1, len(sequence))
    refrwindow = sequence[windowstart:windowend]
    altwindow = '{:s}{:s}{:s}'.format(
        sequence[windowstart:position], duplseq, sequence[position:windowend]
    )

    return refrseq, altseq, refrwindow, altwindow

def mutate_deletion(sequence, position, length, ksize=31):
    delseq = sequence[position:position + length]
    altseq = sequence[position-1]
    refrseq = altseq + delseq

    windowstart = max(position - ksize + 1, 0)
    windowend = min(position + length + ksize - 1, len(sequence))
    refrwindow = sequence[windowstart:windowend]
    altwindow = '{:s}{:s}'.format(
        sequence[windowstart:position], sequence[position + length:windowend]
    )

    return refrseq, altseq, refrwindow, altwindow

def generate_mutations(sequences, n=10, inversions=False, ksize=31, rng=None):
    if not rng:
        seed = random.randrange(sys.maxsize)
        print('[kevlar::gentrio] using random seed', seed, file=sys.stderr)
        rng = random.Random(seed)
    elif isinstance(rng, int):
        rng = random.Random(rng)

    types = ['snv', 'ins', 'del']
    weights = [0.7, 0.15, 0.15]
    if inversions:
        # types.append('inv')
        raise NotImplementedError('feature pending')

    for _ in range(n):
        seqid = rng.choice(list(sequences.keys()))
        seq = sequences[seqid]
        seqlength = len(sequences[seqid])
        position = rng.randint(0, seqlength)
        muttype = weighted_choice(types, weights, rng)

        if muttype == 'snv':
            offset = rng.randint(1, 3)
            refrseq, altseq, refrwindow, altwindow = mutate_snv(
                seq, position, offset, ksize
            )
        elif muttype == 'ins':
            length = rng.randint(5, 350)
            duplpos = rng.randint(0, seqlength)
            refrseq, altseq, refrwindow, altwindow = mutate_insertion(
                seq, position, length, duplpos, ksize
            )
        else:
            assert muttype == 'del'
            length = rng.randint(5, 350)
            refrseq, altseq, refrwindow, altwindow = mutate_deletion(
                seq, position, length, ksize
            )
        yield Variant(seqid, position, refrseq, altseq, VW=altwindow,
                      RW=refrwindow)

def pick_inheritance_genotypes(rng):
    genotype_codes = rng.choice(inheritance_scenarios)
    genotypes = list()
    for code in genotype_codes:
        if code == 0:
            genotype = '0/0'
        elif code == 2:
            genotype = '1/1'
        else:
            genotype = rng.choice(['0/1', '1/0'])
        genotypes.append(genotype)
    return tuple(genotypes)

def simulate_variant_genotypes(sequences, ninh=20, ndenovo=10, rng=None):
    if rng is None:
        seed = random.randrange(sys.maxsize)
        print('[kevlar::gentrio] using random seed', seed, file=sys.stderr)
        rng = random.Random(seed)
    if isinstance(rng, int):
        rng = random.Random(rng)

    mutator = generate_mutations(sequences, n=ninh, rng=rng)
    for variant in mutator:
        genotypes = pick_inheritance_genotypes(rng)
        gtstring = ','.join(genotypes)
        variant.info['GT'] = gtstring
        yield variant

    mutator = generate_mutations(sequences, n=ndenovo, rng=rng)
    for variant in mutator:
        genotypes = (rng.choice(['0/1', '1/0']), '0/0', '0/0')
        gtstring = ','.join(genotypes)
        variant.info['GT'] = gtstring
        yield variant

def apply_mutation(sequence, position, refr, alt):
    prefix = sequence[:position]
    if len(refr) == len(alt):  # SNV
        suffix = sequence[position+1:]
        return prefix + alt + suffix
    elif len(refr) < len(alt):  # Insertion
        suffix = sequence[position+1:]
        return prefix + alt[1:] + suffix
    else:  # Deletion
        dellength = len(refr) - len(alt)
        suffix = sequence[position+dellength:]
        return prefix + suffix

def gentrio(sequences, outstreams, ninh=20, ndenovo=10, seed=None):
    assert len(outstreams) == 3
    mutator = simulate_variant_genotypes(
        sequences, ninh=ninh, ndenovo=ndenovo, rng=seed
    )
    variants = list(mutator)
    variants.sort(key=lambda v: v.position, reverse=True)

    for seqid, sequence in sequences.items():
        for ind in range(3):  # proband mother father
            haploseqs = [str(sequence), str(sequence)]
            for variant in variants:
                if variant.seqid != seqid:
                    continue
                genotype = variant.genotypes[ind]
                haplotypes = (genotype[0], genotype[2])
                for hapindex in range(2):
                    if haplotypes[hapindex] == '0':
                        continue
                    haploseqs[hapindex] = apply_mutation(
                        haploseqs[hapindex], variant.position, variant._refr,
                        variant._alt
                    )
            print('>', seqid, '_haplo1\n', haploseqs[0], sep='',
                  file=outstreams[ind])
            print('>', seqid, '_haplo2\n', haploseqs[1], sep='',
                  file=outstreams[ind])

    variants.sort(key=lambda v: (v.seqid, v.position))
    for variant in variants:
        yield variant

def main(args):
    seqfile = kevlar.open(args.genome, 'r')
    genomeseqs = kevlar.seqio.parse_seq_dict(seqfile)

    samples = ('proband', 'mother', 'father')
    outfiles = ['{:s}-{:s}.fasta'.format(args.prefix, s) for s in samples]
    outstreams = [kevlar.open(outfile, 'w') for outfile in outfiles]

    vcfout = None
    if args.vcf:
        vcfout = kevlar.open(args.vcf, 'w')

    if vcfout:
        kevlar.vcf_header(vcfout, source='kevlar::gentrio', infoheader=True)
    mutator = gentrio(genomeseqs, outstreams, ninh=args.inherited,
                      ndenovo=args.de_novo, seed=args.seed)
    for variant in mutator:
        if vcfout:
            print(variant.vcf, file=vcfout)

    for outstream in outstreams:
        outstream.close()
<commit_msg>An effort to ensure determinism across platforms<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/dib-lab/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

from collections import defaultdict, namedtuple
import random
import sys
import kevlar
from kevlar.call import Variant

# Mappings for SNVs
nucl_to_index = {'A': 0, 'C': 1, 'G': 2, 'T': 3}
index_to_nucl = {0: 'A', 1: 'C', 2: 'G', 3: 'T'}

"""
Viable inheritance scenarios

Each scenario is a 3-tuple showing the genotype of the trio for a particular
variant (child, mother, father).
- 0 refers to homozygous reference (0/0)
- 1 refers to heterzygous (0/1 or 1/0)
- 2 refers to homozygous alternate (1/1)

Combinations of genotypes that are not valid inheritance scenarios are
excluded.
"""
inheritance_scenarios = [

    (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 0, 1), (1, 0, 2), (1, 1, 0),
    (1, 1, 1), (1, 1, 2), (1, 2, 0), (1, 2, 1), (2, 1, 1), (2, 1, 2),
    (2, 2, 1), (2, 2, 2),
]

def weighted_choice(values, weights, rng=random.Random()):
    """Stolen shamelessly from https://stackoverflow.com/a/3679747/459780."""
    assert len(values) == len(weights)
    total = sum(weights)
    r = rng.uniform(0, total)
    cumsum = 0
    for v, w in zip(values, weights):
        if cumsum + w >= r:
            return v
        cumsum += w
    assert False  # pragma: no cover

def mutate_snv(sequence, position, offset, ksize=31):
    orignucl = sequence[position]
    nuclindex = nucl_to_index[orignucl]
    newindex = (nuclindex + offset) % 4
    newnucl = index_to_nucl[newindex]

    windowstart = max(position - ksize + 1, 0)
    windowend = min(position + ksize, len(sequence))
    refrwindow = sequence[windowstart:windowend]
    altwindow = '{:s}{:s}{:s}'.format(
        sequence[windowstart:position], newnucl,
        sequence[position+1:windowend]
    )

    return orignucl, newnucl, refrwindow, altwindow

def mutate_insertion(sequence, position, length, duplpos, ksize=31):
    duplseq = sequence[duplpos:duplpos + length]
    refrseq = sequence[position - 1]
    altseq = refrseq + duplseq

    windowstart = max(position - ksize + 1, 0)
    windowend = min(position + ksize - 1, len(sequence))
    refrwindow = sequence[windowstart:windowend]
    altwindow = '{:s}{:s}{:s}'.format(
        sequence[windowstart:position], duplseq, sequence[position:windowend]
    )

    return refrseq, altseq, refrwindow, altwindow

def mutate_deletion(sequence, position, length, ksize=31):
    delseq = sequence[position:position + length]
    altseq = sequence[position-1]
    refrseq = altseq + delseq

    windowstart = max(position - ksize + 1, 0)
    windowend = min(position + length + ksize - 1, len(sequence))
    refrwindow = sequence[windowstart:windowend]
    altwindow = '{:s}{:s}'.format(
        sequence[windowstart:position], sequence[position + length:windowend]
    )

    return refrseq, altseq, refrwindow, altwindow

def generate_mutations(sequences, n=10, inversions=False, ksize=31, rng=None):
    if rng is None:
        seed = random.randrange(sys.maxsize)
        print('[kevlar::gentrio] using random seed', seed, file=sys.stderr)
        rng = random.Random(seed)
    if isinstance(rng, int):
        rng = random.Random(rng)

    types = ['snv', 'ins', 'del']
    weights = [0.7, 0.15, 0.15]
    if inversions:
        # types.append('inv')
        raise NotImplementedError('feature pending')

    for _ in range(n):
        seqid = rng.choice(list(sorted(sequences.keys())))
        seq = sequences[seqid]
        seqlength = len(sequences[seqid])
        position = rng.randint(0, seqlength)
        muttype = weighted_choice(types, weights, rng)

        if muttype == 'snv':
            offset = rng.randint(1, 3)
            refrseq, altseq, refrwindow, altwindow = mutate_snv(
                seq, position, offset, ksize
            )
        elif muttype == 'ins':
            length = rng.randint(5, 350)
            duplpos = rng.randint(0, seqlength)
            refrseq, altseq, refrwindow, altwindow = mutate_insertion(
                seq, position, length, duplpos, ksize
            )
        else:
            assert muttype == 'del'
            length = rng.randint(5, 350)
            refrseq, altseq, refrwindow, altwindow = mutate_deletion(
                seq, position, length, ksize
            )
        yield Variant(seqid, position, refrseq, altseq, VW=altwindow,
                      RW=refrwindow)

def pick_inheritance_genotypes(rng):
    genotype_codes = rng.choice(inheritance_scenarios)
    genotypes = list()
    for code in genotype_codes:
        if code == 0:
            genotype = '0/0'
        elif code == 2:
            genotype = '1/1'
        else:
            genotype = rng.choice(['0/1', '1/0'])
        genotypes.append(genotype)
    return tuple(genotypes)

def simulate_variant_genotypes(sequences, ninh=20, ndenovo=10, rng=None):
    if rng is None:
        seed = random.randrange(sys.maxsize)
        print('[kevlar::gentrio] using random seed', seed, file=sys.stderr)
        rng = random.Random(seed)
    if isinstance(rng, int):
        rng = random.Random(rng)

    mutator = generate_mutations(sequences, n=ninh, rng=rng)
    for variant in mutator:
        genotypes = pick_inheritance_genotypes(rng)
        gtstring = ','.join(genotypes)
        variant.info['GT'] = gtstring
        yield variant

    mutator = generate_mutations(sequences, n=ndenovo, rng=rng)
    for variant in mutator:
        genotypes = (rng.choice(['0/1', '1/0']), '0/0', '0/0')
        gtstring = ','.join(genotypes)
        variant.info['GT'] = gtstring
        yield variant

def apply_mutation(sequence, position, refr, alt):
    prefix = sequence[:position]
    if len(refr) == len(alt):  # SNV
        suffix = sequence[position+1:]
        return prefix + alt + suffix
    elif len(refr) < len(alt):  # Insertion
        suffix = sequence[position+1:]
        return prefix + alt[1:] + suffix
    else:  # Deletion
        dellength = len(refr) - len(alt)
        suffix = sequence[position+dellength:]
        return prefix + suffix

def gentrio(sequences, outstreams, ninh=20, ndenovo=10, seed=None):
    assert len(outstreams) == 3
    mutator = simulate_variant_genotypes(
        sequences, ninh=ninh, ndenovo=ndenovo, rng=seed
    )
    variants = list(mutator)
    variants.sort(key=lambda v: v.position, reverse=True)

    for seqid, sequence in sequences.items():
        for ind in range(3):  # proband mother father
            haploseqs = [str(sequence), str(sequence)]
            for variant in variants:
                if variant.seqid != seqid:
                    continue
                genotype = variant.genotypes[ind]
                haplotypes = (genotype[0], genotype[2])
                for hapindex in range(2):
                    if haplotypes[hapindex] == '0':
                        continue
                    haploseqs[hapindex] = apply_mutation(
                        haploseqs[hapindex], variant.position, variant._refr,
                        variant._alt
                    )
            print('>', seqid, '_haplo1\n', haploseqs[0], sep='',
                  file=outstreams[ind])
            print('>', seqid, '_haplo2\n', haploseqs[1], sep='',
                  file=outstreams[ind])

    variants.sort(key=lambda v: (v.seqid, v.position))
    for variant in variants:
        yield variant

def main(args):
    seqfile = kevlar.open(args.genome, 'r')
    genomeseqs = kevlar.seqio.parse_seq_dict(seqfile)

    samples = ('proband', 'mother', 'father')
    outfiles = ['{:s}-{:s}.fasta'.format(args.prefix, s) for s in samples]
    outstreams = [kevlar.open(outfile, 'w') for outfile in outfiles]

    vcfout = None
    if args.vcf:
        vcfout = kevlar.open(args.vcf, 'w')

    if vcfout:
        kevlar.vcf_header(vcfout, source='kevlar::gentrio', infoheader=True)
    mutator = gentrio(genomeseqs, outstreams, ninh=args.inherited,
                      ndenovo=args.de_novo, seed=args.seed)
    for variant in mutator:
        if vcfout:
            print(variant.vcf, file=vcfout)

    for outstream in outstreams:
        outstream.close()
<|endoftext|>


<commit_before># KlupuNG
# Copyright (C) 2013 Koodilehto Osk <http://koodilehto.fi>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import klupung

class Policymaker(klupung.db.Model):
    __tablename__ = "policymaker"

    # Columns
    id = klupung.db.Column(klupung.db.Integer,
                           primary_key=True)
    abbreviation = klupung.db.Column(klupung.db.String(20),
                                     unique=True,
                                     nullable=False)
    name = klupung.db.Column(klupung.db.String(50),
                             unique=True,
                             nullable=False)

    # Relationships
    meetings = klupung.db.relationship("Meeting")

    def __init__(self, abbreviation, name):
        self.abbreviation = abbreviation
        self.name = name

class Meeting(klupung.db.Model):
    __tablename__ = "meeting"

    # Columns
    id = klupung.db.Column(klupung.db.Integer,
                           primary_key=True)
    start_datetime = klupung.db.Column(klupung.db.DateTime,
                                       nullable=False)
    policymaker_id = klupung.db.Column(klupung.db.Integer,
                                       klupung.db.ForeignKey("policymaker.id"),
                                       nullable=False)

    # Relationships
    meeting_documents = klupung.db.relationship("MeetingDocument")
    policymaker = klupung.db.relationship("Policymaker")
    __table_args__ = (
        klupung.db.UniqueConstraint("policymaker_id", "start_datetime"),
        )

    def __init__(self, start_datetime, policymaker_id):
        self.start_datetime = start_datetime
        self.policymaker_id = policymaker_id

class MeetingDocument(klupung.db.Model):
    __tablename__ = "meeting_document"

    # Columns
    id = klupung.db.Column(klupung.db.Integer, primary_key=True)
    meeting_id = klupung.db.Column(klupung.db.Integer,
                                   klupung.db.ForeignKey("meeting.id"),
                                   nullable=False)
    origin_url = klupung.db.Column(klupung.db.Text)
    origin_id = klupung.db.Column(klupung.db.String(40),
                                  unique=True,
                                  nullable=False)
    publish_datetime = klupung.db.Column(klupung.db.DateTime)

    # Relationships
    meeting = klupung.db.relationship("Meeting")

    def __init__(self, origin_url, meeting_id, origin_id, publish_datetime):
        self.origin_url = origin_url
        self.meeting_id = meeting_id
        self.origin_id = origin_id
        self.publish_datetime = publish_datetime
<commit_msg>Put model classes in alphabetical order<commit_after># KlupuNG
# Copyright (C) 2013 Koodilehto Osk <http://koodilehto.fi>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import klupung

class Meeting(klupung.db.Model):
    __tablename__ = "meeting"

    # Columns
    id = klupung.db.Column(klupung.db.Integer,
                           primary_key=True)
    start_datetime = klupung.db.Column(klupung.db.DateTime,
                                       nullable=False)
    policymaker_id = klupung.db.Column(klupung.db.Integer,
                                       klupung.db.ForeignKey("policymaker.id"),
                                       nullable=False)

    # Relationships
    meeting_documents = klupung.db.relationship("MeetingDocument")
    policymaker = klupung.db.relationship("Policymaker")
    __table_args__ = (
        klupung.db.UniqueConstraint("policymaker_id", "start_datetime"),
        )

    def __init__(self, start_datetime, policymaker_id):
        self.start_datetime = start_datetime
        self.policymaker_id = policymaker_id

class MeetingDocument(klupung.db.Model):
    __tablename__ = "meeting_document"

    # Columns
    id = klupung.db.Column(klupung.db.Integer, primary_key=True)
    meeting_id = klupung.db.Column(klupung.db.Integer,
                                   klupung.db.ForeignKey("meeting.id"),
                                   nullable=False)
    origin_url = klupung.db.Column(klupung.db.Text)
    origin_id = klupung.db.Column(klupung.db.String(40),
                                  unique=True,
                                  nullable=False)
    publish_datetime = klupung.db.Column(klupung.db.DateTime)

    # Relationships
    meeting = klupung.db.relationship("Meeting")

    def __init__(self, origin_url, meeting_id, origin_id, publish_datetime):
        self.origin_url = origin_url
        self.meeting_id = meeting_id
        self.origin_id = origin_id
        self.publish_datetime = publish_datetime

class Policymaker(klupung.db.Model):
    __tablename__ = "policymaker"

    # Columns
    id = klupung.db.Column(klupung.db.Integer,
                           primary_key=True)
    abbreviation = klupung.db.Column(klupung.db.String(20),
                                     unique=True,
                                     nullable=False)
    name = klupung.db.Column(klupung.db.String(50),
                             unique=True,
                             nullable=False)

    # Relationships
    meetings = klupung.db.relationship("Meeting")

    def __init__(self, abbreviation, name):
        self.abbreviation = abbreviation
        self.name = name
<|endoftext|>


<commit_before># coding=utf-8

#import the app
from app import app

#import flask things
from flask import g, request, render_template, redirect, url_for, flash, abort
from flask.ext.login import current_user, login_required

from werkzeug import secure_filename

#import models
from app.structures.models.user import *
from app.structures.models.gradebook import *
from app.structures.models.course import *

#import forms
from app.structures.forms import SubmitAssignmentForm

#import app helpers
from app.helpers.filestorage import *
from app.helpers.autograder import gradeSubmission

#import other libraries
import os, datetime, shutil

@app.route('/assignments/submit/<pid>', defaults={'uid': None})
@app.route('/assignments/submit/<pid>/<uid>')
@login_required
def submitAssignment(pid, uid):
  '''
  Function Type: View Function
  Template: student/submit.html
  Purpose: Allow a student to submit files for a homework assignment problem.

  Inputs:
    pid: The object ID for the problem being submitted to

  Template Parameters:
    course: The course which contains the problem
    assignment: The assignment group containing the problem
    problem: The problem object specified by <pid>
    form: A SubmitAssignmentForm which has had the partner field filled with
    the students from course.

  Forms Handled: None
  '''
  try:
    p = Problem.objects.get(id=pid)
    c,a = p.getParents()

    #UID lets us know if this is the student submitting or an instructor
    #submitting for a student
    if uid == None:
      #If the student is submitting for themselves only do it if the problem
      #is open
      if not p.isOpen:
        flash("This problem has been closed for submissions.")
        abort(403)

      #For security purposes we reject anyone who isnt in this class
      if not ( c in current_user.courseStudent):
        abort(403)
      u = current_user
    else:
      u = User.objects.get(id=uid)
      if not (c in current_user.courseInstructor):
        abort(403)

    saf = SubmitAssignmentForm()
    users = [(str(x.id), x.username) for x in User.objects.filter(courseStudent=c) if not x.username == u.username]
    users.sort(key=lambda x: x[1])
    saf.partner.choices = [("None", "None")] + users

    return render_template("student/submit.html", \
                            course=c, assignment=a, problem=p,\
                            form=saf, user=u)
  except (Problem.DoesNotExist, Course.DoesNotExist, AssignmentGroup.DoesNotExist):
    #If either p can't be found or we can't get its parents then 404
    abort(404)

#Endpoint for uploading files

@app.route('/assignments/submit/<pid>/<uid>/upload', methods=['POST'])
@login_required
def uploadFiles(pid, uid):
  '''
  Function Type: Callback-Redirect Function
  Purpose: Create a new submission and put the files in the backing filesystem.
  If a partner is specified create the partner's submission as well and link the
  two submissions.

  Inputs:
    pid: The object ID for the problem to be submitted to.

  Forms Handled:
    SubmitAssignmentForm: Contains the files being submitted and the partner
    choice.
  '''
  now = datetime.datetime.utcnow()
  try:
    p = Problem.objects.get(id=pid)
    c, a = p.getParents()
    user = User.objects.get(id=uid)
    #For security purposes we send anyone who isnt in this class to the index
    if not ( c in current_user.courseStudent or c in current_user.courseInstructor):
      abort(403)

    if request.method == "POST":
      form = SubmitAssignmentForm(request.form)
      form.partner.choices = [("None", "None")] + [(str(x.id), x.username) for x in User.objects.filter(courseStudent=c) if not x.username == user.username]
      if form.validate():

        #Make sure the partner exists before doing anything
        if form.partner.data != "None":
          partner = User.objects.get(id=form.partner.data)

        #Make sure the FS is mounted before we try to do anything with the FS
        checkMounted()
        #Create the submissions
        userSub, userSubPath = createSubmission(p, user, now)
        userSub.save()

        #Save the files to the folder
        fileList = request.files.getlist("files")
        error = saveFiles(userSubPath, fileList)

        if error != None:
          #Remove the files
          shutil.rmtree(userSubPath)
          #remove the submission from the submission list
          p.studentSubmissions[user.keyOfUsername()].submissions = p.studentSubmissions[user.keyOfUsername()].submissions[:-1]
          #delete the submission
          userSub.delete()
          raise error

        missingFiles, missingStrict = ensureFiles(p, userSubPath)

        if not len(missingStrict) == 0:
          #Remove the files
          shutil.rmtree(userSubPath)
          #remove the submission from the submission list
          p.studentSubmissions[user.keyOfUsername()].submissions = p.studentSubmissions[user.keyOfUsername()].submissions[:-1]

          if len(p.studentSubmissions[user.keyOfUsername()].submissions) == 0:
            del p.studentSubmissions[user.keyOfUsername()]
            p.save()

          #delete the submission
          userSub.delete()
          flash("Your submission was missing files required for acceptance", "error")
          for m in missingStrict:
            flash("Missing required file: " + m, "error")
          return redirect(url_for("submitAssignment", pid=p.id))

        if not len(missingFiles) == 0:
          flash("Required files for autograding are missing from your submission. \
                It has been accepted but autograding will not run.", "warning")
          for m in missingFiles:
            flash("Missing required file: " + m, "warning")

        #Prepare everything before setting up the partner submission
        if form.partner.data != "None":
          partnerSub, partnerSubPath = createSubmission(p, partner, now)
          partnerSub.save()

          #Link the submissions
          userSub.partner = partner
          userSub.partnerSubmission = partnerSub
          userSub.save()
          partnerSub.partner = user
          partnerSub.partnerSubmission = userSub
          partnerSub.save()

          #remove the folder so we can copy the other folder
          shutil.rmtree(partnerSubPath)
          shutil.copytree(userSubPath, partnerSubPath)

        #Save our submissions to the database
        p.save()
        p.gradeColumn.save()

        gradeSubmission.delay(p.id, user.id, p.getSubmissionNumber(user))
        if form.partner.data != "None":
          gradeSubmission.delay(p.id, partner.id, p.getSubmissionNumber(partner))
      else:
        flash("We could not validate your submission. If this keeps occurring please contact your professor.", "warning")
    if g.user.id == user.id:
      return redirect(url_for('studentAssignments', cid=c.id))
    else:
      return redirect(url_for('instructorViewStudent', cid=c.id, uid=user.id))
  except (Problem.DoesNotExist, Course.DoesNotExist, AssignmentGroup.DoesNotExist):
    #If either p can't be found or we can't get its parents then 404
    abort(404)
  except User.DoesNotExist:
    flash("We couldn't find the partner you specified.", "error")
    abort(500)

#
# Helpers
#

def saveFiles(filePath, files):
  try:
    for f in files:
      filename = secure_filename(f.filename)
      if filename == "":
        continue
      f.save(os.path.join(filePath, filename))
      processZip(filePath, filename)
    return None
  except Exception as e:
    flash("One of your files has caused our system to crash."+\
    " This most commonly happens with zip files which contain two copies of files with the same name or a file which has the same name as the zip file itself."+\
    " Please look at the error and try to correct this issue."+\
    " If this is not your issue please report a bug to the system administrator", "warning")
    return e

def ensureFiles(problem, filePath):
  reqFiles = problem.getRequiredFiles()
  strictFiles = problem.getStrictFiles()
  for root, dirs, files in os.walk(filePath):
    for f in files:
      if f in reqFiles:
        reqFiles.remove(f)
      if f in strictFiles:
        strictFiles.remove(f)
  return reqFiles, strictFiles

def processZip(filePath, fileName):
  from zipfile import ZipFile, is_zipfile
  if not is_zipfile(os.path.join(filePath, fileName)):
    return

  z = ZipFile(os.path.join(filePath, fileName))

  extractPath = os.path.join(filePath, '.extracted')
  z.extractall(extractPath)

  if all([os.path.isdir(os.path.join(extractPath, f)) for f in os.listdir(extractPath)]):
    for d in [ f for f in os.listdir(extractPath) if os.path.isdir(os.path.join(extractPath,f)) ]:
      for f in os.listdir(os.path.join(extractPath, d)):
        shutil.move(os.path.join(extractPath, d, f), filePath)
  else:
    for f in os.listdir(extractPath):
      shutil.move(os.path.join(extractPath, f), filePath)

  shutil.rmtree(os.path.join(filePath, '.extracted'))

def createSubmission(problem, user, now=datetime.datetime.utcnow()):
  '''
  Function Type: Helper Function
  Purpose: Create a new submission for a user

  Inputs:
    problem: The problem object for this submission
    user: The user object for this submission
  '''
  #Make sure the user has a submisssion list
  if user.keyOfUsername() not in problem.studentSubmissions:
    problem.studentSubmissions[user.keyOfUsername()] = StudentSubmissionList()
    problem.save()

  #Create a grade entry
  grade = GBGrade()
  grade.save()
  problem.gradeColumn.scores[user.keyOfUsername()] = grade
  problem.gradeColumn.save()

  #Get the submission number
  subNum = len(problem.studentSubmissions[user.keyOfUsername()].submissions)+1

  c, a = problem.getParents()
  filePath = getSubmissionPath(c, a, problem, user, subNum)
  ensurePathExists(filePath)

  #Create a submission
  sub = Submission()
  sub.submitter = user
  sub.grade = grade
  sub.problem = problem
  sub.submissionTime = now

  if problem.duedate < sub.submissionTime:
    sub.isLate = True

  sub.save()

  #Set the submission reference in the grade
  grade.submission = sub
  grade.save()

  #Add the submission to the submission list
  problem.studentSubmissions[user.keyOfUsername()].addSubmission(sub)

  sub.save()

  return sub, filePath
<commit_msg>Changed how file are marked late: 42 minute grace period added<commit_after># coding=utf-8

#import the app
from app import app

#import flask things
from flask import g, request, render_template, redirect, url_for, flash, abort
from flask.ext.login import current_user, login_required

from werkzeug import secure_filename

#import models
from app.structures.models.user import *
from app.structures.models.gradebook import *
from app.structures.models.course import *

#import forms
from app.structures.forms import SubmitAssignmentForm

#import app helpers
from app.helpers.filestorage import *
from app.helpers.autograder import gradeSubmission

#import other libraries
import os, datetime, shutil

@app.route('/assignments/submit/<pid>', defaults={'uid': None})
@app.route('/assignments/submit/<pid>/<uid>')
@login_required
def submitAssignment(pid, uid):
  '''
  Function Type: View Function
  Template: student/submit.html
  Purpose: Allow a student to submit files for a homework assignment problem.

  Inputs:
    pid: The object ID for the problem being submitted to

  Template Parameters:
    course: The course which contains the problem
    assignment: The assignment group containing the problem
    problem: The problem object specified by <pid>
    form: A SubmitAssignmentForm which has had the partner field filled with
    the students from course.

  Forms Handled: None
  '''
  try:
    p = Problem.objects.get(id=pid)
    c,a = p.getParents()

    #UID lets us know if this is the student submitting or an instructor
    #submitting for a student
    if uid == None:
      #If the student is submitting for themselves only do it if the problem
      #is open
      if not p.isOpen:
        flash("This problem has been closed for submissions.")
        abort(403)

      #For security purposes we reject anyone who isnt in this class
      if not ( c in current_user.courseStudent):
        abort(403)
      u = current_user
    else:
      u = User.objects.get(id=uid)
      if not (c in current_user.courseInstructor):
        abort(403)

    saf = SubmitAssignmentForm()
    users = [(str(x.id), x.username) for x in User.objects.filter(courseStudent=c) if not x.username == u.username]
    users.sort(key=lambda x: x[1])
    saf.partner.choices = [("None", "None")] + users

    return render_template("student/submit.html", \
                            course=c, assignment=a, problem=p,\
                            form=saf, user=u)
  except (Problem.DoesNotExist, Course.DoesNotExist, AssignmentGroup.DoesNotExist):
    #If either p can't be found or we can't get its parents then 404
    abort(404)

#Endpoint for uploading files

@app.route('/assignments/submit/<pid>/<uid>/upload', methods=['POST'])
@login_required
def uploadFiles(pid, uid):
  '''
  Function Type: Callback-Redirect Function
  Purpose: Create a new submission and put the files in the backing filesystem.
  If a partner is specified create the partner's submission as well and link the
  two submissions.

  Inputs:
    pid: The object ID for the problem to be submitted to.

  Forms Handled:
    SubmitAssignmentForm: Contains the files being submitted and the partner
    choice.
  '''
  now = datetime.datetime.utcnow()
  try:
    p = Problem.objects.get(id=pid)
    c, a = p.getParents()
    user = User.objects.get(id=uid)
    #For security purposes we send anyone who isnt in this class to the index
    if not ( c in current_user.courseStudent or c in current_user.courseInstructor):
      abort(403)

    if request.method == "POST":
      form = SubmitAssignmentForm(request.form)
      form.partner.choices = [("None", "None")] + [(str(x.id), x.username) for x in User.objects.filter(courseStudent=c) if not x.username == user.username]
      if form.validate():

        #Make sure the partner exists before doing anything
        if form.partner.data != "None":
          partner = User.objects.get(id=form.partner.data)

        #Make sure the FS is mounted before we try to do anything with the FS
        checkMounted()
        #Create the submissions
        userSub, userSubPath = createSubmission(p, user, now)
        userSub.save()

        #Save the files to the folder
        fileList = request.files.getlist("files")
        error = saveFiles(userSubPath, fileList)

        if error != None:
          #Remove the files
          shutil.rmtree(userSubPath)
          #remove the submission from the submission list
          p.studentSubmissions[user.keyOfUsername()].submissions = p.studentSubmissions[user.keyOfUsername()].submissions[:-1]
          #delete the submission
          userSub.delete()
          raise error

        missingFiles, missingStrict = ensureFiles(p, userSubPath)

        if not len(missingStrict) == 0:
          #Remove the files
          shutil.rmtree(userSubPath)
          #remove the submission from the submission list
          p.studentSubmissions[user.keyOfUsername()].submissions = p.studentSubmissions[user.keyOfUsername()].submissions[:-1]

          if len(p.studentSubmissions[user.keyOfUsername()].submissions) == 0:
            del p.studentSubmissions[user.keyOfUsername()]
            p.save()

          #delete the submission
          userSub.delete()
          flash("Your submission was missing files required for acceptance", "error")
          for m in missingStrict:
            flash("Missing required file: " + m, "error")
          return redirect(url_for("submitAssignment", pid=p.id))

        if not len(missingFiles) == 0:
          flash("Required files for autograding are missing from your submission. \
                It has been accepted but autograding will not run.", "warning")
          for m in missingFiles:
            flash("Missing required file: " + m, "warning")

        #Prepare everything before setting up the partner submission
        if form.partner.data != "None":
          partnerSub, partnerSubPath = createSubmission(p, partner, now)
          partnerSub.save()

          #Link the submissions
          userSub.partner = partner
          userSub.partnerSubmission = partnerSub
          userSub.save()
          partnerSub.partner = user
          partnerSub.partnerSubmission = userSub
          partnerSub.save()

          #remove the folder so we can copy the other folder
          shutil.rmtree(partnerSubPath)
          shutil.copytree(userSubPath, partnerSubPath)

        #Save our submissions to the database
        p.save()
        p.gradeColumn.save()

        gradeSubmission.delay(p.id, user.id, p.getSubmissionNumber(user))
        if form.partner.data != "None":
          gradeSubmission.delay(p.id, partner.id, p.getSubmissionNumber(partner))
      else:
        flash("We could not validate your submission. If this keeps occurring please contact your professor.", "warning")
    if g.user.id == user.id:
      return redirect(url_for('studentAssignments', cid=c.id))
    else:
      return redirect(url_for('instructorViewStudent', cid=c.id, uid=user.id))
  except (Problem.DoesNotExist, Course.DoesNotExist, AssignmentGroup.DoesNotExist):
    #If either p can't be found or we can't get its parents then 404
    abort(404)
  except User.DoesNotExist:
    flash("We couldn't find the partner you specified.", "error")
    abort(500)

#
# Helpers
#

def saveFiles(filePath, files):
  try:
    for f in files:
      filename = secure_filename(f.filename)
      if filename == "":
        continue
      f.save(os.path.join(filePath, filename))
      processZip(filePath, filename)
    return None
  except Exception as e:
    flash("One of your files has caused our system to crash."+\
    " This most commonly happens with zip files which contain two copies of files with the same name or a file which has the same name as the zip file itself."+\
    " Please look at the error and try to correct this issue."+\
    " If this is not your issue please report a bug to the system administrator", "warning")
    return e

def ensureFiles(problem, filePath):
  reqFiles = problem.getRequiredFiles()
  strictFiles = problem.getStrictFiles()
  for root, dirs, files in os.walk(filePath):
    for f in files:
      if f in reqFiles:
        reqFiles.remove(f)
      if f in strictFiles:
        strictFiles.remove(f)
  return reqFiles, strictFiles

def processZip(filePath, fileName):
  from zipfile import ZipFile, is_zipfile
  if not is_zipfile(os.path.join(filePath, fileName)):
    return

  z = ZipFile(os.path.join(filePath, fileName))

  extractPath = os.path.join(filePath, '.extracted')
  z.extractall(extractPath)

  if all([os.path.isdir(os.path.join(extractPath, f)) for f in os.listdir(extractPath)]):
    for d in [ f for f in os.listdir(extractPath) if os.path.isdir(os.path.join(extractPath,f)) ]:
      for f in os.listdir(os.path.join(extractPath, d)):
        shutil.move(os.path.join(extractPath, d, f), filePath)
  else:
    for f in os.listdir(extractPath):
      shutil.move(os.path.join(extractPath, f), filePath)

  shutil.rmtree(os.path.join(filePath, '.extracted'))

def createSubmission(problem, user, now=datetime.datetime.utcnow()):
  '''
  Function Type: Helper Function
  Purpose: Create a new submission for a user

  Inputs:
    problem: The problem object for this submission
    user: The user object for this submission
  '''
  #Make sure the user has a submisssion list
  if user.keyOfUsername() not in problem.studentSubmissions:
    problem.studentSubmissions[user.keyOfUsername()] = StudentSubmissionList()
    problem.save()

  #Create a grade entry
  grade = GBGrade()
  grade.save()
  problem.gradeColumn.scores[user.keyOfUsername()] = grade
  problem.gradeColumn.save()

  #Get the submission number
  subNum = len(problem.studentSubmissions[user.keyOfUsername()].submissions)+1

  c, a = problem.getParents()
  filePath = getSubmissionPath(c, a, problem, user, subNum)
  ensurePathExists(filePath)

  #Create a submission
  sub = Submission()
  sub.submitter = user
  sub.grade = grade
  sub.problem = problem
  sub.submissionTime = now
  GRACE_PERIOD = 42
  dueDateWithGrace = problem.duedate + datetime.timedelta(minutes=GRACE_PERIOD)

  if dueDateWithGrace < sub.submissionTime:
    sub.isLate = True

  sub.save()

  #Set the submission reference in the grade
  grade.submission = sub
  grade.save()

  #Add the submission to the submission list
  problem.studentSubmissions[user.keyOfUsername()].addSubmission(sub)

  sub.save()

  return sub, filePath
<|endoftext|>


<commit_before>from __future__ import print_function
import multiprocessing
import os
import hashlib
import argparse
import llnl.util.tty as tty
import ectl.util
import ectl
import ectl.cmd
import ectl.cdlparams
from ectl import pathutil,rundir,xhash,srcdir,launchers
import ectl.config
import ectl.rundeck
from ectl.rundeck import legacy
import subprocess
import base64
import re
import datetime
import sys
from spack.util import executable
from giss import pyar, ioutil
import importlib
import netCDF4

MODELE_CONTROL_PYAR = 'modele-control.pyar'

def buildhash(rd, src_dir):
    hash = hashlib.md5()
    xhash.update(rd, hash)
    xhash.update(src_dir, hash)    # Source directory
    return hash.hexdigest()
#    return base64.b32encode(hash.digest()).lower()

def pkghash(rd, src_dir):
    hash = hashlib.md5()
    xhash.update(rd, hash)
    srcdir.update_hash(src_dir, hash)
    return hash.hexdigest()
#    return base64.b32encode(hash.digest()).lower()

def good_pkg_dir(pkg_dir):
    """Determines that a pkg_dir has all binaries needed to run."""
    for file in ('lib/libmodele.so', 'bin/modelexe'):
        if not os.path.isfile(os.path.join(pkg_dir, file)):
            return False
    return True

def set_link(src, dst):
    """Like doing ln -s src dst"""
    if os.path.islink(dst):
        if os.path.abspath(os.path.realpath(dst)) == os.path.abspath(src):
            return
        os.remove(dst)
    src_rel = os.path.relpath(src, start=os.path.split(dst)[0])
    os.symlink(src_rel, dst)

def setup(run, rundeck=None, src=None, pkgbuild=False, rebuild=False, jobs=None, unpack=True):

    # Move parameters to different name to maintain SSA coding style below.
    args_run = run
    args_rundeck = rundeck
    args_src = src
    args_pkgbuild = pkgbuild
    args_rebuild = rebuild
    args_jobs = jobs

    args_run = os.path.abspath(args_run)

    # ---------------
    # Get ectl directories
    config = ectl.config.Config(run=run)
    print('-------- Ectl Config:')
    print('    ectl:   %s' % config.ectl)
    print('    runs:   %s' % config.runs)
    print('    builds: %s' % config.builds)
    print('    pkgs:   %s' % config.pkgs)

    # Get src, build and pkg directories the last time setup was run.
    # (None if they don't exist)
    old = rundir.FollowLinks(run)
    status = rundir.Status(run)

    print('\nRun: %s' % run)
    print('-------- Old Setup:')
    print('    rundeck: %s' % old.rundeck)
#    print('    run:     %s' % old.run)
    print('    src:     %s' % old.src)
    print('    build:   %s' % old.build)
    print('    pkg:     %s' % old.pkg)
    print('    status:  %d' % status.status)

    # ----- Determine the rundeck
    new_rundeck = os.path.abspath(args_rundeck) if args_rundeck is not None else None
    print('args_rundeck', args_rundeck)
    print('new_rundeck', new_rundeck)

    rundeck = new_rundeck or old.rundeck
    if rundeck is None:
        raise ValueError('No rundeck specified!')
    if (status.status > launchers.INITIAL) and (old.rundeck is not None) and (rundeck != old.rundeck):
        raise ValueError('Cannot change rundeck (to %s)' % (rundeck))
#        tty.warn('Rundeck changing from %s to %s' % (old.rundeck, rundeck))

    # -------- Determine the src
    new_src = args_src or pathutil.modele_root(rundeck)
    src = new_src or old.src
    if src is None:
        raise ValueError('No source directory specified!')
    if (status.status > launchers.INITIAL) and (old.src is not None) and (src != old.src):
        raise ValueError('Cannot change src (to %s)' % src)

    if not os.path.isdir(src):
        raise ValueError('src %s does not exist!' % src)

    # ===========================================
    # Construct/merge the rundeck
    git = executable.which('git')

    # ----- Create the rundeck repo (if it doesn't already exist)
    print('========= BEGIN Rundeck Management')
    config_dir = os.path.join(args_run, 'config')
    rundeck_R = os.path.join(config_dir, 'rundeck.R')

    # The ModelE directory associated with our rundeck
    # If the rundeck is outside a ModelE directory, use
    # src instead.
    rundeck_src = pathutil.modele_root(rundeck) or src

    template_path = [os.path.join(rundeck_src, 'templates')]

    if not os.path.exists(config_dir):
        # Create a new rundeck.R
        try:
            os.makedirs(config_dir)
        except OSError:
            pass
        with ectl.util.working_dir(config_dir):
            git('init', echo=sys.stdout)
            git('checkout', '-b', 'upstream', echo=sys.stdout)

            # Copy the rundeck from original location (templates?)
            print('$ <generating {0}>'.format(rundeck_R))
            with open(rundeck_R, 'w') as fout:
                for line in legacy.preprocessor(rundeck, template_path):
                    fout.write(line.raw)

            git('add', 'rundeck.R', echo=sys.stdout)
            git('commit', '-a', '-m', 'Initial commit from {0}'.format(rundeck), echo=sys.stdout)

            # Put it on the user branch (where we normally will reside)
            git('checkout', '-b', 'user', echo=sys.stdout)

    else:
        # Update/merge the rundeck
        # If there are unresolved conflicts, this will raise an exception

        with ectl.util.working_dir(config_dir):
            try:
                # ----- Check in changes from user
                git('checkout', 'user', echo=sys.stdout)    # Exception on the first command

                # Add all .cdl files
                cdls = [x for x in os.listdir('.') if x.endswith('.cdl')]
                if len(cdls) > 0:
                    git('add', *cdls)

                # Check in changes from user
                git('commit', '-a', '-m', 'Changes from user', echo=sys.stdout, fail_on_error=False)

                # Check in changes from upstream
                git('checkout', 'upstream', echo=sys.stdout)
                # Copy the rundeck from original location (templates?)
                with open(rundeck_R, 'w') as fout:
                    for line in legacy.preprocessor(rundeck, template_path):
                        fout.write(line.raw)
                git('commit', '-a', '-m', 'Changes from upstream', echo=sys.stdout, fail_on_error=False)

                # Merge upstream changes into user
                git('checkout', 'user', echo=sys.stdout)
                git('merge', 'upstream', '-m', 'Merged changes', echo=sys.stdout)    # Will raise if merge needs help
            except:
                print('Error merging rundeck; do you have unresolved conflicts?')

                if 'EDITOR' in os.environ:
                    EDITOR = os.environ['EDITOR'].split(' ')
                    print('EDITOR', EDITOR)
                    editor = executable.which(EDITOR[0])
                    args = EDITOR[1:] + [rundeck_R]
                    editor(*args, echo=sys.stdout)
                else:
                    print('You need to edit the file to resolve conflicts:')
                    print(rundeck_R)
                print('When you are done resolving conflicts, do:')
                print('    ectl merge {0}'.format(args_run))
                print('    ectl setup {0}'.format(args_run))

                sys.exit(1)
    print('========= END Rundeck Management')

    rd = ectl.rundeck.load(rundeck_R, modele_root=src)
    # =====================================

    # ------ Determine build; cannot change
    build_hash = buildhash(rd, src)
    build = os.path.join(config.builds, build_hash)
    if (status.status > launchers.INITIAL) and (old.build is not None) and (build != old.build):
        raise ValueError('Cannot change build to %s', build)

    # ------ Determine pkg
    pkgbuild = args_pkgbuild or old.pkgbuild
    if pkgbuild:
        pkg = os.path.join(config.pkgs, 'pkg-' + os.path.split(build)[1])
    else:
        pkg_hash = pkghash(rd, src)
        pkg = os.path.join(config.pkgs, pkg_hash)

    print('-------- New Setup:')
    print('    rundeck: %s' % rundeck)
    print('    src:     %s' % src)
    print('    build:   %s' % build)
    print('    pkg:     %s' % pkg)

    # ------------- Set directory symlinks
    set_link(rundeck, os.path.join(args_run, 'upstream.R'))
    set_link(rundeck_R, os.path.join(args_run, 'rundeck.R'))
    set_link(src, os.path.join(args_run, 'src'))
    set_link(build, os.path.join(args_run, 'build'))
    set_link(pkg, os.path.join(args_run, 'pkg'))

    # ------ Re-build only if our pkg is not good
    if args_rebuild or pkgbuild or (not good_pkg_dir(pkg)) or (old.pkg is None):

        try:

            # Unpack CMake build files if a modele-control.pyar file exists
            # Do not overwrite existing build files
            with ectl.util.working_dir(src):
                if unpack:
                    print('Adding files from modele-control.pyar')
                    print('      ', os.path.realpath(MODELE_CONTROL_PYAR))
                    with open(MODELE_CONTROL_PYAR) as fin:
                        pyar.unpack_archive(fin, '.')

                if args_jobs is None:
                    # number of jobs spack has to build with.
                    jobs = multiprocessing.cpu_count()
                else:
                    jobs = args_jobs

                # Create the build dir if it doesn't already exist
                if not os.path.isdir(build):
                    os.makedirs(build)
                os.chdir(build)

                # Only run CMake if no Makefile.  (If Makefile is out
                # of date, CMake will automatically re-run with 'make'
                # command)
                if not os.path.exists('Makefile'):

                    # Read the shebang out of setup.py to get around 80-char limit
                    modele_setup_py = os.path.join(src, 'modele-setup.py')
                    env = dict(os.environ)
                    if False:
                        cmd = []
                        with open(modele_setup_py, 'r') as fin:
                            line = next(fin)
                            if line[0:2] == '#!':
                                python = line[2:].strip()

                                # Make sure this looks like python, not something else
                                if python.index('python') != 0:
                                    cmd.append(python)
                        del env['PYTHONPATH']    # Python2 in CMake build does its own thing
                    else:
                        cmd = ['python3']

                    try:
                        cmd += [modele_setup_py,
                            '-DRUNDECK=%s' % rundeck_R,
                            '-DCMAKE_INSTALL_PREFIX=%s' % pkg,
                            src]
                        print('setup calling', cmd)
                        subprocess.check_call(cmd, env=env)
                    except OSError as err:
                        sys.stderr.write(' '.join(cmd) + '\n')
                        sys.stderr.write('%s\n' % err)
                        raise ValueError('Problem running %s.  Have you run spack setup on your source directory?' % os.path.join(src, 'modele-setup.py'))

                # Now that we have a makefile, run make!
                subprocess.check_call(['make', 'install', '-j%d' % jobs])
        finally:
            if False:
                # Remove files from modele-control.pyar
                if os.path.exists(MODELE_CONTROL_PYAR):
                    print('Removing files from modele-control.pyar')
                    with open(MODELE_CONTROL_PYAR) as fin:
                        for fname in pyar.list_archive(fin):
                            # print('Removing %s' % fname)
                            try:
                                os.remove(fname)
                            except OSError:
                                pass

    # ------------------ Download input files
    download_dir = os.environ['MODELE_ORIGIN_DIR']
    good = ectl.cdlparams.resolve_cdls_in_dir(os.path.join(args_run, 'config'), download_dir=download_dir)

    rd.params.files.resolve(
        file_path=ectl.paths.default_file,
        download_dir=download_dir)

    if not good:
        raise Exception('Problem resolving one or more input filesnames')

    # ---- Create data file symlinks and I file
    # (Just so the user can see what it will be; this is
    # re-done in launch.py)
    rundir.make_rundir(rd, args_run)

    # ---- Run setup scripts...
    for fname in os.listdir(os.path.join(args_run, 'config')):
        if not fname.endswith('.nc'):
            continue

        # Obtain list of setup functions we need to call
        setup_fns = list()
        with netCDF4.Dataset(os.path.join(args_run, 'config', fname), 'r') as nc:
            setups = nc.variables['setups']
            for attr in setups.ncattrs():
                path = getattr(setups, attr).split('.')
                module = importlib.import_module('.'.join(path[:-1]))
                setup_fns.append(getattr(module, path[-1]))

        with ioutil.pushd(args_run):
            for setup_fn in setup_fns:
                setup_fn(args_run, rd)
<commit_msg>Mine CMakeCache.txt for install prefix to know better when to re-run cmake<commit_after>from __future__ import print_function
import multiprocessing
import os
import hashlib
import argparse
import llnl.util.tty as tty
import ectl.util
import ectl
import ectl.cmd
import ectl.cdlparams
from ectl import pathutil,rundir,xhash,srcdir,launchers
import ectl.config
import ectl.rundeck
from ectl.rundeck import legacy
import subprocess
import base64
import re
import datetime
import sys
from spack.util import executable
from giss import pyar, ioutil
import importlib
import netCDF4

MODELE_CONTROL_PYAR = 'modele-control.pyar'

def buildhash(rd, src_dir):
    hash = hashlib.md5()
    xhash.update(rd, hash)
    xhash.update(src_dir, hash)    # Source directory
    return hash.hexdigest()
#    return base64.b32encode(hash.digest()).lower()

def pkghash(rd, src_dir):
    hash = hashlib.md5()
    xhash.update(rd, hash)
    srcdir.update_hash(src_dir, hash)
    return hash.hexdigest()
#    return base64.b32encode(hash.digest()).lower()

def good_pkg_dir(pkg_dir):
    """Determines that a pkg_dir has all binaries needed to run."""
    for file in ('lib/libmodele.so', 'bin/modelexe'):
        if not os.path.isfile(os.path.join(pkg_dir, file)):
            return False
    return True

def set_link(src, dst):
    """Like doing ln -s src dst"""
    print('set_link', src, dst)
    if os.path.islink(dst):
        if os.path.abspath(os.path.realpath(dst)) == os.path.abspath(src):
            return
        os.remove(dst)
    src_rel = os.path.relpath(src, start=os.path.split(dst)[0])
    os.symlink(src_rel, dst)

cmakeRE = re.compile('(.*?)=(.*)')
def read_cmake_cache(fname):
    vars = {}
    try:
        with open(fname, 'r') as fin:
            for line in fin:
                line = line.partition('#')[0].rstrip()
                line = line.partition('//')[0].rstrip()
                match = cmakeRE.match(line)
                if match is not None:
                    vars[match.group(1)] = match.group(2)
    except:
        pass

    return vars

def setup(run, rundeck=None, src=None, pkgbuild=False, rebuild=False, jobs=None, unpack=True):

    # Move parameters to different name to maintain SSA coding style below.
    args_run = run
    args_rundeck = rundeck
    args_src = src
    args_pkgbuild = pkgbuild
    args_rebuild = rebuild
    args_jobs = jobs

    args_run = os.path.abspath(args_run)

    # ---------------
    # Get ectl directories
    config = ectl.config.Config(run=run)
    print('-------- Ectl Config:')
    print('    ectl:   %s' % config.ectl)
    print('    runs:   %s' % config.runs)
    print('    builds: %s' % config.builds)
    print('    pkgs:   %s' % config.pkgs)

    # Get src, build and pkg directories the last time setup was run.
    # (None if they don't exist)
    old = rundir.FollowLinks(run)
    status = rundir.Status(run)

    print('\nRun: %s' % run)
    print('-------- Old Setup:')
    print('    rundeck: %s' % old.rundeck)
#    print('    run:     %s' % old.run)
    print('    src:     %s' % old.src)
    print('    build:   %s' % old.build)
    print('    pkg:     %s' % old.pkg)
    print('    status:  %d' % status.status)

    # ----- Determine the rundeck
    new_rundeck = os.path.abspath(args_rundeck) if args_rundeck is not None else None
    print('args_rundeck', args_rundeck)
    print('new_rundeck', new_rundeck)

    rundeck = new_rundeck or old.rundeck
    if rundeck is None:
        raise ValueError('No rundeck specified!')

    if (status.status > launchers.INITIAL) and (old.rundeck is not None) and (rundeck != old.rundeck):
        raise ValueError('Cannot change rundeck (to %s)' % (rundeck))
#        tty.warn('Rundeck changing from %s to %s' % (old.rundeck, rundeck))

    # -------- Determine the src
    new_src = args_src or pathutil.modele_root(rundeck)
    src = new_src or old.src
    if src is None:
        raise ValueError('No source directory specified!')
    if (status.status > launchers.INITIAL) and (old.src is not None) and (src != old.src):
        raise ValueError('Cannot change src (to %s)' % src)

    if not os.path.isdir(src):
        raise ValueError('src %s does not exist!' % src)

    # ===========================================
    # Construct/merge the rundeck
    git = executable.which('git')

    # ----- Create the rundeck repo (if it doesn't already exist)
    print('========= BEGIN Rundeck Management')
    config_dir = os.path.join(args_run, 'config')
    rundeck_R = os.path.join(config_dir, 'rundeck.R')

    # The ModelE directory associated with our rundeck
    # If the rundeck is outside a ModelE directory, use
    # src instead.
    rundeck_src = pathutil.modele_root(rundeck) or src

    template_path = [os.path.join(rundeck_src, 'templates')]

    if not os.path.exists(config_dir):
        # Create a new rundeck.R
        try:
            os.makedirs(config_dir)
        except OSError:
            pass
        with ectl.util.working_dir(config_dir):
            git('init', echo=sys.stdout)
            git('checkout', '-b', 'upstream', echo=sys.stdout)

            # Copy the rundeck from original location (templates?)
            print('$ <generating {0}>'.format(rundeck_R))
            with open(rundeck_R, 'w') as fout:
                for line in legacy.preprocessor(rundeck, template_path):
                    fout.write(line.raw)

            git('add', 'rundeck.R', echo=sys.stdout)
            git('commit', '-a', '-m', 'Initial commit from {0}'.format(rundeck), echo=sys.stdout)

            # Put it on the user branch (where we normally will reside)
            git('checkout', '-b', 'user', echo=sys.stdout)

    else:
        # Update/merge the rundeck
        # If there are unresolved conflicts, this will raise an exception

        with ectl.util.working_dir(config_dir):
            try:
                # ----- Check in changes from user
                git('checkout', 'user', echo=sys.stdout)    # Exception on the first command

                # Add all .cdl files
                cdls = [x for x in os.listdir('.') if x.endswith('.cdl')]
                if len(cdls) > 0:
                    git('add', *cdls)

                # Check in changes from user
                git('commit', '-a', '-m', 'Changes from user', echo=sys.stdout, fail_on_error=False)

                # Check in changes from upstream
                git('checkout', 'upstream', echo=sys.stdout)
                # Copy the rundeck from original location (templates?)
                with open(rundeck_R, 'w') as fout:
                    for line in legacy.preprocessor(rundeck, template_path):
                        fout.write(line.raw)
                git('commit', '-a', '-m', 'Changes from upstream', echo=sys.stdout, fail_on_error=False)

                # Merge upstream changes into user
                git('checkout', 'user', echo=sys.stdout)
                git('merge', 'upstream', '-m', 'Merged changes', echo=sys.stdout)    # Will raise if merge needs help
            except:
                print('Error merging rundeck; do you have unresolved conflicts?')

                if 'EDITOR' in os.environ:
                    EDITOR = os.environ['EDITOR'].split(' ')
                    print('EDITOR', EDITOR)
                    editor = executable.which(EDITOR[0])
                    args = EDITOR[1:] + [rundeck_R]
                    editor(*args, echo=sys.stdout)
                else:
                    print('You need to edit the file to resolve conflicts:')
                    print(rundeck_R)
                print('When you are done resolving conflicts, do:')
                print('    ectl merge {0}'.format(args_run))
                print('    ectl setup {0}'.format(args_run))

                sys.exit(1)
    print('========= END Rundeck Management')

    rd = ectl.rundeck.load(rundeck_R, modele_root=src)
    # =====================================

    # ------ Determine build; cannot change
    build_hash = buildhash(rd, src)
    build = os.path.join(config.builds, build_hash)
    if (status.status > launchers.INITIAL) and (old.build is not None) and (build != old.build):
        raise ValueError('Cannot change build to %s', build)

    # ------ Determine pkg
    pkgbuild = args_pkgbuild or old.pkgbuild
    if pkgbuild:
        pkg = os.path.join(config.pkgs, 'pkg-' + os.path.split(build)[1])
    else:
        pkg_hash = pkghash(rd, src)
        pkg = os.path.join(config.pkgs, pkg_hash)

    print('-------- New Setup:')
    print('    rundeck: %s' % rundeck)
    print('    src:     %s' % src)
    print('    build:   %s' % build)
    print('    pkg:     %s' % pkg)

    # ------------- Set directory symlinks
    set_link(rundeck, os.path.join(args_run, 'upstream.R'))
    set_link(rundeck_R, os.path.join(args_run, 'rundeck.R'))
    set_link(src, os.path.join(args_run, 'src'))
    set_link(build, os.path.join(args_run, 'build'))
    set_link(pkg, os.path.join(args_run, 'pkg'))

    # ------ Re-build only if our pkg is not good
    if args_rebuild or pkgbuild or (not good_pkg_dir(pkg)) or (old.pkg is None):

        try:

            # Unpack CMake build files if a modele-control.pyar file exists
            # Do not overwrite existing build files
            with ectl.util.working_dir(src):
                if unpack:
                    print('Adding files from modele-control.pyar')
                    print('      ', os.path.realpath(MODELE_CONTROL_PYAR))
                    with open(MODELE_CONTROL_PYAR) as fin:
                        pyar.unpack_archive(fin, '.')

                if args_jobs is None:
                    # number of jobs spack has to build with.
                    jobs = multiprocessing.cpu_count()
                else:
                    jobs = args_jobs

                # Create the build dir if it doesn't already exist
                if not os.path.isdir(build):
                    os.makedirs(build)
                os.chdir(build)

                # Only run CMake if no Makefile.  (If Makefile is out
                # of date, CMake will automatically re-run with 'make'
                # command)
                cmake = read_cmake_cache(os.path.join(pkg, 'CMakeCache.txt'))
                run_cmake = ('CMAKE_INSTALL_PREFIX:PATH' not in cmake) \
                    or (cmake['CMAKE_INSTALL_PREFIX:PATH'] != pkg) \
                    or (not os.path.exists('Makefile')) \
                    or args_rebuild
                if run_cmake:
                    print('============ CMake')

                    # Read the shebang out of setup.py to get around 80-char limit
                    modele_setup_py = os.path.join(src, 'modele-setup.py')
                    env = dict(os.environ)
                    if False:
                        cmd = []
                        with open(modele_setup_py, 'r') as fin:
                            line = next(fin)
                            if line[0:2] == '#!':
                                python = line[2:].strip()

                                # Make sure this looks like python, not something else
                                if python.index('python') != 0:
                                    cmd.append(python)
                        del env['PYTHONPATH']    # Python2 in CMake build does its own thing
                    else:
                        cmd = ['python3']

                    try:
                        cmd += [modele_setup_py,
                            '-DRUNDECK=%s' % rundeck_R,
                            '-DCMAKE_INSTALL_PREFIX=%s' % pkg,
                            src]
                        print('setup calling', cmd)
                        subprocess.check_call(cmd, env=env)
                    except OSError as err:
                        sys.stderr.write(' '.join(cmd) + '\n')
                        sys.stderr.write('%s\n' % err)
                        raise ValueError('Problem running %s.  Have you run spack setup on your source directory?' % os.path.join(src, 'modele-setup.py'))

                # Now that we have a makefile, run make!
                print('============ Make')
                subprocess.check_call(['make', 'install', '-j%d' % jobs])
        finally:
            if False:
                # Remove files from modele-control.pyar
                if os.path.exists(MODELE_CONTROL_PYAR):
                    print('Removing files from modele-control.pyar')
                    with open(MODELE_CONTROL_PYAR) as fin:
                        for fname in pyar.list_archive(fin):
                            # print('Removing %s' % fname)
                            try:
                                os.remove(fname)
                            except OSError:
                                pass

    # ------------------ Download input files
    download_dir = os.environ['MODELE_ORIGIN_DIR']
    good = ectl.cdlparams.resolve_cdls_in_dir(os.path.join(args_run, 'config'), download_dir=download_dir)

    rd.params.files.resolve(
        file_path=ectl.paths.default_file,
        download_dir=download_dir)

    if not good:
        raise Exception('Problem resolving one or more input filesnames')

    # ---- Create data file symlinks and I file
    # (Just so the user can see what it will be; this is
    # re-done in launch.py)
    rundir.make_rundir(rd, args_run)

    # ---- Run setup scripts...
    for fname in os.listdir(os.path.join(args_run, 'config')):
        if not fname.endswith('.nc'):
            continue

        # Obtain list of setup functions we need to call
        setup_fns = list()
        with netCDF4.Dataset(os.path.join(args_run, 'config', fname), 'r') as nc:
            setups = nc.variables['setups']
            for attr in setups.ncattrs():
                path = getattr(setups, attr).split('.')
                module = importlib.import_module('.'.join(path[:-1]))
                setup_fns.append(getattr(module, path[-1]))

        with ioutil.pushd(args_run):
            for setup_fn in setup_fns:
                setup_fn(args_run, rd)
<|endoftext|>


<commit_before># Krisk is a module to bridge E-Charts to python data science ecosystem

from krisk.util import init_notebook
import krisk.plot.api as plot
# Override chart to prevent all variables to be accessed except two below
import krisk.chart.api as chart
from krisk.chart.api import rcParams, Chart

__version__ = '0.1.11'

def _jupyter_nbextension_paths():
    return [dict(
            section="notebook",
            src="static", # relative to 'krisk' directory
            dest="krisk", # directory in "nbextension"
            require="krisk/krisk-require"  # relative to nbextension path
        )]<commit_msg>delete unused comment<commit_after># Krisk is a module to bridge E-Charts to python data science ecosystem

from krisk.util import init_notebook
import krisk.plot.api as plot
import krisk.chart.api as chart
from krisk.chart.api import rcParams, Chart

__version__ = '0.1.11'

def _jupyter_nbextension_paths():
    return [dict(
            section="notebook",
            src="static", # relative to 'krisk' directory
            dest="krisk", # directory in "nbextension"
            require="krisk/krisk-require"  # relative to nbextension path
        )]<|endoftext|>