<commit_before>"""Code to handle panels in the mongo database"""
import datetime as dt
import logging
import math
from copy import deepcopy

import pymongo
from bson import ObjectId

from scout.build import build_panel
from scout.exceptions import IntegrityError
from scout.parse.panel import get_omim_panel_genes
from scout.utils.date import get_date

LOG = logging.getLogger(__name__)

class PanelHandler:
    """Code to handle interactions with the mongo database for panels"""

    def load_panel(self, parsed_panel):
        """Load a gene panel based on the info sent
        A panel object is built and integrity checks are made.
        The panel object is then loaded into the database.

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

            panel_info(dict): {
                'file': <path to panel file>(str),
                'institute': <institute>(str),
                'type': <panel type>(str),
                'date': date,
                'version': version,
                'panel_name': panel_id,
                'full_name': name,
            }
        """
        panel_obj = build_panel(parsed_panel, self)

        self.add_gene_panel(panel_obj)

    def load_omim_panel(self, genemap2_lines, mim2gene_lines, institute=None):
        """Create and load the OMIM-AUTO panel

        If the panel already exists, update with new information and increase version

        Args:
            genemap_lines(iterable(str)): The genemap2 file information
            mim2gene_lines(iterable(str)): The mim2genes file information
            institute(str): What institute that is responsible. Default: 'cust002'

        """
        institute = institute or "cust002"
        existing_panel = self.gene_panel(panel_id="OMIM-AUTO")
        if not existing_panel:
            LOG.warning("OMIM-AUTO does not exists in database")
            LOG.info("Creating a first version")

        version = 1.0
        if existing_panel:
            version = float(math.floor(existing_panel["version"]) + 1)

        LOG.info("Setting version to %s", version)

        date_string = None
        # Get the correct date when omim files where released
        for line in genemap2_lines:
            if "Generated" in line:
                date_string = line.split(":")[-1].strip()
                break

        date_obj = get_date(date_string)

        if existing_panel:
            if existing_panel["date"] == date_obj:
                LOG.warning("There is no new version of OMIM")
                return

        panel_data = {
            "path": None,
            "type": "clinical",
            "date": date_obj,
            "panel_id": "OMIM-AUTO",
            "institute": institute,
            "version": version,
            "display_name": "OMIM-AUTO",
            "genes": [],
        }

        alias_genes = self.genes_by_alias()

        genes = get_omim_panel_genes(
            genemap2_lines=genemap2_lines,
            mim2gene_lines=mim2gene_lines,
            alias_genes=alias_genes,
        )

        for gene in genes:
            panel_data["genes"].append(gene)

        panel_obj = build_panel(panel_data, self)

        if existing_panel:
            new_genes = self.compare_mim_panels(existing_panel, panel_obj)
            if not new_genes:
                LOG.info("The new version of omim does not differ from the old one")
                LOG.info("No update is added")
                return
            self.update_mim_version(
                new_genes, panel_obj, old_version=existing_panel["version"]
            )

        self.add_gene_panel(panel_obj)

    @staticmethod
    def compare_mim_panels(existing_panel, new_panel):
        """Check if the latest version of OMIM differs from the most recent in database
           Return all genes that where not in the previous version.

        Args:
            existing_panel(dict)
            new_panel(dict)

        Returns:
            new_genes(set(str))
        """
        existing_genes = {gene["hgnc_id"] for gene in existing_panel["genes"]}
        new_genes = {gene["hgnc_id"] for gene in new_panel["genes"]}

        return new_genes.difference(existing_genes)

    @staticmethod
    def update_mim_version(new_genes, new_panel, old_version):
        """Set the correct version for each gene
        Loop over the genes in the new panel

        Args:
            new_genes(set(str)): Set with the new gene symbols
            new_panel(dict)

        """
        LOG.info("Updating versions for new genes")
        version = new_panel["version"]
        nr_genes = 0
        for nr_genes, gene in enumerate(new_panel["genes"]):
            gene_symbol = gene["hgnc_id"]
            # If the gene is new we add the version
            if gene_symbol in new_genes:
                gene["database_entry_version"] = version
                continue
            # If the gene is old it will have the previous version
            gene["database_entry_version"] = old_version

        LOG.info("Updated %s genes", nr_genes)

    def add_gene_panel(self, panel_obj):
        """Add a gene panel to the database

            Args:
                panel_obj(dict)
        """
        panel_name = panel_obj["panel_name"]
        panel_version = panel_obj["version"]
        display_name = panel_obj.get("display_name", panel_name)

        if self.gene_panel(panel_name, panel_version):
            raise IntegrityError(
                "Panel {0} with version {1} already"
                " exist in database".format(panel_name, panel_version)
            )
        LOG.info(
            "loading panel %s, version %s to database", display_name, panel_version
        )
        LOG.info("Nr genes in panel: %s", len(panel_obj.get("genes", [])))
        result = self.panel_collection.insert_one(panel_obj)
        LOG.debug("Panel saved")
        return result.inserted_id

    def panel(self, panel_id):
        """Fetch a gene panel by '_id'.

        Args:
            panel_id (str, ObjectId): str or ObjectId of document ObjectId

        Returns:
            dict: panel object or `None` if panel not found
        """
        if not isinstance(panel_id, ObjectId):
            panel_id = ObjectId(panel_id)
        panel_obj = self.panel_collection.find_one({"_id": panel_id})
        return panel_obj

    def delete_panel(self, panel_obj):
        """Delete a panel by '_id'.

        Args:
            panel_obj(dict)

        Returns:
            res(pymongo.DeleteResult)
        """
        res = self.panel_collection.delete_one({"_id": panel_obj["_id"]})
        LOG.warning(
            "Deleting panel %s, version %s"
            % (panel_obj["panel_name"], panel_obj["version"])
        )
        return res

    def gene_panel(self, panel_id, version=None):
        """Fetch a gene panel.

        If no panel is sent return all panels

        Args:
            panel_id (str): unique id for the panel
            version (str): version of the panel. If 'None' latest version will be returned

        Returns:
            gene_panel: gene panel object
        """
        query = {"panel_name": panel_id}
        if version:
            LOG.info(
                "Fetch gene panel {0}, version {1} from database".format(
                    panel_id, version
                )
            )
            query["version"] = version
            return self.panel_collection.find_one(query)

        LOG.info("Fetching gene panels %s from database", panel_id)
        res = self.panel_collection.find(query).sort("version", -1)

        for panel in res:
            return panel

        LOG.info("No gene panel found")

        return None

    def gene_panels(self, panel_id=None, institute_id=None, version=None):
        """Return all gene panels

        If panel_id return all versions of panels by that panel name

        Args:
            panel_id(str)

        Returns:
            cursor(pymongo.cursor)
        """
        query = {}
        if panel_id:
            query["panel_name"] = panel_id
            if version:
                query["version"] = version
        if institute_id:
            query["institute"] = institute_id

        return self.panel_collection.find(query)

    def hgnc_to_panels(self, hgnc_id):
        """Get a list of gene panel objects for a hgnc_id

            Args:
                hgnc_id(int)

            Returns:
                hgnc_panels(dict): A dictionary with hgnc as keys and lists of
                                   gene panel objects as values
        """
        return self.panel_collection.find({"genes.hgnc_id": hgnc_id})

    def gene_to_panels(self, case_obj):
        """Fetch all gene panels and group them by gene

            Args:
                case_obj(scout.models.Case)
            Returns:
                gene_dict(dict): A dictionary with gene as keys and a set of
                                 panel names as value
        """
        LOG.info("Building gene to panels")
        gene_dict = {}

        for panel_info in case_obj.get("panels", []):
            panel_name = panel_info["panel_name"]
            panel_version = panel_info["version"]
            panel_obj = self.gene_panel(panel_name, version=panel_version)
            if not panel_obj:
                ## Raise exception here???
                LOG.warning(
                    "Panel: {0}, version {1} does not exist in database".format(
                        panel_name, panel_version
                    )
                )

            for gene in panel_obj["genes"]:
                hgnc_id = gene["hgnc_id"]

                if hgnc_id not in gene_dict:
                    gene_dict[hgnc_id] = set([panel_name])
                    continue

                gene_dict[hgnc_id].add(panel_name)

        LOG.info("Gene to panels done")

        return gene_dict

    def update_panel(self, panel_obj, version=None, date_obj=None, maintainer=None):
        """Replace a existing gene panel with a new one

        Keeps the object id

        Args:
            panel_obj(dict)
            version(float)
            date_obj(datetime.datetime)

        Returns:
            updated_panel(dict)
        """
        LOG.info("Updating panel %s", panel_obj["panel_name"])
        # update date of panel to "today"
        date = panel_obj["date"]
        if version:
            LOG.info(
                "Updating version from %s to version %s", panel_obj["version"], version
            )
            panel_obj["version"] = version
            # Updating version should not update date
            if date_obj:
                date = date_obj
        elif maintainer is not None:
            LOG.info(
                "Updating maintainer from {} to {}".format(
                    panel_obj.get("maintainer"), maintainer
                )
            )
            panel_obj["maintainer"] = maintainer
        else:
            date = date_obj or dt.datetime.now()
        panel_obj["date"] = date

        updated_panel = self.panel_collection.find_one_and_replace(
            {"_id": panel_obj["_id"]},
            panel_obj,
            return_document=pymongo.ReturnDocument.AFTER,
        )

        return updated_panel

    def add_pending(self, panel_obj, hgnc_gene, action, info=None):
        """Add a pending action to a gene panel

        Store the pending actions in panel.pending

        Args:
            panel_obj(dict): The panel that is about to be updated
            hgnc_gene(dict)
            action(str): choices=['add','delete','edit']
            info(dict): additional gene info (disease_associated_transcripts,
                        reduced_penetrance, mosaicism, database_entry_version,
                        inheritance_models, custom_inheritance_models, comment)

        Returns:
            updated_panel(dict):

        """

        valid_actions = ["add", "delete", "edit"]
        if action not in valid_actions:
            raise ValueError("Invalid action {0}".format(action))

        info = info or {}
        pending_action = {
            "hgnc_id": hgnc_gene["hgnc_id"],
            "action": action,
            "info": info,
            "symbol": hgnc_gene["hgnc_symbol"],
        }

        updated_panel = self.panel_collection.find_one_and_update(
            {"_id": panel_obj["_id"]},
            {"$addToSet": {"pending": pending_action}},
            return_document=pymongo.ReturnDocument.AFTER,
        )

        return updated_panel

    def reset_pending(self, panel_obj):
        """Reset the pending status of a gene panel

        Args:
            panel_obj(dict): panel in database to update

        Returns:
            updated_panel(dict): the updated gene panel
        """

        if "pending" in panel_obj:
            del panel_obj["pending"]

        updated_panel = self.panel_collection.find_one_and_replace(
            {"_id": panel_obj["_id"]},
            panel_obj,
            return_document=pymongo.ReturnDocument.AFTER,
        )
        return updated_panel

    def apply_pending(self, panel_obj, version):
        """Apply the pending changes to an existing gene panel or create a new version of the same panel.

        Args:
            panel_obj(dict): panel in database to update
            version(double): panel version to update

        Returns:
            inserted_id(str): id of updated panel or the new one
        """

        updates = {}
        new_panel = deepcopy(panel_obj)
        new_panel["pending"] = []
        new_panel["date"] = dt.datetime.now()
        info_fields = [
            "disease_associated_transcripts",
            "inheritance_models",
            "custom_inheritance_models",
            "reduced_penetrance",
            "mosaicism",
            "database_entry_version",
            "comment",
        ]
        new_genes = []

        for update in panel_obj.get("pending", []):
            hgnc_id = update["hgnc_id"]

            # If action is add we create a new gene object
            if update["action"] != "add":
                updates[hgnc_id] = update
                continue
            info = update.get("info", {})
            gene_obj = {"hgnc_id": hgnc_id, "symbol": update["symbol"]}

            for field in info_fields:
                if field in info:
                    gene_obj[field] = info[field]
            new_genes.append(gene_obj)

        for gene in panel_obj["genes"]:
            hgnc_id = gene["hgnc_id"]

            if hgnc_id not in updates:
                new_genes.append(gene)
                continue

            current_update = updates[hgnc_id]
            action = current_update["action"]
            info = current_update["info"]

            # If action is delete we do not add the gene to new genes
            if action == "delete":
                continue

            if action == "edit":
                for field in info_fields:
                    if field in info:
                        gene[field] = info[field]
                new_genes.append(gene)

        new_panel["genes"] = new_genes
        new_panel["version"] = float(version)

        inserted_id = None
        # if the same version of the panel should be updated
        if new_panel["version"] == panel_obj["version"]:
            # replace panel_obj with new_panel
            result = self.panel_collection.find_one_and_replace(
                {"_id": panel_obj["_id"]},
                new_panel,
                return_document=pymongo.ReturnDocument.AFTER,
            )
            inserted_id = result["_id"]
        else:  # create a new version of the same panel
            new_panel.pop("_id")

            # archive the old panel
            panel_obj["is_archived"] = True
            self.update_panel(panel_obj=panel_obj, date_obj=panel_obj["date"])

            # insert the new panel
            inserted_id = self.panel_collection.insert_one(new_panel).inserted_id

        return inserted_id

    def latest_panels(self, institute_id):
        """Return the latest version of each panel."""
        panel_names = self.gene_panels(institute_id=institute_id).distinct("panel_name")
        for panel_name in panel_names:
            panel_obj = self.gene_panel(panel_name)
            yield panel_obj

    def clinical_symbols(self, case_obj):
        """Return all the clinical gene symbols for a case."""
        panel_ids = [panel["panel_id"] for panel in case_obj["panels"]]
        query = self.panel_collection.aggregate(
            [
                {"$match": {"_id": {"$in": panel_ids}}},
                {"$unwind": "$genes"},
                {"$group": {"_id": "$genes.symbol"}},
            ]
        )
        return set(item["_id"] for item in query)
<commit_msg>Update docstring.<commit_after>"""Code to handle panels in the mongo database"""
import datetime as dt
import logging
import math
from copy import deepcopy

import pymongo
from bson import ObjectId

from scout.build import build_panel
from scout.exceptions import IntegrityError
from scout.parse.panel import get_omim_panel_genes
from scout.utils.date import get_date

LOG = logging.getLogger(__name__)

class PanelHandler:
    """Code to handle interactions with the mongo database for panels"""

    def load_panel(self, parsed_panel):
        """Load a gene panel based on the info sent
        A panel object is built and integrity checks are made.
        The panel object is then loaded into the database.

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

            panel_info(dict): {
                'file': <path to panel file>(str),
                'institute': <institute>(str),
                'type': <panel type>(str),
                'date': date,
                'version': version,
                'panel_name': panel_id,
                'full_name': name,
            }
        """
        panel_obj = build_panel(parsed_panel, self)

        self.add_gene_panel(panel_obj)

    def load_omim_panel(self, genemap2_lines, mim2gene_lines, institute=None):
        """Create and load the OMIM-AUTO panel

        If the panel already exists, update with new information and increase version

        Args:
            genemap_lines(iterable(str)): The genemap2 file information
            mim2gene_lines(iterable(str)): The mim2genes file information
            institute(str): What institute that is responsible. Default: 'cust002'

        """
        institute = institute or "cust002"
        existing_panel = self.gene_panel(panel_id="OMIM-AUTO")
        if not existing_panel:
            LOG.warning("OMIM-AUTO does not exists in database")
            LOG.info("Creating a first version")

        version = 1.0
        if existing_panel:
            version = float(math.floor(existing_panel["version"]) + 1)

        LOG.info("Setting version to %s", version)

        date_string = None
        # Get the correct date when omim files where released
        for line in genemap2_lines:
            if "Generated" in line:
                date_string = line.split(":")[-1].strip()
                break

        date_obj = get_date(date_string)

        if existing_panel:
            if existing_panel["date"] == date_obj:
                LOG.warning("There is no new version of OMIM")
                return

        panel_data = {
            "path": None,
            "type": "clinical",
            "date": date_obj,
            "panel_id": "OMIM-AUTO",
            "institute": institute,
            "version": version,
            "display_name": "OMIM-AUTO",
            "genes": [],
        }

        alias_genes = self.genes_by_alias()

        genes = get_omim_panel_genes(
            genemap2_lines=genemap2_lines,
            mim2gene_lines=mim2gene_lines,
            alias_genes=alias_genes,
        )

        for gene in genes:
            panel_data["genes"].append(gene)

        panel_obj = build_panel(panel_data, self)

        if existing_panel:
            new_genes = self.compare_mim_panels(existing_panel, panel_obj)
            if not new_genes:
                LOG.info("The new version of omim does not differ from the old one")
                LOG.info("No update is added")
                return
            self.update_mim_version(
                new_genes, panel_obj, old_version=existing_panel["version"]
            )

        self.add_gene_panel(panel_obj)

    @staticmethod
    def compare_mim_panels(existing_panel, new_panel):
        """Check if the latest version of OMIM differs from the most recent in database
           Return all genes that where not in the previous version.

        Args:
            existing_panel(dict)
            new_panel(dict)

        Returns:
            new_genes(set(str))
        """
        existing_genes = {gene["hgnc_id"] for gene in existing_panel["genes"]}
        new_genes = {gene["hgnc_id"] for gene in new_panel["genes"]}

        return new_genes.difference(existing_genes)

    @staticmethod
    def update_mim_version(new_genes, new_panel, old_version):
        """Set the correct version for each gene
        Loop over the genes in the new panel

        Args:
            new_genes(set(str)): Set with the new gene symbols
            new_panel(dict)

        """
        LOG.info("Updating versions for new genes")
        version = new_panel["version"]
        nr_genes = 0
        for nr_genes, gene in enumerate(new_panel["genes"]):
            gene_symbol = gene["hgnc_id"]
            # If the gene is new we add the version
            if gene_symbol in new_genes:
                gene["database_entry_version"] = version
                continue
            # If the gene is old it will have the previous version
            gene["database_entry_version"] = old_version

        LOG.info("Updated %s genes", nr_genes)

    def add_gene_panel(self, panel_obj):
        """Add a gene panel to the database

            Args:
                panel_obj(dict)
        """
        panel_name = panel_obj["panel_name"]
        panel_version = panel_obj["version"]
        display_name = panel_obj.get("display_name", panel_name)

        if self.gene_panel(panel_name, panel_version):
            raise IntegrityError(
                "Panel {0} with version {1} already"
                " exist in database".format(panel_name, panel_version)
            )
        LOG.info(
            "loading panel %s, version %s to database", display_name, panel_version
        )
        LOG.info("Nr genes in panel: %s", len(panel_obj.get("genes", [])))
        result = self.panel_collection.insert_one(panel_obj)
        LOG.debug("Panel saved")
        return result.inserted_id

    def panel(self, panel_id):
        """Fetch a gene panel by '_id'.

        Args:
            panel_id (str, ObjectId): str or ObjectId of document ObjectId

        Returns:
            dict: panel object or `None` if panel not found
        """
        if not isinstance(panel_id, ObjectId):
            panel_id = ObjectId(panel_id)
        panel_obj = self.panel_collection.find_one({"_id": panel_id})
        return panel_obj

    def delete_panel(self, panel_obj):
        """Delete a panel by '_id'.

        Args:
            panel_obj(dict)

        Returns:
            res(pymongo.DeleteResult)
        """
        res = self.panel_collection.delete_one({"_id": panel_obj["_id"]})
        LOG.warning(
            "Deleting panel %s, version %s"
            % (panel_obj["panel_name"], panel_obj["version"])
        )
        return res

    def gene_panel(self, panel_id, version=None):
        """Fetch a gene panel.

        If no panel is sent return all panels

        Args:
            panel_id (str): unique id for the panel
            version (str): version of the panel. If 'None' latest version will be returned

        Returns:
            gene_panel: gene panel object
        """
        query = {"panel_name": panel_id}
        if version:
            LOG.info(
                "Fetch gene panel {0}, version {1} from database".format(
                    panel_id, version
                )
            )
            query["version"] = version
            return self.panel_collection.find_one(query)

        LOG.info("Fetching gene panels %s from database", panel_id)
        res = self.panel_collection.find(query).sort("version", -1)

        for panel in res:
            return panel

        LOG.info("No gene panel found")

        return None

    def gene_panels(self, panel_id=None, institute_id=None, version=None):
        """Return all gene panels

        If panel_id return all versions of panels by that panel name

        Args:
            panel_id(str)

        Returns:
            cursor(pymongo.cursor)
        """
        query = {}
        if panel_id:
            query["panel_name"] = panel_id
            if version:
                query["version"] = version
        if institute_id:
            query["institute"] = institute_id

        return self.panel_collection.find(query)

    def hgnc_to_panels(self, hgnc_id):
        """Get a list of gene panel objects for a hgnc_id

            Args:
                hgnc_id(int)

            Returns:
                hgnc_panels(dict): A dictionary with hgnc as keys and lists of
                                   gene panel objects as values
        """
        return self.panel_collection.find({"genes.hgnc_id": hgnc_id})

    def gene_to_panels(self, case_obj):
        """Fetch all gene panels and group them by gene

            Args:
                case_obj(scout.models.Case)
            Returns:
                gene_dict(dict): A dictionary with gene as keys and a set of
                                 panel names as value
        """
        LOG.info("Building gene to panels")
        gene_dict = {}

        for panel_info in case_obj.get("panels", []):
            panel_name = panel_info["panel_name"]
            panel_version = panel_info["version"]
            panel_obj = self.gene_panel(panel_name, version=panel_version)
            if not panel_obj:
                ## Raise exception here???
                LOG.warning(
                    "Panel: {0}, version {1} does not exist in database".format(
                        panel_name, panel_version
                    )
                )

            for gene in panel_obj["genes"]:
                hgnc_id = gene["hgnc_id"]

                if hgnc_id not in gene_dict:
                    gene_dict[hgnc_id] = set([panel_name])
                    continue

                gene_dict[hgnc_id].add(panel_name)

        LOG.info("Gene to panels done")

        return gene_dict

    def update_panel(self, panel_obj, version=None, date_obj=None, maintainer=None):
        """Replace a existing gene panel with a new one

        Keeps the object id

        Args:
            panel_obj(dict)
            version(float)
            date_obj(datetime.datetime)
            maintainer(list(user._id))

        Returns:
            updated_panel(dict)
        """
        LOG.info("Updating panel %s", panel_obj["panel_name"])
        # update date of panel to "today"
        date = panel_obj["date"]
        if version:
            LOG.info(
                "Updating version from %s to version %s", panel_obj["version"], version
            )
            panel_obj["version"] = version
            # Updating version should not update date
            if date_obj:
                date = date_obj
        elif maintainer is not None:
            LOG.info(
                "Updating maintainer from {} to {}".format(
                    panel_obj.get("maintainer"), maintainer
                )
            )
            panel_obj["maintainer"] = maintainer
        else:
            date = date_obj or dt.datetime.now()
        panel_obj["date"] = date

        updated_panel = self.panel_collection.find_one_and_replace(
            {"_id": panel_obj["_id"]},
            panel_obj,
            return_document=pymongo.ReturnDocument.AFTER,
        )

        return updated_panel

    def add_pending(self, panel_obj, hgnc_gene, action, info=None):
        """Add a pending action to a gene panel

        Store the pending actions in panel.pending

        Args:
            panel_obj(dict): The panel that is about to be updated
            hgnc_gene(dict)
            action(str): choices=['add','delete','edit']
            info(dict): additional gene info (disease_associated_transcripts,
                        reduced_penetrance, mosaicism, database_entry_version,
                        inheritance_models, custom_inheritance_models, comment)

        Returns:
            updated_panel(dict):

        """

        valid_actions = ["add", "delete", "edit"]
        if action not in valid_actions:
            raise ValueError("Invalid action {0}".format(action))

        info = info or {}
        pending_action = {
            "hgnc_id": hgnc_gene["hgnc_id"],
            "action": action,
            "info": info,
            "symbol": hgnc_gene["hgnc_symbol"],
        }

        updated_panel = self.panel_collection.find_one_and_update(
            {"_id": panel_obj["_id"]},
            {"$addToSet": {"pending": pending_action}},
            return_document=pymongo.ReturnDocument.AFTER,
        )

        return updated_panel

    def reset_pending(self, panel_obj):
        """Reset the pending status of a gene panel

        Args:
            panel_obj(dict): panel in database to update

        Returns:
            updated_panel(dict): the updated gene panel
        """

        if "pending" in panel_obj:
            del panel_obj["pending"]

        updated_panel = self.panel_collection.find_one_and_replace(
            {"_id": panel_obj["_id"]},
            panel_obj,
            return_document=pymongo.ReturnDocument.AFTER,
        )
        return updated_panel

    def apply_pending(self, panel_obj, version):
        """Apply the pending changes to an existing gene panel or create a new version of the same panel.

        Args:
            panel_obj(dict): panel in database to update
            version(double): panel version to update

        Returns:
            inserted_id(str): id of updated panel or the new one
        """

        updates = {}
        new_panel = deepcopy(panel_obj)
        new_panel["pending"] = []
        new_panel["date"] = dt.datetime.now()
        info_fields = [
            "disease_associated_transcripts",
            "inheritance_models",
            "custom_inheritance_models",
            "reduced_penetrance",
            "mosaicism",
            "database_entry_version",
            "comment",
        ]
        new_genes = []

        for update in panel_obj.get("pending", []):
            hgnc_id = update["hgnc_id"]

            # If action is add we create a new gene object
            if update["action"] != "add":
                updates[hgnc_id] = update
                continue
            info = update.get("info", {})
            gene_obj = {"hgnc_id": hgnc_id, "symbol": update["symbol"]}

            for field in info_fields:
                if field in info:
                    gene_obj[field] = info[field]
            new_genes.append(gene_obj)

        for gene in panel_obj["genes"]:
            hgnc_id = gene["hgnc_id"]

            if hgnc_id not in updates:
                new_genes.append(gene)
                continue

            current_update = updates[hgnc_id]
            action = current_update["action"]
            info = current_update["info"]

            # If action is delete we do not add the gene to new genes
            if action == "delete":
                continue

            if action == "edit":
                for field in info_fields:
                    if field in info:
                        gene[field] = info[field]
                new_genes.append(gene)

        new_panel["genes"] = new_genes
        new_panel["version"] = float(version)

        inserted_id = None
        # if the same version of the panel should be updated
        if new_panel["version"] == panel_obj["version"]:
            # replace panel_obj with new_panel
            result = self.panel_collection.find_one_and_replace(
                {"_id": panel_obj["_id"]},
                new_panel,
                return_document=pymongo.ReturnDocument.AFTER,
            )
            inserted_id = result["_id"]
        else:  # create a new version of the same panel
            new_panel.pop("_id")

            # archive the old panel
            panel_obj["is_archived"] = True
            self.update_panel(panel_obj=panel_obj, date_obj=panel_obj["date"])

            # insert the new panel
            inserted_id = self.panel_collection.insert_one(new_panel).inserted_id

        return inserted_id

    def latest_panels(self, institute_id):
        """Return the latest version of each panel."""
        panel_names = self.gene_panels(institute_id=institute_id).distinct("panel_name")
        for panel_name in panel_names:
            panel_obj = self.gene_panel(panel_name)
            yield panel_obj

    def clinical_symbols(self, case_obj):
        """Return all the clinical gene symbols for a case."""
        panel_ids = [panel["panel_id"] for panel in case_obj["panels"]]
        query = self.panel_collection.aggregate(
            [
                {"$match": {"_id": {"$in": panel_ids}}},
                {"$unwind": "$genes"},
                {"$group": {"_id": "$genes.symbol"}},
            ]
        )
        return set(item["_id"] for item in query)
<|endoftext|>


<commit_before>import gensim, logging
import decimal
#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
import words.dataretrieval
import os
import copy
# possible parameters: avg valence, avg arousal, avg valence top 5 words, avg arousal top 5 words, average tfidf of a word in the chunk, cosine distance for a word pair, N closest neighbors for a word
# chunk has the format [['word', 'word],['word','word']]
# docs is a list of Document_Data objects

NNsize = 300
minWords = 5
filePath = '/mnt/vol/matrices/'
#filePath = 'C:/Users/L/Documents/School/'

def averageValence(docs): # average valence of a list of documents
    result = decimal.Decimal(0.0)
    for doc in docs:
        result = result + doc.average_valence_doc
    return result/len(docs)

def averageArousal(docs): # average arousal of a list of documents
    result = decimal.Decimal(0.0)
    for doc in docs:
        result = result + doc.average_arousal_doc
    return result/len(docs)

def averageValenceTopFive(docs):
    chunk = []
    for document in docs:
        chunk.append(words.dataretrieval.getWordsInDocument(document))
    dictionary = gensim.corpora.Dictionary(chunk)
    corpus = [dictionary.doc2bow(text) for text in chunk]
    tfidf = gensim.models.TfidfModel(corpus)
    totalChunk = 0.0
    for doc in corpus:
        print("CHUNK!!!", chunk)
        print("CORPUS!!!", corpus)
        print("DOC!!!", doc)
        print("TFIDF!!!", tfidf[doc])
        tfidfs = getTopFiveWords(tfidf[doc])
        totalDoc = 0.0
        for item in tfidfs:
            print("tfidfs", tfidfs)
            print("tok2id", dictionary.token2id)
            for wd,num in dictionary.token2id.items():
                print("wd and num", wd,num)
                if (num == item[0]):
                    word = wd
                    break
            totalDoc = totalDoc + words.dataretrieval.getValence(word)
        totalChunk = totalChunk + totalDoc/len(tfidfs)
    return totalChunk/len(chunk)

def averageArousalTopFive(docs):
    chunk = []
    for document in docs:
        chunk.append(words.dataretrieval.getWordsInDocument(document))
    dictionary = gensim.corpora.Dictionary(chunk)
    corpus = [dictionary.doc2bow(text) for text in chunk]
    tfidf = gensim.models.TfidfModel(corpus)
    totalChunk = 0.0
    for doc in chunk:
        tfidfs = getTopFiveWords(tfidf[doc])
        totalDoc = 0.0
        for item in tfidfs:
            for wd,num in dictionary.token2id:
                if (num == item[0]):
                    word = wd
                    break
            totalDoc = totalDoc + words.dataretrieval.getArousal(word)
        totalChunk = totalChunk + totalDoc/len(tfidfs)
    return totalChunk/len(chunk)

# helper
def getTopFiveWords(tfidfsDoc):
    result = []
    tfidfs = copy.deepcopy(tfidfsDoc)
    topWord = getTopWord(tfidfs)
    while ((topWord != None) and (len(result)<5)):
        result.append(topWord)
        tfidfs.remove(topWord)
        topWord = getTopWord(tfidfs)
    return result

# helper
def getTopWord(tfidfs):
    current = None
    for item in tfidfs:
        if (current == None):
            current = item
        else:
            if (item[1]>current[1]):
                current = item
    return current

# each corpus is a list of documents
# each document is a list of tuples
# each tuple is an id,count pair, where id maps to a word and count is the number of times the word appears in the document
# pass in a chunk as a list of documents, where each document is a list of words, something like [['word', 'word],['word','word']]
# dictionary = corpora.Dictionary(texts): map each word appearing in the corpus "texts" to an id
# dictionary.doc2bow(text): using the mapping of the dictionary, convert text to gensim's corpus format where each text is a list of words
# corpus = [dictionary.doc2bow(text) for text in texts]: pass in texts as a list of lists of words
# not sure if bugged... seems to be sort of working
def averageTfidfOfWord(chunk, word):
    dictionary = gensim.corpora.Dictionary(chunk)
    corpus = [dictionary.doc2bow(text) for text in chunk]
    tfidf = gensim.models.TfidfModel(corpus)
    #print(tfidf)
    #print(corpus)
    totalTfidf = 0.0
    docCount = 0.0
    wordId = dictionary.token2id[word] # THIS CAUSES ERROR IF WORD NOT IN CHUNK
    for doc in corpus:
        #print(doc)
        #print(tfidf[doc])
        for item in tfidf[doc]:
            if (item[0] == wordId):
                totalTfidf = totalTfidf + item[1]
                break
        docCount = docCount + 1
    return totalTfidf/docCount
    #corpusTfidf = tfidf[corpus]
    #for doc in corpusTfidf:
        #for wordd in doc:
            #if (wordd[0] == wordId):
                #totalTfidf = totalTfidf + wordd[1]
                #break

def cosDistanceOfPair(chunk, word1, word2, cbow, hashStr, chunkDate):
    if (cbow==True):
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=0)
    else:
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=1)
    #model.save(filePath+hashStr+'/'+word1+word2+'/'+str(chunkDate)) # save models to /mnt/vol/matrices/somehash/word1word2/somedate. email the user by zipping the somehash folder
    saveMatrix(model, word1+word2, hashStr, chunkDate)
    return model.similarity(word1, word2)

def nClosestNeighboursOfWord(chunk, word, N, cbow, hashStr, chunkDate):
    if (cbow==True):
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=0)
    else:
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=1)
    #model.save(filePath+hashStr+'/'+word+'/'+str(chunkDate)) # save models to /mnt/vol/matrices/somehash/someword/somedate. email the user by zipping the somehash folder
    saveMatrix(model, word, hashStr, chunkDate)
    return model.most_similar(positive=[word], topn=N)

def wordFrequency(chunk, word):
    result = 0.0
    for doc in chunk:
        result = result + doc.count(word)
    return result

# fullFreq is frequency in full corpus
#def relativeWordFrequency(chunk, word, fullFreq):
    #return wordFrequency(chunk,word)/fullFreq # MUST CHECK IF fullFreq = 0

def relativeWordFrequency(chunk, word):
    wordCount = 0.0
    totalWordCount = 0.0
    for doc in chunk:
        wordCount = wordCount + doc.count(word)
        totalWordCount = totalWordCount + len(doc)
    return wordCount/totalWordCount # multiply by 1000000 to get occurences per million

def probX(chunk, x):
    count = 0.0
    for doc in chunk:
        if x in doc:
            count = count + 1.0
    return count/len(chunk)

def probXAndY(chunk, x, y):
    # probability that an article in the chunk has both x and y?
    # all articles with x and y / total articles
    count = 0.0
    for doc in chunk:
        if ((x in doc) and (y in doc)):
            count = count + 1.0
    return count/len(chunk)

def probXAndNotY(chunk, x, y):
    count = 0.0
    for doc in chunk:
        if ((x in doc) and (y not in doc)):
            count = count + 1.0
    return count/len(chunk)

def probXGivenY(chunk, x, y):
    # probability that an article has x given that it has y?
    # probXAndY / probY
    return probXAndY(chunk,x,y)/probX(chunk,y)

def probXGivenNotY(chunk, x, y):
    return probXAndNotY(chunk, x, y)/(1.0 - probX(chunk, y))

    #notY = 1.0 - probX(chunk, y)
    #if (notY == 0):
        #return 0.0
    #xAndNotY = probX(chunk, x)*notY
    #return xAndNotY/notY

# return 1 if probX = 0, 2 if probX = 1, and 0 if no error
def probException(chunk, x):
    pX = probX(chunk, x)
    if (pX.is_integer()):
        if (pX == 0.0):
            return 1
        if (pX == 1.0):
            return 2
        return 0
    else:
        return 0

def wordNotInChunkException(chunk, word):
    wordCount = 0
    for doc in chunk:
        if word in doc:
            wordCount = wordCount + doc.count(word)
            if (wordCount >= minWords):
                return False
    return True

def saveMatrix(model, word, hashStr, chunkDate):
    path = filePath+hashStr+'/'+word+'/'+str(chunkDate)
    try:
        os.makedirs(path)
    except OSError:
        if not os.path.isdir(path):
            raise
    try:
        model.wv.save_word2vec_format(path+'/model', fvocab=path+'/vocab')
    except:
        print('model save failed')
        raise<commit_msg>fixing<commit_after>import gensim, logging
import decimal
#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
import words.dataretrieval
import os
import copy
# possible parameters: avg valence, avg arousal, avg valence top 5 words, avg arousal top 5 words, average tfidf of a word in the chunk, cosine distance for a word pair, N closest neighbors for a word
# chunk has the format [['word', 'word],['word','word']]
# docs is a list of Document_Data objects

NNsize = 300
minWords = 5
filePath = '/mnt/vol/matrices/'
#filePath = 'C:/Users/L/Documents/School/'

def averageValence(docs): # average valence of a list of documents
    result = decimal.Decimal(0.0)
    for doc in docs:
        result = result + doc.average_valence_doc
    return result/len(docs)

def averageArousal(docs): # average arousal of a list of documents
    result = decimal.Decimal(0.0)
    for doc in docs:
        result = result + doc.average_arousal_doc
    return result/len(docs)

def averageValenceTopFive(docs):
    chunk = []
    for document in docs:
        chunk.append(words.dataretrieval.getWordsInDocument(document))
    dictionary = gensim.corpora.Dictionary(chunk)
    corpus = [dictionary.doc2bow(text) for text in chunk]
    tfidf = gensim.models.TfidfModel(corpus)
    totalChunk = 0.0
    for doc in corpus:
        #print("CHUNK!!!", chunk)
        #print("CORPUS!!!", corpus)
        #print("DOC!!!", doc)
        #print("TFIDF!!!", tfidf[doc])
        tfidfs = getTopFiveWords(tfidf[doc])
        totalDoc = 0.0
        for item in tfidfs:
            #print("tfidfs", tfidfs)
            #print("tok2id", dictionary.token2id)
            for wd,num in dictionary.token2id.items():
                #print("wd and num", wd,num)
                if (num == item[0]):
                    word = wd
                    break
            totalDoc = totalDoc + words.dataretrieval.getValence(word)
        totalChunk = totalChunk + totalDoc/len(tfidfs)
    return totalChunk/len(chunk)

def averageArousalTopFive(docs):
    chunk = []
    for document in docs:
        chunk.append(words.dataretrieval.getWordsInDocument(document))
    dictionary = gensim.corpora.Dictionary(chunk)
    corpus = [dictionary.doc2bow(text) for text in chunk]
    tfidf = gensim.models.TfidfModel(corpus)
    totalChunk = 0.0
    for doc in corpus:
        tfidfs = getTopFiveWords(tfidf[doc])
        totalDoc = 0.0
        for item in tfidfs:
            for wd,num in dictionary.token2id.items():
                if (num == item[0]):
                    word = wd
                    break
            totalDoc = totalDoc + words.dataretrieval.getArousal(word)
        totalChunk = totalChunk + totalDoc/len(tfidfs)
    return totalChunk/len(chunk)

# helper
def getTopFiveWords(tfidfsDoc):
    result = []
    tfidfs = copy.deepcopy(tfidfsDoc)
    topWord = getTopWord(tfidfs)
    while ((topWord != None) and (len(result)<5)):
        result.append(topWord)
        tfidfs.remove(topWord)
        topWord = getTopWord(tfidfs)
    return result

# helper
def getTopWord(tfidfs):
    current = None
    for item in tfidfs:
        if (current == None):
            current = item
        else:
            if (item[1]>current[1]):
                current = item
    return current

# each corpus is a list of documents
# each document is a list of tuples
# each tuple is an id,count pair, where id maps to a word and count is the number of times the word appears in the document
# pass in a chunk as a list of documents, where each document is a list of words, something like [['word', 'word],['word','word']]
# dictionary = corpora.Dictionary(texts): map each word appearing in the corpus "texts" to an id
# dictionary.doc2bow(text): using the mapping of the dictionary, convert text to gensim's corpus format where each text is a list of words
# corpus = [dictionary.doc2bow(text) for text in texts]: pass in texts as a list of lists of words
# not sure if bugged... seems to be sort of working
def averageTfidfOfWord(chunk, word):
    dictionary = gensim.corpora.Dictionary(chunk)
    corpus = [dictionary.doc2bow(text) for text in chunk]
    tfidf = gensim.models.TfidfModel(corpus)
    #print(tfidf)
    #print(corpus)
    totalTfidf = 0.0
    docCount = 0.0
    wordId = dictionary.token2id[word] # THIS CAUSES ERROR IF WORD NOT IN CHUNK
    for doc in corpus:
        #print(doc)
        #print(tfidf[doc])
        for item in tfidf[doc]:
            if (item[0] == wordId):
                totalTfidf = totalTfidf + item[1]
                break
        docCount = docCount + 1
    return totalTfidf/docCount
    #corpusTfidf = tfidf[corpus]
    #for doc in corpusTfidf:
        #for wordd in doc:
            #if (wordd[0] == wordId):
                #totalTfidf = totalTfidf + wordd[1]
                #break

def cosDistanceOfPair(chunk, word1, word2, cbow, hashStr, chunkDate):
    if (cbow==True):
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=0)
    else:
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=1)
    #model.save(filePath+hashStr+'/'+word1+word2+'/'+str(chunkDate)) # save models to /mnt/vol/matrices/somehash/word1word2/somedate. email the user by zipping the somehash folder
    saveMatrix(model, word1+word2, hashStr, chunkDate)
    return model.similarity(word1, word2)

def nClosestNeighboursOfWord(chunk, word, N, cbow, hashStr, chunkDate):
    if (cbow==True):
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=0)
    else:
        model = gensim.models.Word2Vec(chunk, size=NNsize, min_count=minWords, sg=1)
    #model.save(filePath+hashStr+'/'+word+'/'+str(chunkDate)) # save models to /mnt/vol/matrices/somehash/someword/somedate. email the user by zipping the somehash folder
    saveMatrix(model, word, hashStr, chunkDate)
    return model.most_similar(positive=[word], topn=N)

def wordFrequency(chunk, word):
    result = 0.0
    for doc in chunk:
        result = result + doc.count(word)
    return result

# fullFreq is frequency in full corpus
#def relativeWordFrequency(chunk, word, fullFreq):
    #return wordFrequency(chunk,word)/fullFreq # MUST CHECK IF fullFreq = 0

def relativeWordFrequency(chunk, word):
    wordCount = 0.0
    totalWordCount = 0.0
    for doc in chunk:
        wordCount = wordCount + doc.count(word)
        totalWordCount = totalWordCount + len(doc)
    return wordCount/totalWordCount # multiply by 1000000 to get occurences per million

def probX(chunk, x):
    count = 0.0
    for doc in chunk:
        if x in doc:
            count = count + 1.0
    return count/len(chunk)

def probXAndY(chunk, x, y):
    # probability that an article in the chunk has both x and y?
    # all articles with x and y / total articles
    count = 0.0
    for doc in chunk:
        if ((x in doc) and (y in doc)):
            count = count + 1.0
    return count/len(chunk)

def probXAndNotY(chunk, x, y):
    count = 0.0
    for doc in chunk:
        if ((x in doc) and (y not in doc)):
            count = count + 1.0
    return count/len(chunk)

def probXGivenY(chunk, x, y):
    # probability that an article has x given that it has y?
    # probXAndY / probY
    return probXAndY(chunk,x,y)/probX(chunk,y)

def probXGivenNotY(chunk, x, y):
    return probXAndNotY(chunk, x, y)/(1.0 - probX(chunk, y))

    #notY = 1.0 - probX(chunk, y)
    #if (notY == 0):
        #return 0.0
    #xAndNotY = probX(chunk, x)*notY
    #return xAndNotY/notY

# return 1 if probX = 0, 2 if probX = 1, and 0 if no error
def probException(chunk, x):
    pX = probX(chunk, x)
    if (pX.is_integer()):
        if (pX == 0.0):
            return 1
        if (pX == 1.0):
            return 2
        return 0
    else:
        return 0

def wordNotInChunkException(chunk, word):
    wordCount = 0
    for doc in chunk:
        if word in doc:
            wordCount = wordCount + doc.count(word)
            if (wordCount >= minWords):
                return False
    return True

def saveMatrix(model, word, hashStr, chunkDate):
    path = filePath+hashStr+'/'+word+'/'+str(chunkDate)
    try:
        os.makedirs(path)
    except OSError:
        if not os.path.isdir(path):
            raise
    try:
        model.wv.save_word2vec_format(path+'/model', fvocab=path+'/vocab')
    except:
        print('model save failed')
        raise<|endoftext|>


<commit_before>from ironman.hardware import HardwareManager, HardwareMap, HardwareNode
manager = HardwareManager()
manager.add(HardwareMap(file('xadc.xml').read(), 'xadc'))

from ironman.communicator import Jarvis, ComplexIO
j = Jarvis()
j.set_hardware_manager(manager)

'''
root@zc706-zynq7:~# ls /sys/devices/soc0/amba@0/f8007100.ps7-xadc/iio\:device0/
dev                        in_voltage2_vccbram_raw    in_voltage6_vrefp_scale
events                     in_voltage2_vccbram_scale  in_voltage7_vrefn_raw
in_temp0_offset            in_voltage3_vccpint_raw    in_voltage7_vrefn_scale
in_temp0_raw               in_voltage3_vccpint_scale  name
in_temp0_scale             in_voltage4_vccpaux_raw    power
in_voltage0_vccint_raw     in_voltage4_vccpaux_scale  sampling_frequency
in_voltage0_vccint_scale   in_voltage5_vccoddr_raw    subsystem
in_voltage1_vccaux_raw     in_voltage5_vccoddr_scale  uevent
in_voltage1_vccaux_scale   in_voltage6_vrefp_raw
'''

@j.register('xadc')
class XADCController(ComplexIO):
    __base__ = "/sys/devices/soc0/amba@0/f8007100.ps7-xadc/iio:device0/"
    __f__ = {
                0:   __base__+"in_temp0_offset",
                1:   __base__+"in_temp0_raw",
                2:   __base__+"in_temp0_scale",
                17:  __base__+"in_voltage0_vccint_raw",
                18:  __base__+"in_voltage0_vccint_scale",
                33:  __base__+"in_voltage1_vccaux_raw",
                34:  __base__+"in_voltage1_vccaux_scale",
                49:  __base__+"in_voltage2_vccbram_raw",
                50:  __base__+"in_voltage2_vccbram_scale",
                65:  __base__+"in_voltage3_vccpint_raw",
                66:  __base__+"in_voltage3_vccpint_scale",
                81:  __base__+"in_voltage4_vccpaux_raw",
                82:  __base__+"in_voltage4_vccpaux_scale",
                97:  __base__+"in_voltage5_vccoddr_raw",
                98:  __base__+"in_voltage5_vccoddr_scale",
                113: __base__+"in_voltage6_vrefp_raw",
                114: __base__+"in_voltage6_vrefp_scale",
                129: __base__+"in_voltage7_vrefn_raw",
                130: __base__+"in_voltage7_vrefn_scale"
            }

from ironman.constructs.ipbus import PacketHeaderStruct, ControlHeaderStruct
def buildResponsePacket(packet):
    data = ''
    data += PacketHeaderStruct.build(packet.struct.header)
    for transaction, response in zip(packet.struct.data, packet.response):
        data += ControlHeaderStruct.build(transaction)
        data += response.encode("hex").decode("hex")
    return data

from ironman.server import ServerFactory
from ironman.packet import IPBusPacket
from twisted.internet import reactor
from twisted.internet.defer import Deferred

reactor.listenUDP(8888, ServerFactory('udp', lambda: Deferred().addCallback(IPBusPacket).addCallback(j).addCallback(buildResponsePacket)))
reactor.run()
<commit_msg>add static directory listing implementation<commit_after>from ironman.hardware import HardwareManager, HardwareMap, HardwareNode
manager = HardwareManager()
manager.add(HardwareMap(file('xadc.yml').read(), 'xadc'))

from ironman.communicator import Jarvis, ComplexIO
j = Jarvis()
j.set_hardware_manager(manager)

'''
root@zc706-zynq7:~# ls /sys/devices/soc0/amba@0/f8007100.ps7-xadc/iio\:device0/
dev                        in_voltage2_vccbram_raw    in_voltage6_vrefp_scale
events                     in_voltage2_vccbram_scale  in_voltage7_vrefn_raw
in_temp0_offset            in_voltage3_vccpint_raw    in_voltage7_vrefn_scale
in_temp0_raw               in_voltage3_vccpint_scale  name
in_temp0_scale             in_voltage4_vccpaux_raw    power
in_voltage0_vccint_raw     in_voltage4_vccpaux_scale  sampling_frequency
in_voltage0_vccint_scale   in_voltage5_vccoddr_raw    subsystem
in_voltage1_vccaux_raw     in_voltage5_vccoddr_scale  uevent
in_voltage1_vccaux_scale   in_voltage6_vrefp_raw
'''

@j.register('xadc')
class XADCController(ComplexIO):
    __base__ = "/sys/devices/soc0/amba@0/f8007100.ps7-xadc/iio:device0/"
    __f__ = {
                0:   __base__+"in_temp0_offset",
                1:   __base__+"in_temp0_raw",
                2:   __base__+"in_temp0_scale",
                17:  __base__+"in_voltage0_vccint_raw",
                18:  __base__+"in_voltage0_vccint_scale",
                33:  __base__+"in_voltage1_vccaux_raw",
                34:  __base__+"in_voltage1_vccaux_scale",
                49:  __base__+"in_voltage2_vccbram_raw",
                50:  __base__+"in_voltage2_vccbram_scale",
                65:  __base__+"in_voltage3_vccpint_raw",
                66:  __base__+"in_voltage3_vccpint_scale",
                81:  __base__+"in_voltage4_vccpaux_raw",
                82:  __base__+"in_voltage4_vccpaux_scale",
                97:  __base__+"in_voltage5_vccoddr_raw",
                98:  __base__+"in_voltage5_vccoddr_scale",
                113: __base__+"in_voltage6_vrefp_raw",
                114: __base__+"in_voltage6_vrefp_scale",
                129: __base__+"in_voltage7_vrefn_raw",
                130: __base__+"in_voltage7_vrefn_scale"
            }

from ironman.constructs.ipbus import PacketHeaderStruct, ControlHeaderStruct
def buildResponsePacket(packet):
    data = ''
    data += PacketHeaderStruct.build(packet.struct.header)
    for transaction, response in zip(packet.struct.data, packet.response):
        data += ControlHeaderStruct.build(transaction)
        data += response.encode("hex").decode("hex")
    return data

from ironman.history import History
h = History()

from ironman.server import ServerFactory
from ironman.packet import IPBusPacket
from twisted.internet import reactor
from twisted.internet.defer import Deferred

reactor.listenUDP(8888, ServerFactory('udp', lambda: Deferred().addCallback(IPBusPacket).addCallback(j).addCallback(buildResponsePacket).addCallback(h.record)))

'''set up a web server from top level'''
# Site, an IProtocolFactory which glues a listening server port (IListeningPort) to the HTTPChannel implementation
from twisted.web.server import Site
# File, an IResource which glues the HTTP protocol implementation to the filesystem
from twisted.web.static import File
reactor.listenTCP(8000, Site(File("/")))

reactor.run()
<|endoftext|>


<commit_before>#!/opt/anaconda/bin/python
# coding=utf-8

# DroughtMonitoring.py

# PROJECT: Integrating climate change risks in agriculture and health sectors in Samoa
# Author: Nicolas Fauchereau, NIWA
# Date: August 2014
# can test using:
# `./DroughtMonitoring.py base_path stations from_date to_date`

### ===========================================================================
# imports
import os, sys
import time
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

### ===========================================================================
# Settings
### ===========================================================================
print("Programme: DroughtMonitoring.py\n")

# set to `True` is testing on a local dataset instead of a clide table
local = False

### ===========================================================================
### defines minimum year (station needs to have been opened before that date)
### ===========================================================================
min_year = 1972

### ===========================================================================
### defines the window (days)
### ===========================================================================
window = 90

### ===========================================================================
### defines the minimum percentage of obs to calculate an avearge in the window
### ===========================================================================
min_per = 0.6

### ===========================================================================
### defines the `alert` levels (in percentage of normal)
### ===========================================================================
level_1 = 40
level_2 = 10

# prints the system date
print("Date: %s\n" % ( time.strftime("%d-%m-%Y") ))

### ===========================================================================
# CLIDE utilities
### ===========================================================================
if local:
    sys.path.append('../clidesc')
    # clide: the interface to clide itself
    from clide import *
    # utils: some utility functions
    from utils import *
else:
    # source path from the absolute path containing the script
    source_path = os.path.dirname(os.path.realpath(sys.argv[0]))
    sys.path.append(os.path.join(source_path, '../common'))
    # clide: the interface to clide itself
    from clide import *
    # utils: some utility functions
    from utils import *
    # opens the connection
    conn = clidesc_open(base_path)

### ===========================================================================
# STATION DETAILS
### ===========================================================================
if local:
    sName = 'Station_Name'
else:
    station = clidesc_stations(conn, stations)
    # get the primary name
    sName = station['name_primary'][0]
    # get the start date
    sDate = station['start_date'][0]

### ===========================================================================
# tests the opening data of the station
### ===========================================================================
if pd.to_datetime(sDate) > datetime(min_year,1,1):
    print("! WARNING, the station has been opened after 1972, not enough data to calculate normals\n")
    sys.exit(1)

### ===========================================================================
# DATA SETUP
### ===========================================================================

if local:
    # if local loads a example 24 hours rainfall dataset
	iFile = '../data/table_rain24h.csv'
	iData = pd.read_csv(iFile, index_col=0)
else:
    #loads the data from clide
    iData = clidesc_rain24h(conn, stations, datetime(min_year, 1 1).strftime("%Y-%m-%d"), to_date)

clidesc_progress(base_path, 10)

# The DataFrame from clide is likely to contain missing indexes
# so we create a continuous datetime index and reindex

daterange = pd.period_range(start = datetime(min_year, 1 1).strftime("%Y-%m-%d"), end = to_date, freq = 'D').to_datetime()

iData = iData.reindex(daterange)

### ===========================================================================
### calculates the rolling cumulative rainfall over the period
### ===========================================================================

min_periods = np.int(np.floor(min_per * window))
datars = pd.rolling_sum(iData[['rain_24h']], window, min_periods=min_periods)

clidesc_progress(base_path, 20)

### ===========================================================================
### calculates monthly means (will be displayed by the green bars in the graphics)
### ===========================================================================

datam = iData[['rain_24h']].resample('1M', how='sum')
datam.columns = [['monthly means']]

clidesc_progress(base_path, 30)

### ===========================================================================
### calculates the normals for the [window] days periods
### ===========================================================================

normal = ['1972-01-01','2012-12-31']

normals = datars[normal[0]:normal[-1]]

normals['year'] = normals.index.year
normals['month'] = normals.index.month
normals['day'] = normals.index.day

datars['year'] = datars.index.year
datars['month'] = datars.index.month
datars['day'] = datars.index.day

clim = normals.groupby(['month','day'])[['rain_24h']].mean()

clim['month'] = clim.index.get_level_values(0)
clim['day'] = clim.index.get_level_values(1)

merged_df_mean = pd.merge(datars, clim, how='left', on=['month','day'])

merged_df_mean = merged_df_mean.sort(['year','month','day'], axis=0)

merged_df_mean.columns = ['rain_24h', 'year', 'month','day', 'clim']

clidesc_progress(base_path, 50)

### ===========================================================================
### calculates the anomalies (in percentage of normals) for the [window] days
### periods and add these to the DataFrame
### ===========================================================================

merged_df_mean['anomalies'] = (merged_df_mean['rain_24h'] / merged_df_mean['clim']) * 100.

clidesc_progress(base_path, 60)

merged_df_mean.index = datars.index

merged_df_mean['monthly means'] = datam['monthly means']

merged_df_mean['monthly means'].fillna(method='bfill', inplace=True)

### ===========================================================================
### Selects the period (from_date to the end)
### ===========================================================================

subsetper = merged_df_mean[from_date::]

clidesc_progress(base_path, 70)

### ===========================================================================
### plot
### ===========================================================================

f, ax = plt.subplots(figsize=(12,8))

f.subplots_adjust(bottom=0.15, left=0.08, right=1)

### get the axes position and shrink the width to 80 % (for legend placement)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.78, box.height])

### first ax: monthtly average rainfall (mm)

ax.plot(subsetper.index, subsetper['monthly means'], color='g', alpha=0.5)
ax.fill_between(subsetper.index, subsetper['monthly means'],color='g', alpha=0.3)
ax.set_ylim(0, max(subsetper['monthly means'] + 0.2 * subsetper['monthly means']))
ax.set_ylabel("monthly (calendar month) rainfall (mm)")
[l.set_rotation(90) for l in ax.xaxis.get_ticklabels()]
ax.grid('off') # no grid lines for the monthly rainfall

### second ax: [window] days anomalies (% of normal)

ax2 = ax.twinx()
box = ax2.get_position()
ax2.set_position([box.x0, box.y0, box.width * 0.78, box.height])

ax2.plot(subsetper.index, subsetper['anomalies'], lw=2, color='b', label='90 days rainfall', zorder=12)

# horizontal lines
ax2.axhline(100, color='k', zorder=10)
ax2.axhline(level_1, color='r', zorder=10)
ax2.axhline(level_2, color='r', zorder=10, lw=2)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * 100., \
                 where=subsetper['anomalies'] > 100., color='b', interpolate=True, alpha=0.05)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * level_2, \
                 where=subsetper['anomalies'] < level_2, color='#700000', interpolate=True, alpha=0.9)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * level_1, \
                 where=subsetper['anomalies'] < level_1, color='r', interpolate=True, alpha=0.4)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * 100., \
                 where=subsetper['anomalies'] < 100., color='r', interpolate=True, alpha=0.05)

ax2.set_ylabel('percentage of normal', fontsize=16)

ax2.grid('on') # gridlines ON for the percentage anomalies

ax2.set_ylim(0, subsetper['anomalies'].max()+5)

# this proves necessary or the (possible) missing values
# at the end are ignored in the plot ...
ax2.set_xlim(subsetper.index[0], subsetper.index[-1])

# title
ax2.set_title("{} days cumulative rainfall anomalies (% of normal)\nending {}".format(window, subsetper.index[-1].strftime("%d %B %Y")), fontsize=16)

### legend section
p1 = plt.Rectangle((0, 0), 1, 1, fc="r", alpha=0.05)
p2 = plt.Rectangle((0, 0), 1, 1, fc="r", alpha=0.4)
p3 = plt.Rectangle((0, 0), 1, 1, fc="#700000", alpha=0.9)
leg = plt.legend([p1, p2, p3], ['< mean', '< {}%'.format(level_1), '< {}%'.format()],\
          loc='upper left', bbox_to_anchor = (1.035, 1.01), fontsize=19)
leg.draw_frame(False)

# saves the figure in png (dpi = 200)
f.savefig(os.path.join(base_path,'drought_monitoring_{}days_WS.png'.format(window)), dpi=200)

clidesc_progress(base_path, 100)

clidesc_close(base_path, conn)
<commit_msg>made the selection of normal more explicit in the DroughtMonitoring script<commit_after>#!/opt/anaconda/bin/python
# coding=utf-8

# DroughtMonitoring.py

# PROJECT: Integrating climate change risks in agriculture and health sectors in Samoa
# Author: Nicolas Fauchereau, NIWA
# Date: August 2014
# can test using:
# `./DroughtMonitoring.py base_path stations from_date to_date`

### ===========================================================================
# imports
import os, sys
import time
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

### ===========================================================================
# Settings
### ===========================================================================
print("Programme: DroughtMonitoring.py\n")

# set to `True` is testing on a local dataset instead of a clide table
local = False

### ===========================================================================
### defines starting and end years for the normal calculation
### if the station has been open AFTER `norma_year_start`, the script will fail
### ===========================================================================
norm_year_start = 1972
norm_year_end = 2012

### ===========================================================================
### defines the window (days)
### ===========================================================================
window = 90

### ===========================================================================
### defines the minimum percentage of obs to calculate an average in the window
### ===========================================================================
min_per = 0.6

### ===========================================================================
### defines the `alert` levels (in percentage of normal)
### ===========================================================================
level_1 = 40
level_2 = 10

# prints the system date
print("Date: %s\n" % ( time.strftime("%d-%m-%Y") ))

### ===========================================================================
# CLIDE utilities
### ===========================================================================
if local:
    sys.path.append('../clidesc')
    # clide: the interface to clide itself
    from clide import *
    # utils: some utility functions
    from utils import *
else:
    # source path from the absolute path containing the script
    source_path = os.path.dirname(os.path.realpath(sys.argv[0]))
    sys.path.append(os.path.join(source_path, '../common'))
    # clide: the interface to clide itself
    from clide import *
    # utils: some utility functions
    from utils import *
    # opens the connection
    conn = clidesc_open(base_path)

### ===========================================================================
# STATION DETAILS
### ===========================================================================
if local:
    sName = 'Station_Name'
else:
    station = clidesc_stations(conn, stations)
    # get the primary name
    sName = station['name_primary'][0]
    # get the start date
    sDate = station['start_date'][0]

### ===========================================================================
# tests the opening data of the station
### ===========================================================================
if pd.to_datetime(sDate) > datetime(norm_year_start,1,1):
    print("! WARNING, the station has been opened after {}, not enough data to calculate normals\n".format(str(norm_year_start)))
    sys.exit(1)

### ===========================================================================
# DATA SETUP
### ===========================================================================

if local:
    # if local loads a example 24 hours rainfall dataset
	iFile = '../data/table_rain24h.csv'
	iData = pd.read_csv(iFile, index_col=0)
else:
    #loads the data from clide
    iData = clidesc_rain24h(conn, stations, datetime(norm_year_start, 1 1).strftime("%Y-%m-%d"), to_date)

clidesc_progress(base_path, 10)

# The DataFrame from clide is likely to contain missing indexes
# so we create a continuous datetime index and reindex

daterange = pd.period_range(start = datetime(norm_year_start, 1 1).strftime("%Y-%m-%d"), end = to_date, freq = 'D').to_datetime()

iData = iData.reindex(daterange)

### ===========================================================================
### calculates the rolling cumulative rainfall over the period
### ===========================================================================

min_periods = np.int(np.floor(min_per * window))
datars = pd.rolling_sum(iData[['rain_24h']], window, min_periods=min_periods)

clidesc_progress(base_path, 20)

### ===========================================================================
### calculates monthly means (will be displayed by the green bars in the graphics)
### ===========================================================================

datam = iData[['rain_24h']].resample('1M', how='sum')
datam.columns = [['monthly means']]

clidesc_progress(base_path, 30)

### ===========================================================================
### calculates the normals for the [window] days periods
### ===========================================================================

normal = ['{}-01-01'.format(str(norm_year_start)),'{}-12-31'.format(str(norm_year_end))]

normals = datars[normal[0]:normal[-1]]

normals['year'] = normals.index.year
normals['month'] = normals.index.month
normals['day'] = normals.index.day

datars['year'] = datars.index.year
datars['month'] = datars.index.month
datars['day'] = datars.index.day

clim = normals.groupby(['month','day'])[['rain_24h']].mean()

clim['month'] = clim.index.get_level_values(0)
clim['day'] = clim.index.get_level_values(1)

merged_df_mean = pd.merge(datars, clim, how='left', on=['month','day'])

merged_df_mean = merged_df_mean.sort(['year','month','day'], axis=0)

merged_df_mean.columns = ['rain_24h', 'year', 'month','day', 'clim']

clidesc_progress(base_path, 50)

### ===========================================================================
### calculates the anomalies (in percentage of normals) for the [window] days
### periods and add these to the DataFrame
### ===========================================================================

merged_df_mean['anomalies'] = (merged_df_mean['rain_24h'] / merged_df_mean['clim']) * 100.

clidesc_progress(base_path, 60)

merged_df_mean.index = datars.index

merged_df_mean['monthly means'] = datam['monthly means']

merged_df_mean['monthly means'].fillna(method='bfill', inplace=True)

### ===========================================================================
### Selects the period (from_date to the end)
### ===========================================================================

subsetper = merged_df_mean[from_date::]

clidesc_progress(base_path, 70)

### ===========================================================================
### plot
### ===========================================================================

f, ax = plt.subplots(figsize=(12,8))

f.subplots_adjust(bottom=0.15, left=0.08, right=1)

### get the axes position and shrink the width to ~80 % (for legend placement)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.78, box.height])

### first ax: monthtly average rainfall (mm)

ax.plot(subsetper.index, subsetper['monthly means'], color='g', alpha=0.5)
ax.fill_between(subsetper.index, subsetper['monthly means'],color='g', alpha=0.3)
ax.set_ylim(0, max(subsetper['monthly means'] + 0.2 * subsetper['monthly means']))
ax.set_ylabel("monthly (calendar month) rainfall (mm)")
[l.set_rotation(90) for l in ax.xaxis.get_ticklabels()]
ax.grid('off') # no grid lines for the monthly rainfall

### second ax: [window] days anomalies (% of normal)

ax2 = ax.twinx()
box = ax2.get_position()
ax2.set_position([box.x0, box.y0, box.width * 0.78, box.height])

ax2.plot(subsetper.index, subsetper['anomalies'], lw=2, color='b', label='90 days rainfall', zorder=12)

# horizontal lines
ax2.axhline(100, color='k', zorder=10)
ax2.axhline(level_1, color='r', zorder=10)
ax2.axhline(level_2, color='r', zorder=10, lw=2)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * 100., \
                 where=subsetper['anomalies'] > 100., color='b', interpolate=True, alpha=0.05)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * level_2, \
                 where=subsetper['anomalies'] < level_2, color='#700000', interpolate=True, alpha=0.9)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * level_1, \
                 where=subsetper['anomalies'] < level_1, color='r', interpolate=True, alpha=0.4)

ax2.fill_between(subsetper.index, subsetper['anomalies'], np.ones(len(subsetper)) * 100., \
                 where=subsetper['anomalies'] < 100., color='r', interpolate=True, alpha=0.05)

ax2.set_ylabel('percentage of normal', fontsize=16)

ax2.grid('on') # gridlines ON for the percentage anomalies

ax2.set_ylim(0, subsetper['anomalies'].max()+5)

# this proves necessary or the (possible) missing values
# at the end are ignored in the plot ...
ax2.set_xlim(subsetper.index[0], subsetper.index[-1])

# title
ax2.set_title("{} days cumulative rainfall anomalies (% of normal)\nending {}".format(window, subsetper.index[-1].strftime("%d %B %Y")), fontsize=16)

### legend section
p1 = plt.Rectangle((0, 0), 1, 1, fc="r", alpha=0.05)
p2 = plt.Rectangle((0, 0), 1, 1, fc="r", alpha=0.4)
p3 = plt.Rectangle((0, 0), 1, 1, fc="#700000", alpha=0.9)
leg = plt.legend([p1, p2, p3], ['< mean', '< {}%'.format(level_1), '< {}%'.format()],\
          loc='upper left', bbox_to_anchor = (1.035, 1.01), fontsize=19)
leg.draw_frame(False)

# saves the figure in png (dpi = 200)
f.savefig(os.path.join(base_path,'drought_monitoring_{}days_WS.png'.format(window)), dpi=200)

clidesc_progress(base_path, 100)

clidesc_close(base_path, conn)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import os
import StringIO

from six import string_types, text_type

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.query import QuerySet
from django.http import HttpResponse
from django.shortcuts import render
from django.template import Context
from django.template.loader import get_template
from django.utils import timezone
from django.utils.text import slugify
from uuidfield import UUIDField
from wagtail.contrib.wagtailroutablepage.models import RoutablePageMixin, route
from wagtail.wagtailadmin.edit_handlers import FieldPanel
from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore.utils import resolve_model_string
from wagtail.wagtailsearch import index
from wagtail.wagtailsearch.backends import get_search_backend
from xhtml2pdf import pisa

from .views import frontend

JOBINDEX_MODEL_CLASSES = []
_JOBINDEX_CONTENT_TYPES = []

def get_jobindex_content_types():
    global _JOBINDEX_CONTENT_TYPES
    if len(_JOBINDEX_CONTENT_TYPES) != len(JOBINDEX_MODEL_CLASSES):
        _JOBINDEX_CONTENT_TYPES = [
            ContentType.objects.get_for_model(cls)
            for cls in JOBINDEX_MODEL_CLASSES]
    return _JOBINDEX_CONTENT_TYPES

class JobIndexMixin(RoutablePageMixin):
    job_model = None
    subpage_types = []

    @route(r'^(?P<uuid>[0-9a-f-]+)/$', name='job')
    def v_job(s, r, **k):
        return frontend.job_detail(r, s, **k)

    @route(r'^(?P<uuid>[0-9a-f-]+)/pdf/$', name='job_pdf')
    def v_job_pdf(s, r, **k):
        return frontend.job_pdf(r, s, **k)

    @classmethod
    def get_job_model(cls):
        if isinstance(cls.job_model, models.Model):
            return cls.job_model
        elif isinstance(cls.job_model, string_types):
            return resolve_model_string(cls.job_model, cls._meta.app_label)
        else:
            raise ValueError('Can not resolve {0}.job_model in to a model: {1!r}'.format(
                cls.__name__, cls.job_model))

class AbstractJobQuerySet(QuerySet):
    def search(self, query_string, fields=None, backend='default'):
        """
        This runs a search query on all the pages in the QuerySet
        """
        search_backend = get_search_backend(backend)
        return search_backend.search(query_string, self)

class AbstractJob(models.Model):
    jobindex = models.ForeignKey(Page)
    uuid = UUIDField(auto=True, null=True, default=None)
    issue_date = models.DateTimeField('Issue date', default=timezone.now)
    last_updated = models.DateTimeField(auto_now=True)
    amount = models.DecimalField(
        max_digits=6,
        decimal_places=2,
        help_text='Without dollar sign($)')

    # User creating the job
    user = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True)

    panels = [
        FieldPanel('amount'),
    ]

    objects = AbstractJobQuerySet.as_manager()

    class Meta:
        abstract = True

    def get_nice_url(self):
        return slugify(text_type(self))

    def get_template(self, request):
        try:
            return self.template
        except AttributeError:
            return '{0}/{1}.html'.format(self._meta.app_label, self._meta.model_name)

    def url(self):
        jobindex = self.jobindex.specific
        url = jobindex.url + jobindex.reverse_subpage('job', kwargs={
            'uuid': str(self.uuid)})
        return url

    def serve(self, request):
        return render(request, self.get_template(request), {
            'self': self.jobindex.specific,
            'job': self,
        })

    def serve_pdf(self, request):
        # Convert HTML URIs to absolute system paths
        def link_callback(uri, rel):
            # use short variable names
            sUrl = settings.STATIC_URL
            sRoot = settings.STATIC_ROOT
            mUrl = settings.MEDIA_URL
            mRoot = settings.MEDIA_ROOT

            # convert URIs to absolute system paths
            if uri.startswith(mUrl):
                path = os.path.join(mRoot, uri.replace(mUrl, ""))
            elif uri.startswith(sUrl):
                path = os.path.join(sRoot, uri.replace(sUrl, ""))

            # make sure that file exists
            if not os.path.isfile(path):
                    raise Exception(
                            'media URI must start with %s or %s' % \
                            (sUrl, mUrl))
            return path

        # Render html content through html template with context
        template = get_template(settings.PDF_TEMPLATE)
        html = template.render(Context({'job': self}))
        print type(self)

        # Write PDF to file
        file = StringIO.StringIO()
        pisaStatus = pisa.CreatePDF(
            html,
            dest=file,
            link_callback=link_callback)

        # Return PDF document through a Django HTTP response
        file.seek(0)
        return HttpResponse(file, content_type='application/pdf')

# Need to import this down here to prevent circular imports :(
<commit_msg>Fixed py2 print statement<commit_after>from __future__ import absolute_import, unicode_literals, print_function

import os
import StringIO

from six import string_types, text_type

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.query import QuerySet
from django.http import HttpResponse
from django.shortcuts import render
from django.template import Context
from django.template.loader import get_template
from django.utils import timezone
from django.utils.text import slugify
from uuidfield import UUIDField
from wagtail.contrib.wagtailroutablepage.models import RoutablePageMixin, route
from wagtail.wagtailadmin.edit_handlers import FieldPanel
from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore.utils import resolve_model_string
from wagtail.wagtailsearch import index
from wagtail.wagtailsearch.backends import get_search_backend
from xhtml2pdf import pisa

from .views import frontend

JOBINDEX_MODEL_CLASSES = []
_JOBINDEX_CONTENT_TYPES = []

def get_jobindex_content_types():
    global _JOBINDEX_CONTENT_TYPES
    if len(_JOBINDEX_CONTENT_TYPES) != len(JOBINDEX_MODEL_CLASSES):
        _JOBINDEX_CONTENT_TYPES = [
            ContentType.objects.get_for_model(cls)
            for cls in JOBINDEX_MODEL_CLASSES]
    return _JOBINDEX_CONTENT_TYPES

class JobIndexMixin(RoutablePageMixin):
    job_model = None
    subpage_types = []

    @route(r'^(?P<uuid>[0-9a-f-]+)/$', name='job')
    def v_job(s, r, **k):
        return frontend.job_detail(r, s, **k)

    @route(r'^(?P<uuid>[0-9a-f-]+)/pdf/$', name='job_pdf')
    def v_job_pdf(s, r, **k):
        return frontend.job_pdf(r, s, **k)

    @classmethod
    def get_job_model(cls):
        if isinstance(cls.job_model, models.Model):
            return cls.job_model
        elif isinstance(cls.job_model, string_types):
            return resolve_model_string(cls.job_model, cls._meta.app_label)
        else:
            raise ValueError('Can not resolve {0}.job_model in to a model: {1!r}'.format(
                cls.__name__, cls.job_model))

class AbstractJobQuerySet(QuerySet):
    def search(self, query_string, fields=None, backend='default'):
        """
        This runs a search query on all the pages in the QuerySet
        """
        search_backend = get_search_backend(backend)
        return search_backend.search(query_string, self)

class AbstractJob(models.Model):
    jobindex = models.ForeignKey(Page)
    uuid = UUIDField(auto=True, null=True, default=None)
    issue_date = models.DateTimeField('Issue date', default=timezone.now)
    last_updated = models.DateTimeField(auto_now=True)
    amount = models.DecimalField(
        max_digits=6,
        decimal_places=2,
        help_text='Without dollar sign($)')

    # User creating the job
    user = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True)

    panels = [
        FieldPanel('amount'),
    ]

    objects = AbstractJobQuerySet.as_manager()

    class Meta:
        abstract = True

    def get_nice_url(self):
        return slugify(text_type(self))

    def get_template(self, request):
        try:
            return self.template
        except AttributeError:
            return '{0}/{1}.html'.format(self._meta.app_label, self._meta.model_name)

    def url(self):
        jobindex = self.jobindex.specific
        url = jobindex.url + jobindex.reverse_subpage('job', kwargs={
            'uuid': str(self.uuid)})
        return url

    def serve(self, request):
        return render(request, self.get_template(request), {
            'self': self.jobindex.specific,
            'job': self,
        })

    def serve_pdf(self, request):
        # Convert HTML URIs to absolute system paths
        def link_callback(uri, rel):
            # use short variable names
            sUrl = settings.STATIC_URL
            sRoot = settings.STATIC_ROOT
            mUrl = settings.MEDIA_URL
            mRoot = settings.MEDIA_ROOT

            # convert URIs to absolute system paths
            if uri.startswith(mUrl):
                path = os.path.join(mRoot, uri.replace(mUrl, ""))
            elif uri.startswith(sUrl):
                path = os.path.join(sRoot, uri.replace(sUrl, ""))

            # make sure that file exists
            if not os.path.isfile(path):
                    raise Exception(
                            'media URI must start with %s or %s' % \
                            (sUrl, mUrl))
            return path

        # Render html content through html template with context
        template = get_template(settings.PDF_TEMPLATE)
        html = template.render(Context({'job': self}))
        print(type(self))

        # Write PDF to file
        file = StringIO.StringIO()
        pisaStatus = pisa.CreatePDF(
            html,
            dest=file,
            link_callback=link_callback)

        # Return PDF document through a Django HTTP response
        file.seek(0)
        return HttpResponse(file, content_type='application/pdf')

# Need to import this down here to prevent circular imports :(
<|endoftext|>