<commit_before>__author__ = 'Frederik Diehl'

import threading.lock
from apsis.models.experiment import Experiment
from apsis.models.candidate import Candidate
from apsis.utilities.optimizer_utils import check_optimizer, build_queue_optimizer
from apsis.utilities.file_utils import ensure_directory_exists
import matplotlib.pyplot as plt
from apsis.utilities.plot_utils import plot_lists, create_figure
import datetime
import os
import time
from apsis.utilities.logging_utils import get_logger
import Queue
import sys

import multiprocessing
import signal

class BasicExperimentAssistant(object):
    """
    This ExperimentAssistant assists with executing experiments.

    It provides methods for getting candidates to evaluate, returning the
    evaluated Candidate and administrates the optimizer.

    Attributes
    ----------
    optimizer : Optimizer
        This is an optimizer implementing the corresponding functions: It
        gets an experiment instance, and returns one or multiple candidates
        which should be evaluated next.
    optimizer_arguments : dict
        These are arguments for the optimizer. Refer to their documentation
        as to which are available.
    experiment : Experiment
        The experiment this assistant assists with.
    write_directory_base : string or None
        The directory to write all results to. If not
        given, a directory with timestamp will automatically be created
        in write_directory_base
    csv_write_frequency : int
        States how often the csv file should be written to.
        If set to 0 no results will be written.
    logger : logging.logger
        The logger for this class.
    """

    AVAILABLE_STATUS = ["finished", "pausing", "working"]

    optimizer = None
    optimizer_arguments = None
    experiment = None

    write_directory_base = None
    experiment_directory_base = None
    csv_write_frequency = None
    csv_steps_written = 0

    logger = None

    def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
                 minimization=True, write_directory_base="/tmp/APSIS_WRITING",
                 experiment_directory_base=None, csv_write_frequency=1):
        """
        Initializes the BasicExperimentAssistant.

        Parameters
        ----------
        name : string
            The name of the experiment. This does not have to be unique, but is
            for human orientation.
        optimizer : Optimizer instance or string
            This is an optimizer implementing the corresponding functions: It
            gets an experiment instance, and returns one or multiple candidates
            which should be evaluated next.
            Alternatively, it can be a string corresponding to the optimizer,
            as defined by apsis.utilities.optimizer_utils.
        param_defs : dict of ParamDef.
            This is the parameter space defining the experiment.
        experiment : Experiment
            Preinitialize this assistant with an existing experiment.
        optimizer_arguments=None : dict
            These are arguments for the optimizer. Refer to their documentation
            as to which are available.
        minimization=True : bool
            Whether the problem is one of minimization or maximization.
        write_directory_base : string, optional
            The global base directory for all writing. Will only be used
            for creation of experiment_directory_base if this is not given.
        experiment_directory_base : string or None, optional
            The directory to write all the results to. If not
            given a directory with timestamp will automatically be created
            in write_directory_base
        csv_write_frequency : int, optional
            States how often the csv file should be written to.
            If set to 0 no results will be written.
        """
        self.logger = get_logger(self)
        self.logger.info("Initializing experiment assistant.")
        self.optimizer = optimizer
        self.optimizer_arguments = optimizer_arguments

        if experiment is None:
            self.experiment = Experiment(name, param_defs, minimization)
        else:
            self.experiment = experiment

        self.csv_write_frequency = csv_write_frequency

        if self.csv_write_frequency != 0:
            self.write_directory_base = write_directory_base
            if experiment_directory_base is not None:
                self.experiment_directory_base = experiment_directory_base
                ensure_directory_exists(self.experiment_directory_base)
            else:
                self._create_experiment_directory()
        self.logger.info("Experiment assistant successfully initialized.")

    def get_next_candidate(self):
        """
        Returns the Candidate next to evaluate.

        Returns
        -------
        next_candidate : Candidate or None
            The Candidate object that should be evaluated next. May be None.
        """
        self.logger.info("Returning next candidate.")
        self.optimizer = check_optimizer(self.optimizer,
                                optimizer_arguments=self.optimizer_arguments)
        if not self.experiment.candidates_pending:
            self.experiment.candidates_pending.extend(
                self.optimizer.get_next_candidates(self.experiment))
        next_candidate = self.experiment.candidates_pending.pop()
        self.logger.info("next candidate found: %s" %next_candidate)
        return next_candidate

    def update(self, candidate, status="finished"):
        """
        Updates the experiment_assistant with the status of an experiment
        evaluation.

        Parameters
        ----------
        candidate : Candidate
            The Candidate object whose status is updated.
        status : {"finished", "pausing", "working"}
            A string defining the status change. Can be one of the following:
            - finished: The Candidate is now finished.
            - pausing: The evaluation of Candidate has been paused and can be
                resumed by another worker.
            - working: The Candidate is now being worked on by a worker.

        """
        if status not in self.AVAILABLE_STATUS:
            message = ("status not in %s but %s."
                             %(str(self.AVAILABLE_STATUS), str(status)))
            self.logger.error(message)
            raise ValueError(message)

        if not isinstance(candidate, Candidate):
            message = ("candidate %s not a Candidate instance."
                             %str(candidate))
            self.logger.error(message)
            raise ValueError(message)

        self.logger.info("Got new %s of candidate %s with parameters %s"
                         " and result %s" %(status, candidate, candidate.params,
                                            candidate.result))

        if status == "finished":
            self.experiment.add_finished(candidate)
            #Also delete all pending candidates from the experiment - we have
            #new data available.
            self.experiment.candidates_pending = []

            #invoke the writing to files
            step = len(self.experiment.candidates_finished)
            if self.csv_write_frequency != 0 and step != 0 \
                    and step % self.csv_write_frequency == 0:
                self._append_to_detailed_csv()

        elif status == "pausing":
            self.experiment.add_pausing(candidate)
        elif status == "working":
            self.experiment.add_working(candidate)

    def get_best_candidate(self):
        """
        Returns the best candidate to date.

        Returns
        -------
        best_candidate : candidate or None
            Returns a candidate if there is a best one (which corresponds to
            at least one candidate evaluated) or None if none exists.
        """
        return self.experiment.best_candidate

    def _append_to_detailed_csv(self):
        if len(self.experiment.candidates_finished) <= self.csv_steps_written:
            return

        #create file and header if
        wHeader = False
        if self.csv_steps_written == 0:
            #set use header
            wHeader = True

        csv_string, steps_included = self.experiment.to_csv_results(wHeader=wHeader,
                                            fromIndex=self.csv_steps_written)

        #write
        filename = os.path.join(self.experiment_directory_base,
                                self.experiment.name + "_results.csv")

        with open(filename, 'a+') as detailed_file:
            detailed_file.write(csv_string)

        self.csv_steps_written += steps_included

    def _create_experiment_directory(self):
        global_start_date = time.time()

        date_name = datetime.datetime.utcfromtimestamp(
                global_start_date).strftime("%Y-%m-%d_%H:%M:%S")

        self.experiment_directory_base = os.path.join(self.write_directory_base,
                                    self.experiment.name + "_" + date_name)

        ensure_directory_exists(self.experiment_directory_base)

class PrettyExperimentAssistant(BasicExperimentAssistant):
    """
    A 'prettier' version of the experiment assistant, mostly through plots.
    """

    def plot_result_per_step(self, show_plot=True, ax=None, color="b",
                             plot_min=None, plot_max=None):
        """
        Returns (and plots) the plt.figure plotting the results over the steps.

        Parameters
        ----------
        show_plot : bool, optional
            Whether to show the plot after creation.
        ax : None or matplotlib.Axes, optional
            The ax to update. If None, a new figure will be created.
        color : string, optional
            A string representing a pyplot color.
        plot_min : float, optional
            The smallest value to plot on the y axis.
        plot_max : float, optional
            The biggest value to plot on the y axis.

        Returns
        -------
        ax: plt.Axes
            The Axes containing the results over the steps.
        """

        plots = self._best_result_per_step_dicts(color)
        if self.experiment.minimization_problem:
            legend_loc = 'upper right'
        else:
            legend_loc = 'upper left'

        plot_options = {
            "legend_loc": legend_loc,
            "x_label": "steps",
            "y_label": "result",
            "title": "Plot of %s result over the steps."
                     %(str(self.experiment.name))
        }
        fig, ax = plot_lists(plots, ax=ax, fig_options=plot_options, plot_min=plot_min, plot_max=plot_max)

        if show_plot:
            plt.show(True)

        return ax

    def _best_result_per_step_dicts(self, color="b"):
        """
        Returns a dict to use with plot_utils.

        Parameters
        ----------
        color="b": string
            A pyplot-color representing string. Both plots will have that
            color.

        Returns
        -------
        dicts: list of dicts
            Two dicts, one for step_eval, one for step_best, and their
            corresponding definitions.
        """
        x, step_eval, step_best = self._best_result_per_step_data()

        step_eval_dict = {
            "x": x,
            "y": step_eval,
            "type": "scatter",
            "label": "%s, current result" %(str(self.experiment.name)),
            "color": color
        }

        step_best_dict = {
            "x": x,
            "y": step_best,
            "type": "line",
            "color": color,
            "label": "%s, best result" %(str(self.experiment.name))
        }

        #print [step_eval_dict, step_best_dict]

        return [step_eval_dict, step_best_dict]

    def _best_result_per_step_data(self):
        """
        This internal function returns goodness of the results by step.

        This returns an x coordinate, and for each of them a value for the
        currently evaluated result and the best found result.

        Returns
        -------
        x: list of ints
            The results per step. Should usually be [0, ..., maxSteps]
        step_evaluation: list of floats
            The result of the evaluated candidate during the corresponding step
        step_best: list of floats
            The best result that has been found until then.

        """
        x = []
        step_evaluation = []
        step_best = []
        best_candidate = None
        for i, e in enumerate(self.experiment.candidates_finished):
            x.append(i)
            step_evaluation.append(e.result)
            if self.experiment.better_cand(e, best_candidate):
                best_candidate = e
                step_best.append(e.result)

            else:
                step_best.append(best_candidate.result)
        return x, step_evaluation, step_best

class ParallelExperimentAssistant(PrettyExperimentAssistant, multiprocessing.Process):
    rcv_queue = None

    optimizer_queue = None
    optimizer_process = None

    def __init__(self, name, optimizer, param_defs,
                 experiment=None, optimizer_arguments=None, minimization=True,
                 write_directory_base="/tmp/APSIS_WRITING",
                 experiment_directory_base=None, csv_write_frequency=1,
                 max_time_without_update=None):

        self.rcv_queue = multiprocessing.Queue()
        self.return_queue = multiprocessing.Queue()

        self._build_new_optimizer()
        super(ParallelExperimentAssistant, self).\
            __init__(name, optimizer, param_defs, experiment=experiment,
                     optimizer_arguments=optimizer_arguments, minimization=minimization,
                     write_directory_base=write_directory_base, experiment_directory_base=experiment_directory_base,
                     csv_write_frequency=csv_write_frequency)

    def run(self):
        while True:
            msg = self.rcv_queue.get(block=True)
            result = getattr(self, "_" + msg["action"])(msg)
            self.return_queue.put(result, block=True)

    def get_next_candidate(self):
        conn_rcv, conn_send = multiprocessing.Pipe(duplex=False)
        msg = {"action": "get_next_candidate",
               "return_pipe": conn_send}
        self.rcv_queue.put(msg)
        next_candidate = conn_rcv.get(block=True)
        conn_rcv.close()
        return next_candidate

    def _get_next_candidate(self, msg):
        self.logger.info("Returning next candidate.")
        next_candidate = None
        while next_candidate is not None:
            if not self.experiment.candidates_pending:
                next_candidate = self.experiment.candidates_pending.pop()
            else:
                if not self.optimizer_queue.empty():
                    next_candidate = self.optimizer_queue.get()
        msg["return_pipe"].put(next_candidate)

    def update(self, candidate, status="finished"):
        msg = {
            "action": "update",
            "candidate": candidate,
            "status": status
        }
        self.rcv_queue.put(msg)

    def _update(self, msg):
        status = msg["status"]
        candidate = msg["candidate"]
        if status not in self.AVAILABLE_STATUS:
            message = ("status not in %s but %s."
                             %(str(self.AVAILABLE_STATUS), str(status)))
            self.logger.error(message)
            raise ValueError(message)

        if not isinstance(candidate, Candidate):
            message = ("candidate %s not a Candidate instance."
                             %str(candidate))
            self.logger.error(message)
            raise ValueError(message)

        self.logger.info("Got new %s of candidate %s with parameters %s"
                         " and result %s" %(status, candidate, candidate.params,
                                            candidate.result))

        if status == "finished":
            self.experiment.add_finished(candidate)
            #Also delete all pending candidates from the experiment - we have
            #new data available.
            self.experiment.candidates_pending = []

            #invoke the writing to files
            step = len(self.experiment.candidates_finished)
            if self.csv_write_frequency != 0 and step != 0 \
                    and step % self.csv_write_frequency == 0:
                self._append_to_detailed_csv()
            #and build a new optimizer.
            # This sends SIGINT to the optimizer process, which is used to
            # terminate the process irrespective of its current computation.
            # (But, since it will be catched, it can still do cleanup)
            os.kill(self.optimizer_process.pid, signal.SIGINT)
            self.optimizer_queue.close()

            # And we rebuild the new optimizer.
            self._build_new_optimizer()

        elif status == "pausing":
            self.experiment.add_pausing(candidate)
        elif status == "working":
            self.experiment.add_working(candidate)

    def _build_new_optimizer(self):
        self.optimizer_queue = multiprocessing.Queue()

        self.optimizer = build_queue_optimizer(self.optimizer, self.experiment,
                            self.optimizer_queue,
                            optimizer_arguments=self.optimizer_arguments)
        self.optimizer.start()

    def get_best_candidate(self):
        conn_rcv, conn_send = multiprocessing.Pipe(duplex=False)
        msg = {
            "action": "get_best_candidate",
            "return_pipe": conn_send
        }
        self.rcv_queue.put(msg)
        best_candidate = conn_rcv.get(block=True)
        conn_rcv.close()
        return best_candidate

    def _get_best_candidate(self, msg):
        msg["return_pipe"].put(self.experiment.best_candidate)<commit_msg>ExpAssistant: Deleted unnecessary return queueu initialization.<commit_after>__author__ = 'Frederik Diehl'

import threading.lock
from apsis.models.experiment import Experiment
from apsis.models.candidate import Candidate
from apsis.utilities.optimizer_utils import check_optimizer, build_queue_optimizer
from apsis.utilities.file_utils import ensure_directory_exists
import matplotlib.pyplot as plt
from apsis.utilities.plot_utils import plot_lists, create_figure
import datetime
import os
import time
from apsis.utilities.logging_utils import get_logger
import Queue
import sys

import multiprocessing
import signal

class BasicExperimentAssistant(object):
    """
    This ExperimentAssistant assists with executing experiments.

    It provides methods for getting candidates to evaluate, returning the
    evaluated Candidate and administrates the optimizer.

    Attributes
    ----------
    optimizer : Optimizer
        This is an optimizer implementing the corresponding functions: It
        gets an experiment instance, and returns one or multiple candidates
        which should be evaluated next.
    optimizer_arguments : dict
        These are arguments for the optimizer. Refer to their documentation
        as to which are available.
    experiment : Experiment
        The experiment this assistant assists with.
    write_directory_base : string or None
        The directory to write all results to. If not
        given, a directory with timestamp will automatically be created
        in write_directory_base
    csv_write_frequency : int
        States how often the csv file should be written to.
        If set to 0 no results will be written.
    logger : logging.logger
        The logger for this class.
    """

    AVAILABLE_STATUS = ["finished", "pausing", "working"]

    optimizer = None
    optimizer_arguments = None
    experiment = None

    write_directory_base = None
    experiment_directory_base = None
    csv_write_frequency = None
    csv_steps_written = 0

    logger = None

    def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
                 minimization=True, write_directory_base="/tmp/APSIS_WRITING",
                 experiment_directory_base=None, csv_write_frequency=1):
        """
        Initializes the BasicExperimentAssistant.

        Parameters
        ----------
        name : string
            The name of the experiment. This does not have to be unique, but is
            for human orientation.
        optimizer : Optimizer instance or string
            This is an optimizer implementing the corresponding functions: It
            gets an experiment instance, and returns one or multiple candidates
            which should be evaluated next.
            Alternatively, it can be a string corresponding to the optimizer,
            as defined by apsis.utilities.optimizer_utils.
        param_defs : dict of ParamDef.
            This is the parameter space defining the experiment.
        experiment : Experiment
            Preinitialize this assistant with an existing experiment.
        optimizer_arguments=None : dict
            These are arguments for the optimizer. Refer to their documentation
            as to which are available.
        minimization=True : bool
            Whether the problem is one of minimization or maximization.
        write_directory_base : string, optional
            The global base directory for all writing. Will only be used
            for creation of experiment_directory_base if this is not given.
        experiment_directory_base : string or None, optional
            The directory to write all the results to. If not
            given a directory with timestamp will automatically be created
            in write_directory_base
        csv_write_frequency : int, optional
            States how often the csv file should be written to.
            If set to 0 no results will be written.
        """
        self.logger = get_logger(self)
        self.logger.info("Initializing experiment assistant.")
        self.optimizer = optimizer
        self.optimizer_arguments = optimizer_arguments

        if experiment is None:
            self.experiment = Experiment(name, param_defs, minimization)
        else:
            self.experiment = experiment

        self.csv_write_frequency = csv_write_frequency

        if self.csv_write_frequency != 0:
            self.write_directory_base = write_directory_base
            if experiment_directory_base is not None:
                self.experiment_directory_base = experiment_directory_base
                ensure_directory_exists(self.experiment_directory_base)
            else:
                self._create_experiment_directory()
        self.logger.info("Experiment assistant successfully initialized.")

    def get_next_candidate(self):
        """
        Returns the Candidate next to evaluate.

        Returns
        -------
        next_candidate : Candidate or None
            The Candidate object that should be evaluated next. May be None.
        """
        self.logger.info("Returning next candidate.")
        self.optimizer = check_optimizer(self.optimizer,
                                optimizer_arguments=self.optimizer_arguments)
        if not self.experiment.candidates_pending:
            self.experiment.candidates_pending.extend(
                self.optimizer.get_next_candidates(self.experiment))
        next_candidate = self.experiment.candidates_pending.pop()
        self.logger.info("next candidate found: %s" %next_candidate)
        return next_candidate

    def update(self, candidate, status="finished"):
        """
        Updates the experiment_assistant with the status of an experiment
        evaluation.

        Parameters
        ----------
        candidate : Candidate
            The Candidate object whose status is updated.
        status : {"finished", "pausing", "working"}
            A string defining the status change. Can be one of the following:
            - finished: The Candidate is now finished.
            - pausing: The evaluation of Candidate has been paused and can be
                resumed by another worker.
            - working: The Candidate is now being worked on by a worker.

        """
        if status not in self.AVAILABLE_STATUS:
            message = ("status not in %s but %s."
                             %(str(self.AVAILABLE_STATUS), str(status)))
            self.logger.error(message)
            raise ValueError(message)

        if not isinstance(candidate, Candidate):
            message = ("candidate %s not a Candidate instance."
                             %str(candidate))
            self.logger.error(message)
            raise ValueError(message)

        self.logger.info("Got new %s of candidate %s with parameters %s"
                         " and result %s" %(status, candidate, candidate.params,
                                            candidate.result))

        if status == "finished":
            self.experiment.add_finished(candidate)
            #Also delete all pending candidates from the experiment - we have
            #new data available.
            self.experiment.candidates_pending = []

            #invoke the writing to files
            step = len(self.experiment.candidates_finished)
            if self.csv_write_frequency != 0 and step != 0 \
                    and step % self.csv_write_frequency == 0:
                self._append_to_detailed_csv()

        elif status == "pausing":
            self.experiment.add_pausing(candidate)
        elif status == "working":
            self.experiment.add_working(candidate)

    def get_best_candidate(self):
        """
        Returns the best candidate to date.

        Returns
        -------
        best_candidate : candidate or None
            Returns a candidate if there is a best one (which corresponds to
            at least one candidate evaluated) or None if none exists.
        """
        return self.experiment.best_candidate

    def _append_to_detailed_csv(self):
        if len(self.experiment.candidates_finished) <= self.csv_steps_written:
            return

        #create file and header if
        wHeader = False
        if self.csv_steps_written == 0:
            #set use header
            wHeader = True

        csv_string, steps_included = self.experiment.to_csv_results(wHeader=wHeader,
                                            fromIndex=self.csv_steps_written)

        #write
        filename = os.path.join(self.experiment_directory_base,
                                self.experiment.name + "_results.csv")

        with open(filename, 'a+') as detailed_file:
            detailed_file.write(csv_string)

        self.csv_steps_written += steps_included

    def _create_experiment_directory(self):
        global_start_date = time.time()

        date_name = datetime.datetime.utcfromtimestamp(
                global_start_date).strftime("%Y-%m-%d_%H:%M:%S")

        self.experiment_directory_base = os.path.join(self.write_directory_base,
                                    self.experiment.name + "_" + date_name)

        ensure_directory_exists(self.experiment_directory_base)

class PrettyExperimentAssistant(BasicExperimentAssistant):
    """
    A 'prettier' version of the experiment assistant, mostly through plots.
    """

    def plot_result_per_step(self, show_plot=True, ax=None, color="b",
                             plot_min=None, plot_max=None):
        """
        Returns (and plots) the plt.figure plotting the results over the steps.

        Parameters
        ----------
        show_plot : bool, optional
            Whether to show the plot after creation.
        ax : None or matplotlib.Axes, optional
            The ax to update. If None, a new figure will be created.
        color : string, optional
            A string representing a pyplot color.
        plot_min : float, optional
            The smallest value to plot on the y axis.
        plot_max : float, optional
            The biggest value to plot on the y axis.

        Returns
        -------
        ax: plt.Axes
            The Axes containing the results over the steps.
        """

        plots = self._best_result_per_step_dicts(color)
        if self.experiment.minimization_problem:
            legend_loc = 'upper right'
        else:
            legend_loc = 'upper left'

        plot_options = {
            "legend_loc": legend_loc,
            "x_label": "steps",
            "y_label": "result",
            "title": "Plot of %s result over the steps."
                     %(str(self.experiment.name))
        }
        fig, ax = plot_lists(plots, ax=ax, fig_options=plot_options, plot_min=plot_min, plot_max=plot_max)

        if show_plot:
            plt.show(True)

        return ax

    def _best_result_per_step_dicts(self, color="b"):
        """
        Returns a dict to use with plot_utils.

        Parameters
        ----------
        color="b": string
            A pyplot-color representing string. Both plots will have that
            color.

        Returns
        -------
        dicts: list of dicts
            Two dicts, one for step_eval, one for step_best, and their
            corresponding definitions.
        """
        x, step_eval, step_best = self._best_result_per_step_data()

        step_eval_dict = {
            "x": x,
            "y": step_eval,
            "type": "scatter",
            "label": "%s, current result" %(str(self.experiment.name)),
            "color": color
        }

        step_best_dict = {
            "x": x,
            "y": step_best,
            "type": "line",
            "color": color,
            "label": "%s, best result" %(str(self.experiment.name))
        }

        #print [step_eval_dict, step_best_dict]

        return [step_eval_dict, step_best_dict]

    def _best_result_per_step_data(self):
        """
        This internal function returns goodness of the results by step.

        This returns an x coordinate, and for each of them a value for the
        currently evaluated result and the best found result.

        Returns
        -------
        x: list of ints
            The results per step. Should usually be [0, ..., maxSteps]
        step_evaluation: list of floats
            The result of the evaluated candidate during the corresponding step
        step_best: list of floats
            The best result that has been found until then.

        """
        x = []
        step_evaluation = []
        step_best = []
        best_candidate = None
        for i, e in enumerate(self.experiment.candidates_finished):
            x.append(i)
            step_evaluation.append(e.result)
            if self.experiment.better_cand(e, best_candidate):
                best_candidate = e
                step_best.append(e.result)

            else:
                step_best.append(best_candidate.result)
        return x, step_evaluation, step_best

class ParallelExperimentAssistant(PrettyExperimentAssistant, multiprocessing.Process):
    rcv_queue = None

    optimizer_queue = None
    optimizer_process = None

    def __init__(self, name, optimizer, param_defs,
                 experiment=None, optimizer_arguments=None, minimization=True,
                 write_directory_base="/tmp/APSIS_WRITING",
                 experiment_directory_base=None, csv_write_frequency=1,
                 max_time_without_update=None):

        self.rcv_queue = multiprocessing.Queue()

        self._build_new_optimizer()
        super(ParallelExperimentAssistant, self).\
            __init__(name, optimizer, param_defs, experiment=experiment,
                     optimizer_arguments=optimizer_arguments, minimization=minimization,
                     write_directory_base=write_directory_base, experiment_directory_base=experiment_directory_base,
                     csv_write_frequency=csv_write_frequency)

    def run(self):
        while True:
            msg = self.rcv_queue.get(block=True)
            result = getattr(self, "_" + msg["action"])(msg)
            self.return_queue.put(result, block=True)

    def get_next_candidate(self):
        conn_rcv, conn_send = multiprocessing.Pipe(duplex=False)
        msg = {"action": "get_next_candidate",
               "return_pipe": conn_send}
        self.rcv_queue.put(msg)
        next_candidate = conn_rcv.get(block=True)
        conn_rcv.close()
        return next_candidate

    def _get_next_candidate(self, msg):
        self.logger.info("Returning next candidate.")
        next_candidate = None
        while next_candidate is not None:
            if not self.experiment.candidates_pending:
                next_candidate = self.experiment.candidates_pending.pop()
            else:
                if not self.optimizer_queue.empty():
                    next_candidate = self.optimizer_queue.get()
        msg["return_pipe"].put(next_candidate)

    def update(self, candidate, status="finished"):
        msg = {
            "action": "update",
            "candidate": candidate,
            "status": status
        }
        self.rcv_queue.put(msg)

    def _update(self, msg):
        status = msg["status"]
        candidate = msg["candidate"]
        if status not in self.AVAILABLE_STATUS:
            message = ("status not in %s but %s."
                             %(str(self.AVAILABLE_STATUS), str(status)))
            self.logger.error(message)
            raise ValueError(message)

        if not isinstance(candidate, Candidate):
            message = ("candidate %s not a Candidate instance."
                             %str(candidate))
            self.logger.error(message)
            raise ValueError(message)

        self.logger.info("Got new %s of candidate %s with parameters %s"
                         " and result %s" %(status, candidate, candidate.params,
                                            candidate.result))

        if status == "finished":
            self.experiment.add_finished(candidate)
            #Also delete all pending candidates from the experiment - we have
            #new data available.
            self.experiment.candidates_pending = []

            #invoke the writing to files
            step = len(self.experiment.candidates_finished)
            if self.csv_write_frequency != 0 and step != 0 \
                    and step % self.csv_write_frequency == 0:
                self._append_to_detailed_csv()
            #and build a new optimizer.
            # This sends SIGINT to the optimizer process, which is used to
            # terminate the process irrespective of its current computation.
            # (But, since it will be catched, it can still do cleanup)
            os.kill(self.optimizer_process.pid, signal.SIGINT)
            self.optimizer_queue.close()

            # And we rebuild the new optimizer.
            self._build_new_optimizer()

        elif status == "pausing":
            self.experiment.add_pausing(candidate)
        elif status == "working":
            self.experiment.add_working(candidate)

    def _build_new_optimizer(self):
        self.optimizer_queue = multiprocessing.Queue()

        self.optimizer = build_queue_optimizer(self.optimizer, self.experiment,
                            self.optimizer_queue,
                            optimizer_arguments=self.optimizer_arguments)
        self.optimizer.start()

    def get_best_candidate(self):
        conn_rcv, conn_send = multiprocessing.Pipe(duplex=False)
        msg = {
            "action": "get_best_candidate",
            "return_pipe": conn_send
        }
        self.rcv_queue.put(msg)
        best_candidate = conn_rcv.get(block=True)
        conn_rcv.close()
        return best_candidate

    def _get_best_candidate(self, msg):
        msg["return_pipe"].put(self.experiment.best_candidate)<|endoftext|>


<commit_before>#!/usr/local/bin/python -t
# $Id$

"""Gnuplot_test.py -- Exercise the Gnuplot.py module.

Copyright (C) 1998,1999 Michael Haggerty

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at
your option) any later version.  This program is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the GNU General Public License for more details; it is
available at <http://www.fsf.org/copyleft/gpl.html>, or by writing to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.

"""

__cvs_version__ = 'CVS version $Revision$'

import sys, math
import Numeric
from Numeric import *
import Gnuplot
gp = Gnuplot # abbreviation

def wait():
    raw_input('Please press return...\n')

def main():
    """Exercise the Gnuplot module."""

    # Make a temporary file:
    file1 = gp.TempFile() # will be deleted upon exit
    f = open(file1.filename, 'w')
    for x in arange(100)/5. - 10.:
        f.write('%s %s %s\n' % (x, math.cos(x), math.sin(x)))
    f.close()

    g = gp.Gnuplot()

    ############### test Func ########################################
    print 'Plot a gnuplot-generated function'
    g.plot(gp.Func('sin(x)'))
    wait()

    print 'Set title and axis labels and try replot()'
    g.title('Title')
    g.xlabel('x')
    g.ylabel('y')
    g.replot()
    wait()

    print 'Style linespoints'
    g.plot(gp.Func('sin(x)', with='linespoints'))
    wait()
    print 'title=None'
    g.plot(gp.Func('sin(x)', title=None))
    wait()
    print 'title="Sine of x"'
    g.plot(gp.Func('sin(x)', title='Sine of x'))
    wait()

    print 'Change Func attributes after construction:'
    f = gp.Func('sin(x)')
    print 'Original'
    g.plot(f)
    wait()
    print 'Style linespoints'
    f.set_option(with='linespoints')
    g.plot(f)
    wait()
    print 'title=None'
    f.set_option(title=None)
    g.plot(f)
    wait()
    print 'title="Sine of x"'
    f.set_option(title='Sine of x')
    g.plot(f)
    wait()

    ############### test File ########################################
    print 'Generate a File from a filename'
    g.plot(gp.File(file1.filename))
    wait()
    print 'Generate a File given a TempFile object'
    g.plot(gp.File(file1))
    wait()

    print 'Style lines'
    g.plot(gp.File(file1.filename, with='lines'))
    wait()
    print 'using=1, using=(1,)'
    g.plot(gp.File(file1.filename, using=1, with='lines'),
           gp.File(file1.filename, using=(1,), with='points'))
    wait()
    print 'using=(1,2), using="1:3"'
    g.plot(gp.File(file1.filename, using=(1,2)),
           gp.File(file1.filename, using='1:3'))
    wait()
    print 'title=None'
    g.plot(gp.File(file1.filename, title=None))
    wait()
    print 'title="title"'
    g.plot(gp.File(file1.filename, title='title'))
    wait()

    print 'Change File attributes after construction:'
    f = gp.File(file1.filename)
    print 'Original'
    g.plot(f)
    wait()
    print 'Style linespoints'
    f.set_option(with='linespoints')
    g.plot(f)
    wait()
    print 'using=(1,3)'
    f.set_option(using=(1,3))
    g.plot(f)
    wait()
    print 'title=None'
    f.set_option(title=None)
    g.plot(f)
    wait()

    ############### test Data ########################################
    x = arange(100)/5. - 10.
    y1 = Numeric.cos(x)
    y2 = Numeric.sin(x)
    d = Numeric.array((x,y1,y2))

    print 'Plot Data, specified column-by-column'
    g.plot(gp.Data(x,y1))
    wait()
    print 'Plot Data, specified by an array'
    g.plot(gp.Data(d))
    wait()
    print 'with="lp 4 4"'
    g.plot(gp.Data(d, with='lp 4 4'))
    wait()
    print 'cols=0'
    g.plot(gp.Data(d, cols=0))
    wait()
    print 'cols=(0,1), cols=(0,2)'
    g.plot(gp.Data(d, cols=(0,1)),
           gp.Data(d, cols=(0,2)))
    wait()
    print 'title=None'
    g.plot(gp.Data(d, title=None))
    wait()
    print 'title="Cosine of x"'
    g.plot(gp.Data(d, title='Cosine of x'))
    wait()

    ############### test shortcuts ###################################
    print 'plot Func and Data using shortcuts'
    g.plot('sin(x)', d)
    wait()

    ############### test GridData ####################################
    # set up x and y values at which the function will be tabulated:
    x = arange(35)/2.0
    y = arange(30)/10.0 - 1.5
    # Make a 2-d array containing a function of x and y.  First create
    # xm and ym which contain the x and y values in a matrix form that
    # can be `broadcast' into a matrix of the appropriate shape:
    xm = x[:,NewAxis]
    ym = y[NewAxis,:]
    m = (sin(xm) + 0.1*xm) - ym**2
    g('set parametric')
    g('set data style lines')
    g('set hidden')
    g('set contour base')
    g.xlabel('x')
    g.ylabel('y')
    # The `binary=1' option would cause communication with gnuplot to
    # be in binary format, which is considerably faster and uses less
    # disk space.  (This only works with the splot command due to
    # limitations of gnuplot.)  `binary=1' is the default, but here we
    # disable binary because older versions of gnuplot don't allow
    # binary data.  Change this to `binary=1' (or omit the binary
    # option) to get the advantage of binary format.
    g.splot(gp.GridData(m,x,y, binary=0))

    ############### test HardCopy ####################################
    g.ylabel('x^2') # take advantage of enhanced postscript mode
    print ('\n******** Generating postscript file '
           '"gnuplot_test_plot.ps" ********\n')
    g.hardcopy('gnuplot_test_plot.ps', enhanced=1, color=1)

if __name__ == '__main__':
    main()

<commit_msg>Moved wait()s.<commit_after>#!/usr/local/bin/python -t
# $Id$

"""Gnuplot_test.py -- Exercise the Gnuplot.py module.

Copyright (C) 1998,1999 Michael Haggerty

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at
your option) any later version.  This program is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the GNU General Public License for more details; it is
available at <http://www.fsf.org/copyleft/gpl.html>, or by writing to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.

"""

__cvs_version__ = 'CVS version $Revision$'

import sys, math
import Numeric
from Numeric import *
import Gnuplot
gp = Gnuplot # abbreviation

def wait():
    raw_input('Press return to show results...\n')

def main():
    """Exercise the Gnuplot module."""

    # Make a temporary file:
    file1 = gp.TempFile() # will be deleted upon exit
    f = open(file1.filename, 'w')
    for x in arange(100)/5. - 10.:
        f.write('%s %s %s\n' % (x, math.cos(x), math.sin(x)))
    f.close()

    g = gp.Gnuplot()

    ############### test Func ########################################
    print 'Plot a gnuplot-generated function'
    wait()
    g.plot(gp.Func('sin(x)'))

    print 'Set title and axis labels and try replot()'
    wait()
    g.title('Title')
    g.xlabel('x')
    g.ylabel('y')
    g.replot()

    print 'Style linespoints'
    wait()
    g.plot(gp.Func('sin(x)', with='linespoints'))
    print 'title=None'
    wait()
    g.plot(gp.Func('sin(x)', title=None))
    print 'title="Sine of x"'
    wait()
    g.plot(gp.Func('sin(x)', title='Sine of x'))

    print 'Change Func attributes after construction:'
    f = gp.Func('sin(x)')
    print 'Original'
    wait()
    g.plot(f)
    print 'Style linespoints'
    wait()
    f.set_option(with='linespoints')
    g.plot(f)
    print 'title=None'
    wait()
    f.set_option(title=None)
    g.plot(f)
    print 'title="Sine of x"'
    wait()
    f.set_option(title='Sine of x')
    g.plot(f)

    ############### test File ########################################
    print 'Generate a File from a filename'
    wait()
    g.plot(gp.File(file1.filename))
    print 'Generate a File given a TempFile object'
    wait()
    g.plot(gp.File(file1))

    print 'Style lines'
    wait()
    g.plot(gp.File(file1.filename, with='lines'))
    print 'using=1, using=(1,)'
    wait()
    g.plot(gp.File(file1.filename, using=1, with='lines'),
           gp.File(file1.filename, using=(1,), with='points'))
    print 'using=(1,2), using="1:3"'
    wait()
    g.plot(gp.File(file1.filename, using=(1,2)),
           gp.File(file1.filename, using='1:3'))
    print 'title=None'
    wait()
    g.plot(gp.File(file1.filename, title=None))
    print 'title="title"'
    wait()
    g.plot(gp.File(file1.filename, title='title'))

    print 'Change File attributes after construction:'
    f = gp.File(file1.filename)
    print 'Original'
    wait()
    g.plot(f)
    print 'Style linespoints'
    wait()
    f.set_option(with='linespoints')
    g.plot(f)
    print 'using=(1,3)'
    wait()
    f.set_option(using=(1,3))
    g.plot(f)
    print 'title=None'
    wait()
    f.set_option(title=None)
    g.plot(f)

    ############### test Data ########################################
    x = arange(100)/5. - 10.
    y1 = Numeric.cos(x)
    y2 = Numeric.sin(x)
    d = Numeric.array((x,y1,y2))

    print 'Plot Data, specified column-by-column'
    wait()
    g.plot(gp.Data(x,y1))
    print 'Plot Data, specified by an array'
    wait()
    g.plot(gp.Data(d))
    print 'with="lp 4 4"'
    wait()
    g.plot(gp.Data(d, with='lp 4 4'))
    print 'cols=0'
    wait()
    g.plot(gp.Data(d, cols=0))
    print 'cols=(0,1), cols=(0,2)'
    wait()
    g.plot(gp.Data(d, cols=(0,1)),
           gp.Data(d, cols=(0,2)))
    print 'title=None'
    wait()
    g.plot(gp.Data(d, title=None))
    print 'title="Cosine of x"'
    wait()
    g.plot(gp.Data(d, title='Cosine of x'))

    ############### test shortcuts ###################################
    print 'plot Func and Data using shortcuts'
    wait()
    g.plot('sin(x)', d)

    ############### test GridData ####################################
    # set up x and y values at which the function will be tabulated:
    x = arange(35)/2.0
    y = arange(30)/10.0 - 1.5
    # Make a 2-d array containing a function of x and y.  First create
    # xm and ym which contain the x and y values in a matrix form that
    # can be `broadcast' into a matrix of the appropriate shape:
    xm = x[:,NewAxis]
    ym = y[NewAxis,:]
    m = (sin(xm) + 0.1*xm) - ym**2
    g('set parametric')
    g('set data style lines')
    g('set hidden')
    g('set contour base')
    g.xlabel('x')
    g.ylabel('y')
    # The `binary=1' option would cause communication with gnuplot to
    # be in binary format, which is considerably faster and uses less
    # disk space.  (This only works with the splot command due to
    # limitations of gnuplot.)  `binary=1' is the default, but here we
    # disable binary because older versions of gnuplot don't allow
    # binary data.  Change this to `binary=1' (or omit the binary
    # option) to get the advantage of binary format.
    g.splot(gp.GridData(m,x,y, binary=0))

    ############### test HardCopy ####################################
    g.ylabel('x^2') # take advantage of enhanced postscript mode
    print ('\n******** Generating postscript file '
           '"gnuplot_test_plot.ps" ********\n')
    g.hardcopy('gnuplot_test_plot.ps', enhanced=1, color=1)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import os
import itertools
import networkx
from networkx.readwrite import json_graph
import json
import codecs
from config import CONFIG

def add_edge_weight(graph, node1, node2):
    if graph.has_edge(node1, node2):
        graph[node1][node2]['weight'] += 1
    else:
        graph.add_edge(node1, node2, weight=1)

for span in sorted(CONFIG["spans"]):
	print "\n#%s"%span
	with codecs.open(os.path.join(CONFIG["parsed_data"],span,"references.dat"),"r",encoding="UTF-8") as file:
		# dat file have one trailing blank line at end of file
		data_lines=file.read().split("\n")[:-1]

	references_by_articles = [(l.split("\t")[0],",".join(l.split("\t")[1:])) for l in data_lines]
	print "%s ref occurences in articles"%len(references_by_articles)
	references_by_articles.sort(key=lambda e:e[1])
	del(data_lines)
	# filter ref which occurs un only one article
	references_article_grouped=[(reference,list(ref_arts)) for reference,ref_arts in itertools.groupby(references_by_articles,key=lambda e:e[1])]
	print "filtering references"
	references_by_articles_filtered = [t for _ in (ref_arts for ref,ref_arts in references_article_grouped if len(ref_arts)>=CONFIG["spans"][span]["references"]["occ"]) for t in _]

	#print "filtering reference occ at minimum %s results in removing %s on %s %03.1f%%"%(spans[span]["ref_occ_min"],len(ref_to_trash),len(references_article_grouped),100*float(len(ref_to_trash))/(len(ref_to_keep)+len(ref_to_trash)))
	#group by articles and create ref network

	# sort on article
	print "sorting references"
	references_by_articles_filtered.sort(key=lambda e:e[0])
	# group by article
	print "processing edges for references... (go for a tea)"
	g=networkx.Graph()
	for article,art_refs in itertools.groupby(references_by_articles_filtered,key=lambda e:e[0]):
		#one link between ref cited by same article
		for r1,r2 in itertools.combinations((r for a,r in art_refs),2):
			add_edge_weight(g,r1,r2)
	print "remove edges with weight < %s"%CONFIG["spans"][span]["references"]["weight"]
	g.remove_edges_from((r1,r2) for (r1,r2,d) in g.edges(data=True) if d['weight'] <CONFIG["spans"][span]["references"]["weight"])
	print "remove nodes with degree = 0"
	g.remove_nodes_from(r for (r,d) in g.degree_iter() if d <1)
	networkx.set_node_attributes(g, 'type', "reference")

	if CONFIG["export_ref_format"] =="gexf":
		print "write gexf export"
		networkx.write_gexf(g,os.path.join(CONFIG["parsed_data"],span,"%s.gexf"%span),encoding="UTF-8")
	elif CONFIG["export_ref_format"] == "edgelist":
		print "write csv export"
		networkx.write_weighted_edgelist(g,os.path.join(CONFIG["parsed_data"],span,"%s.csv"%span),delimiter="\t")
	elif CONFIG["export_ref_format"] == "pajek":
		print "write pajek export"
		networkx.write_pajek(g,os.path.join(CONFIG["parsed_data"],span,"%s.net"%span),encoding='UTF-8')
	elif CONFIG["export_ref_format"] == "json":
		print "write json export"
		data = json_graph.node_link_data(g)
		json.dump(data,open(os.path.join(CONFIG["parsed_data"],span,"%s.json"%span),"w"),encoding='UTF-8')
	else:
		print  "no export compatible export format specified"

<commit_msg>add occurence_count as node attr<commit_after>import os
import itertools
import networkx
from networkx.readwrite import json_graph
import json
import codecs
from config import CONFIG

def add_edge_weight(graph, node1, node2):
    if graph.has_edge(node1, node2):
        graph[node1][node2]['weight'] += 1
    else:
        graph.add_edge(node1, node2, weight=1)

for span in sorted(CONFIG["spans"]):
	print "\n#%s"%span
	with codecs.open(os.path.join(CONFIG["parsed_data"],span,"references.dat"),"r",encoding="UTF-8") as file:
		# dat file have one trailing blank line at end of file
		data_lines=file.read().split("\n")[:-1]

	references_by_articles = [(l.split("\t")[0],",".join(l.split("\t")[1:])) for l in data_lines]
	print "%s ref occurences in articles"%len(references_by_articles)
	references_by_articles.sort(key=lambda e:e[1])
	del(data_lines)
	# filter ref which occurs un only one article
	references_article_grouped=[(reference,list(ref_arts)) for reference,ref_arts in itertools.groupby(references_by_articles,key=lambda e:e[1])]
	references_occs=dict([(reference,len(list(ref_arts))) for reference,ref_arts in references_article_grouped if len(ref_arts)>=CONFIG["spans"][span]["references"]["occ"]])
	print "filtering references"
	references_by_articles_filtered = [t for _ in (ref_arts for ref,ref_arts in references_article_grouped if len(ref_arts)>=CONFIG["spans"][span]["references"]["occ"]) for t in _]

	#print "filtering reference occ at minimum %s results in removing %s on %s %03.1f%%"%(spans[span]["ref_occ_min"],len(ref_to_trash),len(references_article_grouped),100*float(len(ref_to_trash))/(len(ref_to_keep)+len(ref_to_trash)))
	#group by articles and create ref network

	# sort on article
	print "sorting references"
	references_by_articles_filtered.sort(key=lambda e:e[0])
	# group by article
	print "processing edges for references... (go for a tea)"
	g=networkx.Graph()
	for article,art_refs in itertools.groupby(references_by_articles_filtered,key=lambda e:e[0]):
		#one link between ref cited by same article
		for r1,r2 in itertools.combinations((r for a,r in art_refs),2):
			g.add_node(r1,type="references",occurence_count=references_occs[r1])
			g.add_node(r2,type="references",occurence_count=references_occs[r2])
			add_edge_weight(g,r1,r2)
	print "remove edges with weight < %s"%CONFIG["spans"][span]["references"]["weight"]
	g.remove_edges_from((r1,r2) for (r1,r2,d) in g.edges(data=True) if d['weight'] <CONFIG["spans"][span]["references"]["weight"])
	print "remove nodes with degree = 0"
	g.remove_nodes_from(r for (r,d) in g.degree_iter() if d <1)
	networkx.set_node_attributes(g, 'type', "reference")

	if CONFIG["export_ref_format"] =="gexf":
		print "write gexf export"
		networkx.write_gexf(g,os.path.join(CONFIG["parsed_data"],span,"%s.gexf"%span),encoding="UTF-8")
	elif CONFIG["export_ref_format"] == "edgelist":
		print "write csv export"
		networkx.write_weighted_edgelist(g,os.path.join(CONFIG["parsed_data"],span,"%s.csv"%span),delimiter="\t")
	elif CONFIG["export_ref_format"] == "pajek":
		print "write pajek export"
		networkx.write_pajek(g,os.path.join(CONFIG["parsed_data"],span,"%s.net"%span),encoding='UTF-8')
	elif CONFIG["export_ref_format"] == "json":
		print "write json export"
		data = json_graph.node_link_data(g)
		json.dump(data,open(os.path.join(CONFIG["parsed_data"],span,"%s.json"%span),"w"),encoding='UTF-8')
	else:
		print  "no export compatible export format specified"

<|endoftext|>


<commit_before>import logging
import sys
import django
from django.db import transaction
django.setup()

from osf.models import Node
from scripts import utils as script_utils
from website import settings
from website.app import init_app

from keen import scoped_keys

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def valid_keen_key(keenio_read_key, node_id):
    try:
        ret = scoped_keys.decrypt(settings.KEEN['public']['master_key'], keenio_read_key)
        return ret['filters'][0]['property_value'] == node_id
    except Exception as error:
        logger.info('Error on {}:'.format(node_id))
        logger.exception(error)
        return True

def update():
    queryset = Node.objects.filter(preprints__isnull=False, is_public=True, is_deleted=False)
    print('Checking {} preprint nodes'.format(queryset.count()))
    count = 0
    updated_nodes = []
    for node in queryset:
        if not valid_keen_key(node.keenio_read_key, node._id):
            count += 1
            node.keenio_read_key = node.generate_keenio_read_key()
            node.save()
            updated_nodes.append(node._id)
    print('Updated {} node keen key(s), subset for QA = {}'.format(count, str(updated_nodes[0:10])))

def main():
    dry_run = '--dry' in sys.argv
    if not dry_run:
        script_utils.add_file_logger(logger, __file__)
    init_app(set_backends=True, routes=False)
    with transaction.atomic():
        update()
        if dry_run:
            raise RuntimeError('Dry mode -- rolling back transaction')

if __name__ == '__main__':
    main()
<commit_msg>Update exception handling; Add note<commit_after>import logging
import sys
import django
from django.db import transaction
django.setup()

from osf.models import Node
from scripts import utils as script_utils
from website import settings
from website.app import init_app

from keen import scoped_keys

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def valid_keen_key(keenio_read_key, node_id):
    try:
        ret = scoped_keys.decrypt(settings.KEEN['public']['master_key'], keenio_read_key)
        return ret['filters'][0]['property_value'] == node_id
    except Exception as error:
        logger.exception('Error on {}:'.format(node_id))
        # Returns True to bypass updating of the key -- failed nodes should be investigated separately
        return True

def update():
    queryset = Node.objects.filter(preprints__isnull=False, is_public=True, is_deleted=False)
    print('Checking {} preprint nodes'.format(queryset.count()))
    count = 0
    updated_nodes = []
    for node in queryset:
        if not valid_keen_key(node.keenio_read_key, node._id):
            count += 1
            node.keenio_read_key = node.generate_keenio_read_key()
            node.save()
            updated_nodes.append(node._id)
    print('Updated {} node keen key(s), subset for QA = {}'.format(count, str(updated_nodes[0:10])))

def main():
    dry_run = '--dry' in sys.argv
    if not dry_run:
        script_utils.add_file_logger(logger, __file__)
    init_app(set_backends=True, routes=False)
    with transaction.atomic():
        update()
        if dry_run:
            raise RuntimeError('Dry mode -- rolling back transaction')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Copyright 2014-2016 University of Illinois

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

file: userProfiles/models.py

Author: Jon Gunderson

"""

from __future__                 import absolute_import
from django.db                  import models
from django.contrib.auth.models import User
from registration.signals       import user_registered
from timezone_field             import TimeZoneField

from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.template.loader   import render_to_string

from django.db.models import Q

from django.contrib import messages

import markdown

from accounts.models            import AccountType
from subscriptions.models       import Payment
from websiteResultGroups.models import WebsiteReportGroup
from reports.models             import WebsiteReport
from stats.models               import StatsUser
from django.contrib.sites.models import Site

from contact.models              import Announcement

from fae2.settings import DEFAULT_ACCOUNT_TYPE
from fae2.settings import SHIBBOLETH_ENABLED

from django.core.mail import send_mail
from fae2.settings import EMAIL_HOST_USER
from fae2.settings import ADMIN_EMAIL

from datetime import datetime

from datetime import date

def get_profile(user):
    try:
        profile = UserProfile.objects.get(user=user)
    except:
        atype = AccountType.objects.get(type_id=DEFAULT_ACCOUNT_TYPE)
        profile = UserProfile(user=user, account_type=atype)
        profile.save()

        # check for shibboleth user

    try:
        stats = StatsUser.objects.get(user=user)
    except ObjectDoesNotExist:
        wsrg =  WebsiteReportGroup(title="Summary of results for " + str(user))
        wsrg.save()
        stats = StatsUser(user=user, ws_report_group=wsrg)
        stats.save()

    return profile

SUBSCRIPTION_STATUS_CHOICES = (
    ('FREE',    'Free'),
    ('CURRENT',  'Current'),
    ('EXPIRED',  'Expired'),
    ('SPECIAL',  'Special')
)

class UserProfile(models.Model):

    user          = models.OneToOneField(User, related_name="profile")

    account_type             = models.ForeignKey(AccountType, related_name="user_profiles")
    enable_any_account_types = models.BooleanField(default=False)

    subscription_start      = models.DateField(null=True, blank=True)
    subscription_end        = models.DateField(null=True, blank=True)
    subscription_payments   = models.IntegerField(default=0) # in dollars
    subscription_daily_rate = models.IntegerField(default=0) # in cents
    subscription_status     = models.CharField(max_length=8, choices=SUBSCRIPTION_STATUS_CHOICES, default="FREE")
    subscription_days       = models.IntegerField(default=0)

    top_level_domain = models.CharField(max_length=8, null=True, blank=True)
    domain           = models.CharField(max_length=64, null=True, blank=True)

    org                 = models.CharField(max_length=128, blank=True)
    dept                = models.CharField(max_length=128, blank=True)
    email_announcements = models.BooleanField(default=True)

    timezone = TimeZoneField(default='America/Chicago')

    def __unicode__(self):
        return self.user.username

    def set_domain_info(self):
        # If shibboleth headers did not find an e-mail address, populate with the username
        if not len(self.user.email):
            if self.user.username.find('@') > 0 and self.user.username.find('.') > 0:
                self.user.email = self.user.username
                self.user.save()

        email = self.user.email

        if email:
            email = email.strip()
            parts = email.split('.')
            l = len(parts)
            if l > 1:
                top    = parts[l-1]
                parts  = parts[l-2].split('@')
                if len(parts) > 1:
                  domain = parts[len(parts)-1]
                else:
                  domain = parts[0]

                if len(top) < 8 and len(domain) < 64:
                  self.top_level_domain = top
                  self.domain = domain
                  self.save()

    def set_payments(self, amount):
        self.subscription_payments = amount
        self.update_daily_rate()

    def add_payment(self, amount):
        self.subscription_payments += amount
        self.update_daily_rate()

    def subtract_payment(self, amount):
        self.subscription_payments -= amount

        if self.subscription_payments < 0:
            self.subscription_payments = 0

        self.update_daily_rate()

    def update_subscription_status(self):
        self.enable_any_account_types = True

        if self.subscription_end:
            self.enable_any_account_types = False

            date1 = date(self.subscription_end.year, self.subscription_end.month, self.subscription_end.day)
            date2 = date.today()
            delta = date1 - date2

            self.subscription_days = delta.days

            if self.subscription_days >= 0:
                self.subscription_status = 'CURRENT'
            else:
                self.subscription_status     = 'EXPIRED'
                self.subscription_payments   = 0
                self.subscription_daily_rate = 0
                self.account_type            = AccountType.objects.get(type_id=1)

            if self.subscription_days < 7:
                self.enable_any_account_types = True

        self.save()

        self.update_daily_rate()

        return self.subscription_status

    def update_institutional_subscription(self):

        if not self.domain:
            self.set_domain_info()

        if not SHIBBOLETH_ENABLED:
            return False

        try:
            ip = InstitutionalProfile.objects.get(top_level_domain=self.top_level_domain, domain=self.domain)
        except:
            try:
                ip = InstitutionalProfile.objects.get(top_level_domain=self.top_level_domain, alt_domain=self.domain)
            except:
                ip = False

        if ip:

            ip.users.add(self.user)
            ip.save()

        info(str(self) + " added to " + str(ip))

        if ip.account_type.shibboleth:

          self.subscription_status  = ip.subscription_status
          self.subscription_end     = ip.subscription_end
          self.subscription_start   = ip.subscription_start
          self.subscription_days    = ip.subscription_days

          if ip.subscription_status == 'CURRENT':
              self.account_type = ip.account_type

          if ip.subscription_status == 'EXPIRED':
              self.account_type = ip_free

          self.save()

        return ip

    def get_last_subscription(self):
        try:
            payments = Payment.objects.filter(Q(status='PMT_APPROV') | Q(status='PMT_NOCOST'), user=self.user).latest('reference_time')
        except:
            payments = False

        if payments:
            return payments

        return payments

    def check_for_subscription_messages(self, request):

        subscription_url = reverse('update_subscription')

        self.update_subscription_status()

        if self.subscription_status == 'CURRENT' and self.subscription_days < 8:
            messages.warning(request, render_to_string('accounts/subscription_current.txt', {'url': subscription_url, 'days': self.subscription_days}))
        elif self.subscription_status == 'EXPIRED' and self.subscription_days == -1:
            messages.warning(request, render_to_string('accounts/subscription_expired.txt', {'url': subscription_url, 'days': self.subscription_days}))

    def check_for_email_subscription_notifications(self):

        topic = ""

        if self.subscription_status == 'CURRENT':
            if self.subscription_days == 7:
                topic = "FAE subscription expires in " + str(self.subscription_days) + "days"
            elif self.subscription_days == 1:
                topic = "FAE subscription expires in one day"
            elif self.subscription_days == 0:
                topic = "FAE subscription expires today"

        if self.subscription_status == 'EXPIRED' and self.subscription_days == -1:
            topic = "FAE subscription expired yesterday"

        if topic:
            site = Site.objects.get_current()
            message = render_to_string('accounts/email_message.txt', {'user_profile': self, 'subscription_url': str(site) + reverse('update_subscription') })
            send_mail(topic, message, EMAIL_HOST_USER, [self.user.email], fail_silently=False)

        return

    def update_daily_rate(self):
        self.subscription_daily_rate = 0;

        if self.subscription_payments > 0 and self.subscription_end and self.subscription_start:
            date1 = date(self.subscription_end.year, self.subscription_end.month, self.subscription_end.day)
            date2 = date(self.subscription_start.year, self.subscription_start.month, self.subscription_start.day)
            delta = date1 - date2

            if delta.days > 0:
              self.subscription_daily_rate = (100 * self.subscription_payments) / delta.days

        self.save()

    def get_active_reports(self):

        user_reports = WebsiteReport.objects.filter(user=self.user).filter(status='C')
        reports = user_reports[0:self.account_type.max_archive]
        old_reports  = user_reports[self.account_type.max_archive:]

        return [reports, old_reports]

# creates new UserProfile when new user registers
def user_registered_callback(sender, user, request, **kwargs):

    profile = UserProfile(user = user)
    profile.account_type = AccountType.objects.get(type_id=DEFAULT_ACCOUNT_TYPE)
    profile.org = ''
    profile.save()

    # Update first and last name for user
    user.first_name = request.POST['first_name']
    user.last_name = request.POST['last_name']
    user.save()

    wsrg =  WebsiteReportGroup(title="Summary of results for " + str(user))
    wsrg.save()
    user_stats = StatsUser(user=user, ws_report_group=wsrg)
    user_stats.save()

user_registered.connect(user_registered_callback)

class InstitutionalProfile(models.Model):
    id = models.AutoField(primary_key=True)

    title           = models.CharField(max_length=64,  blank=True, default="")
    contact1_name   = models.CharField(max_length=32,  blank=True, default="")
    contact1_title  = models.CharField(max_length=32,  blank=True, default="")
    contact1_email  = models.EmailField(max_length=64, blank=True, default="")
    contact1_phone  = models.CharField(max_length=16,  blank=True, default="")

    contact2_name   = models.CharField(max_length=32,  blank=True, default="")
    contact2_title  = models.CharField(max_length=32,  blank=True, default="")
    contact2_email  = models.EmailField(max_length=64, blank=True, default="")
    contact2_phone  = models.CharField(max_length=16,  blank=True, default="")

    account_type     = models.ForeignKey(AccountType, related_name="institional_subscriptions")
    top_level_domain = models.CharField(max_length=8, blank=True, default="")
    domain           = models.CharField(max_length=64, blank=True, default="")
    alt_domain       = models.CharField(max_length=64, blank=True, default="")
    authentication   = models.CharField(max_length=64, blank=True, default="")

    subscription_status   = models.CharField(max_length=8, choices=SUBSCRIPTION_STATUS_CHOICES, default="FREE")
    subscription_days       = models.IntegerField(default=0)

    subscription_start    = models.DateField(null=True, blank=True)
    subscription_end      = models.DateField(null=True, blank=True)
    subscription_payment  = models.IntegerField(default=0) # in dollars
    last_payment          = models.IntegerField(default=0) # in dollars

    users = models.ManyToManyField(User, related_name="institional_subscriptions", blank=True, default=None)

    class Meta:
        verbose_name        = "Institutional Profile"
        verbose_name_plural = "Institutional Profiles"
        ordering = ['account_type']

    def __str__(self):
        return self.domain + '.' + self.top_level_domain

    def update_subscription_status(self):

        if self.subscription_end:
            date1 = date(self.subscription_end.year, self.subscription_end.month, self.subscription_end.day)
            date2 = date.today()
            delta = date1 - date2

            self.subscription_days = delta.days

            if self.subscription_days >= 0:
                self.subscription_status = 'CURRENT'
            else:
                self.subscription_status     = 'EXPIRED'
                self.subscription_payments   = 0
                self.account_type            = AccountType.objects.get(type_id=16)

        self.save()

    def check_for_email_subscription_notifications(self):

        topic = ""

        if self.subscription_status == 'CURRENT':
            if self.subscription_days == 7 or self.subscription_days == 14:
                topic = "FAE Institutional Subscription expires in " + str(self.subscription_days) + "days"
            elif self.subscription_days == 1:
                topic = "FAE Institutional Subscription expires in one day"
            elif self.subscription_days == 0:
                topic = "FAE Institutional Subscription expires today"

        if self.subscription_status == 'EXPIRED' and self.subscription_days == -1:
            topic = "FAE Institutional Subscription expired yesterday"

        if topic:
            emails = []
            if self.contact1_email:
                emails.append(self.contact1_email)
            if self.contact2_email:
                emails.append(self.contact2_email)

            site = Site.objects.get_current()
            message = render_to_string('accounts/email_institutional_message.txt', {'institutional_profile': self, 'subscription_url': str(site) + reverse('contact_form') })
            send_mail(topic, message, EMAIL_HOST_USER, emails, fail_silently=False)

<commit_msg>updating initialization and updating of inst subscriptions<commit_after>"""
Copyright 2014-2016 University of Illinois

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

file: userProfiles/models.py

Author: Jon Gunderson

"""

from __future__                 import absolute_import
from django.db                  import models
from django.contrib.auth.models import User
from registration.signals       import user_registered
from timezone_field             import TimeZoneField

from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.template.loader   import render_to_string

from django.db.models import Q

from django.contrib import messages

import markdown

from accounts.models            import AccountType
from subscriptions.models       import Payment
from websiteResultGroups.models import WebsiteReportGroup
from reports.models             import WebsiteReport
from stats.models               import StatsUser
from django.contrib.sites.models import Site

from contact.models              import Announcement

from fae2.settings import DEFAULT_ACCOUNT_TYPE
from fae2.settings import SHIBBOLETH_ENABLED

from django.core.mail import send_mail
from fae2.settings import EMAIL_HOST_USER
from fae2.settings import ADMIN_EMAIL

from datetime import datetime

from datetime import date

def get_profile(user):
    try:
        profile = UserProfile.objects.get(user=user)
    except:
        atype = AccountType.objects.get(type_id=DEFAULT_ACCOUNT_TYPE)
        profile = UserProfile(user=user, account_type=atype)
        profile.save()

        # check for shibboleth user

    try:
        stats = StatsUser.objects.get(user=user)
    except ObjectDoesNotExist:
        wsrg =  WebsiteReportGroup(title="Summary of results for " + str(user))
        wsrg.save()
        stats = StatsUser(user=user, ws_report_group=wsrg)
        stats.save()

    return profile

SUBSCRIPTION_STATUS_CHOICES = (
    ('FREE',    'Free'),
    ('CURRENT',  'Current'),
    ('EXPIRED',  'Expired'),
    ('SPECIAL',  'Special')
)

class UserProfile(models.Model):

    user          = models.OneToOneField(User, related_name="profile")

    account_type             = models.ForeignKey(AccountType, related_name="user_profiles")
    enable_any_account_types = models.BooleanField(default=False)

    subscription_start      = models.DateField(null=True, blank=True)
    subscription_end        = models.DateField(null=True, blank=True)
    subscription_payments   = models.IntegerField(default=0) # in dollars
    subscription_daily_rate = models.IntegerField(default=0) # in cents
    subscription_status     = models.CharField(max_length=8, choices=SUBSCRIPTION_STATUS_CHOICES, default="FREE")
    subscription_days       = models.IntegerField(default=0)

    top_level_domain = models.CharField(max_length=8, null=True, blank=True)
    domain           = models.CharField(max_length=64, null=True, blank=True)

    org                 = models.CharField(max_length=128, blank=True)
    dept                = models.CharField(max_length=128, blank=True)
    email_announcements = models.BooleanField(default=True)

    timezone = TimeZoneField(default='America/Chicago')

    def __unicode__(self):
        return self.user.username

    def set_domain_info(self):
        # If shibboleth headers did not find an e-mail address, populate with the username
        if not len(self.user.email):
            if self.user.username.find('@') > 0 and self.user.username.find('.') > 0:
                self.user.email = self.user.username
                self.user.save()

        email = self.user.email

        if email:
            email = email.strip()
            parts = email.split('.')
            l = len(parts)
            if l > 1:
                top    = parts[l-1]
                parts  = parts[l-2].split('@')
                if len(parts) > 1:
                  domain = parts[len(parts)-1]
                else:
                  domain = parts[0]

                if len(top) < 8 and len(domain) < 64:
                  self.top_level_domain = top
                  self.domain = domain
                  self.save()

    def set_payments(self, amount):
        self.subscription_payments = amount
        self.update_daily_rate()

    def add_payment(self, amount):
        self.subscription_payments += amount
        self.update_daily_rate()

    def subtract_payment(self, amount):
        self.subscription_payments -= amount

        if self.subscription_payments < 0:
            self.subscription_payments = 0

        self.update_daily_rate()

    def update_subscription_status(self):
        self.enable_any_account_types = True

        if self.subscription_end:
            self.enable_any_account_types = False

            date1 = date(self.subscription_end.year, self.subscription_end.month, self.subscription_end.day)
            date2 = date.today()
            delta = date1 - date2

            self.subscription_days = delta.days

            if self.subscription_days >= 0:
                self.subscription_status = 'CURRENT'
            else:
                self.subscription_status     = 'EXPIRED'
                self.subscription_payments   = 0
                self.subscription_daily_rate = 0
                self.account_type            = AccountType.objects.get(type_id=1)

            if self.subscription_days < 7:
                self.enable_any_account_types = True

        self.save()

        self.update_daily_rate()

        return self.subscription_status

    def update_institutional_subscription(self):

        if not self.domain:
            self.set_domain_info()

        if not SHIBBOLETH_ENABLED:
            return False

        try:
            ip = InstitutionalProfile.objects.get(top_level_domain=self.top_level_domain, domain=self.domain)
        except:
            try:
                ip = InstitutionalProfile.objects.get(top_level_domain=self.top_level_domain, alt_domain=self.domain)
            except:
                ip = False

        if ip:
            ip.users.add(self.user)
            ip.save()

        if ip.account_type.shibboleth:

          self.subscription_status  = ip.subscription_status
          self.subscription_end     = ip.subscription_end
          self.subscription_start   = ip.subscription_start
          self.subscription_days    = ip.subscription_days

          if ip.subscription_status == 'CURRENT':
              self.account_type = ip.account_type

          if ip.subscription_status == 'EXPIRED':
              self.account_type = ip_free

          self.save()

        return ip

    def get_last_subscription(self):
        try:
            payments = Payment.objects.filter(Q(status='PMT_APPROV') | Q(status='PMT_NOCOST'), user=self.user).latest('reference_time')
        except:
            payments = False

        if payments:
            return payments

        return payments

    def check_for_subscription_messages(self, request):

        subscription_url = reverse('update_subscription')

        self.update_subscription_status()

        if self.subscription_status == 'CURRENT' and self.subscription_days < 8:
            messages.warning(request, render_to_string('accounts/subscription_current.txt', {'url': subscription_url, 'days': self.subscription_days}))
        elif self.subscription_status == 'EXPIRED' and self.subscription_days == -1:
            messages.warning(request, render_to_string('accounts/subscription_expired.txt', {'url': subscription_url, 'days': self.subscription_days}))

    def check_for_email_subscription_notifications(self):

        topic = ""

        if self.subscription_status == 'CURRENT':
            if self.subscription_days == 7:
                topic = "FAE subscription expires in " + str(self.subscription_days) + "days"
            elif self.subscription_days == 1:
                topic = "FAE subscription expires in one day"
            elif self.subscription_days == 0:
                topic = "FAE subscription expires today"

        if self.subscription_status == 'EXPIRED' and self.subscription_days == -1:
            topic = "FAE subscription expired yesterday"

        if topic:
            site = Site.objects.get_current()
            message = render_to_string('accounts/email_message.txt', {'user_profile': self, 'subscription_url': str(site) + reverse('update_subscription') })
            send_mail(topic, message, EMAIL_HOST_USER, [self.user.email], fail_silently=False)

        return

    def update_daily_rate(self):
        self.subscription_daily_rate = 0;

        if self.subscription_payments > 0 and self.subscription_end and self.subscription_start:
            date1 = date(self.subscription_end.year, self.subscription_end.month, self.subscription_end.day)
            date2 = date(self.subscription_start.year, self.subscription_start.month, self.subscription_start.day)
            delta = date1 - date2

            if delta.days > 0:
              self.subscription_daily_rate = (100 * self.subscription_payments) / delta.days

        self.save()

    def get_active_reports(self):

        user_reports = WebsiteReport.objects.filter(user=self.user).filter(status='C')
        reports = user_reports[0:self.account_type.max_archive]
        old_reports  = user_reports[self.account_type.max_archive:]

        return [reports, old_reports]

# creates new UserProfile when new user registers
def user_registered_callback(sender, user, request, **kwargs):

    profile = UserProfile(user = user)
    profile.account_type = AccountType.objects.get(type_id=DEFAULT_ACCOUNT_TYPE)
    profile.org = ''
    profile.save()

    # Update first and last name for user
    user.first_name = request.POST['first_name']
    user.last_name = request.POST['last_name']
    user.save()

    wsrg =  WebsiteReportGroup(title="Summary of results for " + str(user))
    wsrg.save()
    user_stats = StatsUser(user=user, ws_report_group=wsrg)
    user_stats.save()

user_registered.connect(user_registered_callback)

class InstitutionalProfile(models.Model):
    id = models.AutoField(primary_key=True)

    title           = models.CharField(max_length=64,  blank=True, default="")
    contact1_name   = models.CharField(max_length=32,  blank=True, default="")
    contact1_title  = models.CharField(max_length=32,  blank=True, default="")
    contact1_email  = models.EmailField(max_length=64, blank=True, default="")
    contact1_phone  = models.CharField(max_length=16,  blank=True, default="")

    contact2_name   = models.CharField(max_length=32,  blank=True, default="")
    contact2_title  = models.CharField(max_length=32,  blank=True, default="")
    contact2_email  = models.EmailField(max_length=64, blank=True, default="")
    contact2_phone  = models.CharField(max_length=16,  blank=True, default="")

    account_type     = models.ForeignKey(AccountType, related_name="institional_subscriptions")
    top_level_domain = models.CharField(max_length=8, blank=True, default="")
    domain           = models.CharField(max_length=64, blank=True, default="")
    alt_domain       = models.CharField(max_length=64, blank=True, default="")
    authentication   = models.CharField(max_length=64, blank=True, default="")

    subscription_status   = models.CharField(max_length=8, choices=SUBSCRIPTION_STATUS_CHOICES, default="FREE")
    subscription_days       = models.IntegerField(default=0)

    subscription_start    = models.DateField(null=True, blank=True)
    subscription_end      = models.DateField(null=True, blank=True)
    subscription_payment  = models.IntegerField(default=0) # in dollars
    last_payment          = models.IntegerField(default=0) # in dollars

    users = models.ManyToManyField(User, related_name="institional_subscriptions", blank=True, default=None)

    class Meta:
        verbose_name        = "Institutional Profile"
        verbose_name_plural = "Institutional Profiles"
        ordering = ['account_type']

    def __str__(self):
        return self.domain + '.' + self.top_level_domain

    def update_subscription_status(self):

        if self.subscription_end:
            date1 = date(self.subscription_end.year, self.subscription_end.month, self.subscription_end.day)
            date2 = date.today()
            delta = date1 - date2

            self.subscription_days = delta.days

            if self.subscription_days >= 0:
                self.subscription_status = 'CURRENT'
            else:
                self.subscription_status     = 'EXPIRED'
                self.subscription_payments   = 0
                self.account_type            = AccountType.objects.get(type_id=16)

        self.save()

    def check_for_email_subscription_notifications(self):

        topic = ""

        if self.subscription_status == 'CURRENT':
            if self.subscription_days == 7 or self.subscription_days == 14:
                topic = "FAE Institutional Subscription expires in " + str(self.subscription_days) + "days"
            elif self.subscription_days == 1:
                topic = "FAE Institutional Subscription expires in one day"
            elif self.subscription_days == 0:
                topic = "FAE Institutional Subscription expires today"

        if self.subscription_status == 'EXPIRED' and self.subscription_days == -1:
            topic = "FAE Institutional Subscription expired yesterday"

        if topic:
            emails = []
            if self.contact1_email:
                emails.append(self.contact1_email)
            if self.contact2_email:
                emails.append(self.contact2_email)

            site = Site.objects.get_current()
            message = render_to_string('accounts/email_institutional_message.txt', {'institutional_profile': self, 'subscription_url': str(site) + reverse('contact_form') })
            send_mail(topic, message, EMAIL_HOST_USER, emails, fail_silently=False)

<|endoftext|>


<commit_before>import requests

from gophish.models import Error
'''
api.py

Base API endpoint class that abstracts basic CRUD operations.
'''

class APIEndpoint(object):
    """
    Represents an API endpoint for Gophish, containing common patterns
    for CRUD operations.
    """

    def __init__(self, api, endpoint=None, cls=None):
        """ Creates an instance of the APIEndpoint class.

        Args:
            api - Gophish.client - The authenticated REST client
            endpoint - str - The URL path to the resource endpoint
            cls - gophish.models.Model - The Class to use when parsing results
        """
        self.api = api
        self.endpoint = endpoint
        self._cls = cls

    def get(self,
            resource_id=None,
            resource_action=None,
            resource_cls=None,
            single_resource=False):
        """ Gets the details for one or more resources by ID

        Args:
            cls - gophish.models.Model - The resource class
            resource_id - str - The endpoint (URL path) for the resource
            resource_action - str - An action to perform on the resource
            resource_cls - cls - A class to use for parsing, if different than the base resource
            single_resource - bool - An override to tell Gophish that even
                though we aren't requesting a single resource, we expect a single response object

        Returns:
            One or more instances of cls parsed from the returned JSON
        """

        endpoint = self.endpoint

        if not resource_cls:
            resource_cls = self._cls

        if resource_id:
            endpoint = '{}/{}'.format(endpoint, resource_id)

        if resource_action:
            endpoint = '{}/{}'.format(endpoint, resource_action)

        response = self.api.execute("GET", endpoint)
        if not response.ok:
            return Error.parse(response.json())

        if resource_id or single_resource:
            return resource_cls.parse(response.json())

        return [resource_cls.parse(resource) for resource in response.json()]

    def post(self, resource):
        """ Creates a new instance of the resource.

        Args:
            resource - gophish.models.Model - The resource instance

        """
        response = self.api.execute(
            "POST", self.endpoint, json=(resource.as_dict()))

        if not response.ok:
            return Error.parse(response.json())

        return self._cls.parse(response.json())

    def put(self, resource):
        """ Edits an existing resource

        Args:
            resource - gophish.models.Model - The resource instance
        """

        endpoint = self.endpoint

        if resource.id:
            endpoint = '{}/{}'.format(endpoint, resource.id)

        response = self.api.execute("PUT", endpoint, json=resource.as_json())

        if not respose.ok:
            return Error.parse(response.json())

        return self._cls.parse(response.json())

    def delete(self, resource_id):
        """ Deletes an existing resource

        Args:
            resource_id - int - The resource ID to be deleted
        """

        endpoint = '{}/{}'.format(self.endpoint, resource_id)

        response = self.api.execute("DELETE", endpoint)

        if not response.ok:
            return Error.parse(response.json())

        return self._cls.parse(response.json())
<commit_msg>Fixed typo #8<commit_after>import requests

from gophish.models import Error
'''
api.py

Base API endpoint class that abstracts basic CRUD operations.
'''

class APIEndpoint(object):
    """
    Represents an API endpoint for Gophish, containing common patterns
    for CRUD operations.
    """

    def __init__(self, api, endpoint=None, cls=None):
        """ Creates an instance of the APIEndpoint class.

        Args:
            api - Gophish.client - The authenticated REST client
            endpoint - str - The URL path to the resource endpoint
            cls - gophish.models.Model - The Class to use when parsing results
        """
        self.api = api
        self.endpoint = endpoint
        self._cls = cls

    def get(self,
            resource_id=None,
            resource_action=None,
            resource_cls=None,
            single_resource=False):
        """ Gets the details for one or more resources by ID

        Args:
            cls - gophish.models.Model - The resource class
            resource_id - str - The endpoint (URL path) for the resource
            resource_action - str - An action to perform on the resource
            resource_cls - cls - A class to use for parsing, if different than the base resource
            single_resource - bool - An override to tell Gophish that even
                though we aren't requesting a single resource, we expect a single response object

        Returns:
            One or more instances of cls parsed from the returned JSON
        """

        endpoint = self.endpoint

        if not resource_cls:
            resource_cls = self._cls

        if resource_id:
            endpoint = '{}/{}'.format(endpoint, resource_id)

        if resource_action:
            endpoint = '{}/{}'.format(endpoint, resource_action)

        response = self.api.execute("GET", endpoint)
        if not response.ok:
            return Error.parse(response.json())

        if resource_id or single_resource:
            return resource_cls.parse(response.json())

        return [resource_cls.parse(resource) for resource in response.json()]

    def post(self, resource):
        """ Creates a new instance of the resource.

        Args:
            resource - gophish.models.Model - The resource instance

        """
        response = self.api.execute(
            "POST", self.endpoint, json=(resource.as_dict()))

        if not response.ok:
            return Error.parse(response.json())

        return self._cls.parse(response.json())

    def put(self, resource):
        """ Edits an existing resource

        Args:
            resource - gophish.models.Model - The resource instance
        """

        endpoint = self.endpoint

        if resource.id:
            endpoint = '{}/{}'.format(endpoint, resource.id)

        response = self.api.execute("PUT", endpoint, json=resource.as_json())

        if not response.ok:
            return Error.parse(response.json())

        return self._cls.parse(response.json())

    def delete(self, resource_id):
        """ Deletes an existing resource

        Args:
            resource_id - int - The resource ID to be deleted
        """

        endpoint = '{}/{}'.format(self.endpoint, resource_id)

        response = self.api.execute("DELETE", endpoint)

        if not response.ok:
            return Error.parse(response.json())

        return self._cls.parse(response.json())
<|endoftext|>