<commit_before>import openpathsampling as paths
from openpathsampling.todict import OPSNamed

from openpathsampling.analysis.move_strategy import levels as strategy_levels
import openpathsampling.analysis.move_strategy as strategies

import sys

class MoveScheme(OPSNamed):
    """
    Creates a move decision tree based on `MoveStrategy` instances.

    Attributes
    ----------
    movers : dict
        Dictionary mapping mover group as key to list of movers
    strategies : dict
        Dictionary mapping level (number) to list of strategies
    root_mover : PathMover
        Root of the move decision tree (`None` until tree is built)
    """
    def __init__(self, network):
        self.movers = {}
        self.movers = network.movers # TODO: legacy
        self.network = network
        self.strategies = {}
        self.balance_partners = {}
        self.choice_probability = {}
        self.root_mover = None

        self._mover_acceptance = {} # used in analysis

    def append(self, strategies, levels=None):
        """
        Adds new strategies to this scheme, organized by `level`.

        Parameters
        ----------
        strategies : MoveStrategy or list of MoveStrategy
            strategies to add to this scheme
        levels : integer or list of integer or None
            levels to associate with each strategy. If None, strategy.level.
        """
        # first we clean up the input: strategies is a list of MoveStrategy;
        # levels is a list of integers
        try:
            strategies = list(strategies)
        except TypeError:
            strategies = [strategies]

        if levels is not None:
            try:
                levels = list(levels)
            except TypeError:
                levels = [levels]*len(strategies)
        else:
            levels = []
            for strat in strategies:
                levels.append(strat.level)

        # now we put everything into appropriate dictionaries
        for strat, lev in zip(strategies, levels):
            try:
                self.strategies[lev].append(strat)
            except KeyError:
                self.strategies[lev] = [strat]

    # TODO: it might be nice to have a way to "lock" this once it has been
    # saved. That would prevent a (stupid) user from trying to rebuild a
    # custom-modified tree.
    def build_move_decision_tree(self):
        for lev in sorted(self.strategies.keys()):
            for strat in self.strategies[lev]:
                self.apply_strategy(strat)

    # TODO: should I make this a property? make root_mover into
    # _move_decision_tree? allow the user to directly set it? rename as
    # move_scheme? separated from building until some of that is clarified
    def move_decision_tree(self, rebuild=False):
        """
        Returns the move decision tree.

        Parameters
        ----------
        rebuild : bool, optional
            Whether to rebuild the tree, or use the previously build version
            (default is False, if no tree exists, sets to True)

        Returns
        -------
        PathMover
            Root mover of the move decision tree
        """
        if self.root_mover is None:
            rebuild=True
        if rebuild:
            self.choice_probability = {}
            self.build_move_decision_tree()
        return self.root_mover

    def apply_strategy(self, strategy):
        """
        Applies given strategy to the scheme as it stands.

        This is the tool used in the process of building up the move
        decision tree.

        Parameters
        ----------
        strategy : MoveStrategy
            the strategy to apply
        """
        movers = strategy.make_movers(self)
        group = strategy.group
        if strategy_levels.level_type(strategy.level) == strategy_levels.GLOBAL:
            # shortcut out for the global-level stuff
            self.root_mover = movers
        elif strategy.replace_signatures:
            self.movers[group] = movers
        elif strategy.replace_movers:
            try:
                n_existing = len(self.movers[group])
            except KeyError:
                # if the group doesn't exist, set it to these movers
                self.movers[group] = movers
            else:
                # Note that the following means that if the list of new
                # movers includes two movers with the same sig, the second
                # will overwrite the first. This is desired behavior. On the
                # other hand, if the list of old movers in the group already
                # has two movers with the same signature, then both should
                # be overwritten.
                existing_sigs = {}
                for i in range(n_existing):
                    key = self.movers[group][i].ensemble_signature
                    try:
                        existing_sigs[key].append(i)
                    except KeyError:
                        existing_sigs[key] = [i]

                # For each mover, if its signature exists in the existing
                # movers, replace the existing. Otherwise, append it to the
                # list.
                for mover in movers:
                    m_sig = mover.ensemble_signature
                    if m_sig in existing_sigs.keys():
                        for idx in existing_sigs[m_sig]:
                            self.movers[group][idx] = mover
                    else:
                        self.movers[group].append(mover)
        else:
            try:
                self.movers[group].extend(movers)
            except KeyError:
                self.movers[group] = movers

    def ensembles_for_move_tree(self, root=None):
        """
        Finds the list of all ensembles in the move tree starting at `root`.

        Parameters
        ----------
        root : PathMover
            Mover to act as root of this tree (can be a subtree). Default is
            `None`, in which case `self.root_mover` is used.

        Returns
        -------
        list of Ensemble
            ensembles which appear in this (sub)tree
        """
        if root is None:
            root = self.root_mover
        movers = root.map_pre_order(lambda x : x)
        mover_ensemble_dict = {}
        for m in movers:
            input_sig = m.input_ensembles
            output_sig = m.output_ensembles
            for ens in input_sig + output_sig:
                mover_ensemble_dict[ens] = 1
        mover_ensembles = mover_ensemble_dict.keys()
        return mover_ensembles

    def find_hidden_ensembles(self, root=None):
        """
        All ensembles which exist in the move scheme but not in the network.

        Parameters
        ----------
        root : PathMover
            Mover to act as root of this tree (can be a subtree). Default is
            `None`, in which case `self.root_mover` is used.

        Returns
        -------
        set of Ensemble
            "hidden" ensembles; the ensembles which are in the scheme but
            not the network.
        """
        unhidden_ensembles = set(self.network.all_ensembles)
        mover_ensembles = set(self.ensembles_for_move_tree(root))
        hidden_ensembles = mover_ensembles - unhidden_ensembles
        return hidden_ensembles

    def find_unused_ensembles(self, root=None):
        """
        All ensembles which exist in the network but not in the move scheme.

        Parameters
        ----------
        root : PathMover
            Mover to act as root of this tree (can be a subtree). Default is
            `None`, in which case `self.root_mover` is used.

        Returns
        -------
        set of Ensemble
            "unused" ensembles; the ensembles which are in the network but
            not the scheme.
        """
        unhidden_ensembles = set(self.network.all_ensembles)
        mover_ensembles = set(self.ensembles_for_move_tree(root))
        unused_ensembles = unhidden_ensembles - mover_ensembles
        return unused_ensembles

    def check_for_root(self, fcn_name):
        """
        Raises runtime warning if self.root_mover not set.

        Some functions are only valid after the decision tree has been
        built. This complains if the tree is not there.
        """
        if self.root_mover is None:
            warnstr = ("Can't use {fcn_name} before building the move " +
                       "decision tree").format(fcn_name=fcn_name)
            raise RuntimeWarning(warnstr)

    def build_balance_partners(self):
        """
        Create list of balance partners for all movers in groups.

        The balance partners are the movers in the same mover group which
        have the opposite ensemble signature (input and output switched).
        These are used when dynamically calculating detailed balance.

        Note
        ----
        Currently, every mover in a group must have exactly one balance
        partner.  In the future, this might be relaxed to "at least one".
        """
        self.check_for_root("build_balance_partners")
        for groupname in self.movers.keys():
            group = self.movers[groupname]
            for mover in group:
                partner_sig_set = (set(mover.output_ensembles),
                                   set(mover.input_ensembles))
                partners = [m for m in group
                            if m.ensemble_signature_set==partner_sig_set]
                self.balance_partners[mover] = partners
                if len(partners) != 1:
                    warnstr = "Mover {0}: number of balance partners is {1}"
                    raise RuntimeWarning(warnstr.format(mover, len(partners)))
        return self.balance_partners

    def _move_summary_line(self, move_name, n_accepted, n_trials,
                           n_total_trials, indentation):
        line = ("* "*indentation + str(move_name) +
                " ran " + str(float(n_trials)/n_total_trials*100) +
                "% of the cycles with acceptance " + str(n_accepted) + "/" +
                str(n_trials) + " (" + str(float(n_accepted) / n_trials) +
                ") \n")
        return line

    def move_acceptance(self, storage):
        for step in storage.steps:
            delta = step.change
            for m in delta:
                acc = 1 if m.accepted else 0
                key = (m.mover, str(delta.key(m)))
                try:
                    self._mover_acceptance[key][0] += acc
                    self._mover_acceptance[key][1] += 1
                except KeyError:
                    self._mover_acceptance[key] = [acc, 1]

    def move_summary(self, storage, movers=None, output=sys.stdout, depth=0):
        """
        Provides a summary of the movers in `storage` based on this transition.

        The summary includes the number of moves attempted and the
        acceptance rate. In some cases, extra lines are printed for each of
        the submoves.

        Parameters
        ----------
        storage : Storage
            The storage object
        movers : None or string or list of PathMover
            If None, provides a short summary of the keys in self.mover. If
            a string, provides a short summary using that string as a key in
            the `movers` dict. If a mover or list of movers, provides
            summary for each of those movers.
        output : file
            file to direct output
        depth : integer or None
            depth of submovers to show: if integer, shows that many
            submovers for each move; if None, shows all submovers
        """
        my_movers = { }
        if movers is None:
            movers = self.movers.keys()
        if type(movers) is str:
            movers = self.movers[movers]
        for key in movers:
            try:
                my_movers[key] = self.movers[key]
            except KeyError:
                my_movers[key] = [key]

        stats = { }
        for groupname in my_movers.keys():
            stats[groupname] = [0, 0]

        if self._mover_acceptance == { }:
            self.move_acceptance(storage)

        tot_trials = len(storage.steps)
        for groupname in my_movers.keys():
            group = my_movers[groupname]
            for mover in group:
                key_iter = (k for k in self._mover_acceptance.keys()
                            if k[0] is mover)

                for k in key_iter:
                    stats[groupname][0] += self._mover_acceptance[k][0]
                    stats[groupname][1] += self._mover_acceptance[k][1]

        for groupname in my_movers.keys():
            line = self._move_summary_line(
                move_name=groupname,
                n_accepted=stats[groupname][0],
                n_trials=stats[groupname][1],
                n_total_trials=tot_trials,
                indentation=0
            )
            output.write(line)

class DefaultScheme(MoveScheme):
    """
    Just a MoveScheme with the full set of default strategies: nearest
    neighbor repex, uniform selection one-way shooting, minus move, and
    path reversals, all structured as choose move type then choose specific
    move.
    """
    def __init__(self, network):
        super(DefaultScheme, self).__init__(network)
        self.append(strategies.NearestNeighborRepExStrategy())
        self.append(strategies.OneWayShootingStrategy())
        self.append(strategies.PathReversalStrategy())
        self.append(strategies.DefaultStrategy())
        self.append(strategies.MinusMoveStrategy())

        msouters = self.network.special_ensembles['ms_outer']
        for ms in msouters.keys():
            self.append(strategies.OneWayShootingStrategy(
                ensembles=[ms],
                group="ms_outer_shooting"
            ))
            self.append(strategies.PathReversalStrategy(
                ensembles=[ms],
                replace=False
            ))
            ms_neighbors = [t.ensembles[-1] for t in msouters[ms]]
            pairs = [[ms, neighb] for neighb in ms_neighbors]
            self.append(strategies.SelectedPairsRepExStrategy(
                ensembles=pairs
            ))

<commit_msg>Draft of sanity check<commit_after>import openpathsampling as paths
from openpathsampling.todict import OPSNamed

from openpathsampling.analysis.move_strategy import levels as strategy_levels
import openpathsampling.analysis.move_strategy as strategies

import sys

class MoveScheme(OPSNamed):
    """
    Creates a move decision tree based on `MoveStrategy` instances.

    Attributes
    ----------
    movers : dict
        Dictionary mapping mover group as key to list of movers
    strategies : dict
        Dictionary mapping level (number) to list of strategies
    root_mover : PathMover
        Root of the move decision tree (`None` until tree is built)
    """
    def __init__(self, network):
        self.movers = {}
        self.movers = network.movers # TODO: legacy
        self.network = network
        self.strategies = {}
        self.balance_partners = {}
        self.choice_probability = {}
        self.root_mover = None

        self._mover_acceptance = {} # used in analysis

    def append(self, strategies, levels=None):
        """
        Adds new strategies to this scheme, organized by `level`.

        Parameters
        ----------
        strategies : MoveStrategy or list of MoveStrategy
            strategies to add to this scheme
        levels : integer or list of integer or None
            levels to associate with each strategy. If None, strategy.level.
        """
        # first we clean up the input: strategies is a list of MoveStrategy;
        # levels is a list of integers
        try:
            strategies = list(strategies)
        except TypeError:
            strategies = [strategies]

        if levels is not None:
            try:
                levels = list(levels)
            except TypeError:
                levels = [levels]*len(strategies)
        else:
            levels = []
            for strat in strategies:
                levels.append(strat.level)

        # now we put everything into appropriate dictionaries
        for strat, lev in zip(strategies, levels):
            try:
                self.strategies[lev].append(strat)
            except KeyError:
                self.strategies[lev] = [strat]

    # TODO: it might be nice to have a way to "lock" this once it has been
    # saved. That would prevent a (stupid) user from trying to rebuild a
    # custom-modified tree.
    def build_move_decision_tree(self):
        for lev in sorted(self.strategies.keys()):
            for strat in self.strategies[lev]:
                self.apply_strategy(strat)

    # TODO: should I make this a property? make root_mover into
    # _move_decision_tree? allow the user to directly set it? rename as
    # move_scheme? separated from building until some of that is clarified
    def move_decision_tree(self, rebuild=False):
        """
        Returns the move decision tree.

        Parameters
        ----------
        rebuild : bool, optional
            Whether to rebuild the tree, or use the previously build version
            (default is False, if no tree exists, sets to True)

        Returns
        -------
        PathMover
            Root mover of the move decision tree
        """
        if self.root_mover is None:
            rebuild=True
        if rebuild:
            self.choice_probability = {}
            self.build_move_decision_tree()
        return self.root_mover

    def apply_strategy(self, strategy):
        """
        Applies given strategy to the scheme as it stands.

        This is the tool used in the process of building up the move
        decision tree.

        Parameters
        ----------
        strategy : MoveStrategy
            the strategy to apply
        """
        movers = strategy.make_movers(self)
        group = strategy.group
        if strategy_levels.level_type(strategy.level) == strategy_levels.GLOBAL:
            # shortcut out for the global-level stuff
            self.root_mover = movers
        elif strategy.replace_signatures:
            self.movers[group] = movers
        elif strategy.replace_movers:
            try:
                n_existing = len(self.movers[group])
            except KeyError:
                # if the group doesn't exist, set it to these movers
                self.movers[group] = movers
            else:
                # Note that the following means that if the list of new
                # movers includes two movers with the same sig, the second
                # will overwrite the first. This is desired behavior. On the
                # other hand, if the list of old movers in the group already
                # has two movers with the same signature, then both should
                # be overwritten.
                existing_sigs = {}
                for i in range(n_existing):
                    key = self.movers[group][i].ensemble_signature
                    try:
                        existing_sigs[key].append(i)
                    except KeyError:
                        existing_sigs[key] = [i]

                # For each mover, if its signature exists in the existing
                # movers, replace the existing. Otherwise, append it to the
                # list.
                for mover in movers:
                    m_sig = mover.ensemble_signature
                    if m_sig in existing_sigs.keys():
                        for idx in existing_sigs[m_sig]:
                            self.movers[group][idx] = mover
                    else:
                        self.movers[group].append(mover)
        else:
            try:
                self.movers[group].extend(movers)
            except KeyError:
                self.movers[group] = movers

    def ensembles_for_move_tree(self, root=None):
        """
        Finds the list of all ensembles in the move tree starting at `root`.

        Parameters
        ----------
        root : PathMover
            Mover to act as root of this tree (can be a subtree). Default is
            `None`, in which case `self.root_mover` is used.

        Returns
        -------
        list of Ensemble
            ensembles which appear in this (sub)tree
        """
        if root is None:
            root = self.root_mover
        movers = root.map_pre_order(lambda x : x)
        mover_ensemble_dict = {}
        for m in movers:
            input_sig = m.input_ensembles
            output_sig = m.output_ensembles
            for ens in input_sig + output_sig:
                mover_ensemble_dict[ens] = 1
        mover_ensembles = mover_ensemble_dict.keys()
        return mover_ensembles

    def find_hidden_ensembles(self, root=None):
        """
        All ensembles which exist in the move scheme but not in the network.

        Parameters
        ----------
        root : PathMover
            Mover to act as root of this tree (can be a subtree). Default is
            `None`, in which case `self.root_mover` is used.

        Returns
        -------
        set of Ensemble
            "hidden" ensembles; the ensembles which are in the scheme but
            not the network.
        """
        unhidden_ensembles = set(self.network.all_ensembles)
        mover_ensembles = set(self.ensembles_for_move_tree(root))
        hidden_ensembles = mover_ensembles - unhidden_ensembles
        return hidden_ensembles

    def find_unused_ensembles(self, root=None):
        """
        All ensembles which exist in the network but not in the move scheme.

        Parameters
        ----------
        root : PathMover
            Mover to act as root of this tree (can be a subtree). Default is
            `None`, in which case `self.root_mover` is used.

        Returns
        -------
        set of Ensemble
            "unused" ensembles; the ensembles which are in the network but
            not the scheme.
        """
        unhidden_ensembles = set(self.network.all_ensembles)
        mover_ensembles = set(self.ensembles_for_move_tree(root))
        unused_ensembles = unhidden_ensembles - mover_ensembles
        return unused_ensembles

    def check_for_root(self, fcn_name):
        """
        Raises runtime warning if self.root_mover not set.

        Some functions are only valid after the decision tree has been
        built. This complains if the tree is not there.
        """
        if self.root_mover is None:
            warnstr = ("Can't use {fcn_name} before building the move " +
                       "decision tree").format(fcn_name=fcn_name)
            raise RuntimeWarning(warnstr)

    def build_balance_partners(self):
        """
        Create list of balance partners for all movers in groups.

        The balance partners are the movers in the same mover group which
        have the opposite ensemble signature (input and output switched).
        These are used when dynamically calculating detailed balance.

        Note
        ----
        Currently, every mover in a group must have exactly one balance
        partner.  In the future, this might be relaxed to "at least one".
        """
        self.check_for_root("build_balance_partners")
        for groupname in self.movers.keys():
            group = self.movers[groupname]
            for mover in group:
                partner_sig_set = (set(mover.output_ensembles),
                                   set(mover.input_ensembles))
                partners = [m for m in group
                            if m.ensemble_signature_set==partner_sig_set]
                self.balance_partners[mover] = partners
                if len(partners) != 1:
                    warnstr = "Mover {0}: number of balance partners is {1}"
                    raise RuntimeWarning(warnstr.format(mover, len(partners)))
        return self.balance_partners

    def sanity_check(self):
        # check that all sampling ensembles are used
        sampling_transitions = self.network.sampling_transitions
        all_sampling_ensembles = sum(sampling_transitions.ensembles, [])
        unused = self.find_unused_ensembles()
        for ens in unused:
            assert(ens not in all_sampling_ensembles)

        # check that choice_probability adds up
        total_choice = sum(self.choice_probability.values())
        assert(abs(total_choice - 1.0) < 1e-7)

        # check for duplicated movers in groups
        all_movers = sum(self.movers.values(), [])
        all_unique_movers = set(all_movers)
        assert(len(all_movers) == len(all_unique_movers))

    def _move_summary_line(self, move_name, n_accepted, n_trials,
                           n_total_trials, indentation):
        line = ("* "*indentation + str(move_name) +
                " ran " + str(float(n_trials)/n_total_trials*100) +
                "% of the cycles with acceptance " + str(n_accepted) + "/" +
                str(n_trials) + " (" + str(float(n_accepted) / n_trials) +
                ") \n")
        return line

    def move_acceptance(self, storage):
        for step in storage.steps:
            delta = step.change
            for m in delta:
                acc = 1 if m.accepted else 0
                key = (m.mover, str(delta.key(m)))
                try:
                    self._mover_acceptance[key][0] += acc
                    self._mover_acceptance[key][1] += 1
                except KeyError:
                    self._mover_acceptance[key] = [acc, 1]

    def move_summary(self, storage, movers=None, output=sys.stdout, depth=0):
        """
        Provides a summary of the movers in `storage` based on this transition.

        The summary includes the number of moves attempted and the
        acceptance rate. In some cases, extra lines are printed for each of
        the submoves.

        Parameters
        ----------
        storage : Storage
            The storage object
        movers : None or string or list of PathMover
            If None, provides a short summary of the keys in self.mover. If
            a string, provides a short summary using that string as a key in
            the `movers` dict. If a mover or list of movers, provides
            summary for each of those movers.
        output : file
            file to direct output
        depth : integer or None
            depth of submovers to show: if integer, shows that many
            submovers for each move; if None, shows all submovers
        """
        my_movers = { }
        if movers is None:
            movers = self.movers.keys()
        if type(movers) is str:
            movers = self.movers[movers]
        for key in movers:
            try:
                my_movers[key] = self.movers[key]
            except KeyError:
                my_movers[key] = [key]

        stats = { }
        for groupname in my_movers.keys():
            stats[groupname] = [0, 0]

        if self._mover_acceptance == { }:
            self.move_acceptance(storage)

        tot_trials = len(storage.steps)
        for groupname in my_movers.keys():
            group = my_movers[groupname]
            for mover in group:
                key_iter = (k for k in self._mover_acceptance.keys()
                            if k[0] is mover)

                for k in key_iter:
                    stats[groupname][0] += self._mover_acceptance[k][0]
                    stats[groupname][1] += self._mover_acceptance[k][1]

        for groupname in my_movers.keys():
            line = self._move_summary_line(
                move_name=groupname,
                n_accepted=stats[groupname][0],
                n_trials=stats[groupname][1],
                n_total_trials=tot_trials,
                indentation=0
            )
            output.write(line)

class DefaultScheme(MoveScheme):
    """
    Just a MoveScheme with the full set of default strategies: nearest
    neighbor repex, uniform selection one-way shooting, minus move, and
    path reversals, all structured as choose move type then choose specific
    move.
    """
    def __init__(self, network):
        super(DefaultScheme, self).__init__(network)
        self.append(strategies.NearestNeighborRepExStrategy())
        self.append(strategies.OneWayShootingStrategy())
        self.append(strategies.PathReversalStrategy())
        self.append(strategies.DefaultStrategy())
        self.append(strategies.MinusMoveStrategy())

        msouters = self.network.special_ensembles['ms_outer']
        for ms in msouters.keys():
            self.append(strategies.OneWayShootingStrategy(
                ensembles=[ms],
                group="ms_outer_shooting"
            ))
            self.append(strategies.PathReversalStrategy(
                ensembles=[ms],
                replace=False
            ))
            ms_neighbors = [t.ensembles[-1] for t in msouters[ms]]
            pairs = [[ms, neighb] for neighb in ms_neighbors]
            self.append(strategies.SelectedPairsRepExStrategy(
                ensembles=pairs
            ))

<|endoftext|>


<commit_before># ---
# jupyter:
#   jupytext:
#     formats: ipynb,.pct.py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.4.0
#   kernelspec:
#     display_name: 'Python 3.7.4 64-bit (''gpflow2'': conda)'
#     name: python37464bitgpflow2conda97d7f93b14fc49f496551bb0d0f2e18e
# ---

# %% [markdown]
# # Monitoring Optimisation
#
# In this notebook we cover how to monitor the model and certain metrics during optimisation.
#
# ## Setup

# %%
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

import gpflow
from gpflow.ci_utils import ci_niter

np.random.seed(0)

# %% [markdown]
# The monitoring functionality lives in `gpflow.monitor`.
# For now, we import `ModelToTensorBoard`, `ImageToTensorBoard`, `ScalarToTensorBoard` monitoring tasks and `MonitorTaskGroup` and `Monitor`.

# %%
from gpflow.monitor import (
    ImageToTensorBoard,
    ModelToTensorBoard,
    Monitor,
    MonitorTaskGroup,
    ScalarToTensorBoard,
)

# %% [markdown]
# ## Set up data and model

# %%
# Define some configuration constants.

num_data = 100
noise_std = 0.1
optimisation_steps = ci_niter(100)

# %%
# Create dummy data.

X = np.random.randn(num_data, 1)  # [N, 2]
Y = np.sin(X) + 0.5 * np.cos(X) + np.random.randn(*X.shape) * noise_std  # [N, 1]
plt.plot(X, Y, "o")

# %%
# Set up model and print

kernel = gpflow.kernels.SquaredExponential(lengthscales=[1.0, 2.0]) + gpflow.kernels.Linear()
model = gpflow.models.GPR((X, Y), kernel, noise_variance=noise_std ** 2)
model

# %%
# We define a function that plots the model's prediction (in the form of samples) together with the data.
# Importantly, this function has no other argument than `fig: matplotlib.figure.Figure` and `ax: matplotlib.figure.Axes`.

def plot_prediction(fig, ax):
    Xnew = np.linspace(X.min() - 0.5, X.max() + 0.5, 100).reshape(-1, 1)
    Ypred = model.predict_f_samples(Xnew, full_cov=True, num_samples=20)
    ax.plot(Xnew.flatten(), np.squeeze(Ypred).T, "C1", alpha=0.2)
    ax.plot(X, Y, "o")

# Let's check if the function does the desired plotting
fig = plt.figure()
ax = fig.subplots()
plot_prediction(fig, ax)
plt.show()

# %% [markdown]
# ## Set up monitoring tasks
#
# We now define the `MonitorTask`s that will be executed during the optimisation.
# For this tutorial we set up three tasks:
# - `ModelToTensorBoard`: writes the models hyper-parameters such as `likelihood.variance` and `kernel.lengthscales` to a TensorBoard.
# - `ImageToTensorBoard`: writes custom matplotlib images to a TensorBoard.
# - `ScalarToTensorBoard`: writes any scalar value to a TensorBoard. Here, we use it to write the model's training objective.

# %%
log_dir = "logs"  # Directory where TensorBoard files will be written.
model_task = ModelToTensorBoard(log_dir, model)
image_task = ImageToTensorBoard(log_dir, plot_prediction, "image_samples")
lml_task = ScalarToTensorBoard(log_dir, lambda: model.log_likelihood(), "lml")

# %% [markdown]
# We now group the tasks in a set of fast and slow tasks and pass them to the monitor.
# This allows us to execute the groups at a different frequency.

# %%
# Plotting tasks can be quite slow. We want to run them less frequently.
# We group them in a `MonitorTaskGroup` and set the period to 5.
slow_tasks = MonitorTaskGroup(image_task, period=5)

# The other tasks are fast. We run them at each iteration of the optimisation.
fast_tasks = MonitorTaskGroup([model_task, lml_task], period=1)

# Both groups are passed to the monitor.
# `slow_tasks` will be run five times less frequently than `fast_tasks`.
monitor = Monitor(fast_tasks, slow_tasks)

# %%
@tf.function
def closure():
    return -model.log_likelihood()

opt = tf.optimizers.Adam()

for step in range(optimisation_steps):
    opt.minimize(closure, model.trainable_variables)
    monitor(step)  # <-- run the monitoring

# %% [markdown]
# TensorBoard is accessible through the browser, after launching the server by running `tensorboard --logdir ${logdir}`. See the [TensorFlow documentation on TensorBoard](https://www.tensorflow.org/tensorboard/get_started) for more information.

# %% [markdown]
# For optimal performance, we can also wrap the monitor call inside `tf.function`:

# %%
opt = tf.optimizers.Adam()

log_dir = f"{log_dir}/compiled"
model_task = ModelToTensorBoard(log_dir, model)
lml_task = ScalarToTensorBoard(log_dir, lambda: model.log_likelihood(), "lml")
# Note that the `ImageToTensorBoard` task cannot be compiled, and is omitted from the monitoring
monitor = Monitor(MonitorTaskGroup([model_task, lml_task]))

# %% [markdown]
# In the optimisation loop below we use `tf.range` (rather than Python's built-in range) to avoid re-tracing the `step` function each time.

@tf.function
def step(i):
    opt.minimize(lambda: -1.0 * model.log_likelihood(), model.trainable_variables)
    monitor(i)

# Notice the tf.range
for i in tf.range(optimisation_steps):
    step(i)
<commit_msg>Set kernelspec to Py3 in monitoring.pct.py (#1386)<commit_after># ---
# jupyter:
#   jupytext:
#     formats: ipynb,.pct.py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.4.0
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% [markdown]
# # Monitoring Optimisation
#
# In this notebook we cover how to monitor the model and certain metrics during optimisation.
#
# ## Setup

# %%
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

import gpflow
from gpflow.ci_utils import ci_niter

np.random.seed(0)

# %% [markdown]
# The monitoring functionality lives in `gpflow.monitor`.
# For now, we import `ModelToTensorBoard`, `ImageToTensorBoard`, `ScalarToTensorBoard` monitoring tasks and `MonitorTaskGroup` and `Monitor`.

# %%
from gpflow.monitor import (
    ImageToTensorBoard,
    ModelToTensorBoard,
    Monitor,
    MonitorTaskGroup,
    ScalarToTensorBoard,
)

# %% [markdown]
# ## Set up data and model

# %%
# Define some configuration constants.

num_data = 100
noise_std = 0.1
optimisation_steps = ci_niter(100)

# %%
# Create dummy data.

X = np.random.randn(num_data, 1)  # [N, 2]
Y = np.sin(X) + 0.5 * np.cos(X) + np.random.randn(*X.shape) * noise_std  # [N, 1]
plt.plot(X, Y, "o")

# %%
# Set up model and print

kernel = gpflow.kernels.SquaredExponential(lengthscales=[1.0, 2.0]) + gpflow.kernels.Linear()
model = gpflow.models.GPR((X, Y), kernel, noise_variance=noise_std ** 2)
model

# %%
# We define a function that plots the model's prediction (in the form of samples) together with the data.
# Importantly, this function has no other argument than `fig: matplotlib.figure.Figure` and `ax: matplotlib.figure.Axes`.

def plot_prediction(fig, ax):
    Xnew = np.linspace(X.min() - 0.5, X.max() + 0.5, 100).reshape(-1, 1)
    Ypred = model.predict_f_samples(Xnew, full_cov=True, num_samples=20)
    ax.plot(Xnew.flatten(), np.squeeze(Ypred).T, "C1", alpha=0.2)
    ax.plot(X, Y, "o")

# Let's check if the function does the desired plotting
fig = plt.figure()
ax = fig.subplots()
plot_prediction(fig, ax)
plt.show()

# %% [markdown]
# ## Set up monitoring tasks
#
# We now define the `MonitorTask`s that will be executed during the optimisation.
# For this tutorial we set up three tasks:
# - `ModelToTensorBoard`: writes the models hyper-parameters such as `likelihood.variance` and `kernel.lengthscales` to a TensorBoard.
# - `ImageToTensorBoard`: writes custom matplotlib images to a TensorBoard.
# - `ScalarToTensorBoard`: writes any scalar value to a TensorBoard. Here, we use it to write the model's training objective.

# %%
log_dir = "logs"  # Directory where TensorBoard files will be written.
model_task = ModelToTensorBoard(log_dir, model)
image_task = ImageToTensorBoard(log_dir, plot_prediction, "image_samples")
lml_task = ScalarToTensorBoard(log_dir, lambda: model.log_likelihood(), "lml")

# %% [markdown]
# We now group the tasks in a set of fast and slow tasks and pass them to the monitor.
# This allows us to execute the groups at a different frequency.

# %%
# Plotting tasks can be quite slow. We want to run them less frequently.
# We group them in a `MonitorTaskGroup` and set the period to 5.
slow_tasks = MonitorTaskGroup(image_task, period=5)

# The other tasks are fast. We run them at each iteration of the optimisation.
fast_tasks = MonitorTaskGroup([model_task, lml_task], period=1)

# Both groups are passed to the monitor.
# `slow_tasks` will be run five times less frequently than `fast_tasks`.
monitor = Monitor(fast_tasks, slow_tasks)

# %%
@tf.function
def closure():
    return -model.log_likelihood()

opt = tf.optimizers.Adam()

for step in range(optimisation_steps):
    opt.minimize(closure, model.trainable_variables)
    monitor(step)  # <-- run the monitoring

# %% [markdown]
# TensorBoard is accessible through the browser, after launching the server by running `tensorboard --logdir ${logdir}`. See the [TensorFlow documentation on TensorBoard](https://www.tensorflow.org/tensorboard/get_started) for more information.

# %% [markdown]
# For optimal performance, we can also wrap the monitor call inside `tf.function`:

# %%
opt = tf.optimizers.Adam()

log_dir = f"{log_dir}/compiled"
model_task = ModelToTensorBoard(log_dir, model)
lml_task = ScalarToTensorBoard(log_dir, lambda: model.log_likelihood(), "lml")
# Note that the `ImageToTensorBoard` task cannot be compiled, and is omitted from the monitoring
monitor = Monitor(MonitorTaskGroup([model_task, lml_task]))

# %% [markdown]
# In the optimisation loop below we use `tf.range` (rather than Python's built-in range) to avoid re-tracing the `step` function each time.

@tf.function
def step(i):
    opt.minimize(lambda: -1.0 * model.log_likelihood(), model.trainable_variables)
    monitor(i)

# Notice the tf.range
for i in tf.range(optimisation_steps):
    step(i)
<|endoftext|>


<commit_before>PASS = ""
CHANNEL = ""
HOST = "irc.freenode.net"
PORT = 6667
NICK = ""
OWNER = "unaffiliated/wolfbot_admin1"
ADMINS = ("unaffiliated/wolfbot_admin1", "unaffiliated/wolfbot_admin2")
CMD_CHAR = "!"
CHANGING_HOST_QUIT_MESSAGE = "Changing host"<commit_msg>test 111<commit_after>PASS = ""
CHANNEL = ""
HOST = "irc.freenode.net"
PORT = 6667
NICK = "wolfbot"
OWNER = "unaffiliated/wolfbot_admin1"
ADMINS = ("unaffiliated/wolfbot_admin1", "unaffiliated/wolfbot_admin2")
CMD_CHAR = "!"
CHANGING_HOST_QUIT_MESSAGE = "Changing host"<|endoftext|>


<commit_before>import json
import logging
import os
from threading import Timer, enumerate
from datetime import datetime
from requestlogger import WSGILogger, ApacheFormatter
from logging.handlers import TimedRotatingFileHandler
import bottle
from bottle_sqlite import SQLitePlugin
from bottle import run, request
from bitcoinrpc.authproxy import AuthServiceProxy
import time
import credit
import payout
import database
import load_config
from utils import AddressCheck
from exchanges import Bittrex, BTER, CCEDK, Poloniex, TestExchange

__author__ = 'sammoth'

app = bottle.Bottle()
bottle.debug(True)

# Create the log directory
if not os.path.isdir('logs'):
    os.mkdir('logs')

# Set the WSGI Logger facility
log_time = int(time.time())
handlers = [TimedRotatingFileHandler('logs/server-{}.log'.format(log_time),
                                     when='midnight'), ]
server = WSGILogger(app, handlers, ApacheFormatter())

# Build the application Logger
log = logging.Logger('ALP')
rotating_file = TimedRotatingFileHandler('logs/alp-{}.log'.format(log_time),
                                         when='midnight')
stream = logging.StreamHandler()
formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s',
                              datefmt='%y-%m-%d %H:%M:%S')
stream.setFormatter(formatter)
rotating_file.setFormatter(formatter)
log.addHandler(rotating_file)
log.addHandler(stream)

# Create the database if one doesn't exist
database.build(log)

# Install the SQLite plugin
app.install(SQLitePlugin(dbfile='pool.db', keyword='db'))

# Create the Exchange wrapper objects
wrappers = {'bittrex': Bittrex(),
            'bter': BTER(),
            'ccedk': CCEDK(),
            'poloniex': Poloniex(),
            'test_exchange': TestExchange()}

log.info('load pool config')
app.config.load_config('pool_config')

log.info('load exchange config')
load_config.load(app, 'exchange_config')

log.info('set up a json-rpc connection with nud')
rpc = AuthServiceProxy("http://{}:{}@{}:{}".format(app.config['rpc.user'],
                                                   app.config['rpc.pass'],
                                                   app.config['rpc.host'],
                                                   app.config['rpc.port']))

if os.getenv('RUN_TIMERS', '1') == '1':
    log.info('running timers')
    # Set the timer for credits
    credit_timer = Timer(60.0, credit.credit,
                         kwargs={'app': app, 'rpc': rpc, 'log': log})
    credit_timer.name = 'credit_timer'
    if 'credit_timer' not in enumerate():
        credit_timer.daemon = True
        credit_timer.start()
    # Set the timer for payouts
    payout_timer = Timer(86400.0, payout.pay,
                         kwargs={'rpc': rpc, 'log': log})
    payout_timer.name = 'payout_timer'
    if 'payout_timer' not in enumerate():
        payout_timer.daemon = True
        payout_timer.start()

    def check_threads():
        check_threads_timer = Timer(10, check_threads)
        check_threads_timer.name = 'check_threads_timer'
        check_threads_timer.daemon = True
        check_threads_timer.start()
        known_threads = []
        for t in enumerate():
            if t.name not in ['payout_timer', 'credit_timer']:
                continue
            log.debug('known threads %s', known_threads)
            log.debug('thread name %s', t.name)
            if t.name not in known_threads:
                known_threads.append(t.name)
            else:
                log.debug('killed duplicate thread %s', t.name)
                t.cancel()
            log.debug('known threads 2 %s', known_threads)

    check_threads_timer = Timer(10, check_threads)
    check_threads_timer.name = 'check_threads_timer'
    check_threads_timer.daemon = True
    check_threads_timer.start()

def check_headers(headers):
    """
    Ensure the correct headers get passed in the request
    :param headers:
    :return:
    """
    if headers.get('Content-Type') != 'application/json':
        log.warn('invalid header - need app/json')
        return False
    return True

@app.get('/')
def root():
    """
    Show a default page with info about the server health
    :return:
    """
    return {'success': True, 'message': 'ALP Server is operational'}

@app.post('/register')
def register(db):
    """
    Register a new user on the server
    Requires:
        user - API public Key
        address - Valid NBT payout address
        exchange - supported exchange
        unit - supported currency
    :return:
    """
    log.info('/register')
    # Check the content type
    if not check_headers(request.headers):
        return {'success': False, 'message': 'Content-Type header must be set to '
                                             '\'application/json\''}
    # Get the post parameters
    try:
        user = request.json.get('user')
        address = request.json.get('address')
        exchange = request.json.get('exchange')
        unit = request.json.get('unit')
    except AttributeError:
        log.warn('no json found in request')
        return {'success': False, 'message': 'no json found in request'}
    # check for missing parameters
    if not user:
        log.warn('no user provided')
        return {'success': False, 'message': 'no user provided'}
    if not address:
        log.warn('no address provided')
        return {'success': False, 'message': 'no address provided'}
    if not exchange:
        log.warn('no exchange provided')
        return {'success': False, 'message': 'no exchange provided'}
    if not unit:
        log.warn('no unit provided')
        return {'success': False, 'message': 'no unit provided'}
    # Check for a valid address
    if address[0] != 'B':
        log.warn('%s is not a valid NBT address. No \'B\'', address)
        return {'success': False, 'message': '{} is not a valid NBT address. It '
                                             'should start with a \'B\''.format(address)}
    address_check = AddressCheck()
    if not address_check.check_checksum(address):
        log.warn('%s is not a valid NBT address. Bad checksum', address)
        return {'success': False, 'message': '{} is not a valid NBT address. The '
                                             'checksum doesn\'t match'.format(address)}
    # Check that the requests exchange is supported by the server
    if exchange not in app.config['exchanges']:
        log.warn('%s is not supported', exchange)
        return {'success': False, 'message': '{} is not supported'.format(exchange)}
    # Check that the unit is supported on the server
    if unit not in app.config['{}.units'.format(exchange)]:
        log.warn('%s is not supported on %s', unit, exchange)
        return {'success': False, 'message': '{} is not supported on {}'.format(unit,
                                                                                exchange)}
    # Check if the user already exists in the database
    check = db.execute("SELECT id FROM users WHERE user=? AND address=? AND "
                       "exchange=? AND unit=?;", (user, address, exchange,
                                                  unit)).fetchone()
    if check:
        log.warn('user is already registered')
        return {'success': False, 'message': 'user is already registered'}
    db.execute("INSERT INTO users ('user','address','exchange','unit') VALUES (?,?,?,?)",
               (user, address, exchange, unit))
    log.info('user %s successfully registered', user)
    return {'success': True, 'message': 'user successfully registered'}

@app.post('/liquidity')
def liquidity(db):
    """
    Allow the user to submit liquidity validations to the server
    Will be multiple times per minute
    With the submitted data get the users orders and update the database accordingly
    Requires:
        user - API public Key
        req - dictionary to be used to get open orders
        sign - result of signing req with API private key
        exchange - valid, supported exchange
        unit - valid, supported unit
    :return:
    """
    log.info('/liquidity')
    # Check the content type
    if not check_headers(request.headers):
        return {'success': False, 'message': 'Content-Type header must be set to '
                                             '\'application/json\''}
    # Get the post parameters
    try:
        user = request.json.get('user')
        sign = request.json.get('sign')
        exchange = request.json.get('exchange')
        unit = request.json.get('unit')
        req = request.json.get('req')
    except AttributeError:
        log.warn('no json found in request')
        return {'success': False, 'message': 'no json found in request'}
    if not user:
        log.warn('no user provided')
        return {'success': False, 'message': 'no user provided'}
    if not sign:
        log.warn('no sign provided')
        return {'success': False, 'message': 'no sign provided'}
    if not exchange:
        log.warn('no exchange provided')
        return {'success': False, 'message': 'no exchange provided'}
    if not unit:
        log.warn('no unit provided')
        return {'success': False, 'message': 'no unit provided'}
    if not req:
        log.warn('no req provided')
        return {'success': False, 'message': 'no req provided'}
    # use the submitted data to request the users orders
    orders = wrappers[exchange].validate_request(user, unit, req, sign)
    price = get_price()
    # clear existing orders for the user
    log.info('clear existing orders for user %s', user)
    db.execute("DELETE FROM orders WHERE user=? AND exchange=? AND unit=?", (user,
                                                                             exchange,
                                                                             unit))
    # Loop through the orders
    for order in orders:
        # Calculate how the order price is from the known good price
        order_deviation = 1.0 - (min(order['price'], price) / max(order['price'], price))
        # Using the server tolerance determine if the order is tier 1 or tier 2
        # Only tier 1 is compensated by the server
        tier = 'tier_2'
        if order_deviation <= app.config['{}.{}.{}.tolerance'.format(exchange, unit,
                                                                     order['type'])]:
            tier = 'tier_1'
        # save the order details
        db.execute("INSERT INTO orders ('user','tier','order_id','order_amount',"
                   "'order_type','exchange','unit') VALUES (?,?,?,?,?,?,?)",
                   (user, tier, str(order['id']), float(order['amount']),
                    str(order['type']), exchange, unit))
    log.info('user %s orders saved for validation', user)
    return {'success': True, 'message': 'orders saved for validation'}

@app.get('/exchanges')
def exchanges():
    """
    Show the app.config as it pertains to the supported exchanges
    :return:
    """
    log.info('/exchanges')
    data = {}
    for exchange in app.config['exchanges']:
        if exchange not in data:
            data[exchange] = {}
        for unit in app.config['{}.units'.format(exchange)]:
            data[exchange][unit] = {'ask': {'tolerance': app.config['{}.{}.ask.'
                                                                    'tolerance'
                                                                    ''.format(exchange,
                                                                              unit)],
                                            'tier_1': {
                                                'reward': app.config['{}.{}.ask.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_1')]
                                                },
                                            'tier_2': {
                                                'reward': app.config['{}.{}.ask.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_2')]
                                                }
                                            },

                                    'bid': {'tolerance': app.config['{}.{}.bid.'
                                                                    'tolerance'
                                                                    ''.format(exchange,
                                                                              unit)],
                                            'tier_1': {
                                                'reward': app.config['{}.{}.bid.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_1')]
                                                },
                                            'tier_2': {
                                                'reward': app.config['{}.{}.bid.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_2')]
                                                }
                                            }
                                    }
    return data

@app.get('/status')
def status(db):
    """
    Display the overall pool status.
    This will be the number of users and the amount of liquidity for each tier, side,
    unit, exchange for the last full round and the current round
    :param db:
    :return:

    We want to display

    * Total liquidity provided by pool
    * Total tier_1 by pool
    * Total tier_2 by pool
    * total liquidity by exchange
    * total tier_1 by exchange
    * total tier_2 by exchange
    * total liquidity by exchange/pair
    * total tier_1 by exchange/pair
    * total tier_2 by exchange/pair
    * total liquidity by exchange/pair/side
    * total tier_1 by exchange/pair/side
    * total tier_2 by exchange/pair/side

    * number of users
    * number of active users

    * amount 1 NBT will be rewarded currently

    """
    log.info('/status')
    # build the blank data object
    data = {'last_credit_time': 'no credits yet',
            'total_liquidity': 0.0,
            'total_liquidity_bid': 0.0,
            'total_liquidity_ask': 0.0,
            'total_liquidity_tier_1': 0.0,
            'total_liquidity_tier_1_bid': 0.0,
            'total_liquidity_tier_1_ask': 0.0,
            'total_liquidity_tier_2': 0.0,
            'total_liquidity_tier_2_bid': 0.0,
            'total_liquidity_tier_2_ask': 0.0}

    # get the latest credit data from the credits field
    credit_data = db.execute("SELECT * FROM credits WHERE time=("
                             "SELECT value FROM info WHERE key='last_credit_time'"
                             ")").fetchall()
    log.debug(credit_data)
    # parse the data
    # id INTEGER PRIMARY KEY, time NUMBER, user TEXT, exchange TEXT, unit TEXT,
    # tier TEXT, side TEXT, provided NUMBER, total NUMBER, percentage NUMBER,
    # reward NUMBER, paid INTEGER
    for cred in credit_data:
        # update the last_credit_time
        data['last_credit_time'] = cred[1]
        # increment the total liquidity (this is total over entire pool)
        data['total_liquidity'] += cred[7]
        # increment buy and sell side totals
        if cred[6] == 'bid':
            data['total_liquidity_bid'] += cred[7]
        else:
            data['total_liquidity_ask'] += cred[7]
        # increment tier_1 totals
        if cred[5] == 'tier_1':
            data['total_liquidity_tier_1'] += cred[7]
            if cred[6] == 'bid':
                data['total_liquidity_tier_1_bid'] += cred[7]
            else:
                data['total_liquidity_tier_1_bid'] += cred[7]
        # increment tier_2 totals
        if cred[5] == 'tier_2':
            data['total_liquidity_tier_2'] += cred[7]
            if cred[6] == 'bid':
                data['total_liquidity_tier_2_bid'] += cred[7]
            else:
                data['total_liquidity_tier_2_ask'] += cred[7]

    return {'status': True, 'message': data}

def get_price():
    """
    Set the server price primarily from the NuBot streaming server but falling back to
    feeds if that fails
    :return:
    """
    return 1234

@app.error(code=500)
def error500(error):
    return json.dumps({'success': False, 'message': '500 error'})

@app.error(code=502)
def error502(error):
    return json.dumps({'success': False, 'message': '502 error'})

@app.error(code=503)
def error503(error):
    return json.dumps({'success': False, 'message': '503 error'})

@app.error(code=404)
def error404(error):
    return json.dumps({'success': False, 'message': '404 {} not found'
                                                    ''.format(request.url)})

@app.error(code=405)
def error405(error):
    return json.dumps({'success': False, 'message': '405 error. '
                                                    'Incorrect HTTP method used'})

if __name__ == '__main__':
    # Run the server
    run(server, host='localhost', port=int(os.environ.get("PORT", 3333)), debug=True)
<commit_msg>update threading init to allow for gunicorn hook<commit_after>import json
import logging
import os
from threading import Timer, enumerate
from datetime import datetime
from requestlogger import WSGILogger, ApacheFormatter
from logging.handlers import TimedRotatingFileHandler
import bottle
from bottle_sqlite import SQLitePlugin
from bottle import run, request
from bitcoinrpc.authproxy import AuthServiceProxy
import time
import credit
import payout
import database
import load_config
from utils import AddressCheck
from exchanges import Bittrex, BTER, CCEDK, Poloniex, TestExchange

__author__ = 'sammoth'

app = bottle.Bottle()
bottle.debug(True)

# Create the log directory
if not os.path.isdir('logs'):
    os.mkdir('logs')

# Set the WSGI Logger facility
log_time = int(time.time())
handlers = [TimedRotatingFileHandler('logs/server-{}.log'.format(log_time),
                                     when='midnight'), ]
server = WSGILogger(app, handlers, ApacheFormatter())

# Build the application Logger
log = logging.Logger('ALP')
rotating_file = TimedRotatingFileHandler('logs/alp-{}.log'.format(log_time),
                                         when='midnight')
stream = logging.StreamHandler()
formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s',
                              datefmt='%y-%m-%d %H:%M:%S')
stream.setFormatter(formatter)
rotating_file.setFormatter(formatter)
log.addHandler(rotating_file)
log.addHandler(stream)

# Create the database if one doesn't exist
database.build(log)

# Install the SQLite plugin
app.install(SQLitePlugin(dbfile='pool.db', keyword='db'))

# Create the Exchange wrapper objects
wrappers = {'bittrex': Bittrex(),
            'bter': BTER(),
            'ccedk': CCEDK(),
            'poloniex': Poloniex(),
            'test_exchange': TestExchange()}

log.info('load pool config')
app.config.load_config('pool_config')

log.info('load exchange config')
load_config.load(app, 'exchange_config')

log.info('set up a json-rpc connection with nud')
rpc = AuthServiceProxy("http://{}:{}@{}:{}".format(app.config['rpc.user'],
                                                   app.config['rpc.pass'],
                                                   app.config['rpc.host'],
                                                   app.config['rpc.port']))

def run_credit_timer():
    log.info('running credit timer')
    credit_timer = Timer(60.0, credit.credit,
                         kwargs={'app': app, 'rpc': rpc, 'log': log})
    credit_timer.name = 'credit_timer'
    credit_timer.daemon = True
    credit_timer.start()

def run_payout_timer():
    log.info('running payout timer')
    payout_timer = Timer(86400.0, payout.pay,
                         kwargs={'rpc': rpc, 'log': log})
    payout_timer.name = 'payout_timer'
    payout_timer.daemon = True
    payout_timer.start()

if os.getenv('RUN_TIMERS', '0') == '1':
    # Set the timer for credits
    run_credit_timer()
    # Set the timer for payouts
    run_payout_timer()

def check_headers(headers):
    """
    Ensure the correct headers get passed in the request
    :param headers:
    :return:
    """
    if headers.get('Content-Type') != 'application/json':
        log.warn('invalid header - need app/json')
        return False
    return True

@app.get('/')
def root():
    """
    Show a default page with info about the server health
    :return:
    """
    return {'success': True, 'message': 'ALP Server is operational'}

@app.post('/register')
def register(db):
    """
    Register a new user on the server
    Requires:
        user - API public Key
        address - Valid NBT payout address
        exchange - supported exchange
        unit - supported currency
    :return:
    """
    log.info('/register')
    # Check the content type
    if not check_headers(request.headers):
        return {'success': False, 'message': 'Content-Type header must be set to '
                                             '\'application/json\''}
    # Get the post parameters
    try:
        user = request.json.get('user')
        address = request.json.get('address')
        exchange = request.json.get('exchange')
        unit = request.json.get('unit')
    except AttributeError:
        log.warn('no json found in request')
        return {'success': False, 'message': 'no json found in request'}
    # check for missing parameters
    if not user:
        log.warn('no user provided')
        return {'success': False, 'message': 'no user provided'}
    if not address:
        log.warn('no address provided')
        return {'success': False, 'message': 'no address provided'}
    if not exchange:
        log.warn('no exchange provided')
        return {'success': False, 'message': 'no exchange provided'}
    if not unit:
        log.warn('no unit provided')
        return {'success': False, 'message': 'no unit provided'}
    # Check for a valid address
    if address[0] != 'B':
        log.warn('%s is not a valid NBT address. No \'B\'', address)
        return {'success': False, 'message': '{} is not a valid NBT address. It '
                                             'should start with a \'B\''.format(address)}
    address_check = AddressCheck()
    if not address_check.check_checksum(address):
        log.warn('%s is not a valid NBT address. Bad checksum', address)
        return {'success': False, 'message': '{} is not a valid NBT address. The '
                                             'checksum doesn\'t match'.format(address)}
    # Check that the requests exchange is supported by the server
    if exchange not in app.config['exchanges']:
        log.warn('%s is not supported', exchange)
        return {'success': False, 'message': '{} is not supported'.format(exchange)}
    # Check that the unit is supported on the server
    if unit not in app.config['{}.units'.format(exchange)]:
        log.warn('%s is not supported on %s', unit, exchange)
        return {'success': False, 'message': '{} is not supported on {}'.format(unit,
                                                                                exchange)}
    # Check if the user already exists in the database
    check = db.execute("SELECT id FROM users WHERE user=? AND address=? AND "
                       "exchange=? AND unit=?;", (user, address, exchange,
                                                  unit)).fetchone()
    if check:
        log.warn('user is already registered')
        return {'success': False, 'message': 'user is already registered'}
    db.execute("INSERT INTO users ('user','address','exchange','unit') VALUES (?,?,?,?)",
               (user, address, exchange, unit))
    log.info('user %s successfully registered', user)
    return {'success': True, 'message': 'user successfully registered'}

@app.post('/liquidity')
def liquidity(db):
    """
    Allow the user to submit liquidity validations to the server
    Will be multiple times per minute
    With the submitted data get the users orders and update the database accordingly
    Requires:
        user - API public Key
        req - dictionary to be used to get open orders
        sign - result of signing req with API private key
        exchange - valid, supported exchange
        unit - valid, supported unit
    :return:
    """
    log.info('/liquidity')
    # Check the content type
    if not check_headers(request.headers):
        return {'success': False, 'message': 'Content-Type header must be set to '
                                             '\'application/json\''}
    # Get the post parameters
    try:
        user = request.json.get('user')
        sign = request.json.get('sign')
        exchange = request.json.get('exchange')
        unit = request.json.get('unit')
        req = request.json.get('req')
    except AttributeError:
        log.warn('no json found in request')
        return {'success': False, 'message': 'no json found in request'}
    if not user:
        log.warn('no user provided')
        return {'success': False, 'message': 'no user provided'}
    if not sign:
        log.warn('no sign provided')
        return {'success': False, 'message': 'no sign provided'}
    if not exchange:
        log.warn('no exchange provided')
        return {'success': False, 'message': 'no exchange provided'}
    if not unit:
        log.warn('no unit provided')
        return {'success': False, 'message': 'no unit provided'}
    if not req:
        log.warn('no req provided')
        return {'success': False, 'message': 'no req provided'}
    # use the submitted data to request the users orders
    orders = wrappers[exchange].validate_request(user, unit, req, sign)
    price = get_price()
    # clear existing orders for the user
    log.info('clear existing orders for user %s', user)
    db.execute("DELETE FROM orders WHERE user=? AND exchange=? AND unit=?", (user,
                                                                             exchange,
                                                                             unit))
    # Loop through the orders
    for order in orders:
        # Calculate how the order price is from the known good price
        order_deviation = 1.0 - (min(order['price'], price) / max(order['price'], price))
        # Using the server tolerance determine if the order is tier 1 or tier 2
        # Only tier 1 is compensated by the server
        tier = 'tier_2'
        if order_deviation <= app.config['{}.{}.{}.tolerance'.format(exchange, unit,
                                                                     order['type'])]:
            tier = 'tier_1'
        # save the order details
        db.execute("INSERT INTO orders ('user','tier','order_id','order_amount',"
                   "'order_type','exchange','unit') VALUES (?,?,?,?,?,?,?)",
                   (user, tier, str(order['id']), float(order['amount']),
                    str(order['type']), exchange, unit))
    log.info('user %s orders saved for validation', user)
    return {'success': True, 'message': 'orders saved for validation'}

@app.get('/exchanges')
def exchanges():
    """
    Show the app.config as it pertains to the supported exchanges
    :return:
    """
    log.info('/exchanges')
    data = {}
    for exchange in app.config['exchanges']:
        if exchange not in data:
            data[exchange] = {}
        for unit in app.config['{}.units'.format(exchange)]:
            data[exchange][unit] = {'ask': {'tolerance': app.config['{}.{}.ask.'
                                                                    'tolerance'
                                                                    ''.format(exchange,
                                                                              unit)],
                                            'tier_1': {
                                                'reward': app.config['{}.{}.ask.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_1')]
                                                },
                                            'tier_2': {
                                                'reward': app.config['{}.{}.ask.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_2')]
                                                }
                                            },

                                    'bid': {'tolerance': app.config['{}.{}.bid.'
                                                                    'tolerance'
                                                                    ''.format(exchange,
                                                                              unit)],
                                            'tier_1': {
                                                'reward': app.config['{}.{}.bid.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_1')]
                                                },
                                            'tier_2': {
                                                'reward': app.config['{}.{}.bid.{}'
                                                                     '.reward'
                                                                     ''.format(exchange,
                                                                               unit,
                                                                               'tier_2')]
                                                }
                                            }
                                    }
    return data

@app.get('/status')
def status(db):
    """
    Display the overall pool status.
    This will be the number of users and the amount of liquidity for each tier, side,
    unit, exchange for the last full round and the current round
    :param db:
    :return:

    We want to display

    * Total liquidity provided by pool
    * Total tier_1 by pool
    * Total tier_2 by pool
    * total liquidity by exchange
    * total tier_1 by exchange
    * total tier_2 by exchange
    * total liquidity by exchange/pair
    * total tier_1 by exchange/pair
    * total tier_2 by exchange/pair
    * total liquidity by exchange/pair/side
    * total tier_1 by exchange/pair/side
    * total tier_2 by exchange/pair/side

    * number of users
    * number of active users

    * amount 1 NBT will be rewarded currently

    """
    log.info('/status')

    # get the last credit time
    last_credit_time = db.execute("SELECT value FROM info WHERE key=?",
                                  'last_credit_time').fetchone()

    # build the blank data object
    data = {'last_credit_time': last_credit_time,
            'total_liquidity': 0.0,
            'total_liquidity_bid': 0.0,
            'total_liquidity_ask': 0.0,
            'total_liquidity_tier_1': 0.0,
            'total_liquidity_tier_1_bid': 0.0,
            'total_liquidity_tier_1_ask': 0.0,
            'total_liquidity_tier_2': 0.0,
            'total_liquidity_tier_2_bid': 0.0,
            'total_liquidity_tier_2_ask': 0.0}

    # get the latest credit data from the credits field
    credit_data = db.execute("SELECT * FROM credits WHERE time=?",
                             last_credit_time).fetchall()
    log.debug(credit_data)
    # parse the data
    # id INTEGER PRIMARY KEY, time NUMBER, user TEXT, exchange TEXT, unit TEXT,
    # tier TEXT, side TEXT, provided NUMBER, total NUMBER, percentage NUMBER,
    # reward NUMBER, paid INTEGER
    for cred in credit_data:
        # increment the total liquidity (this is total over entire pool)
        data['total_liquidity'] += cred[7]
        # increment buy and sell side totals
        if cred[6] == 'bid':
            data['total_liquidity_bid'] += cred[7]
        else:
            data['total_liquidity_ask'] += cred[7]
        # increment tier_1 totals
        if cred[5] == 'tier_1':
            data['total_liquidity_tier_1'] += cred[7]
            if cred[6] == 'bid':
                data['total_liquidity_tier_1_bid'] += cred[7]
            else:
                data['total_liquidity_tier_1_bid'] += cred[7]
        # increment tier_2 totals
        if cred[5] == 'tier_2':
            data['total_liquidity_tier_2'] += cred[7]
            if cred[6] == 'bid':
                data['total_liquidity_tier_2_bid'] += cred[7]
            else:
                data['total_liquidity_tier_2_ask'] += cred[7]

    return {'status': True, 'message': data}

def get_price():
    """
    Set the server price primarily from the NuBot streaming server but falling back to
    feeds if that fails
    :return:
    """
    return 1234

@app.error(code=500)
def error500(error):
    return json.dumps({'success': False, 'message': '500 error'})

@app.error(code=502)
def error502(error):
    return json.dumps({'success': False, 'message': '502 error'})

@app.error(code=503)
def error503(error):
    return json.dumps({'success': False, 'message': '503 error'})

@app.error(code=404)
def error404(error):
    return json.dumps({'success': False, 'message': '404 {} not found'
                                                    ''.format(request.url)})

@app.error(code=405)
def error405(error):
    return json.dumps({'success': False, 'message': '405 error. '
                                                    'Incorrect HTTP method used'})

if __name__ == '__main__':
    # Run the server
    run(server, host='localhost', port=int(os.environ.get("PORT", 3333)), debug=True)
<|endoftext|>


<commit_before>from __future__ import print_function
from distutils import ccompiler
from distutils import sysconfig
import os
from os import path
import sys

import pkg_resources
import setuptools

from install import build
from install import utils

required_cython_version = pkg_resources.parse_version('0.24.0')

MODULES = [
    {
        'name': 'cuda',
        'file': [
            'cupy.core.core',
            'cupy.core.flags',
            'cupy.core.internal',
            'cupy.cuda.cublas',
            'cupy.cuda.curand',
            'cupy.cuda.device',
            'cupy.cuda.driver',
            'cupy.cuda.memory',
            'cupy.cuda.pinned_memory',
            'cupy.cuda.profiler',
            'cupy.cuda.nvtx',
            'cupy.cuda.function',
            'cupy.cuda.runtime',
            'cupy.util',
        ],
        'include': [
            'cublas_v2.h',
            'cuda.h',
            'cuda_profiler_api.h',
            'cuda_runtime.h',
            'curand.h',
            'nvToolsExt.h',
        ],
        'libraries': [
            'cublas',
            'cuda',
            'cudart',
            'curand',
            'nvToolsExt',
        ],
        'check_method': build.check_cuda_version,
    },
    {
        'name': 'cudnn',
        'file': [
            'cupy.cuda.cudnn',
        ],
        'include': [
            'cudnn.h',
        ],
        'libraries': [
            'cudnn',
        ],
        'check_method': build.check_cudnn_version,
    },
    {
        'name': 'cusolver',
        'file': [
            'cupy.cuda.cusolver',
        ],
        'include': [
            'cusolverDn.h',
        ],
        'libraries': [
            'cusolver',
        ],
        'check_method': build.check_cusolver_version,
    }
]

if sys.platform == 'win32':
    mod_cuda = MODULES[0]
    mod_cuda['libraries'].remove('nvToolsExt')
    if utils.search_on_path(['nvToolsExt64_1.dll']) is None:
        mod_cuda['file'].remove('cupy.cuda.nvtx')
        mod_cuda['include'].remove('nvToolsExt.h')
        utils.print_warning(
            'Cannot find nvToolsExt. nvtx was disabled.')
    else:
        mod_cuda['libraries'].append('nvToolsExt64_1')

def check_readthedocs_environment():
    return os.environ.get('READTHEDOCS', None) == 'True'

def check_library(compiler, includes=(), libraries=(),
                  include_dirs=(), library_dirs=()):

    source = ''.join(['#include <%s>\n' % header for header in includes])
    source += 'int main(int argc, char* argv[]) {return 0;}'
    try:
        # We need to try to build a shared library because distutils
        # uses different option to build an executable and a shared library.
        # Especially when a user build an executable, distutils does not use
        # LDFLAGS environment variable.
        build.build_shlib(compiler, source, libraries,
                          include_dirs, library_dirs)
    except Exception as e:
        print(e)
        return False
    return True

def make_extensions(options, compiler, use_cython):
    """Produce a list of Extension instances which passed to cythonize()."""

    no_cuda = options['no_cuda']
    settings = build.get_compiler_setting()

    include_dirs = settings['include_dirs']

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]
    if sys.platform != 'win32':
        settings['runtime_library_dirs'] = settings['library_dirs']
    if sys.platform == 'darwin':
        args = settings.setdefault('extra_link_args', [])
        args.append(
            '-Wl,' + ','.join('-rpath,' + p
                              for p in settings['library_dirs']))
        # -rpath is only supported when targetting Mac OS X 10.5 or later
        args.append('-mmacosx-version-min=10.5')

    # This is a workaround for Anaconda.
    # Anaconda installs libstdc++ from GCC 4.8 and it is not compatible
    # with GCC 5's new ABI.
    settings['define_macros'].append(('_GLIBCXX_USE_CXX11_ABI', '0'))

    if options['linetrace']:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))

    ret = []
    ext = '.pyx' if use_cython else '.cpp'
    for module in MODULES:
        print('Include directories:', settings['include_dirs'])
        print('Library directories:', settings['library_dirs'])

        if not no_cuda:
            if not check_library(compiler,
                                 includes=module['include'],
                                 include_dirs=settings['include_dirs']):
                utils.print_warning(
                    'Include files not found: %s' % module['include'],
                    'Skip installing %s support' % module['name'],
                    'Check your CFLAGS environment variable')
                continue

            if not check_library(compiler,
                                 libraries=module['libraries'],
                                 library_dirs=settings['library_dirs']):
                utils.print_warning(
                    'Cannot link libraries: %s' % module['libraries'],
                    'Skip installing %s support' % module['name'],
                    'Check your LDFLAGS environment variable')
                continue

            if 'check_method' in module and \
               not module['check_method'](compiler, settings):
                continue

        s = settings.copy()
        if not no_cuda:
            s['libraries'] = module['libraries']

        ret.extend([
            setuptools.Extension(f, [path.join(*f.split('.')) + ext], **s)
            for f in module['file']])
    return ret

def parse_args():
    cupy_profile = '--cupy-profile' in sys.argv
    if cupy_profile:
        sys.argv.remove('--cupy-profile')
    cupy_coverage = '--cupy-coverage' in sys.argv
    if cupy_coverage:
        sys.argv.remove('--cupy-coverage')
    no_cuda = '--cupy-no-cuda' in sys.argv
    if no_cuda:
        sys.argv.remove('--cupy-no-cuda')

    arg_options = {
        'profile': cupy_profile,
        'linetrace': cupy_coverage,
        'annotate': cupy_coverage,
        'no_cuda': no_cuda,
    }
    if check_readthedocs_environment():
        arg_options['no_cuda'] = True
    return arg_options

def check_cython_version():
    try:
        import Cython
        cython_version = pkg_resources.parse_version(Cython.__version__)
        return cython_version >= required_cython_version
    except ImportError:
        return False

def cythonize(extensions, arg_options):
    import Cython.Build

    directive_keys = ('linetrace', 'profile')
    directives = {key: arg_options[key] for key in directive_keys}

    cythonize_option_keys = ('annotate',)
    cythonize_options = {key: arg_options[key]
                         for key in cythonize_option_keys}

    return Cython.Build.cythonize(
        extensions, verbose=True,
        compiler_directives=directives, **cythonize_options)

def check_extensions(extensions):
    for x in extensions:
        for f in x.sources:
            if not path.isfile(f):
                msg = ('Missing file: %s\n' % f +
                       'Please install Cython. ' +
                       'Please also check the version of Cython.\n' +
                       'See http://docs.chainer.org/en/stable/install.html')
                raise RuntimeError(msg)

def get_ext_modules():
    arg_options = parse_args()
    print('Options:', arg_options)

    # We need to call get_config_vars to initialize _config_vars in distutils
    # see #1849
    sysconfig.get_config_vars()
    compiler = ccompiler.new_compiler()
    sysconfig.customize_compiler(compiler)

    use_cython = check_cython_version()
    extensions = make_extensions(arg_options, compiler, use_cython)

    if use_cython:
        extensions = cythonize(extensions, arg_options)

    check_extensions(extensions)
    return extensions
<commit_msg>Add dependency to opnemp for cusolver<commit_after>from __future__ import print_function
from distutils import ccompiler
from distutils import sysconfig
import os
from os import path
import sys

import pkg_resources
import setuptools

from install import build
from install import utils

required_cython_version = pkg_resources.parse_version('0.24.0')

MODULES = [
    {
        'name': 'cuda',
        'file': [
            'cupy.core.core',
            'cupy.core.flags',
            'cupy.core.internal',
            'cupy.cuda.cublas',
            'cupy.cuda.curand',
            'cupy.cuda.device',
            'cupy.cuda.driver',
            'cupy.cuda.memory',
            'cupy.cuda.pinned_memory',
            'cupy.cuda.profiler',
            'cupy.cuda.nvtx',
            'cupy.cuda.function',
            'cupy.cuda.runtime',
            'cupy.util',
        ],
        'include': [
            'cublas_v2.h',
            'cuda.h',
            'cuda_profiler_api.h',
            'cuda_runtime.h',
            'curand.h',
            'nvToolsExt.h',
        ],
        'libraries': [
            'cublas',
            'cuda',
            'cudart',
            'curand',
            'nvToolsExt',
        ],
        'check_method': build.check_cuda_version,
    },
    {
        'name': 'cudnn',
        'file': [
            'cupy.cuda.cudnn',
        ],
        'include': [
            'cudnn.h',
        ],
        'libraries': [
            'cudnn',
        ],
        'check_method': build.check_cudnn_version,
    },
    {
        'name': 'cusolver',
        'file': [
            'cupy.cuda.cusolver',
        ],
        'include': [
            'cusolverDn.h',
        ],
        'libraries': [
            'cusolver',
        ],
        'check_method': build.check_cusolver_version,
    }
]

if sys.platform == 'win32':
    mod_cuda = MODULES[0]
    mod_cuda['libraries'].remove('nvToolsExt')
    if utils.search_on_path(['nvToolsExt64_1.dll']) is None:
        mod_cuda['file'].remove('cupy.cuda.nvtx')
        mod_cuda['include'].remove('nvToolsExt.h')
        utils.print_warning(
            'Cannot find nvToolsExt. nvtx was disabled.')
    else:
        mod_cuda['libraries'].append('nvToolsExt64_1')

def check_readthedocs_environment():
    return os.environ.get('READTHEDOCS', None) == 'True'

def check_library(compiler, includes=(), libraries=(),
                  include_dirs=(), library_dirs=()):

    source = ''.join(['#include <%s>\n' % header for header in includes])
    source += 'int main(int argc, char* argv[]) {return 0;}'
    try:
        # We need to try to build a shared library because distutils
        # uses different option to build an executable and a shared library.
        # Especially when a user build an executable, distutils does not use
        # LDFLAGS environment variable.
        build.build_shlib(compiler, source, libraries,
                          include_dirs, library_dirs)
    except Exception as e:
        print(e)
        return False
    return True

def make_extensions(options, compiler, use_cython):
    """Produce a list of Extension instances which passed to cythonize()."""

    no_cuda = options['no_cuda']
    settings = build.get_compiler_setting()

    include_dirs = settings['include_dirs']

    settings['include_dirs'] = [
        x for x in include_dirs if path.exists(x)]
    settings['library_dirs'] = [
        x for x in settings['library_dirs'] if path.exists(x)]
    if sys.platform != 'win32':
        settings['runtime_library_dirs'] = settings['library_dirs']
    if sys.platform == 'darwin':
        args = settings.setdefault('extra_link_args', [])
        args.append(
            '-Wl,' + ','.join('-rpath,' + p
                              for p in settings['library_dirs']))
        # -rpath is only supported when targetting Mac OS X 10.5 or later
        args.append('-mmacosx-version-min=10.5')

    # This is a workaround for Anaconda.
    # Anaconda installs libstdc++ from GCC 4.8 and it is not compatible
    # with GCC 5's new ABI.
    settings['define_macros'].append(('_GLIBCXX_USE_CXX11_ABI', '0'))

    if options['linetrace']:
        settings['define_macros'].append(('CYTHON_TRACE', '1'))
        settings['define_macros'].append(('CYTHON_TRACE_NOGIL', '1'))
    if no_cuda:
        settings['define_macros'].append(('CUPY_NO_CUDA', '1'))

    ret = []
    ext = '.pyx' if use_cython else '.cpp'
    for module in MODULES:
        print('Include directories:', settings['include_dirs'])
        print('Library directories:', settings['library_dirs'])

        if not no_cuda:
            if not check_library(compiler,
                                 includes=module['include'],
                                 include_dirs=settings['include_dirs']):
                utils.print_warning(
                    'Include files not found: %s' % module['include'],
                    'Skip installing %s support' % module['name'],
                    'Check your CFLAGS environment variable')
                continue

            if not check_library(compiler,
                                 libraries=module['libraries'],
                                 library_dirs=settings['library_dirs']):
                utils.print_warning(
                    'Cannot link libraries: %s' % module['libraries'],
                    'Skip installing %s support' % module['name'],
                    'Check your LDFLAGS environment variable')
                continue

            if 'check_method' in module and \
               not module['check_method'](compiler, settings):
                continue

        s = settings.copy()
        if not no_cuda:
            s['libraries'] = module['libraries']

        if module['name'] == 'cusolver':
            args = s.setdefault('extra_link_args', [])
            # openmp is required for cusolver
            if compiler.compiler_type == 'unix' and sys.platform != 'darwin':
                # In mac environment, openmp is not required.
                args.append('-fopenmp')
            elif compiler.compiler_type == 'msvc':
                args.append('/openmp')

        ret.extend([
            setuptools.Extension(f, [path.join(*f.split('.')) + ext], **s)
            for f in module['file']])
    return ret

def parse_args():
    cupy_profile = '--cupy-profile' in sys.argv
    if cupy_profile:
        sys.argv.remove('--cupy-profile')
    cupy_coverage = '--cupy-coverage' in sys.argv
    if cupy_coverage:
        sys.argv.remove('--cupy-coverage')
    no_cuda = '--cupy-no-cuda' in sys.argv
    if no_cuda:
        sys.argv.remove('--cupy-no-cuda')

    arg_options = {
        'profile': cupy_profile,
        'linetrace': cupy_coverage,
        'annotate': cupy_coverage,
        'no_cuda': no_cuda,
    }
    if check_readthedocs_environment():
        arg_options['no_cuda'] = True
    return arg_options

def check_cython_version():
    try:
        import Cython
        cython_version = pkg_resources.parse_version(Cython.__version__)
        return cython_version >= required_cython_version
    except ImportError:
        return False

def cythonize(extensions, arg_options):
    import Cython.Build

    directive_keys = ('linetrace', 'profile')
    directives = {key: arg_options[key] for key in directive_keys}

    cythonize_option_keys = ('annotate',)
    cythonize_options = {key: arg_options[key]
                         for key in cythonize_option_keys}

    return Cython.Build.cythonize(
        extensions, verbose=True,
        compiler_directives=directives, **cythonize_options)

def check_extensions(extensions):
    for x in extensions:
        for f in x.sources:
            if not path.isfile(f):
                msg = ('Missing file: %s\n' % f +
                       'Please install Cython. ' +
                       'Please also check the version of Cython.\n' +
                       'See http://docs.chainer.org/en/stable/install.html')
                raise RuntimeError(msg)

def get_ext_modules():
    arg_options = parse_args()
    print('Options:', arg_options)

    # We need to call get_config_vars to initialize _config_vars in distutils
    # see #1849
    sysconfig.get_config_vars()
    compiler = ccompiler.new_compiler()
    sysconfig.customize_compiler(compiler)

    use_cython = check_cython_version()
    extensions = make_extensions(arg_options, compiler, use_cython)

    if use_cython:
        extensions = cythonize(extensions, arg_options)

    check_extensions(extensions)
    return extensions
<|endoftext|>