<commit_before># -*- coding: utf-8 -*-
"""
=================================
pyglmnet for Poisson distribution
=================================

This is an example demonstrating how pyglmnet works.

"""

# Author: Pavan Ramkumar
# License: MIT

import numpy as np
import scipy.sparse as sps
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt

########################################################
# Here are inputs that you can provide when you instantiate the `GLM` class.
# If not provided, it will be set to the respective defaults
#
# - `distr`: str, `'poisson'` or `'normal'` or `'binomial'` or `'multinomial'`
#     default: `'poisson'`
# - `alpha`: float, the weighting between L1 and L2 norm, default: 0.5
# - `reg_lambda`: array, array of regularized parameters,
#     default: `np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))`
# - `learning_rate`: float, learning rate for gradient descent,
#     default: 1e-4
# - `max_iter`: int, maximum iteration for the model, default: 100

########################################################

# import GLM model
from pyglmnet import GLM

# create regularize parameters for model
reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))
model = GLM(distr='poisson', verbose=False, alpha=0.05,
            max_iter=1000, learning_rate=1e-4,
            reg_lambda=reg_lambda)

##########################################################
# Simulate a dataset
# ------------------
# The ``GLM`` class has a very useful method called ``simulate()``.
#
# Since a canonical link function is already specified by the distribution
# parameters, or provided by the user, ``simulate()`` requires
# only the independent variables ``X`` and the coefficients ``beta0``
# and ``beta``

##########################################################

n_samples, n_features = 10000, 100

# coefficients
beta0 = np.random.normal(0.0, 1.0, 1)
beta = sps.rand(n_features, 1, 0.1)
beta = np.array(beta.todense())

# training data
Xr = np.random.normal(0.0, 1.0, [n_samples, n_features])
yr = model.simulate(beta0, beta, Xr)

# testing data
Xt = np.random.normal(0.0, 1.0, [n_samples, n_features])
yt = model.simulate(beta0, beta, Xt)

##########################################################
# Fit the model
# ^^^^^^^^^^^^^
# Fitting the model is accomplished by a single GLM method called `fit()`.
# You can provide data and output pair `(X, y)` i.e.

##########################################################

scaler = StandardScaler().fit(Xr)
model.fit(scaler.transform(Xr), yr)

##########################################################
# Visualize the fit coefficients
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# The estimated coefficients are stored in an instance variable called ``.fit_``
# which is a list of dictionaries. Each dictionary corresponds to a
# particular ``reg_lambda``

##########################################################

fit_param = model[0].fit_
plt.plot(beta[:], 'bo')
plt.hold(True)
plt.plot(fit_param['beta'][:], 'ro')
plt.show()

##########################################################
# Slicing the model object
# ^^^^^^^^^^^^^^^^^^^^^^^^
# Although the model is fit to all values of reg_lambda specified by a regularization
# path, often we are only interested in further analysis for a particular value of
# ``reg_lambda``. We can easily do this by slicing the object.
#
# For instance model[0] returns an object identical to model but with ``.fit_``
# as a dictionary corresponding to the estimated coefficients for ``reg_lambda[0]``.

##########################################################
# Make predictions based on fit model
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# The ``predict()`` method takes two parameters: a numpy 2d array of independent
# variables and a dictionary of fit parameters. It returns a vector of
# predicted targets.

##########################################################

# Predict targets from test set
yrhat = model[0].predict(scaler.transform(Xr))
ythat = model[0].predict(scaler.transform(Xt))

plt.plot(yt[:100])
plt.hold(True)
plt.plot(ythat[:100], 'r')
plt.xlabel('samples')
plt.ylabel('true and predicted outputs')
plt.show()

##########################################################
# Goodness of fit
# ^^^^^^^^^^^^^^^
# The GLM class provides two methods for evaluating goodness of fit: ``deviance()``
# and ``pseudo_R2()``. Both of them require the true targets and the predicted targets
# as inputs. ``pseudo_R2()`` additionally requires a null model, which is typically
# the mean of the target variables in the training set.

##########################################################

# Compute model deviance
Dr = model[0].deviance(yr, yrhat)
Dt = model[0].deviance(yt, ythat)
print(Dr, Dt)

# Compute pseudo-R2s
R2r = model[0].pseudo_R2(yr, yrhat, np.mean(yr))
R2t = model[0].pseudo_R2(yt, ythat, np.mean(yr))
print(R2r, R2t)

##########################################################
# perform bootstrap
# here you need ``scikits.bootstrap`` to perform the following line
# you can install using: ``pip install scikits.bootstrap``

##########################################################

import scikits.bootstrap as boot
ci = boot.ci((yr, yrhat, np.mean(yr) * np.ones(np.shape(yr)[0])),
              statfunction=model.pseudo_R2,
              n_samples=100, alpha=0.05)
print(ci)

##########################################################
# Multinomial example
# ^^^^^^^^^^^^^^^^^^^
# we can also use ``pyglmnet`` with multinomial case
# where you can provide array of class

##########################################################

from sklearn.datasets import make_classification
from pyglmnet import GLM
X, y = make_classification(n_samples=10000, n_classes=5,
                           n_informative=100, n_features=100, n_redundant=0)

model_mn = GLM(distr='multinomial', alpha=0.01,
               reg_lambda=np.array([0.02, 0.01]), verbose=False)
model_mn.threshold = 1e-5
model_mn.fit(X, y)
y_pred = model_mn[-1].predict(X).argmax(axis=1)
print('Output performance = %f percent' % (y_pred == y).mean())
<commit_msg>remove bootstrap bubble<commit_after># -*- coding: utf-8 -*-
"""
=================================
pyglmnet for Poisson distribution
=================================

This is an example demonstrating how pyglmnet works.

"""

# Author: Pavan Ramkumar
# License: MIT

import numpy as np
import scipy.sparse as sps
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt

########################################################
# Here are inputs that you can provide when you instantiate the `GLM` class.
# If not provided, it will be set to the respective defaults
#
# - `distr`: str, `'poisson'` or `'normal'` or `'binomial'` or `'multinomial'`
#     default: `'poisson'`
# - `alpha`: float, the weighting between L1 and L2 norm, default: 0.5
# - `reg_lambda`: array, array of regularized parameters,
#     default: `np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))`
# - `learning_rate`: float, learning rate for gradient descent,
#     default: 1e-4
# - `max_iter`: int, maximum iteration for the model, default: 100

########################################################

# import GLM model
from pyglmnet import GLM

# create regularize parameters for model
reg_lambda = np.logspace(np.log(0.5), np.log(0.01), 10, base=np.exp(1))
model = GLM(distr='poisson', verbose=False, alpha=0.05,
            max_iter=1000, learning_rate=1e-4,
            reg_lambda=reg_lambda)

##########################################################
# Simulate a dataset
# ------------------
# The ``GLM`` class has a very useful method called ``simulate()``.
#
# Since a canonical link function is already specified by the distribution
# parameters, or provided by the user, ``simulate()`` requires
# only the independent variables ``X`` and the coefficients ``beta0``
# and ``beta``

##########################################################

n_samples, n_features = 10000, 100

# coefficients
beta0 = np.random.normal(0.0, 1.0, 1)
beta = sps.rand(n_features, 1, 0.1)
beta = np.array(beta.todense())

# training data
Xr = np.random.normal(0.0, 1.0, [n_samples, n_features])
yr = model.simulate(beta0, beta, Xr)

# testing data
Xt = np.random.normal(0.0, 1.0, [n_samples, n_features])
yt = model.simulate(beta0, beta, Xt)

##########################################################
# Fit the model
# ^^^^^^^^^^^^^
# Fitting the model is accomplished by a single GLM method called `fit()`.
# You can provide data and output pair `(X, y)` i.e.

##########################################################

scaler = StandardScaler().fit(Xr)
model.fit(scaler.transform(Xr), yr)

##########################################################
# Visualize the fit coefficients
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# The estimated coefficients are stored in an instance variable called ``.fit_``
# which is a list of dictionaries. Each dictionary corresponds to a
# particular ``reg_lambda``

##########################################################

fit_param = model[0].fit_
plt.plot(beta[:], 'bo')
plt.hold(True)
plt.plot(fit_param['beta'][:], 'ro')
plt.show()

##########################################################
# Slicing the model object
# ^^^^^^^^^^^^^^^^^^^^^^^^
# Although the model is fit to all values of reg_lambda specified by a regularization
# path, often we are only interested in further analysis for a particular value of
# ``reg_lambda``. We can easily do this by slicing the object.
#
# For instance model[0] returns an object identical to model but with ``.fit_``
# as a dictionary corresponding to the estimated coefficients for ``reg_lambda[0]``.

##########################################################
# Make predictions based on fit model
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# The ``predict()`` method takes two parameters: a numpy 2d array of independent
# variables and a dictionary of fit parameters. It returns a vector of
# predicted targets.

##########################################################

# Predict targets from test set
yrhat = model[0].predict(scaler.transform(Xr))
ythat = model[0].predict(scaler.transform(Xt))

plt.plot(yt[:100])
plt.hold(True)
plt.plot(ythat[:100], 'r')
plt.xlabel('samples')
plt.ylabel('true and predicted outputs')
plt.show()

##########################################################
# Goodness of fit
# ^^^^^^^^^^^^^^^
# The GLM class provides two methods for evaluating goodness of fit: ``deviance()``
# and ``pseudo_R2()``. Both of them require the true targets and the predicted targets
# as inputs. ``pseudo_R2()`` additionally requires a null model, which is typically
# the mean of the target variables in the training set.

##########################################################

# Compute model deviance
Dr = model[0].deviance(yr, yrhat)
Dt = model[0].deviance(yt, ythat)
print(Dr, Dt)

# Compute pseudo-R2s
R2r = model[0].pseudo_R2(yr, yrhat, np.mean(yr))
R2t = model[0].pseudo_R2(yt, ythat, np.mean(yr))
print(R2r, R2t)

##########################################################
# Multinomial example
# ^^^^^^^^^^^^^^^^^^^
# we can also use ``pyglmnet`` with multinomial case
# where you can provide array of class

##########################################################

from sklearn.datasets import make_classification
X, y = make_classification(n_samples=10000, n_classes=5,
                           n_informative=100, n_features=100, n_redundant=0)

model_mn = GLM(distr='multinomial', alpha=0.01,
               reg_lambda=np.array([0.02, 0.01]), verbose=False)
model_mn.threshold = 1e-5
model_mn.fit(X, y)
y_pred = model_mn[-1].predict(X).argmax(axis=1)
print('Output performance = %f percent' % (y_pred == y).mean())
<|endoftext|>


<commit_before>import logging

from couchdbkit.ext.django.loading import get_db

from dimagi.utils.couch.database import retry_on_couch_error
from dimagi.utils.dates import force_to_datetime
from dimagi.utils.logging import notify_exception

from corehq.apps.auditcare.models import (
    ACCESS_FAILED,
    ACCESS_LOGIN,
    ACCESS_LOGOUT,
    AccessAudit,
    NavigationEventAudit,
)
from corehq.apps.auditcare.utils.migration import (
    AuditCareMigrationUtil,
    get_formatted_datetime_string,
)
from corehq.util.soft_assert import soft_assert

log = logging.getLogger(__name__)

ACCESS_LOOKUP = {
    "login": ACCESS_LOGIN,
    "logout": ACCESS_LOGOUT,
    "login_failed": ACCESS_FAILED,
}

COUCH_QUERY_LIMIT = 1000

def get_couch_key(time):
    if not time:
        return
    return [time.year, time.month, time.day, time.hour, time.minute, time.second]

def get_migration_key(start_time, end_time):
    return get_formatted_datetime_string(start_time) + '_' + get_formatted_datetime_string(end_time)

@retry_on_couch_error
def _get_couch_docs(start_key, end_key):
    db = get_db("auditcare")
    result = db.view(
        "auditcare/all_events",
        startkey=start_key,
        endkey=end_key,
        reduce=False,
        include_docs=True,
        limit=COUCH_QUERY_LIMIT
    )
    return list(result)

def get_events_from_couch(start_key, end_key):
    navigation_objects = []
    audit_objects = []
    count = 0
    next_start_time = None
    couch_docs = _get_couch_docs(start_key, end_key)
    for result in couch_docs:
        doc = result["doc"]
        try:
            next_start_time = force_to_datetime(doc.get("event_date"))
            kwargs = _pick(doc, ["user", "domain", "ip_address", "session_key",
                                "headers", "status_code", "user_agent"])
            kwargs.update({
                "event_date": next_start_time,
                "couch_id": doc["_id"],
            })

            if doc["doc_type"] == "NavigationEventAudit":
                if NavigationEventAudit.objects.filter(couch_id=doc["_id"]).exists():
                    continue
                kwargs.update(_pick(doc, ["headers", "status_code", "view", "view_kwargs"]))
                path, _, params = doc.get("request_path", "").partition("?")
                kwargs.update({
                    "path": path,
                    "params": params,
                })
                navigation_objects.append(NavigationEventAudit(**kwargs))
            elif doc["doc_type"] == "AccessAudit":
                if AccessAudit.objects.filter(couch_id=doc["_id"]).exists():
                    continue
                kwargs.update(_pick(doc, ["http_accept", "trace_id"]))
                access_type = doc.get('access_type')
                kwargs.update({
                    "access_type": ACCESS_LOOKUP.get(doc.get("access_type")),
                    "path": doc.get("path_info"),
                })
                if access_type == "logout":
                    kwargs.update({"path": "accounts/logout"})
                audit_objects.append(AccessAudit(**kwargs))
            count += 1
        except Exception:
            raise Exception(doc['_id'])
    res_obj = {
        "navigation_events": navigation_objects,
        "audit_events": audit_objects,
        "break_query": count < COUCH_QUERY_LIMIT or not next_start_time,
        "next_start_key": get_couch_key(next_start_time),
        "count": count
    }
    return res_obj

def copy_events_to_sql(start_time, end_time):
    util = AuditCareMigrationUtil()
    print(f"Starting batch: {start_time} - {end_time}")
    key = get_migration_key(start_time, end_time)
    end_key = get_couch_key(end_time)
    start_key = get_couch_key(start_time)
    next_start_key = start_key
    util.log_batch_start(key)
    break_query = False
    count = 0
    try:
        while not break_query:
            events_info = get_events_from_couch(next_start_key, end_key)
            next_start_key = events_info['next_start_key']
            NavigationEventAudit.objects.bulk_create(events_info['navigation_events'], ignore_conflicts=True)
            AccessAudit.objects.bulk_create(events_info['audit_events'], ignore_conflicts=True)
            count += events_info['count']
            break_query = events_info['break_query']
    except Exception as e:
        message = f"Error in copy_events_to_sql on doc {str(e)}"
        notify_exception(None, message=message)
        _soft_assert = soft_assert(to="{}@{}.com".format('aphulera', 'dimagi'), notify_admins=False)
        _soft_assert(False, message)
        util.set_batch_as_errored(key)
        return
    util.set_batch_as_finished(key, count)

def _pick(doc, keys):
    return {key: doc.get(key) for key in keys if doc.get(key)}
<commit_msg>fix edge where last records were skipped if batch contains already saved elements<commit_after>import logging

from couchdbkit.ext.django.loading import get_db

from dimagi.utils.couch.database import retry_on_couch_error
from dimagi.utils.dates import force_to_datetime
from dimagi.utils.logging import notify_exception

from corehq.apps.auditcare.models import (
    ACCESS_FAILED,
    ACCESS_LOGIN,
    ACCESS_LOGOUT,
    AccessAudit,
    NavigationEventAudit,
)
from corehq.apps.auditcare.utils.migration import (
    AuditCareMigrationUtil,
    get_formatted_datetime_string,
)
from corehq.util.soft_assert import soft_assert

log = logging.getLogger(__name__)

ACCESS_LOOKUP = {
    "login": ACCESS_LOGIN,
    "logout": ACCESS_LOGOUT,
    "login_failed": ACCESS_FAILED,
}

COUCH_QUERY_LIMIT = 1000

def get_couch_key(time):
    if not time:
        return
    return [time.year, time.month, time.day, time.hour, time.minute, time.second]

def get_migration_key(start_time, end_time):
    return get_formatted_datetime_string(start_time) + '_' + get_formatted_datetime_string(end_time)

@retry_on_couch_error
def _get_couch_docs(start_key, end_key):
    db = get_db("auditcare")
    result = db.view(
        "auditcare/all_events",
        startkey=start_key,
        endkey=end_key,
        reduce=False,
        include_docs=True,
        limit=COUCH_QUERY_LIMIT
    )
    return list(result)

def get_events_from_couch(start_key, end_key):
    navigation_objects = []
    audit_objects = []
    records_saved = 0
    records_returned = 0
    next_start_time = None
    couch_docs = _get_couch_docs(start_key, end_key)
    for result in couch_docs:
        records_returned += 1
        doc = result["doc"]
        try:
            next_start_time = force_to_datetime(doc.get("event_date"))
            kwargs = _pick(doc, ["user", "domain", "ip_address", "session_key",
                                "headers", "status_code", "user_agent"])
            kwargs.update({
                "event_date": next_start_time,
                "couch_id": doc["_id"],
            })

            if doc["doc_type"] == "NavigationEventAudit":
                if NavigationEventAudit.objects.filter(couch_id=doc["_id"]).exists():
                    continue
                kwargs.update(_pick(doc, ["headers", "status_code", "view", "view_kwargs"]))
                path, _, params = doc.get("request_path", "").partition("?")
                kwargs.update({
                    "path": path,
                    "params": params,
                })
                navigation_objects.append(NavigationEventAudit(**kwargs))
            elif doc["doc_type"] == "AccessAudit":
                if AccessAudit.objects.filter(couch_id=doc["_id"]).exists():
                    continue
                kwargs.update(_pick(doc, ["http_accept", "trace_id"]))
                access_type = doc.get('access_type')
                kwargs.update({
                    "access_type": ACCESS_LOOKUP.get(doc.get("access_type")),
                    "path": doc.get("path_info"),
                })
                if access_type == "logout":
                    kwargs.update({"path": "accounts/logout"})
                audit_objects.append(AccessAudit(**kwargs))
            records_saved += 1
        except Exception:
            raise Exception(doc['_id'])
    res_obj = {
        "navigation_events": navigation_objects,
        "audit_events": audit_objects,
        "break_query": records_returned < COUCH_QUERY_LIMIT or not next_start_time,
        "next_start_key": get_couch_key(next_start_time),
        "count": records_saved
    }
    return res_obj

def copy_events_to_sql(start_time, end_time):
    util = AuditCareMigrationUtil()
    print(f"Starting batch: {start_time} - {end_time}")
    key = get_migration_key(start_time, end_time)
    end_key = get_couch_key(end_time)
    start_key = get_couch_key(start_time)
    next_start_key = start_key
    util.log_batch_start(key)
    break_query = False
    count = 0
    try:
        while not break_query:
            events_info = get_events_from_couch(next_start_key, end_key)
            next_start_key = events_info['next_start_key']
            NavigationEventAudit.objects.bulk_create(events_info['navigation_events'], ignore_conflicts=True)
            AccessAudit.objects.bulk_create(events_info['audit_events'], ignore_conflicts=True)
            count += events_info['count']
            break_query = events_info['break_query']
    except Exception as e:
        message = f"Error in copy_events_to_sql on doc {str(e)}"
        notify_exception(None, message=message)
        _soft_assert = soft_assert(to="{}@{}.com".format('aphulera', 'dimagi'), notify_admins=False)
        _soft_assert(False, message)
        util.set_batch_as_errored(key)
        return
    util.set_batch_as_finished(key, count)

def _pick(doc, keys):
    return {key: doc.get(key) for key in keys if doc.get(key)}
<|endoftext|>


<commit_before>
import six
import time
import signal
import multiprocessing
from functools import partial
from astropy.utils.console import (_get_stdout, isatty, isiterable,
                                   human_file_size, _CAN_RESIZE_TERMINAL,
                                   terminal_size, color_print, human_time)

import contextlib
import warnings
try:
    import builtins
except ImportError:
    # python2
    import __builtin__ as builtins

'''
Copyright (c) 2011-2016, Astropy Developers

All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
  list of conditions and the following disclaimer in the documentation and/or
  other materials provided with the distribution.
* Neither the name of the Astropy Team nor the names of its contributors may be
  used to endorse or promote products derived from this software without
  specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

class ProgressBar(six.Iterator):
    """
    A class to display a progress bar in the terminal.

    It is designed to be used either with the ``with`` statement::

        with ProgressBar(len(items)) as bar:
            for item in enumerate(items):
                bar.update()

    or as a generator::

        for item in ProgressBar(items):
            item.process()
    """
    def __init__(self, total_or_items, ipython_widget=False, file=None):
        """
        Parameters
        ----------
        total_or_items : int or sequence
            If an int, the number of increments in the process being
            tracked.  If a sequence, the items to iterate over.

        ipython_widget : bool, optional
            If `True`, the progress bar will display as an IPython
            notebook widget.

        file : writable file-like object, optional
            The file to write the progress bar to.  Defaults to
            `sys.stdout`.  If `file` is not a tty (as determined by
            calling its `isatty` member, if any, or special case hacks
            to detect the IPython console), the progress bar will be
            completely silent.
        """

        ipython_widget = False

        # if ipython_widget:
        #     # Import only if ipython_widget, i.e., widget in IPython
        #     # notebook
        #     if ipython_major_version < 4:
        #         from IPython.html import widgets
        #     else:
        #         from ipywidgets import widgets
        #     from IPython.display import display

        if file is None:
            file = _get_stdout()

        if not isatty(file) and not ipython_widget:
            self.update = self._silent_update
            self._silent = True
        else:
            self._silent = False

        if isiterable(total_or_items):
            self._items = iter(total_or_items)
            self._total = len(total_or_items)
        else:
            try:
                self._total = int(total_or_items)
            except TypeError:
                raise TypeError("First argument must be int or sequence")
            else:
                self._items = iter(range(self._total))

        self._file = file
        self._start_time = time.time()
        self._human_total = human_file_size(self._total)
        self._ipython_widget = ipython_widget

        self._signal_set = False
        if not ipython_widget:
            self._should_handle_resize = (
                _CAN_RESIZE_TERMINAL and self._file.isatty())
            self._handle_resize()
            if self._should_handle_resize:
                signal.signal(signal.SIGWINCH, self._handle_resize)
                self._signal_set = True

        self.update(0)

    def _handle_resize(self, signum=None, frame=None):
        terminal_width = terminal_size(self._file)[1]
        self._bar_length = terminal_width - 37

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if not self._silent:
            if exc_type is None:
                self.update(self._total)
            self._file.write('\n')
            self._file.flush()
            if self._signal_set:
                signal.signal(signal.SIGWINCH, signal.SIG_DFL)

    def __iter__(self):
        return self

    def __next__(self):
        try:
            rv = next(self._items)
        except StopIteration:
            self.__exit__(None, None, None)
            raise
        else:
            self.update()
            return rv

    def update(self, value=None):
        """
        Update progress bar via the console or notebook accordingly.
        """

        # Update self.value
        if value is None:
            value = self._current_value + 1
        self._current_value = value

        # Choose the appropriate environment
        if self._ipython_widget:
            self._update_ipython_widget(value)
        else:
            self._update_console(value)

    def _update_console(self, value=None):
        """
        Update the progress bar to the given value (out of the total
        given to the constructor).
        """

        if self._total == 0:
            frac = 1.0
        else:
            frac = float(value) / float(self._total)

        file = self._file
        write = file.write

        if frac > 1:
            bar_fill = int(self._bar_length)
        else:
            bar_fill = int(float(self._bar_length) * frac)
        write('\r|')
        color_print('=' * bar_fill, 'blue', file=file, end='')
        if bar_fill < self._bar_length:
            color_print('>', 'green', file=file, end='')
            write('-' * (self._bar_length - bar_fill - 1))
        write('|')

        if value >= self._total:
            t = time.time() - self._start_time
            prefix = '     '
        elif value <= 0:
            t = None
            prefix = ''
        else:
            t = ((time.time() - self._start_time) * (1.0 - frac)) / frac
            prefix = ' ETA '
        write(' {0:>4s}/{1:>4s}'.format(
            human_file_size(value),
            self._human_total))
        write(' ({0:>6s}%)'.format('{0:.2f}'.format(frac * 100.0)))
        write(prefix)
        if t is not None:
            write(human_time(t))
        self._file.flush()

    def _update_ipython_widget(self, value=None):
        """
        Update the progress bar to the given value (out of a total
        given to the constructor).

        This method is for use in the IPython notebook 2+.
        """
        pass

        # Create and display an empty progress bar widget,
        # if none exists.
        # if not hasattr(self, '_widget'):
        #     # Import only if an IPython widget, i.e., widget in iPython NB
        #     if ipython_major_version < 4:
        #         from IPython.html import widgets
        #         self._widget = widgets.FloatProgressWidget()
        #     else:
        #         from ipywidgets import widgets
        #         self._widget = widgets.FloatProgress()
        #     from IPython.display import display

        #     display(self._widget)
        #     self._widget.value = 0

        # # Calculate percent completion, and update progress bar
        # percent = (value / self._total) * 100
        # self._widget.value = percent
        # self._widget.description = \
        #     ' ({0:>6s}%)'.format('{0:.2f}'.format(percent))

    def _silent_update(self, value=None):
        pass

    @classmethod
    def map(cls, function, items, multiprocess=False, file=None, chunksize=100,
            item_len=None, nprocesses=None, **pool_kwargs):
        """
        Does a `map` operation while displaying a progress bar with
        percentage complete.

        ::

            def work(i):
                print(i)

            ProgressBar.map(work, range(50))

        Parameters
        ----------
        function : function
            Function to call for each step

        items : sequence
            Sequence where each element is a tuple of arguments to pass to
            *function*.

        multiprocess : bool, optional
            If `True`, use the `multiprocessing` module to distribute each
            task to a different processor core.

        file : writeable file-like object, optional
            The file to write the progress bar to.  Defaults to
            `sys.stdout`.  If `file` is not a tty (as determined by
            calling its `isatty` member, if any), the scrollbar will
            be completely silent.

        step : int, optional
            Update the progress bar at least every *step* steps (default: 100).
            If ``multiprocess`` is `True`, this will affect the size
            of the chunks of ``items`` that are submitted as separate tasks
            to the process pool.  A large step size may make the job
            complete faster if ``items`` is very long.
        """

        results = []

        if file is None:
            file = _get_stdout()

        if item_len is not None:
            assert isinstance(item_len, int)
            if hasattr(items, "__len__"):
                assert item_len == len(items)
        else:
            if hasattr(items, "__len__"):
                item_len = len(items)
            else:
                # Will convert to iterable. Not a good thing to do with
                # large inputs.
                items = list(items)
                item_len = len(items)

        with cls(item_len, file=file) as bar:
            if not multiprocess:
                for i, item in enumerate(items):
                    results.append(function(item))
                    if (i % chunksize) == 0:
                        bar.update(i)
            else:
                max_proc = multiprocessing.cpu_count()
                if nprocesses is None:
                    nprocesses = max_proc
                elif nprocesses > max_proc:
                    nprocesses = max_proc

                if chunksize is None:
                    chunksize = choose_chunksize(nprocesses, item_len)

                pool = multiprocessing.Pool(nprocesses, **pool_kwargs)
                for i, out in enumerate(pool.imap_unordered(function,
                                                            items,
                                                            chunksize=chunksize)):
                    bar.update(i)
                    results.append(out)
                pool.close()
                pool.join()

        return results

'''
Copyright (c) 2014, spectral-cube developers
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

@contextlib.contextmanager
def _map_context(numcores, verbose=False, num_jobs=None, chunksize=None,
                 **pool_kwargs):
    """
    Mapping context manager to allow parallel mapping or regular mapping
    depending on the number of cores specified.
    """

    if verbose:
        if numcores is not None and numcores > 1:
            parallel = True
        else:
            numcores = 1
            parallel = False
        map = lambda func, items: \
            ProgressBar.map(func, items,
                            nprocesses=numcores,
                            multiprocess=parallel,
                            item_len=num_jobs,
                            chunksize=chunksize,
                            **pool_kwargs)
    else:
        if numcores is not None and numcores > 1:
            try:
                import multiprocessing
                pool = multiprocessing.Pool(processes=numcores, **pool_kwargs)
                if chunksize is None:
                    chunksize = 1

                map = partial(pool.map, chunksize=chunksize)
                parallel = True
            except ImportError:
                map = builtins.map
                warnings.warn("Could not import multiprocessing.  "
                              "map will be non-parallel.")
                parallel = False
        else:
            parallel = False
            map = builtins.map

    try:
        yield map
    finally:
        # ProgressBar.map already closes the pool
        if not verbose and parallel:
                pool.close()
                pool.join()

def choose_chunksize(nprocesses, njobs):
    '''
    Split the chunks into roughly equal portions.
    '''
    # Auto split into close to equal chunks
    if njobs % nprocesses == 0:
        chunksize = njobs / nprocesses
    else:
        # Split into smaller chunks that are still
        # roughly equal, but won't have any small
        # leftovers that would slow things down
        chunksize = njobs / (nprocesses + 1)

    return chunksize
<commit_msg>chunksize must always be >0<commit_after>
import six
import time
import signal
import multiprocessing
from functools import partial
from astropy.utils.console import (_get_stdout, isatty, isiterable,
                                   human_file_size, _CAN_RESIZE_TERMINAL,
                                   terminal_size, color_print, human_time)

import contextlib
import warnings
try:
    import builtins
except ImportError:
    # python2
    import __builtin__ as builtins

'''
Copyright (c) 2011-2016, Astropy Developers

All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
  list of conditions and the following disclaimer in the documentation and/or
  other materials provided with the distribution.
* Neither the name of the Astropy Team nor the names of its contributors may be
  used to endorse or promote products derived from this software without
  specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

class ProgressBar(six.Iterator):
    """
    A class to display a progress bar in the terminal.

    It is designed to be used either with the ``with`` statement::

        with ProgressBar(len(items)) as bar:
            for item in enumerate(items):
                bar.update()

    or as a generator::

        for item in ProgressBar(items):
            item.process()
    """
    def __init__(self, total_or_items, ipython_widget=False, file=None):
        """
        Parameters
        ----------
        total_or_items : int or sequence
            If an int, the number of increments in the process being
            tracked.  If a sequence, the items to iterate over.

        ipython_widget : bool, optional
            If `True`, the progress bar will display as an IPython
            notebook widget.

        file : writable file-like object, optional
            The file to write the progress bar to.  Defaults to
            `sys.stdout`.  If `file` is not a tty (as determined by
            calling its `isatty` member, if any, or special case hacks
            to detect the IPython console), the progress bar will be
            completely silent.
        """

        ipython_widget = False

        # if ipython_widget:
        #     # Import only if ipython_widget, i.e., widget in IPython
        #     # notebook
        #     if ipython_major_version < 4:
        #         from IPython.html import widgets
        #     else:
        #         from ipywidgets import widgets
        #     from IPython.display import display

        if file is None:
            file = _get_stdout()

        if not isatty(file) and not ipython_widget:
            self.update = self._silent_update
            self._silent = True
        else:
            self._silent = False

        if isiterable(total_or_items):
            self._items = iter(total_or_items)
            self._total = len(total_or_items)
        else:
            try:
                self._total = int(total_or_items)
            except TypeError:
                raise TypeError("First argument must be int or sequence")
            else:
                self._items = iter(range(self._total))

        self._file = file
        self._start_time = time.time()
        self._human_total = human_file_size(self._total)
        self._ipython_widget = ipython_widget

        self._signal_set = False
        if not ipython_widget:
            self._should_handle_resize = (
                _CAN_RESIZE_TERMINAL and self._file.isatty())
            self._handle_resize()
            if self._should_handle_resize:
                signal.signal(signal.SIGWINCH, self._handle_resize)
                self._signal_set = True

        self.update(0)

    def _handle_resize(self, signum=None, frame=None):
        terminal_width = terminal_size(self._file)[1]
        self._bar_length = terminal_width - 37

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if not self._silent:
            if exc_type is None:
                self.update(self._total)
            self._file.write('\n')
            self._file.flush()
            if self._signal_set:
                signal.signal(signal.SIGWINCH, signal.SIG_DFL)

    def __iter__(self):
        return self

    def __next__(self):
        try:
            rv = next(self._items)
        except StopIteration:
            self.__exit__(None, None, None)
            raise
        else:
            self.update()
            return rv

    def update(self, value=None):
        """
        Update progress bar via the console or notebook accordingly.
        """

        # Update self.value
        if value is None:
            value = self._current_value + 1
        self._current_value = value

        # Choose the appropriate environment
        if self._ipython_widget:
            self._update_ipython_widget(value)
        else:
            self._update_console(value)

    def _update_console(self, value=None):
        """
        Update the progress bar to the given value (out of the total
        given to the constructor).
        """

        if self._total == 0:
            frac = 1.0
        else:
            frac = float(value) / float(self._total)

        file = self._file
        write = file.write

        if frac > 1:
            bar_fill = int(self._bar_length)
        else:
            bar_fill = int(float(self._bar_length) * frac)
        write('\r|')
        color_print('=' * bar_fill, 'blue', file=file, end='')
        if bar_fill < self._bar_length:
            color_print('>', 'green', file=file, end='')
            write('-' * (self._bar_length - bar_fill - 1))
        write('|')

        if value >= self._total:
            t = time.time() - self._start_time
            prefix = '     '
        elif value <= 0:
            t = None
            prefix = ''
        else:
            t = ((time.time() - self._start_time) * (1.0 - frac)) / frac
            prefix = ' ETA '
        write(' {0:>4s}/{1:>4s}'.format(
            human_file_size(value),
            self._human_total))
        write(' ({0:>6s}%)'.format('{0:.2f}'.format(frac * 100.0)))
        write(prefix)
        if t is not None:
            write(human_time(t))
        self._file.flush()

    def _update_ipython_widget(self, value=None):
        """
        Update the progress bar to the given value (out of a total
        given to the constructor).

        This method is for use in the IPython notebook 2+.
        """
        pass

        # Create and display an empty progress bar widget,
        # if none exists.
        # if not hasattr(self, '_widget'):
        #     # Import only if an IPython widget, i.e., widget in iPython NB
        #     if ipython_major_version < 4:
        #         from IPython.html import widgets
        #         self._widget = widgets.FloatProgressWidget()
        #     else:
        #         from ipywidgets import widgets
        #         self._widget = widgets.FloatProgress()
        #     from IPython.display import display

        #     display(self._widget)
        #     self._widget.value = 0

        # # Calculate percent completion, and update progress bar
        # percent = (value / self._total) * 100
        # self._widget.value = percent
        # self._widget.description = \
        #     ' ({0:>6s}%)'.format('{0:.2f}'.format(percent))

    def _silent_update(self, value=None):
        pass

    @classmethod
    def map(cls, function, items, multiprocess=False, file=None, chunksize=100,
            item_len=None, nprocesses=None, **pool_kwargs):
        """
        Does a `map` operation while displaying a progress bar with
        percentage complete.

        ::

            def work(i):
                print(i)

            ProgressBar.map(work, range(50))

        Parameters
        ----------
        function : function
            Function to call for each step

        items : sequence
            Sequence where each element is a tuple of arguments to pass to
            *function*.

        multiprocess : bool, optional
            If `True`, use the `multiprocessing` module to distribute each
            task to a different processor core.

        file : writeable file-like object, optional
            The file to write the progress bar to.  Defaults to
            `sys.stdout`.  If `file` is not a tty (as determined by
            calling its `isatty` member, if any), the scrollbar will
            be completely silent.

        step : int, optional
            Update the progress bar at least every *step* steps (default: 100).
            If ``multiprocess`` is `True`, this will affect the size
            of the chunks of ``items`` that are submitted as separate tasks
            to the process pool.  A large step size may make the job
            complete faster if ``items`` is very long.
        """

        results = []

        if file is None:
            file = _get_stdout()

        if item_len is not None:
            assert isinstance(item_len, int)
            if hasattr(items, "__len__"):
                assert item_len == len(items)
        else:
            if hasattr(items, "__len__"):
                item_len = len(items)
            else:
                # Will convert to iterable. Not a good thing to do with
                # large inputs.
                items = list(items)
                item_len = len(items)

        with cls(item_len, file=file) as bar:
            if not multiprocess:
                for i, item in enumerate(items):
                    results.append(function(item))
                    if (i % chunksize) == 0:
                        bar.update(i)
            else:
                max_proc = multiprocessing.cpu_count()
                if nprocesses is None:
                    nprocesses = max_proc
                elif nprocesses > max_proc:
                    nprocesses = max_proc

                if chunksize is None:
                    chunksize = choose_chunksize(nprocesses, item_len)

                pool = multiprocessing.Pool(nprocesses, **pool_kwargs)
                for i, out in enumerate(pool.imap_unordered(function,
                                                            items,
                                                            chunksize=chunksize)):
                    bar.update(i)
                    results.append(out)
                pool.close()
                pool.join()

        return results

'''
Copyright (c) 2014, spectral-cube developers
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

@contextlib.contextmanager
def _map_context(numcores, verbose=False, num_jobs=None, chunksize=None,
                 **pool_kwargs):
    """
    Mapping context manager to allow parallel mapping or regular mapping
    depending on the number of cores specified.
    """

    if verbose:
        if numcores is not None and numcores > 1:
            parallel = True
        else:
            numcores = 1
            parallel = False
        map = lambda func, items: \
            ProgressBar.map(func, items,
                            nprocesses=numcores,
                            multiprocess=parallel,
                            item_len=num_jobs,
                            chunksize=chunksize,
                            **pool_kwargs)
    else:
        if numcores is not None and numcores > 1:
            try:
                import multiprocessing
                pool = multiprocessing.Pool(processes=numcores, **pool_kwargs)
                if chunksize is None:
                    chunksize = 1

                map = partial(pool.map, chunksize=chunksize)
                parallel = True
            except ImportError:
                map = builtins.map
                warnings.warn("Could not import multiprocessing.  "
                              "map will be non-parallel.")
                parallel = False
        else:
            parallel = False
            map = builtins.map

    try:
        yield map
    finally:
        # ProgressBar.map already closes the pool
        if not verbose and parallel:
                pool.close()
                pool.join()

def choose_chunksize(nprocesses, njobs):
    '''
    Split the chunks into roughly equal portions.
    '''
    # Auto split into close to equal chunks
    if njobs % nprocesses == 0:
        chunksize = njobs / nprocesses
    else:
        # Split into smaller chunks that are still
        # roughly equal, but won't have any small
        # leftovers that would slow things down
        chunksize = njobs / (nprocesses + 1)

    return chunksize if chunksize > 0 else 1
<|endoftext|>


<commit_before>#!/usr/bin/env python

''' Copyright (c) 2013 Potential Ventures Ltd
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Potential Ventures Ltd nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL POTENTIAL VENTURES LTD BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. '''

"""
    Common bus related functionality

    A bus is simply defined as a collection of signals
"""

class Bus(object):
    """
        Wraps up a collection of signals
    """
    def __init__(self, entity, name, signals):
        """
        entity:         SimHandle instance to the entity containing the bus
        name:           name of the bus
        signals:        array of signal names
        """
        self._entity = entity
        self._name = name
        self._signals = {}

        for signal in signals:
            signame = name + "_" + signal
            setattr(self, signal, getattr(entity, signame))
            self._signals[signal] = getattr(self, signal)

    def drive(self, obj, strict=False):
        """
        Drives values onto the bus.

        Args:
            obj (any type) : object with attribute names that match the bus signals

        Kwargs:
            strict (bool)  : Check that all signals are being assigned

        Raises:
            AttributeError
        """
        for name, hdl in self._signals.items():
            if not hasattr(obj, name):
                if strict:
                    raise AttributeError("Unable to drive onto %s.%s because %s is missing attribute %s" %
                        (self._entity.name, self._name, obj.__class__.__name__, name))
                else: continue
            val = getattr(obj, name)
            hdl <= val
<commit_msg>Add support for "optional" bus signals<commit_after>#!/usr/bin/env python

''' Copyright (c) 2013 Potential Ventures Ltd
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Potential Ventures Ltd nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL POTENTIAL VENTURES LTD BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. '''

"""
    Common bus related functionality

    A bus is simply defined as a collection of signals
"""

class Bus(object):
    """
        Wraps up a collection of signals

        Assumes we have a set of signals/nets named:

            entity.bus_name_signal

        for example a bus named "stream_in" with signals ["valid", "data"]
            dut.stream_in_valid
            dut.stream_in_data

        TODO:
            Support for struct/record ports where signals are member names
    """
    def __init__(self, entity, name, signals, optional_signals=[]):
        """
        Args:
            entity (SimHandle):   SimHandle instance to the entity containing the bus
            name (str):           name of the bus
            signals (list):       array of signal names

        Kwargs:
            optiona_signals (list): array of optional signal names
        """
        self._entity = entity
        self._name = name
        self._signals = {}

        for signal in signals:
            signame = name + "_" + signal
            setattr(self, signal, getattr(entity, signame))
            self._signals[signal] = getattr(self, signal)

        # Also support a set of optional signals that don't have to be present
        for signal in optional_signals:
            signame = name + "_" + signal
            try:
                hdl = getattr(entity, signame)
            except AttributeError:
                self._entity.log.debug("Ignoring optional missing signal %s \
                    on bus %s" % signal, name)
                continue
            setattr(self, signal, hdl)
            self._signals[signal] = getattr(self, signal)

    def drive(self, obj, strict=False):
        """
        Drives values onto the bus.

        Args:
            obj (any type) : object with attribute names that match the bus signals

        Kwargs:
            strict (bool)  : Check that all signals are being assigned

        Raises:
            AttributeError
        """
        for name, hdl in self._signals.items():
            if not hasattr(obj, name):
                if strict:
                    raise AttributeError("Unable to drive onto %s.%s because %s is missing attribute %s" %
                        (self._entity.name, self._name, obj.__class__.__name__, name))
                else: continue
            val = getattr(obj, name)
            hdl <= val

    def __le__(self, value):
        """Overload the less than or equal to operator for value assignment"""
        self.drive(value)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import base64
import logging
import psycopg2
import werkzeug

from werkzeug import url_encode

from odoo import api, http, registry, SUPERUSER_ID, _
from odoo.addons.web.controllers.main import binary_content
from odoo.exceptions import AccessError
from odoo.http import request
from odoo.tools import consteq, pycompat

_logger = logging.getLogger(__name__)

class MailController(http.Controller):
    _cp_path = '/mail'

    @classmethod
    def _redirect_to_messaging(cls):
        url = '/web#%s' % url_encode({'action': 'mail.mail_channel_action_client_chat'})
        return werkzeug.utils.redirect(url)

    @classmethod
    def _check_token(cls, token):
        base_link = request.httprequest.path
        params = dict(request.params)
        params.pop('token', '')
        valid_token = request.env['mail.thread']._notify_encode_link(base_link, params)
        return consteq(valid_token, str(token))

    @classmethod
    def _check_token_and_record_or_redirect(cls, model, res_id, token):
        comparison = cls._check_token(token)
        if not comparison:
            _logger.warning(_('Invalid token in route %s') % request.httprequest.url)
            return comparison, None, cls._redirect_to_messaging()
        try:
            record = request.env[model].browse(res_id).exists()
        except Exception:
            record = None
            redirect = cls._redirect_to_messaging()
        else:
            redirect = cls._redirect_to_record(model, res_id)
        return comparison, record, redirect

    @classmethod
    def _redirect_to_record(cls, model, res_id, access_token=None):
        uid = request.session.uid

        # no model / res_id, meaning no possible record -> redirect to login
        if not model or not res_id or model not in request.env:
            return cls._redirect_to_messaging()

        # find the access action using sudo to have the details about the access link
        RecordModel = request.env[model]
        record_sudo = RecordModel.sudo().browse(res_id).exists()
        if not record_sudo:
            # record does not seem to exist -> redirect to login
            return cls._redirect_to_messaging()

        # the record has a window redirection: check access rights
        if uid is not None:
            if not RecordModel.sudo(uid).check_access_rights('read', raise_exception=False):
                return cls._redirect_to_messaging()
            try:
                record_sudo.sudo(uid).check_access_rule('read')
            except AccessError:
                return cls._redirect_to_messaging()
            else:
                record_action = record_sudo.get_access_action(access_uid=uid)
        else:
            record_action = record_sudo.get_access_action()
            if record_action['type'] == 'ir.actions.act_url' and record_action.get('target_type') != 'public':
                return cls._redirect_to_messaging()

        record_action.pop('target_type', None)
        # the record has an URL redirection: use it directly
        if record_action['type'] == 'ir.actions.act_url':
            return werkzeug.utils.redirect(record_action['url'])
        # other choice: act_window (no support of anything else currently)
        elif not record_action['type'] == 'ir.actions.act_window':
            return cls._redirect_to_messaging()

        url_params = {
            'view_type': record_action['view_type'],
            'model': model,
            'id': res_id,
            'active_id': res_id,
            'action': record_action.get('id'),
        }
        view_id = record_sudo.get_formview_id()
        if view_id:
            url_params['view_id'] = view_id

        url = '/web?#%s' % url_encode(url_params)
        return werkzeug.utils.redirect(url)

    @http.route('/mail/receive', type='json', auth='none')
    def receive(self, req):
        """ End-point to receive mail from an external SMTP server. """
        dbs = req.jsonrequest.get('databases')
        for db in dbs:
            message = base64.b64decode(dbs[db])
            try:
                db_registry = registry(db)
                with db_registry.cursor() as cr:
                    env = api.Environment(cr, SUPERUSER_ID, {})
                    env['mail.thread'].message_process(None, message)
            except psycopg2.Error:
                pass
        return True

    @http.route('/mail/read_followers', type='json', auth='user')
    def read_followers(self, follower_ids, res_model):
        followers = []
        is_editable = request.env.user.has_group('base.group_no_one')
        partner_id = request.env.user.partner_id
        follower_id = None
        follower_recs = request.env['mail.followers'].sudo().browse(follower_ids)
        res_ids = follower_recs.mapped('res_id')
        request.env[res_model].browse(res_ids).check_access_rule("read")
        for follower in follower_recs:
            is_uid = partner_id == follower.partner_id
            follower_id = follower.id if is_uid else follower_id
            followers.append({
                'id': follower.id,
                'name': follower.partner_id.name or follower.channel_id.name,
                'email': follower.partner_id.email if follower.partner_id else None,
                'res_model': 'res.partner' if follower.partner_id else 'mail.channel',
                'res_id': follower.partner_id.id or follower.channel_id.id,
                'is_editable': is_editable,
                'is_uid': is_uid,
            })
        return {
            'followers': followers,
            'subtypes': self.read_subscription_data(res_model, follower_id) if follower_id else None
        }

    @http.route('/mail/read_subscription_data', type='json', auth='user')
    def read_subscription_data(self, res_model, follower_id):
        """ Computes:
            - message_subtype_data: data about document subtypes: which are
                available, which are followed if any """
        followers = request.env['mail.followers'].browse(follower_id)

        # find current model subtypes, add them to a dictionary
        subtypes = request.env['mail.message.subtype'].search(['&', ('hidden', '=', False), '|', ('res_model', '=', res_model), ('res_model', '=', False)])
        subtypes_list = [{
            'name': subtype.name,
            'res_model': subtype.res_model,
            'sequence': subtype.sequence,
            'default': subtype.default,
            'internal': subtype.internal,
            'followed': subtype.id in followers.mapped('subtype_ids').ids,
            'parent_model': subtype.parent_id.res_model,
            'id': subtype.id
        } for subtype in subtypes]
        subtypes_list = sorted(subtypes_list, key=lambda it: (it['parent_model'] or '', it['res_model'] or '', it['internal'], it['sequence']))
        return subtypes_list

    @http.route('/mail/view', type='http', auth='none')
    def mail_action_view(self, model=None, res_id=None, access_token=None, **kwargs):
        """ Generic access point from notification emails. The heuristic to
            choose where to redirect the user is the following :

         - find a public URL
         - if none found
          - users with a read access are redirected to the document
          - users without read access are redirected to the Messaging
          - not logged users are redirected to the login page

            models that have an access_token may apply variations on this.
        """
        if res_id and isinstance(res_id, pycompat.string_types):
            res_id = int(res_id)
        return self._redirect_to_record(model, res_id, access_token)

    @http.route('/mail/assign', type='http', auth='user', methods=['GET'])
    def mail_action_assign(self, model, res_id, token=None):
        comparison, record, redirect = self._check_token_and_record_or_redirect(model, int(res_id), token)
        if comparison and record:
            try:
                record.write({'user_id': request.uid})
            except Exception:
                return self._redirect_to_messaging()
        return redirect

    @http.route('/mail/<string:res_model>/<int:res_id>/avatar/<int:partner_id>', type='http', auth='public')
    def avatar(self, res_model, res_id, partner_id):
        headers = [('Content-Type', 'image/png')]
        status = 200
        content = 'R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='  # default image is one white pixel
        if res_model in request.env:
            try:
                # if the current user has access to the document, get the partner avatar as sudo()
                request.env[res_model].browse(res_id).check_access_rule('read')
                if partner_id in request.env[res_model].browse(res_id).sudo().exists().message_ids.mapped('author_id').ids:
                    status, headers, _content = binary_content(model='res.partner', id=partner_id, field='image_medium', default_mimetype='image/png', env=request.env(user=SUPERUSER_ID))
                    # binary content return an empty string and not a placeholder if obj[field] is False
                    if _content != '':
                        content = _content
                    if status == 304:
                        return werkzeug.wrappers.Response(status=304)
            except AccessError:
                pass
        image_base64 = base64.b64decode(content)
        headers.append(('Content-Length', len(image_base64)))
        response = request.make_response(image_base64, headers)
        response.status = str(status)
        return response

    @http.route('/mail/needaction', type='json', auth='user')
    def needaction(self):
        return request.env['res.partner'].get_needaction_count()

    @http.route('/mail/init_messaging', type='json', auth='user')
    def mail_init_messaging(self):
        values = {
            'needaction_inbox_counter': request.env['res.partner'].get_needaction_count(),
            'starred_counter': request.env['res.partner'].get_starred_count(),
            'channel_slots': request.env['mail.channel'].channel_fetch_slot(),
            'commands': request.env['mail.channel'].get_mention_commands(),
            'mention_partner_suggestions': request.env['res.partner'].get_static_mention_suggestions(),
            'shortcodes': request.env['mail.shortcode'].sudo().search_read([], ['shortcode_type', 'source', 'unicode_source', 'substitution', 'description']),
            'menu_id': request.env['ir.model.data'].xmlid_to_res_id('mail.mail_channel_menu_root_chat'),
        }
        return values

    @http.route('/mail/activity/new', type='json', auth='user')
    def mail_activity_new(self, note, activity_type_id=None, date_deadline=None):
        values = {'note': note}
        if date_deadline:
            values['date_deadline'] = date_deadline
        if not activity_type_id:
            activity_type_id = request.env['mail.activity.type'].sudo().search([('category', '=', 'reminder')]).id
        if activity_type_id:
            values['activity_type_id'] = activity_type_id
        activity = request.env['mail.activity'].create(values)
        return activity.id
<commit_msg>[IMP] mail: take first found reminder activity type as default when creating reminders<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import base64
import logging
import psycopg2
import werkzeug

from werkzeug import url_encode

from odoo import api, http, registry, SUPERUSER_ID, _
from odoo.addons.web.controllers.main import binary_content
from odoo.exceptions import AccessError
from odoo.http import request
from odoo.tools import consteq, pycompat

_logger = logging.getLogger(__name__)

class MailController(http.Controller):
    _cp_path = '/mail'

    @classmethod
    def _redirect_to_messaging(cls):
        url = '/web#%s' % url_encode({'action': 'mail.mail_channel_action_client_chat'})
        return werkzeug.utils.redirect(url)

    @classmethod
    def _check_token(cls, token):
        base_link = request.httprequest.path
        params = dict(request.params)
        params.pop('token', '')
        valid_token = request.env['mail.thread']._notify_encode_link(base_link, params)
        return consteq(valid_token, str(token))

    @classmethod
    def _check_token_and_record_or_redirect(cls, model, res_id, token):
        comparison = cls._check_token(token)
        if not comparison:
            _logger.warning(_('Invalid token in route %s') % request.httprequest.url)
            return comparison, None, cls._redirect_to_messaging()
        try:
            record = request.env[model].browse(res_id).exists()
        except Exception:
            record = None
            redirect = cls._redirect_to_messaging()
        else:
            redirect = cls._redirect_to_record(model, res_id)
        return comparison, record, redirect

    @classmethod
    def _redirect_to_record(cls, model, res_id, access_token=None):
        uid = request.session.uid

        # no model / res_id, meaning no possible record -> redirect to login
        if not model or not res_id or model not in request.env:
            return cls._redirect_to_messaging()

        # find the access action using sudo to have the details about the access link
        RecordModel = request.env[model]
        record_sudo = RecordModel.sudo().browse(res_id).exists()
        if not record_sudo:
            # record does not seem to exist -> redirect to login
            return cls._redirect_to_messaging()

        # the record has a window redirection: check access rights
        if uid is not None:
            if not RecordModel.sudo(uid).check_access_rights('read', raise_exception=False):
                return cls._redirect_to_messaging()
            try:
                record_sudo.sudo(uid).check_access_rule('read')
            except AccessError:
                return cls._redirect_to_messaging()
            else:
                record_action = record_sudo.get_access_action(access_uid=uid)
        else:
            record_action = record_sudo.get_access_action()
            if record_action['type'] == 'ir.actions.act_url' and record_action.get('target_type') != 'public':
                return cls._redirect_to_messaging()

        record_action.pop('target_type', None)
        # the record has an URL redirection: use it directly
        if record_action['type'] == 'ir.actions.act_url':
            return werkzeug.utils.redirect(record_action['url'])
        # other choice: act_window (no support of anything else currently)
        elif not record_action['type'] == 'ir.actions.act_window':
            return cls._redirect_to_messaging()

        url_params = {
            'view_type': record_action['view_type'],
            'model': model,
            'id': res_id,
            'active_id': res_id,
            'action': record_action.get('id'),
        }
        view_id = record_sudo.get_formview_id()
        if view_id:
            url_params['view_id'] = view_id

        url = '/web?#%s' % url_encode(url_params)
        return werkzeug.utils.redirect(url)

    @http.route('/mail/receive', type='json', auth='none')
    def receive(self, req):
        """ End-point to receive mail from an external SMTP server. """
        dbs = req.jsonrequest.get('databases')
        for db in dbs:
            message = base64.b64decode(dbs[db])
            try:
                db_registry = registry(db)
                with db_registry.cursor() as cr:
                    env = api.Environment(cr, SUPERUSER_ID, {})
                    env['mail.thread'].message_process(None, message)
            except psycopg2.Error:
                pass
        return True

    @http.route('/mail/read_followers', type='json', auth='user')
    def read_followers(self, follower_ids, res_model):
        followers = []
        is_editable = request.env.user.has_group('base.group_no_one')
        partner_id = request.env.user.partner_id
        follower_id = None
        follower_recs = request.env['mail.followers'].sudo().browse(follower_ids)
        res_ids = follower_recs.mapped('res_id')
        request.env[res_model].browse(res_ids).check_access_rule("read")
        for follower in follower_recs:
            is_uid = partner_id == follower.partner_id
            follower_id = follower.id if is_uid else follower_id
            followers.append({
                'id': follower.id,
                'name': follower.partner_id.name or follower.channel_id.name,
                'email': follower.partner_id.email if follower.partner_id else None,
                'res_model': 'res.partner' if follower.partner_id else 'mail.channel',
                'res_id': follower.partner_id.id or follower.channel_id.id,
                'is_editable': is_editable,
                'is_uid': is_uid,
            })
        return {
            'followers': followers,
            'subtypes': self.read_subscription_data(res_model, follower_id) if follower_id else None
        }

    @http.route('/mail/read_subscription_data', type='json', auth='user')
    def read_subscription_data(self, res_model, follower_id):
        """ Computes:
            - message_subtype_data: data about document subtypes: which are
                available, which are followed if any """
        followers = request.env['mail.followers'].browse(follower_id)

        # find current model subtypes, add them to a dictionary
        subtypes = request.env['mail.message.subtype'].search(['&', ('hidden', '=', False), '|', ('res_model', '=', res_model), ('res_model', '=', False)])
        subtypes_list = [{
            'name': subtype.name,
            'res_model': subtype.res_model,
            'sequence': subtype.sequence,
            'default': subtype.default,
            'internal': subtype.internal,
            'followed': subtype.id in followers.mapped('subtype_ids').ids,
            'parent_model': subtype.parent_id.res_model,
            'id': subtype.id
        } for subtype in subtypes]
        subtypes_list = sorted(subtypes_list, key=lambda it: (it['parent_model'] or '', it['res_model'] or '', it['internal'], it['sequence']))
        return subtypes_list

    @http.route('/mail/view', type='http', auth='none')
    def mail_action_view(self, model=None, res_id=None, access_token=None, **kwargs):
        """ Generic access point from notification emails. The heuristic to
            choose where to redirect the user is the following :

         - find a public URL
         - if none found
          - users with a read access are redirected to the document
          - users without read access are redirected to the Messaging
          - not logged users are redirected to the login page

            models that have an access_token may apply variations on this.
        """
        if res_id and isinstance(res_id, pycompat.string_types):
            res_id = int(res_id)
        return self._redirect_to_record(model, res_id, access_token)

    @http.route('/mail/assign', type='http', auth='user', methods=['GET'])
    def mail_action_assign(self, model, res_id, token=None):
        comparison, record, redirect = self._check_token_and_record_or_redirect(model, int(res_id), token)
        if comparison and record:
            try:
                record.write({'user_id': request.uid})
            except Exception:
                return self._redirect_to_messaging()
        return redirect

    @http.route('/mail/<string:res_model>/<int:res_id>/avatar/<int:partner_id>', type='http', auth='public')
    def avatar(self, res_model, res_id, partner_id):
        headers = [('Content-Type', 'image/png')]
        status = 200
        content = 'R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='  # default image is one white pixel
        if res_model in request.env:
            try:
                # if the current user has access to the document, get the partner avatar as sudo()
                request.env[res_model].browse(res_id).check_access_rule('read')
                if partner_id in request.env[res_model].browse(res_id).sudo().exists().message_ids.mapped('author_id').ids:
                    status, headers, _content = binary_content(model='res.partner', id=partner_id, field='image_medium', default_mimetype='image/png', env=request.env(user=SUPERUSER_ID))
                    # binary content return an empty string and not a placeholder if obj[field] is False
                    if _content != '':
                        content = _content
                    if status == 304:
                        return werkzeug.wrappers.Response(status=304)
            except AccessError:
                pass
        image_base64 = base64.b64decode(content)
        headers.append(('Content-Length', len(image_base64)))
        response = request.make_response(image_base64, headers)
        response.status = str(status)
        return response

    @http.route('/mail/needaction', type='json', auth='user')
    def needaction(self):
        return request.env['res.partner'].get_needaction_count()

    @http.route('/mail/init_messaging', type='json', auth='user')
    def mail_init_messaging(self):
        values = {
            'needaction_inbox_counter': request.env['res.partner'].get_needaction_count(),
            'starred_counter': request.env['res.partner'].get_starred_count(),
            'channel_slots': request.env['mail.channel'].channel_fetch_slot(),
            'commands': request.env['mail.channel'].get_mention_commands(),
            'mention_partner_suggestions': request.env['res.partner'].get_static_mention_suggestions(),
            'shortcodes': request.env['mail.shortcode'].sudo().search_read([], ['shortcode_type', 'source', 'unicode_source', 'substitution', 'description']),
            'menu_id': request.env['ir.model.data'].xmlid_to_res_id('mail.mail_channel_menu_root_chat'),
        }
        return values

    @http.route('/mail/activity/new', type='json', auth='user')
    def mail_activity_new(self, note, activity_type_id=None, date_deadline=None):
        values = {'note': note}
        if date_deadline:
            values['date_deadline'] = date_deadline
        if not activity_type_id:
            activity_type_id = request.env['mail.activity.type'].sudo().search([('category', '=', 'reminder')], limit=1).id
        if activity_type_id:
            values['activity_type_id'] = activity_type_id
        activity = request.env['mail.activity'].create(values)
        return activity.id
<|endoftext|>


<commit_before>"""
FOIATemplate allows users to create their own request template language

These can be shared among organizations or specified as the defaults
for certain jurisdictions or agencies
"""

# Django
from django.contrib.auth.models import User
from django.db import models

# MuckRock
from muckrock.foia.querysets import FOIATemplateQuerySet

def make_tag(tag, title):
    """Make a tooltip tag for an html template placeholder"""
    return (tag, f'{tag}')

class FOIATemplate(models.Model):
    """A custom template for public records request language"""

    name = models.CharField(max_length=255, unique=True)
    user = models.ForeignKey(User, on_delete=models.PROTECT, related_name="templates")
    jurisdiction = models.ForeignKey(
        "jurisdiction.Jurisdiction",
        related_name="templates",
        on_delete=models.PROTECT,
        blank=True,
        null=True,
    )
    template = models.TextField()

    objects = FOIATemplateQuerySet.as_manager()

    class Meta:
        verbose_name = "FOIA Template"

    def __str__(self):
        return self.name

    def _handle_users(self, tags, user, jurisdiction, **kwargs):
        """Handle user names and proxies"""
        if user.is_authenticated:
            tags.append(("{ name }", user.profile.full_name))
        elif kwargs.get("html"):
            tags.append(make_tag("{ name }", "This will be replaced by your full name"))

        proxy = kwargs.get("proxy")
        if proxy:
            jurisdiction_name = (
                jurisdiction.name if jurisdiction else "{ jurisdiction }"
            )
            tags.append(
                (
                    "{ closing }",
                    f"This request is filed by {proxy.profile.full_name}, a citizen of "
                    f"{jurisdiction_name}, in coordination with "
                    f"{user.profile.full_name}.",
                )
            )
        elif user.is_authenticated:
            tags.append(("{ closing }", f"Sincerely,\n\n{user.profile.full_name}"))
        elif kwargs.get("html"):
            tags.append(
                make_tag(
                    "{ closing }",
                    "This will be replaced by a suitable closing for the letter",
                )
            )

    def render(self, agency, user, requested_docs, **kwargs):
        """Render this template for a given agency"""
        template = self.template
        jurisdiction = kwargs.pop(
            "jurisdiction", agency.jurisdiction if agency else None
        )
        tags = [
            ("{ law name }", jurisdiction.get_law_name()),
            ("{ short name }", jurisdiction.get_law_name(abbrev=True)),
            ("{ days }", jurisdiction.get_days()),
            ("{ waiver }", jurisdiction.get_waiver()),
            ("{ requested docs }", requested_docs),
        ]
        if agency:
            tags.append(("{ agency name }", agency.name))

        self._handle_users(tags, user, jurisdiction, **kwargs)

        for tag, replace in tags:
            template = template.replace(tag, str(replace))
        return template

    def render_generic(self, user, requested_docs, **kwargs):
        """Render this template in a generic manner"""
        if not kwargs.get("html"):
            return self.template

        template = self.template

        tags = [
            make_tag(
                "{ law name }", "This will be replaced by the relevant transparency law"
            ),
            make_tag(
                "{ short name }",
                "This will be replaced by the abbreviation for the "
                "relevant transparency law",
            ),
            make_tag(
                "{ days }",
                "Number of says statue requires or default of 10 days if no requirement",
            ),
            make_tag(
                "{ agency name }", "This will be replaced by the name of the agency"
            ),
            make_tag(
                "{ waiver }",
                "This will be replaced by jurisdiction approriate fee waiver language "
                "if available, or generic fee waiver language otherwise",
            ),
            ("{ requested docs }", requested_docs),
        ]

        self._handle_users(tags, user, None, **kwargs)

        for tag, replace in tags:
            template = template.replace(tag, str(replace))
        return template
<commit_msg>fix proxy language<commit_after>"""
FOIATemplate allows users to create their own request template language

These can be shared among organizations or specified as the defaults
for certain jurisdictions or agencies
"""

# Django
from django.contrib.auth.models import User
from django.db import models

# MuckRock
from muckrock.foia.querysets import FOIATemplateQuerySet

def make_tag(tag, title):
    """Make a tooltip tag for an html template placeholder"""
    return (tag, f'{tag}')

class FOIATemplate(models.Model):
    """A custom template for public records request language"""

    name = models.CharField(max_length=255, unique=True)
    user = models.ForeignKey(User, on_delete=models.PROTECT, related_name="templates")
    jurisdiction = models.ForeignKey(
        "jurisdiction.Jurisdiction",
        related_name="templates",
        on_delete=models.PROTECT,
        blank=True,
        null=True,
    )
    template = models.TextField()

    objects = FOIATemplateQuerySet.as_manager()

    class Meta:
        verbose_name = "FOIA Template"

    def __str__(self):
        return self.name

    def _handle_users(self, tags, user, jurisdiction, **kwargs):
        """Handle user names and proxies"""
        if user.is_authenticated:
            tags.append(("{ name }", user.profile.full_name))
        elif kwargs.get("html"):
            tags.append(make_tag("{ name }", "This will be replaced by your full name"))

        proxy = kwargs.get("proxy")
        if proxy:
            jurisdiction_name = (
                jurisdiction.legal.name if jurisdiction else "{ jurisdiction }"
            )
            coordination_clause = (
                ""
                if proxy == user
                else ", in coorindatation with {user.profile.full_name}"
            )
            tags.append(
                (
                    "{ closing }",
                    f"This request is filed by {proxy.profile.full_name}, a citizen of "
                    f"{jurisdiction_name}{coordination_clause}.",
                )
            )
        elif user.is_authenticated:
            tags.append(("{ closing }", f"Sincerely,\n\n{user.profile.full_name}"))
        elif kwargs.get("html"):
            tags.append(
                make_tag(
                    "{ closing }",
                    "This will be replaced by a suitable closing for the letter",
                )
            )

    def render(self, agency, user, requested_docs, **kwargs):
        """Render this template for a given agency"""
        template = self.template
        jurisdiction = kwargs.pop(
            "jurisdiction", agency.jurisdiction if agency else None
        )
        tags = [
            ("{ law name }", jurisdiction.get_law_name()),
            ("{ short name }", jurisdiction.get_law_name(abbrev=True)),
            ("{ days }", jurisdiction.get_days()),
            ("{ waiver }", jurisdiction.get_waiver()),
            ("{ requested docs }", requested_docs),
        ]
        if agency:
            tags.append(("{ agency name }", agency.name))

        self._handle_users(tags, user, jurisdiction, **kwargs)

        for tag, replace in tags:
            template = template.replace(tag, str(replace))
        return template

    def render_generic(self, user, requested_docs, **kwargs):
        """Render this template in a generic manner"""
        if not kwargs.get("html"):
            return self.template

        template = self.template

        tags = [
            make_tag(
                "{ law name }", "This will be replaced by the relevant transparency law"
            ),
            make_tag(
                "{ short name }",
                "This will be replaced by the abbreviation for the "
                "relevant transparency law",
            ),
            make_tag(
                "{ days }",
                "Number of says statue requires or default of 10 days if no requirement",
            ),
            make_tag(
                "{ agency name }", "This will be replaced by the name of the agency"
            ),
            make_tag(
                "{ waiver }",
                "This will be replaced by jurisdiction approriate fee waiver language "
                "if available, or generic fee waiver language otherwise",
            ),
            ("{ requested docs }", requested_docs),
        ]

        self._handle_users(tags, user, None, **kwargs)

        for tag, replace in tags:
            template = template.replace(tag, str(replace))
        return template
<|endoftext|>


<commit_before>from pyinfra import host, inventory
from pyinfra.modules import files, init, server, yum

SUDO = True

# TODO: perhaps search by name not ip
# update the /etc/hosts file
def update_hosts_file(name, ip):
    name = name.replace('@vagrant/', '')
    files.line(
        {'Add hosts to /etc/hosts'},
        '/etc/hosts',
        r'{} .*'.format(ip),
        replace='{} {}.example.com {}'.format(ip, name, name),
    )

# ensure all hosts are added to each /etc/hosts file
inv = inventory.get_group('@vagrant')
for item in inv:
    update_hosts_file(item.name, item.fact.ipv4_addresses['eth0'])

if host.name == '@vagrant/master':
    server.hostname(
        {'Set the hostname for the Puppet Master'},
        'master.example.com',
    )

if host.name == '@vagrant/agent':
    server.hostname(
        {'Set the hostname for the Puppet Master'},
        'agent.example.com',
    )

if host.fact.linux_name in ['CentOS', 'RedHat']:

    yum.packages(
        {'Install chrony for Network Time Protocol (NTP)'},
        ['chrony'],
    )

    major = host.fact.linux_distribution['major']
    yum.rpm(
        {'Install Puppet Repo'},
        'https://yum.puppet.com/puppet6-release-el-{}.noarch.rpm'
        .format(major),
    )

    files.line(
        {'Ensure SELINUX is disabled'},
        '/etc/sysconfig/selinux',
        r'SELINUX=.*',
        replace='SELINUX=disabled',
    )

    # TODO: reboot after SELINUX is disabled (how to check/easy way to reboot)
    # TODO: how to determine when reboot is complete
    # TODO: run sestatus

if host.name == '@vagrant/master':

    install = yum.packages(
        {'Install puppet server'},
        ['puppetserver'],
    )

    config = files.template(
        {'Manage the puppet master configuration'},
        'templates/master_puppet.conf.j2',
        '/etc/puppetlabs/puppet/puppet.conf',
    )

    # TODO: tune always shows as changed
    # I think it should only show as changed if there really was a difference.
    # Might have to add a suffix to the sed -i option, then move file only if
    # there is a diff. Maybe?
    tune = files.line(
        {'Tune the puppet server jvm to only use 1gb'},
        '/etc/sysconfig/puppetserver',
        r'^JAVA_ARGS=.*$',
        replace='JAVA_ARGS=\\"-Xms1g -Xmx1g -Djruby.logger.class=com.puppetlabs.'
        'jruby_utils.jruby.Slf4jLogger\\"'
    )

    if install.changed or config.changed or tune.changed:
        init.systemd(
            {'Restart and enable puppetserver'},
            'puppetserver',
            running=True,
            restarted=True,
            enabled=True,
        )

    # TODO: firewall-cmd --add-port=8140/tcp --permanent
    # TODO: firewall-cmd --reload

if host.name == '@vagrant/agent':

    yum.packages(
        {'Install puppet agent'},
        ['puppet-agent'],
    )

    files.template(
        {'Manage the puppet agent configuration'},
        'templates/agent_puppet.conf.j2',
        '/etc/puppetlabs/puppet/puppet.conf',
    )

    init.systemd(
        {'Restart and enable puppet agent'},
        'puppet',
        running=True,
        restarted=True,
        enabled=True,
    )
<commit_msg>flake8 fix<commit_after>from pyinfra import host, inventory
from pyinfra.modules import files, init, server, yum

SUDO = True

# TODO: perhaps search by name not ip
# update the /etc/hosts file
def update_hosts_file(name, ip):
    name = name.replace('@vagrant/', '')
    files.line(
        {'Add hosts to /etc/hosts'},
        '/etc/hosts',
        r'{} .*'.format(ip),
        replace='{} {}.example.com {}'.format(ip, name, name),
    )

# ensure all hosts are added to each /etc/hosts file
inv = inventory.get_group('@vagrant')
for item in inv:
    update_hosts_file(item.name, item.fact.ipv4_addresses['eth0'])

if host.name == '@vagrant/master':
    server.hostname(
        {'Set the hostname for the Puppet Master'},
        'master.example.com',
    )

if host.name == '@vagrant/agent':
    server.hostname(
        {'Set the hostname for the Puppet Master'},
        'agent.example.com',
    )

if host.fact.linux_name in ['CentOS', 'RedHat']:

    yum.packages(
        {'Install chrony for Network Time Protocol (NTP)'},
        ['chrony'],
    )

    major = host.fact.linux_distribution['major']
    yum.rpm(
        {'Install Puppet Repo'},
        'https://yum.puppet.com/puppet6-release-el-{}.noarch.rpm'
        .format(major),
    )

    files.line(
        {'Ensure SELINUX is disabled'},
        '/etc/sysconfig/selinux',
        r'SELINUX=.*',
        replace='SELINUX=disabled',
    )

    # TODO: reboot after SELINUX is disabled (how to check/easy way to reboot)
    # TODO: how to determine when reboot is complete
    # TODO: run sestatus

if host.name == '@vagrant/master':

    install = yum.packages(
        {'Install puppet server'},
        ['puppetserver'],
    )

    config = files.template(
        {'Manage the puppet master configuration'},
        'templates/master_puppet.conf.j2',
        '/etc/puppetlabs/puppet/puppet.conf',
    )

    # TODO: tune always shows as changed
    # I think it should only show as changed if there really was a difference.
    # Might have to add a suffix to the sed -i option, then move file only if
    # there is a diff. Maybe?
    tune = files.line(
        {'Tune the puppet server jvm to only use 1gb'},
        '/etc/sysconfig/puppetserver',
        r'^JAVA_ARGS=.*$',
        replace='JAVA_ARGS=\\"-Xms1g -Xmx1g -Djruby.logger.class=com.puppetlabs.'
        'jruby_utils.jruby.Slf4jLogger\\"',
    )

    if install.changed or config.changed or tune.changed:
        init.systemd(
            {'Restart and enable puppetserver'},
            'puppetserver',
            running=True,
            restarted=True,
            enabled=True,
        )

    # TODO: firewall-cmd --add-port=8140/tcp --permanent
    # TODO: firewall-cmd --reload

if host.name == '@vagrant/agent':

    yum.packages(
        {'Install puppet agent'},
        ['puppet-agent'],
    )

    files.template(
        {'Manage the puppet agent configuration'},
        'templates/agent_puppet.conf.j2',
        '/etc/puppetlabs/puppet/puppet.conf',
    )

    init.systemd(
        {'Restart and enable puppet agent'},
        'puppet',
        running=True,
        restarted=True,
        enabled=True,
    )
<|endoftext|>