<commit_before>import lib.getconfig
import lib.pushdata
import lib.record_rate
import subprocess
import datetime
import os
import re

cluster_name = lib.getconfig.getparam('SelfConfig', 'cluster_name')
check_type = 'system'

reaction = -3
warn_percent = 80
rated = True
io_warning_percent = 40

def runcheck():
    local_vars = []
    jsondata = lib.pushdata.JonSon()
    jsondata.prepare_data()
    rate = lib.record_rate.ValueRate()
    timestamp = int(datetime.datetime.now().strftime("%s"))
    devices_blocks = {}
    try:
        for device in os.listdir('/sys/block'):
            if 'ram' not in device and 'loop' not in device:
                devblk = open('/sys/block/'+device+'/queue/hw_sector_size', 'r')
                devices_blocks[device] = devblk.readline().rstrip('\n')
                devblk.close()

        for key, value in list(devices_blocks.items()):
            statsfile = open('/sys/block/' + key + '/stat', 'r')
            stats = statsfile.readline().split()
            read_bytes = int(stats[2]) * int(value)
            write_bytes = int(stats[6]) * int(value)
            reads = 'drive_' + key + '_bytes_read'
            writes = 'drive_' + key + '_bytes_write'
            if rated is True:
                read_rate = rate.record_value_rate(reads, read_bytes, timestamp)
                write_rate = rate.record_value_rate(writes, write_bytes, timestamp)
                local_vars.append({'name': reads, 'timestamp': timestamp, 'value': read_rate, 'reaction': reaction})
                local_vars.append({'name': writes, 'timestamp': timestamp, 'value': write_rate, 'reaction': reaction})

            else:
                local_vars.append({'name': reads, 'timestamp': timestamp, 'value': read_bytes, 'reaction': reaction})
                local_vars.append({'name': writes, 'timestamp': timestamp, 'value': write_bytes, 'reaction': reaction})

            statsfile.close()

        command = 'df'
        p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True, universal_newlines=True)
        output, err = p.communicate()
        for i in output.split("\n"):
            if i.startswith('/'):
                u = re.sub(' +', ' ', i).split(" ")

                local_vars.append({'name': 'drive' + u[0].replace('/dev/', '_') + '_bytes_used',
                                   'timestamp': timestamp, 'value': u[2], 'reaction': reaction})
                local_vars.append({'name': 'drive' + u[0].replace('/dev/', '_') + '_bytes_available',
                                   'timestamp': timestamp, 'value': u[3], 'reaction': reaction})
                local_vars.append({'name': 'drive' + u[0].replace('/dev/', '_') + '_percent_used',
                                   'timestamp': timestamp, 'value': u[4].replace('%', ''), 'chart_type': 'Percent'})

        proc_stats = open('/proc/diskstats')
        for line in proc_stats:
            if "loop" not in line:
                fields = line.strip().split()
                name = 'drive_'+fields[2]+'_io_percent'
                regexp = re.compile(r'\d')
                if regexp.search(name) is None:
                    value = fields[12]
                    reqrate = rate.record_value_rate(name, value, timestamp)
                    if isinstance(reqrate, int):
                        diskrate = reqrate/10
                        local_vars.append({'name': name, 'timestamp': timestamp, 'value': diskrate,  'chart_type': 'Percent'})
        proc_stats.close()
        return  local_vars
    except Exception as e:
        lib.pushdata.print_error(__name__ , (e))
        pass<commit_msg>Removed unnecessary comments<commit_after>import lib.getconfig
import lib.pushdata
import lib.record_rate
import subprocess
import datetime
import os
import re

cluster_name = lib.getconfig.getparam('SelfConfig', 'cluster_name')
check_type = 'system'

reaction = -3
warn_percent = 80
rated = True
io_warning_percent = 40

def runcheck():
    local_vars = []
    jsondata = lib.pushdata.JonSon()
    jsondata.prepare_data()
    rate = lib.record_rate.ValueRate()
    timestamp = int(datetime.datetime.now().strftime("%s"))
    devices_blocks = {}
    try:
        for device in os.listdir('/sys/block'):
            if 'ram' not in device and 'loop' not in device:
                devblk = open('/sys/block/'+device+'/queue/hw_sector_size', 'r')
                devices_blocks[device] = devblk.readline().rstrip('\n')
                devblk.close()

        for key, value in list(devices_blocks.items()):
            statsfile = open('/sys/block/' + key + '/stat', 'r')
            stats = statsfile.readline().split()
            read_bytes = int(stats[2]) * int(value)
            write_bytes = int(stats[6]) * int(value)
            reads = 'drive_' + key + '_bytes_read'
            writes = 'drive_' + key + '_bytes_write'
            if rated is True:
                read_rate = rate.record_value_rate(reads, read_bytes, timestamp)
                write_rate = rate.record_value_rate(writes, write_bytes, timestamp)
                local_vars.append({'name': reads, 'timestamp': timestamp, 'value': read_rate, 'reaction': reaction})
                local_vars.append({'name': writes, 'timestamp': timestamp, 'value': write_rate, 'reaction': reaction})

            else:
                local_vars.append({'name': reads, 'timestamp': timestamp, 'value': read_bytes, 'reaction': reaction})
                local_vars.append({'name': writes, 'timestamp': timestamp, 'value': write_bytes, 'reaction': reaction})

            statsfile.close()

        d = []
        mount = open('/proc/mounts', 'r')

        for l in mount:
            if l[0] == '/':
                l = l.split()
                d.append(l[1])

        mount.close()

        def disk_usage(path):
            st = os.statvfs(path)
            free = st.f_bavail * st.f_frsize
            total = st.f_blocks * st.f_frsize
            used = (st.f_blocks - st.f_bfree) * st.f_frsize
            du = '{:.2f}'.format(used * 100 / float(total))
            return total, used, free, du

        for u in d:
            mrtrics = disk_usage(u)
            if u == '/':
                name = '_rootfs'
            else:
                name = u.replace('/', '_')

            local_vars.append({'name': 'drive' + name + '_bytes_used', 'timestamp': timestamp, 'value': mrtrics[1], 'reaction': reaction})
            local_vars.append({'name': 'drive' + name + '_bytes_available', 'timestamp': timestamp, 'value': mrtrics[2], 'reaction': reaction})
            local_vars.append({'name': 'drive' + name + '_percent_used','timestamp': timestamp, 'value': mrtrics[3], 'chart_type': 'Percent'})
            # print(disk_usage(u))

        proc_stats = open('/proc/diskstats')
        for line in proc_stats:
            if "loop" not in line:
                fields = line.strip().split()
                name = 'drive_'+fields[2]+'_io_percent'
                regexp = re.compile(r'\d')
                if regexp.search(name) is None:
                    value = fields[12]
                    reqrate = rate.record_value_rate(name, value, timestamp)
                    if isinstance(reqrate, int):
                        diskrate = reqrate/10
                        local_vars.append({'name': name, 'timestamp': timestamp, 'value': diskrate,  'chart_type': 'Percent'})
        proc_stats.close()
        return  local_vars
    except Exception as e:
        lib.pushdata.print_error(__name__ , (e))
        pass

<|endoftext|>


<commit_before>#:coding=utf8:

from django.conf.urls import patterns, url
from django.utils.http import urlquote
from django.http import (
    HttpResponseRedirect,
    HttpResponsePermanentRedirect,
    HttpResponseGone,
)

def redirect_to(request, url, permanent=True, **kwargs):
    """
    A copy of the redirect_to view from django.views.generic.simple
    Copied since redirect_to doesn't work for urls that contain non-ascii
    keyword arguments.
    """
    if url is not None:
        klass = (permanent and HttpResponsePermanentRedirect
                 or HttpResponseRedirect)
        quoted_kwargs = {}
        for k, v in kwargs.iteritems():
            quoted_kwargs[k] = urlquote(v)
        return klass(url % quoted_kwargs)
    else:
        return HttpResponseGone()

urlpatterns = patterns(
    '',
    url(r'^page/(?P<page>\d+)/?$', redirect_to, {'url': '/?page=%(page)s'}, name="lifestream_page_redirect"),
    url(r'^projects(?:/.*)?$', redirect_to, {'url': '/'}, name="projects_redirect"),
    #url(r'^en/about(?:/.*)?$', redirect_to, {'url': '/'}, name="en_about_redirect"),
    url(r'^jp/about(?:/.*)?$', redirect_to, {'url': '/en/about/'}, name="jp_about_redirect"),
    url(r'^jp/aboutjp$', redirect_to, {'url': '/en/about/'}, name="jp_about_redirect2"),

    # Legacy urls for the blog
    url(r'^index.php/(?P<locale>\w{2})/?$', redirect_to, {'url': '/%(locale)s/'}),
    url(r'^index.php$', redirect_to, {'url': '/en/'}),
    url(r'^index.php/(?P<locale>\w{2})/(?P<slug>[^/]+)/?$', redirect_to, {'url': '/%(locale)s/%(slug)s'}),

    url(r'^en/(?P<tag_name>.+)\;', redirect_to, {'url': '/en/tag/%(tag_name)s'}),
    url(r'^jp/(?P<tag_name>.+)\;', redirect_to, {'url': '/jp/tag/%(tag_name)s'}),

    # Old lifestream urls.
    url(r'^items/', redirect_to, {'url': '/'}),
)
<commit_msg>Updated an old lifestream URL redirect<commit_after>#:coding=utf8:

from django.conf.urls import patterns, url
from django.utils.http import urlquote
from django.http import (
    HttpResponseRedirect,
    HttpResponsePermanentRedirect,
    HttpResponseGone,
)

def redirect_to(request, url, permanent=True, **kwargs):
    """
    A copy of the redirect_to view from django.views.generic.simple
    Copied since redirect_to doesn't work for urls that contain non-ascii
    keyword arguments.
    """
    if url is not None:
        klass = (permanent and HttpResponsePermanentRedirect
                 or HttpResponseRedirect)
        quoted_kwargs = {}
        for k, v in kwargs.iteritems():
            quoted_kwargs[k] = urlquote(v)
        return klass(url % quoted_kwargs)
    else:
        return HttpResponseGone()

urlpatterns = patterns(
    '',

    url(r'^projects(?:/.*)?$', redirect_to, {'url': '/'}, name="projects_redirect"),

    #url(r'^en/about(?:/.*)?$', redirect_to, {'url': '/'}, name="en_about_redirect"),
    url(r'^jp/about(?:/.*)?$', redirect_to, {'url': '/en/about/'}, name="jp_about_redirect"),
    url(r'^jp/aboutjp$', redirect_to, {'url': '/en/about/'}, name="jp_about_redirect2"),

    # Really old legacy urls from the b2evo blog
    url(r'^index.php/(?P<locale>\w{2})/?$', redirect_to, {'url': '/%(locale)s/'}),
    url(r'^index.php$', redirect_to, {'url': '/en/'}),
    url(r'^index.php/(?P<locale>\w{2})/(?P<slug>[^/]+)/?$', redirect_to, {'url': '/%(locale)s/%(slug)s'}),

    url(r'^en/(?P<tag_name>.+)\;', redirect_to, {'url': '/en/tag/%(tag_name)s'}),
    url(r'^jp/(?P<tag_name>.+)\;', redirect_to, {'url': '/jp/tag/%(tag_name)s'}),

    # Old lifestream urls.
    url(r'^items/', redirect_to, {'url': '/'}),
    url(r'^page/(?P<page>\d+)/?$', redirect_to, {'url': '/'}, name="lifestream_page_redirect"),
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# noinspection PyUnresolvedReferences
from .settings import *

ETC_DIR = os.path.join(BASE_DIR, 'etc')

# Django
DEBUG = False

ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', '23.226.78.111', '.moonfruit.top']

with open(os.path.join(ETC_DIR, 'secret_key.txt')) as f:
    SECRET_KEY = f.read().strip()

STATIC_ROOT = os.path.join(VAR_DIR, 'static')
MEDIA_ROOT = os.path.join(VAR_DIR, 'media')

os.makedirs(STATIC_ROOT, exist_ok=True)
os.makedirs(MEDIA_ROOT, exist_ok=True)

# Security
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_HSTS_SECONDS = 31536000
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_SSL_REDIRECT = True

SESSION_COOKIE_SECURE = True

CSRF_COOKIE_SECURE = True
CSRF_COOKIE_HTTPONLY = True

X_FRAME_OPTIONS = 'DENY'

# Log
LOGGING.update({
    'root': {
        'handlers': ['file'],
        'level': 'DEBUG',
    },
})

# Cache
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
        'LOCATION': '/var/run/memcached.sock',
    }
}

# Email
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
# noinspection SpellCheckingInspection
EMAIL_HOST_PASSWORD = ''

# YYFeed
YYFEED_CACHE = {
    '()': 'yyutil.cache.MemCache',
    'servers': ['/var/run/memcached.sock'],
}
<commit_msg>bug fix host<commit_after># -*- coding: utf-8 -*-
# noinspection PyUnresolvedReferences
from .settings import *

ETC_DIR = os.path.join(BASE_DIR, 'etc')

# Django
DEBUG = False

ALLOWED_HOSTS = ['localhost', '127.0.0.1', '::1', '23.105.201.47', '.moonfruit.top']

with open(os.path.join(ETC_DIR, 'secret_key.txt')) as f:
    SECRET_KEY = f.read().strip()

STATIC_ROOT = os.path.join(VAR_DIR, 'static')
MEDIA_ROOT = os.path.join(VAR_DIR, 'media')

os.makedirs(STATIC_ROOT, exist_ok=True)
os.makedirs(MEDIA_ROOT, exist_ok=True)

# Security
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_HSTS_SECONDS = 31536000
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_SSL_REDIRECT = True

SESSION_COOKIE_SECURE = True

CSRF_COOKIE_SECURE = True
CSRF_COOKIE_HTTPONLY = True

X_FRAME_OPTIONS = 'DENY'

# Log
LOGGING.update({
    'root': {
        'handlers': ['file'],
        'level': 'DEBUG',
    },
})

# Cache
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
        'LOCATION': '/run/memcached/memcached.sock',
    }
}

# Email
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
# noinspection SpellCheckingInspection
EMAIL_HOST_PASSWORD = ''

# YYFeed
YYFEED_CACHE = {
    '()': 'yyutil.cache.MemCache',
    'servers': ['/run/memcached/memcached.sock'],
}
<|endoftext|>


<commit_before>from ..external.qt import *
import numpy as np
from itertools import product
import random

class ControlWidget(QWidget):
    """Control a simulator with buttons or from the interactive console."""

    start_inverting_frames = Signal()
    stop_inverting_frames = Signal()

    def __init__(self, simulator, **kwargs):
        """:param simulator: The simulator object to control."""
        super(ControlWidget, self).__init__(**kwargs)

        self.sim = simulator
        self.timer_delay = 0

        self._setup_ui()

    def _setup_ui(self):
        """Setup the widgets, connect the signals&slots."""
        l = QHBoxLayout(self)
        self.start_button = QPushButton("&Start", self)
        self.stop_button = QPushButton("&Stop", self)
        self.stop_button.hide()
        delay = QSpinBox()
        delay.setMinimum(0)
        delay.setMaximum(10000)
        delay.setSuffix(" ms")
        delay.setValue(self.timer_delay)

        l.addWidget(self.start_button)
        l.addWidget(self.stop_button)
        l.addWidget(delay)

        l.addSpacing(11)
        reset_button = QPushButton("&reset", self)
        reset_button.clicked.connect(self.set_config)
        l.addWidget(reset_button)

        self.zero_percentage = QSpinBox(self)
        self.zero_percentage.setMaximum(99)
        self.zero_percentage.setMinimum(1)
        self.zero_percentage.setValue(50)
        self.zero_percentage.setSuffix("% black")
        l.addWidget(self.zero_percentage)

        self.invert_frames = QCheckBox(self)
        self.invert_frames.setChecked(False)
        self.invert_frames.setText("&invert odd frames")
        self.invert_frames.stateChanged.connect(self.invert_odd)
        l.addWidget(self.invert_frames)

        self.setLayout(l)

        self.start_button.clicked.connect(self.start)
        self.stop_button.clicked.connect(self.stop)
        delay.valueChanged.connect(self.change_delay)

    def invert_odd(self, value):
        if value == Qt.Checked:
            self.start_inverting_frames.emit()
        else:
            self.stop_inverting_frames.emit()

    def start(self):
        """Start running the simulator."""
        self.timer_id = self.startTimer(self.timer_delay)
        self.start_button.hide()
        self.stop_button.show()

    def stop(self):
        """Stop the simulator."""
        self.killTimer(self.timer_id)
        self.stop_button.hide()
        self.start_button.show()

    def change_delay(self, delay):
        """Change the timer delay of the simulator steps."""
        if delay.endswith("ms"):
            delay = delay[:-2]
        self.timer_delay = int(delay)

    def timerEvent(self, event):
        """Step the simulator from the timer.

        .. note::
            This is called by the timer that is controlled by :meth:`start` and
            :meth:`stop`. You should not call it yourself."""
        self.killTimer(self.timer_id)
        self.step()
        self.timer_id = self.startTimer(self.timer_delay)

    def step(self):
        """Step the simulator, update all displays."""
        self.sim.step()

    def fullspeed(self):
        """Run the stepping function without any timer delays."""
        last_time = time.time()
        last_step = 0
        while self.isVisible():
            self.step()
            last_step += 1
            QApplication.processEvents()
            if last_step % 1000 == 1:
                diff, last_time = time.time() - last_time, time.time()
                print last_step, diff

    def set_config(self, conf=None):
        if conf is None:
            conf = np.zeros(self.sim.get_config().shape, self.sim.get_config().dtype)
            positions = product(*[range(size) for size in conf.shape])
            zero_perc = self.zero_percentage.value() / 100.
            for pos in positions:
                if random.random() > zero_perc:
                    conf[pos] = 1

        self.sim.set_config(conf)
<commit_msg>use the black percentage for black, fill the rest up with random states<commit_after>from ..external.qt import *
import numpy as np
from itertools import product
import random

class ControlWidget(QWidget):
    """Control a simulator with buttons or from the interactive console."""

    start_inverting_frames = Signal()
    stop_inverting_frames = Signal()

    def __init__(self, simulator, **kwargs):
        """:param simulator: The simulator object to control."""
        super(ControlWidget, self).__init__(**kwargs)

        self.sim = simulator
        self.timer_delay = 0

        self._setup_ui()

    def _setup_ui(self):
        """Setup the widgets, connect the signals&slots."""
        l = QHBoxLayout(self)
        self.start_button = QPushButton("&Start", self)
        self.stop_button = QPushButton("&Stop", self)
        self.stop_button.hide()
        delay = QSpinBox()
        delay.setMinimum(0)
        delay.setMaximum(10000)
        delay.setSuffix(" ms")
        delay.setValue(self.timer_delay)

        l.addWidget(self.start_button)
        l.addWidget(self.stop_button)
        l.addWidget(delay)

        l.addSpacing(11)
        reset_button = QPushButton("&reset", self)
        reset_button.clicked.connect(self.set_config)
        l.addWidget(reset_button)

        self.zero_percentage = QSpinBox(self)
        self.zero_percentage.setMaximum(99)
        self.zero_percentage.setMinimum(1)
        self.zero_percentage.setValue(50)
        self.zero_percentage.setSuffix("% black")
        l.addWidget(self.zero_percentage)

        self.invert_frames = QCheckBox(self)
        self.invert_frames.setChecked(False)
        self.invert_frames.setText("&invert odd frames")
        self.invert_frames.stateChanged.connect(self.invert_odd)
        l.addWidget(self.invert_frames)

        self.setLayout(l)

        self.start_button.clicked.connect(self.start)
        self.stop_button.clicked.connect(self.stop)
        delay.valueChanged.connect(self.change_delay)

    def invert_odd(self, value):
        if value == Qt.Checked:
            self.start_inverting_frames.emit()
        else:
            self.stop_inverting_frames.emit()

    def start(self):
        """Start running the simulator."""
        self.timer_id = self.startTimer(self.timer_delay)
        self.start_button.hide()
        self.stop_button.show()

    def stop(self):
        """Stop the simulator."""
        self.killTimer(self.timer_id)
        self.stop_button.hide()
        self.start_button.show()

    def change_delay(self, delay):
        """Change the timer delay of the simulator steps."""
        if delay.endswith("ms"):
            delay = delay[:-2]
        self.timer_delay = int(delay)

    def timerEvent(self, event):
        """Step the simulator from the timer.

        .. note::
            This is called by the timer that is controlled by :meth:`start` and
            :meth:`stop`. You should not call it yourself."""
        self.killTimer(self.timer_id)
        self.step()
        self.timer_id = self.startTimer(self.timer_delay)

    def step(self):
        """Step the simulator, update all displays."""
        self.sim.step()

    def fullspeed(self):
        """Run the stepping function without any timer delays."""
        last_time = time.time()
        last_step = 0
        while self.isVisible():
            self.step()
            last_step += 1
            QApplication.processEvents()
            if last_step % 1000 == 1:
                diff, last_time = time.time() - last_time, time.time()
                print last_step, diff

    def set_config(self, conf=None):
        if conf is None:
            conf = np.zeros(self.sim.get_config().shape, self.sim.get_config().dtype)
            positions = product(*[range(size) for size in conf.shape])
            zero_perc = self.zero_percentage.value() / 100.
            for pos in positions:
                if random.random() > zero_perc:
                    conf[pos] = random.choice(self.sim.t.possible_values[1:])

        self.sim.set_config(conf)
<|endoftext|>


<commit_before>from ..external.qt import *

class ControlWidget(QWidget):
    """Control a simulator with buttons or from the interactive console."""

    start_inverting_frames = Signal()
    stop_inverting_frames = Signal()

    def __init__(self, simulator, **kwargs):
        """:param simulator: The simulator object to control."""
        super(ControlWidget, self).__init__(**kwargs)

        self.sim = simulator
        self.timer_delay = 0

        self._setup_ui()

    def _setup_ui(self):
        """Setup the widgets, connect the signals&slots."""
        l = QHBoxLayout(self)
        self.start_button = QPushButton("&Start", self)
        self.stop_button = QPushButton("&Stop", self)
        self.stop_button.hide()
        delay = QSpinBox()
        delay.setMinimum(0)
        delay.setMaximum(10000)
        delay.setSuffix(" ms")
        delay.setValue(self.timer_delay)

        l.addWidget(self.start_button)
        l.addWidget(self.stop_button)
        l.addWidget(delay)

        l.addSpacing(11)
        reset_button = QPushButton("&reset", self)
        reset_button.clicked.connect(self.set_config)
        l.addWidget(reset_button)

        self.zero_percentage = QSpinBox(self)
        self.zero_percentage.setMaximum(99)
        self.zero_percentage.setMinimum(1)
        self.zero_percentage.setValue(50)
        self.zero_percentage.setSuffix("% black")
        l.addWidget(self.zero_percentage)

        self.invert_frames = QCheckBox(self)
        self.invert_frames.setChecked(False)
        self.invert_frames.setText("&invert odd frames")
        self.invert_frames.stateChanged.connect(self.invert_odd)
        l.addWidget(self.invert_frames)

        self.setLayout(l)

        self.start_button.clicked.connect(self.start)
        self.stop_button.clicked.connect(self.stop)
        delay.valueChanged.connect(self.change_delay)

    def invert_odd(self, value):
        if value == Qt.Checked:
            self.start_inverting_frames.emit()
        else:
            self.stop_inverting_frames.emit()

    def start(self):
        """Start running the simulator."""
        self.timer_id = self.startTimer(self.timer_delay)
        self.start_button.hide()
        self.stop_button.show()

    def stop(self):
        """Stop the simulator."""
        self.killTimer(self.timer_id)
        self.stop_button.hide()
        self.start_button.show()

    def change_delay(self, delay):
        """Change the timer delay of the simulator steps."""
        if delay.endswith("ms"):
            delay = delay[:-2]
        self.timer_delay = int(delay)

    def timerEvent(self, event):
        """Step the simulator from the timer.

        .. note::
            This is called by the timer that is controlled by :meth:`start` and
            :meth:`stop`. You should not call it yourself."""
        self.killTimer(self.timer_id)
        self.step()
        self.timer_id = self.startTimer(self.timer_delay)

    def step(self):
        """Step the simulator, update all displays."""
        self.sim.step()

    def fullspeed(self):
        """Run the stepping function without any timer delays."""
        last_time = time.time()
        last_step = 0
        while self.isVisible():
            self.step()
            last_step += 1
            QApplication.processEvents()
            if last_step % 1000 == 1:
                diff, last_time = time.time() - last_time, time.time()
                print last_step, diff

    def set_config(self, conf=None):
        if conf is None:
            conf = np.zeros(self.sim.get_config().shape, self.sim.get_config().dtype)
            positions = product(*[range(size) for size in conf.shape])
            zero_perc = self.zero_percentage.value() / 100.
            for pos in positions:
                if random.random() > zero_perc:
                    conf[pos] = 1

        self.sim.set_config(conf)
<commit_msg>fix imports for control widget.<commit_after>from ..external.qt import *
import numpy as np
from itertools import product
import random

class ControlWidget(QWidget):
    """Control a simulator with buttons or from the interactive console."""

    start_inverting_frames = Signal()
    stop_inverting_frames = Signal()

    def __init__(self, simulator, **kwargs):
        """:param simulator: The simulator object to control."""
        super(ControlWidget, self).__init__(**kwargs)

        self.sim = simulator
        self.timer_delay = 0

        self._setup_ui()

    def _setup_ui(self):
        """Setup the widgets, connect the signals&slots."""
        l = QHBoxLayout(self)
        self.start_button = QPushButton("&Start", self)
        self.stop_button = QPushButton("&Stop", self)
        self.stop_button.hide()
        delay = QSpinBox()
        delay.setMinimum(0)
        delay.setMaximum(10000)
        delay.setSuffix(" ms")
        delay.setValue(self.timer_delay)

        l.addWidget(self.start_button)
        l.addWidget(self.stop_button)
        l.addWidget(delay)

        l.addSpacing(11)
        reset_button = QPushButton("&reset", self)
        reset_button.clicked.connect(self.set_config)
        l.addWidget(reset_button)

        self.zero_percentage = QSpinBox(self)
        self.zero_percentage.setMaximum(99)
        self.zero_percentage.setMinimum(1)
        self.zero_percentage.setValue(50)
        self.zero_percentage.setSuffix("% black")
        l.addWidget(self.zero_percentage)

        self.invert_frames = QCheckBox(self)
        self.invert_frames.setChecked(False)
        self.invert_frames.setText("&invert odd frames")
        self.invert_frames.stateChanged.connect(self.invert_odd)
        l.addWidget(self.invert_frames)

        self.setLayout(l)

        self.start_button.clicked.connect(self.start)
        self.stop_button.clicked.connect(self.stop)
        delay.valueChanged.connect(self.change_delay)

    def invert_odd(self, value):
        if value == Qt.Checked:
            self.start_inverting_frames.emit()
        else:
            self.stop_inverting_frames.emit()

    def start(self):
        """Start running the simulator."""
        self.timer_id = self.startTimer(self.timer_delay)
        self.start_button.hide()
        self.stop_button.show()

    def stop(self):
        """Stop the simulator."""
        self.killTimer(self.timer_id)
        self.stop_button.hide()
        self.start_button.show()

    def change_delay(self, delay):
        """Change the timer delay of the simulator steps."""
        if delay.endswith("ms"):
            delay = delay[:-2]
        self.timer_delay = int(delay)

    def timerEvent(self, event):
        """Step the simulator from the timer.

        .. note::
            This is called by the timer that is controlled by :meth:`start` and
            :meth:`stop`. You should not call it yourself."""
        self.killTimer(self.timer_id)
        self.step()
        self.timer_id = self.startTimer(self.timer_delay)

    def step(self):
        """Step the simulator, update all displays."""
        self.sim.step()

    def fullspeed(self):
        """Run the stepping function without any timer delays."""
        last_time = time.time()
        last_step = 0
        while self.isVisible():
            self.step()
            last_step += 1
            QApplication.processEvents()
            if last_step % 1000 == 1:
                diff, last_time = time.time() - last_time, time.time()
                print last_step, diff

    def set_config(self, conf=None):
        if conf is None:
            conf = np.zeros(self.sim.get_config().shape, self.sim.get_config().dtype)
            positions = product(*[range(size) for size in conf.shape])
            zero_perc = self.zero_percentage.value() / 100.
            for pos in positions:
                if random.random() > zero_perc:
                    conf[pos] = 1

        self.sim.set_config(conf)
<|endoftext|>


<commit_before>"""
Main Zencoder module
"""

import os
import json
import httplib2

class ZencoderError(Exception):
    pass

class HTTPBackend(object):
    """
    Abstracts out an HTTP backend, but defaults to httplib2

    @FIXME: Build in support for supplying arbitrary backends
    """
    def __init__(self, as_xml=False):
        """
        Creates an HTTPBackend object, which abstracts out some of the
        library specific HTTP stuff.
        """
        self.base_url = 'https://app.zencoder.com/api/'

        #TODO investigate httplib2 caching and if it is necessary
        self.http = httplib2.Http()
        self.as_xml = as_xml

        if self.as_xml:
            self.headers = {'Content-Type': 'application/xml',
                            'Accepts': 'application/xml'}
        else:
            self.headers = {'Content-Type': 'application/json',
                            'Accepts': 'application/json'}

    def encode(self, data):
        """
        Encodes data as either JSON or XML, so that it can be passed onto
        the Zencoder API
        """
        if not self.as_xml:
            return json.dumps(data)
        else:
            raise NotImplementedError('Encoding as XML is not supported.')

    def decode(self, raw_body):
        """
        Returns the raw_body as json (the default) or XML
        """
        if not self.as_xml:
            return json.loads(raw_body)

    def post(self, url, body=None):
        """
        Execute a HTTP POST request for the given URL
        """
        response, content = self.http.request(url, method="POST",
                                              body=body,
                                              headers=self.headers)

        return self.process(response, content)

    def process(self, http_response, content):
        """
        Returns HTTP backend agnostic Response data
        """
        code = http_response.status
        body = self.decode(content)
        response = Response(code, body, content, http_response)
        return response

class Zencoder(object):
    """ This is the entry point to the Zencoder API """
    def __init__(self, api_key=None, as_xml=False):
        """
        Initializes Zencoder. You must have a valid API_KEY.

        You can pass in the api_key as an argument, or set
        'ZENCODER_API_KEY' as an environment variable, and it will use
        that, if api_key is unspecified.

        Set as_xml=True to get back xml data instead of the default json.
        """
        self.base_url = 'https://app.zencoder.com/api/'
        if not api_key:
            try:
                self.api_key = os.environ['ZENCODER_API_KEY']
            except KeyError:
                raise ZencoderError('ZENCODER_API_KEY not set')
        else:
            self.api_key = api_key

        self.as_xml = as_xml
        self.job = Job(self.api_key, self.as_xml)
        self.account = Account(self.api_key, self.as_xml)
        self.notification = Notification(self.api_key, self.as_xml)
        self.output = Output(self.api_key, self.as_xml)

    def get_jobs(self):
        url = self.base_url + 'jobs'
        url = '%s?api_key=%s' % (url, self.api_key)
        #data = urllib.urlencode({'api_key': self.api_key})

        request = urllib2.Request(url)

        response = urllib2.urlopen(request).read()
        if not self.as_xml:
            response = json.loads(response)

        return [Job(x) for x in response]

    def create_job(self, input):
        """ creates a zencoder job """
        url = self.base_url + 'jobs'
        data = json.dumps({"api_key":self.api_key, "input": input})

        headers = {'Content-Type': 'application/json',
                   'Accept': 'application/json'}
        req = self.http.request(url, method="POST", body=data,
                              headers=headers)

        return json.loads(req[-1])

class Response(object):
    """ Response object """
    pass

class Output(object):
    """ Output object """
    pass

class Job(object):
    """ Job object """
    def __init__(self, api_key):
        """
        Initialize a job object
        """
        pass

    def create(self, input, outputs=None, options=None):
        """
        Create a job
        """
        pass

    def list(self, page, per_page):
        """
        List some jobs
        """
        pass

    def details(self, job_id):
        """
        Get some job details
        """
        pass

    def resubmit(self, job_id):
        """
        Resubmits a job
        """
        pass

    def cancel(self, job_id):
        """
        Cancels a job
        """
        pass

    def delete(self, job_id):
        """
        Deletes a job
        """
        pass

class Notification(object):
    """ Notification object """
    pass

class Account(object):
    """ Account object """
    pass

<commit_msg>got Job.create working<commit_after>"""
Main Zencoder module
"""

import os
import json
import httplib2

class ZencoderError(Exception):
    pass

class HTTPBackend(object):
    """
    Abstracts out an HTTP backend, but defaults to httplib2

    @FIXME: Build in support for supplying arbitrary backends
    """
    def __init__(self, as_xml=False):
        """
        Creates an HTTPBackend object, which abstracts out some of the
        library specific HTTP stuff.
        """
        self.base_url = 'https://app.zencoder.com/api/'

        #TODO investigate httplib2 caching and if it is necessary
        self.http = httplib2.Http()
        self.as_xml = as_xml

        if self.as_xml:
            self.headers = {'Content-Type': 'application/xml',
                            'Accepts': 'application/xml'}
        else:
            self.headers = {'Content-Type': 'application/json',
                            'Accepts': 'application/json'}

    def encode(self, data):
        """
        Encodes data as either JSON or XML, so that it can be passed onto
        the Zencoder API
        """
        if not self.as_xml:
            return json.dumps(data)
        else:
            raise NotImplementedError('Encoding as XML is not supported.')

    def decode(self, raw_body):
        """
        Returns the raw_body as json (the default) or XML
        """
        if not self.as_xml:
            return json.loads(raw_body)

    def post(self, url, body=None):
        """
        Execute a HTTP POST request for the given URL
        """
        response, content = self.http.request(url, method="POST",
                                              body=body,
                                              headers=self.headers)

        return self.process(response, content)

    def process(self, http_response, content):
        """
        Returns HTTP backend agnostic Response data
        """
        code = http_response.status
        body = self.decode(content)
        response = Response(code, body, content, http_response)
        return response

class Zencoder(object):
    """ This is the entry point to the Zencoder API """
    def __init__(self, api_key=None, as_xml=False):
        """
        Initializes Zencoder. You must have a valid API_KEY.

        You can pass in the api_key as an argument, or set
        'ZENCODER_API_KEY' as an environment variable, and it will use
        that, if api_key is unspecified.

        Set as_xml=True to get back xml data instead of the default json.
        """
        self.base_url = 'https://app.zencoder.com/api/'
        if not api_key:
            try:
                self.api_key = os.environ['ZENCODER_API_KEY']
            except KeyError:
                raise ZencoderError('ZENCODER_API_KEY not set')
        else:
            self.api_key = api_key

        self.as_xml = as_xml
        self.job = Job(self.api_key, self.as_xml)
        self.account = Account(self.api_key, self.as_xml)
        self.notification = Notification(self.api_key, self.as_xml)
        self.output = Output(self.api_key, self.as_xml)

class Response(object):
    """ Response object """
    def __init__(self, code, body, raw_body, raw_response):
        self.code = code
        self.body = body
        self.raw_body = raw_body
        self.raw_response = raw_response

class Output(object):
    """ Output object """
    pass

class Job(HTTPBackend):
    """
    Contains all the methods that can be performed relating to Jobs with
    the Zencoder API
    """
    def __init__(self, api_key, as_xml=False):
        """
        Initialize a job object
        """
        super(Job, self).__init__()
        self.api_key = api_key
        self.as_xml = as_xml
        self.base_url = self.base_url + 'jobs'

    def create(self, input, outputs=None, options=None):
        """
        Create a job

        """
        data = {"api_key": self.api_key, "input": input}
        if outputs:
            data['outputs'] = outputs
        if options:
            data['options'] = options
        return self.post(self.base_url, body=self.encode(data))

    def list(self, page, per_page):
        """
        List some jobs
        """
        pass

    def details(self, job_id):
        """
        Get some job details
        """
        pass

    def resubmit(self, job_id):
        """
        Resubmits a job
        """
        pass

    def cancel(self, job_id):
        """
        Cancels a job
        """
        pass

    def delete(self, job_id):
        """
        Deletes a job
        """
        pass

class Notification(object):
    """ Notification object """
    pass

class Account(object):
    """ Account object """
    pass

<|endoftext|>


<commit_before>from django.conf import settings

if False:
    from zerver.models import UserProfile

from typing import Any, Dict, Optional, Text

from zerver.lib.avatar_hash import gravatar_hash, user_avatar_path_from_ids
from zerver.lib.upload import upload_backend, MEDIUM_AVATAR_SIZE
from zerver.models import UserProfile
from six.moves import urllib

def avatar_url(user_profile, medium=False):
    # type: (UserProfile, bool) -> Text
    return avatar_url_from_dict(
        dict(
            avatar_source=user_profile.avatar_source,
            avatar_version=user_profile.avatar_version,
            email=user_profile.email,
            id=user_profile.id,
            realm_id=user_profile.realm_id),
        medium=medium)

def avatar_url_from_dict(userdict, medium=False):
    # type: (Dict[str, Any], bool) -> Text
    '''
    DEPRECATED: We should start using
                get_avatar_field to populate users,
                particulary for codepaths where the
                client can compute gravatar URLS
                on the client side.
    '''
    url = _get_unversioned_avatar_url(
        userdict['id'],
        userdict['avatar_source'],
        userdict['realm_id'],
        email=userdict['email'],
        medium=medium)
    url += '&version=%d' % (userdict['avatar_version'],)
    return url

def get_avatar_field(user_id,
                     realm_id,
                     email,
                     avatar_source,
                     avatar_version,
                     medium,
                     client_gravatar):
    # type: (int, int, Text, Text, int, bool, bool) -> Optional[Text]
    '''
    Most of the parameters to this function map to fields
    by the same name in UserProfile (avatar_source, realm_id,
    email, etc.).

    Then there are these:

        medium - This means we want a medium-sized avatar. This can
            affect the "s" parameter for gravatar avatars, or it
            can give us something like foo-medium.png for
            user-uploaded avatars.

        client_gravatar - If the client can compute their own
            gravatars, this will be set to True, and we'll avoid
            computing them on the server (mostly to save bandwidth).
    '''

    if client_gravatar:
        '''
        If our client knows how to calculate gravatar hashes, we
        will return None and let the client compute the gravatar
        url.
        '''
        if settings.ENABLE_GRAVATAR:
            if avatar_source == UserProfile.AVATAR_FROM_GRAVATAR:
                return None

    '''
    If we get this far, we'll compute an avatar URL that may be
    either user-uploaded or a gravatar, and then we'll add version
    info to try to avoid stale caches.
    '''
    url = _get_unversioned_avatar_url(
        user_profile_id=user_id,
        avatar_source=avatar_source,
        realm_id=realm_id,
        email=email,
        medium=medium,
    )
    url += '&version=%d' % (avatar_version,)
    return url

def get_gravatar_url(email, avatar_version, medium=False):
    # type: (Text, int, bool) -> Text
    url = _get_unversioned_gravatar_url(email, medium)
    url += '&version=%d' % (avatar_version,)
    return url

def _get_unversioned_gravatar_url(email, medium):
    # type: (Text, bool) -> Text
    if settings.ENABLE_GRAVATAR:
        gravitar_query_suffix = "&s=%s" % (MEDIUM_AVATAR_SIZE,) if medium else ""
        hash_key = gravatar_hash(email)
        return u"https://secure.gravatar.com/avatar/%s?d=identicon%s" % (hash_key, gravitar_query_suffix)
    return settings.DEFAULT_AVATAR_URI+'?x=x'

def _get_unversioned_avatar_url(user_profile_id, avatar_source, realm_id, email=None, medium=False):
    # type: (int, Text, int, Optional[Text], bool) -> Text
    if avatar_source == u'U':
        hash_key = user_avatar_path_from_ids(user_profile_id, realm_id)
        return upload_backend.get_avatar_url(hash_key, medium=medium)
    assert email is not None
    return _get_unversioned_gravatar_url(email, medium)

def absolute_avatar_url(user_profile):
    # type: (UserProfile) -> Text
    """Absolute URLs are used to simplify logic for applications that
    won't be served by browsers, such as rendering GCM notifications."""
    return urllib.parse.urljoin(user_profile.realm.uri, avatar_url(user_profile))
<commit_msg>refactor: Simplify avatar_url.<commit_after>from django.conf import settings

if False:
    from zerver.models import UserProfile

from typing import Any, Dict, Optional, Text

from zerver.lib.avatar_hash import gravatar_hash, user_avatar_path_from_ids
from zerver.lib.upload import upload_backend, MEDIUM_AVATAR_SIZE
from zerver.models import UserProfile
from six.moves import urllib

def avatar_url(user_profile, medium=False, client_gravatar=False):
    # type: (UserProfile, bool, bool) -> Text

    return get_avatar_field(
        user_id=user_profile.id,
        realm_id=user_profile.realm_id,
        email=user_profile.email,
        avatar_source=user_profile.avatar_source,
        avatar_version=user_profile.avatar_version,
        medium=medium,
        client_gravatar=client_gravatar,
    )

def avatar_url_from_dict(userdict, medium=False):
    # type: (Dict[str, Any], bool) -> Text
    '''
    DEPRECATED: We should start using
                get_avatar_field to populate users,
                particulary for codepaths where the
                client can compute gravatar URLS
                on the client side.
    '''
    url = _get_unversioned_avatar_url(
        userdict['id'],
        userdict['avatar_source'],
        userdict['realm_id'],
        email=userdict['email'],
        medium=medium)
    url += '&version=%d' % (userdict['avatar_version'],)
    return url

def get_avatar_field(user_id,
                     realm_id,
                     email,
                     avatar_source,
                     avatar_version,
                     medium,
                     client_gravatar):
    # type: (int, int, Text, Text, int, bool, bool) -> Optional[Text]
    '''
    Most of the parameters to this function map to fields
    by the same name in UserProfile (avatar_source, realm_id,
    email, etc.).

    Then there are these:

        medium - This means we want a medium-sized avatar. This can
            affect the "s" parameter for gravatar avatars, or it
            can give us something like foo-medium.png for
            user-uploaded avatars.

        client_gravatar - If the client can compute their own
            gravatars, this will be set to True, and we'll avoid
            computing them on the server (mostly to save bandwidth).
    '''

    if client_gravatar:
        '''
        If our client knows how to calculate gravatar hashes, we
        will return None and let the client compute the gravatar
        url.
        '''
        if settings.ENABLE_GRAVATAR:
            if avatar_source == UserProfile.AVATAR_FROM_GRAVATAR:
                return None

    '''
    If we get this far, we'll compute an avatar URL that may be
    either user-uploaded or a gravatar, and then we'll add version
    info to try to avoid stale caches.
    '''
    url = _get_unversioned_avatar_url(
        user_profile_id=user_id,
        avatar_source=avatar_source,
        realm_id=realm_id,
        email=email,
        medium=medium,
    )
    url += '&version=%d' % (avatar_version,)
    return url

def get_gravatar_url(email, avatar_version, medium=False):
    # type: (Text, int, bool) -> Text
    url = _get_unversioned_gravatar_url(email, medium)
    url += '&version=%d' % (avatar_version,)
    return url

def _get_unversioned_gravatar_url(email, medium):
    # type: (Text, bool) -> Text
    if settings.ENABLE_GRAVATAR:
        gravitar_query_suffix = "&s=%s" % (MEDIUM_AVATAR_SIZE,) if medium else ""
        hash_key = gravatar_hash(email)
        return u"https://secure.gravatar.com/avatar/%s?d=identicon%s" % (hash_key, gravitar_query_suffix)
    return settings.DEFAULT_AVATAR_URI+'?x=x'

def _get_unversioned_avatar_url(user_profile_id, avatar_source, realm_id, email=None, medium=False):
    # type: (int, Text, int, Optional[Text], bool) -> Text
    if avatar_source == u'U':
        hash_key = user_avatar_path_from_ids(user_profile_id, realm_id)
        return upload_backend.get_avatar_url(hash_key, medium=medium)
    assert email is not None
    return _get_unversioned_gravatar_url(email, medium)

def absolute_avatar_url(user_profile):
    # type: (UserProfile) -> Text
    """Absolute URLs are used to simplify logic for applications that
    won't be served by browsers, such as rendering GCM notifications."""
    return urllib.parse.urljoin(user_profile.realm.uri, avatar_url(user_profile))
<|endoftext|>


<commit_before>import datetime
import logging
from collections import defaultdict
from typing import Any, Dict, List, Set, Tuple, Union

from django.conf import settings
from django.utils.timezone import now as timezone_now

from confirmation.models import one_click_unsubscribe_link
from zerver.context_processors import common_context
from zerver.lib.email_notifications import build_message_list
from zerver.lib.logging_util import log_to_file
from zerver.lib.queue import queue_json_publish
from zerver.lib.send_email import FromAddress, send_future_email
from zerver.lib.url_encoding import encode_stream
from zerver.models import (
    Message,
    Realm,
    RealmAuditLog,
    Recipient,
    Subscription,
    UserActivity,
    UserProfile,
    get_active_streams,
    get_user_profile_by_id,
)

logger = logging.getLogger(__name__)
log_to_file(logger, settings.DIGEST_LOG_PATH)

DIGEST_CUTOFF = 5

TopicKey = Tuple[int, str]

# Digests accumulate 2 types of interesting traffic for a user:
# 1. New streams
# 2. Interesting stream traffic, as determined by the longest and most
#    diversely comment upon topics.

def inactive_since(user_profile: UserProfile, cutoff: datetime.datetime) -> bool:
    # Hasn't used the app in the last DIGEST_CUTOFF (5) days.
    most_recent_visit = [row.last_visit for row in
                         UserActivity.objects.filter(
                             user_profile=user_profile)]

    if not most_recent_visit:
        # This person has never used the app.
        return True

    last_visit = max(most_recent_visit)
    return last_visit < cutoff

def should_process_digest(realm_str: str) -> bool:
    if realm_str in settings.SYSTEM_ONLY_REALMS:
        # Don't try to send emails to system-only realms
        return False
    return True

# Changes to this should also be reflected in
# zerver/worker/queue_processors.py:DigestWorker.consume()
def queue_digest_recipient(user_profile: UserProfile, cutoff: datetime.datetime) -> None:
    # Convert cutoff to epoch seconds for transit.
    event = {"user_profile_id": user_profile.id,
             "cutoff": cutoff.strftime('%s')}
    queue_json_publish("digest_emails", event)

def enqueue_emails(cutoff: datetime.datetime) -> None:
    if not settings.SEND_DIGEST_EMAILS:
        return

    weekday = timezone_now().weekday()
    for realm in Realm.objects.filter(deactivated=False, digest_emails_enabled=True, digest_weekday=weekday):
        if not should_process_digest(realm.string_id):
            continue

        user_profiles = UserProfile.objects.filter(
            realm=realm, is_active=True, is_bot=False, enable_digest_emails=True)

        for user_profile in user_profiles:
            if inactive_since(user_profile, cutoff):
                queue_digest_recipient(user_profile, cutoff)
                logger.info(
                    "User %s is inactive, queuing for potential digest",
                    user_profile.id,
                )

def gather_hot_conversations(
    user_profile: UserProfile,
    stream_ids: List[int],
    cutoff_date: datetime.datetime,
) -> List[Dict[str, Any]]:
    # Gather stream conversations of 2 types:
    # 1. long conversations
    # 2. conversations where many different people participated
    #
    # Returns a list of dictionaries containing the templating
    # information for each hot conversation.

    messages = Message.objects.filter(
        recipient__type=Recipient.STREAM,
        recipient__type_id__in=stream_ids,
        date_sent__gt=cutoff_date).select_related('recipient', 'sender', 'sending_client')

    conversation_length: Dict[TopicKey, int] = defaultdict(int)
    conversation_messages: Dict[TopicKey, List[Message]] = defaultdict(list)
    conversation_senders: Dict[TopicKey, Set[str]] = defaultdict(set)
    for message in messages:
        key = (message.recipient.type_id,
               message.topic_name())

        conversation_messages[key].append(message)

        if not message.sent_by_human():
            # Don't include automated messages in the count.
            continue

        conversation_senders[key].add(message.sender.full_name)
        conversation_length[key] += 1

    topics_by_diversity = list(conversation_senders)
    topics_by_diversity.sort(key=lambda key: conversation_senders[key], reverse=True)

    topics_by_length = list(conversation_length)
    topics_by_diversity.sort(key=lambda key: conversation_length[key], reverse=True)

    # Get up to the 4 best conversations from the diversity list
    # and length list, filtering out overlapping conversations.
    hot_conversations = topics_by_diversity[:2]
    for topic_key in topics_by_length:
        if topic_key not in hot_conversations:
            hot_conversations.append(topic_key)
        if len(hot_conversations) >= 4:
            break

    # There was so much overlap between the diversity and length lists that we
    # still have < 4 conversations. Try to use remaining diversity items to pad
    # out the hot conversations.
    num_convos = len(hot_conversations)
    if num_convos < 4:
        hot_conversations.extend(topics_by_diversity[num_convos:4])

    hot_conversation_render_payloads = []
    for h in hot_conversations:
        users = list(conversation_senders[h])
        count = conversation_length[h]
        messages = conversation_messages[h]

        # We'll display up to 2 messages from the conversation.
        first_few_messages = messages[:2]

        teaser_data = {"participants": users,
                       "count": count - len(first_few_messages),
                       "first_few_messages": build_message_list(
                           user_profile, first_few_messages)}

        hot_conversation_render_payloads.append(teaser_data)
    return hot_conversation_render_payloads

def gather_new_streams(user_profile: UserProfile,
                       threshold: datetime.datetime) -> Tuple[int, Dict[str, List[str]]]:
    if user_profile.is_guest:
        new_streams = list(get_active_streams(user_profile.realm).filter(
            is_web_public=True, date_created__gt=threshold))

    elif user_profile.can_access_public_streams():
        new_streams = list(get_active_streams(user_profile.realm).filter(
            invite_only=False, date_created__gt=threshold))

    base_url = f"{user_profile.realm.uri}/#narrow/stream/"

    streams_html = []
    streams_plain = []

    for stream in new_streams:
        narrow_url = base_url + encode_stream(stream.id, stream.name)
        stream_link = f"{stream.name}"
        streams_html.append(stream_link)
        streams_plain.append(stream.name)

    return len(new_streams), {"html": streams_html, "plain": streams_plain}

def enough_traffic(hot_conversations: str, new_streams: int) -> bool:
    return bool(hot_conversations or new_streams)

def handle_digest_email(user_profile_id: int, cutoff: float,
                        render_to_web: bool = False) -> Union[None, Dict[str, Any]]:
    user_profile = get_user_profile_by_id(user_profile_id)

    # Convert from epoch seconds to a datetime object.
    cutoff_date = datetime.datetime.fromtimestamp(int(cutoff), tz=datetime.timezone.utc)

    context = common_context(user_profile)

    # Start building email template data.
    context.update(
        unsubscribe_link=one_click_unsubscribe_link(user_profile, "digest"),
    )

    home_view_streams = Subscription.objects.filter(
        user_profile=user_profile,
        recipient__type=Recipient.STREAM,
        active=True,
        is_muted=False).values_list('recipient__type_id', flat=True)

    if not user_profile.long_term_idle:
        stream_ids = home_view_streams
    else:
        stream_ids = exclude_subscription_modified_streams(user_profile, home_view_streams, cutoff_date)

    # Gather hot conversations.
    context["hot_conversations"] = gather_hot_conversations(user_profile, stream_ids, cutoff_date)

    # Gather new streams.
    new_streams_count, new_streams = gather_new_streams(
        user_profile, cutoff_date)
    context["new_streams"] = new_streams
    context["new_streams_count"] = new_streams_count

    # TODO: Set has_preheader if we want to include a preheader.

    if render_to_web:
        return context

    # We don't want to send emails containing almost no information.
    if enough_traffic(context["hot_conversations"], new_streams_count):
        logger.info("Sending digest email for user %s", user_profile.id)
        # Send now, as a ScheduledEmail
        send_future_email('zerver/emails/digest', user_profile.realm, to_user_ids=[user_profile.id],
                          from_name="Zulip Digest", from_address=FromAddress.no_reply_placeholder,
                          context=context)
    return None

def exclude_subscription_modified_streams(user_profile: UserProfile,
                                          stream_ids: List[int],
                                          cutoff_date: datetime.datetime) -> List[int]:
    """Exclude streams from given list where users' subscription was modified."""

    events = [
        RealmAuditLog.SUBSCRIPTION_CREATED,
        RealmAuditLog.SUBSCRIPTION_ACTIVATED,
        RealmAuditLog.SUBSCRIPTION_DEACTIVATED,
    ]

    # Streams where the user's subscription was changed
    modified_streams = RealmAuditLog.objects.filter(
        realm=user_profile.realm,
        modified_user=user_profile,
        event_time__gt=cutoff_date,
        event_type__in=events).values_list('modified_stream_id', flat=True)

    return list(set(stream_ids) - set(modified_streams))
<commit_msg>refactor: Rename conversation to topic.<commit_after>import datetime
import logging
from collections import defaultdict
from typing import Any, Dict, List, Set, Tuple, Union

from django.conf import settings
from django.utils.timezone import now as timezone_now

from confirmation.models import one_click_unsubscribe_link
from zerver.context_processors import common_context
from zerver.lib.email_notifications import build_message_list
from zerver.lib.logging_util import log_to_file
from zerver.lib.queue import queue_json_publish
from zerver.lib.send_email import FromAddress, send_future_email
from zerver.lib.url_encoding import encode_stream
from zerver.models import (
    Message,
    Realm,
    RealmAuditLog,
    Recipient,
    Subscription,
    UserActivity,
    UserProfile,
    get_active_streams,
    get_user_profile_by_id,
)

logger = logging.getLogger(__name__)
log_to_file(logger, settings.DIGEST_LOG_PATH)

DIGEST_CUTOFF = 5

TopicKey = Tuple[int, str]

# Digests accumulate 2 types of interesting traffic for a user:
# 1. New streams
# 2. Interesting stream traffic, as determined by the longest and most
#    diversely comment upon topics.

def inactive_since(user_profile: UserProfile, cutoff: datetime.datetime) -> bool:
    # Hasn't used the app in the last DIGEST_CUTOFF (5) days.
    most_recent_visit = [row.last_visit for row in
                         UserActivity.objects.filter(
                             user_profile=user_profile)]

    if not most_recent_visit:
        # This person has never used the app.
        return True

    last_visit = max(most_recent_visit)
    return last_visit < cutoff

def should_process_digest(realm_str: str) -> bool:
    if realm_str in settings.SYSTEM_ONLY_REALMS:
        # Don't try to send emails to system-only realms
        return False
    return True

# Changes to this should also be reflected in
# zerver/worker/queue_processors.py:DigestWorker.consume()
def queue_digest_recipient(user_profile: UserProfile, cutoff: datetime.datetime) -> None:
    # Convert cutoff to epoch seconds for transit.
    event = {"user_profile_id": user_profile.id,
             "cutoff": cutoff.strftime('%s')}
    queue_json_publish("digest_emails", event)

def enqueue_emails(cutoff: datetime.datetime) -> None:
    if not settings.SEND_DIGEST_EMAILS:
        return

    weekday = timezone_now().weekday()
    for realm in Realm.objects.filter(deactivated=False, digest_emails_enabled=True, digest_weekday=weekday):
        if not should_process_digest(realm.string_id):
            continue

        user_profiles = UserProfile.objects.filter(
            realm=realm, is_active=True, is_bot=False, enable_digest_emails=True)

        for user_profile in user_profiles:
            if inactive_since(user_profile, cutoff):
                queue_digest_recipient(user_profile, cutoff)
                logger.info(
                    "User %s is inactive, queuing for potential digest",
                    user_profile.id,
                )

def gather_hot_topics(
    user_profile: UserProfile,
    stream_ids: List[int],
    cutoff_date: datetime.datetime,
) -> List[Dict[str, Any]]:
    # Gather stream topics of 2 types:
    # 1. long topics
    # 2. topics where many different people participated
    #
    # Returns a list of dictionaries containing the templating
    # information for each hot topic.

    messages = Message.objects.filter(
        recipient__type=Recipient.STREAM,
        recipient__type_id__in=stream_ids,
        date_sent__gt=cutoff_date).select_related('recipient', 'sender', 'sending_client')

    topic_length: Dict[TopicKey, int] = defaultdict(int)
    topic_messages: Dict[TopicKey, List[Message]] = defaultdict(list)
    topic_senders: Dict[TopicKey, Set[str]] = defaultdict(set)
    for message in messages:
        key = (message.recipient.type_id,
               message.topic_name())

        topic_messages[key].append(message)

        if not message.sent_by_human():
            # Don't include automated messages in the count.
            continue

        topic_senders[key].add(message.sender.full_name)
        topic_length[key] += 1

    topics_by_diversity = list(topic_senders)
    topics_by_diversity.sort(key=lambda key: topic_senders[key], reverse=True)

    topics_by_length = list(topic_length)
    topics_by_diversity.sort(key=lambda key: topic_length[key], reverse=True)

    # Get up to the 4 best topics from the diversity list
    # and length list, filtering out overlapping topics.
    hot_topics = topics_by_diversity[:2]
    for topic_key in topics_by_length:
        if topic_key not in hot_topics:
            hot_topics.append(topic_key)
        if len(hot_topics) >= 4:
            break

    # There was so much overlap between the diversity and length lists that we
    # still have < 4 topics. Try to use remaining diversity items to pad
    # out the hot topics.
    num_convos = len(hot_topics)
    if num_convos < 4:
        hot_topics.extend(topics_by_diversity[num_convos:4])

    hot_topic_render_payloads = []
    for h in hot_topics:
        users = list(topic_senders[h])
        count = topic_length[h]
        messages = topic_messages[h]

        # We'll display up to 2 messages from the topic.
        first_few_messages = messages[:2]

        teaser_data = {"participants": users,
                       "count": count - len(first_few_messages),
                       "first_few_messages": build_message_list(
                           user_profile, first_few_messages)}

        hot_topic_render_payloads.append(teaser_data)
    return hot_topic_render_payloads

def gather_new_streams(user_profile: UserProfile,
                       threshold: datetime.datetime) -> Tuple[int, Dict[str, List[str]]]:
    if user_profile.is_guest:
        new_streams = list(get_active_streams(user_profile.realm).filter(
            is_web_public=True, date_created__gt=threshold))

    elif user_profile.can_access_public_streams():
        new_streams = list(get_active_streams(user_profile.realm).filter(
            invite_only=False, date_created__gt=threshold))

    base_url = f"{user_profile.realm.uri}/#narrow/stream/"

    streams_html = []
    streams_plain = []

    for stream in new_streams:
        narrow_url = base_url + encode_stream(stream.id, stream.name)
        stream_link = f"{stream.name}"
        streams_html.append(stream_link)
        streams_plain.append(stream.name)

    return len(new_streams), {"html": streams_html, "plain": streams_plain}

def enough_traffic(hot_conversations: str, new_streams: int) -> bool:
    return bool(hot_conversations or new_streams)

def handle_digest_email(user_profile_id: int, cutoff: float,
                        render_to_web: bool = False) -> Union[None, Dict[str, Any]]:
    user_profile = get_user_profile_by_id(user_profile_id)

    # Convert from epoch seconds to a datetime object.
    cutoff_date = datetime.datetime.fromtimestamp(int(cutoff), tz=datetime.timezone.utc)

    context = common_context(user_profile)

    # Start building email template data.
    context.update(
        unsubscribe_link=one_click_unsubscribe_link(user_profile, "digest"),
    )

    home_view_streams = Subscription.objects.filter(
        user_profile=user_profile,
        recipient__type=Recipient.STREAM,
        active=True,
        is_muted=False).values_list('recipient__type_id', flat=True)

    if not user_profile.long_term_idle:
        stream_ids = home_view_streams
    else:
        stream_ids = exclude_subscription_modified_streams(user_profile, home_view_streams, cutoff_date)

    # Gather hot conversations.
    context["hot_conversations"] = gather_hot_topics(user_profile, stream_ids, cutoff_date)

    # Gather new streams.
    new_streams_count, new_streams = gather_new_streams(
        user_profile, cutoff_date)
    context["new_streams"] = new_streams
    context["new_streams_count"] = new_streams_count

    # TODO: Set has_preheader if we want to include a preheader.

    if render_to_web:
        return context

    # We don't want to send emails containing almost no information.
    if enough_traffic(context["hot_conversations"], new_streams_count):
        logger.info("Sending digest email for user %s", user_profile.id)
        # Send now, as a ScheduledEmail
        send_future_email('zerver/emails/digest', user_profile.realm, to_user_ids=[user_profile.id],
                          from_name="Zulip Digest", from_address=FromAddress.no_reply_placeholder,
                          context=context)
    return None

def exclude_subscription_modified_streams(user_profile: UserProfile,
                                          stream_ids: List[int],
                                          cutoff_date: datetime.datetime) -> List[int]:
    """Exclude streams from given list where users' subscription was modified."""

    events = [
        RealmAuditLog.SUBSCRIPTION_CREATED,
        RealmAuditLog.SUBSCRIPTION_ACTIVATED,
        RealmAuditLog.SUBSCRIPTION_DEACTIVATED,
    ]

    # Streams where the user's subscription was changed
    modified_streams = RealmAuditLog.objects.filter(
        realm=user_profile.realm,
        modified_user=user_profile,
        event_time__gt=cutoff_date,
        event_type__in=events).values_list('modified_stream_id', flat=True)

    return list(set(stream_ids) - set(modified_streams))
<|endoftext|>


<commit_before>import zof
from ..http import HttpServer
from ..pktview import pktview_from_list

APP = zof.Application('rest_api')
APP.http_endpoint = '127.0.0.1:8080'

WEB = HttpServer(logger=APP.logger)
# WEB.define_var('dpid', _parse_dpid)

@APP.event('start')
async def start(_):
    await WEB.start(APP.http_endpoint)

@APP.event('stop')
async def stop(_):
    await WEB.stop()

@WEB.get_json('/stats/switches')
def get_switches():
    return [d.datapath_id for d in zof.get_datapaths()]

@WEB.get_json('/stats/flow/{dpid:[0-9A-F:]+}')
async def get_flows(dpid):
    result = await FLOW_REQ.request(datapath_id=_parse_dpid(dpid))
    _translate_flows(result['msg'])
    return {dpid: result['msg']}

@WEB.get_json('/stats/groupdesc/{dpid:[0-9A-F:]+}')
async def get_groupdesc(dpid):
    result = await GROUPDESC_REQ.request(datapath_id=_parse_dpid(dpid))
    _translate_groups(result['msg'])
    return {dpid: result['msg']}

@WEB.get_json('/stats/port/{dpid}/{port_no}')
async def get_portstats(dpid, port_no):
    result = await PORTSTATS_REQ.request(
        datapath_id=_parse_dpid(dpid), port_no=_parse_port(port_no))
    return {dpid: result['msg']}

@WEB.get_json('/stats/port/{dpid}')
async def get_portstats(dpid):
    result = await PORTSTATS_REQ.request(
        datapath_id=_parse_dpid(dpid), port_no='ANY')
    return {dpid: result['msg']}

@WEB.post_json('/stats/portdesc/modify')
async def modify_portdesc(post_data):
    dpid = _parse_dpid(post_data['dpid'])
    port_no = _parse_port(post_data['port_no'])
    port = zof.find_port(datapath_id=dpid, port_no=port_no)
    PORTMOD_REQ.send(
        datapath_id=dpid,
        port_no=port_no,
        hw_addr=port.hw_addr,
        config=post_data['config'],
        mask=post_data['mask'])
    # FIXME(bfish): This code does not handle OpenFlow errors elicited from the PortMod
    # message. Any errors returned will only show up in the log. The barrier here is
    # just a cheap trick to verify that the portmod *should* have been acted on.
    result = await BARRIER_REQ.request(datapath_id=dpid)
    return result['msg']

@WEB.get_json('/stats/portdesc/{dpid}')
async def get_portdesc(dpid):
    result = await PORTDESC_REQ.request(datapath_id=_parse_dpid(dpid))
    return {dpid: result['msg']}

FLOW_REQ = zof.compile('''
    type: REQUEST.FLOW
    msg:
        table_id: ALL
        out_port: ANY
        out_group: ANY
        cookie: 0
        cookie_mask: 0
        match: []
''')

GROUPDESC_REQ = zof.compile('''
    type: REQUEST.GROUP_DESC
''')

PORTSTATS_REQ = zof.compile('''
    type: REQUEST.PORT_STATS
    msg:
        port_no: $port_no
''')

PORTDESC_REQ = zof.compile('''
    type: REQUEST.PORT_DESC
''')

PORTMOD_REQ = zof.compile('''
    type: PORT_MOD
    msg:
        port_no: $port_no
        hw_addr: $hw_addr
        config: [$config]
        mask: [$mask]
''')

BARRIER_REQ = zof.compile('''
    type: BARRIER_REQUEST
''')

def _parse_dpid(dpid):
    if isinstance(dpid, int):
        return _convert_dpid(dpid)
    if ':' in dpid:
        return dpid
    return _convert_dpid(int(dpid, 0))

def _convert_dpid(dpid):
    hexstr = '%16.16x' % dpid
    return ':'.join(hexstr[2 * i:2 * i + 2] for i in range(8))

def _parse_port(port_no):
    if isinstance(port_no, int):
        return port_no
    return int(port_no, 0)

def _translate_flows(msgs):
    for msg in msgs:
        if 'match' in msg:
            msg['match'] = pktview_from_list(msg['match'], slash_notation=True)
        if 'instructions' in msg:
            msg['actions'] = _translate_instructions(msg['instructions'])

def _translate_groups(msgs):
    for msg in msgs:
        for bkt in msg['buckets']:
            if 'actions' in bkt:
                bkt['actions'] = _translate_actions(bkt['actions'])

def _translate_instructions(instrs):
    result = []
    for instr in instrs:
        result += _translate_instruction(instr)
    return result

def _translate_instruction(instr):
    if instr.instruction == 'APPLY_ACTIONS':
        return _translate_actions(instr['actions'])
    return [str(instr)]

def _translate_actions(actions):
    return [_translate_action(act) for act in actions]

def _translate_action(action):
    action_type = action['action']
    if action_type == 'OUTPUT':
        return 'OUTPUT:%s' % action['port_no']
    if action_type == 'GROUP':
        return 'GROUP:%s' % action['group_id']
    if action_type == 'SET_FIELD':
        return 'SET_FIELD: {%s:%s}' % (action['field'].lower(), action['value'])
    if len(action) == 1:
        return '%s' % action_type
    return str(action)

if __name__ == '__main__':
    zof.run()
<commit_msg>Fix zof api.<commit_after>import zof
from ..http import HttpServer
from ..pktview import pktview_from_list

APP = zof.Application('rest_api')
APP.http_endpoint = '127.0.0.1:8080'

WEB = HttpServer(logger=APP.logger)
# WEB.define_var('dpid', _parse_dpid)

@APP.event('start')
async def start(_):
    await WEB.start(APP.http_endpoint)

@APP.event('stop')
async def stop(_):
    await WEB.stop()

@WEB.get_json('/stats/switches')
def get_switches():
    return [d.datapath_id for d in zof.get_datapaths()]

@WEB.get_json('/stats/flow/{dpid:[0-9A-F:]+}')
async def get_flows(dpid):
    result = await FLOW_REQ.request(datapath_id=_parse_dpid(dpid))
    _translate_flows(result['msg'])
    return {dpid: result['msg']}

@WEB.get_json('/stats/groupdesc/{dpid:[0-9A-F:]+}')
async def get_groupdesc(dpid):
    result = await GROUPDESC_REQ.request(datapath_id=_parse_dpid(dpid))
    _translate_groups(result['msg'])
    return {dpid: result['msg']}

@WEB.get_json('/stats/port/{dpid}/{port_no}')
async def get_portstats(dpid, port_no):
    result = await PORTSTATS_REQ.request(
        datapath_id=_parse_dpid(dpid), port_no=_parse_port(port_no))
    return {dpid: result['msg']}

@WEB.get_json('/stats/port/{dpid}')
async def get_portstats(dpid):
    result = await PORTSTATS_REQ.request(
        datapath_id=_parse_dpid(dpid), port_no='ANY')
    return {dpid: result['msg']}

@WEB.post_json('/stats/portdesc/modify')
async def modify_portdesc(post_data):
    dpid = _parse_dpid(post_data['dpid'])
    port_no = _parse_port(post_data['port_no'])
    port = zof.find_port(datapath_id=dpid, port_no=port_no)
    PORTMOD_REQ.send(
        datapath_id=dpid,
        port_no=port_no,
        hw_addr=port.hw_addr,
        config=post_data['config'],
        mask=post_data['mask'])
    # FIXME(bfish): This code does not handle OpenFlow errors elicited from the PortMod
    # message. Any errors returned will only show up in the log. The barrier here is
    # just a cheap trick to verify that the portmod *should* have been acted on.
    result = await BARRIER_REQ.request(datapath_id=dpid)
    return result['msg']

@WEB.get_json('/stats/portdesc/{dpid}')
async def get_portdesc(dpid):
    result = await PORTDESC_REQ.request(datapath_id=_parse_dpid(dpid))
    return {dpid: result['msg']}

FLOW_REQ = zof.compile('''
    type: REQUEST.FLOW
    msg:
        table_id: ALL
        out_port: ANY
        out_group: ANY
        cookie: 0
        cookie_mask: 0
        match: []
''')

GROUPDESC_REQ = zof.compile('''
    type: REQUEST.GROUP_DESC
''')

PORTSTATS_REQ = zof.compile('''
    type: REQUEST.PORT_STATS
    msg:
        port_no: $port_no
''')

PORTDESC_REQ = zof.compile('''
    type: REQUEST.PORT_DESC
''')

PORTMOD_REQ = zof.compile('''
    type: PORT_MOD
    msg:
        port_no: $port_no
        hw_addr: $hw_addr
        config: [$config]
        mask: [$mask]
''')

BARRIER_REQ = zof.compile('''
    type: BARRIER_REQUEST
''')

def _parse_dpid(dpid):
    if isinstance(dpid, int):
        return _convert_dpid(dpid)
    if ':' in dpid:
        return dpid
    return _convert_dpid(int(dpid, 0))

def _convert_dpid(dpid):
    hexstr = '%16.16x' % dpid
    return ':'.join(hexstr[2 * i:2 * i + 2] for i in range(8))

def _parse_port(port_no):
    if isinstance(port_no, int):
        return port_no
    return int(port_no, 0)

def _translate_flows(msgs):
    for msg in msgs:
        if 'match' in msg:
            msg['match'] = pktview_from_list(msg['match'], slash_notation=True)
        if 'instructions' in msg:
            msg['actions'] = _translate_instructions(msg['instructions'])

def _translate_groups(msgs):
    for msg in msgs:
        for bkt in msg['buckets']:
            if 'actions' in bkt:
                bkt['actions'] = _translate_actions(bkt['actions'])

def _translate_instructions(instrs):
    result = []
    for instr in instrs:
        result += _translate_instruction(instr)
    return result

def _translate_instruction(instr):
    if instr['instruction'] == 'APPLY_ACTIONS':
        return _translate_actions(instr['actions'])
    return [str(instr)]

def _translate_actions(actions):
    return [_translate_action(act) for act in actions]

def _translate_action(action):
    action_type = action['action']
    if action_type == 'OUTPUT':
        return 'OUTPUT:%s' % action['port_no']
    if action_type == 'GROUP':
        return 'GROUP:%s' % action['group_id']
    if action_type == 'SET_FIELD':
        return 'SET_FIELD: {%s:%s}' % (action['field'].lower(), action['value'])
    if len(action) == 1:
        return '%s' % action_type
    return str(action)

if __name__ == '__main__':
    zof.run()
<|endoftext|>